more edit

code for cloud doc
2025-12-26 14:49:57 +00:00 · 2024-11-14 13:33:25 -08:00 · 2024-11-13 22:05:09 -08:00 · 2024-11-13 22:03:53 -08:00 · 2024-11-13 21:57:05 -08:00 · 2024-11-13 11:10:56 -08:00
64 changed files with 8339 additions and 2184 deletions
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -31,7 +31,7 @@ jobs:
      - name: Install dependecies needed for ubuntu
        run: |
          sudo apt install -y protobuf-compiler libssl-dev
-          rustup update && rustup default        
+          rustup update && rustup default
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
@@ -41,8 +41,8 @@ jobs:
      - name: Build Python
        working-directory: python
        run: |
-          python -m pip install -e .
-          python -m pip install -r ../docs/requirements.txt
+          python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .
+          python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -r ../docs/requirements.txt
      - name: Set up node
        uses: actions/setup-node@v3
        with:
--- a/.github/workflows/docs_test.yml
+++ b/.github/workflows/docs_test.yml
@@ -49,7 +49,7 @@ jobs:
    - name: Build Python
      working-directory: docs/test
      run:
-        python -m pip install -r requirements.txt
+        python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -r requirements.txt
    - name: Create test files
      run: |
        cd docs/test
--- a/.github/workflows/nodejs.yml
+++ b/.github/workflows/nodejs.yml
@@ -53,6 +53,9 @@ jobs:
        cargo clippy --all --all-features -- -D warnings
        npm ci
        npm run lint-ci
+    - name: Lint examples
+      working-directory: nodejs/examples
+      run: npm ci && npm run lint-ci
  linux:
    name: Linux (NodeJS ${{ matrix.node-version }})
    timeout-minutes: 30
@@ -91,6 +94,19 @@ jobs:
      env:
        S3_TEST: "1"
      run: npm run test
+    - name: Setup examples
+      working-directory: nodejs/examples
+      run: npm ci
+    - name: Test examples
+      working-directory: ./
+      env:
+        OPENAI_API_KEY: test
+        OPENAI_BASE_URL: http://0.0.0.0:8000
+      run: |
+        python ci/mock_openai.py &
+        ss -ltnp | grep :8000
+        cd nodejs/examples
+        npm test
  macos:
    timeout-minutes: 30
    runs-on: "macos-14"
--- a/.github/workflows/npm-publish.yml
+++ b/.github/workflows/npm-publish.yml
@@ -232,21 +232,7 @@ jobs:
    if: startsWith(github.ref, 'refs/tags/v')
    steps:
      - uses: actions/checkout@v4
-      - name: Cache installations
-        id: cache-installs
-        uses: actions/cache@v4
-        with:
-          path: |
-            C:\Program Files\Git
-            C:\BuildTools
-            C:\Program Files (x86)\Windows Kits
-            C:\Program Files\7-Zip
-            C:\protoc
-          key: ${{ runner.os }}-arm64-installs-v1
-          restore-keys: |
-            ${{ runner.os }}-arm64-installs-
      - name: Install Git
-        if: steps.cache-installs.outputs.cache-hit != 'true'
        run: |
          Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
          Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
@@ -263,7 +249,6 @@ jobs:
        with:
          python-version: "3.13"
      - name: Install Visual Studio Build Tools
-        if: steps.cache-installs.outputs.cache-hit != 'true'
        run: |
          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
          Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
@@ -312,7 +297,6 @@ jobs:
        with:
          workspaces: rust
      - name: Install 7-Zip ARM
-        if: steps.cache-installs.outputs.cache-hit != 'true'
        run: |
          New-Item -Path 'C:\7zip' -ItemType Directory
          Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
@@ -322,7 +306,6 @@ jobs:
        run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
        shell: powershell
      - name: Install Protoc v21.12
-        if: steps.cache-installs.outputs.cache-hit != 'true'
        working-directory: C:\
        run: |
          if (Test-Path 'C:\protoc') {
@@ -386,21 +369,7 @@ jobs:
    if: startsWith(github.ref, 'refs/tags/v')
    steps:
      - uses: actions/checkout@v4
-      - name: Cache installations
-        id: cache-installs
-        uses: actions/cache@v4
-        with:
-          path: |
-            C:\Program Files\Git
-            C:\BuildTools
-            C:\Program Files (x86)\Windows Kits
-            C:\Program Files\7-Zip
-            C:\protoc
-          key: ${{ runner.os }}-arm64-installs-v1
-          restore-keys: |
-            ${{ runner.os }}-arm64-installs-
      - name: Install Git
-        if: steps.cache-installs.outputs.cache-hit != 'true'
        run: |
          Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
          Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
@@ -417,7 +386,6 @@ jobs:
        with:
          python-version: "3.13"
      - name: Install Visual Studio Build Tools
-        if: steps.cache-installs.outputs.cache-hit != 'true'
        run: |
          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
          Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
@@ -456,7 +424,6 @@ jobs:
        with:
          workspaces: rust
      - name: Install 7-Zip ARM
-        if: steps.cache-installs.outputs.cache-hit != 'true'
        run: |
          New-Item -Path 'C:\7zip' -ItemType Directory
          Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
@@ -466,7 +433,6 @@ jobs:
        run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
        shell: powershell
      - name: Install Protoc v21.12
-        if: steps.cache-installs.outputs.cache-hit != 'true'
        working-directory: C:\
        run: |
          if (Test-Path 'C:\protoc') {
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -138,7 +138,7 @@ jobs:
        run: rm -rf target/wheels
  windows:
    name: "Windows: ${{ matrix.config.name }}"
-    timeout-minutes: 30
+    timeout-minutes: 60
    strategy:
      matrix:
        config:
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -50,6 +50,7 @@ jobs:
        run: cargo fmt --all -- --check
      - name: Run clippy
        run: cargo clippy --workspace --tests --all-features -- -D warnings
+
  linux:
    timeout-minutes: 30
    # To build all features, we need more disk space than is available
@@ -91,6 +92,7 @@ jobs:
        run: cargo test --all-features
      - name: Run examples
        run: cargo run --example simple
+
  macos:
    timeout-minutes: 30
    strategy:
@@ -118,6 +120,7 @@ jobs:
      - name: Run tests
        # Run with everything except the integration tests.
        run: cargo test --features remote,fp16kernels
+
  windows:
    runs-on: windows-2022
    steps:
@@ -139,24 +142,11 @@ jobs:
          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
          cargo build
          cargo test
+
  windows-arm64:
    runs-on: windows-4x-arm
    steps:
-      - name: Cache installations
-        id: cache-installs
-        uses: actions/cache@v4
-        with:
-          path: |
-            C:\Program Files\Git
-            C:\BuildTools
-            C:\Program Files (x86)\Windows Kits
-            C:\Program Files\7-Zip
-            C:\protoc
-          key: ${{ runner.os }}-arm64-installs-v1
-          restore-keys: |
-            ${{ runner.os }}-arm64-installs-
      - name: Install Git
-        if: steps.cache-installs.outputs.cache-hit != 'true'
        run: |
          Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
          Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
@@ -173,7 +163,6 @@ jobs:
        with:
          python-version: "3.13"
      - name: Install Visual Studio Build Tools
-        if: steps.cache-installs.outputs.cache-hit != 'true'
        run: |
          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
          Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
@@ -217,12 +206,10 @@ jobs:
        run: |
          Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
        shell: powershell
-
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: rust
      - name: Install 7-Zip ARM
-        if: steps.cache-installs.outputs.cache-hit != 'true'
        run: |
          New-Item -Path 'C:\7zip' -ItemType Directory
          Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
@@ -232,12 +219,11 @@ jobs:
        run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
        shell: powershell
      - name: Install Protoc v21.12
-        if: steps.cache-installs.outputs.cache-hit != 'true'
        working-directory: C:\
        run: |
          if (Test-Path 'C:\protoc') {
-              Write-Host "Protoc directory exists, skipping installation"
-              return
+            Write-Host "Protoc directory exists, skipping installation"
+            return
          }
          New-Item -Path 'C:\protoc' -ItemType Directory
          Set-Location C:\protoc
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -21,13 +21,15 @@ categories = ["database-implementations"]
 rust-version = "1.80.0" # TODO: lower this once we upgrade Lance again.

 [workspace.dependencies]
-lance = { "version" = "=0.19.2", "features" = ["dynamodb"], path = "../lance/rust/lance"}
-lance-index = { "version" = "=0.19.2", path = "../lance/rust/lance-index"}
-lance-linalg = { "version" = "=0.19.2", path = "../lance/rust/lance-linalg"}
-lance-testing = { "version" = "=0.19.2", path = "../lance/rust/lance-testing"}
-lance-datafusion = { "version" = "=0.19.2", path = "../lance/rust/lance-datafusion"}
-lance-encoding = { "version" = "=0.19.2", path = "../lance/rust/lance-encoding"}
-lance-table = { "version" = "=0.19.2", path = "../lance/rust/lance-table"}
+lance = { "version" = "=0.19.2", "features" = [
+    "dynamodb",
+], git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
+lance-index = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
+lance-linalg = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
+lance-table = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
+lance-testing = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
+lance-datafusion = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
+lance-encoding = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
 # Note that this one does not include pyarrow
 arrow = { version = "52.2", optional = false }
 arrow-array = "52.2"
--- a/ci/mock_openai.py
+++ b/ci/mock_openai.py
@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+"""A zero-dependency mock OpenAI embeddings API endpoint for testing purposes."""
+import argparse
+import json
+import http.server
+
+
+class MockOpenAIRequestHandler(http.server.BaseHTTPRequestHandler):
+    def do_POST(self):
+        content_length = int(self.headers["Content-Length"])
+        post_data = self.rfile.read(content_length)
+        post_data = json.loads(post_data.decode("utf-8"))
+        # See: https://platform.openai.com/docs/api-reference/embeddings/create
+
+        if isinstance(post_data["input"], str):
+            num_inputs = 1
+        else:
+            num_inputs = len(post_data["input"])
+
+        model = post_data.get("model", "text-embedding-ada-002")
+
+        data = []
+        for i in range(num_inputs):
+            data.append({
+                "object": "embedding",
+                "embedding": [0.1] * 1536,
+                "index": i,
+            })
+
+        response = {
+            "object": "list",
+            "data": data,
+            "model": model,
+            "usage": {
+                "prompt_tokens": 0,
+                "total_tokens": 0,
+            }
+        }
+
+        self.send_response(200)
+        self.send_header("Content-type", "application/json")
+        self.end_headers()
+        self.wfile.write(json.dumps(response).encode("utf-8"))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Mock OpenAI embeddings API endpoint")
+    parser.add_argument("--port", type=int, default=8000, help="Port to listen on")
+    args = parser.parse_args()
+    port = args.port
+
+    print(f"server started on port {port}. Press Ctrl-C to stop.")
+    print(f"To use, set OPENAI_BASE_URL=http://localhost:{port} in your environment.")
+
+    with http.server.HTTPServer(("0.0.0.0", port), MockOpenAIRequestHandler) as server:
+        server.serve_forever()
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -222,10 +222,12 @@ nav:
          - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
      - ☁️ LanceDB Cloud:
          - Overview: cloud/index.md
-          - API reference:
-              - 🐍 Python: python/saas-python.md
-              - 👾 JavaScript: javascript/modules.md
-              - REST API: cloud/rest.md
+          - Quickstart: cloud/quickstart.md
+          - Best Practices: cloud/best_practices.md
+          # - API reference:
+          #     - 🐍 Python: python/saas-python.md
+          #     - 👾 JavaScript: javascript/modules.md
+          #     - REST API: cloud/rest.md

  - Quick start: basic.md
  - Concepts:
@@ -348,10 +350,17 @@ nav:
      - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
  - LanceDB Cloud:
      - Overview: cloud/index.md
-      - API reference:
-          - 🐍 Python: python/saas-python.md
-          - 👾 JavaScript: javascript/modules.md
-          - REST API: cloud/rest.md
+      - Quickstart: cloud/quickstart.md
+      - Work with data: 
+        - Ingest data: cloud/ingest_data.md
+        - Update data: cloud/update_data.md
+        - Build an index: cloud/build_index.md
+        - Vector search: cloud/vector_search.md
+        - Full-text search: cloud/full_text_search.md
+        - Hybrid search: cloud/hybrid_search.md
+        - Metadata Filtering: cloud/metadata_filtering.md
+      - Best Practices: cloud/best_practices.md
+      # - REST API: cloud/rest.md

 extra_css:
  - styles/global.css
--- a/docs/src/ann_indexes.md
+++ b/docs/src/ann_indexes.md
@@ -45,9 +45,9 @@ Lance supports `IVF_PQ` index type by default.
        Creating indexes is done via the [lancedb.Table.createIndex](../js/classes/Table.md/#createIndex) method.

        ```typescript
-        --8<--- "nodejs/examples/ann_indexes.ts:import"
+        --8<--- "nodejs/examples/ann_indexes.test.ts:import"

-        --8<-- "nodejs/examples/ann_indexes.ts:ingest"
+        --8<-- "nodejs/examples/ann_indexes.test.ts:ingest"
        ```

    === "vectordb (deprecated)"
@@ -140,13 +140,15 @@ There are a couple of parameters that can be used to fine-tune the search:

 - **limit** (default: 10): The amount of results that will be returned
 - **nprobes** (default: 20): The number of probes used. A higher number makes search more accurate but also slower.<br/>
-  Most of the time, setting nprobes to cover 5-10% of the dataset should achieve high recall with low latency.<br/>
-  e.g., for 1M vectors divided up into 256 partitions, nprobes should be set to ~20-40.<br/>
-  Note: nprobes is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
+  Most of the time, setting nprobes to cover 5-15% of the dataset should achieve high recall with low latency.<br/>
+    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, `nprobes` should be set to ~20-40. This value can be adjusted to achieve the optimal balance between search latency and search quality. <br/>
+  
 - **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
  A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
-  e.g., for 1M vectors divided into 256 partitions, if you're looking for top 20, then refine_factor=200 reranks the whole partition.<br/>
-  Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
+    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, setting the `refine_factor` to 200 will initially retrieve the top 4,000 candidates (top k * refine_factor) from all searched partitions. These candidates are then reranked to determine the final top 20 results.<br/>
+!!! note 
+    Both `nprobes` and `refine_factor` are only applicable if an ANN index is present. If specified on a table without an ANN index, those parameters are ignored.
+

 === "Python"

@@ -169,7 +171,7 @@ There are a couple of parameters that can be used to fine-tune the search:
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.ts:search1"
+        --8<-- "nodejs/examples/ann_indexes.test.ts:search1"
        ```

    === "vectordb (deprecated)"
@@ -203,7 +205,7 @@ You can further filter the elements returned by a search using a where clause.
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.ts:search2"
+        --8<-- "nodejs/examples/ann_indexes.test.ts:search2"
        ```

    === "vectordb (deprecated)"
@@ -235,7 +237,7 @@ You can select the columns returned by the query using a select clause.
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.ts:search3"
+        --8<-- "nodejs/examples/ann_indexes.test.ts:search3"
        ```

    === "vectordb (deprecated)"
--- a/docs/src/basic.md
+++ b/docs/src/basic.md
@@ -157,7 +157,7 @@ recommend switching to stable releases.
        import * as lancedb from "@lancedb/lancedb";
        import * as arrow from "apache-arrow";

-        --8<-- "nodejs/examples/basic.ts:connect"
+        --8<-- "nodejs/examples/basic.test.ts:connect"
        ```

    === "vectordb (deprecated)"
@@ -212,7 +212,7 @@ table.
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_table"
+        --8<-- "nodejs/examples/basic.test.ts:create_table"
        ```

    === "vectordb (deprecated)"
@@ -268,7 +268,7 @@ similar to a `CREATE TABLE` statement in SQL.
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_empty_table"
+        --8<-- "nodejs/examples/basic.test.ts:create_empty_table"
        ```

    === "vectordb (deprecated)"
@@ -298,7 +298,7 @@ Once created, you can open a table as follows:
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:open_table"
+        --8<-- "nodejs/examples/basic.test.ts:open_table"
        ```

    === "vectordb (deprecated)"
@@ -327,7 +327,7 @@ If you forget the name of your table, you can always get a listing of all table
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:table_names"
+        --8<-- "nodejs/examples/basic.test.ts:table_names"
        ```

    === "vectordb (deprecated)"
@@ -357,7 +357,7 @@ After a table has been created, you can always add more data to it as follows:
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:add_data"
+        --8<-- "nodejs/examples/basic.test.ts:add_data"
        ```

    === "vectordb (deprecated)"
@@ -389,7 +389,7 @@ Once you've embedded the query, you can find its nearest neighbors as follows:
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:vector_search"
+        --8<-- "nodejs/examples/basic.test.ts:vector_search"
        ```

    === "vectordb (deprecated)"
@@ -429,7 +429,7 @@ LanceDB allows you to create an ANN index on a table as follows:
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_index"
+        --8<-- "nodejs/examples/basic.test.ts:create_index"
        ```

    === "vectordb (deprecated)"
@@ -469,7 +469,7 @@ This can delete any number of rows that match the filter.
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:delete_rows"
+        --8<-- "nodejs/examples/basic.test.ts:delete_rows"
        ```

    === "vectordb (deprecated)"
@@ -527,7 +527,7 @@ Use the `drop_table()` method on the database to remove a table.
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:drop_table"
+        --8<-- "nodejs/examples/basic.test.ts:drop_table"
        ```

    === "vectordb (deprecated)"
@@ -561,8 +561,8 @@ You can use the embedding API when working with embedding models. It automatical
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/embedding.ts:imports"
-        --8<-- "nodejs/examples/embedding.ts:openai_embeddings"
+        --8<-- "nodejs/examples/embedding.test.ts:imports"
+        --8<-- "nodejs/examples/embedding.test.ts:openai_embeddings"
        ```

 === "Rust"
--- a/docs/src/cloud/best_practices.md
+++ b/docs/src/cloud/best_practices.md
@@ -0,0 +1,20 @@
+This section provides a set of recommended best practices to help you get the most out of LanceDB Cloud. By following these guidelines, you can optimize your usage of LanceDB Cloud, improve performance, and ensure a smooth experience.
+
+### Should the db connection be created once and keep it open?
+Yes! It is recommended to establish a single db connection and maintain it throughout your interaction with the tables within. 
+
+LanceDB uses `requests.Session()` for connection pooling, which automatically manages connection reuse and cleanup. This approach avoids the overhead of repeatedly establishing HTTP connections, significantly improving efficiency.
+
+### Should a single `open_table` call be made and maintained for subsequent table operations?
+`table = db.open_table()` should be called once and used for all subsequent table operations. If there are changes to the opened table, `table` always reflect the latest version of the data. 
+
+### Row id
+
+### What are the vector indexing types supported by LanceDB Cloud?
+We support `IVF_PQ` and `IVF_HNSW_SQ` as the `index_type` which is passed to `create_index`. LanceDB Cloud tunes the indexing parameters automatically to achieve the best tradeoff betweeln query latency and query quality.
+
+### Do I need to do anything when there is new data added to a table with an existing index?
+No! LanceDB Cloud triggers an asynchronous background job to index the new vectors. This process will either merge the new vectors into the existing index or initiate a complete re-indexing if needed. 
+
+There is a flag `fast_search` in `table.search()` that allows you to control whether the unindexed rows should be searched or not.  
+
--- a/docs/src/cloud/build_index.md
+++ b/docs/src/cloud/build_index.md
@@ -0,0 +1,64 @@
+LanceDB Cloud supports **vector index**, **scalar index** and **full-text search index**. Compared to open-source version, LanceDB Cloud focuses on **automation**:
+
+- If there is a single vector column in the table, the vector column can be inferred from the schema and the index will be automatically created. 
+
+- Indexing parameters will be automatically tuned for customer's data.
+
+## Vector index
+LanceDB has implemented the state-of-art indexing algorithms (more about [IVF-PQ](https://lancedb.github.io/lancedb/concepts/index_ivfpq/) and [HNSW](https://lancedb.github.io/lancedb/concepts/index_hnsw/)). We currently 
+support the _L2_, _Cosine_ and _Dot_ as distance calculation metrics. You can create multiple vector indices within a table.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:create_index"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:create_index"
+    ```
+
+## Scalar index
+LanceDB Cloud and LanceDB Enterprise supports several types of Scalar indices to accelerate search over scalar columns.
+
+- *BTREE*: The most common type is BTREE. This index is inspired by the btree data structure although only the first few layers of the btree are cached in memory. It will perform well on columns with a large number of unique values and few rows per value.
+- *BITMAP*: this index stores a bitmap for each unique value in the column. This index is useful for columns with a finite number of unique values and many rows per value. 
+    - For example, columns that represent "categories", "labels", or "tags"
+- *LABEL_LIST*: a special index that is used to index list columns whose values have a finite set of possibilities. 
+    - For example, a column that contains lists of tags (e.g. ["tag1", "tag2", "tag3"]) can be indexed with a LABEL_LIST index.
+
+You can create multiple scalar indices within a table.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:create_scalar_index"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:create_scalar_index"
+    ```
+
+## Full-text search index
+We provide performant full-text search on LanceDB Cloud, allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
+!!! note ""
+
+    `use_tantivy` is not available with `create_fts_index` on LanceDB Cloud as we used our native implementation, which has better performance comparing to tantivy.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:create_fts_index"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:create_fts_index"
+    ```
--- a/docs/src/cloud/full_text_search.md
+++ b/docs/src/cloud/full_text_search.md
@@ -0,0 +1,14 @@
+The full-text search allows you to 
+incorporate keyword-based search (based on BM25) in your retrieval solutions.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:full_text_search"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:full_text_search"
+    ```
--- a/docs/src/cloud/hybrid_search.md
+++ b/docs/src/cloud/hybrid_search.md
@@ -0,0 +1,10 @@
+We support hybrid search that combines semantic and full-text search via a 
+reranking algorithm of your choice, to get the best of both worlds. LanceDB 
+comes with [built-in rerankers](https://lancedb.github.io/lancedb/reranking/) 
+and you can implement you own _customized reranker_ as well. 
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:hybrid_search"
+    ```
--- a/docs/src/cloud/ingest_data.md
+++ b/docs/src/cloud/ingest_data.md
@@ -0,0 +1,31 @@
+## Insert data
+The LanceDB Cloud SDK for data ingestion remains consistent with our open-source version, 
+ensuring a seamless transition for existing OSS users. 
+!!! note "unsupported parameters in create_table"
+
+    The following two parameters: `mode="overwrite"` and `exist_ok`, are expected to be added by Nov, 2024.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:import-ingest-data"
+    
+    --8<-- "python/python/tests/docs/test_cloud.py:ingest_data"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    --8<-- "nodejs/examples/cloud.test.ts:ingest_data"
+    ```
+
+## Insert large datasets
+It is recommended to use itertators to add large datasets in batches when creating 
+your table in one go. Data will be automatically compacted for the best query performance.
+!!! info "batch size"
+
+    The batch size .
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:ingest_data_in_batch"
+    ```
--- a/docs/src/cloud/metadata_filtering.md
+++ b/docs/src/cloud/metadata_filtering.md
@@ -0,0 +1,33 @@
+LanceDB Cloud supports rich filtering features of query results based on metadata fields. 
+
+By default, _post-filtering_ is performed on the top-k results returned by the vector search. 
+However, _pre-filtering_ is also an option that performs the filter prior to vector search. 
+This can be useful to narrow down on the search space on a very large dataset to reduce query 
+latency.
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:filtering"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+
+    --8<-- "nodejs/examples/cloud.test.ts:filtering"
+    ```
+We also support standard SQL expressions as predicates for filtering operations. 
+It can be used during vector search, update, and deletion operations.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:sql_filtering"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:sql_filtering"
+    ```
--- a/docs/src/cloud/update_data.md
+++ b/docs/src/cloud/update_data.md
@@ -0,0 +1,49 @@
+LanceDB Cloud efficiently manages updates across many tables. 
+Currently, we offer _update_, _merge_insert_, and _delete_.
+
+## update
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:update_data"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:update_data"
+    ```
+
+## merge insert
+This merge insert can add rows, update rows, and remove rows all in a single transaction. 
+It combines new data from a source table with existing data in a target table by using a join.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:merge_insert"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:merge_insert"
+    ```
+
+## delete
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:delete_data"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:delete_data"
+    ```
--- a/docs/src/cloud/vector_search.md
+++ b/docs/src/cloud/vector_search.md
@@ -0,0 +1,21 @@
+Users can also tune the following parameters for better search quality.
+
+ - [nprobes](https://lancedb.github.io/lancedb/js/classes/VectorQuery/#nprobes): 
+ the number of partitions to search (probe).
+ - [refine factor](https://lancedb.github.io/lancedb/js/classes/VectorQuery/#refinefactor): 
+ a multiplier to control how many additional rows are taken during the refine step.
+
+ [Metadata filtering](filtering) combined with the vector search is also supported.
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:vector_search"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:vector_search"
+    ```
--- a/docs/src/embeddings/custom_embedding_function.md
+++ b/docs/src/embeddings/custom_embedding_function.md
@@ -47,9 +47,9 @@ Let's implement `SentenceTransformerEmbeddings` class. All you need to do is imp
 === "TypeScript"

    ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.ts:imports"
+    --8<--- "nodejs/examples/custom_embedding_function.test.ts:imports"

-    --8<--- "nodejs/examples/custom_embedding_function.ts:embedding_impl"
+    --8<--- "nodejs/examples/custom_embedding_function.test.ts:embedding_impl"
    ```


@@ -78,7 +78,7 @@ Now you can use this embedding function to create your table schema and that's i
 === "TypeScript"

    ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.ts:call_custom_function"
+    --8<--- "nodejs/examples/custom_embedding_function.test.ts:call_custom_function"
    ```

 !!! note
--- a/docs/src/embeddings/embedding_functions.md
+++ b/docs/src/embeddings/embedding_functions.md
@@ -94,8 +94,8 @@ the embeddings at all:
    === "@lancedb/lancedb"

        ```ts
-        --8<-- "nodejs/examples/embedding.ts:imports"
-        --8<-- "nodejs/examples/embedding.ts:embedding_function"
+        --8<-- "nodejs/examples/embedding.test.ts:imports"
+        --8<-- "nodejs/examples/embedding.test.ts:embedding_function"
        ```

    === "vectordb (deprecated)"
@@ -150,7 +150,7 @@ need to worry about it when you query the table:
            .toArray()
        ```

-    === "vectordb (deprecated)
+    === "vectordb (deprecated)"

        ```ts
        const results = await table
--- a/docs/src/embeddings/index.md
+++ b/docs/src/embeddings/index.md
@@ -51,8 +51,8 @@ LanceDB registers the OpenAI embeddings function in the registry as `openai`. Yo
 === "TypeScript"

    ```typescript
-    --8<--- "nodejs/examples/embedding.ts:imports"
-    --8<--- "nodejs/examples/embedding.ts:openai_embeddings"
+    --8<--- "nodejs/examples/embedding.test.ts:imports"
+    --8<--- "nodejs/examples/embedding.test.ts:openai_embeddings"
    ```

 === "Rust"
@@ -121,12 +121,10 @@ class Words(LanceModel):
    vector: Vector(func.ndims()) = func.VectorField()

 table = db.create_table("words", schema=Words)
-table.add(
-    [
-        {"text": "hello world"},
-        {"text": "goodbye world"}
-    ]
-    )
+table.add([
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+])

 query = "greetings"
 actual = table.search(query).limit(1).to_pydantic(Words)[0]
--- a/docs/src/guides/tables.md
+++ b/docs/src/guides/tables.md
@@ -85,13 +85,13 @@ Initialize a LanceDB connection and create a table


        ```ts
-        --8<-- "nodejs/examples/basic.ts:create_table"
+        --8<-- "nodejs/examples/basic.test.ts:create_table"
        ```

        This will infer the schema from the provided data. If you want to explicitly provide a schema, you can use `apache-arrow` to declare a schema

        ```ts
-        --8<-- "nodejs/examples/basic.ts:create_table_with_schema"
+        --8<-- "nodejs/examples/basic.test.ts:create_table_with_schema"
        ```

        !!! info "Note"
@@ -100,14 +100,14 @@ Initialize a LanceDB connection and create a table
            passed in will NOT be appended to the table in that case.

        ```ts
-        --8<-- "nodejs/examples/basic.ts:create_table_exists_ok"
+        --8<-- "nodejs/examples/basic.test.ts:create_table_exists_ok"
        ```

        Sometimes you want to make sure that you start fresh. If you want to
        overwrite the table, you can pass in mode: "overwrite" to the createTable function.

        ```ts
-        --8<-- "nodejs/examples/basic.ts:create_table_overwrite"
+        --8<-- "nodejs/examples/basic.test.ts:create_table_overwrite"
        ```

    === "vectordb (deprecated)"
@@ -227,7 +227,7 @@ LanceDB supports float16 data type!
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_f16_table"
+        --8<-- "nodejs/examples/basic.test.ts:create_f16_table"
        ```

    === "vectordb (deprecated)"
@@ -455,7 +455,7 @@ You can create an empty table for scenarios where you want to add data to the ta
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_empty_table"
+        --8<-- "nodejs/examples/basic.test.ts:create_empty_table"
        ```

    === "vectordb (deprecated)"
--- a/docs/src/notebooks/hybrid_search.ipynb
+++ b/docs/src/notebooks/hybrid_search.ipynb
--- a/docs/src/search.md
+++ b/docs/src/search.md
@@ -58,9 +58,9 @@ db.create_table("my_vectors", data=data)
    === "@lancedb/lancedb"

        ```ts
-        --8<-- "nodejs/examples/search.ts:import"
+        --8<-- "nodejs/examples/search.test.ts:import"

-        --8<-- "nodejs/examples/search.ts:search1"
+        --8<-- "nodejs/examples/search.test.ts:search1"
        ```


@@ -89,7 +89,7 @@ By default, `l2` will be used as metric type. You can specify the metric type as
    === "@lancedb/lancedb"

        ```ts
-        --8<-- "nodejs/examples/search.ts:search2"
+        --8<-- "nodejs/examples/search.test.ts:search2"
        ```

    === "vectordb (deprecated)"
--- a/docs/src/sql.md
+++ b/docs/src/sql.md
@@ -49,7 +49,7 @@ const tbl = await db.createTable('myVectors', data)
    === "@lancedb/lancedb"

        ```ts
-        --8<-- "nodejs/examples/filtering.ts:search"
+        --8<-- "nodejs/examples/filtering.test.ts:search"
        ```

    === "vectordb (deprecated)"
@@ -91,7 +91,7 @@ For example, the following filter string is acceptable:
    === "@lancedb/lancedb"

        ```ts
-        --8<-- "nodejs/examples/filtering.ts:vec_search"
+        --8<-- "nodejs/examples/filtering.test.ts:vec_search"
        ```

    === "vectordb (deprecated)"
@@ -169,7 +169,7 @@ You can also filter your data without search.
    === "@lancedb/lancedb"

        ```ts
-        --8<-- "nodejs/examples/filtering.ts:sql_search"
+        --8<-- "nodejs/examples/filtering.test.ts:sql_search"
        ```

    === "vectordb (deprecated)"
--- a/docs/test/md_testing.py
+++ b/docs/test/md_testing.py
@@ -22,7 +22,8 @@ excluded_globs = [
    "../src/embeddings/available_embedding_models/text_embedding_functions/*.md",
    "../src/embeddings/available_embedding_models/multimodal_embedding_functions/*.md",
    "../src/rag/*.md",
-    "../src/rag/advanced_techniques/*.md"
+    "../src/rag/advanced_techniques/*.md",
+    "../src/cloud/*.md"


 ]
--- a/nodejs/biome.json
+++ b/nodejs/biome.json
@@ -9,7 +9,8 @@
      "**/native.js",
      "**/native.d.ts",
      "**/npm/**/*",
-      "**/.vscode/**"
+      "**/.vscode/**",
+      "./examples/*"
    ]
  },
  "formatter": {
--- a/nodejs/examples/ann_indexes.test.ts
+++ b/nodejs/examples/ann_indexes.test.ts
@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<-- [start:import]
+import * as lancedb from "@lancedb/lancedb";
+import { VectorQuery } from "@lancedb/lancedb";
+// --8<-- [end:import]
+import { withTempDirectory } from "./util.ts";
+
+test("ann index examples", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:ingest]
+    const db = await lancedb.connect(databaseDir);
+
+    const data = Array.from({ length: 5_000 }, (_, i) => ({
+      vector: Array(128).fill(i),
+      id: `${i}`,
+      content: "",
+      longId: `${i}`,
+    }));
+
+    const table = await db.createTable("my_vectors", data, {
+      mode: "overwrite",
+    });
+    await table.createIndex("vector", {
+      config: lancedb.Index.ivfPq({
+        numPartitions: 10,
+        numSubVectors: 16,
+      }),
+    });
+    // --8<-- [end:ingest]
+
+    // --8<-- [start:search1]
+    const search = table.search(Array(128).fill(1.2)).limit(2) as VectorQuery;
+    const results1 = await search.nprobes(20).refineFactor(10).toArray();
+    // --8<-- [end:search1]
+    expect(results1.length).toBe(2);
+
+    // --8<-- [start:search2]
+    const results2 = await table
+      .search(Array(128).fill(1.2))
+      .where("id != '1141'")
+      .limit(2)
+      .toArray();
+    // --8<-- [end:search2]
+    expect(results2.length).toBe(2);
+
+    // --8<-- [start:search3]
+    const results3 = await table
+      .search(Array(128).fill(1.2))
+      .select(["id"])
+      .limit(2)
+      .toArray();
+    // --8<-- [end:search3]
+    expect(results3.length).toBe(2);
+  });
+}, 100_000);
--- a/nodejs/examples/ann_indexes.ts
+++ b/nodejs/examples/ann_indexes.ts
@@ -1,49 +0,0 @@
-// --8<-- [start:import]
-import * as lancedb from "@lancedb/lancedb";
-// --8<-- [end:import]
-
-// --8<-- [start:ingest]
-const db = await lancedb.connect("/tmp/lancedb/");
-
-const data = Array.from({ length: 10_000 }, (_, i) => ({
-  vector: Array(1536).fill(i),
-  id: `${i}`,
-  content: "",
-  longId: `${i}`,
-}));
-
-const table = await db.createTable("my_vectors", data, { mode: "overwrite" });
-await table.createIndex("vector", {
-  config: lancedb.Index.ivfPq({
-    numPartitions: 16,
-    numSubVectors: 48,
-  }),
-});
-// --8<-- [end:ingest]
-
-// --8<-- [start:search1]
-const _results1 = await table
-  .search(Array(1536).fill(1.2))
-  .limit(2)
-  .nprobes(20)
-  .refineFactor(10)
-  .toArray();
-// --8<-- [end:search1]
-
-// --8<-- [start:search2]
-const _results2 = await table
-  .search(Array(1536).fill(1.2))
-  .where("id != '1141'")
-  .limit(2)
-  .toArray();
-// --8<-- [end:search2]
-
-// --8<-- [start:search3]
-const _results3 = await table
-  .search(Array(1536).fill(1.2))
-  .select(["id"])
-  .limit(2)
-  .toArray();
-// --8<-- [end:search3]
-
-console.log("Ann indexes: done");
--- a/nodejs/examples/basic.test.ts
+++ b/nodejs/examples/basic.test.ts
@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<--  [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import * as arrow from "apache-arrow";
+import {
+  Field,
+  FixedSizeList,
+  Float16,
+  Int32,
+  Schema,
+  Utf8,
+} from "apache-arrow";
+// --8<-- [end:imports]
+import { withTempDirectory } from "./util.ts";
+
+test("basic table examples", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:connect]
+    const db = await lancedb.connect(databaseDir);
+    // --8<-- [end:connect]
+    {
+      // --8<-- [start:create_table]
+      const _tbl = await db.createTable(
+        "myTable",
+        [
+          { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+          { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+        ],
+        { mode: "overwrite" },
+      );
+      // --8<-- [end:create_table]
+
+      const data = [
+        { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+        { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+      ];
+
+      {
+        // --8<-- [start:create_table_exists_ok]
+        const tbl = await db.createTable("myTable", data, {
+          existOk: true,
+        });
+        // --8<-- [end:create_table_exists_ok]
+        expect(await tbl.countRows()).toBe(2);
+      }
+      {
+        // --8<-- [start:create_table_overwrite]
+        const tbl = await db.createTable("myTable", data, {
+          mode: "overwrite",
+        });
+        // --8<-- [end:create_table_overwrite]
+        expect(await tbl.countRows()).toBe(2);
+      }
+    }
+
+    await db.dropTable("myTable");
+
+    {
+      // --8<-- [start:create_table_with_schema]
+      const schema = new arrow.Schema([
+        new arrow.Field(
+          "vector",
+          new arrow.FixedSizeList(
+            2,
+            new arrow.Field("item", new arrow.Float32(), true),
+          ),
+        ),
+        new arrow.Field("item", new arrow.Utf8(), true),
+        new arrow.Field("price", new arrow.Float32(), true),
+      ]);
+      const data = [
+        { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+        { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+      ];
+      const tbl = await db.createTable("myTable", data, {
+        schema,
+      });
+      // --8<-- [end:create_table_with_schema]
+      expect(await tbl.countRows()).toBe(2);
+    }
+
+    {
+      // --8<-- [start:create_empty_table]
+
+      const schema = new arrow.Schema([
+        new arrow.Field("id", new arrow.Int32()),
+        new arrow.Field("name", new arrow.Utf8()),
+      ]);
+
+      const emptyTbl = await db.createEmptyTable("empty_table", schema);
+      // --8<-- [end:create_empty_table]
+      expect(await emptyTbl.countRows()).toBe(0);
+    }
+    {
+      // --8<-- [start:open_table]
+      const _tbl = await db.openTable("myTable");
+      // --8<-- [end:open_table]
+    }
+
+    {
+      // --8<-- [start:table_names]
+      const tableNames = await db.tableNames();
+      // --8<-- [end:table_names]
+      expect(tableNames).toEqual(["empty_table", "myTable"]);
+    }
+
+    const tbl = await db.openTable("myTable");
+    {
+      // --8<-- [start:add_data]
+      const data = [
+        { vector: [1.3, 1.4], item: "fizz", price: 100.0 },
+        { vector: [9.5, 56.2], item: "buzz", price: 200.0 },
+      ];
+      await tbl.add(data);
+      // --8<-- [end:add_data]
+    }
+    {
+      // --8<-- [start:vector_search]
+      const res = await tbl.search([100, 100]).limit(2).toArray();
+      // --8<-- [end:vector_search]
+      expect(res.length).toBe(2);
+    }
+    {
+      const data = Array.from({ length: 1000 })
+        .fill(null)
+        .map(() => ({
+          vector: [Math.random(), Math.random()],
+          item: "autogen",
+          price: Math.round(Math.random() * 100),
+        }));
+
+      await tbl.add(data);
+    }
+
+    // --8<-- [start:create_index]
+    await tbl.createIndex("vector");
+    // --8<-- [end:create_index]
+
+    // --8<-- [start:delete_rows]
+    await tbl.delete('item = "fizz"');
+    // --8<-- [end:delete_rows]
+
+    // --8<-- [start:drop_table]
+    await db.dropTable("myTable");
+    // --8<-- [end:drop_table]
+    await db.dropTable("empty_table");
+
+    {
+      // --8<-- [start:create_f16_table]
+      const db = await lancedb.connect(databaseDir);
+      const dim = 16;
+      const total = 10;
+      const f16Schema = new Schema([
+        new Field("id", new Int32()),
+        new Field(
+          "vector",
+          new FixedSizeList(dim, new Field("item", new Float16(), true)),
+          false,
+        ),
+      ]);
+      const data = lancedb.makeArrowTable(
+        Array.from(Array(total), (_, i) => ({
+          id: i,
+          vector: Array.from(Array(dim), Math.random),
+        })),
+        { schema: f16Schema },
+      );
+      const _table = await db.createTable("f16_tbl", data);
+      // --8<-- [end:create_f16_table]
+      await db.dropTable("f16_tbl");
+    }
+  });
+});
--- a/nodejs/examples/basic.ts
+++ b/nodejs/examples/basic.ts
@@ -1,162 +0,0 @@
-// --8<--  [start:imports]
-import * as lancedb from "@lancedb/lancedb";
-import * as arrow from "apache-arrow";
-import {
-  Field,
-  FixedSizeList,
-  Float16,
-  Int32,
-  Schema,
-  Utf8,
-} from "apache-arrow";
-
-// --8<-- [end:imports]
-
-// --8<-- [start:connect]
-const uri = "/tmp/lancedb/";
-const db = await lancedb.connect(uri);
-// --8<-- [end:connect]
-{
-  // --8<-- [start:create_table]
-  const tbl = await db.createTable(
-    "myTable",
-    [
-      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-    ],
-    { mode: "overwrite" },
-  );
-  // --8<-- [end:create_table]
-
-  const data = [
-    { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-    { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-  ];
-
-  {
-    // --8<-- [start:create_table_exists_ok]
-    const tbl = await db.createTable("myTable", data, {
-      existsOk: true,
-    });
-    // --8<-- [end:create_table_exists_ok]
-  }
-  {
-    // --8<-- [start:create_table_overwrite]
-    const _tbl = await db.createTable("myTable", data, {
-      mode: "overwrite",
-    });
-    // --8<-- [end:create_table_overwrite]
-  }
-}
-
-{
-  // --8<-- [start:create_table_with_schema]
-  const schema = new arrow.Schema([
-    new arrow.Field(
-      "vector",
-      new arrow.FixedSizeList(
-        2,
-        new arrow.Field("item", new arrow.Float32(), true),
-      ),
-    ),
-    new arrow.Field("item", new arrow.Utf8(), true),
-    new arrow.Field("price", new arrow.Float32(), true),
-  ]);
-  const data = [
-    { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-    { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-  ];
-  const _tbl = await db.createTable("myTable", data, {
-    schema,
-  });
-  // --8<-- [end:create_table_with_schema]
-}
-
-{
-  // --8<-- [start:create_empty_table]
-
-  const schema = new arrow.Schema([
-    new arrow.Field("id", new arrow.Int32()),
-    new arrow.Field("name", new arrow.Utf8()),
-  ]);
-
-  const empty_tbl = await db.createEmptyTable("empty_table", schema);
-  // --8<-- [end:create_empty_table]
-}
-{
-  // --8<-- [start:open_table]
-  const _tbl = await db.openTable("myTable");
-  // --8<-- [end:open_table]
-}
-
-{
-  // --8<-- [start:table_names]
-  const tableNames = await db.tableNames();
-  console.log(tableNames);
-  // --8<-- [end:table_names]
-}
-
-const tbl = await db.openTable("myTable");
-{
-  // --8<-- [start:add_data]
-  const data = [
-    { vector: [1.3, 1.4], item: "fizz", price: 100.0 },
-    { vector: [9.5, 56.2], item: "buzz", price: 200.0 },
-  ];
-  await tbl.add(data);
-  // --8<-- [end:add_data]
-}
-{
-  // --8<-- [start:vector_search]
-  const _res = tbl.search([100, 100]).limit(2).toArray();
-  // --8<-- [end:vector_search]
-}
-{
-  const data = Array.from({ length: 1000 })
-    .fill(null)
-    .map(() => ({
-      vector: [Math.random(), Math.random()],
-      item: "autogen",
-      price: Math.round(Math.random() * 100),
-    }));
-
-  await tbl.add(data);
-}
-
-// --8<-- [start:create_index]
-await tbl.createIndex("vector");
-// --8<-- [end:create_index]
-
-// --8<-- [start:delete_rows]
-await tbl.delete('item = "fizz"');
-// --8<-- [end:delete_rows]
-
-// --8<-- [start:drop_table]
-await db.dropTable("myTable");
-// --8<-- [end:drop_table]
-await db.dropTable("empty_table");
-
-{
-  // --8<-- [start:create_f16_table]
-  const db = await lancedb.connect("/tmp/lancedb");
-  const dim = 16;
-  const total = 10;
-  const f16Schema = new Schema([
-    new Field("id", new Int32()),
-    new Field(
-      "vector",
-      new FixedSizeList(dim, new Field("item", new Float16(), true)),
-      false,
-    ),
-  ]);
-  const data = lancedb.makeArrowTable(
-    Array.from(Array(total), (_, i) => ({
-      id: i,
-      vector: Array.from(Array(dim), Math.random),
-    })),
-    { schema: f16Schema },
-  );
-  const _table = await db.createTable("f16_tbl", data);
-  // --8<-- [end:create_f16_table]
-  await db.dropTable("f16_tbl");
-}
--- a/nodejs/examples/cloud.test.ts
+++ b/nodejs/examples/cloud.test.ts
@@ -0,0 +1,230 @@
+// --8<--  [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+// --8<-- [end:imports]
+
+// --8<-- [start:generate_data]
+function genData(numRows: number, numVectorDim: number): any[] {
+    const data = [];
+    for (let i = 0; i < numRows; i++) {
+      const vector = [];
+      for (let j = 0; j < numVectorDim; j++) {
+        vector.push(i + j * 0.1);
+      }
+      data.push({
+        id: i,
+        name: `name_${i}`,
+        vector,
+      });
+    }
+    return data;
+}
+// --8<-- [end:generate_data]
+
+test("cloud quickstart", async () => {
+    {
+        // --8<-- [start:connect]
+        const db = await lancedb.connect({
+            uri: "db://your-project-slug",
+            apiKey: "your-api-key",
+            region: "your-cloud-region",
+        });
+        // --8<-- [end:connect]
+        // --8<-- [start:create_table]
+        const tableName = "myTable"
+        const data = genData(5000, 1536)
+        const table = await db.createTable(tableName, data);
+        // --8<-- [end:create_table]
+        // --8<-- [start:create_index_search]
+        // create a vector index
+        await table.createIndex({
+            column: "vector",
+            metric_type: lancedb.MetricType.Cosine,
+            type: "ivf_pq",
+        });
+        const result = await table.search([0.01, 0.02])
+            .select(["vector", "item"])
+            .limit(1)
+            .execute();
+        // --8<-- [end:create_index_search]
+        // --8<-- [start:drop_table]
+        await db.dropTable(tableName);
+        // --8<-- [end:drop_table]
+    }
+});
+
+test("ingest data", async () => {
+    // --8<-- [start:ingest_data]
+    import { Schema, Field, Float32, FixedSizeList, Utf8 } from "apache-arrow";
+
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+
+    const data = [
+        { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+        { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+        { vector: [10.2, 100.8], item: "baz", price: 30.0},
+        { vector: [1.4, 9.5], item: "fred", price: 40.0},
+    ]
+    // create an empty table with schema
+    const schema = new Schema([
+        new Field(
+        "vector",
+        new FixedSizeList(2, new Field("float32", new Float32())),
+        ),
+        new Field("item", new Utf8()),
+        new Field("price", new Float32()),
+    ]);
+    const tableName = "myTable";
+    const table = await db.createTable({
+        name: tableName,
+        schema,
+    });
+    await table.add(data);
+    // --8<-- [end:ingest_data]
+});
+
+test("update data", async () => {
+    // --8<-- [start:connect_db_and_open_table]
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+    const tableName = "myTable"
+    const table = await db.openTable(tableName);
+    // --8<-- [end:connect_db_and_open_table]
+    // --8<-- [start:update_data]
+    await table.update({
+        where: "price < 20.0",
+        values: { vector: [2, 2], item: "foo-updated" },
+    });
+    // --8<-- [end:update_data]
+    // --8<-- [start:merge_insert]
+      let newData = [
+        {vector: [1, 1], item: 'foo-updated', price: 50.0}
+      ];
+      // upsert
+      await table.mergeInsert("item", newData, {
+        whenMatchedUpdateAll: true,
+        whenNotMatchedInsertAll: true,
+      });
+      // --8<-- [end:merge_insert]
+      // --8<-- [start:delete_data]
+      // delete data
+      const predicate = "price = 30.0";
+      await table.delete(predicate);
+      // --8<-- [end:delete_data]
+});
+
+test("create index", async () => {
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+
+    const tableName = "myTable";
+    const table = await db.openTable(tableName);
+    // --8<-- [start:create_index]
+    // the vector column only needs to be specified when there are 
+    // multiple vector columns or the column is not named as "vector"
+    // L2 is used as the default distance metric
+    await table.createIndex({
+            column: "vector",
+            metric_type: lancedb.MetricType.Cosine,
+        });
+
+    // --8<-- [end:create_index]
+    // --8<-- [start:create_scalar_index]
+    await table.createScalarIndex("item");
+    // --8<-- [end:create_scalar_index]
+    // --8<-- [start:create_fts_index]
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+
+    const tableName = "myTable"
+    const data = [
+        { vector: [3.1, 4.1], text: "Frodo was a happy puppy" },
+        { vector: [5.9, 26.5], text: "There are several kittens playing" },
+    ];
+    const table = createTable(tableName, data);
+    await table.createIndex("text", {
+        config: lancedb.Index.fts(),
+    });
+    // --8<-- [end:create_fts_index]
+});
+
+test("vector search", async () => {
+    // --8<-- [start:vector_search]
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+
+    const tableName = "myTable"
+    const table = await db.openTable(tableName);
+    const result = await table.search([0.4, 1.4])
+        .where("price > 10.0")
+        .prefilter(true)
+        .select(["item", "vector"])
+        .limit(2)
+        .execute();
+    // --8<-- [end:vector_search]
+});
+
+test("full-text search", async () => {
+    // --8<-- [start:full_text_search]
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+
+    const data = [
+    { vector: [3.1, 4.1], text: "Frodo was a happy puppy" },
+    { vector: [5.9, 26.5], text: "There are several kittens playing" },
+    ];
+    const tableName = "myTable"
+    const table = await db.createTable(tableName, data);
+    await table.createIndex("text", {
+        config: lancedb.Index.fts(),
+    });
+
+    await tableName
+        .search("puppy", queryType="fts")
+        .select(["text"])
+        .limit(10)
+        .toArray();
+    // --8<-- [end:full_text_search]
+});
+
+test("metadata filtering", async () => {
+    // --8<-- [start:filtering]
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+    const tableName = "myTable"
+    const table = await db.openTable(tableName);
+    await table
+        .search(Array(2).fill(0.1))
+        .where("(item IN ('foo', 'bar')) AND (price > 10.0)")
+        .postfilter()
+        .toArray();
+    // --8<-- [end:filtering]
+    // --8<-- [start:sql_filtering]
+    await table
+        .search(Array(2).fill(0.1))
+        .where("(item IN ('foo', 'bar')) AND (price > 10.0)")
+        .postfilter()
+        .toArray();
+    // --8<-- [end:sql_filtering]
+});
--- a/nodejs/examples/custom_embedding_function.test.ts
+++ b/nodejs/examples/custom_embedding_function.test.ts
@@ -0,0 +1,76 @@
+import { FeatureExtractionPipeline, pipeline } from "@huggingface/transformers";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<-- [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import {
+  LanceSchema,
+  TextEmbeddingFunction,
+  getRegistry,
+  register,
+} from "@lancedb/lancedb/embedding";
+// --8<-- [end:imports]
+import { withTempDirectory } from "./util.ts";
+
+// --8<-- [start:embedding_impl]
+@register("sentence-transformers")
+class SentenceTransformersEmbeddings extends TextEmbeddingFunction {
+  name = "Xenova/all-miniLM-L6-v2";
+  #ndims!: number;
+  extractor!: FeatureExtractionPipeline;
+
+  async init() {
+    this.extractor = await pipeline("feature-extraction", this.name, {
+      dtype: "fp32",
+    });
+    this.#ndims = await this.generateEmbeddings(["hello"]).then(
+      (e) => e[0].length,
+    );
+  }
+
+  ndims() {
+    return this.#ndims;
+  }
+
+  toJSON() {
+    return {
+      name: this.name,
+    };
+  }
+  async generateEmbeddings(texts: string[]) {
+    const output = await this.extractor(texts, {
+      pooling: "mean",
+      normalize: true,
+    });
+    return output.tolist();
+  }
+}
+// -8<-- [end:embedding_impl]
+
+test("Registry examples", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:call_custom_function]
+    const registry = getRegistry();
+
+    const sentenceTransformer = await registry
+      .get<SentenceTransformersEmbeddings>("sentence-transformers")!
+      .create();
+
+    const schema = LanceSchema({
+      vector: sentenceTransformer.vectorField(),
+      text: sentenceTransformer.sourceField(),
+    });
+
+    const db = await lancedb.connect(databaseDir);
+    const table = await db.createEmptyTable("table", schema, {
+      mode: "overwrite",
+    });
+
+    await table.add([{ text: "hello" }, { text: "world" }]);
+
+    const results = await table.search("greeting").limit(1).toArray();
+    // -8<-- [end:call_custom_function]
+    expect(results.length).toBe(1);
+  });
+}, 100_000);
--- a/nodejs/examples/custom_embedding_function.ts
+++ b/nodejs/examples/custom_embedding_function.ts
@@ -1,64 +0,0 @@
-// --8<-- [start:imports]
-import * as lancedb from "@lancedb/lancedb";
-import {
-  LanceSchema,
-  TextEmbeddingFunction,
-  getRegistry,
-  register,
-} from "@lancedb/lancedb/embedding";
-import { pipeline } from "@xenova/transformers";
-// --8<-- [end:imports]
-
-// --8<-- [start:embedding_impl]
-@register("sentence-transformers")
-class SentenceTransformersEmbeddings extends TextEmbeddingFunction {
-  name = "Xenova/all-miniLM-L6-v2";
-  #ndims!: number;
-  extractor: any;
-
-  async init() {
-    this.extractor = await pipeline("feature-extraction", this.name);
-    this.#ndims = await this.generateEmbeddings(["hello"]).then(
-      (e) => e[0].length,
-    );
-  }
-
-  ndims() {
-    return this.#ndims;
-  }
-
-  toJSON() {
-    return {
-      name: this.name,
-    };
-  }
-  async generateEmbeddings(texts: string[]) {
-    const output = await this.extractor(texts, {
-      pooling: "mean",
-      normalize: true,
-    });
-    return output.tolist();
-  }
-}
-// -8<-- [end:embedding_impl]
-
-// --8<-- [start:call_custom_function]
-const registry = getRegistry();
-
-const sentenceTransformer = await registry
-  .get<SentenceTransformersEmbeddings>("sentence-transformers")!
-  .create();
-
-const schema = LanceSchema({
-  vector: sentenceTransformer.vectorField(),
-  text: sentenceTransformer.sourceField(),
-});
-
-const db = await lancedb.connect("/tmp/db");
-const table = await db.createEmptyTable("table", schema, { mode: "overwrite" });
-
-await table.add([{ text: "hello" }, { text: "world" }]);
-
-const results = await table.search("greeting").limit(1).toArray();
-console.log(results[0].text);
-// -8<-- [end:call_custom_function]
--- a/nodejs/examples/embedding.test.ts
+++ b/nodejs/examples/embedding.test.ts
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<-- [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import "@lancedb/lancedb/embedding/openai";
+import { LanceSchema, getRegistry, register } from "@lancedb/lancedb/embedding";
+import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
+import { type Float, Float32, Utf8 } from "apache-arrow";
+// --8<-- [end:imports]
+import { withTempDirectory } from "./util.ts";
+
+const openAiTest = process.env.OPENAI_API_KEY == null ? test.skip : test;
+
+openAiTest("openai embeddings", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:openai_embeddings]
+    const db = await lancedb.connect(databaseDir);
+    const func = getRegistry()
+      .get("openai")
+      ?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
+
+    const wordsSchema = LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+    const tbl = await db.createEmptyTable("words", wordsSchema, {
+      mode: "overwrite",
+    });
+    await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
+
+    const query = "greetings";
+    const actual = (await tbl.search(query).limit(1).toArray())[0];
+    // --8<-- [end:openai_embeddings]
+    expect(actual).toHaveProperty("text");
+  });
+});
+
+test("custom embedding function", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:embedding_function]
+    const db = await lancedb.connect(databaseDir);
+
+    @register("my_embedding")
+    class MyEmbeddingFunction extends EmbeddingFunction<string> {
+      toJSON(): object {
+        return {};
+      }
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): Float {
+        return new Float32();
+      }
+      async computeQueryEmbeddings(_data: string) {
+        // This is a placeholder for a real embedding function
+        return [1, 2, 3];
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        // This is a placeholder for a real embedding function
+        return Array.from({ length: data.length }).fill([
+          1, 2, 3,
+        ]) as number[][];
+      }
+    }
+
+    const func = new MyEmbeddingFunction();
+
+    const data = [{ text: "pepperoni" }, { text: "pineapple" }];
+
+    // Option 1: manually specify the embedding function
+    const table = await db.createTable("vectors", data, {
+      embeddingFunction: {
+        function: func,
+        sourceColumn: "text",
+        vectorColumn: "vector",
+      },
+      mode: "overwrite",
+    });
+
+    // Option 2: provide the embedding function through a schema
+
+    const schema = LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+
+    const table2 = await db.createTable("vectors2", data, {
+      schema,
+      mode: "overwrite",
+    });
+    // --8<-- [end:embedding_function]
+    expect(await table.countRows()).toBe(2);
+    expect(await table2.countRows()).toBe(2);
+  });
+});
--- a/nodejs/examples/embedding.ts
+++ b/nodejs/examples/embedding.ts
@@ -1,83 +0,0 @@
-// --8<-- [start:imports]
-import * as lancedb from "@lancedb/lancedb";
-import { LanceSchema, getRegistry, register } from "@lancedb/lancedb/embedding";
-import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
-import { type Float, Float32, Utf8 } from "apache-arrow";
-// --8<-- [end:imports]
-
-{
-  // --8<-- [start:openai_embeddings]
-
-  const db = await lancedb.connect("/tmp/db");
-  const func = getRegistry()
-    .get("openai")
-    ?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
-
-  const wordsSchema = LanceSchema({
-    text: func.sourceField(new Utf8()),
-    vector: func.vectorField(),
-  });
-  const tbl = await db.createEmptyTable("words", wordsSchema, {
-    mode: "overwrite",
-  });
-  await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
-
-  const query = "greetings";
-  const actual = (await (await tbl.search(query)).limit(1).toArray())[0];
-
-  // --8<-- [end:openai_embeddings]
-  console.log("result = ", actual.text);
-}
-
-{
-  // --8<-- [start:embedding_function]
-  const db = await lancedb.connect("/tmp/db");
-
-  @register("my_embedding")
-  class MyEmbeddingFunction extends EmbeddingFunction<string> {
-    toJSON(): object {
-      return {};
-    }
-    ndims() {
-      return 3;
-    }
-    embeddingDataType(): Float {
-      return new Float32();
-    }
-    async computeQueryEmbeddings(_data: string) {
-      // This is a placeholder for a real embedding function
-      return [1, 2, 3];
-    }
-    async computeSourceEmbeddings(data: string[]) {
-      // This is a placeholder for a real embedding function
-      return Array.from({ length: data.length }).fill([1, 2, 3]) as number[][];
-    }
-  }
-
-  const func = new MyEmbeddingFunction();
-
-  const data = [{ text: "pepperoni" }, { text: "pineapple" }];
-
-  // Option 1: manually specify the embedding function
-  const table = await db.createTable("vectors", data, {
-    embeddingFunction: {
-      function: func,
-      sourceColumn: "text",
-      vectorColumn: "vector",
-    },
-    mode: "overwrite",
-  });
-
-  // Option 2: provide the embedding function through a schema
-
-  const schema = LanceSchema({
-    text: func.sourceField(new Utf8()),
-    vector: func.vectorField(),
-  });
-
-  const table2 = await db.createTable("vectors2", data, {
-    schema,
-    mode: "overwrite",
-  });
-  // --8<-- [end:embedding_function]
-}
--- a/nodejs/examples/filtering.test.ts
+++ b/nodejs/examples/filtering.test.ts
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+import * as lancedb from "@lancedb/lancedb";
+import { withTempDirectory } from "./util.ts";
+
+test("filtering examples", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    const db = await lancedb.connect(databaseDir);
+
+    const data = Array.from({ length: 10_000 }, (_, i) => ({
+      vector: Array(1536).fill(i),
+      id: i,
+      item: `item ${i}`,
+      strId: `${i}`,
+    }));
+
+    const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
+
+    // --8<-- [start:search]
+    const _result = await tbl
+      .search(Array(1536).fill(0.5))
+      .limit(1)
+      .where("id = 10")
+      .toArray();
+    // --8<-- [end:search]
+
+    // --8<-- [start:vec_search]
+    const result = await (
+      tbl.search(Array(1536).fill(0)) as lancedb.VectorQuery
+    )
+      .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
+      .postfilter()
+      .toArray();
+    // --8<-- [end:vec_search]
+    expect(result.length).toBe(0);
+
+    // --8<-- [start:sql_search]
+    await tbl.query().where("id = 10").limit(10).toArray();
+    // --8<-- [end:sql_search]
+  });
+});
--- a/nodejs/examples/filtering.ts
+++ b/nodejs/examples/filtering.ts
@@ -1,34 +0,0 @@
-import * as lancedb from "@lancedb/lancedb";
-
-const db = await lancedb.connect("data/sample-lancedb");
-
-const data = Array.from({ length: 10_000 }, (_, i) => ({
-  vector: Array(1536).fill(i),
-  id: i,
-  item: `item ${i}`,
-  strId: `${i}`,
-}));
-
-const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
-
-// --8<-- [start:search]
-const _result = await tbl
-  .search(Array(1536).fill(0.5))
-  .limit(1)
-  .where("id = 10")
-  .toArray();
-// --8<-- [end:search]
-
-// --8<-- [start:vec_search]
-await tbl
-  .search(Array(1536).fill(0))
-  .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
-  .postfilter()
-  .toArray();
-// --8<-- [end:vec_search]
-
-// --8<-- [start:sql_search]
-await tbl.query().where("id = 10").limit(10).toArray();
-// --8<-- [end:sql_search]
-
-console.log("SQL search: done");
--- a/nodejs/examples/full_text_search.test.ts
+++ b/nodejs/examples/full_text_search.test.ts
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+import * as lancedb from "@lancedb/lancedb";
+import { withTempDirectory } from "./util.ts";
+
+test("full text search", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    const db = await lancedb.connect(databaseDir);
+
+    const words = [
+      "apple",
+      "banana",
+      "cherry",
+      "date",
+      "elderberry",
+      "fig",
+      "grape",
+    ];
+
+    const data = Array.from({ length: 10_000 }, (_, i) => ({
+      vector: Array(1536).fill(i),
+      id: i,
+      item: `item ${i}`,
+      strId: `${i}`,
+      doc: words[i % words.length],
+    }));
+
+    const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
+
+    await tbl.createIndex("doc", {
+      config: lancedb.Index.fts(),
+    });
+
+    // --8<-- [start:full_text_search]
+    const result = await tbl
+      .query()
+      .nearestToText("apple")
+      .select(["id", "doc"])
+      .limit(10)
+      .toArray();
+    expect(result.length).toBe(10);
+    // --8<-- [end:full_text_search]
+  });
+});
--- a/nodejs/examples/full_text_search.ts
+++ b/nodejs/examples/full_text_search.ts
@@ -1,52 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import * as lancedb from "@lancedb/lancedb";
-
-const db = await lancedb.connect("data/sample-lancedb");
-
-const words = [
-  "apple",
-  "banana",
-  "cherry",
-  "date",
-  "elderberry",
-  "fig",
-  "grape",
-];
-
-const data = Array.from({ length: 10_000 }, (_, i) => ({
-  vector: Array(1536).fill(i),
-  id: i,
-  item: `item ${i}`,
-  strId: `${i}`,
-  doc: words[i % words.length],
-}));
-
-const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
-
-await tbl.createIndex("doc", {
-  config: lancedb.Index.fts(),
-});
-
-// --8<-- [start:full_text_search]
-let result = await tbl
-  .search("apple")
-  .select(["id", "doc"])
-  .limit(10)
-  .toArray();
-console.log(result);
-// --8<-- [end:full_text_search]
-
-console.log("SQL search: done");
--- a/nodejs/examples/jest.config.cjs
+++ b/nodejs/examples/jest.config.cjs
@@ -0,0 +1,6 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  testPathIgnorePatterns: ["./dist"],
+};
--- a/nodejs/examples/jsconfig.json
+++ b/nodejs/examples/jsconfig.json
@@ -1,27 +0,0 @@
-{
-  "compilerOptions": {
-    // Enable latest features
-    "lib": ["ESNext", "DOM"],
-    "target": "ESNext",
-    "module": "ESNext",
-    "moduleDetection": "force",
-    "jsx": "react-jsx",
-    "allowJs": true,
-
-    // Bundler mode
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "verbatimModuleSyntax": true,
-    "noEmit": true,
-
-    // Best practices
-    "strict": true,
-    "skipLibCheck": true,
-    "noFallthroughCasesInSwitch": true,
-
-    // Some stricter flags (disabled by default)
-    "noUnusedLocals": false,
-    "noUnusedParameters": false,
-    "noPropertyAccessFromIndexSignature": false
-  }
-}
--- a/nodejs/examples/package-lock.json
+++ b/nodejs/examples/package-lock.json
--- a/nodejs/examples/package.json
+++ b/nodejs/examples/package.json
@@ -5,24 +5,29 @@
  "main": "index.js",
  "type": "module",
  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
+    "//2": "--testEnvironment is needed to run jest with sentence-transformers",
+    "//3": "See: https://github.com/huggingface/transformers.js/issues/57",
+    "test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
+    "lint": "biome check *.ts && biome format *.ts",
+    "lint-ci": "biome ci .",
+    "lint-fix": "biome check --write *.ts && npm run format",
+    "format": "biome format --write *.ts"
  },
  "author": "Lance Devs",
  "license": "Apache-2.0",
  "dependencies": {
-    "@lancedb/lancedb": "file:../",
-    "@xenova/transformers": "^2.17.2"
+    "@huggingface/transformers": "^3.0.2",
+    "@lancedb/lancedb": "file:../dist",
+    "openai": "^4.29.2",
+    "sharp": "^0.33.5"
  },
  "devDependencies": {
+    "@biomejs/biome": "^1.7.3",
+    "@jest/globals": "^29.7.0",
+    "jest": "^29.7.0",
+    "jest-environment-node-single-context": "^29.4.0",
+    "ts-jest": "^29.2.5",
    "typescript": "^5.5.4"
-  },
-  "compilerOptions": {
-    "target": "ESNext",
-    "module": "ESNext",
-    "moduleResolution": "Node",
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true
  }
 }
--- a/nodejs/examples/search.test.ts
+++ b/nodejs/examples/search.test.ts
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<-- [start:import]
+import * as lancedb from "@lancedb/lancedb";
+// --8<-- [end:import]
+import { withTempDirectory } from "./util.ts";
+
+test("full text search", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    {
+      const db = await lancedb.connect(databaseDir);
+
+      const data = Array.from({ length: 10_000 }, (_, i) => ({
+        vector: Array(128).fill(i),
+        id: `${i}`,
+        content: "",
+        longId: `${i}`,
+      }));
+
+      await db.createTable("my_vectors", data);
+    }
+
+    // --8<-- [start:search1]
+    const db = await lancedb.connect(databaseDir);
+    const tbl = await db.openTable("my_vectors");
+
+    const results1 = await tbl.search(Array(128).fill(1.2)).limit(10).toArray();
+    // --8<-- [end:search1]
+    expect(results1.length).toBe(10);
+
+    // --8<-- [start:search2]
+    const results2 = await (
+      tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery
+    )
+      .distanceType("cosine")
+      .limit(10)
+      .toArray();
+    // --8<-- [end:search2]
+    expect(results2.length).toBe(10);
+  });
+});
--- a/nodejs/examples/search.ts
+++ b/nodejs/examples/search.ts
@@ -1,38 +0,0 @@
-// --8<-- [end:import]
-import * as fs from "node:fs";
-// --8<-- [start:import]
-import * as lancedb from "@lancedb/lancedb";
-
-async function setup() {
-  fs.rmSync("data/sample-lancedb", { recursive: true, force: true });
-  const db = await lancedb.connect("data/sample-lancedb");
-
-  const data = Array.from({ length: 10_000 }, (_, i) => ({
-    vector: Array(1536).fill(i),
-    id: `${i}`,
-    content: "",
-    longId: `${i}`,
-  }));
-
-  await db.createTable("my_vectors", data);
-}
-
-await setup();
-
-// --8<-- [start:search1]
-const db = await lancedb.connect("data/sample-lancedb");
-const tbl = await db.openTable("my_vectors");
-
-const _results1 = await tbl.search(Array(1536).fill(1.2)).limit(10).toArray();
-// --8<-- [end:search1]
-
-// --8<-- [start:search2]
-const _results2 = await tbl
-  .search(Array(1536).fill(1.2))
-  .distanceType("cosine")
-  .limit(10)
-  .toArray();
-console.log(_results2);
-// --8<-- [end:search2]
-
-console.log("search: done");
--- a/nodejs/examples/sentence-transformers.js
+++ b/nodejs/examples/sentence-transformers.js
@@ -1,50 +0,0 @@
-import * as lancedb from "@lancedb/lancedb";
-
-import { LanceSchema, getRegistry } from "@lancedb/lancedb/embedding";
-import { Utf8 } from "apache-arrow";
-
-const db = await lancedb.connect("/tmp/db");
-const func = await getRegistry().get("huggingface").create();
-
-const facts = [
-  "Albert Einstein was a theoretical physicist.",
-  "The capital of France is Paris.",
-  "The Great Wall of China is one of the Seven Wonders of the World.",
-  "Python is a popular programming language.",
-  "Mount Everest is the highest mountain in the world.",
-  "Leonardo da Vinci painted the Mona Lisa.",
-  "Shakespeare wrote Hamlet.",
-  "The human body has 206 bones.",
-  "The speed of light is approximately 299,792 kilometers per second.",
-  "Water boils at 100 degrees Celsius.",
-  "The Earth orbits the Sun.",
-  "The Pyramids of Giza are located in Egypt.",
-  "Coffee is one of the most popular beverages in the world.",
-  "Tokyo is the capital city of Japan.",
-  "Photosynthesis is the process by which plants make their food.",
-  "The Pacific Ocean is the largest ocean on Earth.",
-  "Mozart was a prolific composer of classical music.",
-  "The Internet is a global network of computers.",
-  "Basketball is a sport played with a ball and a hoop.",
-  "The first computer virus was created in 1983.",
-  "Artificial neural networks are inspired by the human brain.",
-  "Deep learning is a subset of machine learning.",
-  "IBM's Watson won Jeopardy! in 2011.",
-  "The first computer programmer was Ada Lovelace.",
-  "The first chatbot was ELIZA, created in the 1960s.",
-].map((text) => ({ text }));
-
-const factsSchema = LanceSchema({
-  text: func.sourceField(new Utf8()),
-  vector: func.vectorField(),
-});
-
-const tbl = await db.createTable("facts", facts, {
-  mode: "overwrite",
-  schema: factsSchema,
-});
-
-const query = "How many bones are in the human body?";
-const actual = await tbl.search(query).limit(1).toArray();
-
-console.log("Answer: ", actual[0]["text"]);
--- a/nodejs/examples/sentence-transformers.test.ts
+++ b/nodejs/examples/sentence-transformers.test.ts
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+import { withTempDirectory } from "./util.ts";
+
+import * as lancedb from "@lancedb/lancedb";
+import "@lancedb/lancedb/embedding/transformers";
+import { LanceSchema, getRegistry } from "@lancedb/lancedb/embedding";
+import { Utf8 } from "apache-arrow";
+
+test("full text search", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    const db = await lancedb.connect(databaseDir);
+    const func = await getRegistry().get("huggingface").create();
+
+    const facts = [
+      "Albert Einstein was a theoretical physicist.",
+      "The capital of France is Paris.",
+      "The Great Wall of China is one of the Seven Wonders of the World.",
+      "Python is a popular programming language.",
+      "Mount Everest is the highest mountain in the world.",
+      "Leonardo da Vinci painted the Mona Lisa.",
+      "Shakespeare wrote Hamlet.",
+      "The human body has 206 bones.",
+      "The speed of light is approximately 299,792 kilometers per second.",
+      "Water boils at 100 degrees Celsius.",
+      "The Earth orbits the Sun.",
+      "The Pyramids of Giza are located in Egypt.",
+      "Coffee is one of the most popular beverages in the world.",
+      "Tokyo is the capital city of Japan.",
+      "Photosynthesis is the process by which plants make their food.",
+      "The Pacific Ocean is the largest ocean on Earth.",
+      "Mozart was a prolific composer of classical music.",
+      "The Internet is a global network of computers.",
+      "Basketball is a sport played with a ball and a hoop.",
+      "The first computer virus was created in 1983.",
+      "Artificial neural networks are inspired by the human brain.",
+      "Deep learning is a subset of machine learning.",
+      "IBM's Watson won Jeopardy! in 2011.",
+      "The first computer programmer was Ada Lovelace.",
+      "The first chatbot was ELIZA, created in the 1960s.",
+    ].map((text) => ({ text }));
+
+    const factsSchema = LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+
+    const tbl = await db.createTable("facts", facts, {
+      mode: "overwrite",
+      schema: factsSchema,
+    });
+
+    const query = "How many bones are in the human body?";
+    const actual = await tbl.search(query).limit(1).toArray();
+
+    expect(actual[0]["text"]).toBe("The human body has 206 bones.");
+  });
+});
--- a/nodejs/examples/tsconfig.json
+++ b/nodejs/examples/tsconfig.json
@@ -0,0 +1,17 @@
+{
+  "include": ["*.test.ts"],
+  "compilerOptions": {
+    "target": "es2022",
+    "module": "NodeNext",
+    "declaration": true,
+    "outDir": "./dist",
+    "strict": true,
+    "allowJs": true,
+    "resolveJsonModule": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "moduleResolution": "NodeNext",
+    "allowImportingTsExtensions": true,
+    "emitDeclarationOnly": true
+  }
+}
--- a/nodejs/examples/util.ts
+++ b/nodejs/examples/util.ts
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import * as fs from "fs";
+import { tmpdir } from "os";
+import * as path from "path";
+
+export async function withTempDirectory(
+  fn: (tempDir: string) => Promise<void>,
+) {
+  const tmpDirPath = fs.mkdtempSync(path.join(tmpdir(), "temp-dir-"));
+  try {
+    await fn(tmpDirPath);
+  } finally {
+    fs.rmSync(tmpDirPath, { recursive: true });
+  }
+}
--- a/nodejs/jest.config.js
+++ b/nodejs/jest.config.js
@@ -4,4 +4,5 @@ module.exports = {
  testEnvironment: "node",
  moduleDirectories: ["node_modules", "./dist"],
  moduleFileExtensions: ["js", "ts"],
+  modulePathIgnorePatterns: ["<rootDir>/examples/"],
 };
--- a/nodejs/lancedb/embedding/transformers.ts
+++ b/nodejs/lancedb/embedding/transformers.ts
@@ -47,8 +47,8 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
  string,
  Partial<XenovaTransformerOptions>
 > {
-  #model?: import("@xenova/transformers").PreTrainedModel;
-  #tokenizer?: import("@xenova/transformers").PreTrainedTokenizer;
+  #model?: import("@huggingface/transformers").PreTrainedModel;
+  #tokenizer?: import("@huggingface/transformers").PreTrainedTokenizer;
  #modelName: XenovaTransformerOptions["model"];
  #initialized = false;
  #tokenizerOptions: XenovaTransformerOptions["tokenizerOptions"];
@@ -92,18 +92,19 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
    try {
      // SAFETY:
      // since typescript transpiles `import` to `require`, we need to do this in an unsafe way
-      // We can't use `require` because `@xenova/transformers` is an ESM module
+      // We can't use `require` because `@huggingface/transformers` is an ESM module
      // and we can't use `import` directly because typescript will transpile it to `require`.
      // and we want to remain compatible with both ESM and CJS modules
      // so we use `eval` to bypass typescript for this specific import.
-      transformers = await eval('import("@xenova/transformers")');
+      transformers = await eval('import("@huggingface/transformers")');
    } catch (e) {
-      throw new Error(`error loading @xenova/transformers\nReason: ${e}`);
+      throw new Error(`error loading @huggingface/transformers\nReason: ${e}`);
    }

    try {
      this.#model = await transformers.AutoModel.from_pretrained(
        this.#modelName,
+        { dtype: "fp32" },
      );
    } catch (e) {
      throw new Error(
@@ -128,7 +129,8 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
    } else {
      const config = this.#model!.config;

-      const ndims = config["hidden_size"];
+      // biome-ignore lint/style/useNamingConvention: we don't control this name.
+      const ndims = (config as unknown as { hidden_size: number }).hidden_size;
      if (!ndims) {
        throw new Error(
          "hidden_size not found in model config, you may need to manually specify the embedding dimensions. ",
@@ -183,7 +185,7 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
 }

 const tensorDiv = (
-  src: import("@xenova/transformers").Tensor,
+  src: import("@huggingface/transformers").Tensor,
  divBy: number,
 ) => {
  for (let i = 0; i < src.data.length; ++i) {
--- a/nodejs/lancedb/query.ts
+++ b/nodejs/lancedb/query.ts
@@ -571,4 +571,9 @@ export class Query extends QueryBase<NativeQuery> {
      return new VectorQuery(vectorQuery);
    }
  }
+
+  nearestToText(query: string, columns?: string[]): Query {
+    this.doCall((inner) => inner.fullTextSearch(query, columns));
+    return this;
+  }
 }
--- a/nodejs/package-lock.json
+++ b/nodejs/package-lock.json
--- a/nodejs/package.json
+++ b/nodejs/package.json
@@ -85,7 +85,7 @@
    "reflect-metadata": "^0.2.2"
  },
  "optionalDependencies": {
-    "@xenova/transformers": ">=2.17 < 3",
+    "@huggingface/transformers": "^3.0.2",
    "openai": "^4.29.2"
  },
  "peerDependencies": {
--- a/nodejs/tsconfig.json
+++ b/nodejs/tsconfig.json
@@ -12,7 +12,7 @@
    "experimentalDecorators": true,
    "moduleResolution": "Node"
  },
-  "exclude": ["./dist/*"],
+  "exclude": ["./dist/*", "./examples/*"],
  "typedocOptions": {
    "entryPoints": ["lancedb/index.ts"],
    "out": "../docs/src/javascript/",
--- a/python/python/lancedb/embeddings/registry.py
+++ b/python/python/lancedb/embeddings/registry.py
@@ -1,15 +1,6 @@
-#  Copyright (c) 2023. LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 import json
 from typing import Dict, Optional

@@ -170,7 +161,7 @@ def register(name):
    return __REGISTRY__.get_instance().register(name)


-def get_registry():
+def get_registry() -> EmbeddingFunctionRegistry:
    """
    Utility function to get the global instance of the registry

--- a/python/python/lancedb/index.py
+++ b/python/python/lancedb/index.py
@@ -110,16 +110,7 @@ class FTS:
        remove_stop_words: bool = False,
        ascii_folding: bool = False,
    ):
-        self._inner = LanceDbIndex.fts(
-            with_position=with_position,
-            base_tokenizer=base_tokenizer,
-            language=language,
-            max_token_length=max_token_length,
-            lower_case=lower_case,
-            stem=stem,
-            remove_stop_words=remove_stop_words,
-            ascii_folding=ascii_folding,
-        )
+        self._inner = LanceDbIndex.fts(with_position=with_position)


 class HnswPq:
--- a/python/python/lancedb/remote/table.py
+++ b/python/python/lancedb/remote/table.py
@@ -132,25 +132,8 @@ class RemoteTable(Table):
        *,
        replace: bool = False,
        with_position: bool = True,
-        # tokenizer configs:
-        base_tokenizer: str = "simple",
-        language: str = "English",
-        max_token_length: Optional[int] = 40,
-        lower_case: bool = True,
-        stem: bool = False,
-        remove_stop_words: bool = False,
-        ascii_folding: bool = False,
    ):
-        config = FTS(
-            with_position=with_position,
-            base_tokenizer=base_tokenizer,
-            language=language,
-            max_token_length=max_token_length,
-            lower_case=lower_case,
-            stem=stem,
-            remove_stop_words=remove_stop_words,
-            ascii_folding=ascii_folding,
-        )
+        config = FTS(with_position=with_position)
        self._loop.run_until_complete(
            self._table.create_index(column, config=config, replace=replace)
        )
--- a/python/python/lancedb/rerankers/voyageai.py
+++ b/python/python/lancedb/rerankers/voyageai.py
@@ -13,7 +13,7 @@

 import os
 from functools import cached_property
-from typing import Union, Optional
+from typing import Optional

 import pyarrow as pa

--- a/python/python/lancedb/table.py
+++ b/python/python/lancedb/table.py
@@ -73,6 +73,21 @@ pl = safe_import_polars()
 QueryType = Literal["vector", "fts", "hybrid", "auto"]


+def _pd_schema_without_embedding_funcs(
+    schema: Optional[pa.Schema], columns: List[str]
+) -> Optional[pa.Schema]:
+    """Return a schema without any embedding function columns"""
+    if schema is None:
+        return None
+    embedding_functions = EmbeddingFunctionRegistry.get_instance().parse_functions(
+        schema.metadata
+    )
+    if not embedding_functions:
+        return schema
+    columns = set(columns)
+    return pa.schema([field for field in schema if field.name in columns])
+
+
 def _coerce_to_table(data, schema: Optional[pa.Schema] = None) -> pa.Table:
    if _check_for_hugging_face(data):
        # Huggingface datasets
@@ -103,10 +118,10 @@ def _coerce_to_table(data, schema: Optional[pa.Schema] = None) -> pa.Table:
        elif isinstance(data[0], pa.RecordBatch):
            return pa.Table.from_batches(data, schema=schema)
        else:
-            return pa.Table.from_pylist(data)
+            return pa.Table.from_pylist(data, schema=schema)
    elif _check_for_pandas(data) and isinstance(data, pd.DataFrame):
-        # Do not add schema here, since schema may contains the vector column
-        table = pa.Table.from_pandas(data, preserve_index=False)
+        raw_schema = _pd_schema_without_embedding_funcs(schema, data.columns.to_list())
+        table = pa.Table.from_pandas(data, preserve_index=False, schema=raw_schema)
        # Do not serialize Pandas metadata
        meta = table.schema.metadata if table.schema.metadata is not None else {}
        meta = {k: v for k, v in meta.items() if k != b"pandas"}
@@ -172,6 +187,8 @@ def sanitize_create_table(
        schema = schema.to_arrow_schema()

    if data is not None:
+        if metadata is None and schema is not None:
+            metadata = schema.metadata
        data, schema = _sanitize_data(
            data,
            schema,
--- a/python/python/tests/docs/test_cloud.py
+++ b/python/python/tests/docs/test_cloud.py
@@ -0,0 +1,293 @@
+# --8<-- [start:imports]
+# --8<-- [start:import-lancedb]
+# --8<-- [start:import-ingest-data]
+import lancedb
+import pyarrow as pa
+# --8<-- [end:import-ingest-data]
+import numpy as np
+
+# --8<-- [end:import-lancedb]
+# --8<-- [end:imports]
+# --8<-- [start:gen_data]
+def gen_data(total_rows: int, ndims: int = 1536):
+    return pa.RecordBatch.from_pylist(
+        [
+            {
+                "vector": np.random.rand(ndims).astype(np.float32).tolist(),
+                "id": i,
+                "name": "name_" + str(i),
+            }
+            for i in range(total_rows)
+        ],
+    ).to_pandas()
+
+
+# --8<-- [end:gen_data]
+
+
+def test_cloud_quickstart():
+    # --8<-- [start:connect]
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="your-cloud-region"
+    )
+    # --8<-- [end:connect]
+    # --8<-- [start:create_table]
+    table_name = "myTable"
+    table = db.create_table(table_name, data=gen_data(5000))
+    # --8<-- [end:create_table]
+    # --8<-- [start:create_index_search]
+    # create a vector index
+    table.create_index("cosine", vector_column_name="vector")
+    result = table.search([0.01, 0.02]).select(["vector", "item"]).limit(1).to_pandas()
+    print(result)
+    # --8<-- [end:create_index_search]
+    # --8<-- [start:drop_table]
+    db.drop_table(table_name)
+    # --8<-- [end:drop_table]
+
+
+def test_ingest_data():
+    # --8<-- [start:ingest_data]
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+
+    # create an empty table with schema
+    table_name = "myTable"
+    data = [
+        {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
+        {"vector": [10.2, 100.8], "item": "baz", "price": 30.0},
+        {"vector": [1.4, 9.5], "item": "fred", "price": 40.0},
+    ]
+    schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float32(), 2)),
+            pa.field("item", pa.utf8()),
+            pa.field("price", pa.float32()),
+        ]
+    )
+    table = db.create_table(table_name, schema=schema)
+    table.add(data)
+    # --8<-- [end:ingest_data]
+    # --8<-- [start:ingest_data_in_batch]
+    def make_batches():
+        for i in range(5):
+            yield pa.RecordBatch.from_arrays(
+                [
+                    pa.array([[3.1, 4.1], [5.9, 26.5]], pa.list_(pa.float32(), 2)),
+                    pa.array(["foo", "bar"]),
+                    pa.array([10.0, 20.0]),
+                ],
+                ["vector", "item", "price"],
+            )
+
+    schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float32(), 2)),
+            pa.field("item", pa.utf8()),
+            pa.field("price", pa.float32()),
+        ]
+    )
+    db.create_table("table2", make_batches(), schema=schema)
+    # --8<-- [end:ingest_data_in_batch]
+
+
+def test_updates():
+    # --8<-- [start:update_data]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+    table_name = "myTable"
+    table = db.open_table(table_name)
+    table.update(where="price < 20.0", values={"vector": [2, 2], "item": "foo-updated"})
+    # --8<-- [end:update_data]
+    # --8<-- [start:merge_insert]
+    table = db.open_table(table_name)
+    # upsert
+    new_data = [{"vector": [1, 1], "item": "foo-updated", "price": 50.0}]
+    table.merge_insert(
+        "item"
+    ).when_matched_update_all().when_not_matched_insert_all().execute(new_data)
+    # --8<-- [end:merge_insert]
+    # --8<-- [start:delete_data]
+    table_name = "myTable"
+    table = db.open_table(table_name)
+    # delete data
+    predicate = "price = 30.0"
+    table.delete(predicate)
+    # --8<-- [end:delete_data]
+
+
+def test_create_index():
+    # --8<-- [start:create_index]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+
+    table_name = "myTable"
+    table = db.open_table(table_name)
+    # the vector column only needs to be specified when there are
+    # multiple vector columns or the column is not named as "vector"
+    # L2 is used as the default distance metric
+    table.create_index(metric="cosine", vector_column_name="vector")
+    # --8<-- [end:create_index]
+
+
+def test_create_scalar_index():
+    # --8<-- [start:create_scalar_index]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+
+    table_name = "myTable"
+    table = db.open_table(table_name)
+    # default is BTree
+    table.create_scalar_index("item", index_type="BITMAP")
+    # --8<-- [end:create_scalar_index]
+
+
+def test_create_fts_index():
+    # --8<-- [start:create_fts_index]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+
+    table_name = "myTable"
+    data = [
+        {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+        {"vector": [5.9, 26.5], "text": "There are several kittens playing"},
+    ]
+    table = db.create_table(table_name, data=data)
+    table.create_fts_index("text")
+    # --8<-- [end:create_fts_index]
+
+
+def test_search():
+    # --8<-- [start:vector_search]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+
+    table_name = "myTable"
+    table = db.open_table(table_name)
+    query = [0.4, 1.4]
+    result = (
+        table.search(query)
+        .where("price > 10.0", prefilter=True)
+        .select(["item", "vector"])
+        .limit(2)
+        .to_pandas()
+    )
+    print(result)
+    # --8<-- [end:vector_search]
+    # --8<-- [start:full_text_search]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+    table_name = "myTable"
+    table = db.create_table(
+        table_name,
+        data=[
+            {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+            {"vector": [5.9, 26.5], "text": "There are several kittens playing"},
+        ],
+    )
+
+    table.create_fts_index("text")
+    table.search("puppy", query_type="fts").limit(10).select(["text"]).to_list()
+    # --8<-- [end:full_text_search]
+    # --8<-- [start:hybrid_search]
+    import os
+
+    import lancedb
+    import openai
+    from lancedb.embeddings import get_registry
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.rerankers import RRFReranker
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+
+    # Configuring the environment variable OPENAI_API_KEY
+    if "OPENAI_API_KEY" not in os.environ:
+        # OR set the key here as a variable
+        openai.api_key = "sk-..."
+    embeddings = get_registry().get("openai").create()
+
+    class Documents(LanceModel):
+        text: str = embeddings.SourceField()
+        vector: Vector(embeddings.ndims()) = embeddings.VectorField()
+
+    table_name = "myTable"
+    table = db.create_table(table_name, schema=Documents)
+    data = [
+        {"text": "rebel spaceships striking from a hidden base"},
+        {"text": "have won their first victory against the evil Galactic Empire"},
+        {"text": "during the battle rebel spies managed to steal secret plans"},
+        {"text": "to the Empire's ultimate weapon the Death Star"},
+    ]
+    table.add(data=data)
+    table.create_index("L2", "vector")
+    table.create_fts_index("text")
+
+    # you can use table.list_indices() to make sure indices have been created
+    reranker = RRFReranker()
+    result = (
+        table.search(
+            "flower moon",
+            query_type="hybrid",
+            vector_column_name="vector",
+            fts_columns="text",
+        )
+        .rerank(reranker)
+        .limit(10)
+        .to_pandas()
+    )
+    print(result)
+    # --8<-- [end:hybrid_search]
+
+
+def test_filtering():
+    # --8<-- [start:filtering]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+    table_name = "myTable"
+    table = db.open_table(table_name)
+    result = (
+        table.search([100, 102])
+        .where("(item IN ('foo', 'bar')) AND (price > 10.0)")
+        .to_arrow()
+    )
+    print(result)
+    # --8<-- [end:filtering]
+    # --8<-- [start:sql_filtering]
+    table.search([100, 102]).where(
+        "(item IN ('foo', 'bar')) AND (price > 10.0)"
+    ).to_arrow()
+    # --8<-- [end:sql_filtering]
--- a/python/python/tests/test_embeddings.py
+++ b/python/python/tests/test_embeddings.py
@@ -1,15 +1,6 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 from typing import List, Union
 from unittest.mock import MagicMock, patch

@@ -18,6 +9,7 @@ import lancedb
 import numpy as np
 import pyarrow as pa
 import pytest
+import pandas as pd
 from lancedb.conftest import MockTextEmbeddingFunction
 from lancedb.embeddings import (
    EmbeddingFunctionConfig,
@@ -129,6 +121,142 @@ def test_embedding_with_bad_results(tmp_path):
    # assert tbl["vector"].null_count == 1


+def test_with_existing_vectors(tmp_path):
+    @register("mock-embedding")
+    class MockEmbeddingFunction(TextEmbeddingFunction):
+        def ndims(self):
+            return 128
+
+        def generate_embeddings(
+            self, texts: Union[List[str], np.ndarray]
+        ) -> List[np.array]:
+            return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
+
+    registry = get_registry()
+    model = registry.get("mock-embedding").create()
+
+    class Schema(LanceModel):
+        text: str = model.SourceField()
+        vector: Vector(model.ndims()) = model.VectorField()
+
+    db = lancedb.connect(tmp_path)
+    tbl = db.create_table("test", schema=Schema, mode="overwrite")
+    tbl.add([{"text": "hello world", "vector": np.zeros(128).tolist()}])
+
+    embeddings = tbl.to_arrow()["vector"].to_pylist()
+    assert not np.any(embeddings), "all zeros"
+
+
+def test_embedding_function_with_pandas(tmp_path):
+    @register("mock-embedding")
+    class _MockEmbeddingFunction(TextEmbeddingFunction):
+        def ndims(self):
+            return 128
+
+        def generate_embeddings(
+            self, texts: Union[List[str], np.ndarray]
+        ) -> List[np.array]:
+            return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
+
+    registery = get_registry()
+    func = registery.get("mock-embedding").create()
+
+    class TestSchema(LanceModel):
+        text: str = func.SourceField()
+        val: int
+        vector: Vector(func.ndims()) = func.VectorField()
+
+    df = pd.DataFrame(
+        {
+            "text": ["hello world", "goodbye world"],
+            "val": [1, 2],
+            "not-used": ["s1", "s3"],
+        }
+    )
+    db = lancedb.connect(tmp_path)
+    tbl = db.create_table("test", schema=TestSchema, mode="overwrite", data=df)
+    schema = tbl.schema
+    assert schema.field("text").type == pa.string()
+    assert schema.field("val").type == pa.int64()
+    assert schema.field("vector").type == pa.list_(pa.float32(), 128)
+
+    df = pd.DataFrame(
+        {
+            "text": ["extra", "more"],
+            "val": [4, 5],
+            "misc-col": ["s1", "s3"],
+        }
+    )
+    tbl.add(df)
+
+    assert tbl.count_rows() == 4
+    embeddings = tbl.to_arrow()["vector"]
+    assert embeddings.null_count == 0
+
+    df = pd.DataFrame(
+        {
+            "text": ["with", "embeddings"],
+            "val": [6, 7],
+            "vector": [np.zeros(128).tolist(), np.zeros(128).tolist()],
+        }
+    )
+    tbl.add(df)
+
+    embeddings = tbl.search().where("val > 5").to_arrow()["vector"].to_pylist()
+    assert not np.any(embeddings), "all zeros"
+
+
+def test_multiple_embeddings_for_pandas(tmp_path):
+    @register("mock-embedding")
+    class MockFunc1(TextEmbeddingFunction):
+        def ndims(self):
+            return 128
+
+        def generate_embeddings(
+            self, texts: Union[List[str], np.ndarray]
+        ) -> List[np.array]:
+            return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
+
+    @register("mock-embedding2")
+    class MockFunc2(TextEmbeddingFunction):
+        def ndims(self):
+            return 512
+
+        def generate_embeddings(
+            self, texts: Union[List[str], np.ndarray]
+        ) -> List[np.array]:
+            return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
+
+    registery = get_registry()
+    func1 = registery.get("mock-embedding").create()
+    func2 = registery.get("mock-embedding2").create()
+
+    class TestSchema(LanceModel):
+        text: str = func1.SourceField()
+        val: int
+        vec1: Vector(func1.ndims()) = func1.VectorField()
+        prompt: str = func2.SourceField()
+        vec2: Vector(func2.ndims()) = func2.VectorField()
+
+    df = pd.DataFrame(
+        {
+            "text": ["hello world", "goodbye world"],
+            "val": [1, 2],
+            "prompt": ["hello", "goodbye"],
+        }
+    )
+    db = lancedb.connect(tmp_path)
+    tbl = db.create_table("test", schema=TestSchema, mode="overwrite", data=df)
+
+    schema = tbl.schema
+    assert schema.field("text").type == pa.string()
+    assert schema.field("val").type == pa.int64()
+    assert schema.field("vec1").type == pa.list_(pa.float32(), 128)
+    assert schema.field("prompt").type == pa.string()
+    assert schema.field("vec2").type == pa.list_(pa.float32(), 512)
+    assert tbl.count_rows() == 2
+
+
@pytest.mark.slow
 def test_embedding_function_rate_limit(tmp_path):
    def _get_schema_from_model(model):
Author	SHA1	Message	Date
qzhu	a503845c9f	more edit	2024-11-14 13:33:25 -08:00
qzhu	955a295026	code for cloud doc	2024-11-13 22:05:09 -08:00
qzhu	b70fa3892e	code for cloud doc	2024-11-13 22:03:53 -08:00
qzhu	31fb3b3b5c	first edit	2024-11-13 21:57:05 -08:00
Will Jones	0fd8a50bd7	ci(node): run examples in CI (#1796 ) This is done as setup for a PR that will fix the OpenAI dependency issue. * [x] FTS examples * [x] Setup mock openai * [x] Ran `npm audit fix` * [x] sentences embeddings test * [x] Double check formatting of docs examples	2024-11-13 11:10:56 -08:00
Umut Hope YILDIRIM	9f228feb0e	ci: remove cache to fix build issues on windows arm runner (#1820 )	2024-11-13 09:27:10 -08:00
Ayush Chaurasia	90e9c52d0a	docs: update hybrid search example to latest langchain (#1824 ) Co-authored-by: qzhu <qian@lancedb.com>	2024-11-12 20:06:25 -08:00
Will Jones	68974a4e06	ci: add index URL to fix failing docs build (#1823 )	2024-11-12 16:54:22 -08:00
Lei Xu	4c9bab0d92	fix: use pandas with pydantic embedding column (#1818 ) * Make Pandas `DataFrame` works with embedding function + Subset of columns * Make `lancedb.create_table()` work with embedding function	2024-11-11 14:48:56 -08:00
QianZhu	5117aecc38	docs: search param explanation for OSS doc (#1815 ) ![Screenshot 2024-11-09 at 11 09 14 AM](https://github.com/user-attachments/assets/2aeba016-aeff-4658-85c6-8640285ba0c9)	2024-11-11 11:57:17 -08:00