Bump version: 0.25.2-beta.3 → 0.25.2

Expand the custom redirection links list to include storage page

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.21.2"
+current_version = "0.22.2-beta.2"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/actions/create-failure-issue/action.yml

@@ -0,0 +1,45 @@
+name: Create Failure Issue
+description: Creates a GitHub issue if any jobs in the workflow failed
+
+inputs:
+  job-results:
+    description: 'JSON string of job results from needs context'
+    required: true
+  workflow-name:
+    description: 'Name of the workflow'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Check for failures and create issue
+      shell: bash
+      env:
+        JOB_RESULTS: ${{ inputs.job-results }}
+        WORKFLOW_NAME: ${{ inputs.workflow-name }}
+        RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        # Check if any job failed
+        if echo "$JOB_RESULTS" | jq -e 'to_entries | any(.value.result == "failure")' > /dev/null; then
+          echo "Detected job failures, creating issue..."
+
+          # Extract failed job names
+          FAILED_JOBS=$(echo "$JOB_RESULTS" | jq -r 'to_entries | map(select(.value.result == "failure")) | map(.key) | join(", ")')
+
+          # Create issue with workflow name, failed jobs, and run URL
+          gh issue create \
+            --title "$WORKFLOW_NAME Failed ($FAILED_JOBS)" \
+            --body "The workflow **$WORKFLOW_NAME** failed during execution.
+
+        **Failed jobs:** $FAILED_JOBS
+
+        **Run URL:** $RUN_URL
+
+        Please investigate the failed jobs and address any issues." \
+            --label "ci"
+
+          echo "Issue created successfully"
+        else
+          echo "No job failures detected, skipping issue creation"
+        fi

.github/workflows/cargo-publish.yml

@@ -38,3 +38,17 @@ jobs:
       - name: Publish the package
         run: |
           cargo publish -p lancedb --all-features --token ${{ steps.auth.outputs.token }}
+  report-failure:
+    name: Report Workflow Failure
+    runs-on: ubuntu-latest
+    needs: [build]
+    if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/create-failure-issue
+        with:
+          job-results: ${{ toJSON(needs) }}
+          workflow-name: ${{ github.workflow }}

.github/workflows/docs.yml

@@ -56,22 +56,12 @@ jobs:
         with:
           node-version: 20
           cache: 'npm'
-          cache-dependency-path: node/package-lock.json
+          cache-dependency-path: docs/package-lock.json
       - name: Install node dependencies
-        working-directory: node
+        working-directory: nodejs
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build node
-        working-directory: node
-        run: |
-          npm ci
-          npm run build
-          npm run tsc
-      - name: Create markdown files
-        working-directory: node
-        run: |
-          npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
       - name: Build docs
         working-directory: docs
         run: |

.github/workflows/docs_test.yml

@@ -24,7 +24,8 @@ env:
 jobs:
   test-python:
     name: Test doc python code
-    runs-on: ubuntu-24.04
+    runs-on: warp-ubuntu-2204-x64-8x
+    timeout-minutes: 60
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -58,51 +59,3 @@ jobs:
       run: |
         cd docs/test/python
         for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
-  test-node:
-    name: Test doc nodejs code
-    runs-on: ubuntu-24.04
-    timeout-minutes: 60
-    strategy:
-      fail-fast: false
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Print CPU capabilities
-      run: cat /proc/cpuinfo
-    - name: Set up Node
-      uses: actions/setup-node@v4
-      with:
-        node-version: 20
-    - name: Install protobuf
-      run: |
-        sudo apt update
-        sudo apt install -y protobuf-compiler
-    - name: Install dependecies needed for ubuntu
-      run: |
-        sudo apt install -y libssl-dev
-        rustup update && rustup default
-    - name: Rust cache
-      uses: swatinem/rust-cache@v2
-    - name: Install node dependencies
-      run: |
-        sudo swapoff -a
-        sudo fallocate -l 8G /swapfile
-        sudo chmod 600 /swapfile
-        sudo mkswap /swapfile
-        sudo swapon /swapfile
-        sudo swapon --show
-        cd node
-        npm ci
-        npm run build-release
-        cd ../docs
-        npm install
-    - name: Test
-      env:
-        LANCEDB_URI: ${{ secrets.LANCEDB_URI }}
-        LANCEDB_DEV_API_KEY: ${{ secrets.LANCEDB_DEV_API_KEY }}
-      run: |
-        cd docs
-        npm t

.github/workflows/java-publish.yml

@@ -43,7 +43,6 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          toolchain: "1.81.0"
           cache-workspaces: "./java/core/lancedb-jni"
           # Disable full debug symbol generation to speed up CI build and keep memory down
           # "1" means line tables only, which is useful for panic tracebacks.
@@ -112,3 +111,17 @@ jobs:
         env:
           SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
           SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}
+  report-failure:
+    name: Report Workflow Failure
+    runs-on: ubuntu-latest
+    needs: [linux-arm64, linux-x86, macos-arm64]
+    if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/create-failure-issue
+        with:
+          job-results: ${{ toJSON(needs) }}
+          workflow-name: ${{ github.workflow }}

.github/workflows/nodejs.yml

@@ -6,6 +6,7 @@ on:
       - main
   pull_request:
     paths:
+      - Cargo.toml
       - nodejs/**
       - .github/workflows/nodejs.yml
       - docker-compose.yml
@@ -79,7 +80,7 @@ jobs:
       with:
         node-version: ${{ matrix.node-version }}
         cache: 'npm'
-        cache-dependency-path: node/package-lock.json
+        cache-dependency-path: nodejs/package-lock.json
     - uses: Swatinem/rust-cache@v2
     - name: Install dependencies
       run: |
@@ -116,7 +117,7 @@ jobs:
         set -e
         npm ci
         npm run docs
-        if ! git diff --exit-code -- . ':(exclude)Cargo.lock'; then
+        if ! git diff --exit-code -- ../ ':(exclude)Cargo.lock'; then
           echo "Docs need to be updated"
           echo "Run 'npm run docs', fix any warnings, and commit the changes."
           exit 1
@@ -137,7 +138,7 @@ jobs:
       with:
         node-version: 20
         cache: 'npm'
-        cache-dependency-path: node/package-lock.json
+        cache-dependency-path: nodejs/package-lock.json
     - uses: Swatinem/rust-cache@v2
     - name: Install dependencies
       run: |

.github/workflows/npm-publish.yml

@@ -365,3 +365,17 @@ jobs:
             ARGS="$ARGS --tag preview"
           fi
           npm publish $ARGS
+  report-failure:
+    name: Report Workflow Failure
+    runs-on: ubuntu-latest
+    needs: [build-lancedb, test-lancedb, publish]
+    if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/create-failure-issue
+        with:
+          job-results: ${{ toJSON(needs) }}
+          workflow-name: ${{ github.workflow }}

.github/workflows/pypi-publish.yml

@@ -56,7 +56,7 @@ jobs:
           pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           fury_token: ${{ secrets.FURY_TOKEN }}
   mac:
-    timeout-minutes: 60
+    timeout-minutes: 90
     runs-on: ${{ matrix.config.runner }}
     strategy:
       matrix:
@@ -64,7 +64,7 @@ jobs:
           - target: x86_64-apple-darwin
             runner: macos-13
           - target: aarch64-apple-darwin
-            runner: macos-14
+            runner: warp-macos-14-arm64-6x
     env:
       MACOSX_DEPLOYMENT_TARGET: 10.15
     steps:
@@ -173,3 +173,17 @@ jobs:
           generate_release_notes: false
           name: Python LanceDB v${{ steps.extract_version.outputs.version }}
           body: ${{ steps.python_release_notes.outputs.changelog }}
+  report-failure:
+    name: Report Workflow Failure
+    runs-on: ubuntu-latest
+    needs: [linux, mac, windows]
+    permissions:
+      contents: read
+      issues: write
+    if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/create-failure-issue
+        with:
+          job-results: ${{ toJSON(needs) }}
+          workflow-name: ${{ github.workflow }}

.github/workflows/python.yml

@@ -6,6 +6,7 @@ on:
       - main
   pull_request:
     paths:
+      - Cargo.toml
       - python/**
       - .github/workflows/python.yml
 

.github/workflows/rust.yml

@@ -96,6 +96,7 @@ jobs:
       # Need up-to-date compilers for kernels
       CC: clang-18
       CXX: clang++-18
+      GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -117,15 +118,17 @@ jobs:
           sudo chmod 600 /swapfile
           sudo mkswap /swapfile
           sudo swapon /swapfile
-      - name: Start S3 integration test environment
-        working-directory: .
-        run: docker compose up --detach --wait
       - name: Build
         run: cargo build --all-features --tests --locked --examples
-      - name: Run tests
-        run: cargo test --all-features --locked
+      - name: Run feature tests
+        run: make -C ./lancedb feature-tests
       - name: Run examples
         run: cargo run --example simple --locked
+      - name: Run remote tests
+        # Running this requires access to secrets, so skip if this is
+        # a PR from a fork.
+        if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
+        run: make -C ./lancedb remote-tests
 
   macos:
     timeout-minutes: 30

.github/workflows/trigger-vectordb-recipes.yml

@@ -1,26 +0,0 @@
-name: Trigger vectordb-recipers workflow
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    paths:
-      - .github/workflows/trigger-vectordb-recipes.yml
-  workflow_dispatch:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Trigger vectordb-recipes workflow
-        uses: actions/github-script@v6
-        with:
-          github-token: ${{ secrets.VECTORDB_RECIPES_ACTION_TOKEN }}
-          script: |
-            const result = await github.rest.actions.createWorkflowDispatch({
-                owner: 'lancedb',
-                repo: 'vectordb-recipes',
-                workflow_id: 'examples-test.yml',
-                ref: 'main'
-            });
-            console.log(result);

.gitignore

@@ -31,9 +31,6 @@ python/dist
 *.node
 **/node_modules
 **/.DS_Store
-node/dist
-node/examples/**/package-lock.json
-node/examples/**/dist
 nodejs/lancedb/native*
 dist
 

CLAUDE.md

@@ -11,14 +11,70 @@ Project layout:
 * `nodejs`: The Typescript bindings, using napi-rs
 * `java`: The Java bindings
 
-(`rust/ffi` and `node/` are for a deprecated package. You can ignore them.)
-
 Common commands:
 
-* Check for compiler errors: `cargo check --features remote --tests --examples`
-* Run tests: `cargo test --features remote --tests`
-* Run specific test: `cargo test --features remote -p <package_name> --test <test_name>`
-* Lint: `cargo clippy --features remote --tests --examples`
+* Check for compiler errors: `cargo check --quiet --features remote --tests --examples`
+* Run tests: `cargo test --quiet --features remote --tests`
+* Run specific test: `cargo test --quiet --features remote -p <package_name> --test <test_name>`
+* Lint: `cargo clippy --quiet --features remote --tests --examples`
 * Format: `cargo fmt --all`
 
 Before committing changes, run formatting.
+
+## Coding tips
+
+* When writing Rust doctests for things that require a connection or table reference,
+  write them as a function instead of a fully executable test. This allows type checking
+  to run but avoids needing a full test environment. For example:
+    ```rust
+    /// ```
+    /// use lance_index::scalar::FullTextSearchQuery;
+    /// use lancedb::query::{QueryBase, ExecutableQuery};
+    ///
+    /// # use lancedb::Table;
+    /// # async fn query(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
+    /// let results = table.query()
+    ///     .full_text_search(FullTextSearchQuery::new("hello world".into()))
+    ///     .execute()
+    ///     .await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    ```
+
+## Example plan: adding a new method on Table
+
+Adding a new method involves first adding it to the Rust core, then exposing it
+in the Python and TypeScript bindings. There are both local and remote tables.
+Remote tables are implemented via a HTTP API and require the `remote` cargo
+feature flag to be enabled. Python has both sync and async methods.
+
+Rust core changes:
+
+1. Add method on `Table` struct in `rust/lancedb/src/table.rs` (calls `BaseTable` trait).
+2. Add method to `BaseTable` trait in `rust/lancedb/src/table.rs`.
+3. Implement new trait method on `NativeTable` in `rust/lancedb/src/table.rs`.
+    * Test with unit test in `rust/lancedb/src/table.rs`.
+4. Implement new trait method on `RemoteTable` in `rust/lancedb/src/remote/table.rs`.
+    * Test with unit test in `rust/lancedb/src/remote/table.rs` against mocked endpoint.
+
+Python bindings changes:
+
+1. Add PyO3 method binding in `python/src/table.rs`. Run `make develop` to compile bindings.
+2. Add types for PyO3 method in `python/python/lancedb/_lancedb.pyi`.
+3. Add method to `AsyncTable` class in `python/python/lancedb/table.py`.
+4. Add abstract method to `Table` abstract base class in `python/python/lancedb/table.py`.
+5. Add concrete sync method to `LanceTable` class in `python/python/lancedb/table.py`.
+    * Should use `LOOP.run()` to call the corresponding `AsyncTable` method.
+6. Add concrete sync method to `RemoteTable` class in `python/python/lancedb/remote/table.py`.
+7. Add unit test in `python/tests/test_table.py`.
+
+TypeScript bindings changes:
+
+1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
+2. Run `npm run build` to generate TypeScript definitions.
+3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
+4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
+    * Note: despite the name, this class is also used for remote tables.
+5. Add test in `nodejs/__test__/table.test.ts`.
+6. Run `npm run docs` to generate TypeScript documentation.

Cargo.toml

@@ -1,10 +1,5 @@
 [workspace]
-members = [
-    "rust/lancedb",
-    "nodejs",
-    "python",
-    "java/core/lancedb-jni",
-]
+members = ["rust/lancedb", "nodejs", "python", "java/core/lancedb-jni"]
 # Python package needs to be built by maturin.
 exclude = ["python"]
 resolver = "2"
@@ -20,32 +15,30 @@ categories = ["database-implementations"]
 rust-version = "1.78.0"
 
 [workspace.dependencies]
-lance = { "version" = "=0.32.1", "features" = [
-    "dynamodb",
-], "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
-lance-io = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
-lance-index = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
-lance-linalg = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
-lance-table = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
-lance-testing = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
-lance-datafusion = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
-lance-encoding = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
+lance = { "version" = "=0.38.2", default-features = false, "features" = ["dynamodb"] }
+lance-io = { "version" = "=0.38.2", default-features = false }
+lance-index = "=0.38.2"
+lance-linalg = "=0.38.2"
+lance-table = "=0.38.2"
+lance-testing = "=0.38.2"
+lance-datafusion = "=0.38.2"
+lance-encoding = "=0.38.2"
+lance-namespace = "0.0.18"
 # Note that this one does not include pyarrow
-arrow = { version = "55.1", optional = false }
-arrow-array = "55.1"
-arrow-data = "55.1"
-arrow-ipc = "55.1"
-arrow-ord = "55.1"
-arrow-schema = "55.1"
-arrow-arith = "55.1"
-arrow-cast = "55.1"
+arrow = { version = "56.2", optional = false }
+arrow-array = "56.2"
+arrow-data = "56.2"
+arrow-ipc = "56.2"
+arrow-ord = "56.2"
+arrow-schema = "56.2"
+arrow-cast = "56.2"
 async-trait = "0"
-datafusion = { version = "48.0", default-features = false }
-datafusion-catalog = "48.0"
-datafusion-common = { version = "48.0", default-features = false }
-datafusion-execution = "48.0"
-datafusion-expr = "48.0"
-datafusion-physical-plan = "48.0"
+datafusion = { version = "50.1", default-features = false }
+datafusion-catalog = "50.1"
+datafusion-common = { version = "50.1", default-features = false }
+datafusion-execution = "50.1"
+datafusion-expr = "50.1"
+datafusion-physical-plan = "50.1"
 env_logger = "0.11"
 half = { "version" = "2.6.0", default-features = false, features = [
     "num-traits",
@@ -58,16 +51,24 @@ pin-project = "1.0.7"
 snafu = "0.8"
 url = "2"
 num-traits = "0.2"
-rand = "0.9"
 regex = "1.10"
 lazy_static = "1"
 semver = "1.0.25"
+crunchy = "0.2.4"
 # Temporary pins to work around downstream issues
 # https://github.com/apache/arrow-rs/commit/2fddf85afcd20110ce783ed5b4cdeb82293da30b
 chrono = "=0.4.41"
-# https://github.com/RustCrypto/formats/issues/1684
-base64ct = "=1.6.0"
-# Workaround for: https://github.com/eira-fransham/crunchy/issues/13
-crunchy = "=0.2.2"
 # Workaround for: https://github.com/Lokathor/bytemuck/issues/306
 bytemuck_derive = ">=1.8.1, <1.9.0"
+
+# This is only needed when we reference preview releases of lance
+# [patch.crates-io]
+# # Force to use the same lance version as the rest of the project to avoid duplicate dependencies
+# lance = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
+# lance-io = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
+# lance-index = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
+# lance-linalg = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
+# lance-table = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
+# lance-testing = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
+# lance-datafusion = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
+# lance-encoding = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }

ci/create_lancedb_test_connection.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+export RUST_LOG=info
+exec ./lancedb server --port 0 --sql-port 0  --data-dir "${1}"

ci/run_with_docker_compose.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+#
+# A script for running the given command together with a docker compose environment.
+#
+
+# Bring down the docker setup once the command is done running.
+tear_down() {
+    docker compose -p fixture down
+}
+trap tear_down EXIT
+
+set +xe
+
+# Clean up any existing docker setup and bring up a new one.
+docker compose -p fixture up --detach --wait || exit 1
+
+"${@}"

ci/run_with_test_connection.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+#
+# A script for running the given command together with the lancedb cli.
+#
+
+die() {
+    echo $?
+    exit 1
+}
+
+check_command_exists() {
+    command="${1}"
+    which ${command} &> /dev/null || \
+        die "Unable to locate command: ${command}. Did you install it?"
+}
+
+if [[ ! -e ./lancedb ]]; then
+    if [[ -v SOPHON_READ_TOKEN ]]; then
+        INPUT="lancedb-linux-x64"
+        gh release \
+            --repo lancedb/lancedb \
+            download ci-support-binaries \
+            --pattern "${INPUT}" \
+            || die "failed to fetch cli."
+        check_command_exists openssl
+        openssl enc -aes-256-cbc \
+            -d -pbkdf2 \
+            -pass "env:SOPHON_READ_TOKEN" \
+            -in "${INPUT}" \
+            -out ./lancedb-linux-x64.tar.gz \
+            || die "openssl failed"
+        TARGET="${INPUT}.tar.gz"
+    else
+        ARCH="x64"
+        if [[ $OSTYPE == 'darwin'* ]]; then
+            UNAME=$(uname -m)
+            if [[ $UNAME == 'arm64' ]]; then
+                ARCH='arm64'
+            fi
+            OSTYPE="macos"
+        elif [[ $OSTYPE == 'linux'* ]]; then
+            if [[ $UNAME == 'aarch64' ]]; then
+                ARCH='arm64'
+            fi
+            OSTYPE="linux"
+        else
+            die "unknown OSTYPE: $OSTYPE"
+        fi
+
+        check_command_exists gh
+        TARGET="lancedb-${OSTYPE}-${ARCH}.tar.gz"
+        gh release \
+            --repo lancedb/sophon \
+            download lancedb-cli-v0.0.3 \
+            --pattern "${TARGET}" \
+            || die "failed to fetch cli."
+    fi
+
+    check_command_exists tar
+    tar xvf "${TARGET}" || die "tar failed."
+    [[ -e ./lancedb ]] || die "failed to extract lancedb."
+fi
+
+SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
+export CREATE_LANCEDB_TEST_CONNECTION_SCRIPT="${SCRIPT_DIR}/create_lancedb_test_connection.sh"
+
+"${@}"

ci/set_lance_version.py

@@ -1,4 +1,5 @@
 import argparse
+import re
 import sys
 import json
 
@@ -18,8 +19,12 @@ def run_command(command: str) -> str:
 
 def get_latest_stable_version() -> str:
     version_line = run_command("cargo info lance | grep '^version:'")
-    version = version_line.split(" ")[1].strip()
-    return version
+    # Example output: "version: 0.35.0 (latest 0.37.0)"
+    match = re.search(r'\(latest ([0-9.]+)\)', version_line)
+    if match:
+        return match.group(1)
+    # Fallback: use the first version after 'version:'
+    return version_line.split("version:")[1].split()[0].strip()
 
 
 def get_latest_preview_version() -> str:
@@ -54,6 +59,52 @@ def extract_features(line: str) -> list:
     return []
 
 
+def extract_default_features(line: str) -> bool:
+    """
+    Checks if default-features = false is present in a line in Cargo.toml.
+    Example: 'lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"] }'
+    Returns: True if default-features = false is present, False otherwise
+    """
+    import re
+
+    match = re.search(r'default-features\s*=\s*false', line)
+    return match is not None
+
+
+def dict_to_toml_line(package_name: str, config: dict) -> str:
+    """
+    Converts a configuration dictionary to a TOML dependency line.
+    Dictionary insertion order is preserved (Python 3.7+), so the caller
+    controls the order of fields in the output.
+
+    Args:
+        package_name: The name of the package (e.g., "lance", "lance-io")
+        config: Dictionary with keys like "version", "path", "git", "tag", "features", "default-features"
+                The order of keys in this dict determines the order in the output.
+
+    Returns:
+        A properly formatted TOML line with a trailing newline
+    """
+    # If only version is specified, use simple format
+    if len(config) == 1 and "version" in config:
+        return f'{package_name} = "{config["version"]}"\n'
+
+    # Otherwise, use inline table format
+    parts = []
+    for key, value in config.items():
+        if key == "default-features" and not value:
+            parts.append("default-features = false")
+        elif key == "features":
+            parts.append(f'"features" = {json.dumps(value)}')
+        elif isinstance(value, str):
+            parts.append(f'"{key}" = "{value}"')
+        else:
+            # This shouldn't happen with our current usage
+            parts.append(f'"{key}" = {json.dumps(value)}')
+
+    return f'{package_name} = {{ {", ".join(parts)} }}\n'
+
+
 def update_cargo_toml(line_updater):
     """
     Updates the Cargo.toml file by applying the line_updater function to each line.
@@ -66,21 +117,28 @@ def update_cargo_toml(line_updater):
     lance_line = ""
     is_parsing_lance_line = False
     for line in lines:
-        if line.startswith("lance"):
-            # Update the line using the provided function
-            if line.strip().endswith("}"):
+        if line.startswith("lance") and not line.startswith("lance-namespace"):
+            # Check if this is a single-line or multi-line entry
+            # Single-line entries either:
+            # 1. End with } (complete inline table)
+            # 2. End with " (simple version string)
+            # Multi-line entries start with { but don't end with }
+            if line.strip().endswith("}") or line.strip().endswith('"'):
+                # Single-line entry - process immediately
                 new_lines.append(line_updater(line))
-            else:
+            elif "{" in line and not line.strip().endswith("}"):
+                # Multi-line entry - start accumulating
                 lance_line = line
                 is_parsing_lance_line = True
+            else:
+                # Single-line entry without quotes or braces (shouldn't happen but handle it)
+                new_lines.append(line_updater(line))
         elif is_parsing_lance_line:
             lance_line += line
             if line.strip().endswith("}"):
                 new_lines.append(line_updater(lance_line))
                 lance_line = ""
                 is_parsing_lance_line = False
-            else:
-                print("doesn't end with }:", line)
         else:
             # Keep the line unchanged
             new_lines.append(line)
@@ -92,18 +150,25 @@ def update_cargo_toml(line_updater):
 def set_stable_version(version: str):
     """
     Sets lines to
-    lance = { "version" = "=0.29.0", "features" = ["dynamodb"] }
-    lance-io = "=0.29.0"
+    lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"] }
+    lance-io = { "version" = "=0.29.0", default-features = false }
     ...
     """
 
     def line_updater(line: str) -> str:
         package_name = line.split("=", maxsplit=1)[0].strip()
+
+        # Build config in desired order: version, default-features, features
+        config = {"version": f"={version}"}
+
+        if extract_default_features(line):
+            config["default-features"] = False
+
         features = extract_features(line)
         if features:
-            return f'{package_name} = {{ "version" = "={version}", "features" = {json.dumps(features)} }}\n'
-        else:
-            return f'{package_name} = "={version}"\n'
+            config["features"] = features
+
+        return dict_to_toml_line(package_name, config)
 
     update_cargo_toml(line_updater)
 
@@ -111,19 +176,29 @@ def set_stable_version(version: str):
 def set_preview_version(version: str):
     """
     Sets lines to
-    lance = { "version" = "=0.29.0", "features" = ["dynamodb"], tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" }
-    lance-io = { version = "=0.29.0", tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" }
+    lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"], "tag" = "v0.29.0-beta.2", "git" = "https://github.com/lancedb/lance.git" }
+    lance-io = { "version" = "=0.29.0", default-features = false, "tag" = "v0.29.0-beta.2", "git" = "https://github.com/lancedb/lance.git" }
     ...
     """
 
     def line_updater(line: str) -> str:
         package_name = line.split("=", maxsplit=1)[0].strip()
-        features = extract_features(line)
         base_version = version.split("-")[0]  # Get the base version without beta suffix
+
+        # Build config in desired order: version, default-features, features, tag, git
+        config = {"version": f"={base_version}"}
+
+        if extract_default_features(line):
+            config["default-features"] = False
+
+        features = extract_features(line)
         if features:
-            return f'{package_name} = {{ "version" = "={base_version}", "features" = {json.dumps(features)}, "tag" = "v{version}", "git" = "https://github.com/lancedb/lance.git" }}\n'
-        else:
-            return f'{package_name} = {{ "version" = "={base_version}", "tag" = "v{version}", "git" = "https://github.com/lancedb/lance.git" }}\n'
+            config["features"] = features
+
+        config["tag"] = f"v{version}"
+        config["git"] = "https://github.com/lancedb/lance.git"
+
+        return dict_to_toml_line(package_name, config)
 
     update_cargo_toml(line_updater)
 
@@ -131,18 +206,25 @@ def set_preview_version(version: str):
 def set_local_version():
     """
     Sets lines to
-    lance = { path = "../lance/rust/lance", features = ["dynamodb"] }
-    lance-io = { path = "../lance/rust/lance-io" }
+    lance = { "path" = "../lance/rust/lance", default-features = false, "features" = ["dynamodb"] }
+    lance-io = { "path" = "../lance/rust/lance-io", default-features = false }
     ...
     """
 
     def line_updater(line: str) -> str:
         package_name = line.split("=", maxsplit=1)[0].strip()
+
+        # Build config in desired order: path, default-features, features
+        config = {"path": f"../lance/rust/{package_name}"}
+
+        if extract_default_features(line):
+            config["default-features"] = False
+
         features = extract_features(line)
         if features:
-            return f'{package_name} = {{ "path" = "../lance/rust/{package_name}", "features" = {json.dumps(features)} }}\n'
-        else:
-            return f'{package_name} = {{ "path" = "../lance/rust/{package_name}" }}\n'
+            config["features"] = features
+
+        return dict_to_toml_line(package_name, config)
 
     update_cargo_toml(line_updater)
 

ci/update_lockfiles.sh

@@ -15,16 +15,13 @@ cargo metadata --quiet > /dev/null
 pushd nodejs || exit 1
 npm install --package-lock-only --silent
 popd
-pushd node || exit 1
-npm install --package-lock-only --silent
-popd
 
 if git diff --quiet --exit-code; then
   echo "No lockfile changes to commit; skipping amend."
 elif $AMEND; then
-  git add Cargo.lock nodejs/package-lock.json node/package-lock.json
+  git add Cargo.lock nodejs/package-lock.json
   git commit --amend --no-edit
 else
-  git add Cargo.lock nodejs/package-lock.json node/package-lock.json
+  git add Cargo.lock nodejs/package-lock.json
   git commit -m "Update lockfiles"
 fi

docs/mkdocs.yml

@@ -70,6 +70,23 @@ plugins:
   - mkdocs-jupyter
   - render_swagger:
       allow_arbitrary_locations: true
+  - redirects:
+      redirect_maps:
+        # Redirect the home page and other top-level markdown files. This enables maximum SEO benefit
+        # other sub-pages are handled by the ingected js in overrides/partials/header.html
+        'index.md': 'https://lancedb.com/docs/'
+        'guides/tables.md': 'https://lancedb.com/docs/tables/'
+        'ann_indexes.md': 'https://lancedb.com/docs/indexing/'
+        'basic.md': 'https://lancedb.com/docs/quickstart/'
+        'faq.md': 'https://lancedb.com/docs/faq/'
+        'embeddings/understanding_embeddings.md': 'https://lancedb.com/docs/embedding/'
+        'integrations.md': 'https://lancedb.com/docs/integrations/'
+        'examples.md': 'https://lancedb.com/docs/tutorials/'
+        'concepts/vector_search.md': 'https://lancedb.com/docs/search/vector-search/'
+        'troubleshooting.md': 'https://lancedb.com/docs/troubleshooting/'
+        'guides/storage.md': 'https://lancedb.com/docs/storage/integrations'
+
+
 
 markdown_extensions:
   - admonition

docs/overrides/partials/header.html

@@ -19,7 +19,13 @@
   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
   IN THE SOFTWARE.
 -->
-
+<div id="deprecation-banner" style="background-color: #f8d7da; color: #721c24; padding: 1em; text-align: center;">                                                           
+    <p style="margin: 0; font-size: 1.1em;">                                                                                                           
+        <strong>This documentation site is deprecated.</strong>                                                                           
+        Please visit our new documentation site at <a href="https://lancedb.com/docs" style="color: #721c24; text-decoration: underline;">
+          lancedb.com/docs</a> for the latest information.                                                                                        
+    </p>                                                                                                                                               
+  </div>  
 {% set class = "md-header" %}
 {% if "navigation.tabs.sticky" in features %}
   {% set class = class ~ " md-header--shadow md-header--lifted" %}
@@ -150,9 +156,9 @@
 
     <div style="margin-left: 10px; margin-right: 5px;">
         <a href="https://discord.com/invite/zMM32dvNtd" target="_blank" rel="noopener noreferrer">
-            <svg fill="#FFFFFF" xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 50 50" width="25px" height="25px"><path d="M 41.625 10.769531 C 37.644531 7.566406 31.347656 7.023438 31.078125 7.003906 C 30.660156 6.96875 30.261719 7.203125 30.089844 7.589844 C 30.074219 7.613281 29.9375 7.929688 29.785156 8.421875 C 32.417969 8.867188 35.652344 9.761719 38.578125 11.578125 C 39.046875 11.867188 39.191406 12.484375 38.902344 12.953125 C 38.710938 13.261719 38.386719 13.429688 38.050781 13.429688 C 37.871094 13.429688 37.6875 13.378906 37.523438 13.277344 C 32.492188 10.15625 26.210938 10 25 10 C 23.789063 10 17.503906 10.15625 12.476563 13.277344 C 12.007813 13.570313 11.390625 13.425781 11.101563 12.957031 C 10.808594 12.484375 10.953125 11.871094 11.421875 11.578125 C 14.347656 9.765625 17.582031 8.867188 20.214844 8.425781 C 20.0625 7.929688 19.925781 7.617188 19.914063 7.589844 C 19.738281 7.203125 19.34375 6.960938 18.921875 7.003906 C 18.652344 7.023438 12.355469 7.566406 8.320313 10.8125 C 6.214844 12.761719 2 24.152344 2 34 C 2 34.175781 2.046875 34.34375 2.132813 34.496094 C 5.039063 39.605469 12.972656 40.941406 14.78125 41 C 14.789063 41 14.800781 41 14.8125 41 C 15.132813 41 15.433594 40.847656 15.621094 40.589844 L 17.449219 38.074219 C 12.515625 36.800781 9.996094 34.636719 9.851563 34.507813 C 9.4375 34.144531 9.398438 33.511719 9.765625 33.097656 C 10.128906 32.683594 10.761719 32.644531 11.175781 33.007813 C 11.234375 33.0625 15.875 37 25 37 C 34.140625 37 38.78125 33.046875 38.828125 33.007813 C 39.242188 32.648438 39.871094 32.683594 40.238281 33.101563 C 40.601563 33.515625 40.5625 34.144531 40.148438 34.507813 C 40.003906 34.636719 37.484375 36.800781 32.550781 38.074219 L 34.378906 40.589844 C 34.566406 40.847656 34.867188 41 35.1875 41 C 35.199219 41 35.210938 41 35.21875 41 C 37.027344 40.941406 44.960938 39.605469 47.867188 34.496094 C 47.953125 34.34375 48 34.175781 48 34 C 48 24.152344 43.785156 12.761719 41.625 10.769531 Z M 18.5 30 C 16.566406 30 15 28.210938 15 26 C 15 23.789063 16.566406 22 18.5 22 C 20.433594 22 22 23.789063 22 26 C 22 28.210938 20.433594 30 18.5 30 Z M 31.5 30 C 29.566406 30 28 28.210938 28 26 C 28 23.789063 29.566406 22 31.5 22 C 33.433594 22 35 23.789063 35 26 C 35 28.210938 33.433594 30 31.5 30 Z"/></svg>
-        </a>
-    </div>
+          <svg fill="#FFFFFF" xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 50 50" width="25px" height="25px"><path d="M 41.625 10.769531 C 37.644531 7.566406 31.347656 7.023438 31.078125 7.003906 C 30.660156 6.96875 30.261719 7.203125 30.089844 7.589844 C 30.074219 7.613281 29.9375 7.929688 29.785156 8.421875 C 32.417969 8.867188 35.652344 9.761719 38.578125 11.578125 C 39.046875 11.867188 39.191406 12.484375 38.902344 12.953125 C 38.710938 13.261719 38.386719 13.429688 38.050781 13.429688 C 37.871094 13.429688 37.6875 13.378906 37.523438 13.277344 C 32.492188 10.15625 26.210938 10 25 10 C 23.789063 10 17.503906 10.15625 12.476563 13.277344 C 12.007813 13.570313 11.390625 13.425781 11.101563 12.957031 C 10.808594 12.484375 10.953125 11.871094 11.421875 11.578125 C 14.347656 9.765625 17.582031 8.867188 20.214844 8.425781 C 20.0625 7.929688 19.925781 7.617188 19.914063 7.589844 C 19.738281 7.203125 19.34375 6.960938 18.921875 7.003906 C 18.652344 7.023438 12.355469 7.566406 8.320313 10.8125 C 6.214844 12.761719 2 24.152344 2 34 C 2 34.175781 2.046875 34.34375 2.132813 34.496094 C 5.039063 39.605469 12.972656 40.941406 14.78125 41 C 14.789063 41 14.800781 41 14.8125 41 C 15.132813 41 15.433594 40.847656 15.621094 40.589844 L 17.449219 38.074219 C 12.515625 36.800781 9.996094 34.636719 9.851563 34.507813 C 9.4375 34.144531 9.398438 33.511719 9.765625 33.097656 C 10.128906 32.683594 10.761719 32.644531 11.175781 33.007813 C 11.234375 33.0625 15.875 37 25 37 C 34.140625 37 38.78125 33.046875 38.828125 33.007813 C 39.242188 32.648438 39.871094 32.683594 40.238281 33.101563 C 40.601563 33.515625 40.5625 34.144531 40.148438 34.507813 C 40.003906 34.636719 37.484375 36.800781 32.550781 38.074219 L 34.378906 40.589844 C 34.566406 40.847656 34.867188 41 35.1875 41 C 35.199219 41 35.210938 41 35.21875 41 C 37.027344 40.941406 44.960938 39.605469 47.867188 34.496094 C 47.953125 34.34375 48 34.175781 48 34 C 48 24.152344 43.785156 12.761719 41.625 10.769531 Z M 18.5 30 C 16.566406 30 15 28.210938 15 26 C 15 23.789063 16.566406 22 18.5 22 C 20.433594 22 22 23.789063 22 26 C 22 28.210938 20.433594 30 18.5 30 Z M 31.5 30 C 29.566406 30 28 28.210938 28 26 C 28 23.789063 29.566406 22 31.5 22 C 33.433594 22 35 23.789063 35 26 C 35 28.210938 33.433594 30 31.5 30 Z"/></svg>
+      </a>
+  </div>
     <div style="margin-left: 5px; margin-right: 5px;">
         <a href="https://twitter.com/lancedb" target="_blank" rel="noopener noreferrer">
             <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0,0,256,256" width="25px" height="25px" fill-rule="nonzero"><g fill-opacity="0" fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><path d="M0,256v-256h256v256z" id="bgRectangle"></path></g><g fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(4,4)"><path d="M57,17.114c-1.32,1.973 -2.991,3.707 -4.916,5.097c0.018,0.423 0.028,0.847 0.028,1.274c0,13.013 -9.902,28.018 -28.016,28.018c-5.562,0 -12.81,-1.948 -15.095,-4.423c0.772,0.092 1.556,0.138 2.35,0.138c4.615,0 8.861,-1.575 12.23,-4.216c-4.309,-0.079 -7.946,-2.928 -9.199,-6.84c1.96,0.308 4.447,-0.17 4.447,-0.17c0,0 -7.7,-1.322 -7.899,-9.779c2.226,1.291 4.46,1.231 4.46,1.231c0,0 -4.441,-2.734 -4.379,-8.195c0.037,-3.221 1.331,-4.953 1.331,-4.953c8.414,10.361 20.298,10.29 20.298,10.29c0,0 -0.255,-1.471 -0.255,-2.243c0,-5.437 4.408,-9.847 9.847,-9.847c2.832,0 5.391,1.196 7.187,3.111c2.245,-0.443 4.353,-1.263 6.255,-2.391c-0.859,3.44 -4.329,5.448 -4.329,5.448c0,0 2.969,-0.329 5.655,-1.55z"></path></g></g></svg>
@@ -173,4 +179,77 @@
       {% include "partials/tabs.html" %}
     {% endif %}
   {% endif %}
-</header>
+</header>
+
+<script>
+  (function() {
+    function checkPathAndRedirect() {
+      var banner = document.getElementById('deprecation-banner');
+
+      if (document.querySelector('meta[http-equiv="refresh"]')) {
+        return; // The redirects plugin is already handling this page.
+      }
+
+      var currentPath = window.location.pathname;
+
+      var cleanPath = currentPath.endsWith('/') && currentPath.length > 1
+        ? currentPath.slice(0, -1)
+        : currentPath;
+
+      // These are the ONLY paths that should remain on the old site
+      var apiPaths = [
+        '/lancedb/python', 
+        '/lancedb/javascript', 
+        '/lancedb/js',
+        '/lancedb/api_reference'
+      ];
+      
+      var isApiPage = apiPaths.some(function(apiPath) {
+        return cleanPath.startsWith(apiPath);
+      });
+
+      if (isApiPage) {
+        if (banner) {
+          banner.style.display = 'none';
+        }
+      } else {
+        if (banner) {
+          banner.style.display = 'block';
+        }
+        
+        // Add noindex meta tag to prevent indexing of old docs for seo
+        var noindexMeta = document.createElement('meta');
+        noindexMeta.setAttribute('name', 'robots');
+        noindexMeta.setAttribute('content', 'noindex, follow');
+        document.head.appendChild(noindexMeta);
+
+        // Add canonical link to point to the new docs to reward new site for seo
+        var canonicalLink = document.createElement('link');
+        canonicalLink.setAttribute('rel', 'canonical');
+        canonicalLink.setAttribute('href', 'https://lancedb.com/docs');
+        document.head.appendChild(canonicalLink);
+        
+        window.location.replace('https://lancedb.com/docs');
+      }
+    }
+
+    // Run the check only if doc is ready. This makes sure we catch the initial load
+    // and redirect.
+    if (document.readyState === 'loading') {
+      document.addEventListener('DOMContentLoaded', checkPathAndRedirect);
+    } else {
+      checkPathAndRedirect();
+    }
+
+    // Use an interval to handle subsequent navigation clicks.
+    var lastPath = window.location.pathname;
+    setInterval(function() {
+      if (window.location.pathname !== lastPath) {
+        lastPath = window.location.pathname;
+        checkPathAndRedirect();
+      }
+    }, 2000); // keeping it 2 second to make it easy for user to understand
+              // what's happening
+
+  })();
+</script>

docs/requirements.txt

@@ -5,3 +5,4 @@ mkdocstrings[python]==0.25.2
 griffe
 mkdocs-render-swagger-plugin
 pydantic
+mkdocs-redirects

docs/src/concepts/data_management.md

@@ -13,7 +13,7 @@ The following concepts are important to keep in mind:
 - Data is versioned, with each insert operation creating a new version of the dataset and an update to the manifest that tracks versions via metadata
 
 !!! note
-    1. First, each version contains metadata and just the new/updated data in your transaction. So if you have 100 versions, they aren't 100 duplicates of the same data. However, they do have 100x the metadata overhead of a single version, which can result in slower queries.  
+    1. First, each version contains metadata and just the new/updated data in your transaction. So if you have 100 versions, they aren't 100 duplicates of the same data. However, they do have 100x the metadata overhead of a single version, which can result in slower queries.
     2. Second, these versions exist to keep LanceDB scalable and consistent. We do not immediately blow away old versions when creating new ones because other clients might be in the middle of querying the old version. It's important to retain older versions for as long as they might be queried.
 
 ## What are fragments?
@@ -37,6 +37,10 @@ Depending on the use case and dataset, optimal compaction will have different re
 - It’s always better to use *batch* inserts rather than adding 1 row at a time (to avoid too small fragments). If single-row inserts are unavoidable, run compaction on a regular basis to merge them into larger fragments.
 - Keep the number of fragments under 100, which is suitable for most use cases (for *really* large datasets of >500M rows, more fragments might be needed)
 
+!!! note
+
+    LanceDB Cloud/Enterprise supports [auto-compaction](https://docs.lancedb.com/enterprise/architecture/architecture#write-path) which automatically optimizes fragments in the background as data changes.
+
 ## Deletion
 
 Although Lance allows you to delete rows from a dataset, it does not actually delete the data immediately. It simply marks the row as deleted in the `DataFile` that represents a fragment. For a given version of the dataset, each fragment can have up to one deletion file (if no rows were ever deleted from that fragment, it will not have a deletion file). This is important to keep in mind because it means that the data is still there, and can be recovered if needed, as long as that version still exists based on your backup policy.
@@ -50,13 +54,9 @@ Reindexing is the process of updating the index to account for new data, keeping
 
 Both LanceDB OSS and Cloud support reindexing, but the process (at least for now) is different for each, depending on the type of index.
 
-When a reindex job is triggered in the background, the entire data is reindexed, but in the interim as new queries come in, LanceDB will combine results from the existing index with exhaustive kNN search on the new data. This is done to ensure that you're still searching on all your data, but it does come at a performance cost. The more data that you add without reindexing, the impact on latency (due to exhaustive search) can be noticeable.
+In LanceDB OSS, re-indexing happens synchronously when you call either `create_index` or `optimize` on a table. In LanceDB Cloud, re-indexing happens asynchronously as you add and update data in your table.
 
-### Vector reindex
+By default, queries will search new data even if it has yet to be indexed. This is done using brute-force methods, such as kNN for vector search, and combined with the fast index search results. This is done to ensure that you're always searching over all your data, but it does come at a performance cost. Without reindexing, adding more data to a table will make queries slower and more expensive. This behavior can be disabled by setting the [fast_search](https://lancedb.github.io/lancedb/python/python/#lancedb.query.AsyncQuery.fast_search) parameter which will instruct the query to ignore un-indexed data.
 
-* LanceDB Cloud supports incremental reindexing, where a background process will trigger a new index build for you automatically when new data is added to a dataset
+* LanceDB Cloud/Enterprise supports [automatic incremental reindexing](https://docs.lancedb.com/core#vector-index) for vector, scalar, and FTS indices, where a background process will trigger a new index build for you automatically when new data is added or modified in a dataset
 * LanceDB OSS requires you to manually trigger a reindex operation -- we are working on adding incremental reindexing to LanceDB OSS as well
-
-### FTS reindex
-
-FTS reindexing is supported in both LanceDB OSS and Cloud, but requires that it's manually rebuilt once you have a significant enough amount of new data added that needs to be reindexed. We [updated](https://github.com/lancedb/lancedb/pull/762) Tantivy's default heap size from 128MB to 1GB in LanceDB to make it much faster to reindex, by up to 10x from the default settings.

docs/src/js/classes/Connection.md

@@ -25,6 +25,51 @@ the underlying connection has been closed.
 
 ## Methods
 
+### cloneTable()
+
+```ts
+abstract cloneTable(
+   targetTableName,
+   sourceUri,
+   options?): Promise<Table>
+```
+
+Clone a table from a source table.
+
+A shallow clone creates a new table that shares the underlying data files
+with the source table but has its own independent manifest. This allows
+both the source and cloned tables to evolve independently while initially
+sharing the same data, deletion, and index files.
+
+#### Parameters
+
+* **targetTableName**: `string`
+    The name of the target table to create.
+
+* **sourceUri**: `string`
+    The URI of the source table to clone from.
+
+* **options?**
+    Clone options.
+
+* **options.isShallow?**: `boolean`
+    Whether to perform a shallow clone (defaults to true).
+
+* **options.sourceTag?**: `string`
+    The tag of the source table to clone.
+
+* **options.sourceVersion?**: `number`
+    The version of the source table to clone.
+
+* **options.targetNamespace?**: `string`[]
+    The namespace for the target table (defaults to root namespace).
+
+#### Returns
+
+`Promise`&lt;[`Table`](Table.md)&gt;
+
+***
+
 ### close()
 
 ```ts
@@ -45,6 +90,8 @@ Any attempt to use the connection after it is closed will result in an error.
 
 ### createEmptyTable()
 
+#### createEmptyTable(name, schema, options)
+
 ```ts
 abstract createEmptyTable(
    name,
@@ -54,7 +101,7 @@ abstract createEmptyTable(
 
 Creates a new empty Table
 
-#### Parameters
+##### Parameters
 
 * **name**: `string`
     The name of the table.
@@ -63,8 +110,39 @@ Creates a new empty Table
     The schema of the table
 
 * **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+    Additional options (backwards compatibility)
 
-#### Returns
+##### Returns
+
+`Promise`&lt;[`Table`](Table.md)&gt;
+
+#### createEmptyTable(name, schema, namespace, options)
+
+```ts
+abstract createEmptyTable(
+   name,
+   schema,
+   namespace?,
+   options?): Promise<Table>
+```
+
+Creates a new empty Table
+
+##### Parameters
+
+* **name**: `string`
+    The name of the table.
+
+* **schema**: [`SchemaLike`](../type-aliases/SchemaLike.md)
+    The schema of the table
+
+* **namespace?**: `string`[]
+    The namespace to create the table in (defaults to root namespace)
+
+* **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+    Additional options
+
+##### Returns
 
 `Promise`&lt;[`Table`](Table.md)&gt;
 
@@ -72,10 +150,10 @@ Creates a new empty Table
 
 ### createTable()
 
-#### createTable(options)
+#### createTable(options, namespace)
 
 ```ts
-abstract createTable(options): Promise<Table>
+abstract createTable(options, namespace?): Promise<Table>
 ```
 
 Creates a new Table and initialize it with new data.
@@ -85,6 +163,9 @@ Creates a new Table and initialize it with new data.
 * **options**: `object` & `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
     The options object.
 
+* **namespace?**: `string`[]
+    The namespace to create the table in (defaults to root namespace)
+
 ##### Returns
 
 `Promise`&lt;[`Table`](Table.md)&gt;
@@ -110,6 +191,38 @@ Creates a new Table and initialize it with new data.
     to be inserted into the table
 
 * **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+    Additional options (backwards compatibility)
+
+##### Returns
+
+`Promise`&lt;[`Table`](Table.md)&gt;
+
+#### createTable(name, data, namespace, options)
+
+```ts
+abstract createTable(
+   name,
+   data,
+   namespace?,
+   options?): Promise<Table>
+```
+
+Creates a new Table and initialize it with new data.
+
+##### Parameters
+
+* **name**: `string`
+    The name of the table.
+
+* **data**: [`TableLike`](../type-aliases/TableLike.md) \| `Record`&lt;`string`, `unknown`&gt;[]
+    Non-empty Array of Records
+    to be inserted into the table
+
+* **namespace?**: `string`[]
+    The namespace to create the table in (defaults to root namespace)
+
+* **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+    Additional options
 
 ##### Returns
 
@@ -134,11 +247,16 @@ Return a brief description of the connection
 ### dropAllTables()
 
 ```ts
-abstract dropAllTables(): Promise<void>
+abstract dropAllTables(namespace?): Promise<void>
 ```
 
 Drop all tables in the database.
 
+#### Parameters
+
+* **namespace?**: `string`[]
+    The namespace to drop tables from (defaults to root namespace).
+
 #### Returns
 
 `Promise`&lt;`void`&gt;
@@ -148,7 +266,7 @@ Drop all tables in the database.
 ### dropTable()
 
 ```ts
-abstract dropTable(name): Promise<void>
+abstract dropTable(name, namespace?): Promise<void>
 ```
 
 Drop an existing table.
@@ -158,6 +276,9 @@ Drop an existing table.
 * **name**: `string`
     The name of the table to drop.
 
+* **namespace?**: `string`[]
+    The namespace of the table (defaults to root namespace).
+
 #### Returns
 
 `Promise`&lt;`void`&gt;
@@ -181,7 +302,10 @@ Return true if the connection has not been closed
 ### openTable()
 
 ```ts
-abstract openTable(name, options?): Promise<Table>
+abstract openTable(
+   name,
+   namespace?,
+   options?): Promise<Table>
 ```
 
 Open a table in the database.
@@ -191,7 +315,11 @@ Open a table in the database.
 * **name**: `string`
     The name of the table
 
+* **namespace?**: `string`[]
+    The namespace of the table (defaults to root namespace)
+
 * **options?**: `Partial`&lt;[`OpenTableOptions`](../interfaces/OpenTableOptions.md)&gt;
+    Additional options
 
 #### Returns
 
@@ -201,6 +329,8 @@ Open a table in the database.
 
 ### tableNames()
 
+#### tableNames(options)
+
 ```ts
 abstract tableNames(options?): Promise<string[]>
 ```
@@ -209,12 +339,35 @@ List all the table names in this database.
 
 Tables will be returned in lexicographical order.
 
-#### Parameters
+##### Parameters
+
+* **options?**: `Partial`&lt;[`TableNamesOptions`](../interfaces/TableNamesOptions.md)&gt;
+    options to control the
+    paging / start point (backwards compatibility)
+
+##### Returns
+
+`Promise`&lt;`string`[]&gt;
+
+#### tableNames(namespace, options)
+
+```ts
+abstract tableNames(namespace?, options?): Promise<string[]>
+```
+
+List all the table names in this database.
+
+Tables will be returned in lexicographical order.
+
+##### Parameters
+
+* **namespace?**: `string`[]
+    The namespace to list tables from (defaults to root namespace)
 
 * **options?**: `Partial`&lt;[`TableNamesOptions`](../interfaces/TableNamesOptions.md)&gt;
     options to control the
     paging / start point
 
-#### Returns
+##### Returns
 
 `Promise`&lt;`string`[]&gt;

docs/src/js/classes/HeaderProvider.md

@@ -0,0 +1,85 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / HeaderProvider
+
+# Class: `abstract` HeaderProvider
+
+Abstract base class for providing custom headers for each request.
+
+Users can implement this interface to provide dynamic headers for various purposes
+such as authentication (OAuth tokens, API keys), request tracking (correlation IDs),
+custom metadata, or any other header-based requirements. The provider is called
+before each request to ensure fresh header values are always used.
+
+## Examples
+
+Simple JWT token provider:
+```typescript
+class JWTProvider extends HeaderProvider {
+  constructor(private token: string) {
+    super();
+  }
+
+  getHeaders(): Record<string, string> {
+    return { authorization: `Bearer ${this.token}` };
+  }
+}
+```
+
+Provider with request tracking:
+```typescript
+class RequestTrackingProvider extends HeaderProvider {
+  constructor(private sessionId: string) {
+    super();
+  }
+
+  getHeaders(): Record<string, string> {
+    return {
+      "X-Session-Id": this.sessionId,
+      "X-Request-Id": `req-${Date.now()}`
+    };
+  }
+}
+```
+
+## Extended by
+
+- [`StaticHeaderProvider`](StaticHeaderProvider.md)
+- [`OAuthHeaderProvider`](OAuthHeaderProvider.md)
+
+## Constructors
+
+### new HeaderProvider()
+
+```ts
+new HeaderProvider(): HeaderProvider
+```
+
+#### Returns
+
+[`HeaderProvider`](HeaderProvider.md)
+
+## Methods
+
+### getHeaders()
+
+```ts
+abstract getHeaders(): Record<string, string>
+```
+
+Get the latest headers to be added to requests.
+
+This method is called before each request to the remote LanceDB server.
+Implementations should return headers that will be merged with existing headers.
+
+#### Returns
+
+`Record`&lt;`string`, `string`&gt;
+
+Dictionary of header names to values to add to the request.
+
+#### Throws
+
+If unable to fetch headers, the exception will be propagated and the request will fail.

docs/src/js/classes/MergeInsertBuilder.md

@@ -52,6 +52,30 @@ the merge result
 
 ***
 
+### useIndex()
+
+```ts
+useIndex(useIndex): MergeInsertBuilder
+```
+
+Controls whether to use indexes for the merge operation.
+
+When set to `true` (the default), the operation will use an index if available
+on the join key for improved performance. When set to `false`, it forces a full
+table scan even if an index exists. This can be useful for benchmarking or when
+the query optimizer chooses a suboptimal path.
+
+#### Parameters
+
+* **useIndex**: `boolean`
+    Whether to use indices for the merge operation. Defaults to `true`.
+
+#### Returns
+
+[`MergeInsertBuilder`](MergeInsertBuilder.md)
+
+***
+
 ### whenMatchedUpdateAll()
 
 ```ts

docs/src/js/classes/NativeJsHeaderProvider.md

@@ -0,0 +1,29 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / NativeJsHeaderProvider
+
+# Class: NativeJsHeaderProvider
+
+JavaScript HeaderProvider implementation that wraps a JavaScript callback.
+This is the only native header provider - all header provider implementations
+should provide a JavaScript function that returns headers.
+
+## Constructors
+
+### new NativeJsHeaderProvider()
+
+```ts
+new NativeJsHeaderProvider(getHeadersCallback): NativeJsHeaderProvider
+```
+
+Create a new JsHeaderProvider from a JavaScript callback
+
+#### Parameters
+
+* **getHeadersCallback**
+
+#### Returns
+
+[`NativeJsHeaderProvider`](NativeJsHeaderProvider.md)

docs/src/js/classes/OAuthHeaderProvider.md

@@ -0,0 +1,108 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / OAuthHeaderProvider
+
+# Class: OAuthHeaderProvider
+
+Example implementation: OAuth token provider with automatic refresh.
+
+This is an example implementation showing how to manage OAuth tokens
+with automatic refresh when they expire.
+
+## Example
+
+```typescript
+async function fetchToken(): Promise<TokenResponse> {
+  const response = await fetch("https://oauth.example.com/token", {
+    method: "POST",
+    body: JSON.stringify({
+      grant_type: "client_credentials",
+      client_id: "your-client-id",
+      client_secret: "your-client-secret"
+    }),
+    headers: { "Content-Type": "application/json" }
+  });
+  const data = await response.json();
+  return {
+    accessToken: data.access_token,
+    expiresIn: data.expires_in
+  };
+}
+
+const provider = new OAuthHeaderProvider(fetchToken);
+const headers = provider.getHeaders();
+// Returns: {"authorization": "Bearer <your-token>"}
+```
+
+## Extends
+
+- [`HeaderProvider`](HeaderProvider.md)
+
+## Constructors
+
+### new OAuthHeaderProvider()
+
+```ts
+new OAuthHeaderProvider(tokenFetcher, refreshBufferSeconds): OAuthHeaderProvider
+```
+
+Initialize the OAuth provider.
+
+#### Parameters
+
+* **tokenFetcher**
+    Function to fetch new tokens. Should return object with 'accessToken' and optionally 'expiresIn'.
+
+* **refreshBufferSeconds**: `number` = `300`
+    Seconds before expiry to refresh token. Default 300 (5 minutes).
+
+#### Returns
+
+[`OAuthHeaderProvider`](OAuthHeaderProvider.md)
+
+#### Overrides
+
+[`HeaderProvider`](HeaderProvider.md).[`constructor`](HeaderProvider.md#constructors)
+
+## Methods
+
+### getHeaders()
+
+```ts
+getHeaders(): Record<string, string>
+```
+
+Get OAuth headers, refreshing token if needed.
+Note: This is synchronous for now as the Rust implementation expects sync.
+In a real implementation, this would need to handle async properly.
+
+#### Returns
+
+`Record`&lt;`string`, `string`&gt;
+
+Headers with Bearer token authorization.
+
+#### Throws
+
+If unable to fetch or refresh token.
+
+#### Overrides
+
+[`HeaderProvider`](HeaderProvider.md).[`getHeaders`](HeaderProvider.md#getheaders)
+
+***
+
+### refreshToken()
+
+```ts
+refreshToken(): Promise<void>
+```
+
+Manually refresh the token.
+Call this before using getHeaders() to ensure token is available.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;

docs/src/js/classes/Query.md

@@ -14,7 +14,7 @@ A builder for LanceDB queries.
 
 ## Extends
 
-- [`QueryBase`](QueryBase.md)<`NativeQuery`>
+- `StandardQueryBase`<`NativeQuery`>
 
 ## Properties
 
@@ -26,7 +26,7 @@ protected inner: Query | Promise<Query>;
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`inner`](QueryBase.md#inner)
+`StandardQueryBase.inner`
 
 ## Methods
 
@@ -73,7 +73,7 @@ AnalyzeExec verbose=true, metrics=[]
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`analyzePlan`](QueryBase.md#analyzeplan)
+`StandardQueryBase.analyzePlan`
 
 ***
 
@@ -107,7 +107,7 @@ single query)
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`execute`](QueryBase.md#execute)
+`StandardQueryBase.execute`
 
 ***
 
@@ -143,7 +143,7 @@ const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`explainPlan`](QueryBase.md#explainplan)
+`StandardQueryBase.explainPlan`
 
 ***
 
@@ -164,7 +164,7 @@ Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`fastSearch`](QueryBase.md#fastsearch)
+`StandardQueryBase.fastSearch`
 
 ***
 
@@ -194,7 +194,7 @@ Use `where` instead
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`filter`](QueryBase.md#filter)
+`StandardQueryBase.filter`
 
 ***
 
@@ -216,7 +216,7 @@ fullTextSearch(query, options?): this
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`fullTextSearch`](QueryBase.md#fulltextsearch)
+`StandardQueryBase.fullTextSearch`
 
 ***
 
@@ -241,7 +241,7 @@ called then every valid row from the table will be returned.
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`limit`](QueryBase.md#limit)
+`StandardQueryBase.limit`
 
 ***
 
@@ -325,6 +325,10 @@ nearestToText(query, columns?): Query
 offset(offset): this
 ```
 
+Set the number of rows to skip before returning results.
+
+This is useful for pagination.
+
 #### Parameters
 
 * **offset**: `number`
@@ -335,7 +339,7 @@ offset(offset): this
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`offset`](QueryBase.md#offset)
+`StandardQueryBase.offset`
 
 ***
 
@@ -388,7 +392,7 @@ object insertion order is easy to get wrong and `Map` is more foolproof.
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`select`](QueryBase.md#select)
+`StandardQueryBase.select`
 
 ***
 
@@ -410,7 +414,7 @@ Collect the results as an array of objects.
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`toArray`](QueryBase.md#toarray)
+`StandardQueryBase.toArray`
 
 ***
 
@@ -436,7 +440,7 @@ ArrowTable.
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
+`StandardQueryBase.toArrow`
 
 ***
 
@@ -471,7 +475,7 @@ on the filter column(s).
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`where`](QueryBase.md#where)
+`StandardQueryBase.where`
 
 ***
 
@@ -493,4 +497,4 @@ order to perform hybrid search.
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`withRowId`](QueryBase.md#withrowid)
+`StandardQueryBase.withRowId`

docs/src/js/classes/QueryBase.md

@@ -15,12 +15,11 @@ Common methods supported by all query types
 
 ## Extended by
 
-- [`Query`](Query.md)
-- [`VectorQuery`](VectorQuery.md)
+- [`TakeQuery`](TakeQuery.md)
 
 ## Type Parameters
 
-• **NativeQueryType** *extends* `NativeQuery` \| `NativeVectorQuery`
+• **NativeQueryType** *extends* `NativeQuery` \| `NativeVectorQuery` \| `NativeTakeQuery`
 
 ## Implements
 
@@ -141,104 +140,6 @@ const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
 
 ***
 
-### fastSearch()
-
-```ts
-fastSearch(): this
-```
-
-Skip searching un-indexed data. This can make search faster, but will miss
-any data that is not yet indexed.
-
-Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
-
-#### Returns
-
-`this`
-
-***
-
-### ~~filter()~~
-
-```ts
-filter(predicate): this
-```
-
-A filter statement to be applied to this query.
-
-#### Parameters
-
-* **predicate**: `string`
-
-#### Returns
-
-`this`
-
-#### See
-
-where
-
-#### Deprecated
-
-Use `where` instead
-
-***
-
-### fullTextSearch()
-
-```ts
-fullTextSearch(query, options?): this
-```
-
-#### Parameters
-
-* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
-
-* **options?**: `Partial`<[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)>
-
-#### Returns
-
-`this`
-
-***
-
-### limit()
-
-```ts
-limit(limit): this
-```
-
-Set the maximum number of results to return.
-
-By default, a plain search has no limit.  If this method is not
-called then every valid row from the table will be returned.
-
-#### Parameters
-
-* **limit**: `number`
-
-#### Returns
-
-`this`
-
-***
-
-### offset()
-
-```ts
-offset(offset): this
-```
-
-#### Parameters
-
-* **offset**: `number`
-
-#### Returns
-
-`this`
-
-***
-
 ### select()
 
 ```ts
@@ -328,37 +229,6 @@ ArrowTable.
 
 ***
 
-### where()
-
-```ts
-where(predicate): this
-```
-
-A filter statement to be applied to this query.
-
-The filter should be supplied as an SQL query string.  For example:
-
-#### Parameters
-
-* **predicate**: `string`
-
-#### Returns
-
-`this`
-
-#### Example
-
-```ts
-x > 10
-y > 0 AND y < 100
-x > 5 OR y = 'test'
-
-Filtering performance can often be improved by creating a scalar index
-on the filter column(s).
-```
-
-***
-
 ### withRowId()
 
 ```ts

docs/src/js/classes/Session.md

@@ -9,7 +9,8 @@
 A session for managing caches and object stores across LanceDB operations.
 
 Sessions allow you to configure cache sizes for index and metadata caches,
-which can significantly impact performance for large datasets.
+which can significantly impact memory use and performance. They can
+also be re-used across multiple connections to share the same cache state.
 
 ## Constructors
 
@@ -24,8 +25,11 @@ Create a new session with custom cache sizes.
 # Parameters
 
 - `index_cache_size_bytes`: The size of the index cache in bytes.
+  Index data is stored in memory in this cache to speed up queries.
   Defaults to 6GB if not specified.
 - `metadata_cache_size_bytes`: The size of the metadata cache in bytes.
+  The metadata cache stores file metadata and schema information in memory.
+  This cache improves scan and write performance.
   Defaults to 1GB if not specified.
 
 #### Parameters

docs/src/js/classes/StaticHeaderProvider.md

@@ -0,0 +1,70 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / StaticHeaderProvider
+
+# Class: StaticHeaderProvider
+
+Example implementation: A simple header provider that returns static headers.
+
+This is an example implementation showing how to create a HeaderProvider
+for cases where headers don't change during the session.
+
+## Example
+
+```typescript
+const provider = new StaticHeaderProvider({
+  authorization: "Bearer my-token",
+  "X-Custom-Header": "custom-value"
+});
+const headers = provider.getHeaders();
+// Returns: {authorization: 'Bearer my-token', 'X-Custom-Header': 'custom-value'}
+```
+
+## Extends
+
+- [`HeaderProvider`](HeaderProvider.md)
+
+## Constructors
+
+### new StaticHeaderProvider()
+
+```ts
+new StaticHeaderProvider(headers): StaticHeaderProvider
+```
+
+Initialize with static headers.
+
+#### Parameters
+
+* **headers**: `Record`<`string`, `string`>
+    Headers to return for every request.
+
+#### Returns
+
+[`StaticHeaderProvider`](StaticHeaderProvider.md)
+
+#### Overrides
+
+[`HeaderProvider`](HeaderProvider.md).[`constructor`](HeaderProvider.md#constructors)
+
+## Methods
+
+### getHeaders()
+
+```ts
+getHeaders(): Record<string, string>
+```
+
+Return the static headers.
+
+#### Returns
+
+`Record`&lt;`string`, `string`&gt;
+
+Copy of the static headers.
+
+#### Overrides
+
+[`HeaderProvider`](HeaderProvider.md).[`getHeaders`](HeaderProvider.md#getheaders)

docs/src/js/classes/Table.md

@@ -674,6 +674,48 @@ console.log(tags); // { "v1": { version: 1, manifestSize: ... } }
 
 ***
 
+### takeOffsets()
+
+```ts
+abstract takeOffsets(offsets): TakeQuery
+```
+
+Create a query that returns a subset of the rows in the table.
+
+#### Parameters
+
+* **offsets**: `number`[]
+    The offsets of the rows to return.
+
+#### Returns
+
+[`TakeQuery`](TakeQuery.md)
+
+A builder that can be used to parameterize the query.
+
+***
+
+### takeRowIds()
+
+```ts
+abstract takeRowIds(rowIds): TakeQuery
+```
+
+Create a query that returns a subset of the rows in the table.
+
+#### Parameters
+
+* **rowIds**: `number`[]
+    The row ids of the rows to return.
+
+#### Returns
+
+[`TakeQuery`](TakeQuery.md)
+
+A builder that can be used to parameterize the query.
+
+***
+
 ### toArrow()
 
 ```ts

docs/src/js/classes/TakeQuery.md

@@ -0,0 +1,265 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / TakeQuery
+
+# Class: TakeQuery
+
+A query that returns a subset of the rows in the table.
+
+## Extends
+
+- [`QueryBase`](QueryBase.md)<`NativeTakeQuery`>
+
+## Properties
+
+### inner
+
+```ts
+protected inner: TakeQuery | Promise<TakeQuery>;
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`inner`](QueryBase.md#inner)
+
+## Methods
+
+### analyzePlan()
+
+```ts
+analyzePlan(): Promise<string>
+```
+
+Executes the query and returns the physical query plan annotated with runtime metrics.
+
+This is useful for debugging and performance analysis, as it shows how the query was executed
+and includes metrics such as elapsed time, rows processed, and I/O statistics.
+
+#### Returns
+
+`Promise`&lt;`string`&gt;
+
+A query execution plan with runtime metrics for each step.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+
+const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
+
+Example output (with runtime metrics inlined):
+AnalyzeExec verbose=true, metrics=[]
+ ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
+  Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
+   CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
+    GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
+     FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
+      SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
+       KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
+        LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`analyzePlan`](QueryBase.md#analyzeplan)
+
+***
+
+### execute()
+
+```ts
+protected execute(options?): RecordBatchIterator
+```
+
+Execute the query and return the results as an
+
+#### Parameters
+
+* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+
+#### Returns
+
+[`RecordBatchIterator`](RecordBatchIterator.md)
+
+#### See
+
+ - AsyncIterator
+of
+ - RecordBatch.
+
+By default, LanceDb will use many threads to calculate results and, when
+the result set is large, multiple batches will be processed at one time.
+This readahead is limited however and backpressure will be applied if this
+stream is consumed slowly (this constrains the maximum memory used by a
+single query)
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`execute`](QueryBase.md#execute)
+
+***
+
+### explainPlan()
+
+```ts
+explainPlan(verbose): Promise<string>
+```
+
+Generates an explanation of the query execution plan.
+
+#### Parameters
+
+* **verbose**: `boolean` = `false`
+    If true, provides a more detailed explanation. Defaults to false.
+
+#### Returns
+
+`Promise`&lt;`string`&gt;
+
+A Promise that resolves to a string containing the query execution plan explanation.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`explainPlan`](QueryBase.md#explainplan)
+
+***
+
+### select()
+
+```ts
+select(columns): this
+```
+
+Return only the specified columns.
+
+By default a query will return all columns from the table.  However, this can have
+a very significant impact on latency.  LanceDb stores data in a columnar fashion.  This
+means we can finely tune our I/O to select exactly the columns we need.
+
+As a best practice you should always limit queries to the columns that you need.  If you
+pass in an array of column names then only those columns will be returned.
+
+You can also use this method to create new "dynamic" columns based on your existing columns.
+For example, you may not care about "a" or "b" but instead simply want "a + b".  This is often
+seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
+
+To create dynamic columns you can pass in a Map<string, string>.  A column will be returned
+for each entry in the map.  The key provides the name of the column.  The value is
+an SQL string used to specify how the column is calculated.
+
+For example, an SQL query might state `SELECT a + b AS combined, c`.  The equivalent
+input to this method would be:
+
+#### Parameters
+
+* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
+
+#### Returns
+
+`this`
+
+#### Example
+
+```ts
+new Map([["combined", "a + b"], ["c", "c"]])
+
+Columns will always be returned in the order given, even if that order is different than
+the order used when adding the data.
+
+Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
+uses `Object.entries` which should preserve the insertion order of the object.  However,
+object insertion order is easy to get wrong and `Map` is more foolproof.
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`select`](QueryBase.md#select)
+
+***
+
+### toArray()
+
+```ts
+toArray(options?): Promise<any[]>
+```
+
+Collect the results as an array of objects.
+
+#### Parameters
+
+* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+
+#### Returns
+
+`Promise`&lt;`any`[]&gt;
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`toArray`](QueryBase.md#toarray)
+
+***
+
+### toArrow()
+
+```ts
+toArrow(options?): Promise<Table<any>>
+```
+
+Collect the results as an Arrow
+
+#### Parameters
+
+* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+
+#### Returns
+
+`Promise`&lt;`Table`&lt;`any`&gt;&gt;
+
+#### See
+
+ArrowTable.
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
+
+***
+
+### withRowId()
+
+```ts
+withRowId(): this
+```
+
+Whether to return the row id in the results.
+
+This column can be used to match results between different queries. For
+example, to match results from a full text search and a vector search in
+order to perform hybrid search.
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`withRowId`](QueryBase.md#withrowid)

docs/src/js/classes/VectorQuery.md

@@ -16,7 +16,7 @@ This builder can be reused to execute the query many times.
 
 ## Extends
 
-- [`QueryBase`](QueryBase.md)<`NativeVectorQuery`>
+- `StandardQueryBase`<`NativeVectorQuery`>
 
 ## Properties
 
@@ -28,7 +28,7 @@ protected inner: VectorQuery | Promise<VectorQuery>;
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`inner`](QueryBase.md#inner)
+`StandardQueryBase.inner`
 
 ## Methods
 
@@ -91,7 +91,7 @@ AnalyzeExec verbose=true, metrics=[]
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`analyzePlan`](QueryBase.md#analyzeplan)
+`StandardQueryBase.analyzePlan`
 
 ***
 
@@ -248,7 +248,7 @@ single query)
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`execute`](QueryBase.md#execute)
+`StandardQueryBase.execute`
 
 ***
 
@@ -284,7 +284,7 @@ const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`explainPlan`](QueryBase.md#explainplan)
+`StandardQueryBase.explainPlan`
 
 ***
 
@@ -305,7 +305,7 @@ Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`fastSearch`](QueryBase.md#fastsearch)
+`StandardQueryBase.fastSearch`
 
 ***
 
@@ -335,7 +335,7 @@ Use `where` instead
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`filter`](QueryBase.md#filter)
+`StandardQueryBase.filter`
 
 ***
 
@@ -357,7 +357,7 @@ fullTextSearch(query, options?): this
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`fullTextSearch`](QueryBase.md#fulltextsearch)
+`StandardQueryBase.fullTextSearch`
 
 ***
 
@@ -382,7 +382,7 @@ called then every valid row from the table will be returned.
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`limit`](QueryBase.md#limit)
+`StandardQueryBase.limit`
 
 ***
 
@@ -480,6 +480,10 @@ the minimum and maximum to the same value.
 offset(offset): this
 ```
 
+Set the number of rows to skip before returning results.
+
+This is useful for pagination.
+
 #### Parameters
 
 * **offset**: `number`
@@ -490,7 +494,7 @@ offset(offset): this
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`offset`](QueryBase.md#offset)
+`StandardQueryBase.offset`
 
 ***
 
@@ -637,7 +641,7 @@ object insertion order is easy to get wrong and `Map` is more foolproof.
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`select`](QueryBase.md#select)
+`StandardQueryBase.select`
 
 ***
 
@@ -659,7 +663,7 @@ Collect the results as an array of objects.
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`toArray`](QueryBase.md#toarray)
+`StandardQueryBase.toArray`
 
 ***
 
@@ -685,7 +689,7 @@ ArrowTable.
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
+`StandardQueryBase.toArrow`
 
 ***
 
@@ -720,7 +724,7 @@ on the filter column(s).
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`where`](QueryBase.md#where)
+`StandardQueryBase.where`
 
 ***
 
@@ -742,4 +746,4 @@ order to perform hybrid search.
 
 #### Inherited from
 
-[`QueryBase`](QueryBase.md).[`withRowId`](QueryBase.md#withrowid)
+`StandardQueryBase.withRowId`

docs/src/js/functions/connect.md

@@ -6,13 +6,14 @@
 
 # Function: connect()
 
-## connect(uri, options, session)
+## connect(uri, options, session, headerProvider)
 
 ```ts
 function connect(
    uri,
    options?,
-   session?): Promise<Connection>
+   session?,
+   headerProvider?): Promise<Connection>
 ```
 
 Connect to a LanceDB instance at the given URI.
@@ -34,6 +35,8 @@ Accepted formats:
 
 * **session?**: [`Session`](../classes/Session.md)
 
+* **headerProvider?**: [`HeaderProvider`](../classes/HeaderProvider.md) \| () => `Record`&lt;`string`, `string`&gt; \| () => `Promise`&lt;`Record`&lt;`string`, `string`&gt;&gt;
+
 ### Returns
 
 `Promise`&lt;[`Connection`](../classes/Connection.md)&gt;
@@ -55,6 +58,18 @@ const conn = await connect(
 });
 ```
 
+Using with a header provider for per-request authentication:
+```ts
+const provider = new StaticHeaderProvider({
+  "X-API-Key": "my-key"
+});
+const conn = await connectWithHeaderProvider(
+  "db://host:port",
+  options,
+  provider
+);
+```
+
 ## connect(options)
 
 ```ts

docs/src/js/functions/makeArrowTable.md

@@ -13,7 +13,7 @@ function makeArrowTable(
    metadata?): ArrowTable
 ```
 
-An enhanced version of the makeTable function from Apache Arrow
+An enhanced version of the apache-arrow makeTable function from Apache Arrow
 that supports nested fields and embeddings columns.
 
 (typically you do not need to call this function.  It will be called automatically

docs/src/js/globals.md

@@ -20,19 +20,24 @@
 - [BooleanQuery](classes/BooleanQuery.md)
 - [BoostQuery](classes/BoostQuery.md)
 - [Connection](classes/Connection.md)
+- [HeaderProvider](classes/HeaderProvider.md)
 - [Index](classes/Index.md)
 - [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
 - [MatchQuery](classes/MatchQuery.md)
 - [MergeInsertBuilder](classes/MergeInsertBuilder.md)
 - [MultiMatchQuery](classes/MultiMatchQuery.md)
+- [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md)
+- [OAuthHeaderProvider](classes/OAuthHeaderProvider.md)
 - [PhraseQuery](classes/PhraseQuery.md)
 - [Query](classes/Query.md)
 - [QueryBase](classes/QueryBase.md)
 - [RecordBatchIterator](classes/RecordBatchIterator.md)
 - [Session](classes/Session.md)
+- [StaticHeaderProvider](classes/StaticHeaderProvider.md)
 - [Table](classes/Table.md)
 - [TagContents](classes/TagContents.md)
 - [Tags](classes/Tags.md)
+- [TakeQuery](classes/TakeQuery.md)
 - [VectorColumnOptions](classes/VectorColumnOptions.md)
 - [VectorQuery](classes/VectorQuery.md)
 
@@ -73,6 +78,8 @@
 - [TableNamesOptions](interfaces/TableNamesOptions.md)
 - [TableStatistics](interfaces/TableStatistics.md)
 - [TimeoutConfig](interfaces/TimeoutConfig.md)
+- [TlsConfig](interfaces/TlsConfig.md)
+- [TokenResponse](interfaces/TokenResponse.md)
 - [UpdateOptions](interfaces/UpdateOptions.md)
 - [UpdateResult](interfaces/UpdateResult.md)
 - [Version](interfaces/Version.md)

docs/src/js/interfaces/ClientConfig.md

@@ -16,6 +16,14 @@ optional extraHeaders: Record<string, string>;
 
 ***
 
+### idDelimiter?
+
+```ts
+optional idDelimiter: string;
+```
+
+***
+
 ### retryConfig?
 
 ```ts
@@ -32,6 +40,14 @@ optional timeoutConfig: TimeoutConfig;
 
 ***
 
+### tlsConfig?
+
+```ts
+optional tlsConfig: TlsConfig;
+```
+
+***
+
 ### userAgent?
 
 ```ts

docs/src/js/interfaces/IndexOptions.md

@@ -26,6 +26,18 @@ will be used to determine the most useful kind of index to create.
 
 ***
 
+### name?
+
+```ts
+optional name: string;
+```
+
+Optional custom name for the index.
+
+If not provided, a default name will be generated based on the column name.
+
+***
+
 ### replace?
 
 ```ts
@@ -42,8 +54,27 @@ The default is true
 
 ***
 
+### train?
+
+```ts
+optional train: boolean;
+```
+
+Whether to train the index with existing data.
+
+If true (default), the index will be trained with existing data in the table.
+If false, the index will be created empty and populated as new data is added.
+
+Note: This option is only supported for scalar indices. Vector indices always train.
+
+***
+
 ### waitTimeoutSeconds?
 
 ```ts
 optional waitTimeoutSeconds: number;
 ```
+
+Timeout in seconds to wait for index creation to complete.
+
+If not specified, the method will return immediately after starting the index creation.

docs/src/js/interfaces/TimeoutConfig.md

@@ -44,3 +44,17 @@ optional readTimeout: number;
 The timeout for reading data from the server in seconds. Default is 300
 seconds (5 minutes). This can also be set via the environment variable
 `LANCE_CLIENT_READ_TIMEOUT`, as an integer number of seconds.
+
+***
+
+### timeout?
+
+```ts
+optional timeout: number;
+```
+
+The overall timeout for the entire request in seconds. This includes
+connection, send, and read time. If the entire request doesn't complete
+within this time, it will fail. Default is None (no overall timeout).
+This can also be set via the environment variable `LANCE_CLIENT_TIMEOUT`,
+as an integer number of seconds.

docs/src/js/interfaces/TlsConfig.md

@@ -0,0 +1,49 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / TlsConfig
+
+# Interface: TlsConfig
+
+TLS/mTLS configuration for the remote HTTP client.
+
+## Properties
+
+### assertHostname?
+
+```ts
+optional assertHostname: boolean;
+```
+
+Whether to verify the hostname in the server's certificate.
+
+***
+
+### certFile?
+
+```ts
+optional certFile: string;
+```
+
+Path to the client certificate file (PEM format) for mTLS authentication.
+
+***
+
+### keyFile?
+
+```ts
+optional keyFile: string;
+```
+
+Path to the client private key file (PEM format) for mTLS authentication.
+
+***
+
+### sslCaCert?
+
+```ts
+optional sslCaCert: string;
+```
+
+Path to the CA certificate file (PEM format) for server verification.

docs/src/js/interfaces/TokenResponse.md

@@ -0,0 +1,25 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / TokenResponse
+
+# Interface: TokenResponse
+
+Token response from OAuth provider.
+
+## Properties
+
+### accessToken
+
+```ts
+accessToken: string;
+```
+
+***
+
+### expiresIn?
+
+```ts
+optional expiresIn: number;
+```

java/core/lancedb-jni/Cargo.toml

@@ -15,7 +15,7 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-lancedb = { path = "../../../rust/lancedb" }
+lancedb = { path = "../../../rust/lancedb", default-features = false }
 lance = { workspace = true }
 arrow = { workspace = true, features = ["ffi"] }
 arrow-schema.workspace = true
@@ -25,3 +25,6 @@ snafu.workspace = true
 lazy_static.workspace = true
 serde = { version = "^1" }
 serde_json = { version = "1" }
+
+[features]
+default = ["lancedb/default"]

java/core/lancedb-jni/src/ffi.rs

@@ -16,6 +16,7 @@ pub trait JNIEnvExt {
     fn get_integers(&mut self, obj: &JObject) -> Result<Vec<i32>>;
 
     /// Get strings from Java List<String> object.
+    #[allow(dead_code)]
     fn get_strings(&mut self, obj: &JObject) -> Result<Vec<String>>;
 
     /// Get strings from Java String[] object.

java/core/lancedb-jni/src/traits.rs

@@ -6,6 +6,7 @@ use jni::JNIEnv;
 
 use crate::Result;
 
+#[allow(dead_code)]
 pub trait FromJObject<T> {
     fn extract(&self) -> Result<T>;
 }
@@ -39,6 +40,7 @@ impl FromJObject<f64> for JObject<'_> {
     }
 }
 
+#[allow(dead_code)]
 pub trait FromJString {
     fn extract(&self, env: &mut JNIEnv) -> Result<String>;
 }
@@ -66,6 +68,7 @@ pub trait JMapExt {
     fn get_f64(&self, env: &mut JNIEnv, key: &str) -> Result<Option<f64>>;
 }
 
+#[allow(dead_code)]
 fn get_map_value<T>(env: &mut JNIEnv, map: &JMap, key: &str) -> Result<Option<T>>
 where
     for<'a> JObject<'a>: FromJObject<T>,

java/core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>com.lancedb</groupId>
         <artifactId>lancedb-parent</artifactId>
-        <version>0.21.2-final.0</version>
+        <version>0.22.2-beta.2</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

java/lance-namespace/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>com.lancedb</groupId>
         <artifactId>lancedb-parent</artifactId>
-        <version>0.21.2-final.0</version>
+        <version>0.22.2-beta.2</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.21.2-final.0</version>
+    <version>0.22.2-beta.2</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <description>LanceDB Java SDK Parent POM</description>

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.21.2"
+version = "0.22.2-beta.2"
 license.workspace = true
 description.workspace = true
 repository.workspace = true
@@ -18,7 +18,7 @@ arrow-array.workspace = true
 arrow-schema.workspace = true
 env_logger.workspace = true
 futures.workspace = true
-lancedb = { path = "../rust/lancedb" }
+lancedb = { path = "../rust/lancedb", default-features = false }
 napi = { version = "2.16.8", default-features = false, features = [
     "napi9",
     "async"
@@ -36,6 +36,6 @@ aws-lc-rs = "=1.13.0"
 napi-build = "2.1"
 
 [features]
-default = ["remote"]
+default = ["remote", "lancedb/default"]
 fp16kernels = ["lancedb/fp16kernels"]
 remote = ["lancedb/remote"]

nodejs/__test__/arrow.test.ts

@@ -1,17 +1,5 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-import {
-  Bool,
-  Field,
-  Int32,
-  List,
-  Schema,
-  Struct,
-  Uint8,
-  Utf8,
-} from "apache-arrow";
-
 import * as arrow15 from "apache-arrow-15";
 import * as arrow16 from "apache-arrow-16";
 import * as arrow17 from "apache-arrow-17";
@@ -25,11 +13,9 @@ import {
   fromTableToBuffer,
   makeArrowTable,
   makeEmptyTable,
-  tableFromIPC,
 } from "../lancedb/arrow";
 import {
   EmbeddingFunction,
-  FieldOptions,
   FunctionOptions,
 } from "../lancedb/embedding/embedding_function";
 import { EmbeddingFunctionConfig } from "../lancedb/embedding/registry";
@@ -1008,5 +994,64 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         expect(result).toEqual(null);
       });
     });
+
+    describe("boolean null handling", function () {
+      it("should handle null values in nullable boolean fields", () => {
+        const { makeArrowTable } = require("../lancedb/arrow");
+        const schema = new Schema([new Field("test", new arrow.Bool(), true)]);
+
+        // Test with all null values
+        const data = [{ test: null }];
+        const table = makeArrowTable(data, { schema });
+
+        expect(table.numRows).toBe(1);
+        expect(table.schema.names).toEqual(["test"]);
+        expect(table.getChild("test")!.get(0)).toBeNull();
+      });
+
+      it("should handle mixed null and non-null boolean values", () => {
+        const { makeArrowTable } = require("../lancedb/arrow");
+        const schema = new Schema([new Field("test", new Bool(), true)]);
+
+        // Test with mixed values
+        const data = [{ test: true }, { test: null }, { test: false }];
+        const table = makeArrowTable(data, { schema });
+
+        expect(table.numRows).toBe(3);
+        expect(table.getChild("test")!.get(0)).toBe(true);
+        expect(table.getChild("test")!.get(1)).toBeNull();
+        expect(table.getChild("test")!.get(2)).toBe(false);
+      });
+    });
+
+    // Test for the undefined values bug fix
+    describe("undefined values handling", () => {
+      it("should handle mixed undefined and actual values", () => {
+        const schema = new Schema([
+          new Field("text", new Utf8(), true), // nullable
+          new Field("number", new Int32(), true), // nullable
+          new Field("bool", new Bool(), true), // nullable
+        ]);
+
+        const data = [
+          { text: undefined, number: 42, bool: true },
+          { text: "hello", number: undefined, bool: false },
+          { text: "world", number: 123, bool: undefined },
+        ];
+        const table = makeArrowTable(data, { schema });
+
+        const result = table.toArray();
+        expect(result).toHaveLength(3);
+        expect(result[0].text).toBe(null);
+        expect(result[0].number).toBe(42);
+        expect(result[0].bool).toBe(true);
+        expect(result[1].text).toBe("hello");
+        expect(result[1].number).toBe(null);
+        expect(result[1].bool).toBe(false);
+        expect(result[2].text).toBe("world");
+        expect(result[2].number).toBe(123);
+        expect(result[2].bool).toBe(null);
+      });
+    });
   },
 );

nodejs/__test__/connection.test.ts

@@ -203,3 +203,106 @@ describe("given a connection", () => {
     });
   });
 });
+
+describe("clone table functionality", () => {
+  let tmpDir: tmp.DirResult;
+  let db: Connection;
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    db = await connect(tmpDir.name);
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("should clone a table with latest version (default behavior)", async () => {
+    // Create source table with some data
+    const data = [
+      { id: 1, text: "hello", vector: [1.0, 2.0] },
+      { id: 2, text: "world", vector: [3.0, 4.0] },
+    ];
+    const sourceTable = await db.createTable("source", data);
+
+    // Add more data to create a new version
+    const moreData = [{ id: 3, text: "test", vector: [5.0, 6.0] }];
+    await sourceTable.add(moreData);
+
+    // Clone the table (should get latest version with 3 rows)
+    const sourceUri = `${tmpDir.name}/source.lance`;
+    const clonedTable = await db.cloneTable("cloned", sourceUri);
+
+    // Verify cloned table has all 3 rows
+    expect(await clonedTable.countRows()).toBe(3);
+    expect((await db.tableNames()).includes("cloned")).toBe(true);
+  });
+
+  it("should clone a table from a specific version", async () => {
+    // Create source table with initial data
+    const data = [
+      { id: 1, text: "hello", vector: [1.0, 2.0] },
+      { id: 2, text: "world", vector: [3.0, 4.0] },
+    ];
+    const sourceTable = await db.createTable("source", data);
+
+    // Get the initial version
+    const initialVersion = await sourceTable.version();
+
+    // Add more data to create a new version
+    const moreData = [{ id: 3, text: "test", vector: [5.0, 6.0] }];
+    await sourceTable.add(moreData);
+
+    // Verify source now has 3 rows
+    expect(await sourceTable.countRows()).toBe(3);
+
+    // Clone from the initial version (should have only 2 rows)
+    const sourceUri = `${tmpDir.name}/source.lance`;
+    const clonedTable = await db.cloneTable("cloned", sourceUri, {
+      sourceVersion: initialVersion,
+    });
+
+    // Verify cloned table has only the initial 2 rows
+    expect(await clonedTable.countRows()).toBe(2);
+  });
+
+  it("should clone a table from a tagged version", async () => {
+    // Create source table with initial data
+    const data = [
+      { id: 1, text: "hello", vector: [1.0, 2.0] },
+      { id: 2, text: "world", vector: [3.0, 4.0] },
+    ];
+    const sourceTable = await db.createTable("source", data);
+
+    // Create a tag for the current version
+    const tags = await sourceTable.tags();
+    await tags.create("v1.0", await sourceTable.version());
+
+    // Add more data after the tag
+    const moreData = [{ id: 3, text: "test", vector: [5.0, 6.0] }];
+    await sourceTable.add(moreData);
+
+    // Verify source now has 3 rows
+    expect(await sourceTable.countRows()).toBe(3);
+
+    // Clone from the tagged version (should have only 2 rows)
+    const sourceUri = `${tmpDir.name}/source.lance`;
+    const clonedTable = await db.cloneTable("cloned", sourceUri, {
+      sourceTag: "v1.0",
+    });
+
+    // Verify cloned table has only the tagged version's 2 rows
+    expect(await clonedTable.countRows()).toBe(2);
+  });
+
+  it("should fail when attempting deep clone", async () => {
+    // Create source table with some data
+    const data = [
+      { id: 1, text: "hello", vector: [1.0, 2.0] },
+      { id: 2, text: "world", vector: [3.0, 4.0] },
+    ];
+    await db.createTable("source", data);
+
+    // Try to create a deep clone (should fail)
+    const sourceUri = `${tmpDir.name}/source.lance`;
+    await expect(
+      db.cloneTable("cloned", sourceUri, { isShallow: false }),
+    ).rejects.toThrow("Deep clone is not yet implemented");
+  });
+});

nodejs/__test__/embedding.test.ts

@@ -256,6 +256,60 @@ describe("embedding functions", () => {
     expect(actual).toHaveProperty("text");
   });
 
+  it("should handle undefined vector field with embedding function correctly", async () => {
+    @register("undefined_test")
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): Float {
+        return new Float32();
+      }
+      async computeQueryEmbeddings(_data: string) {
+        return [1, 2, 3];
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return Array.from({ length: data.length }).fill([
+          1, 2, 3,
+        ]) as number[][];
+      }
+    }
+    const func = getRegistry()
+      .get<MockEmbeddingFunction>("undefined_test")!
+      .create();
+    const schema = new Schema([
+      new Field("text", new Utf8(), true),
+      new Field(
+        "vector",
+        new FixedSizeList(3, new Field("item", new Float32(), true)),
+        true,
+      ),
+    ]);
+
+    const db = await connect(tmpDir.name);
+    const table = await db.createEmptyTable("test_undefined", schema, {
+      embeddingFunction: {
+        function: func,
+        sourceColumn: "text",
+        vectorColumn: "vector",
+      },
+    });
+
+    // Test that undefined, null, and omitted vector fields all work
+    await table.add([{ text: "test1", vector: undefined }]);
+    await table.add([{ text: "test2", vector: null }]);
+    await table.add([{ text: "test3" }]);
+
+    const rows = await table.query().toArray();
+    expect(rows.length).toBe(3);
+
+    // All rows should have vectors computed by the embedding function
+    for (const row of rows) {
+      expect(row.vector).toBeDefined();
+      expect(JSON.parse(JSON.stringify(row.vector))).toEqual([1, 2, 3]);
+    }
+  });
+
   test.each([new Float16(), new Float32(), new Float64()])(
     "should be able to provide manual embeddings with multiple float datatype",
     async (floatType) => {

nodejs/__test__/remote.test.ts

@@ -3,7 +3,49 @@
 
 import * as http from "http";
 import { RequestListener } from "http";
-import { Connection, ConnectionOptions, connect } from "../lancedb";
+import {
+  ClientConfig,
+  Connection,
+  ConnectionOptions,
+  TlsConfig,
+  connect,
+} from "../lancedb";
+import {
+  HeaderProvider,
+  OAuthHeaderProvider,
+  StaticHeaderProvider,
+} from "../lancedb/header";
+
+// Test-only header providers
+class CustomProvider extends HeaderProvider {
+  getHeaders(): Record<string, string> {
+    return { "X-Custom": "custom-value" };
+  }
+}
+
+class ErrorProvider extends HeaderProvider {
+  private errorMessage: string;
+  public callCount: number = 0;
+
+  constructor(errorMessage: string = "Test error") {
+    super();
+    this.errorMessage = errorMessage;
+  }
+
+  getHeaders(): Record<string, string> {
+    this.callCount++;
+    throw new Error(this.errorMessage);
+  }
+}
+
+class ConcurrentProvider extends HeaderProvider {
+  private counter: number = 0;
+
+  getHeaders(): Record<string, string> {
+    this.counter++;
+    return { "X-Request-Id": String(this.counter) };
+  }
+}
 
 async function withMockDatabase(
   listener: RequestListener,
@@ -148,4 +190,431 @@ describe("remote connection", () => {
       },
     );
   });
+
+  describe("TlsConfig", () => {
+    it("should create TlsConfig with all fields", () => {
+      const tlsConfig: TlsConfig = {
+        certFile: "/path/to/cert.pem",
+        keyFile: "/path/to/key.pem",
+        sslCaCert: "/path/to/ca.pem",
+        assertHostname: false,
+      };
+
+      expect(tlsConfig.certFile).toBe("/path/to/cert.pem");
+      expect(tlsConfig.keyFile).toBe("/path/to/key.pem");
+      expect(tlsConfig.sslCaCert).toBe("/path/to/ca.pem");
+      expect(tlsConfig.assertHostname).toBe(false);
+    });
+
+    it("should create TlsConfig with partial fields", () => {
+      const tlsConfig: TlsConfig = {
+        certFile: "/path/to/cert.pem",
+        keyFile: "/path/to/key.pem",
+      };
+
+      expect(tlsConfig.certFile).toBe("/path/to/cert.pem");
+      expect(tlsConfig.keyFile).toBe("/path/to/key.pem");
+      expect(tlsConfig.sslCaCert).toBeUndefined();
+      expect(tlsConfig.assertHostname).toBeUndefined();
+    });
+
+    it("should create ClientConfig with TlsConfig", () => {
+      const tlsConfig: TlsConfig = {
+        certFile: "/path/to/cert.pem",
+        keyFile: "/path/to/key.pem",
+        sslCaCert: "/path/to/ca.pem",
+        assertHostname: true,
+      };
+
+      const clientConfig: ClientConfig = {
+        userAgent: "test-agent",
+        tlsConfig: tlsConfig,
+      };
+
+      expect(clientConfig.userAgent).toBe("test-agent");
+      expect(clientConfig.tlsConfig).toBeDefined();
+      expect(clientConfig.tlsConfig?.certFile).toBe("/path/to/cert.pem");
+      expect(clientConfig.tlsConfig?.keyFile).toBe("/path/to/key.pem");
+      expect(clientConfig.tlsConfig?.sslCaCert).toBe("/path/to/ca.pem");
+      expect(clientConfig.tlsConfig?.assertHostname).toBe(true);
+    });
+
+    it("should handle empty TlsConfig", () => {
+      const tlsConfig: TlsConfig = {};
+
+      expect(tlsConfig.certFile).toBeUndefined();
+      expect(tlsConfig.keyFile).toBeUndefined();
+      expect(tlsConfig.sslCaCert).toBeUndefined();
+      expect(tlsConfig.assertHostname).toBeUndefined();
+    });
+
+    it("should accept TlsConfig in connection options", () => {
+      const tlsConfig: TlsConfig = {
+        certFile: "/path/to/cert.pem",
+        keyFile: "/path/to/key.pem",
+        sslCaCert: "/path/to/ca.pem",
+        assertHostname: false,
+      };
+
+      // Just verify that the ClientConfig accepts the TlsConfig
+      const clientConfig: ClientConfig = {
+        tlsConfig: tlsConfig,
+      };
+
+      const connectionOptions: ConnectionOptions = {
+        apiKey: "fake",
+        clientConfig: clientConfig,
+      };
+
+      // Verify the configuration structure is correct
+      expect(connectionOptions.clientConfig).toBeDefined();
+      expect(connectionOptions.clientConfig?.tlsConfig).toBeDefined();
+      expect(connectionOptions.clientConfig?.tlsConfig?.certFile).toBe(
+        "/path/to/cert.pem",
+      );
+    });
+  });
+
+  describe("header providers", () => {
+    it("should work with StaticHeaderProvider", async () => {
+      const provider = new StaticHeaderProvider({
+        authorization: "Bearer test-token",
+        "X-Custom": "value",
+      });
+
+      const headers = provider.getHeaders();
+      expect(headers).toEqual({
+        authorization: "Bearer test-token",
+        "X-Custom": "value",
+      });
+
+      // Test that it returns a copy
+      headers["X-Modified"] = "modified";
+      const headers2 = provider.getHeaders();
+      expect(headers2).not.toHaveProperty("X-Modified");
+    });
+
+    it("should pass headers from StaticHeaderProvider to requests", async () => {
+      const provider = new StaticHeaderProvider({
+        "X-Custom-Auth": "secret-token",
+        "X-Request-Source": "test-suite",
+      });
+
+      await withMockDatabase(
+        (req, res) => {
+          expect(req.headers["x-custom-auth"]).toEqual("secret-token");
+          expect(req.headers["x-request-source"]).toEqual("test-suite");
+
+          const body = JSON.stringify({ tables: [] });
+          res.writeHead(200, { "Content-Type": "application/json" }).end(body);
+        },
+        async () => {
+          // Use actual header provider mechanism instead of extraHeaders
+          const conn = await connect(
+            "db://dev",
+            {
+              apiKey: "fake",
+              hostOverride: "http://localhost:8000",
+            },
+            undefined, // session
+            provider, // headerProvider
+          );
+
+          const tableNames = await conn.tableNames();
+          expect(tableNames).toEqual([]);
+        },
+      );
+    });
+
+    it("should work with CustomProvider", () => {
+      const provider = new CustomProvider();
+      const headers = provider.getHeaders();
+      expect(headers).toEqual({ "X-Custom": "custom-value" });
+    });
+
+    it("should handle ErrorProvider errors", () => {
+      const provider = new ErrorProvider("Authentication failed");
+
+      expect(() => provider.getHeaders()).toThrow("Authentication failed");
+      expect(provider.callCount).toBe(1);
+
+      // Test that error is thrown each time
+      expect(() => provider.getHeaders()).toThrow("Authentication failed");
+      expect(provider.callCount).toBe(2);
+    });
+
+    it("should work with ConcurrentProvider", () => {
+      const provider = new ConcurrentProvider();
+
+      const headers1 = provider.getHeaders();
+      const headers2 = provider.getHeaders();
+      const headers3 = provider.getHeaders();
+
+      expect(headers1).toEqual({ "X-Request-Id": "1" });
+      expect(headers2).toEqual({ "X-Request-Id": "2" });
+      expect(headers3).toEqual({ "X-Request-Id": "3" });
+    });
+
+    describe("OAuthHeaderProvider", () => {
+      it("should initialize correctly", () => {
+        const fetcher = () => ({
+          accessToken: "token123",
+          expiresIn: 3600,
+        });
+
+        const provider = new OAuthHeaderProvider(fetcher);
+        expect(provider).toBeInstanceOf(HeaderProvider);
+      });
+
+      it("should fetch token on first use", async () => {
+        let callCount = 0;
+        const fetcher = () => {
+          callCount++;
+          return {
+            accessToken: "token123",
+            expiresIn: 3600,
+          };
+        };
+
+        const provider = new OAuthHeaderProvider(fetcher);
+
+        // Need to manually refresh first due to sync limitation
+        await provider.refreshToken();
+
+        const headers = provider.getHeaders();
+        expect(headers).toEqual({ authorization: "Bearer token123" });
+        expect(callCount).toBe(1);
+
+        // Second call should not fetch again
+        const headers2 = provider.getHeaders();
+        expect(headers2).toEqual({ authorization: "Bearer token123" });
+        expect(callCount).toBe(1);
+      });
+
+      it("should handle tokens without expiry", async () => {
+        const fetcher = () => ({
+          accessToken: "permanent_token",
+        });
+
+        const provider = new OAuthHeaderProvider(fetcher);
+        await provider.refreshToken();
+
+        const headers = provider.getHeaders();
+        expect(headers).toEqual({ authorization: "Bearer permanent_token" });
+      });
+
+      it("should throw error when access_token is missing", async () => {
+        const fetcher = () =>
+          ({
+            expiresIn: 3600,
+          }) as { accessToken?: string; expiresIn?: number };
+
+        const provider = new OAuthHeaderProvider(
+          fetcher as () => {
+            accessToken: string;
+            expiresIn?: number;
+          },
+        );
+
+        await expect(provider.refreshToken()).rejects.toThrow(
+          "Token fetcher did not return 'accessToken'",
+        );
+      });
+
+      it("should handle async token fetchers", async () => {
+        const fetcher = async () => {
+          // Simulate async operation
+          await new Promise((resolve) => setTimeout(resolve, 10));
+          return {
+            accessToken: "async_token",
+            expiresIn: 3600,
+          };
+        };
+
+        const provider = new OAuthHeaderProvider(fetcher);
+        await provider.refreshToken();
+
+        const headers = provider.getHeaders();
+        expect(headers).toEqual({ authorization: "Bearer async_token" });
+      });
+    });
+
+    it("should merge header provider headers with extra headers", async () => {
+      const provider = new StaticHeaderProvider({
+        "X-From-Provider": "provider-value",
+      });
+
+      await withMockDatabase(
+        (req, res) => {
+          expect(req.headers["x-from-provider"]).toEqual("provider-value");
+          expect(req.headers["x-extra-header"]).toEqual("extra-value");
+
+          const body = JSON.stringify({ tables: [] });
+          res.writeHead(200, { "Content-Type": "application/json" }).end(body);
+        },
+        async () => {
+          // Use header provider with additional extraHeaders
+          const conn = await connect(
+            "db://dev",
+            {
+              apiKey: "fake",
+              hostOverride: "http://localhost:8000",
+              clientConfig: {
+                extraHeaders: {
+                  "X-Extra-Header": "extra-value",
+                },
+              },
+            },
+            undefined, // session
+            provider, // headerProvider
+          );
+
+          const tableNames = await conn.tableNames();
+          expect(tableNames).toEqual([]);
+        },
+      );
+    });
+  });
+
+  describe("header provider integration", () => {
+    it("should work with TypeScript StaticHeaderProvider", async () => {
+      let requestCount = 0;
+
+      await withMockDatabase(
+        (req, res) => {
+          requestCount++;
+
+          // Check headers are present on each request
+          expect(req.headers["authorization"]).toEqual("Bearer test-token-123");
+          expect(req.headers["x-custom"]).toEqual("custom-value");
+
+          // Return different responses based on the endpoint
+          if (req.url === "/v1/table/test_table/describe/") {
+            const body = JSON.stringify({
+              name: "test_table",
+              schema: { fields: [] },
+            });
+            res
+              .writeHead(200, { "Content-Type": "application/json" })
+              .end(body);
+          } else {
+            const body = JSON.stringify({ tables: ["test_table"] });
+            res
+              .writeHead(200, { "Content-Type": "application/json" })
+              .end(body);
+          }
+        },
+        async () => {
+          // Create provider with static headers
+          const provider = new StaticHeaderProvider({
+            authorization: "Bearer test-token-123",
+            "X-Custom": "custom-value",
+          });
+
+          // Connect with the provider
+          const conn = await connect(
+            "db://dev",
+            {
+              apiKey: "fake",
+              hostOverride: "http://localhost:8000",
+            },
+            undefined, // session
+            provider, // headerProvider
+          );
+
+          // Make multiple requests to verify headers are sent each time
+          const tables1 = await conn.tableNames();
+          expect(tables1).toEqual(["test_table"]);
+
+          const tables2 = await conn.tableNames();
+          expect(tables2).toEqual(["test_table"]);
+
+          // Verify headers were sent with each request
+          expect(requestCount).toBeGreaterThanOrEqual(2);
+        },
+      );
+    });
+
+    it("should work with JavaScript function provider", async () => {
+      let requestId = 0;
+
+      await withMockDatabase(
+        (req, res) => {
+          // Check dynamic header is present
+          expect(req.headers["x-request-id"]).toBeDefined();
+          expect(req.headers["x-request-id"]).toMatch(/^req-\d+$/);
+
+          const body = JSON.stringify({ tables: [] });
+          res.writeHead(200, { "Content-Type": "application/json" }).end(body);
+        },
+        async () => {
+          // Create a JavaScript function that returns dynamic headers
+          const getHeaders = async () => {
+            requestId++;
+            return {
+              "X-Request-Id": `req-${requestId}`,
+              "X-Timestamp": new Date().toISOString(),
+            };
+          };
+
+          // Connect with the function directly
+          const conn = await connect(
+            "db://dev",
+            {
+              apiKey: "fake",
+              hostOverride: "http://localhost:8000",
+            },
+            undefined, // session
+            getHeaders, // headerProvider
+          );
+
+          // Make requests - each should have different headers
+          const tables = await conn.tableNames();
+          expect(tables).toEqual([]);
+        },
+      );
+    });
+
+    it("should support OAuth-like token refresh pattern", async () => {
+      let tokenVersion = 0;
+
+      await withMockDatabase(
+        (req, res) => {
+          // Verify authorization header
+          const authHeader = req.headers["authorization"];
+          expect(authHeader).toBeDefined();
+          expect(authHeader).toMatch(/^Bearer token-v\d+$/);
+
+          const body = JSON.stringify({ tables: [] });
+          res.writeHead(200, { "Content-Type": "application/json" }).end(body);
+        },
+        async () => {
+          // Simulate OAuth token fetcher
+          const fetchToken = async () => {
+            tokenVersion++;
+            return {
+              authorization: `Bearer token-v${tokenVersion}`,
+            };
+          };
+
+          // Connect with the function directly
+          const conn = await connect(
+            "db://dev",
+            {
+              apiKey: "fake",
+              hostOverride: "http://localhost:8000",
+            },
+            undefined, // session
+            fetchToken, // headerProvider
+          );
+
+          // Each request will fetch a new token
+          await conn.tableNames();
+
+          // Token should be different on next request
+          await conn.tableNames();
+        },
+      );
+    });
+  });
 });

nodejs/__test__/sanitize.test.ts

@@ -0,0 +1,184 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import * as arrow from "../lancedb/arrow";
+import { sanitizeField, sanitizeType } from "../lancedb/sanitize";
+
+describe("sanitize", function () {
+  describe("sanitizeType function", function () {
+    it("should handle type objects", function () {
+      const type = new arrow.Int32();
+      const result = sanitizeType(type);
+
+      expect(result.typeId).toBe(arrow.Type.Int);
+      expect((result as arrow.Int).bitWidth).toBe(32);
+      expect((result as arrow.Int).isSigned).toBe(true);
+
+      const floatType = {
+        typeId: 3, // Type.Float = 3
+        precision: 2,
+        toString: () => "Float",
+        isFloat: true,
+        isFixedWidth: true,
+      };
+
+      const floatResult = sanitizeType(floatType);
+      expect(floatResult).toBeInstanceOf(arrow.DataType);
+      expect(floatResult.typeId).toBe(arrow.Type.Float);
+
+      const floatResult2 = sanitizeType({ ...floatType, typeId: () => 3 });
+      expect(floatResult2).toBeInstanceOf(arrow.DataType);
+      expect(floatResult2.typeId).toBe(arrow.Type.Float);
+    });
+
+    const allTypeNameTestCases = [
+      ["null", new arrow.Null()],
+      ["binary", new arrow.Binary()],
+      ["utf8", new arrow.Utf8()],
+      ["bool", new arrow.Bool()],
+      ["int8", new arrow.Int8()],
+      ["int16", new arrow.Int16()],
+      ["int32", new arrow.Int32()],
+      ["int64", new arrow.Int64()],
+      ["uint8", new arrow.Uint8()],
+      ["uint16", new arrow.Uint16()],
+      ["uint32", new arrow.Uint32()],
+      ["uint64", new arrow.Uint64()],
+      ["float16", new arrow.Float16()],
+      ["float32", new arrow.Float32()],
+      ["float64", new arrow.Float64()],
+      ["datemillisecond", new arrow.DateMillisecond()],
+      ["dateday", new arrow.DateDay()],
+      ["timenanosecond", new arrow.TimeNanosecond()],
+      ["timemicrosecond", new arrow.TimeMicrosecond()],
+      ["timemillisecond", new arrow.TimeMillisecond()],
+      ["timesecond", new arrow.TimeSecond()],
+      ["intervaldaytime", new arrow.IntervalDayTime()],
+      ["intervalyearmonth", new arrow.IntervalYearMonth()],
+      ["durationnanosecond", new arrow.DurationNanosecond()],
+      ["durationmicrosecond", new arrow.DurationMicrosecond()],
+      ["durationmillisecond", new arrow.DurationMillisecond()],
+      ["durationsecond", new arrow.DurationSecond()],
+    ] as const;
+
+    it.each(allTypeNameTestCases)(
+      'should map type name "%s" to %s',
+      function (name, expected) {
+        const result = sanitizeType(name);
+        expect(result).toBeInstanceOf(expected.constructor);
+      },
+    );
+
+    const caseVariationTestCases = [
+      ["NULL", new arrow.Null()],
+      ["Utf8", new arrow.Utf8()],
+      ["FLOAT32", new arrow.Float32()],
+      ["DaTedAy", new arrow.DateDay()],
+    ] as const;
+
+    it.each(caseVariationTestCases)(
+      'should be case insensitive for type name "%s" mapped to %s',
+      function (name, expected) {
+        const result = sanitizeType(name);
+        expect(result).toBeInstanceOf(expected.constructor);
+      },
+    );
+
+    it("should throw error for unrecognized type name", function () {
+      expect(() => sanitizeType("invalid_type")).toThrow(
+        "Unrecognized type name in schema: invalid_type",
+      );
+    });
+  });
+
+  describe("sanitizeField function", function () {
+    it("should handle field with string type name", function () {
+      const field = sanitizeField({
+        name: "string_field",
+        type: "utf8",
+        nullable: true,
+        metadata: new Map([["key", "value"]]),
+      });
+
+      expect(field).toBeInstanceOf(arrow.Field);
+      expect(field.name).toBe("string_field");
+      expect(field.type).toBeInstanceOf(arrow.Utf8);
+      expect(field.nullable).toBe(true);
+      expect(field.metadata?.get("key")).toBe("value");
+    });
+
+    it("should handle field with type object", function () {
+      const floatType = {
+        typeId: 3, // Float
+        precision: 32,
+      };
+
+      const field = sanitizeField({
+        name: "float_field",
+        type: floatType,
+        nullable: false,
+      });
+
+      expect(field).toBeInstanceOf(arrow.Field);
+      expect(field.name).toBe("float_field");
+      expect(field.type).toBeInstanceOf(arrow.DataType);
+      expect(field.type.typeId).toBe(arrow.Type.Float);
+      expect((field.type as arrow.Float64).precision).toBe(32);
+      expect(field.nullable).toBe(false);
+    });
+
+    it("should handle field with direct Type instance", function () {
+      const field = sanitizeField({
+        name: "bool_field",
+        type: new arrow.Bool(),
+        nullable: true,
+      });
+
+      expect(field).toBeInstanceOf(arrow.Field);
+      expect(field.name).toBe("bool_field");
+      expect(field.type).toBeInstanceOf(arrow.Bool);
+      expect(field.nullable).toBe(true);
+    });
+
+    it("should throw error for invalid field object", function () {
+      expect(() =>
+        sanitizeField({
+          type: "int32",
+          nullable: true,
+        }),
+      ).toThrow(
+        "The field passed in is missing a `type`/`name`/`nullable` property",
+      );
+
+      // Invalid type
+      expect(() =>
+        sanitizeField({
+          name: "invalid",
+          type: { invalid: true },
+          nullable: true,
+        }),
+      ).toThrow("Expected a Type to have a typeId property");
+
+      // Invalid nullable
+      expect(() =>
+        sanitizeField({
+          name: "invalid_nullable",
+          type: "int32",
+          nullable: "not a boolean",
+        }),
+      ).toThrow("The field passed in had a non-boolean `nullable` property");
+    });
+
+    it("should report error for invalid type name", function () {
+      expect(() =>
+        sanitizeField({
+          name: "invalid_field",
+          type: "invalid_type",
+          nullable: true,
+        }),
+      ).toThrow(
+        "Unable to sanitize type for field: invalid_field due to error: Error: Unrecognized type name in schema: invalid_type",
+      );
+    });
+  });
+});

nodejs/__test__/table.test.ts

@@ -10,7 +10,13 @@ import * as arrow16 from "apache-arrow-16";
 import * as arrow17 from "apache-arrow-17";
 import * as arrow18 from "apache-arrow-18";
 
-import { MatchQuery, PhraseQuery, Table, connect } from "../lancedb";
+import {
+  Connection,
+  MatchQuery,
+  PhraseQuery,
+  Table,
+  connect,
+} from "../lancedb";
 import {
   Table as ArrowTable,
   Field,
@@ -21,6 +27,8 @@ import {
   Int64,
   List,
   Schema,
+  SchemaLike,
+  Type,
   Uint8,
   Utf8,
   makeArrowTable,
@@ -39,7 +47,6 @@ import {
   Operator,
   instanceOfFullTextQuery,
 } from "../lancedb/query";
-import exp = require("constants");
 
 describe.each([arrow15, arrow16, arrow17, arrow18])(
   "Given a table",
@@ -212,8 +219,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       },
     );
 
-    // TODO: https://github.com/lancedb/lancedb/issues/1832
-    it.skip("should be able to omit nullable fields", async () => {
+    it("should be able to omit nullable fields", async () => {
       const db = await connect(tmpDir.name);
       const schema = new arrow.Schema([
         new arrow.Field(
@@ -237,23 +243,36 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       await table.add([data3]);
 
       let res = await table.query().limit(10).toArray();
-      const resVector = res.map((r) => r.get("vector").toArray());
+      const resVector = res.map((r) =>
+        r.vector ? Array.from(r.vector) : null,
+      );
       expect(resVector).toEqual([null, data2.vector, data3.vector]);
-      const resItem = res.map((r) => r.get("item").toArray());
+      const resItem = res.map((r) => r.item);
       expect(resItem).toEqual(["foo", null, "bar"]);
-      const resPrice = res.map((r) => r.get("price").toArray());
+      const resPrice = res.map((r) => r.price);
       expect(resPrice).toEqual([10.0, 2.0, 3.0]);
 
       const data4 = { item: "foo" };
       // We can't omit a column if it's not nullable
-      await expect(table.add([data4])).rejects.toThrow("Invalid user input");
+      await expect(table.add([data4])).rejects.toThrow(
+        "Append with different schema",
+      );
 
       // But we can alter columns to make them nullable
       await table.alterColumns([{ path: "price", nullable: true }]);
       await table.add([data4]);
 
-      res = (await table.query().limit(10).toArray()).map((r) => r.toJSON());
-      expect(res).toEqual([data1, data2, data3, data4]);
+      res = (await table.query().limit(10).toArray()).map((r) => ({
+        ...r.toJSON(),
+        vector: r.vector ? Array.from(r.vector) : null,
+      }));
+      // Rust fills missing nullable fields with null
+      expect(res).toEqual([
+        { ...data1, vector: null },
+        { ...data2, item: null },
+        data3,
+        { ...data4, price: null, vector: null },
+      ]);
     });
 
     it("should be able to insert nullable data for non-nullable fields", async () => {
@@ -287,6 +306,12 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(res2[1].id).toEqual(data2.id);
     });
 
+    it("should support take queries", async () => {
+      await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
+      const res = await table.takeOffsets([1, 2]).toArrow();
+      expect(res.getChild("id")?.toJSON()).toEqual([2, 3]);
+    });
+
     it("should return the table as an instance of an arrow table", async () => {
       const arrowTbl = await table.toArrow();
       expect(arrowTbl).toBeInstanceOf(ArrowTable);
@@ -325,6 +350,43 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       const table = await db.createTable("my_table", data);
       expect(await table.countRows()).toEqual(2);
     });
+
+    it("should allow undefined and omitted nullable vector fields", async () => {
+      // Test for the bug: can't pass undefined or omit vector column
+      const db = await connect("memory://");
+      const schema = new arrow.Schema([
+        new arrow.Field("id", new arrow.Int32(), true),
+        new arrow.Field(
+          "vector",
+          new arrow.FixedSizeList(
+            32,
+            new arrow.Field("item", new arrow.Float32(), true),
+          ),
+          true, // nullable = true
+        ),
+      ]);
+      const table = await db.createEmptyTable("test_table", schema);
+
+      // Should not throw error for undefined value
+      await table.add([{ id: 0, vector: undefined }]);
+
+      // Should not throw error for omitted field
+      await table.add([{ id: 1 }]);
+
+      // Should still work for null
+      await table.add([{ id: 2, vector: null }]);
+
+      // Should still work for actual vector
+      const testVector = new Array(32).fill(0.5);
+      await table.add([{ id: 3, vector: testVector }]);
+      expect(await table.countRows()).toEqual(4);
+
+      const res = await table.query().limit(10).toArray();
+      const resVector = res.map((r) =>
+        r.vector ? Array.from(r.vector) : null,
+      );
+      expect(resVector).toEqual([null, null, null, testVector]);
+    });
   },
 );
 
@@ -482,6 +544,32 @@ describe("merge insert", () => {
         .execute(newData, { timeoutMs: 0 }),
     ).rejects.toThrow("merge insert timed out");
   });
+
+  test("useIndex", async () => {
+    const newData = [
+      { a: 2, b: "x" },
+      { a: 4, b: "z" },
+    ];
+
+    // Test with useIndex(true) - should work fine
+    const result1 = await table
+      .mergeInsert("a")
+      .whenNotMatchedInsertAll()
+      .useIndex(true)
+      .execute(newData);
+
+    expect(result1.numInsertedRows).toBe(1); // Only a=4 should be inserted
+
+    // Test with useIndex(false) - should also work fine
+    const newData2 = [{ a: 5, b: "w" }];
+    const result2 = await table
+      .mergeInsert("a")
+      .whenNotMatchedInsertAll()
+      .useIndex(false)
+      .execute(newData2);
+
+    expect(result2.numInsertedRows).toBe(1); // a=5 should be inserted
+  });
 });
 
 describe("When creating an index", () => {
@@ -557,7 +645,7 @@ describe("When creating an index", () => {
 
     // test offset
     rst = await tbl.query().limit(2).offset(1).nearestTo(queryVec).toArrow();
-    expect(rst.numRows).toBe(1);
+    expect(rst.numRows).toBe(2);
 
     // test nprobes
     rst = await tbl.query().nearestTo(queryVec).limit(2).nprobes(50).toArrow();
@@ -696,7 +784,7 @@ describe("When creating an index", () => {
 
     // test offset
     rst = await tbl.query().limit(2).offset(1).nearestTo(queryVec).toArrow();
-    expect(rst.numRows).toBe(1);
+    expect(rst.numRows).toBe(2);
 
     // test ef
     rst = await tbl.query().limit(2).nearestTo(queryVec).ef(100).toArrow();
@@ -851,6 +939,40 @@ describe("When creating an index", () => {
     expect(stats).toBeUndefined();
   });
 
+  test("should support name and train parameters", async () => {
+    // Test with custom name
+    await tbl.createIndex("vec", {
+      config: Index.ivfPq({ numPartitions: 4 }),
+      name: "my_custom_vector_index",
+    });
+
+    const indices = await tbl.listIndices();
+    expect(indices).toHaveLength(1);
+    expect(indices[0].name).toBe("my_custom_vector_index");
+
+    // Test scalar index with train=false
+    await tbl.createIndex("id", {
+      config: Index.btree(),
+      name: "btree_empty",
+      train: false,
+    });
+
+    const allIndices = await tbl.listIndices();
+    expect(allIndices).toHaveLength(2);
+    expect(allIndices.some((idx) => idx.name === "btree_empty")).toBe(true);
+
+    // Test with both name and train=true (use tags column)
+    await tbl.createIndex("tags", {
+      config: Index.labelList(),
+      name: "tags_trained",
+      train: true,
+    });
+
+    const finalIndices = await tbl.listIndices();
+    expect(finalIndices).toHaveLength(3);
+    expect(finalIndices.some((idx) => idx.name === "tags_trained")).toBe(true);
+  });
+
   test("create ivf_flat with binary vectors", async () => {
     const db = await connect(tmpDir.name);
     const binarySchema = new Schema([
@@ -1389,7 +1511,9 @@ describe("when optimizing a dataset", () => {
 
   it("delete unverified", async () => {
     const version = await table.version();
-    const versionFile = `${tmpDir.name}/${table.name}.lance/_versions/${version - 1}.manifest`;
+    const versionFile = `${tmpDir.name}/${table.name}.lance/_versions/${
+      version - 1
+    }.manifest`;
     fs.rmSync(versionFile);
 
     let stats = await table.optimize({ deleteUnverified: false });
@@ -1903,3 +2027,52 @@ describe("column name options", () => {
     expect(results2.length).toBe(10);
   });
 });
+
+describe("when creating an empty table", () => {
+  let con: Connection;
+  beforeEach(async () => {
+    const tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    con = await connect(tmpDir.name);
+  });
+  afterEach(() => {
+    con.close();
+  });
+
+  it("can create an empty table from an arrow Schema", async () => {
+    const schema = new Schema([
+      new Field("id", new Int64()),
+      new Field("vector", new Float64()),
+    ]);
+    const table = await con.createEmptyTable("test", schema);
+    const actualSchema = await table.schema();
+    expect(actualSchema.fields[0].type.typeId).toBe(Type.Int);
+    expect((actualSchema.fields[0].type as Int64).bitWidth).toBe(64);
+    expect(actualSchema.fields[1].type.typeId).toBe(Type.Float);
+    expect((actualSchema.fields[1].type as Float64).precision).toBe(2);
+  });
+
+  it("can create an empty table from schema that specifies field types by name", async () => {
+    const schemaLike = {
+      fields: [
+        {
+          name: "id",
+          type: "int64",
+          nullable: true,
+        },
+        {
+          name: "vector",
+          type: "float64",
+          nullable: true,
+        },
+      ],
+      metadata: new Map(),
+      names: ["id", "vector"],
+    } satisfies SchemaLike;
+    const table = await con.createEmptyTable("test", schemaLike);
+    const actualSchema = await table.schema();
+    expect(actualSchema.fields[0].type.typeId).toBe(Type.Int);
+    expect((actualSchema.fields[0].type as Int64).bitWidth).toBe(64);
+    expect(actualSchema.fields[1].type.typeId).toBe(Type.Float);
+    expect((actualSchema.fields[1].type as Float64).precision).toBe(2);
+  });
+});

nodejs/biome.json

@@ -48,6 +48,7 @@
         "noUnreachableSuper": "error",
         "noUnsafeFinally": "error",
         "noUnsafeOptionalChaining": "error",
+        "noUnusedImports": "error",
         "noUnusedLabels": "error",
         "noUnusedVariables": "warn",
         "useIsNan": "error",

nodejs/examples/ann_indexes.test.ts

@@ -12,7 +12,7 @@ test("ann index examples", async () => {
     // --8<-- [start:ingest]
     const db = await lancedb.connect(databaseDir);
 
-    const data = Array.from({ length: 5_000 }, (_, i) => ({
+    const data = Array.from({ length: 1_000 }, (_, i) => ({
       vector: Array(128).fill(i),
       id: `${i}`,
       content: "",
@@ -24,8 +24,8 @@ test("ann index examples", async () => {
     });
     await table.createIndex("vector", {
       config: lancedb.Index.ivfPq({
-        numPartitions: 10,
-        numSubVectors: 16,
+        numPartitions: 30,
+        numSubVectors: 8,
       }),
     });
     // --8<-- [end:ingest]

nodejs/lancedb/arrow.ts

@@ -41,7 +41,6 @@ import {
   vectorFromArray as badVectorFromArray,
   makeBuilder,
   makeData,
-  makeTable,
 } from "apache-arrow";
 import { Buffers } from "apache-arrow/data";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
@@ -74,7 +73,7 @@ export type FieldLike =
   | {
       type: string;
       name: string;
-      nullable?: boolean;
+      nullable: boolean;
       metadata?: Map<string, string>;
     };
 
@@ -279,7 +278,7 @@ export class MakeArrowTableOptions {
 }
 
 /**
- * An enhanced version of the {@link makeTable} function from Apache Arrow
+ * An enhanced version of the apache-arrow makeTable function from Apache Arrow
  * that supports nested fields and embeddings columns.
  *
  * (typically you do not need to call this function.  It will be called automatically
@@ -512,7 +511,11 @@ function* rowPathsAndValues(
     if (isObject(value)) {
       yield* rowPathsAndValues(value, [...basePath, key]);
     } else {
-      yield [[...basePath, key], value];
+      // Skip undefined values - they should be treated the same as missing fields
+      // for embedding function purposes
+      if (value !== undefined) {
+        yield [[...basePath, key], value];
+      }
     }
   }
 }
@@ -701,7 +704,7 @@ function transposeData(
       }
       return current;
     });
-    return makeVector(values, field.type);
+    return makeVector(values, field.type, undefined, field.nullable);
   }
 }
 
@@ -748,9 +751,30 @@ function makeVector(
   values: unknown[],
   type?: DataType,
   stringAsDictionary?: boolean,
+  nullable?: boolean,
   // biome-ignore lint/suspicious/noExplicitAny: skip
 ): Vector<any> {
   if (type !== undefined) {
+    // Convert undefined values to null for nullable fields
+    if (nullable) {
+      values = values.map((v) => (v === undefined ? null : v));
+    }
+
+    // workaround for: https://github.com/apache/arrow-js/issues/68
+    if (DataType.isBool(type)) {
+      const hasNonNullValue = values.some((v) => v !== null && v !== undefined);
+      if (!hasNonNullValue) {
+        const nullBitmap = new Uint8Array(Math.ceil(values.length / 8));
+        const data = makeData({
+          type: type,
+          length: values.length,
+          nullCount: values.length,
+          nullBitmap,
+        });
+        return arrowMakeVector(data);
+      }
+    }
+
     // No need for inference, let Arrow create it
     if (type instanceof Int) {
       if (DataType.isInt(type) && type.bitWidth === 64) {
@@ -875,7 +899,12 @@ async function applyEmbeddingsFromMetadata(
   for (const field of schema.fields) {
     if (!(field.name in columns)) {
       const nullValues = new Array(table.numRows).fill(null);
-      columns[field.name] = makeVector(nullValues, field.type);
+      columns[field.name] = makeVector(
+        nullValues,
+        field.type,
+        undefined,
+        field.nullable,
+      );
     }
   }
 
@@ -939,7 +968,12 @@ async function applyEmbeddings<T>(
     } else if (schema != null) {
       const destField = schema.fields.find((f) => f.name === destColumn);
       if (destField != null) {
-        newColumns[destColumn] = makeVector([], destField.type);
+        newColumns[destColumn] = makeVector(
+          [],
+          destField.type,
+          undefined,
+          destField.nullable,
+        );
       } else {
         throw new Error(
           `Attempt to apply embeddings to an empty table failed because schema was missing embedding column '${destColumn}'`,
@@ -1251,19 +1285,36 @@ function validateSchemaEmbeddings(
     if (isFixedSizeList(field.type)) {
       field = sanitizeField(field);
       if (data.length !== 0 && data?.[0]?.[field.name] === undefined) {
+        // Check if there's an embedding function registered for this field
+        let hasEmbeddingFunction = false;
+
+        // Check schema metadata for embedding functions
         if (schema.metadata.has("embedding_functions")) {
           const embeddings = JSON.parse(
             schema.metadata.get("embedding_functions")!,
           );
-          if (
-            // biome-ignore lint/suspicious/noExplicitAny: we don't know the type of `f`
-            embeddings.find((f: any) => f["vectorColumn"] === field.name) ===
-            undefined
-          ) {
+          // biome-ignore lint/suspicious/noExplicitAny: we don't know the type of `f`
+          if (embeddings.find((f: any) => f["vectorColumn"] === field.name)) {
+            hasEmbeddingFunction = true;
+          }
+        }
+
+        // Check passed embedding function parameter
+        if (embeddings && embeddings.vectorColumn === field.name) {
+          hasEmbeddingFunction = true;
+        }
+
+        // If the field is nullable AND there's no embedding function, allow undefined/omitted values
+        if (field.nullable && !hasEmbeddingFunction) {
+          fields.push(field);
+        } else {
+          // Either not nullable OR has embedding function - require explicit values
+          if (hasEmbeddingFunction) {
+            // Don't add to missingEmbeddingFields since this is expected to be filled by embedding function
+            fields.push(field);
+          } else {
             missingEmbeddingFields.push(field);
           }
-        } else {
-          missingEmbeddingFields.push(field);
         }
       } else {
         fields.push(field);

nodejs/lancedb/connection.ts

@@ -3,7 +3,6 @@
 
 import {
   Data,
-  Schema,
   SchemaLike,
   TableLike,
   fromTableToStreamBuffer,
@@ -159,17 +158,33 @@ export abstract class Connection {
    *
    * Tables will be returned in lexicographical order.
    * @param {Partial<TableNamesOptions>} options - options to control the
-   * paging / start point
+   * paging / start point (backwards compatibility)
    *
    */
   abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
+  /**
+   * List all the table names in this database.
+   *
+   * Tables will be returned in lexicographical order.
+   * @param {string[]} namespace - The namespace to list tables from (defaults to root namespace)
+   * @param {Partial<TableNamesOptions>} options - options to control the
+   * paging / start point
+   *
+   */
+  abstract tableNames(
+    namespace?: string[],
+    options?: Partial<TableNamesOptions>,
+  ): Promise<string[]>;
 
   /**
    * Open a table in the database.
    * @param {string} name - The name of the table
+   * @param {string[]} namespace - The namespace of the table (defaults to root namespace)
+   * @param {Partial<OpenTableOptions>} options - Additional options
    */
   abstract openTable(
     name: string,
+    namespace?: string[],
     options?: Partial<OpenTableOptions>,
   ): Promise<Table>;
 
@@ -178,6 +193,7 @@ export abstract class Connection {
    * @param {object} options - The options object.
    * @param {string} options.name - The name of the table.
    * @param {Data} options.data - Non-empty Array of Records to be inserted into the table
+   * @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
    *
    */
   abstract createTable(
@@ -185,40 +201,99 @@ export abstract class Connection {
       name: string;
       data: Data;
     } & Partial<CreateTableOptions>,
+    namespace?: string[],
   ): Promise<Table>;
   /**
    * Creates a new Table and initialize it with new data.
    * @param {string} name - The name of the table.
    * @param {Record<string, unknown>[] | TableLike} data - Non-empty Array of Records
    * to be inserted into the table
+   * @param {Partial<CreateTableOptions>} options - Additional options (backwards compatibility)
    */
   abstract createTable(
     name: string,
     data: Record<string, unknown>[] | TableLike,
     options?: Partial<CreateTableOptions>,
   ): Promise<Table>;
+  /**
+   * Creates a new Table and initialize it with new data.
+   * @param {string} name - The name of the table.
+   * @param {Record<string, unknown>[] | TableLike} data - Non-empty Array of Records
+   * to be inserted into the table
+   * @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
+   * @param {Partial<CreateTableOptions>} options - Additional options
+   */
+  abstract createTable(
+    name: string,
+    data: Record<string, unknown>[] | TableLike,
+    namespace?: string[],
+    options?: Partial<CreateTableOptions>,
+  ): Promise<Table>;
 
   /**
    * Creates a new empty Table
    * @param {string} name - The name of the table.
    * @param {Schema} schema - The schema of the table
+   * @param {Partial<CreateTableOptions>} options - Additional options (backwards compatibility)
    */
   abstract createEmptyTable(
     name: string,
     schema: import("./arrow").SchemaLike,
     options?: Partial<CreateTableOptions>,
   ): Promise<Table>;
+  /**
+   * Creates a new empty Table
+   * @param {string} name - The name of the table.
+   * @param {Schema} schema - The schema of the table
+   * @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
+   * @param {Partial<CreateTableOptions>} options - Additional options
+   */
+  abstract createEmptyTable(
+    name: string,
+    schema: import("./arrow").SchemaLike,
+    namespace?: string[],
+    options?: Partial<CreateTableOptions>,
+  ): Promise<Table>;
 
   /**
    * Drop an existing table.
    * @param {string} name The name of the table to drop.
+   * @param {string[]} namespace The namespace of the table (defaults to root namespace).
    */
-  abstract dropTable(name: string): Promise<void>;
+  abstract dropTable(name: string, namespace?: string[]): Promise<void>;
 
   /**
    * Drop all tables in the database.
+   * @param {string[]} namespace The namespace to drop tables from (defaults to root namespace).
    */
-  abstract dropAllTables(): Promise<void>;
+  abstract dropAllTables(namespace?: string[]): Promise<void>;
+
+  /**
+   * Clone a table from a source table.
+   *
+   * A shallow clone creates a new table that shares the underlying data files
+   * with the source table but has its own independent manifest. This allows
+   * both the source and cloned tables to evolve independently while initially
+   * sharing the same data, deletion, and index files.
+   *
+   * @param {string} targetTableName - The name of the target table to create.
+   * @param {string} sourceUri - The URI of the source table to clone from.
+   * @param {object} options - Clone options.
+   * @param {string[]} options.targetNamespace - The namespace for the target table (defaults to root namespace).
+   * @param {number} options.sourceVersion - The version of the source table to clone.
+   * @param {string} options.sourceTag - The tag of the source table to clone.
+   * @param {boolean} options.isShallow - Whether to perform a shallow clone (defaults to true).
+   */
+  abstract cloneTable(
+    targetTableName: string,
+    sourceUri: string,
+    options?: {
+      targetNamespace?: string[];
+      sourceVersion?: number;
+      sourceTag?: string;
+      isShallow?: boolean;
+    },
+  ): Promise<Table>;
 }
 
 /** @hideconstructor */
@@ -243,16 +318,39 @@ export class LocalConnection extends Connection {
     return this.inner.display();
   }
 
-  async tableNames(options?: Partial<TableNamesOptions>): Promise<string[]> {
-    return this.inner.tableNames(options?.startAfter, options?.limit);
+  async tableNames(
+    namespaceOrOptions?: string[] | Partial<TableNamesOptions>,
+    options?: Partial<TableNamesOptions>,
+  ): Promise<string[]> {
+    // Detect if first argument is namespace array or options object
+    let namespace: string[] | undefined;
+    let tableNamesOptions: Partial<TableNamesOptions> | undefined;
+
+    if (Array.isArray(namespaceOrOptions)) {
+      // First argument is namespace array
+      namespace = namespaceOrOptions;
+      tableNamesOptions = options;
+    } else {
+      // First argument is options object (backwards compatibility)
+      namespace = undefined;
+      tableNamesOptions = namespaceOrOptions;
+    }
+
+    return this.inner.tableNames(
+      namespace ?? [],
+      tableNamesOptions?.startAfter,
+      tableNamesOptions?.limit,
+    );
   }
 
   async openTable(
     name: string,
+    namespace?: string[],
     options?: Partial<OpenTableOptions>,
   ): Promise<Table> {
     const innerTable = await this.inner.openTable(
       name,
+      namespace ?? [],
       cleanseStorageOptions(options?.storageOptions),
       options?.indexCacheSize,
     );
@@ -260,6 +358,28 @@ export class LocalConnection extends Connection {
     return new LocalTable(innerTable);
   }
 
+  async cloneTable(
+    targetTableName: string,
+    sourceUri: string,
+    options?: {
+      targetNamespace?: string[];
+      sourceVersion?: number;
+      sourceTag?: string;
+      isShallow?: boolean;
+    },
+  ): Promise<Table> {
+    const innerTable = await this.inner.cloneTable(
+      targetTableName,
+      sourceUri,
+      options?.targetNamespace ?? [],
+      options?.sourceVersion ?? null,
+      options?.sourceTag ?? null,
+      options?.isShallow ?? true,
+    );
+
+    return new LocalTable(innerTable);
+  }
+
   private getStorageOptions(
     options?: Partial<CreateTableOptions>,
   ): Record<string, string> | undefined {
@@ -286,14 +406,44 @@ export class LocalConnection extends Connection {
     nameOrOptions:
       | string
       | ({ name: string; data: Data } & Partial<CreateTableOptions>),
-    data?: Record<string, unknown>[] | TableLike,
+    dataOrNamespace?: Record<string, unknown>[] | TableLike | string[],
+    namespaceOrOptions?: string[] | Partial<CreateTableOptions>,
     options?: Partial<CreateTableOptions>,
   ): Promise<Table> {
     if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
-      const { name, data, ...options } = nameOrOptions;
-
-      return this.createTable(name, data, options);
+      // First overload: createTable(options, namespace?)
+      const { name, data, ...createOptions } = nameOrOptions;
+      const namespace = dataOrNamespace as string[] | undefined;
+      return this._createTableImpl(name, data, namespace, createOptions);
     }
+
+    // Second overload: createTable(name, data, namespace?, options?)
+    const name = nameOrOptions;
+    const data = dataOrNamespace as Record<string, unknown>[] | TableLike;
+
+    // Detect if third argument is namespace array or options object
+    let namespace: string[] | undefined;
+    let createOptions: Partial<CreateTableOptions> | undefined;
+
+    if (Array.isArray(namespaceOrOptions)) {
+      // Third argument is namespace array
+      namespace = namespaceOrOptions;
+      createOptions = options;
+    } else {
+      // Third argument is options object (backwards compatibility)
+      namespace = undefined;
+      createOptions = namespaceOrOptions;
+    }
+
+    return this._createTableImpl(name, data, namespace, createOptions);
+  }
+
+  private async _createTableImpl(
+    name: string,
+    data: Data,
+    namespace?: string[],
+    options?: Partial<CreateTableOptions>,
+  ): Promise<Table> {
     if (data === undefined) {
       throw new Error("data is required");
     }
@@ -302,9 +452,10 @@ export class LocalConnection extends Connection {
     const storageOptions = this.getStorageOptions(options);
 
     const innerTable = await this.inner.createTable(
-      nameOrOptions,
+      name,
       buf,
       mode,
+      namespace ?? [],
       storageOptions,
     );
 
@@ -314,39 +465,55 @@ export class LocalConnection extends Connection {
   async createEmptyTable(
     name: string,
     schema: import("./arrow").SchemaLike,
+    namespaceOrOptions?: string[] | Partial<CreateTableOptions>,
     options?: Partial<CreateTableOptions>,
   ): Promise<Table> {
-    let mode: string = options?.mode ?? "create";
-    const existOk = options?.existOk ?? false;
+    // Detect if third argument is namespace array or options object
+    let namespace: string[] | undefined;
+    let createOptions: Partial<CreateTableOptions> | undefined;
+
+    if (Array.isArray(namespaceOrOptions)) {
+      // Third argument is namespace array
+      namespace = namespaceOrOptions;
+      createOptions = options;
+    } else {
+      // Third argument is options object (backwards compatibility)
+      namespace = undefined;
+      createOptions = namespaceOrOptions;
+    }
+
+    let mode: string = createOptions?.mode ?? "create";
+    const existOk = createOptions?.existOk ?? false;
 
     if (mode === "create" && existOk) {
       mode = "exist_ok";
     }
     let metadata: Map<string, string> | undefined = undefined;
-    if (options?.embeddingFunction !== undefined) {
-      const embeddingFunction = options.embeddingFunction;
+    if (createOptions?.embeddingFunction !== undefined) {
+      const embeddingFunction = createOptions.embeddingFunction;
       const registry = getRegistry();
       metadata = registry.getTableMetadata([embeddingFunction]);
     }
 
-    const storageOptions = this.getStorageOptions(options);
+    const storageOptions = this.getStorageOptions(createOptions);
     const table = makeEmptyTable(schema, metadata);
     const buf = await fromTableToBuffer(table);
     const innerTable = await this.inner.createEmptyTable(
       name,
       buf,
       mode,
+      namespace ?? [],
       storageOptions,
     );
     return new LocalTable(innerTable);
   }
 
-  async dropTable(name: string): Promise<void> {
-    return this.inner.dropTable(name);
+  async dropTable(name: string, namespace?: string[]): Promise<void> {
+    return this.inner.dropTable(name, namespace ?? []);
   }
 
-  async dropAllTables(): Promise<void> {
-    return this.inner.dropAllTables();
+  async dropAllTables(namespace?: string[]): Promise<void> {
+    return this.inner.dropAllTables(namespace ?? []);
   }
 }
 

nodejs/lancedb/header.ts

@@ -0,0 +1,253 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+/**
+ * Header providers for LanceDB remote connections.
+ *
+ * This module provides a flexible header management framework for LanceDB remote
+ * connections, allowing users to implement custom header strategies for
+ * authentication, request tracking, custom metadata, or any other header-based
+ * requirements.
+ *
+ * @module header
+ */
+
+/**
+ * Abstract base class for providing custom headers for each request.
+ *
+ * Users can implement this interface to provide dynamic headers for various purposes
+ * such as authentication (OAuth tokens, API keys), request tracking (correlation IDs),
+ * custom metadata, or any other header-based requirements. The provider is called
+ * before each request to ensure fresh header values are always used.
+ *
+ * @example
+ * Simple JWT token provider:
+ * ```typescript
+ * class JWTProvider extends HeaderProvider {
+ *   constructor(private token: string) {
+ *     super();
+ *   }
+ *
+ *   getHeaders(): Record<string, string> {
+ *     return { authorization: `Bearer ${this.token}` };
+ *   }
+ * }
+ * ```
+ *
+ * @example
+ * Provider with request tracking:
+ * ```typescript
+ * class RequestTrackingProvider extends HeaderProvider {
+ *   constructor(private sessionId: string) {
+ *     super();
+ *   }
+ *
+ *   getHeaders(): Record<string, string> {
+ *     return {
+ *       "X-Session-Id": this.sessionId,
+ *       "X-Request-Id": `req-${Date.now()}`
+ *     };
+ *   }
+ * }
+ * ```
+ */
+export abstract class HeaderProvider {
+  /**
+   * Get the latest headers to be added to requests.
+   *
+   * This method is called before each request to the remote LanceDB server.
+   * Implementations should return headers that will be merged with existing headers.
+   *
+   * @returns Dictionary of header names to values to add to the request.
+   * @throws If unable to fetch headers, the exception will be propagated and the request will fail.
+   */
+  abstract getHeaders(): Record<string, string>;
+}
+
+/**
+ * Example implementation: A simple header provider that returns static headers.
+ *
+ * This is an example implementation showing how to create a HeaderProvider
+ * for cases where headers don't change during the session.
+ *
+ * @example
+ * ```typescript
+ * const provider = new StaticHeaderProvider({
+ *   authorization: "Bearer my-token",
+ *   "X-Custom-Header": "custom-value"
+ * });
+ * const headers = provider.getHeaders();
+ * // Returns: {authorization: 'Bearer my-token', 'X-Custom-Header': 'custom-value'}
+ * ```
+ */
+export class StaticHeaderProvider extends HeaderProvider {
+  private _headers: Record<string, string>;
+
+  /**
+   * Initialize with static headers.
+   * @param headers - Headers to return for every request.
+   */
+  constructor(headers: Record<string, string>) {
+    super();
+    this._headers = { ...headers };
+  }
+
+  /**
+   * Return the static headers.
+   * @returns Copy of the static headers.
+   */
+  getHeaders(): Record<string, string> {
+    return { ...this._headers };
+  }
+}
+
+/**
+ * Token response from OAuth provider.
+ * @public
+ */
+export interface TokenResponse {
+  accessToken: string;
+  expiresIn?: number;
+}
+
+/**
+ * Example implementation: OAuth token provider with automatic refresh.
+ *
+ * This is an example implementation showing how to manage OAuth tokens
+ * with automatic refresh when they expire.
+ *
+ * @example
+ * ```typescript
+ * async function fetchToken(): Promise<TokenResponse> {
+ *   const response = await fetch("https://oauth.example.com/token", {
+ *     method: "POST",
+ *     body: JSON.stringify({
+ *       grant_type: "client_credentials",
+ *       client_id: "your-client-id",
+ *       client_secret: "your-client-secret"
+ *     }),
+ *     headers: { "Content-Type": "application/json" }
+ *   });
+ *   const data = await response.json();
+ *   return {
+ *     accessToken: data.access_token,
+ *     expiresIn: data.expires_in
+ *   };
+ * }
+ *
+ * const provider = new OAuthHeaderProvider(fetchToken);
+ * const headers = provider.getHeaders();
+ * // Returns: {"authorization": "Bearer <your-token>"}
+ * ```
+ */
+export class OAuthHeaderProvider extends HeaderProvider {
+  private _tokenFetcher: () => Promise<TokenResponse> | TokenResponse;
+  private _refreshBufferSeconds: number;
+  private _currentToken: string | null = null;
+  private _tokenExpiresAt: number | null = null;
+  private _refreshPromise: Promise<void> | null = null;
+
+  /**
+   * Initialize the OAuth provider.
+   * @param tokenFetcher - Function to fetch new tokens. Should return object with 'accessToken' and optionally 'expiresIn'.
+   * @param refreshBufferSeconds - Seconds before expiry to refresh token. Default 300 (5 minutes).
+   */
+  constructor(
+    tokenFetcher: () => Promise<TokenResponse> | TokenResponse,
+    refreshBufferSeconds: number = 300,
+  ) {
+    super();
+    this._tokenFetcher = tokenFetcher;
+    this._refreshBufferSeconds = refreshBufferSeconds;
+  }
+
+  /**
+   * Check if token needs refresh.
+   */
+  private _needsRefresh(): boolean {
+    if (this._currentToken === null) {
+      return true;
+    }
+
+    if (this._tokenExpiresAt === null) {
+      // No expiration info, assume token is valid
+      return false;
+    }
+
+    // Refresh if we're within the buffer time of expiration
+    const now = Date.now() / 1000;
+    return now >= this._tokenExpiresAt - this._refreshBufferSeconds;
+  }
+
+  /**
+   * Refresh the token if it's expired or close to expiring.
+   */
+  private async _refreshTokenIfNeeded(): Promise<void> {
+    if (!this._needsRefresh()) {
+      return;
+    }
+
+    // If refresh is already in progress, wait for it
+    if (this._refreshPromise) {
+      await this._refreshPromise;
+      return;
+    }
+
+    // Start refresh
+    this._refreshPromise = (async () => {
+      try {
+        const tokenData = await this._tokenFetcher();
+
+        this._currentToken = tokenData.accessToken;
+        if (!this._currentToken) {
+          throw new Error("Token fetcher did not return 'accessToken'");
+        }
+
+        // Set expiration if provided
+        if (tokenData.expiresIn) {
+          this._tokenExpiresAt = Date.now() / 1000 + tokenData.expiresIn;
+        } else {
+          // Token doesn't expire or expiration unknown
+          this._tokenExpiresAt = null;
+        }
+      } finally {
+        this._refreshPromise = null;
+      }
+    })();
+
+    await this._refreshPromise;
+  }
+
+  /**
+   * Get OAuth headers, refreshing token if needed.
+   * Note: This is synchronous for now as the Rust implementation expects sync.
+   * In a real implementation, this would need to handle async properly.
+   * @returns Headers with Bearer token authorization.
+   * @throws If unable to fetch or refresh token.
+   */
+  getHeaders(): Record<string, string> {
+    // For simplicity in this example, we assume the token is already fetched
+    // In a real implementation, this would need to handle the async nature properly
+    if (!this._currentToken && !this._refreshPromise) {
+      // Synchronously trigger refresh - this is a limitation of the current implementation
+      throw new Error(
+        "Token not initialized. Call refreshToken() first or use async initialization.",
+      );
+    }
+
+    if (!this._currentToken) {
+      throw new Error("Failed to obtain OAuth token");
+    }
+
+    return { authorization: `Bearer ${this._currentToken}` };
+  }
+
+  /**
+   * Manually refresh the token.
+   * Call this before using getHeaders() to ensure token is available.
+   */
+  async refreshToken(): Promise<void> {
+    this._currentToken = null; // Force refresh
+    await this._refreshTokenIfNeeded();
+  }
+}

nodejs/lancedb/index.ts

@@ -10,9 +10,15 @@ import {
 import {
   ConnectionOptions,
   Connection as LanceDbConnection,
+  JsHeaderProvider as NativeJsHeaderProvider,
   Session,
 } from "./native.js";
 
+import { HeaderProvider } from "./header";
+
+// Re-export native header provider for use with connectWithHeaderProvider
+export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
+
 export {
   AddColumnsSql,
   ConnectionOptions,
@@ -21,6 +27,7 @@ export {
   ClientConfig,
   TimeoutConfig,
   RetryConfig,
+  TlsConfig,
   OptimizeStats,
   CompactionStats,
   RemovalStats,
@@ -59,6 +66,7 @@ export {
   Query,
   QueryBase,
   VectorQuery,
+  TakeQuery,
   QueryExecutionOptions,
   FullTextSearchOptions,
   RecordBatchIterator,
@@ -92,6 +100,13 @@ export {
   ColumnAlteration,
 } from "./table";
 
+export {
+  HeaderProvider,
+  StaticHeaderProvider,
+  OAuthHeaderProvider,
+  TokenResponse,
+} from "./header";
+
 export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
 
 export * as embedding from "./embedding";
@@ -130,11 +145,27 @@ export { IntoSql, packBits } from "./util";
  *   {storageOptions: {timeout: "60s"}
  * });
  * ```
+ * @example
+ * Using with a header provider for per-request authentication:
+ * ```ts
+ * const provider = new StaticHeaderProvider({
+ *   "X-API-Key": "my-key"
+ * });
+ * const conn = await connectWithHeaderProvider(
+ *   "db://host:port",
+ *   options,
+ *   provider
+ * );
+ * ```
  */
 export async function connect(
   uri: string,
   options?: Partial<ConnectionOptions>,
   session?: Session,
+  headerProvider?:
+    | HeaderProvider
+    | (() => Record<string, string>)
+    | (() => Promise<Record<string, string>>),
 ): Promise<Connection>;
 /**
  * Connect to a LanceDB instance at the given URI.
@@ -168,18 +199,58 @@ export async function connect(
 ): Promise<Connection>;
 export async function connect(
   uriOrOptions: string | (Partial<ConnectionOptions> & { uri: string }),
-  options?: Partial<ConnectionOptions>,
+  optionsOrSession?: Partial<ConnectionOptions> | Session,
+  sessionOrHeaderProvider?:
+    | Session
+    | HeaderProvider
+    | (() => Record<string, string>)
+    | (() => Promise<Record<string, string>>),
+  headerProvider?:
+    | HeaderProvider
+    | (() => Record<string, string>)
+    | (() => Promise<Record<string, string>>),
 ): Promise<Connection> {
   let uri: string | undefined;
   let finalOptions: Partial<ConnectionOptions> = {};
+  let finalHeaderProvider:
+    | HeaderProvider
+    | (() => Record<string, string>)
+    | (() => Promise<Record<string, string>>)
+    | undefined;
 
   if (typeof uriOrOptions !== "string") {
+    // First overload: connect(options)
     const { uri: uri_, ...opts } = uriOrOptions;
     uri = uri_;
     finalOptions = opts;
   } else {
+    // Second overload: connect(uri, options?, session?, headerProvider?)
     uri = uriOrOptions;
-    finalOptions = options || {};
+
+    // Handle optionsOrSession parameter
+    if (optionsOrSession && "inner" in optionsOrSession) {
+      // Second param is session, so no options provided
+      finalOptions = {};
+    } else {
+      // Second param is options
+      finalOptions = (optionsOrSession as Partial<ConnectionOptions>) || {};
+    }
+
+    // Handle sessionOrHeaderProvider parameter
+    if (
+      sessionOrHeaderProvider &&
+      (typeof sessionOrHeaderProvider === "function" ||
+        "getHeaders" in sessionOrHeaderProvider)
+    ) {
+      // Third param is header provider
+      finalHeaderProvider = sessionOrHeaderProvider as
+        | HeaderProvider
+        | (() => Record<string, string>)
+        | (() => Promise<Record<string, string>>);
+    } else {
+      // Third param is session, header provider is fourth param
+      finalHeaderProvider = headerProvider;
+    }
   }
 
   if (!uri) {
@@ -190,6 +261,26 @@ export async function connect(
   (<ConnectionOptions>finalOptions).storageOptions = cleanseStorageOptions(
     (<ConnectionOptions>finalOptions).storageOptions,
   );
-  const nativeConn = await LanceDbConnection.new(uri, finalOptions);
+
+  // Create native header provider if one was provided
+  let nativeProvider: NativeJsHeaderProvider | undefined;
+  if (finalHeaderProvider) {
+    if (typeof finalHeaderProvider === "function") {
+      nativeProvider = new NativeJsHeaderProvider(finalHeaderProvider);
+    } else if (
+      finalHeaderProvider &&
+      typeof finalHeaderProvider.getHeaders === "function"
+    ) {
+      nativeProvider = new NativeJsHeaderProvider(async () =>
+        finalHeaderProvider.getHeaders(),
+      );
+    }
+  }
+
+  const nativeConn = await LanceDbConnection.new(
+    uri,
+    finalOptions,
+    nativeProvider,
+  );
   return new LocalConnection(nativeConn);
 }

nodejs/lancedb/indices.ts

@@ -700,5 +700,27 @@ export interface IndexOptions {
    */
   replace?: boolean;
 
+  /**
+   * Timeout in seconds to wait for index creation to complete.
+   *
+   * If not specified, the method will return immediately after starting the index creation.
+   */
   waitTimeoutSeconds?: number;
+
+  /**
+   * Optional custom name for the index.
+   *
+   * If not provided, a default name will be generated based on the column name.
+   */
+  name?: string;
+
+  /**
+   * Whether to train the index with existing data.
+   *
+   * If true (default), the index will be trained with existing data in the table.
+   * If false, the index will be created empty and populated as new data is added.
+   *
+   * Note: This option is only supported for scalar indices. Vector indices always train.
+   */
+  train?: boolean;
 }

nodejs/lancedb/merge.ts

@@ -70,6 +70,23 @@ export class MergeInsertBuilder {
       this.#schema,
     );
   }
+
+  /**
+   * Controls whether to use indexes for the merge operation.
+   *
+   * When set to `true` (the default), the operation will use an index if available
+   * on the join key for improved performance. When set to `false`, it forces a full
+   * table scan even if an index exists. This can be useful for benchmarking or when
+   * the query optimizer chooses a suboptimal path.
+   *
+   * @param useIndex - Whether to use indices for the merge operation. Defaults to `true`.
+   */
+  useIndex(useIndex: boolean): MergeInsertBuilder {
+    return new MergeInsertBuilder(
+      this.#native.useIndex(useIndex),
+      this.#schema,
+    );
+  }
   /**
    * Executes the merge insert operation
    *

nodejs/lancedb/query.ts

@@ -15,6 +15,7 @@ import {
   RecordBatchIterator as NativeBatchIterator,
   Query as NativeQuery,
   Table as NativeTable,
+  TakeQuery as NativeTakeQuery,
   VectorQuery as NativeVectorQuery,
 } from "./native";
 import { Reranker } from "./rerankers";
@@ -50,7 +51,7 @@ export class RecordBatchIterator implements AsyncIterator<RecordBatch> {
 /* eslint-enable */
 
 class RecordBatchIterable<
-  NativeQueryType extends NativeQuery | NativeVectorQuery,
+  NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
 > implements AsyncIterable<RecordBatch>
 {
   private inner: NativeQueryType;
@@ -107,8 +108,9 @@ export interface FullTextSearchOptions {
  *
  * @hideconstructor
  */
-export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
-  implements AsyncIterable<RecordBatch>
+export class QueryBase<
+  NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
+> implements AsyncIterable<RecordBatch>
 {
   /**
    * @hidden
@@ -133,56 +135,6 @@ export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
       fn(this.inner);
     }
   }
-  /**
-   * A filter statement to be applied to this query.
-   *
-   * The filter should be supplied as an SQL query string.  For example:
-   * @example
-   * x > 10
-   * y > 0 AND y < 100
-   * x > 5 OR y = 'test'
-   *
-   * Filtering performance can often be improved by creating a scalar index
-   * on the filter column(s).
-   */
-  where(predicate: string): this {
-    this.doCall((inner: NativeQueryType) => inner.onlyIf(predicate));
-    return this;
-  }
-  /**
-   * A filter statement to be applied to this query.
-   * @see where
-   * @deprecated Use `where` instead
-   */
-  filter(predicate: string): this {
-    return this.where(predicate);
-  }
-
-  fullTextSearch(
-    query: string | FullTextQuery,
-    options?: Partial<FullTextSearchOptions>,
-  ): this {
-    let columns: string[] | null = null;
-    if (options) {
-      if (typeof options.columns === "string") {
-        columns = [options.columns];
-      } else if (Array.isArray(options.columns)) {
-        columns = options.columns;
-      }
-    }
-
-    this.doCall((inner: NativeQueryType) => {
-      if (typeof query === "string") {
-        inner.fullTextSearch({
-          query: query,
-          columns: columns,
-        });
-      } else {
-        inner.fullTextSearch({ query: query.inner });
-      }
-    });
-    return this;
-  }
 
   /**
    * Return only the specified columns.
@@ -241,33 +193,6 @@ export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
     return this;
   }
 
-  /**
-   * Set the maximum number of results to return.
-   *
-   * By default, a plain search has no limit.  If this method is not
-   * called then every valid row from the table will be returned.
-   */
-  limit(limit: number): this {
-    this.doCall((inner: NativeQueryType) => inner.limit(limit));
-    return this;
-  }
-
-  offset(offset: number): this {
-    this.doCall((inner: NativeQueryType) => inner.offset(offset));
-    return this;
-  }
-
-  /**
-   * Skip searching un-indexed data. This can make search faster, but will miss
-   * any data that is not yet indexed.
-   *
-   * Use {@link Table#optimize} to index all un-indexed data.
-   */
-  fastSearch(): this {
-    this.doCall((inner: NativeQueryType) => inner.fastSearch());
-    return this;
-  }
-
   /**
    * Whether to return the row id in the results.
    *
@@ -403,6 +328,100 @@ export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
   }
 }
 
+export class StandardQueryBase<
+    NativeQueryType extends NativeQuery | NativeVectorQuery,
+  >
+  extends QueryBase<NativeQueryType>
+  implements ExecutableQuery
+{
+  constructor(inner: NativeQueryType | Promise<NativeQueryType>) {
+    super(inner);
+  }
+
+  /**
+   * A filter statement to be applied to this query.
+   *
+   * The filter should be supplied as an SQL query string.  For example:
+   * @example
+   * x > 10
+   * y > 0 AND y < 100
+   * x > 5 OR y = 'test'
+   *
+   * Filtering performance can often be improved by creating a scalar index
+   * on the filter column(s).
+   */
+  where(predicate: string): this {
+    this.doCall((inner: NativeQueryType) => inner.onlyIf(predicate));
+    return this;
+  }
+  /**
+   * A filter statement to be applied to this query.
+   * @see where
+   * @deprecated Use `where` instead
+   */
+  filter(predicate: string): this {
+    return this.where(predicate);
+  }
+
+  fullTextSearch(
+    query: string | FullTextQuery,
+    options?: Partial<FullTextSearchOptions>,
+  ): this {
+    let columns: string[] | null = null;
+    if (options) {
+      if (typeof options.columns === "string") {
+        columns = [options.columns];
+      } else if (Array.isArray(options.columns)) {
+        columns = options.columns;
+      }
+    }
+
+    this.doCall((inner: NativeQueryType) => {
+      if (typeof query === "string") {
+        inner.fullTextSearch({
+          query: query,
+          columns: columns,
+        });
+      } else {
+        inner.fullTextSearch({ query: query.inner });
+      }
+    });
+    return this;
+  }
+
+  /**
+   * Set the maximum number of results to return.
+   *
+   * By default, a plain search has no limit.  If this method is not
+   * called then every valid row from the table will be returned.
+   */
+  limit(limit: number): this {
+    this.doCall((inner: NativeQueryType) => inner.limit(limit));
+    return this;
+  }
+
+  /**
+   * Set the number of rows to skip before returning results.
+   *
+   * This is useful for pagination.
+   */
+  offset(offset: number): this {
+    this.doCall((inner: NativeQueryType) => inner.offset(offset));
+    return this;
+  }
+
+  /**
+   * Skip searching un-indexed data. This can make search faster, but will miss
+   * any data that is not yet indexed.
+   *
+   * Use {@link Table#optimize} to index all un-indexed data.
+   */
+  fastSearch(): this {
+    this.doCall((inner: NativeQueryType) => inner.fastSearch());
+    return this;
+  }
+}
+
 /**
  * An interface for a query that can be executed
  *
@@ -419,7 +438,7 @@ export interface ExecutableQuery {}
  *
  * @hideconstructor
  */
-export class VectorQuery extends QueryBase<NativeVectorQuery> {
+export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   /**
    * @hidden
    */
@@ -679,13 +698,24 @@ export class VectorQuery extends QueryBase<NativeVectorQuery> {
   }
 }
 
+/**
+ * A query that returns a subset of the rows in the table.
+ *
+ * @hideconstructor
+ */
+export class TakeQuery extends QueryBase<NativeTakeQuery> {
+  constructor(inner: NativeTakeQuery) {
+    super(inner);
+  }
+}
+
 /** A builder for LanceDB queries.
  *
  * @see {@link Table#query}, {@link Table#search}
  *
  * @hideconstructor
  */
-export class Query extends QueryBase<NativeQuery> {
+export class Query extends StandardQueryBase<NativeQuery> {
   /**
    * @hidden
    */

nodejs/lancedb/sanitize.ts

@@ -326,6 +326,9 @@ export function sanitizeDictionary(typeLike: object) {
 
 // biome-ignore lint/suspicious/noExplicitAny: skip
 export function sanitizeType(typeLike: unknown): DataType<any> {
+  if (typeof typeLike === "string") {
+    return dataTypeFromName(typeLike);
+  }
   if (typeof typeLike !== "object" || typeLike === null) {
     throw Error("Expected a Type but object was null/undefined");
   }
@@ -447,7 +450,7 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
     case Type.DurationSecond:
       return new DurationSecond();
     default:
-      throw new Error("Unrecoginized type id in schema: " + typeId);
+      throw new Error("Unrecognized type id in schema: " + typeId);
   }
 }
 
@@ -467,7 +470,15 @@ export function sanitizeField(fieldLike: unknown): Field {
       "The field passed in is missing a `type`/`name`/`nullable` property",
     );
   }
-  const type = sanitizeType(fieldLike.type);
+  let type: DataType;
+  try {
+    type = sanitizeType(fieldLike.type);
+  } catch (error: unknown) {
+    throw Error(
+      `Unable to sanitize type for field: ${fieldLike.name} due to error: ${error}`,
+      { cause: error },
+    );
+  }
   const name = fieldLike.name;
   if (!(typeof name === "string")) {
     throw Error("The field passed in had a non-string `name` property");
@@ -581,3 +592,46 @@ function sanitizeData(
     },
   );
 }
+
+const constructorsByTypeName = {
+  null: () => new Null(),
+  binary: () => new Binary(),
+  utf8: () => new Utf8(),
+  bool: () => new Bool(),
+  int8: () => new Int8(),
+  int16: () => new Int16(),
+  int32: () => new Int32(),
+  int64: () => new Int64(),
+  uint8: () => new Uint8(),
+  uint16: () => new Uint16(),
+  uint32: () => new Uint32(),
+  uint64: () => new Uint64(),
+  float16: () => new Float16(),
+  float32: () => new Float32(),
+  float64: () => new Float64(),
+  datemillisecond: () => new DateMillisecond(),
+  dateday: () => new DateDay(),
+  timenanosecond: () => new TimeNanosecond(),
+  timemicrosecond: () => new TimeMicrosecond(),
+  timemillisecond: () => new TimeMillisecond(),
+  timesecond: () => new TimeSecond(),
+  intervaldaytime: () => new IntervalDayTime(),
+  intervalyearmonth: () => new IntervalYearMonth(),
+  durationnanosecond: () => new DurationNanosecond(),
+  durationmicrosecond: () => new DurationMicrosecond(),
+  durationmillisecond: () => new DurationMillisecond(),
+  durationsecond: () => new DurationSecond(),
+} as const;
+
+type MappableTypeName = keyof typeof constructorsByTypeName;
+
+export function dataTypeFromName(typeName: string): DataType {
+  const normalizedTypeName = typeName.toLowerCase() as MappableTypeName;
+  const _constructor = constructorsByTypeName[normalizedTypeName];
+
+  if (!_constructor) {
+    throw new Error("Unrecognized type name in schema: " + typeName);
+  }
+
+  return _constructor();
+}

nodejs/lancedb/table.ts

@@ -35,6 +35,7 @@ import {
 import {
   FullTextQuery,
   Query,
+  TakeQuery,
   VectorQuery,
   instanceOfFullTextQuery,
 } from "./query";
@@ -336,6 +337,20 @@ export abstract class Table {
    */
   abstract query(): Query;
 
+  /**
+   * Create a query that returns a subset of the rows in the table.
+   * @param offsets The offsets of the rows to return.
+   * @returns A builder that can be used to parameterize the query.
+   */
+  abstract takeOffsets(offsets: number[]): TakeQuery;
+
+  /**
+   * Create a query that returns a subset of the rows in the table.
+   * @param rowIds The row ids of the rows to return.
+   * @returns A builder that can be used to parameterize the query.
+   */
+  abstract takeRowIds(rowIds: number[]): TakeQuery;
+
   /**
    * Create a search query to find the nearest neighbors
    * of the given query
@@ -647,6 +662,8 @@ export class LocalTable extends Table {
       column,
       options?.replace,
       options?.waitTimeoutSeconds,
+      options?.name,
+      options?.train,
     );
   }
 
@@ -665,6 +682,14 @@ export class LocalTable extends Table {
     await this.inner.waitForIndex(indexNames, timeoutSeconds);
   }
 
+  takeOffsets(offsets: number[]): TakeQuery {
+    return new TakeQuery(this.inner.takeOffsets(offsets));
+  }
+
+  takeRowIds(rowIds: number[]): TakeQuery {
+    return new TakeQuery(this.inner.takeRowIds(rowIds));
+  }
+
   query(): Query {
     return new Query(this.inner);
   }

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.21.2",
+	"version": "0.22.2-beta.2",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/darwin-x64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.21.2",
+	"version": "0.22.2-beta.2",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.21.2",
+	"version": "0.22.2-beta.2",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.21.2",
+	"version": "0.22.2-beta.2",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.21.2",
+	"version": "0.22.2-beta.2",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.21.2",
+	"version": "0.22.2-beta.2",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.21.2",
+  "version": "0.22.2-beta.2",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.21.2",
+	"version": "0.22.2-beta.2",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.21.2",
+  "version": "0.22.2-beta.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.21.2",
+      "version": "0.22.2-beta.2",
       "cpu": [
         "x64",
         "arm64"
@@ -5549,10 +5549,11 @@
       "dev": true
     },
     "node_modules/brace-expansion": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
-      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "version": "1.1.12",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
+      "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0",
         "concat-map": "0.0.1"
@@ -5629,6 +5630,20 @@
       "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
       "dev": true
     },
+    "node_modules/call-bind-apply-helpers": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
+      "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
+      "devOptional": true,
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "function-bind": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/camelcase": {
       "version": "5.3.1",
       "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
@@ -6032,6 +6047,21 @@
         "node": ">=6.0.0"
       }
     },
+    "node_modules/dunder-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
+      "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
+      "devOptional": true,
+      "license": "MIT",
+      "dependencies": {
+        "call-bind-apply-helpers": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "gopd": "^1.2.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/eastasianwidth": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
@@ -6071,6 +6101,55 @@
         "is-arrayish": "^0.2.1"
       }
     },
+    "node_modules/es-define-property": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
+      "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
+      "devOptional": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-errors": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+      "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+      "devOptional": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-object-atoms": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
+      "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
+      "devOptional": true,
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-set-tostringtag": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
+      "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
+      "devOptional": true,
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.6",
+        "has-tostringtag": "^1.0.2",
+        "hasown": "^2.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/escalade": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
@@ -6510,13 +6589,16 @@
       }
     },
     "node_modules/form-data": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
-      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
+      "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
       "devOptional": true,
+      "license": "MIT",
       "dependencies": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
+        "es-set-tostringtag": "^2.1.0",
+        "hasown": "^2.0.2",
         "mime-types": "^2.1.12"
       },
       "engines": {
@@ -6575,7 +6657,7 @@
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
       "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
-      "dev": true,
+      "devOptional": true,
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
@@ -6598,6 +6680,31 @@
         "node": "6.* || 8.* || >= 10.*"
       }
     },
+    "node_modules/get-intrinsic": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
+      "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
+      "devOptional": true,
+      "license": "MIT",
+      "dependencies": {
+        "call-bind-apply-helpers": "^1.0.2",
+        "es-define-property": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "es-object-atoms": "^1.1.1",
+        "function-bind": "^1.1.2",
+        "get-proto": "^1.0.1",
+        "gopd": "^1.2.0",
+        "has-symbols": "^1.1.0",
+        "hasown": "^2.0.2",
+        "math-intrinsics": "^1.1.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/get-package-type": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz",
@@ -6607,6 +6714,20 @@
         "node": ">=8.0.0"
       }
     },
+    "node_modules/get-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
+      "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
+      "devOptional": true,
+      "license": "MIT",
+      "dependencies": {
+        "dunder-proto": "^1.0.1",
+        "es-object-atoms": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/get-stream": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
@@ -6698,6 +6819,19 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/gopd": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
+      "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
+      "devOptional": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/graceful-fs": {
       "version": "4.2.11",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
@@ -6724,11 +6858,41 @@
         "node": ">=8"
       }
     },
+    "node_modules/has-symbols": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
+      "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
+      "devOptional": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/has-tostringtag": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
+      "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
+      "devOptional": true,
+      "license": "MIT",
+      "dependencies": {
+        "has-symbols": "^1.0.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/hasown": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz",
-      "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==",
-      "dev": true,
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
+      "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
+      "devOptional": true,
+      "license": "MIT",
       "dependencies": {
         "function-bind": "^1.1.2"
       },
@@ -7943,6 +8107,16 @@
       "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
       "dev": true
     },
+    "node_modules/math-intrinsics": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
+      "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
+      "devOptional": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/md5": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
@@ -8053,9 +8227,10 @@
       }
     },
     "node_modules/minizlib/node_modules/brace-expansion": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
+      "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
+      "license": "MIT",
       "optional": true,
       "dependencies": {
         "balanced-match": "^1.0.0"
@@ -9201,10 +9376,11 @@
       "dev": true
     },
     "node_modules/tmp": {
-      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz",
-      "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==",
+      "version": "0.2.5",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz",
+      "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==",
       "dev": true,
+      "license": "MIT",
       "engines": {
         "node": ">=14.14"
       }
@@ -9349,10 +9525,11 @@
       }
     },
     "node_modules/typedoc/node_modules/brace-expansion": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
+      "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0"
       }
@@ -9602,10 +9779,11 @@
       }
     },
     "node_modules/typescript-eslint/node_modules/brace-expansion": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
+      "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0"
       }

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.21.2",
+  "version": "0.22.2-beta.2",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",

nodejs/src/connection.rs

@@ -2,12 +2,14 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 use std::collections::HashMap;
+use std::sync::Arc;
 
 use lancedb::database::CreateTableMode;
 use napi::bindgen_prelude::*;
 use napi_derive::*;
 
 use crate::error::NapiErrorExt;
+use crate::header::JsHeaderProvider;
 use crate::table::Table;
 use crate::ConnectionOptions;
 use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection};
@@ -45,7 +47,11 @@ impl Connection {
 impl Connection {
     /// Create a new Connection instance from the given URI.
     #[napi(factory)]
-    pub async fn new(uri: String, options: ConnectionOptions) -> napi::Result<Self> {
+    pub async fn new(
+        uri: String,
+        options: ConnectionOptions,
+        header_provider: Option<&JsHeaderProvider>,
+    ) -> napi::Result<Self> {
         let mut builder = ConnectBuilder::new(&uri);
         if let Some(interval) = options.read_consistency_interval {
             builder =
@@ -57,8 +63,16 @@ impl Connection {
             }
         }
 
+        // Create client config, optionally with header provider
         let client_config = options.client_config.unwrap_or_default();
-        builder = builder.client_config(client_config.into());
+        let mut rust_config: lancedb::remote::ClientConfig = client_config.into();
+
+        if let Some(provider) = header_provider {
+            rust_config.header_provider =
+                Some(Arc::new(provider.clone()) as Arc<dyn lancedb::remote::HeaderProvider>);
+        }
+
+        builder = builder.client_config(rust_config);
 
         if let Some(api_key) = options.api_key {
             builder = builder.api_key(&api_key);
@@ -100,10 +114,12 @@ impl Connection {
     #[napi(catch_unwind)]
     pub async fn table_names(
         &self,
+        namespace: Vec<String>,
         start_after: Option<String>,
         limit: Option<u32>,
     ) -> napi::Result<Vec<String>> {
         let mut op = self.get_inner()?.table_names();
+        op = op.namespace(namespace);
         if let Some(start_after) = start_after {
             op = op.start_after(start_after);
         }
@@ -125,6 +141,7 @@ impl Connection {
         name: String,
         buf: Buffer,
         mode: String,
+        namespace: Vec<String>,
         storage_options: Option<HashMap<String, String>>,
     ) -> napi::Result<Table> {
         let batches = ipc_file_to_batches(buf.to_vec())
@@ -132,6 +149,8 @@ impl Connection {
         let mode = Self::parse_create_mode_str(&mode)?;
         let mut builder = self.get_inner()?.create_table(&name, batches).mode(mode);
 
+        builder = builder.namespace(namespace);
+
         if let Some(storage_options) = storage_options {
             for (key, value) in storage_options {
                 builder = builder.storage_option(key, value);
@@ -147,6 +166,7 @@ impl Connection {
         name: String,
         schema_buf: Buffer,
         mode: String,
+        namespace: Vec<String>,
         storage_options: Option<HashMap<String, String>>,
     ) -> napi::Result<Table> {
         let schema = ipc_file_to_schema(schema_buf.to_vec()).map_err(|e| {
@@ -157,6 +177,9 @@ impl Connection {
             .get_inner()?
             .create_empty_table(&name, schema)
             .mode(mode);
+
+        builder = builder.namespace(namespace);
+
         if let Some(storage_options) = storage_options {
             for (key, value) in storage_options {
                 builder = builder.storage_option(key, value);
@@ -170,10 +193,14 @@ impl Connection {
     pub async fn open_table(
         &self,
         name: String,
+        namespace: Vec<String>,
         storage_options: Option<HashMap<String, String>>,
         index_cache_size: Option<u32>,
     ) -> napi::Result<Table> {
         let mut builder = self.get_inner()?.open_table(&name);
+
+        builder = builder.namespace(namespace);
+
         if let Some(storage_options) = storage_options {
             for (key, value) in storage_options {
                 builder = builder.storage_option(key, value);
@@ -186,14 +213,50 @@ impl Connection {
         Ok(Table::new(tbl))
     }
 
+    #[napi(catch_unwind)]
+    pub async fn clone_table(
+        &self,
+        target_table_name: String,
+        source_uri: String,
+        target_namespace: Vec<String>,
+        source_version: Option<i64>,
+        source_tag: Option<String>,
+        is_shallow: bool,
+    ) -> napi::Result<Table> {
+        let mut builder = self
+            .get_inner()?
+            .clone_table(&target_table_name, &source_uri);
+
+        builder = builder.target_namespace(target_namespace);
+
+        if let Some(version) = source_version {
+            builder = builder.source_version(version as u64);
+        }
+
+        if let Some(tag) = source_tag {
+            builder = builder.source_tag(tag);
+        }
+
+        builder = builder.is_shallow(is_shallow);
+
+        let tbl = builder.execute().await.default_error()?;
+        Ok(Table::new(tbl))
+    }
+
     /// Drop table with the name. Or raise an error if the table does not exist.
     #[napi(catch_unwind)]
-    pub async fn drop_table(&self, name: String) -> napi::Result<()> {
-        self.get_inner()?.drop_table(&name).await.default_error()
+    pub async fn drop_table(&self, name: String, namespace: Vec<String>) -> napi::Result<()> {
+        self.get_inner()?
+            .drop_table(&name, &namespace)
+            .await
+            .default_error()
     }
 
     #[napi(catch_unwind)]
-    pub async fn drop_all_tables(&self) -> napi::Result<()> {
-        self.get_inner()?.drop_all_tables().await.default_error()
+    pub async fn drop_all_tables(&self, namespace: Vec<String>) -> napi::Result<()> {
+        self.get_inner()?
+            .drop_all_tables(&namespace)
+            .await
+            .default_error()
     }
 }

nodejs/src/header.rs

@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use napi::{
+    bindgen_prelude::*,
+    threadsafe_function::{ErrorStrategy, ThreadsafeFunction},
+};
+use napi_derive::napi;
+use std::collections::HashMap;
+use std::sync::Arc;
+
+/// JavaScript HeaderProvider implementation that wraps a JavaScript callback.
+/// This is the only native header provider - all header provider implementations
+/// should provide a JavaScript function that returns headers.
+#[napi]
+pub struct JsHeaderProvider {
+    get_headers_fn: Arc<ThreadsafeFunction<(), ErrorStrategy::CalleeHandled>>,
+}
+
+impl Clone for JsHeaderProvider {
+    fn clone(&self) -> Self {
+        Self {
+            get_headers_fn: self.get_headers_fn.clone(),
+        }
+    }
+}
+
+#[napi]
+impl JsHeaderProvider {
+    /// Create a new JsHeaderProvider from a JavaScript callback
+    #[napi(constructor)]
+    pub fn new(get_headers_callback: JsFunction) -> Result<Self> {
+        let get_headers_fn = get_headers_callback
+            .create_threadsafe_function(0, |ctx| Ok(vec![ctx.value]))
+            .map_err(|e| {
+                Error::new(
+                    Status::GenericFailure,
+                    format!("Failed to create threadsafe function: {}", e),
+                )
+            })?;
+
+        Ok(Self {
+            get_headers_fn: Arc::new(get_headers_fn),
+        })
+    }
+}
+
+#[cfg(feature = "remote")]
+#[async_trait::async_trait]
+impl lancedb::remote::HeaderProvider for JsHeaderProvider {
+    async fn get_headers(&self) -> lancedb::error::Result<HashMap<String, String>> {
+        // Call the JavaScript function asynchronously
+        let promise: Promise<HashMap<String, String>> =
+            self.get_headers_fn.call_async(Ok(())).await.map_err(|e| {
+                lancedb::error::Error::Runtime {
+                    message: format!("Failed to call JavaScript get_headers: {}", e),
+                }
+            })?;
+
+        // Await the promise result
+        promise.await.map_err(|e| lancedb::error::Error::Runtime {
+            message: format!("JavaScript get_headers failed: {}", e),
+        })
+    }
+}
+
+impl std::fmt::Debug for JsHeaderProvider {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "JsHeaderProvider")
+    }
+}

nodejs/src/lib.rs

@@ -8,6 +8,7 @@ use napi_derive::*;
 
 mod connection;
 mod error;
+mod header;
 mod index;
 mod iterator;
 pub mod merge;

nodejs/src/merge.rs

@@ -43,6 +43,13 @@ impl NativeMergeInsertBuilder {
         self.inner.timeout(Duration::from_millis(timeout as u64));
     }
 
+    #[napi]
+    pub fn use_index(&self, use_index: bool) -> Self {
+        let mut this = self.clone();
+        this.inner.use_index(use_index);
+        this
+    }
+
     #[napi(catch_unwind)]
     pub async fn execute(&self, buf: Buffer) -> napi::Result<MergeResult> {
         let data = ipc_file_to_batches(buf.to_vec())

nodejs/src/query.rs

@@ -12,6 +12,7 @@ use lancedb::query::Query as LanceDbQuery;
 use lancedb::query::QueryBase;
 use lancedb::query::QueryExecutionOptions;
 use lancedb::query::Select;
+use lancedb::query::TakeQuery as LanceDbTakeQuery;
 use lancedb::query::VectorQuery as LanceDbVectorQuery;
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
@@ -319,6 +320,79 @@ impl VectorQuery {
     }
 }
 
+#[napi]
+pub struct TakeQuery {
+    inner: LanceDbTakeQuery,
+}
+
+#[napi]
+impl TakeQuery {
+    pub fn new(query: LanceDbTakeQuery) -> Self {
+        Self { inner: query }
+    }
+
+    #[napi]
+    pub fn select(&mut self, columns: Vec<(String, String)>) {
+        self.inner = self.inner.clone().select(Select::dynamic(&columns));
+    }
+
+    #[napi]
+    pub fn select_columns(&mut self, columns: Vec<String>) {
+        self.inner = self.inner.clone().select(Select::columns(&columns));
+    }
+
+    #[napi]
+    pub fn with_row_id(&mut self) {
+        self.inner = self.inner.clone().with_row_id();
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn execute(
+        &self,
+        max_batch_length: Option<u32>,
+        timeout_ms: Option<u32>,
+    ) -> napi::Result<RecordBatchIterator> {
+        let mut execution_opts = QueryExecutionOptions::default();
+        if let Some(max_batch_length) = max_batch_length {
+            execution_opts.max_batch_length = max_batch_length;
+        }
+        if let Some(timeout_ms) = timeout_ms {
+            execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64))
+        }
+        let inner_stream = self
+            .inner
+            .execute_with_options(execution_opts)
+            .await
+            .map_err(|e| {
+                napi::Error::from_reason(format!(
+                    "Failed to execute query stream: {}",
+                    convert_error(&e)
+                ))
+            })?;
+        Ok(RecordBatchIterator::new(inner_stream))
+    }
+
+    #[napi]
+    pub async fn explain_plan(&self, verbose: bool) -> napi::Result<String> {
+        self.inner.explain_plan(verbose).await.map_err(|e| {
+            napi::Error::from_reason(format!(
+                "Failed to retrieve the query plan: {}",
+                convert_error(&e)
+            ))
+        })
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn analyze_plan(&self) -> napi::Result<String> {
+        self.inner.analyze_plan().await.map_err(|e| {
+            napi::Error::from_reason(format!(
+                "Failed to execute analyze plan: {}",
+                convert_error(&e)
+            ))
+        })
+    }
+}
+
 #[napi]
 #[derive(Debug, Clone)]
 pub struct JsFullTextQuery {
@@ -406,6 +480,7 @@ impl JsFullTextQuery {
     }
 
     #[napi(factory)]
+    #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
     pub fn boolean_query(queries: Vec<(String, &JsFullTextQuery)>) -> napi::Result<Self> {
         let mut sub_queries = Vec::with_capacity(queries.len());
         for (occur, q) in queries {

nodejs/src/remote.rs

@@ -69,6 +69,20 @@ pub struct RetryConfig {
     pub statuses: Option<Vec<u16>>,
 }
 
+/// TLS/mTLS configuration for the remote HTTP client.
+#[napi(object)]
+#[derive(Debug, Default)]
+pub struct TlsConfig {
+    /// Path to the client certificate file (PEM format) for mTLS authentication.
+    pub cert_file: Option<String>,
+    /// Path to the client private key file (PEM format) for mTLS authentication.
+    pub key_file: Option<String>,
+    /// Path to the CA certificate file (PEM format) for server verification.
+    pub ssl_ca_cert: Option<String>,
+    /// Whether to verify the hostname in the server's certificate.
+    pub assert_hostname: Option<bool>,
+}
+
 #[napi(object)]
 #[derive(Debug, Default)]
 pub struct ClientConfig {
@@ -76,6 +90,8 @@ pub struct ClientConfig {
     pub retry_config: Option<RetryConfig>,
     pub timeout_config: Option<TimeoutConfig>,
     pub extra_headers: Option<HashMap<String, String>>,
+    pub id_delimiter: Option<String>,
+    pub tls_config: Option<TlsConfig>,
 }
 
 impl From<TimeoutConfig> for lancedb::remote::TimeoutConfig {
@@ -106,6 +122,17 @@ impl From<RetryConfig> for lancedb::remote::RetryConfig {
     }
 }
 
+impl From<TlsConfig> for lancedb::remote::TlsConfig {
+    fn from(config: TlsConfig) -> Self {
+        Self {
+            cert_file: config.cert_file,
+            key_file: config.key_file,
+            ssl_ca_cert: config.ssl_ca_cert,
+            assert_hostname: config.assert_hostname.unwrap_or(true),
+        }
+    }
+}
+
 impl From<ClientConfig> for lancedb::remote::ClientConfig {
     fn from(config: ClientConfig) -> Self {
         Self {
@@ -115,6 +142,9 @@ impl From<ClientConfig> for lancedb::remote::ClientConfig {
             retry_config: config.retry_config.map(Into::into).unwrap_or_default(),
             timeout_config: config.timeout_config.map(Into::into).unwrap_or_default(),
             extra_headers: config.extra_headers.unwrap_or_default(),
+            id_delimiter: config.id_delimiter,
+            tls_config: config.tls_config.map(Into::into),
+            header_provider: None, // the header provider is set separately later
         }
     }
 }

nodejs/src/session.rs

@@ -94,7 +94,7 @@ impl napi::bindgen_prelude::FromNapiValue for Session {
         env: napi::sys::napi_env,
         napi_val: napi::sys::napi_value,
     ) -> napi::Result<Self> {
-        let object: napi::bindgen_prelude::ClassInstance<Session> =
+        let object: napi::bindgen_prelude::ClassInstance<Self> =
             napi::bindgen_prelude::ClassInstance::from_napi_value(env, napi_val)?;
         let copy = object.clone();
         Ok(copy)

nodejs/src/table.rs

@@ -15,7 +15,7 @@ use napi_derive::napi;
 use crate::error::NapiErrorExt;
 use crate::index::Index;
 use crate::merge::NativeMergeInsertBuilder;
-use crate::query::{Query, VectorQuery};
+use crate::query::{Query, TakeQuery, VectorQuery};
 
 #[napi]
 pub struct Table {
@@ -114,6 +114,8 @@ impl Table {
         column: String,
         replace: Option<bool>,
         wait_timeout_s: Option<i64>,
+        name: Option<String>,
+        train: Option<bool>,
     ) -> napi::Result<()> {
         let lancedb_index = if let Some(index) = index {
             index.consume()?
@@ -128,6 +130,12 @@ impl Table {
             builder =
                 builder.wait_timeout(std::time::Duration::from_secs(timeout.try_into().unwrap()));
         }
+        if let Some(name) = name {
+            builder = builder.name(name);
+        }
+        if let Some(train) = train {
+            builder = builder.train(train);
+        }
         builder.execute().await.default_error()
     }
 
@@ -187,6 +195,44 @@ impl Table {
         Ok(Query::new(self.inner_ref()?.query()))
     }
 
+    #[napi(catch_unwind)]
+    pub fn take_offsets(&self, offsets: Vec<i64>) -> napi::Result<TakeQuery> {
+        Ok(TakeQuery::new(
+            self.inner_ref()?.take_offsets(
+                offsets
+                    .into_iter()
+                    .map(|o| {
+                        u64::try_from(o).map_err(|e| {
+                            napi::Error::from_reason(format!(
+                                "Failed to convert offset to u64: {}",
+                                e
+                            ))
+                        })
+                    })
+                    .collect::<Result<Vec<_>>>()?,
+            ),
+        ))
+    }
+
+    #[napi(catch_unwind)]
+    pub fn take_row_ids(&self, row_ids: Vec<i64>) -> napi::Result<TakeQuery> {
+        Ok(TakeQuery::new(
+            self.inner_ref()?.take_row_ids(
+                row_ids
+                    .into_iter()
+                    .map(|o| {
+                        u64::try_from(o).map_err(|e| {
+                            napi::Error::from_reason(format!(
+                                "Failed to convert row id to u64: {}",
+                                e
+                            ))
+                        })
+                    })
+                    .collect::<Result<Vec<_>>>()?,
+            ),
+        ))
+    }
+
     #[napi(catch_unwind)]
     pub fn vector_search(&self, vector: Float32Array) -> napi::Result<VectorQuery> {
         self.query()?.nearest_to(vector)

nodejs/typedoc.json

@@ -2,6 +2,7 @@
   "intentionallyNotExported": [
     "lancedb/native.d.ts:Query",
     "lancedb/native.d.ts:VectorQuery",
+    "lancedb/native.d.ts:TakeQuery",
     "lancedb/native.d.ts:RecordBatchIterator",
     "lancedb/native.d.ts:NativeMergeInsertBuilder"
   ],

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.24.2"
+current_version = "0.25.2"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.24.2"
+version = "0.25.2"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true
@@ -14,11 +14,12 @@ name = "_lancedb"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "55.1", features = ["pyarrow"] }
+arrow = { version = "56.2", features = ["pyarrow"] }
+async-trait = "0.1"
 lancedb = { path = "../rust/lancedb", default-features = false }
 env_logger.workspace = true
-pyo3 = { version = "0.24", features = ["extension-module", "abi3-py39"] }
-pyo3-async-runtimes = { version = "0.24", features = [
+pyo3 = { version = "0.25", features = ["extension-module", "abi3-py39"] }
+pyo3-async-runtimes = { version = "0.25", features = [
     "attributes",
     "tokio-runtime",
 ] }
@@ -27,12 +28,12 @@ futures.workspace = true
 tokio = { version = "1.40", features = ["sync"] }
 
 [build-dependencies]
-pyo3-build-config = { version = "0.24", features = [
+pyo3-build-config = { version = "0.25", features = [
     "extension-module",
     "abi3-py39",
 ] }
 
 [features]
-default = ["remote"]
+default = ["remote", "lancedb/default"]
 fp16kernels = ["lancedb/fp16kernels"]
 remote = ["lancedb/remote"]

python/pyproject.toml

@@ -5,11 +5,12 @@ dynamic = ["version"]
 dependencies = [
     "deprecation",
     "numpy",
-    "overrides>=0.7",
+    "overrides>=0.7; python_version<'3.12'",
     "packaging",
     "pyarrow>=16",
     "pydantic>=1.10",
     "tqdm>=4.27.0",
+    "lance-namespace>=0.0.16"
 ]
 description = "lancedb"
 authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]

python/python/lancedb/__init__.py

@@ -19,6 +19,7 @@ from .remote.db import RemoteDBConnection
 from .schema import vector
 from .table import AsyncTable
 from ._lancedb import Session
+from .namespace import connect_namespace, LanceNamespaceDBConnection
 
 
 def connect(
@@ -221,6 +222,7 @@ async def connect_async(
 __all__ = [
     "connect",
     "connect_async",
+    "connect_namespace",
     "AsyncConnection",
     "AsyncTable",
     "URI",
@@ -228,6 +230,7 @@ __all__ = [
     "vector",
     "DBConnection",
     "LanceDBConnection",
+    "LanceNamespaceDBConnection",
     "RemoteDBConnection",
     "Session",
     "__version__",

python/python/lancedb/_lancedb.pyi

@@ -21,14 +21,28 @@ class Session:
 
 class Connection(object):
     uri: str
+    async def is_open(self): ...
+    async def close(self): ...
+    async def list_namespaces(
+        self,
+        namespace: List[str],
+        page_token: Optional[str],
+        limit: Optional[int],
+    ) -> List[str]: ...
+    async def create_namespace(self, namespace: List[str]) -> None: ...
+    async def drop_namespace(self, namespace: List[str]) -> None: ...
     async def table_names(
-        self, start_after: Optional[str], limit: Optional[int]
+        self,
+        namespace: List[str],
+        start_after: Optional[str],
+        limit: Optional[int],
     ) -> list[str]: ...
     async def create_table(
         self,
         name: str,
         mode: str,
         data: pa.RecordBatchReader,
+        namespace: List[str] = [],
         storage_options: Optional[Dict[str, str]] = None,
     ) -> Table: ...
     async def create_empty_table(
@@ -36,10 +50,34 @@ class Connection(object):
         name: str,
         mode: str,
         schema: pa.Schema,
+        namespace: List[str] = [],
         storage_options: Optional[Dict[str, str]] = None,
     ) -> Table: ...
-    async def rename_table(self, old_name: str, new_name: str) -> None: ...
-    async def drop_table(self, name: str) -> None: ...
+    async def open_table(
+        self,
+        name: str,
+        namespace: List[str] = [],
+        storage_options: Optional[Dict[str, str]] = None,
+        index_cache_size: Optional[int] = None,
+    ) -> Table: ...
+    async def clone_table(
+        self,
+        target_table_name: str,
+        source_uri: str,
+        target_namespace: List[str] = [],
+        source_version: Optional[int] = None,
+        source_tag: Optional[str] = None,
+        is_shallow: bool = True,
+    ) -> Table: ...
+    async def rename_table(
+        self,
+        cur_name: str,
+        new_name: str,
+        cur_namespace: List[str] = [],
+        new_namespace: List[str] = [],
+    ) -> None: ...
+    async def drop_table(self, name: str, namespace: List[str] = []) -> None: ...
+    async def drop_all_tables(self, namespace: List[str] = []) -> None: ...
 
 class Table:
     def name(self) -> str: ...
@@ -59,6 +97,10 @@ class Table:
         column: str,
         index: Union[IvfFlat, IvfPq, HnswPq, HnswSq, BTree, Bitmap, LabelList, FTS],
         replace: Optional[bool],
+        wait_timeout: Optional[object],
+        *,
+        name: Optional[str],
+        train: Optional[bool],
     ): ...
     async def list_versions(self) -> List[Dict[str, Any]]: ...
     async def version(self) -> int: ...
@@ -91,6 +133,7 @@ class Tags:
     async def update(self, tag: str, version: int): ...
 
 class IndexConfig:
+    name: str
     index_type: str
     columns: List[str]
 

python/python/lancedb/db.py

@@ -6,10 +6,18 @@ from __future__ import annotations
 
 from abc import abstractmethod
 from pathlib import Path
+import sys
 from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Optional, Union
 
+if sys.version_info >= (3, 12):
+    from typing import override
+
+    class EnforceOverrides:
+        pass
+else:
+    from overrides import EnforceOverrides, override  # type: ignore
+
 from lancedb.embeddings.registry import EmbeddingFunctionRegistry
-from overrides import EnforceOverrides, override  # type: ignore
 
 from lancedb.common import data_to_reader, sanitize_uri, validate_schema
 from lancedb.background_loop import LOOP
@@ -43,14 +51,70 @@ if TYPE_CHECKING:
 class DBConnection(EnforceOverrides):
     """An active LanceDB connection interface."""
 
+    def list_namespaces(
+        self,
+        namespace: List[str] = [],
+        page_token: Optional[str] = None,
+        limit: int = 10,
+    ) -> Iterable[str]:
+        """List immediate child namespace names in the given namespace.
+
+        Parameters
+        ----------
+        namespace: List[str], default []
+            The parent namespace to list namespaces in.
+            Empty list represents root namespace.
+        page_token: str, optional
+            The token to use for pagination. If not present, start from the beginning.
+        limit: int, default 10
+            The size of the page to return.
+
+        Returns
+        -------
+        Iterable of str
+            List of immediate child namespace names
+        """
+        return []
+
+    def create_namespace(self, namespace: List[str]) -> None:
+        """Create a new namespace.
+
+        Parameters
+        ----------
+        namespace: List[str]
+            The namespace identifier to create.
+        """
+        raise NotImplementedError(
+            "Namespace operations are not supported for this connection type"
+        )
+
+    def drop_namespace(self, namespace: List[str]) -> None:
+        """Drop a namespace.
+
+        Parameters
+        ----------
+        namespace: List[str]
+            The namespace identifier to drop.
+        """
+        raise NotImplementedError(
+            "Namespace operations are not supported for this connection type"
+        )
+
     @abstractmethod
     def table_names(
-        self, page_token: Optional[str] = None, limit: int = 10
+        self,
+        page_token: Optional[str] = None,
+        limit: int = 10,
+        *,
+        namespace: List[str] = [],
     ) -> Iterable[str]:
         """List all tables in this database, in sorted order
 
         Parameters
         ----------
+        namespace: List[str], default []
+            The namespace to list tables in.
+            Empty list represents root namespace.
         page_token: str, optional
             The token to use for pagination. If not present, start from the beginning.
             Typically, this token is last table name from the previous page.
@@ -77,6 +141,7 @@ class DBConnection(EnforceOverrides):
         fill_value: float = 0.0,
         embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
         *,
+        namespace: List[str] = [],
         storage_options: Optional[Dict[str, str]] = None,
         data_storage_version: Optional[str] = None,
         enable_v2_manifest_paths: Optional[bool] = None,
@@ -87,6 +152,9 @@ class DBConnection(EnforceOverrides):
         ----------
         name: str
             The name of the table.
+        namespace: List[str], default []
+            The namespace to create the table in.
+            Empty list represents root namespace.
         data: The data to initialize the table, *optional*
             User must provide at least one of `data` or `schema`.
             Acceptable types are:
@@ -238,6 +306,7 @@ class DBConnection(EnforceOverrides):
         self,
         name: str,
         *,
+        namespace: List[str] = [],
         storage_options: Optional[Dict[str, str]] = None,
         index_cache_size: Optional[int] = None,
     ) -> Table:
@@ -247,6 +316,9 @@ class DBConnection(EnforceOverrides):
         ----------
         name: str
             The name of the table.
+        namespace: List[str], optional
+            The namespace to open the table from.
+            None or empty list represents root namespace.
         index_cache_size: int, default 256
             **Deprecated**: Use session-level cache configuration instead.
             Create a Session with custom cache sizes and pass it to lancedb.connect().
@@ -272,17 +344,26 @@ class DBConnection(EnforceOverrides):
         """
         raise NotImplementedError
 
-    def drop_table(self, name: str):
+    def drop_table(self, name: str, namespace: List[str] = []):
         """Drop a table from the database.
 
         Parameters
         ----------
         name: str
             The name of the table.
+        namespace: List[str], default []
+            The namespace to drop the table from.
+            Empty list represents root namespace.
         """
         raise NotImplementedError
 
-    def rename_table(self, cur_name: str, new_name: str):
+    def rename_table(
+        self,
+        cur_name: str,
+        new_name: str,
+        cur_namespace: List[str] = [],
+        new_namespace: List[str] = [],
+    ):
         """Rename a table in the database.
 
         Parameters
@@ -291,6 +372,12 @@ class DBConnection(EnforceOverrides):
             The current name of the table.
         new_name: str
             The new name of the table.
+        cur_namespace: List[str], optional
+            The namespace of the current table.
+            None or empty list represents root namespace.
+        new_namespace: List[str], optional
+            The namespace to move the table to.
+            If not specified, defaults to the same as cur_namespace.
         """
         raise NotImplementedError
 
@@ -301,9 +388,15 @@ class DBConnection(EnforceOverrides):
         """
         raise NotImplementedError
 
-    def drop_all_tables(self):
+    def drop_all_tables(self, namespace: List[str] = []):
         """
         Drop all tables from the database
+
+        Parameters
+        ----------
+        namespace: List[str], optional
+            The namespace to drop all tables from.
+            None or empty list represents root namespace.
         """
         raise NotImplementedError
 
@@ -404,18 +497,87 @@ class LanceDBConnection(DBConnection):
         conn = AsyncConnection(await lancedb_connect(self.uri))
         return await conn.table_names(start_after=start_after, limit=limit)
 
+    @override
+    def list_namespaces(
+        self,
+        namespace: List[str] = [],
+        page_token: Optional[str] = None,
+        limit: int = 10,
+    ) -> Iterable[str]:
+        """List immediate child namespace names in the given namespace.
+
+        Parameters
+        ----------
+        namespace: List[str], optional
+            The parent namespace to list namespaces in.
+            None or empty list represents root namespace.
+        page_token: str, optional
+            The token to use for pagination. If not present, start from the beginning.
+        limit: int, default 10
+            The size of the page to return.
+
+        Returns
+        -------
+        Iterable of str
+            List of immediate child namespace names
+        """
+        return LOOP.run(
+            self._conn.list_namespaces(
+                namespace=namespace, page_token=page_token, limit=limit
+            )
+        )
+
+    @override
+    def create_namespace(self, namespace: List[str]) -> None:
+        """Create a new namespace.
+
+        Parameters
+        ----------
+        namespace: List[str]
+            The namespace identifier to create.
+        """
+        LOOP.run(self._conn.create_namespace(namespace=namespace))
+
+    @override
+    def drop_namespace(self, namespace: List[str]) -> None:
+        """Drop a namespace.
+
+        Parameters
+        ----------
+        namespace: List[str]
+            The namespace identifier to drop.
+        """
+        return LOOP.run(self._conn.drop_namespace(namespace=namespace))
+
     @override
     def table_names(
-        self, page_token: Optional[str] = None, limit: int = 10
+        self,
+        page_token: Optional[str] = None,
+        limit: int = 10,
+        *,
+        namespace: List[str] = [],
     ) -> Iterable[str]:
         """Get the names of all tables in the database. The names are sorted.
 
+        Parameters
+        ----------
+        namespace: List[str], optional
+            The namespace to list tables in.
+        page_token: str, optional
+            The token to use for pagination.
+        limit: int, default 10
+            The maximum number of tables to return.
+
         Returns
         -------
         Iterator of str.
             A list of table names.
         """
-        return LOOP.run(self._conn.table_names(start_after=page_token, limit=limit))
+        return LOOP.run(
+            self._conn.table_names(
+                namespace=namespace, start_after=page_token, limit=limit
+            )
+        )
 
     def __len__(self) -> int:
         return len(self.table_names())
@@ -435,12 +597,18 @@ class LanceDBConnection(DBConnection):
         fill_value: float = 0.0,
         embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
         *,
+        namespace: List[str] = [],
         storage_options: Optional[Dict[str, str]] = None,
         data_storage_version: Optional[str] = None,
         enable_v2_manifest_paths: Optional[bool] = None,
     ) -> LanceTable:
         """Create a table in the database.
 
+        Parameters
+        ----------
+        namespace: List[str], optional
+            The namespace to create the table in.
+
         See
         ---
         DBConnection.create_table
@@ -459,6 +627,7 @@ class LanceDBConnection(DBConnection):
             on_bad_vectors=on_bad_vectors,
             fill_value=fill_value,
             embedding_functions=embedding_functions,
+            namespace=namespace,
             storage_options=storage_options,
         )
         return tbl
@@ -468,6 +637,7 @@ class LanceDBConnection(DBConnection):
         self,
         name: str,
         *,
+        namespace: List[str] = [],
         storage_options: Optional[Dict[str, str]] = None,
         index_cache_size: Optional[int] = None,
     ) -> LanceTable:
@@ -477,6 +647,8 @@ class LanceDBConnection(DBConnection):
         ----------
         name: str
             The name of the table.
+        namespace: List[str], optional
+            The namespace to open the table from.
 
         Returns
         -------
@@ -496,26 +668,122 @@ class LanceDBConnection(DBConnection):
         return LanceTable.open(
             self,
             name,
+            namespace=namespace,
             storage_options=storage_options,
             index_cache_size=index_cache_size,
         )
 
+    def clone_table(
+        self,
+        target_table_name: str,
+        source_uri: str,
+        *,
+        target_namespace: List[str] = [],
+        source_version: Optional[int] = None,
+        source_tag: Optional[str] = None,
+        is_shallow: bool = True,
+    ) -> LanceTable:
+        """Clone a table from a source table.
+
+        A shallow clone creates a new table that shares the underlying data files
+        with the source table but has its own independent manifest. This allows
+        both the source and cloned tables to evolve independently while initially
+        sharing the same data, deletion, and index files.
+
+        Parameters
+        ----------
+        target_table_name: str
+            The name of the target table to create.
+        source_uri: str
+            The URI of the source table to clone from.
+        target_namespace: List[str], optional
+            The namespace for the target table.
+            None or empty list represents root namespace.
+        source_version: int, optional
+            The version of the source table to clone.
+        source_tag: str, optional
+            The tag of the source table to clone.
+        is_shallow: bool, default True
+            Whether to perform a shallow clone (True) or deep clone (False).
+            Currently only shallow clone is supported.
+
+        Returns
+        -------
+        A LanceTable object representing the cloned table.
+        """
+        LOOP.run(
+            self._conn.clone_table(
+                target_table_name,
+                source_uri,
+                target_namespace=target_namespace,
+                source_version=source_version,
+                source_tag=source_tag,
+                is_shallow=is_shallow,
+            )
+        )
+        return LanceTable.open(
+            self,
+            target_table_name,
+            namespace=target_namespace,
+        )
+
     @override
-    def drop_table(self, name: str, ignore_missing: bool = False):
+    def drop_table(
+        self,
+        name: str,
+        namespace: List[str] = [],
+        ignore_missing: bool = False,
+    ):
         """Drop a table from the database.
 
         Parameters
         ----------
         name: str
             The name of the table.
+        namespace: List[str], optional
+            The namespace to drop the table from.
         ignore_missing: bool, default False
             If True, ignore if the table does not exist.
         """
-        LOOP.run(self._conn.drop_table(name, ignore_missing=ignore_missing))
+        LOOP.run(
+            self._conn.drop_table(
+                name, namespace=namespace, ignore_missing=ignore_missing
+            )
+        )
 
     @override
-    def drop_all_tables(self):
-        LOOP.run(self._conn.drop_all_tables())
+    def drop_all_tables(self, namespace: List[str] = []):
+        LOOP.run(self._conn.drop_all_tables(namespace=namespace))
+
+    @override
+    def rename_table(
+        self,
+        cur_name: str,
+        new_name: str,
+        cur_namespace: List[str] = [],
+        new_namespace: List[str] = [],
+    ):
+        """Rename a table in the database.
+
+        Parameters
+        ----------
+        cur_name: str
+            The current name of the table.
+        new_name: str
+            The new name of the table.
+        cur_namespace: List[str], optional
+            The namespace of the current table.
+        new_namespace: List[str], optional
+            The namespace to move the table to.
+        """
+        LOOP.run(
+            self._conn.rename_table(
+                cur_name,
+                new_name,
+                cur_namespace=cur_namespace,
+                new_namespace=new_namespace,
+            )
+        )
 
     @deprecation.deprecated(
         deprecated_in="0.15.1",
@@ -588,13 +856,67 @@ class AsyncConnection(object):
     def uri(self) -> str:
         return self._inner.uri
 
+    async def list_namespaces(
+        self,
+        namespace: List[str] = [],
+        page_token: Optional[str] = None,
+        limit: int = 10,
+    ) -> Iterable[str]:
+        """List immediate child namespace names in the given namespace.
+
+        Parameters
+        ----------
+        namespace: List[str], optional
+            The parent namespace to list namespaces in.
+            None or empty list represents root namespace.
+        page_token: str, optional
+            The token to use for pagination. If not present, start from the beginning.
+        limit: int, default 10
+            The size of the page to return.
+
+        Returns
+        -------
+        Iterable of str
+            List of immediate child namespace names (not full paths)
+        """
+        return await self._inner.list_namespaces(
+            namespace=namespace, page_token=page_token, limit=limit
+        )
+
+    async def create_namespace(self, namespace: List[str]) -> None:
+        """Create a new namespace.
+
+        Parameters
+        ----------
+        namespace: List[str]
+            The namespace identifier to create.
+        """
+        await self._inner.create_namespace(namespace)
+
+    async def drop_namespace(self, namespace: List[str]) -> None:
+        """Drop a namespace.
+
+        Parameters
+        ----------
+        namespace: List[str]
+            The namespace identifier to drop.
+        """
+        await self._inner.drop_namespace(namespace)
+
     async def table_names(
-        self, *, start_after: Optional[str] = None, limit: Optional[int] = None
+        self,
+        *,
+        namespace: List[str] = [],
+        start_after: Optional[str] = None,
+        limit: Optional[int] = None,
     ) -> Iterable[str]:
         """List all tables in this database, in sorted order
 
         Parameters
         ----------
+        namespace: List[str], optional
+            The namespace to list tables in.
+            None or empty list represents root namespace.
         start_after: str, optional
             If present, only return names that come lexicographically after the supplied
             value.
@@ -608,7 +930,9 @@ class AsyncConnection(object):
         -------
         Iterable of str
         """
-        return await self._inner.table_names(start_after=start_after, limit=limit)
+        return await self._inner.table_names(
+            namespace=namespace, start_after=start_after, limit=limit
+        )
 
     async def create_table(
         self,
@@ -621,6 +945,7 @@ class AsyncConnection(object):
         fill_value: Optional[float] = None,
         storage_options: Optional[Dict[str, str]] = None,
         *,
+        namespace: List[str] = [],
         embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
     ) -> AsyncTable:
         """Create an [AsyncTable][lancedb.table.AsyncTable] in the database.
@@ -629,6 +954,9 @@ class AsyncConnection(object):
         ----------
         name: str
             The name of the table.
+        namespace: List[str], default []
+            The namespace to create the table in.
+            Empty list represents root namespace.
         data: The data to initialize the table, *optional*
             User must provide at least one of `data` or `schema`.
             Acceptable types are:
@@ -807,6 +1135,7 @@ class AsyncConnection(object):
                 name,
                 mode,
                 schema,
+                namespace=namespace,
                 storage_options=storage_options,
             )
         else:
@@ -815,6 +1144,7 @@ class AsyncConnection(object):
                 name,
                 mode,
                 data,
+                namespace=namespace,
                 storage_options=storage_options,
             )
 
@@ -823,6 +1153,8 @@ class AsyncConnection(object):
     async def open_table(
         self,
         name: str,
+        *,
+        namespace: List[str] = [],
         storage_options: Optional[Dict[str, str]] = None,
         index_cache_size: Optional[int] = None,
     ) -> AsyncTable:
@@ -832,6 +1164,9 @@ class AsyncConnection(object):
         ----------
         name: str
             The name of the table.
+        namespace: List[str], optional
+            The namespace to open the table from.
+            None or empty list represents root namespace.
         storage_options: dict, optional
             Additional options for the storage backend. Options already set on the
             connection will be inherited by the table, but can be overridden here.
@@ -855,42 +1190,125 @@ class AsyncConnection(object):
         -------
         A LanceTable object representing the table.
         """
-        table = await self._inner.open_table(name, storage_options, index_cache_size)
+        table = await self._inner.open_table(
+            name,
+            namespace=namespace,
+            storage_options=storage_options,
+            index_cache_size=index_cache_size,
+        )
         return AsyncTable(table)
 
-    async def rename_table(self, old_name: str, new_name: str):
+    async def clone_table(
+        self,
+        target_table_name: str,
+        source_uri: str,
+        *,
+        target_namespace: List[str] = [],
+        source_version: Optional[int] = None,
+        source_tag: Optional[str] = None,
+        is_shallow: bool = True,
+    ) -> AsyncTable:
+        """Clone a table from a source table.
+
+        A shallow clone creates a new table that shares the underlying data files
+        with the source table but has its own independent manifest. This allows
+        both the source and cloned tables to evolve independently while initially
+        sharing the same data, deletion, and index files.
+
+        Parameters
+        ----------
+        target_table_name: str
+            The name of the target table to create.
+        source_uri: str
+            The URI of the source table to clone from.
+        target_namespace: List[str], optional
+            The namespace for the target table.
+            None or empty list represents root namespace.
+        source_version: int, optional
+            The version of the source table to clone.
+        source_tag: str, optional
+            The tag of the source table to clone.
+        is_shallow: bool, default True
+            Whether to perform a shallow clone (True) or deep clone (False).
+            Currently only shallow clone is supported.
+
+        Returns
+        -------
+        An AsyncTable object representing the cloned table.
+        """
+        table = await self._inner.clone_table(
+            target_table_name,
+            source_uri,
+            target_namespace=target_namespace,
+            source_version=source_version,
+            source_tag=source_tag,
+            is_shallow=is_shallow,
+        )
+        return AsyncTable(table)
+
+    async def rename_table(
+        self,
+        cur_name: str,
+        new_name: str,
+        cur_namespace: List[str] = [],
+        new_namespace: List[str] = [],
+    ):
         """Rename a table in the database.
 
         Parameters
         ----------
-        old_name: str
+        cur_name: str
             The current name of the table.
         new_name: str
             The new name of the table.
+        cur_namespace: List[str], optional
+            The namespace of the current table.
+            None or empty list represents root namespace.
+        new_namespace: List[str], optional
+            The namespace to move the table to.
+            If not specified, defaults to the same as cur_namespace.
         """
-        await self._inner.rename_table(old_name, new_name)
+        await self._inner.rename_table(
+            cur_name, new_name, cur_namespace=cur_namespace, new_namespace=new_namespace
+        )
 
-    async def drop_table(self, name: str, *, ignore_missing: bool = False):
+    async def drop_table(
+        self,
+        name: str,
+        *,
+        namespace: List[str] = [],
+        ignore_missing: bool = False,
+    ):
         """Drop a table from the database.
 
         Parameters
         ----------
         name: str
             The name of the table.
+        namespace: List[str], default []
+            The namespace to drop the table from.
+            Empty list represents root namespace.
         ignore_missing: bool, default False
             If True, ignore if the table does not exist.
         """
         try:
-            await self._inner.drop_table(name)
+            await self._inner.drop_table(name, namespace=namespace)
         except ValueError as e:
             if not ignore_missing:
                 raise e
             if f"Table '{name}' was not found" not in str(e):
                 raise e
 
-    async def drop_all_tables(self):
-        """Drop all tables from the database."""
-        await self._inner.drop_all_tables()
+    async def drop_all_tables(self, namespace: List[str] = []):
+        """Drop all tables from the database.
+
+        Parameters
+        ----------
+        namespace: List[str], optional
+            The namespace to drop all tables from.
+            None or empty list represents root namespace.
+        """
+        await self._inner.drop_all_tables(namespace=namespace)
 
     @deprecation.deprecated(
         deprecated_in="0.15.1",

python/python/lancedb/embeddings/registry.py

@@ -122,7 +122,7 @@ class EmbeddingFunctionRegistry:
             obj["vector_column"]: EmbeddingFunctionConfig(
                 vector_column=obj["vector_column"],
                 source_column=obj["source_column"],
-                function=self.get(obj["name"])(**obj["model"]),
+                function=self.get(obj["name"]).create(**obj["model"]),
             )
             for obj in raw_list
         }

python/python/lancedb/index.py

@@ -251,6 +251,13 @@ class HnswPq:
         results. In most cases, there is no benefit to setting this higher than 500.
         This value should be set to a value that is not less than `ef` in the
         search phase.
+
+    target_partition_size, default is 1,048,576
+
+        The target size of each partition.
+
+        This value controls the tradeoff between search performance and accuracy.
+        faster search but less accurate results as higher value.
     """
 
     distance_type: Literal["l2", "cosine", "dot"] = "l2"
@@ -261,6 +268,7 @@ class HnswPq:
     sample_rate: int = 256
     m: int = 20
     ef_construction: int = 300
+    target_partition_size: Optional[int] = None
 
 
 @dataclass
@@ -351,6 +359,12 @@ class HnswSq:
         This value should be set to a value that is not less than `ef` in the search
         phase.
 
+    target_partition_size, default is 1,048,576
+
+        The target size of each partition.
+
+        This value controls the tradeoff between search performance and accuracy.
+        faster search but less accurate results as higher value.
     """
 
     distance_type: Literal["l2", "cosine", "dot"] = "l2"
@@ -359,6 +373,7 @@ class HnswSq:
     sample_rate: int = 256
     m: int = 20
     ef_construction: int = 300
+    target_partition_size: Optional[int] = None
 
 
 @dataclass
@@ -444,12 +459,20 @@ class IvfFlat:
         cases the default should be sufficient.
 
         The default value is 256.
+
+    target_partition_size, default is 8192
+
+        The target size of each partition.
+
+        This value controls the tradeoff between search performance and accuracy.
+        faster search but less accurate results as higher value.
     """
 
     distance_type: Literal["l2", "cosine", "dot", "hamming"] = "l2"
     num_partitions: Optional[int] = None
     max_iterations: int = 50
     sample_rate: int = 256
+    target_partition_size: Optional[int] = None
 
 
 @dataclass
@@ -564,6 +587,13 @@ class IvfPq:
         cases the default should be sufficient.
 
         The default value is 256.
+
+    target_partition_size, default is 8192
+
+        The target size of each partition.
+
+        This value controls the tradeoff between search performance and accuracy.
+        faster search but less accurate results as higher value.
     """
 
     distance_type: Literal["l2", "cosine", "dot"] = "l2"
@@ -572,6 +602,7 @@ class IvfPq:
     num_bits: int = 8
     max_iterations: int = 50
     sample_rate: int = 256
+    target_partition_size: Optional[int] = None
 
 
 __all__ = [

python/python/lancedb/merge.py

@@ -33,6 +33,7 @@ class LanceMergeInsertBuilder(object):
         self._when_not_matched_by_source_delete = False
         self._when_not_matched_by_source_condition = None
         self._timeout = None
+        self._use_index = True
 
     def when_matched_update_all(
         self, *, where: Optional[str] = None
@@ -78,6 +79,23 @@ class LanceMergeInsertBuilder(object):
             self._when_not_matched_by_source_condition = condition
         return self
 
+    def use_index(self, use_index: bool) -> LanceMergeInsertBuilder:
+        """
+        Controls whether to use indexes for the merge operation.
+
+        When set to `True` (the default), the operation will use an index if available
+        on the join key for improved performance. When set to `False`, it forces a full
+        table scan even if an index exists. This can be useful for benchmarking or when
+        the query optimizer chooses a suboptimal path.
+
+        Parameters
+        ----------
+        use_index: bool
+            Whether to use indices for the merge operation. Defaults to `True`.
+        """
+        self._use_index = use_index
+        return self
+
     def execute(
         self,
         new_data: DATA,

python/python/lancedb/namespace.py

@@ -0,0 +1,406 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+"""
+LanceDB Namespace integration module.
+
+This module provides integration with lance_namespace for managing tables
+through a namespace abstraction.
+"""
+
+from __future__ import annotations
+
+from typing import Dict, Iterable, List, Optional, Union
+import os
+import sys
+
+if sys.version_info >= (3, 12):
+    from typing import override
+else:
+    from overrides import override
+
+from lancedb.db import DBConnection
+from lancedb.table import LanceTable, Table
+from lancedb.util import validate_table_name
+from lancedb.common import validate_schema
+from lancedb.table import sanitize_create_table
+
+from lance_namespace import LanceNamespace, connect as namespace_connect
+from lance_namespace_urllib3_client.models import (
+    ListTablesRequest,
+    DescribeTableRequest,
+    CreateTableRequest,
+    DropTableRequest,
+    ListNamespacesRequest,
+    CreateNamespaceRequest,
+    DropNamespaceRequest,
+    JsonArrowSchema,
+    JsonArrowField,
+    JsonArrowDataType,
+)
+
+import pyarrow as pa
+from datetime import timedelta
+from lancedb.pydantic import LanceModel
+from lancedb.common import DATA
+from lancedb.embeddings import EmbeddingFunctionConfig
+from ._lancedb import Session
+
+
+def _convert_pyarrow_type_to_json(arrow_type: pa.DataType) -> JsonArrowDataType:
+    """Convert PyArrow DataType to JsonArrowDataType."""
+    if pa.types.is_null(arrow_type):
+        type_name = "null"
+    elif pa.types.is_boolean(arrow_type):
+        type_name = "bool"
+    elif pa.types.is_int8(arrow_type):
+        type_name = "int8"
+    elif pa.types.is_uint8(arrow_type):
+        type_name = "uint8"
+    elif pa.types.is_int16(arrow_type):
+        type_name = "int16"
+    elif pa.types.is_uint16(arrow_type):
+        type_name = "uint16"
+    elif pa.types.is_int32(arrow_type):
+        type_name = "int32"
+    elif pa.types.is_uint32(arrow_type):
+        type_name = "uint32"
+    elif pa.types.is_int64(arrow_type):
+        type_name = "int64"
+    elif pa.types.is_uint64(arrow_type):
+        type_name = "uint64"
+    elif pa.types.is_float32(arrow_type):
+        type_name = "float32"
+    elif pa.types.is_float64(arrow_type):
+        type_name = "float64"
+    elif pa.types.is_string(arrow_type):
+        type_name = "utf8"
+    elif pa.types.is_binary(arrow_type):
+        type_name = "binary"
+    elif pa.types.is_list(arrow_type):
+        # For list types, we need more complex handling
+        type_name = "list"
+    elif pa.types.is_fixed_size_list(arrow_type):
+        type_name = "fixed_size_list"
+    else:
+        # Default to string representation for unsupported types
+        type_name = str(arrow_type)
+
+    return JsonArrowDataType(type=type_name)
+
+
+def _convert_pyarrow_schema_to_json(schema: pa.Schema) -> JsonArrowSchema:
+    """Convert PyArrow Schema to JsonArrowSchema."""
+    fields = []
+    for field in schema:
+        json_field = JsonArrowField(
+            name=field.name,
+            type=_convert_pyarrow_type_to_json(field.type),
+            nullable=field.nullable,
+            metadata=field.metadata,
+        )
+        fields.append(json_field)
+
+    return JsonArrowSchema(fields=fields, metadata=schema.metadata)
+
+
+class LanceNamespaceDBConnection(DBConnection):
+    """
+    A LanceDB connection that uses a namespace for table management.
+
+    This connection delegates table URI resolution to a lance_namespace instance,
+    while using the standard LanceTable for actual table operations.
+    """
+
+    def __init__(
+        self,
+        namespace: LanceNamespace,
+        *,
+        read_consistency_interval: Optional[timedelta] = None,
+        storage_options: Optional[Dict[str, str]] = None,
+        session: Optional[Session] = None,
+    ):
+        """
+        Initialize a namespace-based LanceDB connection.
+
+        Parameters
+        ----------
+        namespace : LanceNamespace
+            The namespace instance to use for table management
+        read_consistency_interval : Optional[timedelta]
+            The interval at which to check for updates to the table from other
+            processes. If None, then consistency is not checked.
+        storage_options : Optional[Dict[str, str]]
+            Additional options for the storage backend
+        session : Optional[Session]
+            A session to use for this connection
+        """
+        self._ns = namespace
+        self.read_consistency_interval = read_consistency_interval
+        self.storage_options = storage_options or {}
+        self.session = session
+
+    @override
+    def table_names(
+        self,
+        page_token: Optional[str] = None,
+        limit: int = 10,
+        *,
+        namespace: List[str] = [],
+    ) -> Iterable[str]:
+        request = ListTablesRequest(id=namespace, page_token=page_token, limit=limit)
+        response = self._ns.list_tables(request)
+        return response.tables if response.tables else []
+
+    @override
+    def create_table(
+        self,
+        name: str,
+        data: Optional[DATA] = None,
+        schema: Optional[Union[pa.Schema, LanceModel]] = None,
+        mode: str = "create",
+        exist_ok: bool = False,
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
+        *,
+        namespace: List[str] = [],
+        storage_options: Optional[Dict[str, str]] = None,
+        data_storage_version: Optional[str] = None,
+        enable_v2_manifest_paths: Optional[bool] = None,
+    ) -> Table:
+        if mode.lower() not in ["create", "overwrite"]:
+            raise ValueError("mode must be either 'create' or 'overwrite'")
+        validate_table_name(name)
+
+        # TODO: support passing data
+        if data is not None:
+            raise ValueError(
+                "create_table currently only supports creating empty tables (data=None)"
+            )
+
+        # Prepare schema
+        metadata = None
+        if embedding_functions is not None:
+            from lancedb.embeddings.registry import EmbeddingFunctionRegistry
+
+            registry = EmbeddingFunctionRegistry.get_instance()
+            metadata = registry.get_table_metadata(embedding_functions)
+
+        data, schema = sanitize_create_table(
+            data, schema, metadata, on_bad_vectors, fill_value
+        )
+        validate_schema(schema)
+
+        # Convert PyArrow schema to JsonArrowSchema
+        json_schema = _convert_pyarrow_schema_to_json(schema)
+
+        # Create table request with namespace
+        table_id = namespace + [name]
+        request = CreateTableRequest(id=table_id, var_schema=json_schema)
+
+        # Create empty Arrow IPC stream bytes
+        import pyarrow.ipc as ipc
+        import io
+
+        empty_table = pa.Table.from_arrays(
+            [pa.array([], type=field.type) for field in schema], schema=schema
+        )
+        buffer = io.BytesIO()
+        with ipc.new_stream(buffer, schema) as writer:
+            writer.write_table(empty_table)
+        request_data = buffer.getvalue()
+
+        self._ns.create_table(request, request_data)
+        return self.open_table(
+            name, namespace=namespace, storage_options=storage_options
+        )
+
+    @override
+    def open_table(
+        self,
+        name: str,
+        *,
+        namespace: List[str] = [],
+        storage_options: Optional[Dict[str, str]] = None,
+        index_cache_size: Optional[int] = None,
+    ) -> Table:
+        table_id = namespace + [name]
+        request = DescribeTableRequest(id=table_id)
+        response = self._ns.describe_table(request)
+
+        merged_storage_options = dict()
+        if storage_options:
+            merged_storage_options.update(storage_options)
+        if response.storage_options:
+            merged_storage_options.update(response.storage_options)
+
+        return self._lance_table_from_uri(
+            response.location,
+            storage_options=merged_storage_options,
+            index_cache_size=index_cache_size,
+        )
+
+    @override
+    def drop_table(self, name: str, namespace: List[str] = []):
+        # Use namespace drop_table directly
+        table_id = namespace + [name]
+        request = DropTableRequest(id=table_id)
+        self._ns.drop_table(request)
+
+    @override
+    def rename_table(
+        self,
+        cur_name: str,
+        new_name: str,
+        cur_namespace: List[str] = [],
+        new_namespace: List[str] = [],
+    ):
+        raise NotImplementedError(
+            "rename_table is not supported for namespace connections"
+        )
+
+    @override
+    def drop_database(self):
+        raise NotImplementedError(
+            "drop_database is deprecated, use drop_all_tables instead"
+        )
+
+    @override
+    def drop_all_tables(self, namespace: List[str] = []):
+        for table_name in self.table_names(namespace=namespace):
+            self.drop_table(table_name, namespace=namespace)
+
+    @override
+    def list_namespaces(
+        self,
+        namespace: List[str] = [],
+        page_token: Optional[str] = None,
+        limit: int = 10,
+    ) -> Iterable[str]:
+        """
+        List child namespaces under the given namespace.
+
+        Parameters
+        ----------
+        namespace : Optional[List[str]]
+            The parent namespace to list children from.
+            If None, lists root-level namespaces.
+        page_token : Optional[str]
+            Pagination token for listing results.
+        limit : int
+            Maximum number of namespaces to return.
+
+        Returns
+        -------
+        Iterable[str]
+            Names of child namespaces.
+        """
+        request = ListNamespacesRequest(
+            id=namespace, page_token=page_token, limit=limit
+        )
+        response = self._ns.list_namespaces(request)
+        return response.namespaces if response.namespaces else []
+
+    @override
+    def create_namespace(self, namespace: List[str]) -> None:
+        """
+        Create a new namespace.
+
+        Parameters
+        ----------
+        namespace : List[str]
+            The namespace path to create.
+        """
+        request = CreateNamespaceRequest(id=namespace)
+        self._ns.create_namespace(request)
+
+    @override
+    def drop_namespace(self, namespace: List[str]) -> None:
+        """
+        Drop a namespace.
+
+        Parameters
+        ----------
+        namespace : List[str]
+            The namespace path to drop.
+        """
+        request = DropNamespaceRequest(id=namespace)
+        self._ns.drop_namespace(request)
+
+    def _lance_table_from_uri(
+        self,
+        table_uri: str,
+        *,
+        storage_options: Optional[Dict[str, str]] = None,
+        index_cache_size: Optional[int] = None,
+    ) -> LanceTable:
+        # Extract the base path and table name from the URI
+        if table_uri.endswith(".lance"):
+            base_path = os.path.dirname(table_uri)
+            table_name = os.path.basename(table_uri)[:-6]  # Remove .lance
+        else:
+            raise ValueError(f"Invalid table URI: {table_uri}")
+
+        from lancedb.db import LanceDBConnection
+
+        temp_conn = LanceDBConnection(
+            base_path,
+            read_consistency_interval=self.read_consistency_interval,
+            storage_options={**self.storage_options, **(storage_options or {})},
+            session=self.session,
+        )
+
+        # Open the table using the temporary connection
+        return LanceTable.open(
+            temp_conn,
+            table_name,
+            storage_options=storage_options,
+            index_cache_size=index_cache_size,
+        )
+
+
+def connect_namespace(
+    impl: str,
+    properties: Dict[str, str],
+    *,
+    read_consistency_interval: Optional[timedelta] = None,
+    storage_options: Optional[Dict[str, str]] = None,
+    session: Optional[Session] = None,
+) -> LanceNamespaceDBConnection:
+    """
+    Connect to a LanceDB database through a namespace.
+
+    Parameters
+    ----------
+    impl : str
+        The namespace implementation to use. For examples:
+        - "dir" for DirectoryNamespace
+        - "rest" for REST-based namespace
+        - Full module path for custom implementations
+    properties : Dict[str, str]
+        Configuration properties for the namespace implementation.
+        Different namespace implemenation has different config properties.
+        For example, use DirectoryNamespace with {"root": "/path/to/directory"}
+    read_consistency_interval : Optional[timedelta]
+        The interval at which to check for updates to the table from other
+        processes. If None, then consistency is not checked.
+    storage_options : Optional[Dict[str, str]]
+        Additional options for the storage backend
+    session : Optional[Session]
+        A session to use for this connection
+
+    Returns
+    -------
+    LanceNamespaceDBConnection
+        A namespace-based connection to LanceDB
+    """
+    namespace = namespace_connect(impl, properties)
+
+    # Return the namespace-based connection
+    return LanceNamespaceDBConnection(
+        namespace,
+        read_consistency_interval=read_consistency_interval,
+        storage_options=storage_options,
+        session=session,
+    )

python/python/lancedb/query.py

@@ -28,6 +28,7 @@ import pyarrow.fs as pa_fs
 import pydantic
 
 from lancedb.pydantic import PYDANTIC_VERSION
+from lancedb.background_loop import LOOP
 
 from . import __version__
 from .arrow import AsyncRecordBatchReader
@@ -48,6 +49,7 @@ if TYPE_CHECKING:
     from ._lancedb import FTSQuery as LanceFTSQuery
     from ._lancedb import HybridQuery as LanceHybridQuery
     from ._lancedb import VectorQuery as LanceVectorQuery
+    from ._lancedb import TakeQuery as LanceTakeQuery
     from ._lancedb import PyQueryRequest
     from .common import VEC
     from .pydantic import LanceModel
@@ -910,7 +912,7 @@ class LanceQueryBuilder(ABC):
         ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
           GlobalLimitExec: skip=0, fetch=10
             FilterExec: _distance@2 IS NOT NULL
-              SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
+              SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
                 KNNVectorDistance: metric=l2
                   LanceRead: uri=..., projection=[vector], ...
 
@@ -941,20 +943,22 @@ class LanceQueryBuilder(ABC):
         >>> query = [100, 100]
         >>> plan = table.search(query).analyze_plan()
         >>> print(plan)  # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
-        AnalyzeExec verbose=true, metrics=[]
-          TracedExec, metrics=[]
-            ProjectionExec: expr=[...], metrics=[...]
-              GlobalLimitExec: skip=0, fetch=10, metrics=[...]
+        AnalyzeExec verbose=true, metrics=[], cumulative_cpu=...
+          TracedExec, metrics=[], cumulative_cpu=...
+            ProjectionExec: expr=[...], metrics=[...], cumulative_cpu=...
+              GlobalLimitExec: skip=0, fetch=10, metrics=[...], cumulative_cpu=...
                 FilterExec: _distance@2 IS NOT NULL,
-                metrics=[output_rows=..., elapsed_compute=...]
+                metrics=[output_rows=..., elapsed_compute=...], cumulative_cpu=...
                   SortExec: TopK(fetch=10), expr=[...],
                   preserve_partitioning=[...],
-                  metrics=[output_rows=..., elapsed_compute=..., row_replacements=...]
+                  metrics=[output_rows=..., elapsed_compute=..., row_replacements=...],
+                  cumulative_cpu=...
                     KNNVectorDistance: metric=l2,
-                    metrics=[output_rows=..., elapsed_compute=..., output_batches=...]
+                    metrics=[output_rows=..., elapsed_compute=..., output_batches=...],
+                    cumulative_cpu=...
                       LanceRead: uri=..., projection=[vector], ...
                       metrics=[output_rows=..., elapsed_compute=...,
-                      bytes_read=..., iops=..., requests=...]
+                      bytes_read=..., iops=..., requests=...], cumulative_cpu=...
 
         Returns
         -------
@@ -2041,11 +2045,11 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         >>> plan = table.search(query).explain_plan(True)
         >>> print(plan) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
         ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
-        GlobalLimitExec: skip=0, fetch=10
-          FilterExec: _distance@2 IS NOT NULL
-            SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
-              KNNVectorDistance: metric=l2
-                LanceRead: uri=..., projection=[vector], ...
+          GlobalLimitExec: skip=0, fetch=10
+            FilterExec: _distance@2 IS NOT NULL
+              SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
+                KNNVectorDistance: metric=l2
+                  LanceRead: uri=..., projection=[vector], ...
 
         Parameters
         ----------
@@ -2139,7 +2143,11 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
 
 
 class AsyncQueryBase(object):
-    def __init__(self, inner: Union[LanceQuery, LanceVectorQuery]):
+    """
+    Base class for all async queries (take, scan, vector, fts, hybrid)
+    """
+
+    def __init__(self, inner: Union[LanceQuery, LanceVectorQuery, LanceTakeQuery]):
         """
         Construct an AsyncQueryBase
 
@@ -2149,27 +2157,14 @@ class AsyncQueryBase(object):
         self._inner = inner
 
     def to_query_object(self) -> Query:
+        """
+        Convert the query into a query object
+
+        This is currently experimental but can be useful as the query object is pure
+        python and more easily serializable.
+        """
         return Query.from_inner(self._inner.to_query_request())
 
-    def where(self, predicate: str) -> Self:
-        """
-        Only return rows matching the given predicate
-
-        The predicate should be supplied as an SQL query string.
-
-        Examples
-        --------
-
-        >>> predicate = "x > 10"
-        >>> predicate = "y > 0 AND y < 100"
-        >>> predicate = "x > 5 OR y = 'test'"
-
-        Filtering performance can often be improved by creating a scalar index
-        on the filter column(s).
-        """
-        self._inner.where(predicate)
-        return self
-
     def select(self, columns: Union[List[str], dict[str, str]]) -> Self:
         """
         Return only the specified columns.
@@ -2208,42 +2203,6 @@ class AsyncQueryBase(object):
             raise TypeError("columns must be a list of column names or a dict")
         return self
 
-    def limit(self, limit: int) -> Self:
-        """
-        Set the maximum number of results to return.
-
-        By default, a plain search has no limit.  If this method is not
-        called then every valid row from the table will be returned.
-        """
-        self._inner.limit(limit)
-        return self
-
-    def offset(self, offset: int) -> Self:
-        """
-        Set the offset for the results.
-
-        Parameters
-        ----------
-        offset: int
-            The offset to start fetching results from.
-        """
-        self._inner.offset(offset)
-        return self
-
-    def fast_search(self) -> Self:
-        """
-        Skip searching un-indexed data.
-
-        This can make queries faster, but will miss any data that has not been
-        indexed.
-
-        !!! tip
-            You can add new data into an existing index by calling
-            [AsyncTable.optimize][lancedb.table.AsyncTable.optimize].
-        """
-        self._inner.fast_search()
-        return self
-
     def with_row_id(self) -> Self:
         """
         Include the _rowid column in the results.
@@ -2251,27 +2210,6 @@ class AsyncQueryBase(object):
         self._inner.with_row_id()
         return self
 
-    def postfilter(self) -> Self:
-        """
-        If this is called then filtering will happen after the search instead of
-        before.
-        By default filtering will be performed before the search.  This is how
-        filtering is typically understood to work.  This prefilter step does add some
-        additional latency.  Creating a scalar index on the filter column(s) can
-        often improve this latency.  However, sometimes a filter is too complex or
-        scalar indices cannot be applied to the column.  In these cases postfiltering
-        can be used instead of prefiltering to improve latency.
-        Post filtering applies the filter to the results of the search.  This
-        means we only run the filter on a much smaller set of data.  However, it can
-        cause the query to return fewer than `limit` results (or even no results) if
-        none of the nearest results match the filter.
-        Post filtering happens during the "refine stage" (described in more detail in
-        @see {@link VectorQuery#refineFactor}).  This means that setting a higher refine
-        factor can often help restore some of the results lost by post filtering.
-        """
-        self._inner.postfilter()
-        return self
-
     async def to_batches(
         self,
         *,
@@ -2295,7 +2233,9 @@ class AsyncQueryBase(object):
             complete within the specified time, an error will be raised.
         """
         return AsyncRecordBatchReader(
-            await self._inner.execute(max_batch_length, timeout)
+            await self._inner.execute(
+                max_batch_length=max_batch_length, timeout=timeout
+            )
         )
 
     async def to_arrow(self, timeout: Optional[timedelta] = None) -> pa.Table:
@@ -2429,7 +2369,7 @@ class AsyncQueryBase(object):
         ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
           GlobalLimitExec: skip=0, fetch=10
             FilterExec: _distance@2 IS NOT NULL
-              SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
+              SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
                 KNNVectorDistance: metric=l2
                   LanceRead: uri=..., projection=[vector], ...
 
@@ -2454,7 +2394,98 @@ class AsyncQueryBase(object):
         return await self._inner.analyze_plan()
 
 
-class AsyncQuery(AsyncQueryBase):
+class AsyncStandardQuery(AsyncQueryBase):
+    """
+    Base class for "standard" async queries (all but take currently)
+    """
+
+    def __init__(self, inner: Union[LanceQuery, LanceVectorQuery]):
+        """
+        Construct an AsyncStandardQuery
+
+        This method is not intended to be called directly.  Instead, use the
+        [AsyncTable.query][lancedb.table.AsyncTable.query] method to create a query.
+        """
+        super().__init__(inner)
+
+    def where(self, predicate: str) -> Self:
+        """
+        Only return rows matching the given predicate
+
+        The predicate should be supplied as an SQL query string.
+
+        Examples
+        --------
+
+        >>> predicate = "x > 10"
+        >>> predicate = "y > 0 AND y < 100"
+        >>> predicate = "x > 5 OR y = 'test'"
+
+        Filtering performance can often be improved by creating a scalar index
+        on the filter column(s).
+        """
+        self._inner.where(predicate)
+        return self
+
+    def limit(self, limit: int) -> Self:
+        """
+        Set the maximum number of results to return.
+
+        By default, a plain search has no limit.  If this method is not
+        called then every valid row from the table will be returned.
+        """
+        self._inner.limit(limit)
+        return self
+
+    def offset(self, offset: int) -> Self:
+        """
+        Set the offset for the results.
+
+        Parameters
+        ----------
+        offset: int
+            The offset to start fetching results from.
+        """
+        self._inner.offset(offset)
+        return self
+
+    def fast_search(self) -> Self:
+        """
+        Skip searching un-indexed data.
+
+        This can make queries faster, but will miss any data that has not been
+        indexed.
+
+        !!! tip
+            You can add new data into an existing index by calling
+            [AsyncTable.optimize][lancedb.table.AsyncTable.optimize].
+        """
+        self._inner.fast_search()
+        return self
+
+    def postfilter(self) -> Self:
+        """
+        If this is called then filtering will happen after the search instead of
+        before.
+        By default filtering will be performed before the search.  This is how
+        filtering is typically understood to work.  This prefilter step does add some
+        additional latency.  Creating a scalar index on the filter column(s) can
+        often improve this latency.  However, sometimes a filter is too complex or
+        scalar indices cannot be applied to the column.  In these cases postfiltering
+        can be used instead of prefiltering to improve latency.
+        Post filtering applies the filter to the results of the search.  This
+        means we only run the filter on a much smaller set of data.  However, it can
+        cause the query to return fewer than `limit` results (or even no results) if
+        none of the nearest results match the filter.
+        Post filtering happens during the "refine stage" (described in more detail in
+        @see {@link VectorQuery#refineFactor}).  This means that setting a higher refine
+        factor can often help restore some of the results lost by post filtering.
+        """
+        self._inner.postfilter()
+        return self
+
+
+class AsyncQuery(AsyncStandardQuery):
     def __init__(self, inner: LanceQuery):
         """
         Construct an AsyncQuery
@@ -2588,7 +2619,7 @@ class AsyncQuery(AsyncQueryBase):
         return AsyncFTSQuery(self._inner.nearest_to_text({"query": query}))
 
 
-class AsyncFTSQuery(AsyncQueryBase):
+class AsyncFTSQuery(AsyncStandardQuery):
     """A query for full text search for LanceDB."""
 
     def __init__(self, inner: LanceFTSQuery):
@@ -2867,7 +2898,7 @@ class AsyncVectorQueryBase:
         return self
 
 
-class AsyncVectorQuery(AsyncQueryBase, AsyncVectorQueryBase):
+class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase):
     def __init__(self, inner: LanceVectorQuery):
         """
         Construct an AsyncVectorQuery
@@ -2950,7 +2981,7 @@ class AsyncVectorQuery(AsyncQueryBase, AsyncVectorQueryBase):
         return AsyncRecordBatchReader(results, max_batch_length=max_batch_length)
 
 
-class AsyncHybridQuery(AsyncQueryBase, AsyncVectorQueryBase):
+class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
     """
     A query builder that performs hybrid vector and full text search.
     Results are combined and reranked based on the specified reranker.
@@ -3054,7 +3085,7 @@ class AsyncHybridQuery(AsyncQueryBase, AsyncVectorQueryBase):
             CoalesceBatchesExec: target_batch_size=1024
               GlobalLimitExec: skip=0, fetch=10
                 FilterExec: _distance@2 IS NOT NULL
-                  SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
+                  SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
                     KNNVectorDistance: metric=l2
                       LanceRead: uri=..., projection=[vector], ...
         <BLANKLINE>
@@ -3102,3 +3133,252 @@ class AsyncHybridQuery(AsyncQueryBase, AsyncVectorQueryBase):
         results.append(await self._inner.to_fts_query().analyze_plan())
 
         return "\n".join(results)
+
+
+class AsyncTakeQuery(AsyncQueryBase):
+    """
+    Builder for parameterizing and executing take queries.
+    """
+
+    def __init__(self, inner: LanceTakeQuery):
+        super().__init__(inner)
+
+
+class BaseQueryBuilder(object):
+    """
+    Wraps AsyncQueryBase and provides a synchronous interface
+    """
+
+    def __init__(self, inner: AsyncQueryBase):
+        self._inner = inner
+
+    def to_query_object(self) -> Query:
+        return self._inner.to_query_object()
+
+    def select(self, columns: Union[List[str], dict[str, str]]) -> Self:
+        """
+        Return only the specified columns.
+
+        By default a query will return all columns from the table.  However, this can
+        have a very significant impact on latency.  LanceDb stores data in a columnar
+        fashion.  This
+        means we can finely tune our I/O to select exactly the columns we need.
+
+        As a best practice you should always limit queries to the columns that you need.
+        If you pass in a list of column names then only those columns will be
+        returned.
+
+        You can also use this method to create new "dynamic" columns based on your
+        existing columns. For example, you may not care about "a" or "b" but instead
+        simply want "a + b".  This is often seen in the SELECT clause of an SQL query
+        (e.g. `SELECT a+b FROM my_table`).
+
+        To create dynamic columns you can pass in a dict[str, str].  A column will be
+        returned for each entry in the map.  The key provides the name of the column.
+        The value is an SQL string used to specify how the column is calculated.
+
+        For example, an SQL query might state `SELECT a + b AS combined, c`.  The
+        equivalent input to this method would be `{"combined": "a + b", "c": "c"}`.
+
+        Columns will always be returned in the order given, even if that order is
+        different than the order used when adding the data.
+        """
+        self._inner.select(columns)
+        return self
+
+    def with_row_id(self) -> Self:
+        """
+        Include the _rowid column in the results.
+        """
+        self._inner.with_row_id()
+        return self
+
+    def to_batches(
+        self,
+        *,
+        max_batch_length: Optional[int] = None,
+        timeout: Optional[timedelta] = None,
+    ) -> pa.RecordBatchReader:
+        """
+        Execute the query and return the results as an Apache Arrow RecordBatchReader.
+
+        Parameters
+        ----------
+
+        max_batch_length: Optional[int]
+            The maximum number of selected records in a single RecordBatch object.
+            If not specified, a default batch length is used.
+            It is possible for batches to be smaller than the provided length if the
+            underlying data is stored in smaller chunks.
+        timeout: Optional[timedelta]
+            The maximum time to wait for the query to complete.
+            If not specified, no timeout is applied. If the query does not
+            complete within the specified time, an error will be raised.
+        """
+        async_iter = LOOP.run(self._inner.execute(max_batch_length, timeout))
+
+        def iter_sync():
+            try:
+                while True:
+                    yield LOOP.run(async_iter.__anext__())
+            except StopAsyncIteration:
+                return
+
+        return pa.RecordBatchReader.from_batches(async_iter.schema, iter_sync())
+
+    def to_arrow(self, timeout: Optional[timedelta] = None) -> pa.Table:
+        """
+        Execute the query and collect the results into an Apache Arrow Table.
+
+        This method will collect all results into memory before returning.  If
+        you expect a large number of results, you may want to use
+        [to_batches][lancedb.query.AsyncQueryBase.to_batches]
+
+        Parameters
+        ----------
+        timeout: Optional[timedelta]
+            The maximum time to wait for the query to complete.
+            If not specified, no timeout is applied. If the query does not
+            complete within the specified time, an error will be raised.
+        """
+        return LOOP.run(self._inner.to_arrow(timeout))
+
+    def to_list(self, timeout: Optional[timedelta] = None) -> List[dict]:
+        """
+        Execute the query and return the results as a list of dictionaries.
+
+        Each list entry is a dictionary with the selected column names as keys,
+        or all table columns if `select` is not called. The vector and the "_distance"
+        fields are returned whether or not they're explicitly selected.
+
+        Parameters
+        ----------
+        timeout: Optional[timedelta]
+            The maximum time to wait for the query to complete.
+            If not specified, no timeout is applied. If the query does not
+            complete within the specified time, an error will be raised.
+        """
+        return LOOP.run(self._inner.to_list(timeout))
+
+    def to_pandas(
+        self,
+        flatten: Optional[Union[int, bool]] = None,
+        timeout: Optional[timedelta] = None,
+    ) -> "pd.DataFrame":
+        """
+        Execute the query and collect the results into a pandas DataFrame.
+
+        This method will collect all results into memory before returning.  If you
+        expect a large number of results, you may want to use
+        [to_batches][lancedb.query.AsyncQueryBase.to_batches] and convert each batch to
+        pandas separately.
+
+        Examples
+        --------
+
+        >>> import asyncio
+        >>> from lancedb import connect_async
+        >>> async def doctest_example():
+        ...     conn = await connect_async("./.lancedb")
+        ...     table = await conn.create_table("my_table", data=[{"a": 1, "b": 2}])
+        ...     async for batch in await table.query().to_batches():
+        ...         batch_df = batch.to_pandas()
+        >>> asyncio.run(doctest_example())
+
+        Parameters
+        ----------
+        flatten: Optional[Union[int, bool]]
+            If flatten is True, flatten all nested columns.
+            If flatten is an integer, flatten the nested columns up to the
+            specified depth.
+            If unspecified, do not flatten the nested columns.
+        timeout: Optional[timedelta]
+            The maximum time to wait for the query to complete.
+            If not specified, no timeout is applied. If the query does not
+            complete within the specified time, an error will be raised.
+        """
+        return LOOP.run(self._inner.to_pandas(flatten, timeout))
+
+    def to_polars(
+        self,
+        timeout: Optional[timedelta] = None,
+    ) -> "pl.DataFrame":
+        """
+        Execute the query and collect the results into a Polars DataFrame.
+
+        This method will collect all results into memory before returning.  If you
+        expect a large number of results, you may want to use
+        [to_batches][lancedb.query.AsyncQueryBase.to_batches] and convert each batch to
+        polars separately.
+
+        Parameters
+        ----------
+        timeout: Optional[timedelta]
+            The maximum time to wait for the query to complete.
+            If not specified, no timeout is applied. If the query does not
+            complete within the specified time, an error will be raised.
+
+        Examples
+        --------
+
+        >>> import asyncio
+        >>> import polars as pl
+        >>> from lancedb import connect_async
+        >>> async def doctest_example():
+        ...     conn = await connect_async("./.lancedb")
+        ...     table = await conn.create_table("my_table", data=[{"a": 1, "b": 2}])
+        ...     async for batch in await table.query().to_batches():
+        ...         batch_df = pl.from_arrow(batch)
+        >>> asyncio.run(doctest_example())
+        """
+        return LOOP.run(self._inner.to_polars(timeout))
+
+    def explain_plan(self, verbose: Optional[bool] = False):
+        """Return the execution plan for this query.
+
+        Examples
+        --------
+        >>> import asyncio
+        >>> from lancedb import connect_async
+        >>> async def doctest_example():
+        ...     conn = await connect_async("./.lancedb")
+        ...     table = await conn.create_table("my_table", [{"vector": [99, 99]}])
+        ...     query = [100, 100]
+        ...     plan = await table.query().nearest_to([1, 2]).explain_plan(True)
+        ...     print(plan)
+        >>> asyncio.run(doctest_example()) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
+        ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
+          GlobalLimitExec: skip=0, fetch=10
+            FilterExec: _distance@2 IS NOT NULL
+              SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
+                KNNVectorDistance: metric=l2
+                  LanceRead: uri=..., projection=[vector], ...
+
+        Parameters
+        ----------
+        verbose : bool, default False
+            Use a verbose output format.
+
+        Returns
+        -------
+        plan : str
+        """  # noqa: E501
+        return LOOP.run(self._inner.explain_plan(verbose))
+
+    def analyze_plan(self):
+        """Execute the query and display with runtime metrics.
+
+        Returns
+        -------
+        plan : str
+        """
+        return LOOP.run(self._inner.analyze_plan())
+
+
+class LanceTakeQueryBuilder(BaseQueryBuilder):
+    """
+    Builder for parameterizing and executing take queries.
+    """
+
+    def __init__(self, inner: AsyncTakeQuery):
+        super().__init__(inner)

python/python/lancedb/remote/__init__.py

@@ -8,7 +8,15 @@ from typing import List, Optional
 
 from lancedb import __version__
 
-__all__ = ["TimeoutConfig", "RetryConfig", "ClientConfig"]
+from .header import HeaderProvider
+
+__all__ = [
+    "TimeoutConfig",
+    "RetryConfig",
+    "TlsConfig",
+    "ClientConfig",
+    "HeaderProvider",
+]
 
 
 @dataclass
@@ -112,15 +120,43 @@ class RetryConfig:
     statuses: Optional[List[int]] = None
 
 
+@dataclass
+class TlsConfig:
+    """TLS/mTLS configuration for the remote HTTP client.
+
+    Attributes
+    ----------
+    cert_file: Optional[str]
+        Path to the client certificate file (PEM format) for mTLS authentication.
+    key_file: Optional[str]
+        Path to the client private key file (PEM format) for mTLS authentication.
+    ssl_ca_cert: Optional[str]
+        Path to the CA certificate file (PEM format) for server verification.
+    assert_hostname: bool
+        Whether to verify the hostname in the server's certificate. Default is True.
+        Set to False to disable hostname verification (use with caution).
+    """
+
+    cert_file: Optional[str] = None
+    key_file: Optional[str] = None
+    ssl_ca_cert: Optional[str] = None
+    assert_hostname: bool = True
+
+
 @dataclass
 class ClientConfig:
     user_agent: str = f"LanceDB-Python-Client/{__version__}"
     retry_config: RetryConfig = field(default_factory=RetryConfig)
     timeout_config: Optional[TimeoutConfig] = field(default_factory=TimeoutConfig)
     extra_headers: Optional[dict] = None
+    id_delimiter: Optional[str] = None
+    tls_config: Optional[TlsConfig] = None
+    header_provider: Optional["HeaderProvider"] = None
 
     def __post_init__(self):
         if isinstance(self.retry_config, dict):
             self.retry_config = RetryConfig(**self.retry_config)
         if isinstance(self.timeout_config, dict):
             self.timeout_config = TimeoutConfig(**self.timeout_config)
+        if isinstance(self.tls_config, dict):
+            self.tls_config = TlsConfig(**self.tls_config)