Bump version: 0.29.0 → 0.29.1-beta.0

## Summary
- Bump Lance Rust workspace dependencies to `v7.0.0-beta.13` using
`ci/set_lance_version.py`.
- Update the Java `lance-core` dependency property to `7.0.0-beta.13`.
- Triggering tag:
https://github.com/lance-format/lance/releases/tag/v7.0.0-beta.13

## Verification
- `cargo clippy --workspace --tests --all-features -- -D warnings`
- `cargo fmt --all`

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.28.0-beta.2"
+current_version = "0.29.1-beta.0"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/ISSUE_TEMPLATE/documentation.yml

@@ -18,6 +18,6 @@ body:
       label: Link
       description: >
         Provide a link to the existing documentation, if applicable.
-      placeholder: ex. https://lancedb.com/docs/tables/...
+      placeholder: ex. https://docs.lancedb.com/tables/...
     validations:
       required: false

.github/dependabot.yml

@@ -0,0 +1,18 @@
+version: 2
+
+# Scope: the root Cargo workspace, which produces the Rust binaries we
+# ship to users (the Node.js and Python native extensions). The
+# `rust/lancedb` library crate shares the same lockfile; its consumers
+# pick their own dependency versions, but bumping transitive deps here
+# keeps the binaries we ship current.
+updates:
+  - package-ecosystem: cargo
+    directory: /
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 10
+    groups:
+      rust-minor-patch:
+        update-types:
+          - minor
+          - patch

.github/workflows/codex-fix-ci.yml

@@ -45,7 +45,9 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          # pnpm 11 (used by the nodejs install step below) requires
+          # Node >= 22.13; use 24 since 22 hits EOL in October.
+          node-version: 24
 
       - name: Install Codex CLI
         run: npm install -g @openai/codex
@@ -79,10 +81,14 @@ jobs:
           java-version: '11'
           cache: maven
 
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 11.1.1
       - name: Install Node.js dependencies for TypeScript bindings
         run: |
           cd nodejs
-          npm ci
+          pnpm install --frozen-lockfile
 
       - name: Configure git user
         run: |
@@ -137,7 +143,7 @@ jobs:
                - For Rust test failures: Run the specific test with "cargo test -p <crate> <test_name>"
                - For Python test failures: Build with "cd python && maturin develop" then run "pytest <specific_test_file>::<test_name>"
                - For Java test failures: Run "cd java && mvn test -Dtest=<TestClass>#<testMethod>"
-               - For TypeScript test failures: Run "cd nodejs && npm run build && npm test -- --testNamePattern='<test_name>'"
+               - For TypeScript test failures: Run "cd nodejs && pnpm build && pnpm test -- --testNamePattern='<test_name>'"
                - Do NOT run the full test suite - only run the tests that were failing
 
           7. If the additional guidelines are provided, follow them as well.

.github/workflows/dev.yml

@@ -8,6 +8,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   labeler:
     permissions:

.github/workflows/java-publish.yml

@@ -19,6 +19,9 @@ on:
     paths:
       - .github/workflows/java-publish.yml
 
+permissions:
+  contents: read
+
 jobs:
   publish:
     name: Build and Publish
@@ -40,7 +43,7 @@ jobs:
           server-username: SONATYPE_USER
           server-password: SONATYPE_TOKEN
           gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
-          gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }}
+          gpg-passphrase: MAVEN_GPG_PASSPHRASE
       - name: Set git config
         run: |
           git config --global user.email "dev+gha@lancedb.com"
@@ -55,10 +58,11 @@ jobs:
           echo "use-agent" >> ~/.gnupg/gpg.conf
           echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
           export GPG_TTY=$(tty)
-          ./mvnw --batch-mode -DskipTests -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -pl lancedb-core -am -P deploy-to-ossrh
+          ./mvnw --batch-mode -DskipTests -DpushChanges=false deploy -pl lancedb-core -am -P deploy-to-ossrh
         env:
           SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
           SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}
+          MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
 
   report-failure:
     name: Report Workflow Failure

.github/workflows/java.yml

@@ -24,6 +24,9 @@ on:
       - java/**
       - .github/workflows/java.yml
 
+permissions:
+  contents: read
+
 jobs:
   build-java:
     runs-on: ubuntu-24.04

.github/workflows/license-header-check.yml

@@ -10,6 +10,10 @@ on:
       - nodejs/**
       - java/**
       - .github/workflows/license-header-check.yml
+
+permissions:
+  contents: read
+
 jobs:
   check-licenses:
     runs-on: ubuntu-latest

.github/workflows/nodejs.yml

@@ -15,6 +15,9 @@ on:
       - .github/workflows/nodejs.yml
       - docker-compose.yml
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -39,11 +42,17 @@ jobs:
       with:
         fetch-depth: 0
         lfs: true
+    - uses: pnpm/action-setup@v4
+      with:
+        version: 11.1.1
     - uses: actions/setup-node@v4
       with:
-        node-version: 20
-        cache: 'npm'
-        cache-dependency-path: nodejs/package-lock.json
+        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # in October. The library itself still supports Node >= 18
+        # (see test matrix below).
+        node-version: 24
+        cache: 'pnpm'
+        cache-dependency-path: nodejs/pnpm-lock.yaml
     - uses: actions-rust-lang/setup-rust-toolchain@v1
       with:
         components: rustfmt, clippy
@@ -58,11 +67,13 @@ jobs:
       run: cargo clippy --profile ci --all --all-features -- -D warnings
     - name: Lint Typescript
       run: |
-        npm ci
-        npm run lint-ci
+        pnpm install --frozen-lockfile
+        pnpm lint-ci
     - name: Lint examples
       working-directory: nodejs/examples
-      run: npm ci && npm run lint-ci
+      # The `@lancedb/lancedb` dep points at file:../dist; pnpm errors if
+      # that dir is missing, so create an empty one for lint-only runs.
+      run: mkdir -p ../dist && pnpm install --frozen-lockfile && pnpm lint-ci
   linux:
     name: Linux (NodeJS ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -79,14 +90,18 @@ jobs:
       with:
         fetch-depth: 0
         lfs: true
-    - uses: actions/setup-node@v4
-      name: Setup Node.js 20 for build
+    - uses: pnpm/action-setup@v4
       with:
-        # @napi-rs/cli v3 requires Node >= 20.12 (via @inquirer/prompts@8).
-        # Build always on Node 20; tests run on the matrix version below.
-        node-version: 20
-        cache: 'npm'
-        cache-dependency-path: nodejs/package-lock.json
+        version: 11.1.1
+    - uses: actions/setup-node@v4
+      name: Setup Node.js 24 for build
+      with:
+        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # in October. Build/install runs on Node 24; tests run on the
+        # matrix version below using direct jest invocation.
+        node-version: 24
+        cache: 'pnpm'
+        cache-dependency-path: nodejs/pnpm-lock.yaml
     - uses: Swatinem/rust-cache@v2
     - name: Install dependencies
       run: |
@@ -94,45 +109,52 @@ jobs:
         sudo apt install -y protobuf-compiler libssl-dev
     - name: Build
       run: |
-        npm ci --include=optional
-        npm run build:debug -- --profile ci
+        pnpm install --frozen-lockfile
+        # No `--` separator: pnpm forwards it literally, which would
+        # make napi-rs treat `--profile ci` as a cargo passthrough arg.
+        pnpm build:debug --profile ci
+        pnpm tsc
+    - name: Setup examples
+      working-directory: nodejs/examples
+      run: pnpm install --frozen-lockfile
+    - name: Check docs
+      run: |
+        # We run this as part of the job because the binary needs to be built
+        # first to export the types of the native code.
+        set -e
+        # `pnpm docs` would invoke pnpm's built-in `docs` command, not
+        # the script — use `pnpm run docs`.
+        pnpm run docs
+        if ! git diff --exit-code -- ../ ':(exclude)Cargo.lock'; then
+          echo "Docs need to be updated"
+          echo "Run 'pnpm run docs', fix any warnings, and commit the changes."
+          exit 1
+        fi
     - uses: actions/setup-node@v4
       name: Setup Node.js ${{ matrix.node-version }} for test
       with:
         node-version: ${{ matrix.node-version }}
-    - name: Compile TypeScript
-      run: npm run tsc
     - name: Setup localstack
       working-directory: .
       run: docker compose up --detach --wait
     - name: Test
       env:
         S3_TEST: "1"
-      run: npm run test
-    - name: Setup examples
-      working-directory: nodejs/examples
-      run: npm ci
+        # Newer @smithy/core uses dynamic ESM imports.
+        NODE_OPTIONS: "--experimental-vm-modules"
+      # Invoke jest directly because pnpm 11 itself requires Node 22+
+      # while the matrix tests on older Node versions.
+      run: npx jest --verbose
     - name: Test examples
       working-directory: ./
       env:
         OPENAI_API_KEY: test
         OPENAI_BASE_URL: http://0.0.0.0:8000
+        NODE_OPTIONS: "--experimental-vm-modules"
       run: |
         python ci/mock_openai.py &
         cd nodejs/examples
-        npm test
-    - name: Check docs
-      run: |
-        # We run this as part of the job because the binary needs to be built
-        # first to export the types of the native code.
-        set -e
-        npm ci
-        npm run docs
-        if ! git diff --exit-code -- ../ ':(exclude)Cargo.lock'; then
-          echo "Docs need to be updated"
-          echo "Run 'npm run docs', fix any warnings, and commit the changes."
-          exit 1
-        fi
+        npx jest --testEnvironment jest-environment-node-single-context --verbose
   macos:
     timeout-minutes: 30
     runs-on: "macos-14"
@@ -145,20 +167,28 @@ jobs:
       with:
         fetch-depth: 0
         lfs: true
+    - uses: pnpm/action-setup@v4
+      with:
+        version: 11.1.1
     - uses: actions/setup-node@v4
       with:
-        node-version: 20
-        cache: 'npm'
-        cache-dependency-path: nodejs/package-lock.json
+        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # in October.
+        node-version: 24
+        cache: 'pnpm'
+        cache-dependency-path: nodejs/pnpm-lock.yaml
+    - uses: dtolnay/rust-toolchain@stable
     - uses: Swatinem/rust-cache@v2
     - name: Install dependencies
       run: |
         brew install protobuf
     - name: Build
       run: |
-        npm ci --include=optional
-        npm run build:debug -- --profile ci
-        npm run tsc
+        pnpm install --frozen-lockfile
+        # No `--` separator: pnpm forwards it literally, which would
+        # make napi-rs treat `--profile ci` as a cargo passthrough arg.
+        pnpm build:debug --profile ci
+        pnpm tsc
     - name: Test
       run: |
-        npm run test
+        pnpm test

.github/workflows/npm-publish.yml

@@ -171,13 +171,18 @@ jobs:
         working-directory: nodejs
     steps:
       - uses: actions/checkout@v4
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 11.1.1
       - name: Setup node
         uses: actions/setup-node@v4
-        if: ${{ !matrix.settings.docker }}
         with:
-          node-version: 20
-          cache: npm
-          cache-dependency-path: nodejs/package-lock.json
+          # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+          # in October.
+          node-version: 24
+          cache: pnpm
+          cache-dependency-path: nodejs/pnpm-lock.yaml
       - name: Install
         uses: dtolnay/rust-toolchain@stable
         if: ${{ !matrix.settings.docker }}
@@ -195,7 +200,7 @@ jobs:
             target/
           key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
       - name: Install dependencies
-        run: npm ci
+        run: pnpm install --frozen-lockfile
       - name: Install Zig
         uses: mlugg/setup-zig@v2
         if: ${{ contains(matrix.settings.target, 'musl') }}
@@ -248,7 +253,7 @@ jobs:
       # one to do the upload.
       - name: Make generic artifacts
         if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
-        run: npm run tsc
+        run: pnpm tsc
       - name: Upload Generic Artifacts
         if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
         uses: actions/upload-artifact@v4
@@ -283,14 +288,24 @@ jobs:
         working-directory: nodejs
     steps:
       - uses: actions/checkout@v4
-      - name: Setup node
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 11.1.1
+      - name: Setup Node.js 24 for install
+        uses: actions/setup-node@v4
+        with:
+          # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+          # in October.
+          node-version: 24
+          cache: pnpm
+          cache-dependency-path: nodejs/pnpm-lock.yaml
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: Setup Node.js ${{ matrix.node }} for test
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
-          cache: npm
-          cache-dependency-path: nodejs/package-lock.json
-      - name: Install dependencies
-        run: npm ci
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
@@ -311,7 +326,9 @@ jobs:
       - name: Move built files
         run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
       - name: Test bindings
-        run: npm test
+        # Invoke jest directly because pnpm 11 itself requires Node 22+
+        # while the matrix tests on older Node versions.
+        run: npx jest --verbose
   publish:
     name: Publish
     runs-on: ubuntu-latest
@@ -323,15 +340,19 @@ jobs:
       - test-lancedb
     steps:
       - uses: actions/checkout@v4
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 11.1.1
       - name: Setup node
         uses: actions/setup-node@v4
         with:
           node-version: 24
-          cache: npm
-          cache-dependency-path: nodejs/package-lock.json
+          cache: pnpm
+          cache-dependency-path: nodejs/pnpm-lock.yaml
           registry-url: "https://registry.npmjs.org"
       - name: Install dependencies
-        run: npm ci
+        run: pnpm install --frozen-lockfile
       - uses: actions/download-artifact@v4
         with:
           name: nodejs-dist
@@ -351,7 +372,7 @@ jobs:
       - name: Display structure of downloaded files
         run: find dist && find nodejs-artifacts
       - name: Move artifacts
-        run: npx napi artifacts -d nodejs-artifacts
+        run: pnpm exec napi artifacts -d nodejs-artifacts
       - name: List packages
         run: find npm
       - name: Publish

.github/workflows/pypi-publish.yml

@@ -14,10 +14,16 @@ on:
 env:
   PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
 
+permissions:
+  contents: read
+
 jobs:
   linux:
     name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
     timeout-minutes: 60
+    permissions:
+      id-token: write
+      contents: read
     strategy:
       matrix:
         config:
@@ -57,10 +63,12 @@ jobs:
       - uses: ./.github/workflows/upload_wheel
         if: startsWith(github.ref, 'refs/tags/python-v')
         with:
-          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           fury_token: ${{ secrets.FURY_TOKEN }}
   mac:
     timeout-minutes: 90
+    permissions:
+      id-token: write
+      contents: read
     runs-on: ${{ matrix.config.runner }}
     strategy:
       matrix:
@@ -85,10 +93,12 @@ jobs:
       - uses: ./.github/workflows/upload_wheel
         if: startsWith(github.ref, 'refs/tags/python-v')
         with:
-          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           fury_token: ${{ secrets.FURY_TOKEN }}
   windows:
     timeout-minutes: 60
+    permissions:
+      id-token: write
+      contents: read
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v4
@@ -107,7 +117,6 @@ jobs:
       - uses: ./.github/workflows/upload_wheel
         if: startsWith(github.ref, 'refs/tags/python-v')
         with:
-          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           fury_token: ${{ secrets.FURY_TOKEN }}
   gh-release:
     if: startsWith(github.ref, 'refs/tags/python-v')

.github/workflows/python.yml

@@ -17,6 +17,9 @@ on:
       - .github/workflows/build_windows_wheel/**
       - .github/workflows/run_tests/**
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -108,7 +111,6 @@ jobs:
       - name: Install
         run: |
           pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
-          pip install tantivy
           pip install mlx
       - name: Doctest
         run: pytest --doctest-modules python/lancedb
@@ -227,6 +229,5 @@ jobs:
           pip install "pydantic<2"
           pip install pyarrow==16
           pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
-          pip install tantivy
       - name: Run tests
         run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests

.github/workflows/rust.yml

@@ -9,9 +9,15 @@ on:
       - Cargo.toml
       - Cargo.lock
       - rust-toolchain.toml
+      - deny.toml
       - rust/**
+      - nodejs/Cargo.toml
+      - python/Cargo.toml
       - .github/workflows/rust.yml
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -53,6 +59,17 @@ jobs:
       - name: Run clippy (without remote feature)
         run: cargo clippy --profile ci --workspace --tests -- -D warnings
 
+  deny:
+    # Supply-chain checks: advisories, licenses, banned crates, and source
+    # restrictions. Configuration lives in `deny.toml` at the workspace root.
+    timeout-minutes: 10
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: EmbarkStudios/cargo-deny-action@v2
+        with:
+          command: check advisories bans licenses sources
+
   build-no-lock:
     runs-on: ubuntu-24.04
     timeout-minutes: 30

.github/workflows/update_package_lock_run.yml

@@ -3,6 +3,9 @@ name: Update package-lock.json
 on:
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   publish:
     runs-on: ubuntu-latest

.github/workflows/update_package_lock_run_nodejs.yml

@@ -3,6 +3,9 @@ name: Update NodeJs package-lock.json
 on:
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   publish:
     runs-on: ubuntu-latest

.github/workflows/upload_wheel/action.yml

@@ -2,9 +2,6 @@ name: upload-wheel
 
 description: "Upload wheels to Pypi"
 inputs:
-  pypi_token:
-    required: true
-    description: "release token for the repo"
   fury_token:
     required: true
     description: "release token for the fury repo"
@@ -12,12 +9,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-  - name: Install dependencies
-    shell: bash
-    run: |
-      python -m pip install --upgrade pip
-      pip install twine
-      python3 -m pip install --upgrade pkginfo
   - name: Choose repo
     shell: bash
     id: choose_repo
@@ -27,19 +18,17 @@ runs:
       else
         echo "repo=pypi" >> $GITHUB_OUTPUT
       fi
-  - name: Publish to PyPI
+  - name: Publish to Fury
+    if: steps.choose_repo.outputs.repo == 'fury'
     shell: bash
     env:
       FURY_TOKEN: ${{ inputs.fury_token }}
-      PYPI_TOKEN: ${{ inputs.pypi_token }}
     run: |
-      if [[ ${{ steps.choose_repo.outputs.repo }} == fury ]]; then
-        WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
-        echo "Uploading $WHEEL to Fury"
-        curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/
-      else
-        twine upload --repository ${{ steps.choose_repo.outputs.repo }} \
-          --username __token__ \
-          --password $PYPI_TOKEN \
-          target/wheels/lancedb-*.whl
-      fi
+      WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
+      echo "Uploading $WHEEL to Fury"
+      curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/
+  - name: Publish to PyPI
+    if: steps.choose_repo.outputs.repo == 'pypi'
+    uses: pypa/gh-action-pypi-publish@release/v1
+    with:
+      packages-dir: target/wheels/

Cargo.toml

@@ -1,7 +1,5 @@
 [workspace]
 members = ["rust/lancedb", "nodejs", "python"]
-# Python package needs to be built by maturin.
-exclude = ["python"]
 resolver = "2"
 
 [workspace.package]
@@ -15,40 +13,40 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=5.1.0-beta.2", default-features = false, "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=5.1.0-beta.2", default-features = false, "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=5.1.0-beta.2", default-features = false, "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=7.0.0-beta.13", default-features = false, "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=7.0.0-beta.13", "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=7.0.0-beta.13", "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=7.0.0-beta.13", "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=7.0.0-beta.13", default-features = false, "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=7.0.0-beta.13", "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=7.0.0-beta.13", "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=7.0.0-beta.13", "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=7.0.0-beta.13", default-features = false, "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=7.0.0-beta.13", "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=7.0.0-beta.13", "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=7.0.0-beta.13", "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=7.0.0-beta.13", "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=7.0.0-beta.13", "tag" = "v7.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
-arrow = { version = "57.2", optional = false }
-arrow-array = "57.2"
-arrow-data = "57.2"
-arrow-ipc = "57.2"
-arrow-ord = "57.2"
-arrow-schema = "57.2"
-arrow-select = "57.2"
-arrow-cast = "57.2"
+arrow = { version = "58.0.0", optional = false }
+arrow-array = "58.0.0"
+arrow-data = "58.0.0"
+arrow-ipc = "58.0.0"
+arrow-ord = "58.0.0"
+arrow-schema = "58.0.0"
+arrow-select = "58.0.0"
+arrow-cast = "58.0.0"
 async-trait = "0"
-datafusion = { version = "52.1", default-features = false }
-datafusion-catalog = "52.1"
-datafusion-common = { version = "52.1", default-features = false }
-datafusion-execution = "52.1"
-datafusion-expr = "52.1"
-datafusion-functions = "52.1"
-datafusion-physical-plan = "52.1"
-datafusion-physical-expr = "52.1"
-datafusion-sql = "52.1"
+datafusion = { version = "53.0.0", default-features = false }
+datafusion-catalog = "53.0.0"
+datafusion-common = { version = "53.0.0", default-features = false }
+datafusion-execution = "53.0.0"
+datafusion-expr = "53.0.0"
+datafusion-functions = "53.0.0"
+datafusion-physical-plan = "53.0.0"
+datafusion-physical-expr = "53.0.0"
+datafusion-sql = "53.0.0"
 env_logger = "0.11"
 half = { "version" = "2.7.1", default-features = false, features = [
     "num-traits",
@@ -56,7 +54,7 @@ half = { "version" = "2.7.1", default-features = false, features = [
 futures = "0"
 log = "0.4"
 moka = { version = "0.12", features = ["future"] }
-object_store = "0.12.0"
+object_store = "0.13.2"
 pin-project = "1.0.7"
 rand = "0.9"
 snafu = "0.8"

README.md

@@ -15,7 +15,7 @@
 
 # **The Multimodal AI Lakehouse**
 
-[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://lancedb.com/docs) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦  [**Contributors**](#contributors) 
+[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://docs.lancedb.com) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦  [**Contributors**](#contributors) 
 
 **The ultimate multimodal data platform for AI/ML applications.** 
 
@@ -57,7 +57,7 @@ LanceDB is a central location where developers can build, train and analyze thei
 
 ## **How to Install**:
 
-Follow the [Quickstart](https://lancedb.com/docs/quickstart/) doc to set up LanceDB locally. 
+Follow the [Quickstart](https://docs.lancedb.com/quickstart) doc to set up LanceDB locally. 
 
 **API & SDK:** We also support Python, Typescript and Rust SDKs
 

deny.toml

@@ -0,0 +1,196 @@
+# cargo-deny configuration for LanceDB.
+#
+# Run locally with `cargo deny check`. See
+# https://embarkstudios.github.io/cargo-deny/ for the full reference.
+
+# The set of target triples we care about. cargo-deny will only consider
+# dependencies that are used on at least one of these targets. Keeping this
+# explicit avoids noise from platform-specific crates (e.g. wasm, android,
+# ios) that we never actually ship.
+[graph]
+targets = [
+    "x86_64-unknown-linux-gnu",
+    "aarch64-unknown-linux-gnu",
+    "x86_64-apple-darwin",
+    "aarch64-apple-darwin",
+    "x86_64-pc-windows-msvc",
+    "aarch64-pc-windows-msvc",
+]
+all-features = true
+
+[output]
+feature-depth = 1
+
+# ---------------------------------------------------------------------------
+# Advisories: security vulnerabilities and yanked crates.
+# ---------------------------------------------------------------------------
+[advisories]
+version = 2
+# Fail the check if any crate in the lockfile has been yanked from crates.io.
+# Yanked crates are a signal the author retracted the release (often due to
+# bugs or security issues) and should not be depended on.
+yanked = "deny"
+# Advisory IDs we have explicitly reviewed and chosen to accept. Every
+# entry must include a rationale and, where possible, an upstream issue
+# pointing to a fix. Revisit this list whenever dependencies are updated.
+ignore = [
+    # rsa: Marvin Attack timing side-channel in PKCS#1 v1.5 decryption.
+    # Reached only through opendal → reqsign → rsa. We do not use RSA
+    # decryption in LanceDB ourselves; this is dormant in the signing path.
+    # No fixed release exists upstream as of this writing.
+    # https://rustsec.org/advisories/RUSTSEC-2023-0071
+    { id = "RUSTSEC-2023-0071", reason = "rsa crate via opendal/reqsign; no fixed upstream release" },
+
+    # instant: unmaintained. Pulled in via backoff → instant. Upstream
+    # recommends switching to `web-time`; fix has to come from backoff.
+    # https://rustsec.org/advisories/RUSTSEC-2024-0384
+    { id = "RUSTSEC-2024-0384", reason = "transitive via backoff; waiting on backoff replacement" },
+
+    # paste: unmaintained (author archived the repo). Used transitively by
+    # datafusion and the arrow ecosystem; widespread, no drop-in replacement.
+    # https://rustsec.org/advisories/RUSTSEC-2024-0436
+    { id = "RUSTSEC-2024-0436", reason = "transitive via datafusion; awaiting ecosystem migration" },
+
+    # encoding: unmaintained. Reached through lindera-dictionary, which is
+    # required by the native Lindera tokenizer path. Lindera has not migrated
+    # off this crate yet.
+    # https://rustsec.org/advisories/RUSTSEC-2021-0153
+    { id = "RUSTSEC-2021-0153", reason = "transitive via lindera-dictionary for native Lindera tokenizer" },
+
+    # fast-float: unsound and unmaintained. Reached only through polars-arrow
+    # from the optional Polars integration; replacement requires a Polars
+    # dependency upgrade.
+    # https://rustsec.org/advisories/RUSTSEC-2024-0379
+    { id = "RUSTSEC-2024-0379", reason = "transitive via polars-arrow; waiting on Polars migration" },
+
+    # tantivy: segfault on malformed input due to missing bounds check.
+    # Pulled in via lance for full-text search. We only feed tantivy
+    # documents we construct ourselves, not attacker-controlled bytes.
+    # Tracked for a lance dependency bump.
+    # https://rustsec.org/advisories/RUSTSEC-2025-0003
+    { id = "RUSTSEC-2025-0003", reason = "tantivy via lance; inputs are internally produced, not user-supplied bytes" },
+
+    # backoff: unmaintained. Reached only via async-openai. Replacement
+    # requires async-openai to migrate (or us to drop async-openai).
+    # https://rustsec.org/advisories/RUSTSEC-2025-0012
+    { id = "RUSTSEC-2025-0012", reason = "transitive via async-openai; waiting on upstream migration" },
+
+    # number_prefix: unmaintained. Transitive via indicatif → hf-hub.
+    # No security impact, just maintenance status.
+    # https://rustsec.org/advisories/RUSTSEC-2025-0119
+    { id = "RUSTSEC-2025-0119", reason = "transitive via hf-hub/indicatif; cosmetic formatting crate" },
+
+    # bincode: unmaintained. Reached through lindera and lindera-dictionary,
+    # which are required by the native Lindera tokenizer path. Lindera has not
+    # migrated to another serialization format yet.
+    # https://rustsec.org/advisories/RUSTSEC-2025-0141
+    { id = "RUSTSEC-2025-0141", reason = "transitive via lindera/lindera-dictionary for native Lindera tokenizer" },
+
+    # lru: soundness issue in IterMut. Reached only through aws-sdk-s3 in
+    # LanceDB's dev-dependency graph; LanceDB does not use that iterator
+    # directly. Clearing this requires the AWS SDK chain to update lru.
+    # https://rustsec.org/advisories/RUSTSEC-2026-0002
+    { id = "RUSTSEC-2026-0002", reason = "transitive via aws-sdk-s3 dev-dependency; waiting on AWS SDK lru upgrade" },
+
+    # rustls-webpki 0.101.7 (old major line): name-constraint checks for
+    # URI / wildcard names. Pulled in only via the legacy rustls 0.21 chain
+    # from aws-smithy-http-client. The 0.103 line we actively use is patched.
+    # Clearing the 0.101 copy requires the aws-sdk chain to migrate off
+    # rustls 0.21.
+    # https://rustsec.org/advisories/RUSTSEC-2026-0098
+    # https://rustsec.org/advisories/RUSTSEC-2026-0099
+    { id = "RUSTSEC-2026-0098", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
+    { id = "RUSTSEC-2026-0099", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
+
+    # rustls-webpki 0.101.7: reachable panic in CRL parsing. Same legacy
+    # rustls 0.21 chain from aws-smithy-http-client as above. The 0.103 line
+    # we actively use is upgraded to 0.103.13 which contains the fix.
+    # https://rustsec.org/advisories/RUSTSEC-2026-0104
+    { id = "RUSTSEC-2026-0104", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
+
+    # rand 0.8.5: soundness issue only when ThreadRng reseeds inside a custom
+    # logger. Reached through several transitive chains. LanceDB does not use
+    # rand from a custom logger; upgrade once all pinned chains accept 0.8.6+.
+    # https://rustsec.org/advisories/RUSTSEC-2026-0097
+    { id = "RUSTSEC-2026-0097", reason = "transitive rand 0.8.5; LanceDB does not call ThreadRng from custom logging" },
+]
+
+# ---------------------------------------------------------------------------
+# Licenses: only allow licenses we've reviewed as compatible with Apache-2.0.
+# ---------------------------------------------------------------------------
+[licenses]
+version = 2
+# SPDX identifiers for licenses that are compatible with our Apache-2.0
+# distribution. Additions require legal review.
+allow = [
+    "Apache-2.0",
+    "Apache-2.0 WITH LLVM-exception",
+    "MIT",
+    "BSD-2-Clause",
+    "BSD-3-Clause",
+    "ISC",
+    "Unicode-3.0",
+    "Unicode-DFS-2016",
+    "Zlib",
+    "CC0-1.0",
+    "MPL-2.0",
+    "BSL-1.0",
+    "OpenSSL",
+    # 0BSD ("BSD Zero Clause") is effectively public domain — no attribution
+    # required. Pulled in by `mock_instant`.
+    "0BSD",
+    # bzip2-1.0.6 is the permissive upstream bzip2 license (BSD-like). Pulled
+    # in by `libbz2-rs-sys`, the pure-Rust bzip2 implementation.
+    "bzip2-1.0.6",
+    # CDLA-Permissive-2.0 is a permissive data license used by `webpki-roots`
+    # for the Mozilla CA root bundle. Data-only, distribution-compatible.
+    "CDLA-Permissive-2.0",
+]
+confidence-threshold = 0.8
+# Crates whose license cannot be determined from Cargo metadata but whose
+# license we've manually confirmed from upstream. Keep this list minimal.
+[[licenses.clarify]]
+# polars-arrow-format omits the `license` field in its Cargo.toml, but the
+# upstream repo (pola-rs/polars-arrow-format) is dual-licensed Apache-2.0 OR
+# MIT. See https://github.com/pola-rs/polars-arrow-format/blob/main/LICENSE
+crate = "polars-arrow-format"
+expression = "Apache-2.0 OR MIT"
+license-files = []
+
+# ---------------------------------------------------------------------------
+# Bans: disallow specific crates and flag dependency hygiene issues.
+# ---------------------------------------------------------------------------
+[bans]
+# Warn (not deny) on duplicate versions of the same crate. In a large
+# workspace like this one, duplicates are common and often unavoidable
+# transitively. We surface them to discourage growth, but don't fail CI.
+multiple-versions = "warn"
+# Wildcard version requirements (`foo = "*"`) are a footgun — they let any
+# future release in without review. Ban them outright.
+wildcards = "deny"
+# Internal workspace crates reference each other via `path = "..."`, which
+# cargo-deny sees as a wildcard version. That's fine for private workspace
+# members (not published to crates.io), so allow it specifically for paths.
+allow-wildcard-paths = true
+# Features that, if enabled, should cause the check to fail.
+deny = []
+# Crates to skip when checking for duplicate versions.
+skip = []
+# Similar to `skip`, but also skips the entire transitive subtree.
+skip-tree = []
+
+# ---------------------------------------------------------------------------
+# Sources: restrict where crates can come from.
+# ---------------------------------------------------------------------------
+[sources]
+# Deny any registry other than the ones explicitly listed below.
+unknown-registry = "deny"
+# Deny any git dependency whose host isn't in the allow-list below. This
+# prevents accidental pulls from arbitrary forks.
+unknown-git = "deny"
+allow-registry = ["https://github.com/rust-lang/crates.io-index"]
+# Lance is developed in a sibling repo and pulled as a git dependency until
+# releases are cut to crates.io. Allow that specific host.
+allow-git = [
+    "https://github.com/lance-format/lance",
+]

dockerfiles/Dockerfile

@@ -24,4 +24,4 @@ RUN python --version && \
   rustc --version && \
   protoc --version
 
-RUN pip install --no-cache-dir tantivy lancedb
+RUN pip install --no-cache-dir lancedb

docs/README.md

@@ -1,6 +1,6 @@
 # LanceDB Documentation
 
-LanceDB docs are available at [lancedb.com/docs](https://lancedb.com/docs).
+LanceDB docs are available at [docs.lancedb.com](https://docs.lancedb.com).
 
 The SDK docs are built and deployed automatically by [Github Actions](../.github/workflows/docs.yml)
 whenever a commit is pushed to the `main` branch. So it is possible for the docs to show

docs/src/java/java.md

@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-core</artifactId>
-    <version>0.28.0-beta.2</version>
+    <version>0.29.1-beta.0</version>
 </dependency>
 ```
 

docs/src/js/README.md

@@ -34,7 +34,7 @@ const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
 console.log(results);
 ```
 
-The [quickstart](https://lancedb.com/docs/quickstart/basic-usage/) contains more complete examples.
+The [quickstart](https://docs.lancedb.com/quickstart/) contains more complete examples.
 
 ## Development
 

docs/src/js/_media/CONTRIBUTING.md

@@ -12,20 +12,22 @@ Typescript.
 * `src/`: Rust bindings source code
 * `lancedb/`: Typescript package source code
 * `__test__/`: Unit tests
-* `examples/`: An npm package with the examples shown in the documentation
+* `examples/`: A pnpm package with the examples shown in the documentation
 
 ## Development environment
 
 To set up your development environment, you will need to install the following:
 
-1. Node.js 14 or later
-2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
-3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
+1. Node.js 22 or later (required by pnpm 11)
+2. [pnpm](https://pnpm.io/installation) 11 or later (or run via `corepack enable`,
+   which uses the `packageManager` field in `package.json`)
+3. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
+4. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
 
 Initial setup:
 
 ```shell
-npm install
+pnpm install
 ```
 
 ### Commit Hooks
@@ -39,38 +41,38 @@ pre-commit install
 
 ## Development
 
-Most common development commands can be run using the npm scripts.
+Most common development commands can be run using the pnpm scripts.
 
 Build the package
 
 ```shell
-npm install
-npm run build
+pnpm install
+pnpm build
 ```
 
 Lint:
 
 ```shell
-npm run lint
+pnpm lint
 ```
 
 Format and fix lints:
 
 ```shell
-npm run lint-fix
+pnpm lint-fix
 ```
 
 Run tests:
 
 ```shell
-npm test
+pnpm test
 ```
 
 To run a single test:
 
 ```shell
 # Single file: table.test.ts
-npm test -- table.test.ts
+pnpm test -- table.test.ts
 # Single test: 'merge insert' in table.test.ts
-npm test -- table.test.ts --testNamePattern=merge\ insert
+pnpm test -- table.test.ts --testNamePattern=merge\ insert
 ```

docs/src/js/classes/Connection.md

@@ -148,6 +148,33 @@ Creates a new empty Table
 
 ***
 
+### createNamespace()
+
+```ts
+abstract createNamespace(namespacePath, options?): Promise<CreateNamespaceResponse>
+```
+
+Create a new namespace at the given path.
+
+#### Parameters
+
+* **namespacePath**: `string`[]
+    The namespace path to create.
+
+* **options?**: `Partial`&lt;[`CreateNamespaceOptions`](../interfaces/CreateNamespaceOptions.md)&gt;
+    Creation `mode`
+    ("create" | "exist_ok" | "overwrite") and optional `properties`
+    to attach to the namespace.
+
+#### Returns
+
+`Promise`&lt;[`CreateNamespaceResponse`](../interfaces/CreateNamespaceResponse.md)&gt;
+
+The properties of the
+  created namespace and an optional transaction id.
+
+***
+
 ### createTable()
 
 #### createTable(options, namespacePath)
@@ -230,6 +257,29 @@ Creates a new Table and initialize it with new data.
 
 ***
 
+### describeNamespace()
+
+```ts
+abstract describeNamespace(namespacePath): Promise<DescribeNamespaceResponse>
+```
+
+Describe a namespace, returning its properties.
+
+#### Parameters
+
+* **namespacePath**: `string`[]
+    The namespace path to describe, in
+    parent → child order, e.g. `["analytics", "sales"]`.
+
+#### Returns
+
+`Promise`&lt;[`DescribeNamespaceResponse`](../interfaces/DescribeNamespaceResponse.md)&gt;
+
+The namespace's properties
+  (may be undefined if the namespace has none).
+
+***
+
 ### display()
 
 ```ts
@@ -263,6 +313,36 @@ Drop all tables in the database.
 
 ***
 
+### dropNamespace()
+
+```ts
+abstract dropNamespace(namespacePath, options?): Promise<DropNamespaceResponse>
+```
+
+Drop a namespace.
+
+Use `behavior: "cascade"` to also drop everything contained in the
+namespace (sub-namespaces and tables). The default `"restrict"`
+behavior refuses to drop a non-empty namespace.
+
+#### Parameters
+
+* **namespacePath**: `string`[]
+    The namespace path to drop.
+
+* **options?**: `Partial`&lt;[`DropNamespaceOptions`](../interfaces/DropNamespaceOptions.md)&gt;
+    `mode` ("skip" | "fail"
+    for missing-namespace handling) and `behavior` ("restrict" | "cascade").
+
+#### Returns
+
+`Promise`&lt;[`DropNamespaceResponse`](../interfaces/DropNamespaceResponse.md)&gt;
+
+Any properties returned by
+  the server and an optional transaction id.
+
+***
+
 ### dropTable()
 
 ```ts
@@ -299,6 +379,36 @@ Return true if the connection has not been closed
 
 ***
 
+### listNamespaces()
+
+```ts
+abstract listNamespaces(namespacePath?, options?): Promise<ListNamespacesResponse>
+```
+
+List the immediate child namespaces under the given parent.
+
+Results may be paginated. To retrieve subsequent pages, pass the
+`pageToken` returned by a previous call.
+
+#### Parameters
+
+* **namespacePath?**: `string`[]
+    The parent namespace path. Defaults
+    to the root namespace if omitted.
+
+* **options?**: `Partial`&lt;[`ListNamespacesOptions`](../interfaces/ListNamespacesOptions.md)&gt;
+    Pagination options
+    (`pageToken`, `limit`).
+
+#### Returns
+
+`Promise`&lt;[`ListNamespacesResponse`](../interfaces/ListNamespacesResponse.md)&gt;
+
+Child namespace names and
+  an optional token for fetching the next page.
+
+***
+
 ### openTable()
 
 ```ts
@@ -327,6 +437,29 @@ Open a table in the database.
 
 ***
 
+### renameTable()
+
+```ts
+abstract renameTable(
+   oldName,
+   newName,
+   namespacePath?): Promise<void>
+```
+
+#### Parameters
+
+* **oldName**: `string`
+
+* **newName**: `string`
+
+* **namespacePath?**: `string`[]
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
 ### tableNames()
 
 #### tableNames(options)

docs/src/js/classes/Query.md

@@ -343,6 +343,30 @@ This is useful for pagination.
 
 ***
 
+### orderBy()
+
+```ts
+orderBy(ordering): this
+```
+
+Sort the results by the specified column(s).
+
+#### Parameters
+
+* **ordering**: [`ColumnOrdering`](../interfaces/ColumnOrdering.md) \| [`ColumnOrdering`](../interfaces/ColumnOrdering.md)[]
+
+#### Returns
+
+`this`
+
+This query builder.
+
+#### Inherited from
+
+`StandardQueryBase.orderBy`
+
+***
+
 ### outputSchema()
 
 ```ts

docs/src/js/classes/Scannable.md

@@ -0,0 +1,173 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / Scannable
+
+# Class: Scannable
+
+A data source that can be scanned as a stream of Arrow `RecordBatch`es.
+
+`Scannable` wraps the schema + optional row count + rescannable flag and
+a callback that yields batches one at a time. It is passed to consumers
+(e.g. `Table.add`, `createTable`, `mergeInsert` — follow-up work) that
+need to pull data without materializing the full dataset in JS memory.
+
+Batches cross the JS↔Rust boundary as Arrow IPC Stream messages; a fresh
+writer serializes each batch, and the Rust side decodes it with
+`arrow_ipc::reader::StreamReader`. One batch is in flight at a time.
+
+## Properties
+
+### numRows
+
+```ts
+readonly numRows: null | number;
+```
+
+***
+
+### rescannable
+
+```ts
+readonly rescannable: boolean;
+```
+
+***
+
+### schema
+
+```ts
+readonly schema: Schema<any>;
+```
+
+## Methods
+
+### fromFactory()
+
+```ts
+static fromFactory(
+   schema,
+   factory,
+   opts): Promise<Scannable>
+```
+
+Build a Scannable from an explicit schema and a factory that returns a
+fresh batch iterator on each call.
+
+The factory is invoked once per scan. Each iterator yields
+`RecordBatch`es matching the declared schema. Use this when you need
+direct control over the pull loop — for example, to wrap a streaming
+source whose batches are produced lazily.
+
+#### Parameters
+
+* **schema**: `Schema`&lt;`any`&gt;
+    The Arrow schema of the produced batches.
+
+* **factory**
+    Called at the start of each scan to produce a batch
+    iterator. Must be idempotent when `rescannable` is true.
+
+* **opts**: [`ScannableOptions`](../interfaces/ScannableOptions.md) = `{}`
+    Optional hints. `rescannable` defaults to `true`; set to
+    `false` if calling `factory()` twice would not reproduce the same data.
+
+#### Returns
+
+`Promise`&lt;[`Scannable`](Scannable.md)&gt;
+
+***
+
+### fromIterable()
+
+```ts
+static fromIterable(
+   schema,
+   iter,
+   opts): Promise<Scannable>
+```
+
+Build a Scannable from an iterable of `RecordBatch`es. `rescannable`
+defaults to `false`. Pass an explicit schema so the consumer can
+validate before any batch is pulled.
+
+`opts.rescannable: true` is honest for replayable iterables (Arrays,
+Sets, or custom iterables whose `[Symbol.iterator]()` returns a fresh
+iterator each call). It is rejected for one-shot iterables (generators,
+async generators, or already-an-iterator inputs) because their
+`[Symbol.iterator]()` returns the same exhausted object on the second
+scan. For replayable sources outside this shape, use
+`fromFactory(schema, () => createIter(), { rescannable: true })`.
+
+Note: when `opts.rescannable` is `true`, the constructor calls
+`[Symbol.iterator]()` once on the input to perform the structural check.
+
+#### Parameters
+
+* **schema**: `Schema`&lt;`any`&gt;
+
+* **iter**: `Iterable`&lt;`RecordBatch`&lt;`any`&gt;&gt; \| `AsyncIterable`&lt;`RecordBatch`&lt;`any`&gt;&gt;
+
+* **opts**: [`ScannableOptions`](../interfaces/ScannableOptions.md) = `{}`
+
+#### Returns
+
+`Promise`&lt;[`Scannable`](Scannable.md)&gt;
+
+***
+
+### fromRecordBatchReader()
+
+```ts
+static fromRecordBatchReader(reader, opts): Promise<Scannable>
+```
+
+Build a Scannable from an Arrow `RecordBatchReader`. A reader can only
+be consumed once; `rescannable` defaults to `false`.
+
+The reader must already be opened (via `.open()`) so its `.schema` is
+populated. `RecordBatchReader.from(...)` returns an unopened reader.
+
+`opts.rescannable: true` is rejected because `RecordBatchReader` is a
+self-iterator (its `[Symbol.iterator]()` returns itself), and this
+constructor does not call `reader.reset()` between scans, so a second
+scan would always see an exhausted reader. For genuinely replayable
+sources, use
+`fromFactory(schema, () => openReader(), { rescannable: true })`,
+which mints a fresh reader on each scan.
+
+#### Parameters
+
+* **reader**: `RecordBatchReader`&lt;`any`&gt;
+
+* **opts**: [`ScannableOptions`](../interfaces/ScannableOptions.md) = `{}`
+
+#### Returns
+
+`Promise`&lt;[`Scannable`](Scannable.md)&gt;
+
+***
+
+### fromTable()
+
+```ts
+static fromTable(table, opts): Promise<Scannable>
+```
+
+Build a Scannable from an in-memory Arrow `Table`. Always rescannable;
+the table's batches are replayed on each scan.
+
+The table's row count is authoritative: `opts.numRows` must either be
+omitted or equal to `table.numRows`. `opts.rescannable` of `false` is
+rejected because in-memory Tables are always rescannable.
+
+#### Parameters
+
+* **table**: `Table`&lt;`any`&gt;
+
+* **opts**: [`ScannableOptions`](../interfaces/ScannableOptions.md) = `{}`
+
+#### Returns
+
+`Promise`&lt;[`Scannable`](Scannable.md)&gt;

docs/src/js/classes/Table.md

@@ -501,6 +501,34 @@ Modeled after ``VACUUM`` in PostgreSQL.
 
 ***
 
+### prewarmData()
+
+```ts
+abstract prewarmData(columns?): Promise<void>
+```
+
+Prewarm one or more columns of data in the table.
+
+#### Parameters
+
+* **columns?**: `string`[]
+    The columns to prewarm. If undefined, all columns are prewarmed.
+    This will load the column data into the page cache so that future queries that
+    read those columns avoid the initial cold-start latency.  This call initiates
+    prewarming and returns once the request is accepted; the warming itself may
+    continue in the background.  Calling it on already-prewarmed columns is a
+    no-op on the server.
+    Prewarming is generally useful for columns used in filters or projections.
+    Large columns (e.g. high-dimensional vectors or binary data) may not be
+    practical to prewarm.
+    This feature is currently only supported on remote tables.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
 ### prewarmIndex()
 
 ```ts
@@ -662,6 +690,74 @@ of the given query
 
 ***
 
+### setLsmWriteSpec()
+
+```ts
+abstract setLsmWriteSpec(spec): Promise<void>
+```
+
+Install an [LsmWriteSpec](../interfaces/LsmWriteSpec.md) on this table, selecting Lance's MemWAL
+LSM-style write path for future `mergeInsert` calls.
+
+`LsmWriteSpec` chooses one of three sharding strategies via `specType`:
+
+- `"bucket"` — hash-bucket writes by the single-column unenforced primary
+  key (`column` and `numBuckets` required).
+- `"identity"` — shard by the raw value of a scalar `column`.
+- `"unsharded"` — route every write to a single shard.
+
+All variants require the table to have an unenforced primary key
+([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally
+requires it to be the single column being bucketed.
+
+#### Parameters
+
+* **spec**: [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)
+    The sharding spec to install.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+#### Example
+
+```ts
+await table.setUnenforcedPrimaryKey("id");
+await table.setLsmWriteSpec({
+  specType: "bucket",
+  column: "id",
+  numBuckets: 16,
+  maintainedIndexes: ["id_idx"],
+});
+```
+
+***
+
+### setUnenforcedPrimaryKey()
+
+```ts
+abstract setUnenforcedPrimaryKey(columns): Promise<void>
+```
+
+Set the unenforced primary key for this table to a single column.
+
+"Unenforced" means LanceDB does not check uniqueness on writes; the
+column is recorded in the schema as the primary key for use by features
+such as `merge_insert`. Only single-column primary keys are supported,
+and the key cannot be changed once set.
+
+#### Parameters
+
+* **columns**: `string` \| `string`[]
+    The primary key column. A one-element
+    array is also accepted; passing more than one column is rejected.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
 ### stats()
 
 ```ts
@@ -765,6 +861,23 @@ Return the table as an arrow table
 
 ***
 
+### unsetLsmWriteSpec()
+
+```ts
+abstract unsetLsmWriteSpec(): Promise<void>
+```
+
+Remove the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) from this table, reverting to the standard
+`mergeInsert` write path.
+
+Errors if no spec is currently set.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
 ### update()
 
 #### update(opts)

docs/src/js/classes/VectorQuery.md

@@ -498,6 +498,30 @@ This is useful for pagination.
 
 ***
 
+### orderBy()
+
+```ts
+orderBy(ordering): this
+```
+
+Sort the results by the specified column(s).
+
+#### Parameters
+
+* **ordering**: [`ColumnOrdering`](../interfaces/ColumnOrdering.md) \| [`ColumnOrdering`](../interfaces/ColumnOrdering.md)[]
+
+#### Returns
+
+`this`
+
+This query builder.
+
+#### Inherited from
+
+`StandardQueryBase.orderBy`
+
+***
+
 ### outputSchema()
 
 ```ts

docs/src/js/functions/connectNamespace.md

@@ -0,0 +1,131 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / connectNamespace
+
+# Function: connectNamespace()
+
+## connectNamespace(implName, config, options)
+
+```ts
+function connectNamespace(
+   implName,
+   config,
+   options?): Promise<Connection>
+```
+
+Connect to a LanceDB database through a namespace.
+
+Unlike [connect](connect.md), which routes by URI scheme (local path vs.
+`db://` cloud), `connectNamespace` always returns a namespace-backed
+connection. The `implName` selects the namespace implementation:
+
+- `"dir"` — directory namespace, configured with [DirNamespaceConfig](../interfaces/DirNamespaceConfig.md).
+- `"rest"` — remote REST catalog, configured with [RestNamespaceConfig](../interfaces/RestNamespaceConfig.md).
+- Any other string — full module path for a custom implementation,
+  configured with a free-form string-keyed `properties` map.
+
+### Parameters
+
+* **implName**: `"dir"`
+
+* **config**: [`DirNamespaceConfig`](../interfaces/DirNamespaceConfig.md)
+
+* **options?**: `Partial`&lt;[`ConnectNamespaceOptions`](../interfaces/ConnectNamespaceOptions.md)&gt;
+
+### Returns
+
+`Promise`&lt;[`Connection`](../classes/Connection.md)&gt;
+
+### Examples
+
+```ts
+const db = await connectNamespace("dir", { root: "/path/to/db" });
+await db.createTable("users", [{ id: 1 }]);
+```
+
+```ts
+const db = await connectNamespace("rest", {
+  uri: "https://catalog.example.com",
+  headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
+});
+```
+
+```ts
+const db = await connectNamespace("my.custom.Namespace", {
+  endpoint: "...",
+});
+```
+
+## connectNamespace(implName, config, options)
+
+```ts
+function connectNamespace(
+   implName,
+   config,
+   options?): Promise<Connection>
+```
+
+Connect through the built-in REST namespace.
+
+Configured with [RestNamespaceConfig](../interfaces/RestNamespaceConfig.md). See the function-level
+documentation above for the full surface, examples, and how this
+relates to [connect](connect.md).
+
+### Parameters
+
+* **implName**: `"rest"`
+
+* **config**: [`RestNamespaceConfig`](../interfaces/RestNamespaceConfig.md)
+
+* **options?**: `Partial`&lt;[`ConnectNamespaceOptions`](../interfaces/ConnectNamespaceOptions.md)&gt;
+
+### Returns
+
+`Promise`&lt;[`Connection`](../classes/Connection.md)&gt;
+
+### Example
+
+```ts
+const db = await connectNamespace("rest", {
+  uri: "https://catalog.example.com",
+  headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
+});
+```
+
+## connectNamespace(implName, properties, options)
+
+```ts
+function connectNamespace(
+   implName,
+   properties,
+   options?): Promise<Connection>
+```
+
+Connect through a custom namespace implementation by full module path,
+configured with a free-form string-keyed `properties` map. Use the
+typed overloads above for the built-in `"dir"` and `"rest"` impls.
+
+See the function-level documentation above for examples and how this
+relates to [connect](connect.md).
+
+### Parameters
+
+* **implName**: `string`
+
+* **properties**: `Record`&lt;`string`, `string`&gt;
+
+* **options?**: `Partial`&lt;[`ConnectNamespaceOptions`](../interfaces/ConnectNamespaceOptions.md)&gt;
+
+### Returns
+
+`Promise`&lt;[`Connection`](../classes/Connection.md)&gt;
+
+### Example
+
+```ts
+const db = await connectNamespace("my.custom.Namespace", {
+  endpoint: "...",
+});
+```

docs/src/js/globals.md

@@ -32,6 +32,7 @@
 - [PhraseQuery](classes/PhraseQuery.md)
 - [Query](classes/Query.md)
 - [QueryBase](classes/QueryBase.md)
+- [Scannable](classes/Scannable.md)
 - [Session](classes/Session.md)
 - [StaticHeaderProvider](classes/StaticHeaderProvider.md)
 - [Table](classes/Table.md)
@@ -50,11 +51,19 @@
 - [AlterColumnsResult](interfaces/AlterColumnsResult.md)
 - [ClientConfig](interfaces/ClientConfig.md)
 - [ColumnAlteration](interfaces/ColumnAlteration.md)
+- [ColumnOrdering](interfaces/ColumnOrdering.md)
 - [CompactionStats](interfaces/CompactionStats.md)
+- [ConnectNamespaceOptions](interfaces/ConnectNamespaceOptions.md)
 - [ConnectionOptions](interfaces/ConnectionOptions.md)
+- [CreateNamespaceOptions](interfaces/CreateNamespaceOptions.md)
+- [CreateNamespaceResponse](interfaces/CreateNamespaceResponse.md)
 - [CreateTableOptions](interfaces/CreateTableOptions.md)
 - [DeleteResult](interfaces/DeleteResult.md)
+- [DescribeNamespaceResponse](interfaces/DescribeNamespaceResponse.md)
+- [DirNamespaceConfig](interfaces/DirNamespaceConfig.md)
 - [DropColumnsResult](interfaces/DropColumnsResult.md)
+- [DropNamespaceOptions](interfaces/DropNamespaceOptions.md)
+- [DropNamespaceResponse](interfaces/DropNamespaceResponse.md)
 - [ExecutableQuery](interfaces/ExecutableQuery.md)
 - [FragmentStatistics](interfaces/FragmentStatistics.md)
 - [FragmentSummaryStats](interfaces/FragmentSummaryStats.md)
@@ -69,13 +78,18 @@
 - [IvfFlatOptions](interfaces/IvfFlatOptions.md)
 - [IvfPqOptions](interfaces/IvfPqOptions.md)
 - [IvfRqOptions](interfaces/IvfRqOptions.md)
+- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
+- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
+- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
 - [MergeResult](interfaces/MergeResult.md)
 - [OpenTableOptions](interfaces/OpenTableOptions.md)
 - [OptimizeOptions](interfaces/OptimizeOptions.md)
 - [OptimizeStats](interfaces/OptimizeStats.md)
 - [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
 - [RemovalStats](interfaces/RemovalStats.md)
+- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
 - [RetryConfig](interfaces/RetryConfig.md)
+- [ScannableOptions](interfaces/ScannableOptions.md)
 - [ShuffleOptions](interfaces/ShuffleOptions.md)
 - [SplitCalculatedOptions](interfaces/SplitCalculatedOptions.md)
 - [SplitHashOptions](interfaces/SplitHashOptions.md)
@@ -107,6 +121,7 @@
 
 - [RecordBatchIterator](functions/RecordBatchIterator.md)
 - [connect](functions/connect.md)
+- [connectNamespace](functions/connectNamespace.md)
 - [makeArrowTable](functions/makeArrowTable.md)
 - [packBits](functions/packBits.md)
 - [permutationBuilder](functions/permutationBuilder.md)

docs/src/js/interfaces/ColumnOrdering.md

@@ -0,0 +1,31 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ColumnOrdering
+
+# Interface: ColumnOrdering
+
+## Properties
+
+### ascending?
+
+```ts
+optional ascending: boolean;
+```
+
+***
+
+### columnName
+
+```ts
+columnName: string;
+```
+
+***
+
+### nullsFirst?
+
+```ts
+optional nullsFirst: boolean;
+```

docs/src/js/interfaces/ConnectNamespaceOptions.md

@@ -0,0 +1,54 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ConnectNamespaceOptions
+
+# Interface: ConnectNamespaceOptions
+
+## Properties
+
+### namespaceClientProperties?
+
+```ts
+optional namespaceClientProperties: Record<string, string>;
+```
+
+Extra properties for the backing namespace client.
+
+***
+
+### readConsistencyInterval?
+
+```ts
+optional readConsistencyInterval: number;
+```
+
+The interval, in seconds, at which to check for updates to the table
+from other processes. If None, then consistency is not checked. For
+performance reasons, this is the default. For strong consistency, set
+this to zero seconds. Then every read will check for updates from other
+processes. As a compromise, you can set this to a non-zero value for
+eventual consistency.
+
+***
+
+### session?
+
+```ts
+optional session: Session;
+```
+
+The session to use for this connection. Holds shared caches and other
+session-specific state.
+
+***
+
+### storageOptions?
+
+```ts
+optional storageOptions: Record<string, string>;
+```
+
+Configuration for object storage. The available options are described
+at https://docs.lancedb.com/storage/

docs/src/js/interfaces/ConnectionOptions.md

@@ -41,6 +41,29 @@ for testing purposes.
 
 ***
 
+### manifestEnabled?
+
+```ts
+optional manifestEnabled: boolean;
+```
+
+(For LanceDB OSS only): use directory namespace manifests as the source
+of truth for table metadata. Existing directory-listed root tables are
+migrated into the manifest on access.
+
+***
+
+### namespaceClientProperties?
+
+```ts
+optional namespaceClientProperties: Record<string, string>;
+```
+
+(For LanceDB OSS only): extra properties for the backing namespace
+client used by manifest-enabled native connections.
+
+***
+
 ### readConsistencyInterval?
 
 ```ts
@@ -89,4 +112,4 @@ optional storageOptions: Record<string, string>;
 
 (For LanceDB OSS only): configuration for object storage.
 
-The available options are described at https://lancedb.com/docs/storage/
+The available options are described at https://docs.lancedb.com/storage/

docs/src/js/interfaces/CreateNamespaceOptions.md

@@ -0,0 +1,27 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / CreateNamespaceOptions
+
+# Interface: CreateNamespaceOptions
+
+## Properties
+
+### mode?
+
+```ts
+optional mode: "overwrite" | "create" | "exist_ok";
+```
+
+Creation mode.
+
+***
+
+### properties?
+
+```ts
+optional properties: Record<string, string>;
+```
+
+Properties to set on the new namespace.

docs/src/js/interfaces/CreateNamespaceResponse.md

@@ -0,0 +1,23 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / CreateNamespaceResponse
+
+# Interface: CreateNamespaceResponse
+
+## Properties
+
+### properties?
+
+```ts
+optional properties: Record<string, string>;
+```
+
+***
+
+### transactionId?
+
+```ts
+optional transactionId: string;
+```

docs/src/js/interfaces/CreateTableOptions.md

@@ -97,4 +97,4 @@ Configuration for object storage.
 Options already set on the connection will be inherited by the table,
 but can be overridden here.
 
-The available options are described at https://lancedb.com/docs/storage/
+The available options are described at https://docs.lancedb.com/storage/

docs/src/js/interfaces/DescribeNamespaceResponse.md

@@ -0,0 +1,15 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / DescribeNamespaceResponse
+
+# Interface: DescribeNamespaceResponse
+
+## Properties
+
+### properties?
+
+```ts
+optional properties: Record<string, string>;
+```

docs/src/js/interfaces/DirNamespaceConfig.md

@@ -0,0 +1,47 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / DirNamespaceConfig
+
+# Interface: DirNamespaceConfig
+
+Configuration for the built-in directory namespace (`"dir"`).
+
+The directory namespace stores tables under a single root path (local
+filesystem or object storage URI). See
+[https://docs.lancedb.com/namespaces](https://docs.lancedb.com/namespaces) for the documented surface;
+less-common knobs live under [DirNamespaceConfig.extraProperties](DirNamespaceConfig.md#extraproperties).
+
+## Properties
+
+### extraProperties?
+
+```ts
+optional extraProperties: Record<string, string>;
+```
+
+Additional raw properties passed verbatim to the namespace
+implementation (e.g. `storage.*`, `credential_vendor.*`). Typed
+fields above take precedence on key collision.
+
+***
+
+### manifestEnabled?
+
+```ts
+optional manifestEnabled: boolean;
+```
+
+Whether to maintain a namespace manifest at the root. Required for
+child namespaces. Defaults to true on the impl side.
+
+***
+
+### root
+
+```ts
+root: string;
+```
+
+Root path or URI containing the LanceDB tables.

docs/src/js/interfaces/DropNamespaceOptions.md

@@ -0,0 +1,27 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / DropNamespaceOptions
+
+# Interface: DropNamespaceOptions
+
+## Properties
+
+### behavior?
+
+```ts
+optional behavior: "restrict" | "cascade";
+```
+
+Refuse to drop if non-empty (restrict) or drop recursively (cascade).
+
+***
+
+### mode?
+
+```ts
+optional mode: "fail" | "skip";
+```
+
+Whether to skip if the namespace doesn't exist, or fail.

docs/src/js/interfaces/DropNamespaceResponse.md

@@ -0,0 +1,23 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / DropNamespaceResponse
+
+# Interface: DropNamespaceResponse
+
+## Properties
+
+### properties?
+
+```ts
+optional properties: Record<string, string>;
+```
+
+***
+
+### transactionId?
+
+```ts
+optional transactionId: string[];
+```

docs/src/js/interfaces/ListNamespacesOptions.md

@@ -0,0 +1,27 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ListNamespacesOptions
+
+# Interface: ListNamespacesOptions
+
+## Properties
+
+### limit?
+
+```ts
+optional limit: number;
+```
+
+An optional limit to the number of results to return.
+
+***
+
+### pageToken?
+
+```ts
+optional pageToken: string;
+```
+
+Token from a previous response for pagination.

docs/src/js/interfaces/ListNamespacesResponse.md

@@ -0,0 +1,23 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ListNamespacesResponse
+
+# Interface: ListNamespacesResponse
+
+## Properties
+
+### namespaces
+
+```ts
+namespaces: string[];
+```
+
+***
+
+### pageToken?
+
+```ts
+optional pageToken: string;
+```

docs/src/js/interfaces/LsmWriteSpec.md

@@ -0,0 +1,64 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / LsmWriteSpec
+
+# Interface: LsmWriteSpec
+
+Specification selecting Lance's MemWAL LSM-style write path for
+`mergeInsert`.
+
+`specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`,
+`column` and `numBuckets` are required; for `"identity"`, `column` is
+required.
+
+## Properties
+
+### column?
+
+```ts
+optional column: string;
+```
+
+Bucket and identity variants: the sharding column.
+
+***
+
+### maintainedIndexes?
+
+```ts
+optional maintainedIndexes: string[];
+```
+
+Names of indexes the MemWAL should keep up to date during writes.
+
+***
+
+### numBuckets?
+
+```ts
+optional numBuckets: number;
+```
+
+Bucket variant: the number of buckets, in `[1, 1024]`.
+
+***
+
+### specType
+
+```ts
+specType: "bucket" | "identity" | "unsharded";
+```
+
+One of `"bucket"`, `"identity"`, or `"unsharded"`.
+
+***
+
+### writerConfigDefaults?
+
+```ts
+optional writerConfigDefaults: Record<string, string>;
+```
+
+Default `ShardWriter` configuration recorded in the MemWAL index.

docs/src/js/interfaces/OpenTableOptions.md

@@ -42,4 +42,4 @@ Configuration for object storage.
 Options already set on the connection will be inherited by the table,
 but can be overridden here.
 
-The available options are described at https://lancedb.com/docs/storage/
+The available options are described at https://docs.lancedb.com/storage/

docs/src/js/interfaces/RestNamespaceConfig.md

@@ -0,0 +1,47 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / RestNamespaceConfig
+
+# Interface: RestNamespaceConfig
+
+Configuration for the built-in REST namespace (`"rest"`).
+
+The REST namespace talks to a remote catalog server over HTTP. See
+[https://docs.lancedb.com/namespaces](https://docs.lancedb.com/namespaces) for the documented surface;
+less-common knobs (TLS, metrics) live under
+[RestNamespaceConfig.extraProperties](RestNamespaceConfig.md#extraproperties).
+
+## Properties
+
+### extraProperties?
+
+```ts
+optional extraProperties: Record<string, string>;
+```
+
+Additional raw properties passed verbatim to the namespace
+implementation (e.g. `tls.*`, `ops_metrics_enabled`, `delimiter`).
+Typed fields above take precedence on key collision.
+
+***
+
+### headers?
+
+```ts
+optional headers: Record<string, string>;
+```
+
+HTTP headers forwarded with each request. Keys are passed through
+as-is (e.g. `"x-api-key"`, `"Authorization"`).
+
+***
+
+### uri
+
+```ts
+uri: string;
+```
+
+Catalog endpoint URL.

docs/src/js/interfaces/ScannableOptions.md

@@ -0,0 +1,29 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ScannableOptions
+
+# Interface: ScannableOptions
+
+## Properties
+
+### numRows?
+
+```ts
+optional numRows: number;
+```
+
+Hint about the number of rows. Not validated against the stream.
+
+***
+
+### rescannable?
+
+```ts
+optional rescannable: boolean;
+```
+
+Whether the source can be scanned more than once. Defaults to `true` for
+`fromTable` / `fromFactory` and `false` for `fromIterable` /
+`fromRecordBatchReader`.

docs/src/python/python.md

@@ -94,11 +94,11 @@ of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and
 
 ## Full text search
 
-::: lancedb.fts.create_index
+Use [lancedb.table.Table.create_fts_index][] for the synchronous API or
+[lancedb.table.AsyncTable.create_index][] with [lancedb.index.FTS][] for the
+asynchronous API.
 
-::: lancedb.fts.populate_index
-
-::: lancedb.fts.search_index
+::: lancedb.index.FTS
 
 ## Utilities
 

java/lancedb-core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
       <groupId>com.lancedb</groupId>
       <artifactId>lancedb-parent</artifactId>
-      <version>0.28.0-beta.2</version>
+      <version>0.29.1-beta.0</version>
       <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.28.0-beta.2</version>
+    <version>0.29.1-beta.0</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>5.1.0-beta.2</lance-core.version>
+        <lance-core.version>7.0.0-beta.13</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>

nodejs/AGENTS.md

@@ -3,11 +3,11 @@ The core Rust library is in the `../rust/lancedb` directory, the rust binding
 code is in the `src/` directory and the typescript bindings are in
 the `lancedb/` directory.
 
-Whenever you change the Rust code, you will need to recompile: `npm run build`.
+Whenever you change the Rust code, you will need to recompile: `pnpm build`.
 
 Common commands:
-* Build: `npm run build`
-* Lint: `npm run lint`
-* Fix lints: `npm run lint-fix`
-* Test: `npm test`
-* Run single test file: `npm test __test__/arrow.test.ts`
+* Build: `pnpm build`
+* Lint: `pnpm lint`
+* Fix lints: `pnpm lint-fix`
+* Test: `pnpm test`
+* Run single test file: `pnpm test __test__/arrow.test.ts`

nodejs/CONTRIBUTING.md

@@ -12,20 +12,22 @@ Typescript.
 * `src/`: Rust bindings source code
 * `lancedb/`: Typescript package source code
 * `__test__/`: Unit tests
-* `examples/`: An npm package with the examples shown in the documentation
+* `examples/`: A pnpm package with the examples shown in the documentation
 
 ## Development environment
 
 To set up your development environment, you will need to install the following:
 
-1. Node.js 14 or later
-2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
-3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
+1. Node.js 22 or later (required by pnpm 11)
+2. [pnpm](https://pnpm.io/installation) 11 or later (or run via `corepack enable`,
+   which uses the `packageManager` field in `package.json`)
+3. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
+4. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
 
 Initial setup:
 
 ```shell
-npm install
+pnpm install
 ```
 
 ### Commit Hooks
@@ -39,38 +41,38 @@ pre-commit install
 
 ## Development
 
-Most common development commands can be run using the npm scripts.
+Most common development commands can be run using the pnpm scripts.
 
 Build the package
 
 ```shell
-npm install
-npm run build
+pnpm install
+pnpm build
 ```
 
 Lint:
 
 ```shell
-npm run lint
+pnpm lint
 ```
 
 Format and fix lints:
 
 ```shell
-npm run lint-fix
+pnpm lint-fix
 ```
 
 Run tests:
 
 ```shell
-npm test
+pnpm test
 ```
 
 To run a single test:
 
 ```shell
 # Single file: table.test.ts
-npm test -- table.test.ts
+pnpm test -- table.test.ts
 # Single test: 'merge insert' in table.test.ts
-npm test -- table.test.ts --testNamePattern=merge\ insert
+pnpm test -- table.test.ts --testNamePattern=merge\ insert
 ```

nodejs/Cargo.toml

@@ -1,7 +1,8 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.28.0-beta.2"
+version = "0.29.1-beta.0"
+publish = false
 license.workspace = true
 description.workspace = true
 repository.workspace = true
@@ -15,12 +16,13 @@ crate-type = ["cdylib"]
 async-trait.workspace = true
 arrow-ipc.workspace = true
 arrow-array.workspace = true
-arrow-buffer = "57.2"
+arrow-buffer = "58.0.0"
 half.workspace = true
 arrow-schema.workspace = true
 env_logger.workspace = true
 futures.workspace = true
 lancedb = { path = "../rust/lancedb", default-features = false }
+lance-namespace.workspace = true
 napi = { version = "3.8.3", default-features = false, features = [
     "napi9",
     "async"
@@ -31,8 +33,8 @@ lzma-sys = { version = "0.1", features = ["static"] }
 log.workspace = true
 
 # Pin to resolve build failures; update periodically for security patches.
-aws-lc-sys = "=0.38.0"
-aws-lc-rs = "=1.16.1"
+aws-lc-sys = "=0.40.0"
+aws-lc-rs = "=1.16.3"
 
 [build-dependencies]
 napi-build = "2.3.1"

nodejs/README.md

@@ -30,7 +30,7 @@ const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
 console.log(results);
 ```
 
-The [quickstart](https://lancedb.com/docs/quickstart/basic-usage/) contains more complete examples.
+The [quickstart](https://docs.lancedb.com/quickstart/) contains more complete examples.
 
 ## Development
 

nodejs/__test__/connection.test.ts

@@ -4,7 +4,7 @@
 import { readdirSync } from "fs";
 import { Field, Float64, Schema } from "apache-arrow";
 import * as tmp from "tmp";
-import { Connection, Table, connect } from "../lancedb";
+import { Connection, Table, connect, connectNamespace } from "../lancedb";
 import { LocalTable } from "../lancedb/table";
 
 describe("when connecting", () => {
@@ -81,6 +81,16 @@ describe("given a connection", () => {
     await db.createTable("test4", [{ id: 1 }, { id: 2 }]);
   });
 
+  it("should expose renameTable and reject on OSS listing DB", async () => {
+    await db.createTable("old_name", [{ id: 1 }]);
+
+    await expect(db.renameTable("old_name", "new_name")).rejects.toThrow(
+      "rename_table is not supported in LanceDB OSS",
+    );
+
+    await expect(db.tableNames()).resolves.toEqual(["old_name"]);
+  });
+
   it("should fail if creating table twice, unless overwrite is true", async () => {
     let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
     await expect(tbl.countRows()).resolves.toBe(2);
@@ -306,3 +316,186 @@ describe("clone table functionality", () => {
     ).rejects.toThrow("Deep clone is not yet implemented");
   });
 });
+
+describe("namespaces", () => {
+  let tmpDir: tmp.DirResult;
+  let db: Connection;
+
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    // The local DirectoryNamespace backend only supports child namespaces
+    // when manifest mode is enabled (see lance-namespace-impls/src/dir.rs).
+    db = await connect(tmpDir.name, {
+      // biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust
+      namespaceClientProperties: { manifest_enabled: "true" },
+    });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("should create and describe a namespace", async () => {
+    await db.createNamespace(["myns"]);
+    const desc = await db.describeNamespace(["myns"]);
+    expect(desc).toBeDefined();
+  });
+
+  it("should list namespaces created at the root", async () => {
+    await db.createNamespace(["alpha"]);
+    await db.createNamespace(["beta"]);
+    const list = await db.listNamespaces();
+    expect(list.namespaces).toEqual(expect.arrayContaining(["alpha", "beta"]));
+  });
+
+  it("should list child namespaces under a parent", async () => {
+    await db.createNamespace(["parent"]);
+    await db.createNamespace(["parent", "child"]);
+    const list = await db.listNamespaces(["parent"]);
+    expect(list.namespaces).toContain("child");
+  });
+
+  it("should drop a namespace", async () => {
+    await db.createNamespace(["ephemeral"]);
+    await db.dropNamespace(["ephemeral"]);
+    const list = await db.listNamespaces();
+    expect(list.namespaces).not.toContain("ephemeral");
+  });
+
+  it("should raise an error on any namespace op after close", async () => {
+    await db.close();
+    await expect(db.describeNamespace(["foo"])).rejects.toThrow(
+      "Connection is closed",
+    );
+    await expect(db.listNamespaces()).rejects.toThrow("Connection is closed");
+    await expect(db.createNamespace(["foo"])).rejects.toThrow(
+      "Connection is closed",
+    );
+    await expect(db.dropNamespace(["foo"])).rejects.toThrow(
+      "Connection is closed",
+    );
+  });
+
+  it("should raise an understandable error when describing a non-existent namespace", async () => {
+    await expect(db.describeNamespace(["does-not-exist"])).rejects.toThrow(
+      /not found/i,
+    );
+  });
+
+  it("should raise an error when creating a namespace that already exists", async () => {
+    await db.createNamespace(["dup"]);
+    await expect(db.createNamespace(["dup"])).rejects.toThrow();
+  });
+
+  it("should reject an unrecognized createNamespace mode with a clear error", async () => {
+    await expect(
+      // biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
+      db.createNamespace(["x"], { mode: "frobnicate" as any }),
+    ).rejects.toThrow(/Invalid mode 'frobnicate'/);
+  });
+
+  it("should reject an unrecognized dropNamespace mode with a clear error", async () => {
+    await db.createNamespace(["x"]);
+    await expect(
+      // biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
+      db.dropNamespace(["x"], { mode: "frobnicate" as any }),
+    ).rejects.toThrow(/Invalid mode 'frobnicate'/);
+  });
+
+  it("should reject an unrecognized dropNamespace behavior with a clear error", async () => {
+    await db.createNamespace(["x"]);
+    await expect(
+      // biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
+      db.dropNamespace(["x"], { behavior: "frobnicate" as any }),
+    ).rejects.toThrow(/Invalid behavior 'frobnicate'/);
+  });
+});
+
+describe("connectNamespace", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("connects via the dir implementation and supports table ops", async () => {
+    const db = await connectNamespace("dir", { root: tmpDir.name });
+    await db.createTable("users", [{ id: 1 }, { id: 2 }]);
+    await expect(db.tableNames()).resolves.toContain("users");
+  });
+
+  it("throws a clear error when implName is empty", async () => {
+    await expect(connectNamespace("", {})).rejects.toThrow(
+      "implName must be a non-empty string",
+    );
+  });
+
+  it("throws when the namespace implementation is unknown", async () => {
+    await expect(connectNamespace("not-a-real-impl", {})).rejects.toThrow();
+  });
+
+  it("passes storage options through to the namespace", async () => {
+    const db = await connectNamespace(
+      "dir",
+      { root: tmpDir.name },
+      { storageOptions: { newTableDataStorageVersion: "stable" } },
+    );
+    await db.createTable("plumbing", [{ id: 1 }]);
+    await expect(db.tableNames()).resolves.toContain("plumbing");
+  });
+
+  it("supports child namespaces when manifestEnabled is true on the dir config", async () => {
+    const writer = await connectNamespace("dir", {
+      root: tmpDir.name,
+      manifestEnabled: true,
+    });
+    await writer.createNamespace(["analytics"]);
+    await writer.createTable("orders", [{ id: 1 }, { id: 2 }], ["analytics"]);
+    await writer.close();
+
+    const reader = await connectNamespace("dir", {
+      root: tmpDir.name,
+      manifestEnabled: true,
+    });
+    await expect(reader.tableNames(["analytics"])).resolves.toContain("orders");
+    const orders = await reader.openTable("orders", ["analytics"]);
+    await expect(orders.countRows()).resolves.toBe(2);
+  });
+
+  it("merges extraProperties into the dir config and is overridden by typed fields", async () => {
+    // Two observable assertions:
+    // - Typed `root` overrides extraProperties.root: createTable would fail
+    //   under the bogus path if the override didn't happen.
+    // - extraProperties.manifest_enabled="false" is honored end-to-end. Child
+    //   namespaces require manifest mode (default true), so explicitly
+    //   disabling it via extraProperties must make createNamespace reject. If
+    //   extraProperties pass-through were silently broken, the default would
+    //   let createNamespace succeed.
+    const db = await connectNamespace("dir", {
+      root: tmpDir.name,
+      extraProperties: {
+        root: "/should/be/overridden",
+        // biome-ignore lint/style/useNamingConvention: backend property key
+        manifest_enabled: "false",
+      },
+    });
+    await db.createTable("base", [{ id: 1 }]);
+    await expect(db.tableNames()).resolves.toContain("base");
+    await expect(db.createNamespace(["analytics"])).rejects.toThrow();
+  });
+
+  it("flows unknown top-level keys through when implName is dynamic (no silent drop)", async () => {
+    // Routes via the third overload because `impl` is `string`, not the
+    // literal `"dir"`. The dispatcher still notices the runtime value is
+    // "dir", but unknown keys like `manifest_enabled` must not be silently
+    // dropped during the conversion.
+    //
+    // Asserting a *negative* outcome (manifest disabled -> createNamespace
+    // rejects) is required for observability, since the backend default for
+    // `manifest_enabled` is true.
+    const impl: string = "dir";
+    const db = await connectNamespace(impl, {
+      root: tmpDir.name,
+      // biome-ignore lint/style/useNamingConvention: backend property key
+      manifest_enabled: "false",
+    });
+    await expect(db.createNamespace(["mixed"])).rejects.toThrow();
+  });
+});

nodejs/__test__/query.test.ts

@@ -109,3 +109,209 @@ describe("Query outputSchema", () => {
     expect(schema.fields.length).toBe(3);
   });
 });
+
+describe("Query orderBy", () => {
+  let tmpDir: tmp.DirResult;
+  let table: Table;
+
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const db = await connect(tmpDir.name);
+
+    // Create table with numeric data for sorting
+    const schema = new Schema([
+      new Field("id", new Int64(), true),
+      new Field("score", new Float32(), true),
+      new Field("name", new Utf8(), true),
+    ]);
+
+    const data = makeArrowTable(
+      [
+        { id: 1n, score: 3.5, name: "charlie" },
+        { id: 2n, score: 1.2, name: "alice" },
+        { id: 3n, score: 2.8, name: "bob" },
+        { id: 4n, score: 0.5, name: "david" },
+        { id: 5n, score: 4.1, name: "eve" },
+      ],
+      { schema },
+    );
+    table = await db.createTable("test", data);
+  });
+
+  afterEach(() => {
+    tmpDir.removeCallback();
+  });
+
+  it("should sort by single column ascending", async () => {
+    const results = await table
+      .query()
+      .orderBy({ columnName: "score", ascending: true, nullsFirst: false })
+      .toArray();
+
+    expect(results.length).toBe(5);
+    // Verify ascending order
+    expect(results[0].score).toBeCloseTo(0.5, 0.001);
+    expect(results[1].score).toBeCloseTo(1.2, 0.001);
+    expect(results[2].score).toBeCloseTo(2.8, 0.001);
+    expect(results[3].score).toBeCloseTo(3.5, 0.001);
+    expect(results[4].score).toBeCloseTo(4.1, 0.001);
+  });
+
+  it("should sort by single column descending", async () => {
+    const results = await table
+      .query()
+      .orderBy({ columnName: "score", ascending: false, nullsFirst: false })
+      .toArray();
+
+    expect(results.length).toBe(5);
+    // Verify descending order
+    expect(results[0].score).toBeCloseTo(4.1, 0.001);
+    expect(results[1].score).toBeCloseTo(3.5, 0.001);
+    expect(results[2].score).toBeCloseTo(2.8, 0.001);
+    expect(results[3].score).toBeCloseTo(1.2, 0.001);
+    expect(results[4].score).toBeCloseTo(0.5, 0.001);
+  });
+
+  it("should use ascending as default direction", async () => {
+    const results = await table
+      .query()
+      .orderBy({ columnName: "score" })
+      .toArray();
+
+    expect(results.length).toBe(5);
+    // Verify ascending order (default)
+    expect(results[0].score).toBeCloseTo(0.5, 0.001);
+    expect(results[1].score).toBeCloseTo(1.2, 0.001);
+    expect(results[2].score).toBeCloseTo(2.8, 0.001);
+    expect(results[3].score).toBeCloseTo(3.5, 0.001);
+    expect(results[4].score).toBeCloseTo(4.1, 0.001);
+  });
+
+  it("should sort by string column", async () => {
+    const results = await table
+      .query()
+      .orderBy({ columnName: "name" })
+      .toArray();
+
+    expect(results.length).toBe(5);
+    // Verify alphabetical order
+    expect(results[0].name).toBe("alice");
+    expect(results[1].name).toBe("bob");
+    expect(results[2].name).toBe("charlie");
+    expect(results[3].name).toBe("david");
+    expect(results[4].name).toBe("eve");
+  });
+
+  it("should support method chaining with where", async () => {
+    const results = await table
+      .query()
+      .where("score > 2.0")
+      .orderBy({ columnName: "score" })
+      .toArray();
+    expect(results.length).toBe(3);
+    // Verify filtered and sorted
+    expect(results[0].score).toBeCloseTo(2.8, 0.001);
+    expect(results[1].score).toBeCloseTo(3.5, 0.001);
+    expect(results[2].score).toBeCloseTo(4.1, 0.001);
+  });
+
+  it("should support method chaining with limit", async () => {
+    const results = await table
+      .query()
+      .orderBy({ columnName: "score", ascending: false })
+      .limit(3)
+      .toArray();
+
+    expect(results.length).toBe(3);
+    // Verify top 3 in descending order
+    expect(results[0].score).toBeCloseTo(4.1, 0.001);
+    expect(results[1].score).toBeCloseTo(3.5, 0.001);
+    expect(results[2].score).toBeCloseTo(2.8, 0.001);
+  });
+
+  it("should support method chaining with offset", async () => {
+    const results = await table
+      .query()
+      .orderBy({ columnName: "score" })
+      .offset(2)
+      .limit(2)
+      .toArray();
+
+    expect(results.length).toBe(2);
+    // Verify results skip first 2 and take next 2
+    expect(results[0].score).toBeCloseTo(2.8, 0.001);
+    expect(results[1].score).toBeCloseTo(3.5, 0.001);
+  });
+
+  it("should support method chaining with select", async () => {
+    const results = await table
+      .query()
+      .orderBy({ columnName: "name" })
+      .select(["name", "score"])
+      .toArray();
+
+    expect(results.length).toBe(5);
+    // Verify only selected columns are present
+    expect(Object.keys(results[0])).toEqual(["name", "score"]);
+    expect(Object.keys(results[4])).toEqual(["name", "score"]);
+    // Verify sorted by name
+    expect(results[0].name).toBe("alice");
+    expect(results[4].name).toBe("eve");
+  });
+
+  it("should support complex method chaining", async () => {
+    const results = await table
+      .query()
+      .where("score > 1.0")
+      .orderBy({ columnName: "score", ascending: false })
+      .limit(3)
+      .select(["id", "score", "name"])
+      .toArray();
+
+    expect(results.length).toBe(3);
+    // Verify filtered, sorted, limited, and projected
+    expect(results[0].score).toBeCloseTo(4.1, 0.001);
+    expect(results[1].score).toBeCloseTo(3.5, 0.001);
+    expect(results[2].score).toBeCloseTo(2.8, 0.001);
+    expect(Object.keys(results[0])).toEqual(["id", "score", "name"]);
+  });
+
+  it("should support multi-column ordering and null placement", async () => {
+    const schema = new Schema([
+      new Field("group", new Int64(), true),
+      new Field("score", new Float32(), true),
+      new Field("name", new Utf8(), true),
+    ]);
+
+    const data = makeArrowTable(
+      [
+        { group: 1n, score: null, name: "z" },
+        { group: 1n, score: 1.0, name: "b" },
+        { group: 1n, score: 1.0, name: "a" },
+        { group: 2n, score: 0.5, name: "c" },
+      ],
+      { schema },
+    );
+    const nullTable = await (await connect(tmpDir.name)).createTable(
+      "test_multi_order",
+      data,
+      { mode: "overwrite" },
+    );
+
+    const results = await nullTable
+      .query()
+      .orderBy([
+        { columnName: "group", ascending: true, nullsFirst: false },
+        { columnName: "score", ascending: true, nullsFirst: true },
+        { columnName: "name", ascending: true, nullsFirst: false },
+      ])
+      .toArray();
+
+    expect(results.map((r) => [r.group, r.score, r.name])).toEqual([
+      [1n, null, "z"],
+      [1n, 1.0, "a"],
+      [1n, 1.0, "b"],
+      [2n, 0.5, "c"],
+    ]);
+  });
+});

nodejs/__test__/rerankers.test.ts

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
+import { spawn } from "node:child_process";
+import * as path from "node:path";
 import { RecordBatch } from "apache-arrow";
 import * as tmp from "tmp";
 import { Connection, Index, Table, connect, makeArrowTable } from "../lancedb";
@@ -76,4 +78,91 @@ describe("rerankers", function () {
 
     expect(result).toHaveLength(2);
   });
+
+  it("does not keep process alive after rerank query", async function () {
+    const script = `
+import * as lancedb from "./dist/index.js";
+import * as os from "node:os";
+import * as path from "node:path";
+import * as fs from "node:fs/promises";
+
+const dir = await fs.mkdtemp(path.join(os.tmpdir(), "lancedb-rerank-exit-"));
+const db = await lancedb.connect(dir);
+const table = await db.createTable("test", [{ text: "hello", vector: [1, 2, 3] }], {
+  mode: "overwrite",
+});
+await table.createIndex("text", { config: lancedb.Index.fts() });
+await table.waitForIndex(["text_idx"], 30);
+
+const reranker = await lancedb.rerankers.RRFReranker.create();
+await table
+  .query()
+  .nearestTo([1, 2, 3])
+  .fullTextSearch("hello")
+  .rerank(reranker)
+  .toArray();
+
+table.close();
+db.close();
+`;
+
+    await new Promise<void>((resolve, reject) => {
+      const child = spawn(
+        process.execPath,
+        ["--input-type=module", "-e", script],
+        {
+          cwd: path.resolve(__dirname, ".."),
+          stdio: ["ignore", "pipe", "pipe"],
+        },
+      );
+
+      let stdout = "";
+      let stderr = "";
+
+      child.stdout.on("data", (chunk) => {
+        stdout += chunk.toString();
+      });
+
+      child.stderr.on("data", (chunk) => {
+        stderr += chunk.toString();
+      });
+
+      const timeout = setTimeout(() => {
+        child.kill();
+        reject(
+          new Error(
+            `child process did not exit in time\nstdout:\n${stdout}\nstderr:\n${stderr}`,
+          ),
+        );
+      }, 20_000);
+
+      child.on("error", (err) => {
+        clearTimeout(timeout);
+        reject(err);
+      });
+
+      child.on("exit", (code, signal) => {
+        clearTimeout(timeout);
+        if (signal !== null) {
+          reject(
+            new Error(
+              `child process exited with signal ${signal}\nstdout:\n${stdout}\nstderr:\n${stderr}`,
+            ),
+          );
+          return;
+        }
+
+        if (code !== 0) {
+          reject(
+            new Error(
+              `child process exited with code ${code}\nstdout:\n${stdout}\nstderr:\n${stderr}`,
+            ),
+          );
+          return;
+        }
+
+        resolve();
+      });
+    });
+  });
 });

nodejs/__test__/scannable.test.ts

@@ -0,0 +1,438 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import {
+  Field,
+  Float16,
+  Int32,
+  type RecordBatch,
+  RecordBatchReader,
+  Schema,
+  tableToIPC,
+} from "apache-arrow";
+import { makeArrowTable, makeEmptyTable } from "../lancedb/arrow";
+import { Scannable } from "../lancedb/scannable";
+
+function makeTable() {
+  return makeArrowTable(
+    [
+      { id: 1, name: "a" },
+      { id: 2, name: "b" },
+      { id: 3, name: "c" },
+    ],
+    { vectorColumns: {} },
+  );
+}
+
+async function makeReader(): Promise<RecordBatchReader> {
+  // `RecordBatchReader.from()` returns an unopened reader; `.schema` is only
+  // populated after `.open()`. Opening sync readers is synchronous.
+  const reader = RecordBatchReader.from(tableToIPC(makeTable()));
+  return reader.open() as RecordBatchReader;
+}
+
+describe("Scannable", () => {
+  describe("fromTable", () => {
+    test("reflects schema, numRows, and defaults rescannable=true", async () => {
+      const table = makeTable();
+      const scannable = await Scannable.fromTable(table);
+
+      expect(scannable.schema).toBe(table.schema);
+      expect(scannable.numRows).toBe(table.numRows);
+      expect(scannable.rescannable).toBe(true);
+    });
+
+    test("throws when opts.numRows does not match table.numRows", async () => {
+      await expect(
+        Scannable.fromTable(makeTable(), { numRows: 42 }),
+      ).rejects.toThrow(/does not match table\.numRows/);
+    });
+
+    test("throws when opts.rescannable is false", async () => {
+      await expect(
+        Scannable.fromTable(makeTable(), { rescannable: false }),
+      ).rejects.toThrow(/always rescannable/);
+    });
+  });
+
+  describe("fromRecordBatchReader", () => {
+    test("reflects schema and defaults numRows=null, rescannable=false", async () => {
+      const reader = await makeReader();
+      const scannable = await Scannable.fromRecordBatchReader(reader);
+
+      expect(scannable.schema).toBe(reader.schema);
+      expect(scannable.numRows).toBeNull();
+      expect(scannable.rescannable).toBe(false);
+    });
+
+    test("honors numRows override", async () => {
+      const scannable = await Scannable.fromRecordBatchReader(
+        await makeReader(),
+        { numRows: 3 },
+      );
+
+      expect(scannable.numRows).toBe(3);
+      expect(scannable.rescannable).toBe(false);
+    });
+
+    test("rescannable: false explicit does not throw", async () => {
+      const reader = await makeReader();
+      const scannable = await Scannable.fromRecordBatchReader(reader, {
+        rescannable: false,
+      });
+      expect(scannable.rescannable).toBe(false);
+    });
+
+    test("throws when opts.rescannable is true", async () => {
+      const reader = await makeReader();
+      await expect(
+        Scannable.fromRecordBatchReader(reader, { rescannable: true }),
+      ).rejects.toThrow(/does not accept rescannable/);
+    });
+
+    test("throws when opts.rescannable is true even alongside numRows", async () => {
+      const reader = await makeReader();
+      await expect(
+        Scannable.fromRecordBatchReader(reader, {
+          numRows: 3,
+          rescannable: true,
+        }),
+      ).rejects.toThrow(/does not accept rescannable/);
+    });
+  });
+
+  describe("fromIterable", () => {
+    test("accepts a sync iterable of batches", async () => {
+      const table = makeTable();
+      const scannable = await Scannable.fromIterable(
+        table.schema,
+        table.batches,
+      );
+
+      expect(scannable.schema).toBe(table.schema);
+      expect(scannable.numRows).toBeNull();
+      expect(scannable.rescannable).toBe(false);
+    });
+
+    test("accepts an async iterable of batches", async () => {
+      const table = makeTable();
+      async function* generator(): AsyncGenerator<RecordBatch> {
+        for (const batch of table.batches) {
+          yield batch;
+        }
+      }
+
+      const scannable = await Scannable.fromIterable(table.schema, generator());
+      expect(scannable.schema).toBe(table.schema);
+      expect(scannable.rescannable).toBe(false);
+    });
+
+    describe("rescannable: true detection", () => {
+      // Replayable inputs: [Symbol.iterator]() / [Symbol.asyncIterator]()
+      // returns a fresh iterator each call. Must NOT throw.
+
+      test("Array passes (fresh ArrayIterator each call)", async () => {
+        const table = makeTable();
+        const scannable = await Scannable.fromIterable(
+          table.schema,
+          table.batches,
+          { rescannable: true },
+        );
+        expect(scannable.rescannable).toBe(true);
+      });
+
+      test("Set passes (fresh SetIterator each call)", async () => {
+        const table = makeTable();
+        const set = new Set<RecordBatch>(table.batches);
+        const scannable = await Scannable.fromIterable(table.schema, set, {
+          rescannable: true,
+        });
+        expect(scannable.rescannable).toBe(true);
+      });
+
+      test("custom Iterable returning a fresh iterator passes", async () => {
+        const table = makeTable();
+        const replayable: Iterable<RecordBatch> = {
+          [Symbol.iterator]() {
+            return table.batches[Symbol.iterator]();
+          },
+        };
+        const scannable = await Scannable.fromIterable(
+          table.schema,
+          replayable,
+          { rescannable: true },
+        );
+        expect(scannable.rescannable).toBe(true);
+      });
+
+      test("object with generator method passes (fresh generator each call)", async () => {
+        const table = makeTable();
+        const replayable: Iterable<RecordBatch> = {
+          *[Symbol.iterator]() {
+            for (const batch of table.batches) yield batch;
+          },
+        };
+        const scannable = await Scannable.fromIterable(
+          table.schema,
+          replayable,
+          { rescannable: true },
+        );
+        expect(scannable.rescannable).toBe(true);
+      });
+
+      test("empty Array passes (replayable degenerate case)", async () => {
+        const schema = makeTable().schema;
+        const scannable = await Scannable.fromIterable(
+          schema,
+          [] as RecordBatch[],
+          { rescannable: true },
+        );
+        expect(scannable.rescannable).toBe(true);
+      });
+
+      // One-shot inputs: [Symbol.iterator]() / [Symbol.asyncIterator]()
+      // returns the same object, or the input is already-an-iterator.
+      // Must throw with a /one-shot/ message.
+
+      test("sync generator throws", async () => {
+        const table = makeTable();
+        function* generator(): Generator<RecordBatch> {
+          for (const batch of table.batches) yield batch;
+        }
+        await expect(
+          Scannable.fromIterable(table.schema, generator(), {
+            rescannable: true,
+          }),
+        ).rejects.toThrow(/one-shot/);
+      });
+
+      test("async generator throws", async () => {
+        const table = makeTable();
+        async function* generator(): AsyncGenerator<RecordBatch> {
+          for (const batch of table.batches) yield batch;
+        }
+        await expect(
+          Scannable.fromIterable(table.schema, generator(), {
+            rescannable: true,
+          }),
+        ).rejects.toThrow(/one-shot/);
+      });
+
+      test("empty generator throws (one-shot degenerate case)", async () => {
+        const schema = makeTable().schema;
+        function* generator(): Generator<RecordBatch> {
+          // intentionally empty; yields nothing.
+        }
+        await expect(
+          Scannable.fromIterable(schema, generator(), { rescannable: true }),
+        ).rejects.toThrow(/one-shot/);
+      });
+
+      test("custom self-iterator throws", async () => {
+        const table = makeTable();
+        const batches = table.batches;
+        let i = 0;
+        const oneShot: Iterable<RecordBatch> & Iterator<RecordBatch> = {
+          [Symbol.iterator]() {
+            return this;
+          },
+          next() {
+            if (i >= batches.length) {
+              return { done: true, value: undefined };
+            }
+            return { done: false, value: batches[i++] };
+          },
+        };
+        await expect(
+          Scannable.fromIterable(table.schema, oneShot, { rescannable: true }),
+        ).rejects.toThrow(/one-shot/);
+      });
+
+      test("Array.values() (IterableIterator) throws", async () => {
+        const table = makeTable();
+        const iter = table.batches.values();
+        await expect(
+          Scannable.fromIterable(table.schema, iter, { rescannable: true }),
+        ).rejects.toThrow(/one-shot/);
+      });
+
+      test("raw iterator (only `.next`) throws", async () => {
+        const table = makeTable();
+        const batches = table.batches;
+        let i = 0;
+        const rawIter = {
+          next(): IteratorResult<RecordBatch> {
+            if (i >= batches.length) {
+              return { done: true, value: undefined };
+            }
+            return { done: false, value: batches[i++] };
+          },
+        };
+        await expect(
+          Scannable.fromIterable(
+            table.schema,
+            rawIter as unknown as Iterable<RecordBatch>,
+            { rescannable: true },
+          ),
+        ).rejects.toThrow(/one-shot/);
+      });
+
+      // Edge: null/undefined must not crash the detection helper. The
+      // null check belongs to `normalizeIterator` and only fires when a
+      // scan starts.
+
+      test("null input does not crash detection at construction", async () => {
+        const schema = makeTable().schema;
+        await expect(
+          Scannable.fromIterable(
+            schema,
+            null as unknown as Iterable<RecordBatch>,
+            {
+              rescannable: true,
+            },
+          ),
+        ).resolves.toBeDefined();
+      });
+
+      test("undefined input does not crash detection at construction", async () => {
+        const schema = makeTable().schema;
+        await expect(
+          Scannable.fromIterable(
+            schema,
+            undefined as unknown as Iterable<RecordBatch>,
+            { rescannable: true },
+          ),
+        ).resolves.toBeDefined();
+      });
+
+      // Default (rescannable omitted) skips the check entirely, so even
+      // pathological inputs construct without throwing here.
+
+      test("rescannable omitted skips detection entirely (generator passes)", async () => {
+        const table = makeTable();
+        function* generator(): Generator<RecordBatch> {
+          for (const batch of table.batches) yield batch;
+        }
+        const scannable = await Scannable.fromIterable(
+          table.schema,
+          generator(),
+        );
+        expect(scannable.rescannable).toBe(false);
+      });
+
+      test("rescannable: false explicit skips detection entirely (generator passes)", async () => {
+        const table = makeTable();
+        function* generator(): Generator<RecordBatch> {
+          for (const batch of table.batches) yield batch;
+        }
+        const scannable = await Scannable.fromIterable(
+          table.schema,
+          generator(),
+          { rescannable: false },
+        );
+        expect(scannable.rescannable).toBe(false);
+      });
+    });
+  });
+
+  describe("fromFactory", () => {
+    test("defaults rescannable=true and does not invoke the factory eagerly", async () => {
+      const table = makeTable();
+      const factory = jest.fn(() => table.batches);
+
+      const scannable = await Scannable.fromFactory(table.schema, factory);
+
+      expect(scannable.schema).toBe(table.schema);
+      expect(scannable.rescannable).toBe(true);
+      expect(factory).not.toHaveBeenCalled();
+    });
+
+    test("honors rescannable and numRows overrides", async () => {
+      const table = makeTable();
+      const scannable = await Scannable.fromFactory(
+        table.schema,
+        () => table.batches,
+        { numRows: 7, rescannable: false },
+      );
+
+      expect(scannable.numRows).toBe(7);
+      expect(scannable.rescannable).toBe(false);
+    });
+  });
+
+  describe("validation", () => {
+    test("throws when numRows is negative", async () => {
+      await expect(
+        Scannable.fromFactory(makeTable().schema, () => [], { numRows: -1 }),
+      ).rejects.toThrow(/non-negative/);
+    });
+
+    test("throws when numRows is not an integer", async () => {
+      await expect(
+        Scannable.fromFactory(makeTable().schema, () => [], { numRows: 3.5 }),
+      ).rejects.toThrow(/integer/);
+    });
+  });
+
+  describe("native handle", () => {
+    test("exposes a native handle via inner", async () => {
+      const scannable = await Scannable.fromTable(makeTable());
+      expect(scannable.inner).toBeDefined();
+      expect(typeof scannable.inner).toBe("object");
+      expect(scannable.inner).not.toBeNull();
+    });
+  });
+
+  // Schema-variety construction tests. Each asserts that construction
+  // succeeds against a richer Arrow schema, which transitively exercises
+  // schema serialization and the Rust-side `ipc_file_to_schema` for types
+  // beyond flat primitives.
+  describe("schema variety", () => {
+    test("accepts an empty table", async () => {
+      const schema = new Schema([new Field("id", new Int32(), true)]);
+      const table = makeEmptyTable(schema);
+      const scannable = await Scannable.fromTable(table);
+
+      expect(scannable.numRows).toBe(0);
+      expect(scannable.schema).toBe(table.schema);
+    });
+
+    test("accepts nested struct and list columns", async () => {
+      const table = makeArrowTable(
+        [
+          { id: 1, point: { x: 0, y: 0 }, tags: ["a", "b"] },
+          { id: 2, point: { x: 1, y: 2 }, tags: ["c"] },
+        ],
+        { vectorColumns: {} },
+      );
+      const scannable = await Scannable.fromTable(table);
+
+      expect(scannable.schema).toBe(table.schema);
+      expect(scannable.numRows).toBe(2);
+    });
+
+    test("accepts a FixedSizeList (vector) column", async () => {
+      const table = makeArrowTable(
+        [
+          { id: 1, vec: [1, 2, 3] },
+          { id: 2, vec: [4, 5, 6] },
+        ],
+        { vectorColumns: { vec: { type: new Float16() } } },
+      );
+      const scannable = await Scannable.fromTable(table);
+
+      expect(scannable.schema).toBe(table.schema);
+      expect(scannable.numRows).toBe(2);
+    });
+
+    test("accepts a table with many columns", async () => {
+      const row: Record<string, number> = {};
+      for (let i = 0; i < 50; i++) row[`c${i}`] = i;
+      const table = makeArrowTable([row, row], { vectorColumns: {} });
+      const scannable = await Scannable.fromTable(table);
+
+      expect(scannable.schema.fields.length).toBe(50);
+      expect(scannable.numRows).toBe(2);
+    });
+  });
+});

nodejs/__test__/table.test.ts

@@ -1870,6 +1870,25 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(results.length).toBe(3);
     });
 
+    test("prewarmData errors on local tables", async () => {
+      const db = await connect(tmpDir.name);
+      const data = [
+        { text: "alpha", vector: [0.1, 0.2, 0.3] },
+        { text: "beta", vector: [0.4, 0.5, 0.6] },
+      ];
+      const table = await db.createTable("prewarm_data_test", data);
+
+      // prewarmData is only supported on remote tables. We verify the call
+      // is wired through napi and surfaces the expected error for both
+      // arg shapes (undefined and string[]).
+      await expect(table.prewarmData()).rejects.toThrow(
+        "prewarm_data is currently only supported on remote tables",
+      );
+      await expect(table.prewarmData(["text"])).rejects.toThrow(
+        "prewarm_data is currently only supported on remote tables",
+      );
+    });
+
     test("full text index on list", async () => {
       const db = await connect(tmpDir.name);
       const data = [
@@ -2329,3 +2348,130 @@ describe("when creating a table with Float32Array vectors", () => {
     expect((fsl.children[0].type as Float32).precision).toBe(1);
   });
 });
+
+describe("setUnenforcedPrimaryKey", () => {
+  let tmpDir: tmp.DirResult;
+
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("sets a single-column primary key (string or one-element array)", async () => {
+    const conn = await connect(tmpDir.name);
+    const schema = new arrow.Schema([
+      new arrow.Field("id", new arrow.Int64(), false),
+    ]);
+    const t1 = await conn.createEmptyTable("t1", schema);
+    await t1.setUnenforcedPrimaryKey("id");
+
+    const t2 = await conn.createEmptyTable("t2", schema);
+    await t2.setUnenforcedPrimaryKey(["id"]);
+  });
+
+  it("rejects a compound primary key", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await conn.createEmptyTable(
+      "t",
+      new arrow.Schema([
+        new arrow.Field("id", new arrow.Int64(), false),
+        new arrow.Field("name", new arrow.Utf8(), false),
+      ]),
+    );
+    await expect(
+      table.setUnenforcedPrimaryKey(["id", "name"]),
+    ).rejects.toThrow();
+  });
+
+  it("rejects changing the primary key once set", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await conn.createEmptyTable(
+      "t",
+      new arrow.Schema([
+        new arrow.Field("id", new arrow.Int64(), false),
+        new arrow.Field("name", new arrow.Utf8(), false),
+      ]),
+    );
+    await table.setUnenforcedPrimaryKey("id");
+    await expect(table.setUnenforcedPrimaryKey("name")).rejects.toThrow();
+    await expect(table.setUnenforcedPrimaryKey("id")).rejects.toThrow();
+  });
+});
+
+describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
+  let tmpDir: tmp.DirResult;
+
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  async function makeTable(conn: Connection): Promise<Table> {
+    return await conn.createEmptyTable(
+      "t",
+      new arrow.Schema([new arrow.Field("id", new arrow.Int64(), false)]),
+    );
+  }
+
+  it("installs and removes a bucket spec", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await makeTable(conn);
+
+    await table.setUnenforcedPrimaryKey("id");
+    await table.setLsmWriteSpec({
+      specType: "bucket",
+      column: "id",
+      numBuckets: 4,
+    });
+    await table.unsetLsmWriteSpec();
+    // A second unset errors — there is no spec left to remove.
+    await expect(table.unsetLsmWriteSpec()).rejects.toThrow();
+    // A fresh spec can be installed after unset.
+    await table.setLsmWriteSpec({
+      specType: "bucket",
+      column: "id",
+      numBuckets: 8,
+    });
+  });
+
+  it("installs an unsharded spec", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await makeTable(conn);
+
+    await table.setUnenforcedPrimaryKey("id");
+    await table.setLsmWriteSpec({ specType: "unsharded" });
+    await table.unsetLsmWriteSpec();
+  });
+
+  it("installs an identity spec", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await makeTable(conn);
+
+    await table.setUnenforcedPrimaryKey("id");
+    await table.setLsmWriteSpec({ specType: "identity", column: "id" });
+    await table.unsetLsmWriteSpec();
+  });
+
+  it("rejects an invalid spec", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await makeTable(conn);
+
+    await table.setUnenforcedPrimaryKey("id");
+    // num_buckets out of range.
+    await expect(
+      table.setLsmWriteSpec({
+        specType: "bucket",
+        column: "id",
+        numBuckets: 0,
+      }),
+    ).rejects.toThrow();
+    // Column mismatch.
+    await expect(
+      table.setLsmWriteSpec({
+        specType: "bucket",
+        column: "missing",
+        numBuckets: 4,
+      }),
+    ).rejects.toThrow();
+  });
+});

nodejs/examples/filtering.test.ts

@@ -38,5 +38,14 @@ test("filtering examples", async () => {
     // --8<-- [start:sql_search]
     await tbl.query().where("id = 10").limit(10).toArray();
     // --8<-- [end:sql_search]
+
+    // --8<-- [start:orderby_search]
+    await tbl
+      .query()
+      .where("id > 10")
+      .orderBy({ columnName: "id", ascending: false })
+      .limit(5)
+      .toArray();
+    // --8<-- [end:orderby_search]
   });
 });

nodejs/examples/package.json

@@ -11,16 +11,17 @@
     "test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
     "lint": "biome check *.ts && biome format *.ts",
     "lint-ci": "biome ci .",
-    "lint-fix": "biome check --write *.ts && npm run format",
+    "lint-fix": "biome check --write *.ts && pnpm format",
     "format": "biome format --write *.ts"
   },
   "author": "Lance Devs",
   "license": "Apache-2.0",
+  "packageManager": "pnpm@11.1.1",
   "dependencies": {
-    "@huggingface/transformers": "^3.0.2",
+    "@huggingface/transformers": "3.0.2",
     "@lancedb/lancedb": "file:../dist",
-    "openai": "^4.29.2",
-    "sharp": "^0.33.5"
+    "openai": "4.29.2",
+    "sharp": "0.33.5"
   },
   "devDependencies": {
     "@biomejs/biome": "^1.7.3",

nodejs/examples/pnpm-workspace.yaml

@@ -0,0 +1,13 @@
+# Block resolution of versions less than 24h old (Shai-Hulud window).
+# This is the pnpm 11 default but pinned here so it's visible to
+# reviewers and survives a future pnpm major flipping the default.
+minimumReleaseAge: 1440
+
+# Fail install if a transitive dep tries to run an unapproved script.
+strictDepBuilds: true
+
+allowBuilds:
+  '@biomejs/biome': true
+  onnxruntime-node: true
+  protobufjs: true
+  sharp: true

nodejs/lancedb/arrow.ts

@@ -1291,6 +1291,18 @@ export async function fromRecordBatchToBuffer(
   return Buffer.from(await writer.toUint8Array());
 }
 
+/**
+ * Create a buffer containing a single record batch using the Arrow IPC Stream
+ * serialization. Each call produces a self-contained Stream message (schema +
+ * batch + EOS) suitable for incremental decode by `arrow_ipc::reader::StreamReader`.
+ */
+export async function fromRecordBatchToStreamBuffer(
+  batch: RecordBatch,
+): Promise<Buffer> {
+  const writer = RecordBatchStreamWriter.writeAll([batch]);
+  return Buffer.from(await writer.toUint8Array());
+}
+
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
  *

nodejs/lancedb/connection.ts

@@ -16,6 +16,18 @@ import {
 } from "./arrow";
 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
 import { Connection as LanceDbConnection } from "./native";
+import type {
+  CreateNamespaceResponse,
+  DescribeNamespaceResponse,
+  DropNamespaceResponse,
+  ListNamespacesResponse,
+} from "./native";
+export type {
+  CreateNamespaceResponse,
+  DescribeNamespaceResponse,
+  DropNamespaceResponse,
+  ListNamespacesResponse,
+};
 import { sanitizeTable } from "./sanitize";
 import { LocalTable, Table } from "./table";
 
@@ -42,7 +54,7 @@ export interface CreateTableOptions {
    * Options already set on the connection will be inherited by the table,
    * but can be overridden here.
    *
-   * The available options are described at https://lancedb.com/docs/storage/
+   * The available options are described at https://docs.lancedb.com/storage/
    */
   storageOptions?: Record<string, string>;
 
@@ -78,7 +90,7 @@ export interface OpenTableOptions {
    * Options already set on the connection will be inherited by the table,
    * but can be overridden here.
    *
-   * The available options are described at https://lancedb.com/docs/storage/
+   * The available options are described at https://docs.lancedb.com/storage/
    */
   storageOptions?: Record<string, string>;
   /**
@@ -110,6 +122,28 @@ export interface TableNamesOptions {
   /** An optional limit to the number of results to return. */
   limit?: number;
 }
+
+export interface ListNamespacesOptions {
+  /** Token from a previous response for pagination. */
+  pageToken?: string;
+  /** An optional limit to the number of results to return. */
+  limit?: number;
+}
+
+export interface CreateNamespaceOptions {
+  /** Creation mode. */
+  mode?: "create" | "exist_ok" | "overwrite";
+  /** Properties to set on the new namespace. */
+  properties?: Record<string, string>;
+}
+
+export interface DropNamespaceOptions {
+  /** Whether to skip if the namespace doesn't exist, or fail. */
+  mode?: "skip" | "fail";
+  /** Refuse to drop if non-empty (restrict) or drop recursively (cascade). */
+  behavior?: "restrict" | "cascade";
+}
+
 /**
  * A LanceDB Connection that allows you to open tables and create new ones.
  *
@@ -262,12 +296,81 @@ export abstract class Connection {
    */
   abstract dropTable(name: string, namespacePath?: string[]): Promise<void>;
 
+  abstract renameTable(
+    oldName: string,
+    newName: string,
+    namespacePath?: string[],
+  ): Promise<void>;
+
   /**
    * Drop all tables in the database.
    * @param {string[]} namespacePath The namespace path to drop tables from (defaults to root namespace).
    */
   abstract dropAllTables(namespacePath?: string[]): Promise<void>;
 
+  /**
+   * Describe a namespace, returning its properties.
+   *
+   * @param {string[]} namespacePath - The namespace path to describe, in
+   *   parent → child order, e.g. `["analytics", "sales"]`.
+   * @returns {Promise<DescribeNamespaceResponse>} The namespace's properties
+   *   (may be undefined if the namespace has none).
+   */
+  abstract describeNamespace(
+    namespacePath: string[],
+  ): Promise<DescribeNamespaceResponse>;
+
+  /**
+   * List the immediate child namespaces under the given parent.
+   *
+   * Results may be paginated. To retrieve subsequent pages, pass the
+   * `pageToken` returned by a previous call.
+   *
+   * @param {string[]} namespacePath - The parent namespace path. Defaults
+   *   to the root namespace if omitted.
+   * @param {Partial<ListNamespacesOptions>} options - Pagination options
+   *   (`pageToken`, `limit`).
+   * @returns {Promise<ListNamespacesResponse>} Child namespace names and
+   *   an optional token for fetching the next page.
+   */
+  abstract listNamespaces(
+    namespacePath?: string[],
+    options?: Partial<ListNamespacesOptions>,
+  ): Promise<ListNamespacesResponse>;
+
+  /**
+   * Create a new namespace at the given path.
+   *
+   * @param {string[]} namespacePath - The namespace path to create.
+   * @param {Partial<CreateNamespaceOptions>} options - Creation `mode`
+   *   ("create" | "exist_ok" | "overwrite") and optional `properties`
+   *   to attach to the namespace.
+   * @returns {Promise<CreateNamespaceResponse>} The properties of the
+   *   created namespace and an optional transaction id.
+   */
+  abstract createNamespace(
+    namespacePath: string[],
+    options?: Partial<CreateNamespaceOptions>,
+  ): Promise<CreateNamespaceResponse>;
+
+  /**
+   * Drop a namespace.
+   *
+   * Use `behavior: "cascade"` to also drop everything contained in the
+   * namespace (sub-namespaces and tables). The default `"restrict"`
+   * behavior refuses to drop a non-empty namespace.
+   *
+   * @param {string[]} namespacePath - The namespace path to drop.
+   * @param {Partial<DropNamespaceOptions>} options - `mode` ("skip" | "fail"
+   *   for missing-namespace handling) and `behavior` ("restrict" | "cascade").
+   * @returns {Promise<DropNamespaceResponse>} Any properties returned by
+   *   the server and an optional transaction id.
+   */
+  abstract dropNamespace(
+    namespacePath: string[],
+    options?: Partial<DropNamespaceOptions>,
+  ): Promise<DropNamespaceResponse>;
+
   /**
    * Clone a table from a source table.
    *
@@ -512,9 +615,56 @@ export class LocalConnection extends Connection {
     return this.inner.dropTable(name, namespacePath ?? []);
   }
 
+  async renameTable(
+    oldName: string,
+    newName: string,
+    namespacePath?: string[],
+  ): Promise<void> {
+    return this.inner.renameTable(oldName, newName, namespacePath ?? []);
+  }
+
   async dropAllTables(namespacePath?: string[]): Promise<void> {
     return this.inner.dropAllTables(namespacePath ?? []);
   }
+
+  describeNamespace(
+    namespacePath: string[],
+  ): Promise<DescribeNamespaceResponse> {
+    return this.inner.describeNamespace(namespacePath);
+  }
+
+  listNamespaces(
+    namespacePath?: string[],
+    options?: Partial<ListNamespacesOptions>,
+  ): Promise<ListNamespacesResponse> {
+    return this.inner.listNamespaces(
+      namespacePath ?? [],
+      options?.pageToken,
+      options?.limit,
+    );
+  }
+
+  createNamespace(
+    namespacePath: string[],
+    options?: Partial<CreateNamespaceOptions>,
+  ): Promise<CreateNamespaceResponse> {
+    return this.inner.createNamespace(
+      namespacePath,
+      options?.mode,
+      options?.properties,
+    );
+  }
+
+  dropNamespace(
+    namespacePath: string[],
+    options?: Partial<DropNamespaceOptions>,
+  ): Promise<DropNamespaceResponse> {
+    return this.inner.dropNamespace(
+      namespacePath,
+      options?.mode,
+      options?.behavior,
+    );
+  }
 }
 
 /**

nodejs/lancedb/index.ts

@@ -8,6 +8,7 @@ import {
 } from "./connection";
 
 import {
+  ConnectNamespaceOptions,
   ConnectionOptions,
   Connection as LanceDbConnection,
   JsHeaderProvider as NativeJsHeaderProvider,
@@ -22,6 +23,7 @@ export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
 export {
   AddColumnsSql,
   ConnectionOptions,
+  ConnectNamespaceOptions,
   IndexStatistics,
   IndexConfig,
   ClientConfig,
@@ -62,6 +64,13 @@ export {
   CreateTableOptions,
   TableNamesOptions,
   OpenTableOptions,
+  ListNamespacesOptions,
+  CreateNamespaceOptions,
+  DropNamespaceOptions,
+  ListNamespacesResponse,
+  CreateNamespaceResponse,
+  DropNamespaceResponse,
+  DescribeNamespaceResponse,
 } from "./connection";
 
 export { Session } from "./native.js";
@@ -73,6 +82,7 @@ export {
   VectorQuery,
   TakeQuery,
   QueryExecutionOptions,
+  ColumnOrdering,
   FullTextSearchOptions,
   RecordBatchIterator,
   FullTextQuery,
@@ -103,6 +113,7 @@ export {
   UpdateOptions,
   OptimizeOptions,
   Version,
+  LsmWriteSpec,
   ColumnAlteration,
 } from "./table";
 
@@ -117,6 +128,7 @@ export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
 
 export * as embedding from "./embedding";
 export { permutationBuilder, PermutationBuilder } from "./permutation";
+export { Scannable, ScannableOptions } from "./scannable";
 export * as rerankers from "./rerankers";
 export {
   SchemaLike,
@@ -293,3 +305,197 @@ export async function connect(
   );
   return new LocalConnection(nativeConn);
 }
+
+/**
+ * Configuration for the built-in directory namespace (`"dir"`).
+ *
+ * The directory namespace stores tables under a single root path (local
+ * filesystem or object storage URI). See
+ * {@link https://docs.lancedb.com/namespaces} for the documented surface;
+ * less-common knobs live under {@link DirNamespaceConfig.extraProperties}.
+ */
+export interface DirNamespaceConfig {
+  /** Root path or URI containing the LanceDB tables. */
+  root: string;
+  /**
+   * Whether to maintain a namespace manifest at the root. Required for
+   * child namespaces. Defaults to true on the impl side.
+   */
+  manifestEnabled?: boolean;
+  /**
+   * Additional raw properties passed verbatim to the namespace
+   * implementation (e.g. `storage.*`, `credential_vendor.*`). Typed
+   * fields above take precedence on key collision.
+   */
+  extraProperties?: Record<string, string>;
+}
+
+/**
+ * Configuration for the built-in REST namespace (`"rest"`).
+ *
+ * The REST namespace talks to a remote catalog server over HTTP. See
+ * {@link https://docs.lancedb.com/namespaces} for the documented surface;
+ * less-common knobs (TLS, metrics) live under
+ * {@link RestNamespaceConfig.extraProperties}.
+ */
+export interface RestNamespaceConfig {
+  /** Catalog endpoint URL. */
+  uri: string;
+  /**
+   * HTTP headers forwarded with each request. Keys are passed through
+   * as-is (e.g. `"x-api-key"`, `"Authorization"`).
+   */
+  headers?: Record<string, string>;
+  /**
+   * Additional raw properties passed verbatim to the namespace
+   * implementation (e.g. `tls.*`, `ops_metrics_enabled`, `delimiter`).
+   * Typed fields above take precedence on key collision.
+   */
+  extraProperties?: Record<string, string>;
+}
+
+function dirConfigToProperties(
+  config: DirNamespaceConfig,
+): Record<string, string> {
+  // Spread the whole input so that unknown keys (e.g. a raw `manifest_enabled`
+  // passed via the dynamic-impl path) flow through instead of being dropped.
+  // Typed transformations layer on top.
+  const { manifestEnabled, extraProperties, ...rest } = config;
+  const properties: Record<string, string> = {
+    ...(extraProperties ?? {}),
+    ...(rest as Record<string, string>),
+  };
+  if (manifestEnabled !== undefined) {
+    properties.manifest_enabled = String(manifestEnabled);
+  }
+  return properties;
+}
+
+function restConfigToProperties(
+  config: RestNamespaceConfig,
+): Record<string, string> {
+  const { headers, extraProperties, ...rest } = config;
+  const properties: Record<string, string> = {
+    ...(extraProperties ?? {}),
+    ...(rest as Record<string, string>),
+  };
+  if (headers) {
+    for (const [name, value] of Object.entries(headers)) {
+      properties[`headers.${name}`] = value;
+    }
+  }
+  return properties;
+}
+
+/**
+ * Connect to a LanceDB database through a namespace.
+ *
+ * Unlike {@link connect}, which routes by URI scheme (local path vs.
+ * `db://` cloud), `connectNamespace` always returns a namespace-backed
+ * connection. The `implName` selects the namespace implementation:
+ *
+ * - `"dir"` — directory namespace, configured with {@link DirNamespaceConfig}.
+ * - `"rest"` — remote REST catalog, configured with {@link RestNamespaceConfig}.
+ * - Any other string — full module path for a custom implementation,
+ *   configured with a free-form string-keyed `properties` map.
+ *
+ * @example Typed dir namespace
+ * ```ts
+ * const db = await connectNamespace("dir", { root: "/path/to/db" });
+ * await db.createTable("users", [{ id: 1 }]);
+ * ```
+ *
+ * @example Typed REST namespace with auth headers
+ * ```ts
+ * const db = await connectNamespace("rest", {
+ *   uri: "https://catalog.example.com",
+ *   headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
+ * });
+ * ```
+ *
+ * @example Custom implementation with raw properties
+ * ```ts
+ * const db = await connectNamespace("my.custom.Namespace", {
+ *   endpoint: "...",
+ * });
+ * ```
+ */
+export function connectNamespace(
+  implName: "dir",
+  config: DirNamespaceConfig,
+  options?: Partial<ConnectNamespaceOptions>,
+): Promise<Connection>;
+/**
+ * Connect through the built-in REST namespace.
+ *
+ * Configured with {@link RestNamespaceConfig}. See the function-level
+ * documentation above for the full surface, examples, and how this
+ * relates to {@link connect}.
+ *
+ * @example
+ * ```ts
+ * const db = await connectNamespace("rest", {
+ *   uri: "https://catalog.example.com",
+ *   headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
+ * });
+ * ```
+ */
+export function connectNamespace(
+  implName: "rest",
+  config: RestNamespaceConfig,
+  options?: Partial<ConnectNamespaceOptions>,
+): Promise<Connection>;
+/**
+ * Connect through a custom namespace implementation by full module path,
+ * configured with a free-form string-keyed `properties` map. Use the
+ * typed overloads above for the built-in `"dir"` and `"rest"` impls.
+ *
+ * See the function-level documentation above for examples and how this
+ * relates to {@link connect}.
+ *
+ * @example
+ * ```ts
+ * const db = await connectNamespace("my.custom.Namespace", {
+ *   endpoint: "...",
+ * });
+ * ```
+ */
+export function connectNamespace(
+  implName: string,
+  properties: Record<string, string>,
+  options?: Partial<ConnectNamespaceOptions>,
+): Promise<Connection>;
+export async function connectNamespace(
+  implName: string,
+  configOrProperties:
+    | DirNamespaceConfig
+    | RestNamespaceConfig
+    | Record<string, string>,
+  options?: Partial<ConnectNamespaceOptions>,
+): Promise<Connection> {
+  let properties: Record<string, string>;
+  if (implName === "dir") {
+    properties = dirConfigToProperties(
+      configOrProperties as DirNamespaceConfig,
+    );
+  } else if (implName === "rest") {
+    properties = restConfigToProperties(
+      configOrProperties as RestNamespaceConfig,
+    );
+  } else {
+    properties = configOrProperties as Record<string, string>;
+  }
+
+  const finalOptions: ConnectNamespaceOptions = (options ??
+    {}) as ConnectNamespaceOptions;
+  finalOptions.storageOptions = cleanseStorageOptions(
+    finalOptions.storageOptions,
+  );
+
+  const nativeConn = await LanceDbConnection.newWithNamespace(
+    implName,
+    properties,
+    finalOptions,
+  );
+  return new LocalConnection(nativeConn);
+}

nodejs/lancedb/query.ts

@@ -79,6 +79,12 @@ export interface QueryExecutionOptions {
   timeoutMs?: number;
 }
 
+export interface ColumnOrdering {
+  columnName: string;
+  ascending?: boolean;
+  nullsFirst?: boolean;
+}
+
 /**
  * Options that control the behavior of a full text search
  */
@@ -417,6 +423,21 @@ export class StandardQueryBase<
     return this;
   }
 
+  /**
+   * Sort the results by the specified column(s).
+   * @returns This query builder.
+   */
+  orderBy(ordering: ColumnOrdering | ColumnOrdering[]): this {
+    const orderings = Array.isArray(ordering) ? ordering : [ordering];
+    const normalized = orderings.map((o) => ({
+      columnName: o.columnName,
+      ascending: o.ascending ?? true,
+      nullsFirst: o.nullsFirst ?? false,
+    }));
+    this.doCall((inner) => inner.orderBy(normalized));
+    return this;
+  }
+
   /**
    * Skip searching un-indexed data. This can make search faster, but will miss
    * any data that is not yet indexed.

nodejs/lancedb/scannable.ts

@@ -0,0 +1,274 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import {
+  Table as ArrowTable,
+  RecordBatch,
+  RecordBatchReader,
+  Schema,
+} from "apache-arrow";
+import {
+  fromRecordBatchToStreamBuffer,
+  fromTableToBuffer,
+  makeEmptyTable,
+} from "./arrow";
+import { NapiScannable } from "./native.js";
+
+export interface ScannableOptions {
+  /** Hint about the number of rows. Not validated against the stream. */
+  numRows?: number;
+  /**
+   * Whether the source can be scanned more than once. Defaults to `true` for
+   * `fromTable` / `fromFactory` and `false` for `fromIterable` /
+   * `fromRecordBatchReader`.
+   */
+  rescannable?: boolean;
+}
+
+/**
+ * A data source that can be scanned as a stream of Arrow `RecordBatch`es.
+ *
+ * `Scannable` wraps the schema + optional row count + rescannable flag and
+ * a callback that yields batches one at a time. It is passed to consumers
+ * (e.g. `Table.add`, `createTable`, `mergeInsert` — follow-up work) that
+ * need to pull data without materializing the full dataset in JS memory.
+ *
+ * Batches cross the JS↔Rust boundary as Arrow IPC Stream messages; a fresh
+ * writer serializes each batch, and the Rust side decodes it with
+ * `arrow_ipc::reader::StreamReader`. One batch is in flight at a time.
+ */
+export class Scannable {
+  readonly schema: Schema;
+  readonly numRows: number | null;
+  readonly rescannable: boolean;
+
+  /** @hidden */
+  private readonly native: NapiScannable;
+
+  private constructor(
+    native: NapiScannable,
+    schema: Schema,
+    numRows: number | null,
+    rescannable: boolean,
+  ) {
+    this.native = native;
+    this.schema = schema;
+    this.numRows = numRows;
+    this.rescannable = rescannable;
+  }
+
+  /** @hidden Access the native handle for passing through to Rust consumers. */
+  get inner(): NapiScannable {
+    return this.native;
+  }
+
+  /**
+   * Build a Scannable from an explicit schema and a factory that returns a
+   * fresh batch iterator on each call.
+   *
+   * The factory is invoked once per scan. Each iterator yields
+   * `RecordBatch`es matching the declared schema. Use this when you need
+   * direct control over the pull loop — for example, to wrap a streaming
+   * source whose batches are produced lazily.
+   *
+   * @param schema - The Arrow schema of the produced batches.
+   * @param factory - Called at the start of each scan to produce a batch
+   *   iterator. Must be idempotent when `rescannable` is true.
+   * @param opts - Optional hints. `rescannable` defaults to `true`; set to
+   *   `false` if calling `factory()` twice would not reproduce the same data.
+   */
+  static async fromFactory(
+    schema: Schema,
+    factory: () =>
+      | AsyncIterable<RecordBatch>
+      | Iterable<RecordBatch>
+      | AsyncIterator<RecordBatch>
+      | Iterator<RecordBatch>,
+    opts: ScannableOptions = {},
+  ): Promise<Scannable> {
+    const numRows = opts.numRows ?? null;
+    if (numRows != null && !Number.isInteger(numRows)) {
+      throw new TypeError("numRows must be an integer");
+    }
+    const rescannable = opts.rescannable ?? true;
+
+    let iter: AsyncIterator<RecordBatch> | Iterator<RecordBatch> | null = null;
+    const getNextBatch = async (isStart: boolean): Promise<Buffer | null> => {
+      // `isStart` is true on the first pull of every new scan_as_stream.
+      // Drop any cached iterator so factory() is re-invoked for the next scan
+      if (isStart) {
+        iter = null;
+      }
+      if (iter === null) {
+        iter = normalizeIterator(factory());
+      }
+      const result = await iter.next();
+      if (result.done) {
+        iter = null;
+        return null;
+      }
+      return fromRecordBatchToStreamBuffer(result.value);
+    };
+
+    const schemaBuf = await fromTableToBuffer(makeEmptyTable(schema));
+    const native = new NapiScannable(
+      schemaBuf,
+      numRows,
+      rescannable,
+      getNextBatch,
+    );
+    return new Scannable(native, schema, numRows, rescannable);
+  }
+
+  /**
+   * Build a Scannable from an in-memory Arrow `Table`. Always rescannable;
+   * the table's batches are replayed on each scan.
+   *
+   * The table's row count is authoritative: `opts.numRows` must either be
+   * omitted or equal to `table.numRows`. `opts.rescannable` of `false` is
+   * rejected because in-memory Tables are always rescannable.
+   */
+  static async fromTable(
+    table: ArrowTable,
+    opts: ScannableOptions = {},
+  ): Promise<Scannable> {
+    if (opts.numRows != null && opts.numRows !== table.numRows) {
+      throw new TypeError(
+        `opts.numRows (${opts.numRows}) does not match table.numRows (${table.numRows}). ` +
+          `The table's row count is authoritative; omit numRows or pass the matching value.`,
+      );
+    }
+    if (opts.rescannable === false) {
+      throw new TypeError(
+        `fromTable does not accept rescannable: false. ` +
+          `In-memory Arrow Tables are always rescannable; omit the option or pass true.`,
+      );
+    }
+    return Scannable.fromFactory(table.schema, () => table.batches, {
+      numRows: table.numRows,
+      rescannable: true,
+    });
+  }
+
+  /**
+   * Build a Scannable from an iterable of `RecordBatch`es. `rescannable`
+   * defaults to `false`. Pass an explicit schema so the consumer can
+   * validate before any batch is pulled.
+   *
+   * `opts.rescannable: true` is honest for replayable iterables (Arrays,
+   * Sets, or custom iterables whose `[Symbol.iterator]()` returns a fresh
+   * iterator each call). It is rejected for one-shot iterables (generators,
+   * async generators, or already-an-iterator inputs) because their
+   * `[Symbol.iterator]()` returns the same exhausted object on the second
+   * scan. For replayable sources outside this shape, use
+   * `fromFactory(schema, () => createIter(), { rescannable: true })`.
+   *
+   * Note: when `opts.rescannable` is `true`, the constructor calls
+   * `[Symbol.iterator]()` once on the input to perform the structural check.
+   */
+  static async fromIterable(
+    schema: Schema,
+    iter: AsyncIterable<RecordBatch> | Iterable<RecordBatch>,
+    opts: ScannableOptions = {},
+  ): Promise<Scannable> {
+    if (opts.rescannable === true && isOneShotIterable(iter)) {
+      throw new TypeError(
+        `fromIterable: rescannable: true is not honest for one-shot iterables ` +
+          `(generators, async generators, or iterators where [Symbol.iterator]() ` +
+          `returns the same object). The source would be exhausted after the first scan. ` +
+          `Use fromFactory(schema, () => createIter(), { rescannable: true }) for sources ` +
+          `where each call mints a fresh iterator.`,
+      );
+    }
+    return Scannable.fromFactory(schema, () => iter, {
+      numRows: opts.numRows,
+      rescannable: opts.rescannable ?? false,
+    });
+  }
+
+  /**
+   * Build a Scannable from an Arrow `RecordBatchReader`. A reader can only
+   * be consumed once; `rescannable` defaults to `false`.
+   *
+   * The reader must already be opened (via `.open()`) so its `.schema` is
+   * populated. `RecordBatchReader.from(...)` returns an unopened reader.
+   *
+   * `opts.rescannable: true` is rejected because `RecordBatchReader` is a
+   * self-iterator (its `[Symbol.iterator]()` returns itself), and this
+   * constructor does not call `reader.reset()` between scans, so a second
+   * scan would always see an exhausted reader. For genuinely replayable
+   * sources, use
+   * `fromFactory(schema, () => openReader(), { rescannable: true })`,
+   * which mints a fresh reader on each scan.
+   */
+  static async fromRecordBatchReader(
+    reader: RecordBatchReader,
+    opts: ScannableOptions = {},
+  ): Promise<Scannable> {
+    if (opts.rescannable === true) {
+      throw new TypeError(
+        `fromRecordBatchReader does not accept rescannable: true. ` +
+          `RecordBatchReader is a self-iterator (its [Symbol.iterator]() ` +
+          `returns itself) and would be exhausted after the first scan. ` +
+          `Use fromFactory(schema, () => openReader(), { rescannable: true }) ` +
+          `for sources where each call mints a fresh reader.`,
+      );
+    }
+    return Scannable.fromFactory(reader.schema, () => reader, {
+      numRows: opts.numRows,
+      rescannable: false,
+    });
+  }
+}
+
+function normalizeIterator<T>(
+  source: AsyncIterable<T> | Iterable<T> | AsyncIterator<T> | Iterator<T>,
+): AsyncIterator<T> | Iterator<T> {
+  if (source == null) {
+    throw new TypeError("Scannable factory returned null/undefined");
+  }
+  if (
+    typeof (source as AsyncIterable<T>)[Symbol.asyncIterator] === "function"
+  ) {
+    return (source as AsyncIterable<T>)[Symbol.asyncIterator]();
+  }
+  if (typeof (source as Iterable<T>)[Symbol.iterator] === "function") {
+    return (source as Iterable<T>)[Symbol.iterator]();
+  }
+  // Already an iterator (has `.next`).
+  if (typeof (source as Iterator<T>).next === "function") {
+    return source as Iterator<T>;
+  }
+  throw new TypeError("Scannable factory returned a non-iterable value");
+}
+
+// A "self-iterator" returns the same object from `[Symbol.iterator]()` /
+// `[Symbol.asyncIterator]()`. Generators behave this way, so they exhaust
+// after one pass. Replayable iterables (Array, Set, custom) return a fresh
+// iterator each call. Detection mirrors `normalizeIterator`'s ordering so
+// classification matches scan-time behavior.
+function isOneShotIterable(
+  source: AsyncIterable<unknown> | Iterable<unknown>,
+): boolean {
+  // null/undefined are not one-shot in any meaningful sense; let
+  // `normalizeIterator` raise the actual error at scan time.
+  if (source == null) return false;
+  const ref = source as unknown;
+  if (
+    typeof (source as AsyncIterable<unknown>)[Symbol.asyncIterator] ===
+    "function"
+  ) {
+    const it = (source as AsyncIterable<unknown>)[
+      Symbol.asyncIterator
+    ]() as unknown;
+    return it === ref;
+  }
+  if (typeof (source as Iterable<unknown>)[Symbol.iterator] === "function") {
+    const it = (source as Iterable<unknown>)[Symbol.iterator]() as unknown;
+    return it === ref;
+  }
+  // Already-an-iterator (has `.next` but no `Symbol.iterator`) is by
+  // definition one-shot.
+  if (typeof (source as { next?: unknown }).next === "function") return true;
+  return false;
+}

nodejs/lancedb/table.ts

@@ -106,6 +106,27 @@ export interface Version {
   metadata: Record<string, string>;
 }
 
+/**
+ * Specification selecting Lance's MemWAL LSM-style write path for
+ * `mergeInsert`.
+ *
+ * `specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`,
+ * `column` and `numBuckets` are required; for `"identity"`, `column` is
+ * required.
+ */
+export interface LsmWriteSpec {
+  /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
+  specType: "bucket" | "identity" | "unsharded";
+  /** Bucket and identity variants: the sharding column. */
+  column?: string;
+  /** Bucket variant: the number of buckets, in `[1, 1024]`. */
+  numBuckets?: number;
+  /** Names of indexes the MemWAL should keep up to date during writes. */
+  maintainedIndexes?: string[];
+  /** Default `ShardWriter` configuration recorded in the MemWAL index. */
+  writerConfigDefaults?: Record<string, string>;
+}
+
 /**
  * A Table is a collection of Records in a LanceDB Database.
  *
@@ -285,6 +306,25 @@ export abstract class Table {
    */
   abstract prewarmIndex(name: string): Promise<void>;
 
+  /**
+   * Prewarm one or more columns of data in the table.
+   *
+   * @param columns The columns to prewarm. If undefined, all columns are prewarmed.
+   *
+   * This will load the column data into the page cache so that future queries that
+   * read those columns avoid the initial cold-start latency.  This call initiates
+   * prewarming and returns once the request is accepted; the warming itself may
+   * continue in the background.  Calling it on already-prewarmed columns is a
+   * no-op on the server.
+   *
+   * Prewarming is generally useful for columns used in filters or projections.
+   * Large columns (e.g. high-dimensional vectors or binary data) may not be
+   * practical to prewarm.
+   *
+   * This feature is currently only supported on remote tables.
+   */
+  abstract prewarmData(columns?: string[]): Promise<void>;
+
   /**
    * Waits for asynchronous indexing to complete on the table.
    *
@@ -430,6 +470,54 @@ export abstract class Table {
    * containing the new version number of the table after dropping the columns.
    */
   abstract dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
+  /**
+   * Set the unenforced primary key for this table to a single column.
+   *
+   * "Unenforced" means LanceDB does not check uniqueness on writes; the
+   * column is recorded in the schema as the primary key for use by features
+   * such as `merge_insert`. Only single-column primary keys are supported,
+   * and the key cannot be changed once set.
+   * @param {string | string[]} columns The primary key column. A one-element
+   * array is also accepted; passing more than one column is rejected.
+   * @returns {Promise<void>}
+   */
+  abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
+  /**
+   * Install an {@link LsmWriteSpec} on this table, selecting Lance's MemWAL
+   * LSM-style write path for future `mergeInsert` calls.
+   *
+   * `LsmWriteSpec` chooses one of three sharding strategies via `specType`:
+   *
+   * - `"bucket"` — hash-bucket writes by the single-column unenforced primary
+   *   key (`column` and `numBuckets` required).
+   * - `"identity"` — shard by the raw value of a scalar `column`.
+   * - `"unsharded"` — route every write to a single shard.
+   *
+   * All variants require the table to have an unenforced primary key
+   * ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally
+   * requires it to be the single column being bucketed.
+   * @param {LsmWriteSpec} spec The sharding spec to install.
+   * @returns {Promise<void>}
+   * @example
+   * ```ts
+   * await table.setUnenforcedPrimaryKey("id");
+   * await table.setLsmWriteSpec({
+   *   specType: "bucket",
+   *   column: "id",
+   *   numBuckets: 16,
+   *   maintainedIndexes: ["id_idx"],
+   * });
+   * ```
+   */
+  abstract setLsmWriteSpec(spec: LsmWriteSpec): Promise<void>;
+  /**
+   * Remove the {@link LsmWriteSpec} from this table, reverting to the standard
+   * `mergeInsert` write path.
+   *
+   * Errors if no spec is currently set.
+   * @returns {Promise<void>}
+   */
+  abstract unsetLsmWriteSpec(): Promise<void>;
   /** Retrieve the version of the table */
 
   abstract version(): Promise<number>;
@@ -710,6 +798,10 @@ export class LocalTable extends Table {
     await this.inner.prewarmIndex(name);
   }
 
+  async prewarmData(columns?: string[]): Promise<void> {
+    await this.inner.prewarmData(columns);
+  }
+
   async waitForIndex(
     indexNames: string[],
     timeoutSeconds: number,
@@ -874,6 +966,19 @@ export class LocalTable extends Table {
     return await this.inner.dropColumns(columnNames);
   }
 
+  async setUnenforcedPrimaryKey(columns: string | string[]): Promise<void> {
+    const cols = typeof columns === "string" ? [columns] : columns;
+    return await this.inner.setUnenforcedPrimaryKey(cols);
+  }
+
+  async setLsmWriteSpec(spec: LsmWriteSpec): Promise<void> {
+    return await this.inner.setLsmWriteSpec(spec);
+  }
+
+  async unsetLsmWriteSpec(): Promise<void> {
+    return await this.inner.unsetLsmWriteSpec();
+  }
+
   async version(): Promise<number> {
     return await this.inner.version();
   }

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.28.0-beta.2",
+	"version": "0.29.1-beta.0",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.28.0-beta.2",
+	"version": "0.29.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.28.0-beta.2",
+	"version": "0.29.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.28.0-beta.2",
+	"version": "0.29.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.28.0-beta.2",
+	"version": "0.29.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.28.0-beta.2",
+  "version": "0.29.1-beta.0",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.28.0-beta.2",
+	"version": "0.29.1-beta.0",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.28.0-beta.2",
+  "version": "0.29.1-beta.0",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -38,15 +38,15 @@
     "url": "https://github.com/lancedb/lancedb"
   },
   "devDependencies": {
-    "@aws-sdk/client-dynamodb": "^3.33.0",
-    "@aws-sdk/client-kms": "^3.33.0",
-    "@aws-sdk/client-s3": "^3.33.0",
+    "@aws-sdk/client-dynamodb": "3.1003.0",
+    "@aws-sdk/client-kms": "3.1003.0",
+    "@aws-sdk/client-s3": "3.1003.0",
     "@biomejs/biome": "^1.7.3",
     "@jest/globals": "^29.7.0",
-    "@napi-rs/cli": "^3.5.1",
+    "@napi-rs/cli": "3.5.1",
     "@types/axios": "^0.14.0",
     "@types/jest": "^29.1.2",
-    "@types/node": "^22.7.4",
+    "@types/node": "22.7.4",
     "@types/tmp": "^0.2.6",
     "apache-arrow-15": "npm:apache-arrow@15.0.0",
     "apache-arrow-16": "npm:apache-arrow@16.0.0",
@@ -57,9 +57,9 @@
     "shx": "^0.3.4",
     "tmp": "^0.2.3",
     "ts-jest": "^29.1.2",
-    "typedoc": "^0.26.4",
-    "typedoc-plugin-markdown": "^4.2.1",
-    "typescript": "^5.5.4",
+    "typedoc": "0.26.4",
+    "typedoc-plugin-markdown": "4.2.1",
+    "typescript": "5.5.4",
     "typescript-eslint": "^7.1.0"
   },
   "ava": {
@@ -68,16 +68,16 @@
   "engines": {
     "node": ">= 18"
   },
+  "packageManager": "pnpm@11.1.1",
   "cpu": ["x64", "arm64"],
   "os": ["darwin", "linux", "win32"],
   "scripts": {
     "artifacts": "napi artifacts",
     "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir lancedb",
-    "postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
+    "postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/ && node -e \"require('fs').writeFileSync('dist/package.json', JSON.stringify({name:'@lancedb/lancedb',type:'commonjs'}))\"",
     "build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir dist",
-    "postbuild:release": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
-    "build": "npm run build:debug && npm run tsc",
-    "build-release": "npm run build:release && npm run tsc",
+    "build": "pnpm build:debug && pnpm tsc",
+    "build-release": "pnpm build:release && pnpm tsc",
     "tsc": "tsc -b",
     "posttsc": "shx cp lancedb/native.d.ts dist/native.d.ts",
     "lint-ci": "biome ci .",
@@ -87,7 +87,7 @@
     "lint-fix": "biome check --write . && biome format --write .",
     "prepublishOnly": "napi prepublish -t npm",
     "test": "jest --verbose",
-    "integration": "S3_TEST=1 npm run test",
+    "integration": "S3_TEST=1 pnpm test",
     "universal": "napi universalize",
     "version": "napi version"
   },
@@ -95,8 +95,8 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@huggingface/transformers": "^3.0.2",
-    "openai": "^4.29.2"
+    "@huggingface/transformers": "3.0.2",
+    "openai": "4.29.2"
   },
   "peerDependencies": {
     "apache-arrow": ">=15.0.0 <=18.1.0"

nodejs/pnpm-workspace.yaml

@@ -0,0 +1,18 @@
+# Flat node_modules layout. The @napi-rs/cli build step fails to locate
+# the cdylib artifact under pnpm's isolated layout; the hoisted linker
+# mirrors npm's structure and unblocks the native build.
+nodeLinker: hoisted
+
+# Block resolution of versions less than 24h old (Shai-Hulud window).
+# This is the pnpm 11 default but pinned here so it's visible to
+# reviewers and survives a future pnpm major flipping the default.
+minimumReleaseAge: 1440
+
+# Fail install if a transitive dep tries to run an unapproved script.
+strictDepBuilds: true
+
+allowBuilds:
+  '@biomejs/biome': true
+  onnxruntime-node: true
+  protobufjs: true
+  sharp: true

nodejs/src/connection.rs

@@ -8,12 +8,16 @@ use lancedb::database::{CreateTableMode, Database};
 use napi::bindgen_prelude::*;
 use napi_derive::*;
 
+use crate::ConnectNamespaceOptions;
 use crate::ConnectionOptions;
 use crate::error::NapiErrorExt;
 use crate::header::JsHeaderProvider;
 use crate::table::Table;
-use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection};
+use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, connect_namespace};
 
+use lance_namespace::models::{
+    CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest,
+};
 use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
 
 #[napi]
@@ -21,6 +25,29 @@ pub struct Connection {
     inner: Option<LanceDBConnection>,
 }
 
+#[napi(object)]
+pub struct DescribeNamespaceResponse {
+    pub properties: Option<HashMap<String, String>>,
+}
+
+#[napi(object)]
+pub struct ListNamespacesResponse {
+    pub namespaces: Vec<String>,
+    pub page_token: Option<String>,
+}
+
+#[napi(object)]
+pub struct CreateNamespaceResponse {
+    pub properties: Option<HashMap<String, String>>,
+    pub transaction_id: Option<String>,
+}
+
+#[napi(object)]
+pub struct DropNamespaceResponse {
+    pub properties: Option<HashMap<String, String>>,
+    pub transaction_id: Option<Vec<String>>,
+}
+
 impl Connection {
     pub(crate) fn inner_new(inner: LanceDBConnection) -> Self {
         Self { inner: Some(inner) }
@@ -67,6 +94,12 @@ impl Connection {
                 builder = builder.storage_option(key, value);
             }
         }
+        if let Some(manifest_enabled) = options.manifest_enabled {
+            builder = builder.manifest_enabled(manifest_enabled);
+        }
+        if let Some(namespace_client_properties) = options.namespace_client_properties {
+            builder = builder.namespace_client_properties(namespace_client_properties);
+        }
 
         // Create client config, optionally with header provider
         let client_config = options.client_config.unwrap_or_default();
@@ -100,6 +133,39 @@ impl Connection {
         Ok(Self::inner_new(builder.execute().await.default_error()?))
     }
 
+    /// Create a new Connection instance backed by a namespace implementation.
+    #[napi(factory)]
+    pub async fn new_with_namespace(
+        impl_name: String,
+        properties: HashMap<String, String>,
+        options: ConnectNamespaceOptions,
+    ) -> napi::Result<Self> {
+        if impl_name.is_empty() {
+            return Err(napi::Error::from_reason(
+                "implName must be a non-empty string",
+            ));
+        }
+
+        let mut builder = connect_namespace(&impl_name, properties);
+        if let Some(interval) = options.read_consistency_interval {
+            builder =
+                builder.read_consistency_interval(std::time::Duration::from_secs_f64(interval));
+        }
+        if let Some(storage_options) = options.storage_options {
+            for (key, value) in storage_options {
+                builder = builder.storage_option(key, value);
+            }
+        }
+        if let Some(namespace_client_properties) = options.namespace_client_properties {
+            builder = builder.namespace_client_properties(namespace_client_properties);
+        }
+        if let Some(session) = options.session {
+            builder = builder.session(session.inner.clone());
+        }
+
+        Ok(Self::inner_new(builder.execute().await.default_error()?))
+    }
+
     #[napi]
     pub fn display(&self) -> napi::Result<String> {
         Ok(self.get_inner()?.to_string())
@@ -262,9 +328,149 @@ impl Connection {
             .default_error()
     }
 
+    #[napi(catch_unwind)]
+    pub async fn rename_table(
+        &self,
+        old_name: String,
+        new_name: String,
+        namespace_path: Option<Vec<String>>,
+    ) -> napi::Result<()> {
+        let ns = namespace_path.unwrap_or_default();
+        self.get_inner()?
+            .rename_table(&old_name, &new_name, &ns, &ns)
+            .await
+            .default_error()
+    }
+
     #[napi(catch_unwind)]
     pub async fn drop_all_tables(&self, namespace_path: Option<Vec<String>>) -> napi::Result<()> {
         let ns = namespace_path.unwrap_or_default();
         self.get_inner()?.drop_all_tables(&ns).await.default_error()
     }
+
+    #[napi(catch_unwind)]
+    /// Describe a namespace and return its properties.
+    pub async fn describe_namespace(
+        &self,
+        namespace_path: Vec<String>,
+    ) -> napi::Result<DescribeNamespaceResponse> {
+        let req = DescribeNamespaceRequest {
+            id: Some(namespace_path),
+            ..Default::default()
+        };
+        let resp = self
+            .get_inner()?
+            .describe_namespace(req)
+            .await
+            .default_error()?;
+        Ok(DescribeNamespaceResponse {
+            properties: resp.properties,
+        })
+    }
+
+    #[napi(catch_unwind)]
+    /// List child namespaces under the given namespace path
+    pub async fn list_namespaces(
+        &self,
+        namespace_path: Option<Vec<String>>,
+        page_token: Option<String>,
+        limit: Option<u32>,
+    ) -> napi::Result<ListNamespacesResponse> {
+        let req = ListNamespacesRequest {
+            id: namespace_path,
+            page_token,
+            limit: limit.map(|l| l as i32),
+            ..Default::default()
+        };
+        let resp = self
+            .get_inner()?
+            .list_namespaces(req)
+            .await
+            .default_error()?;
+        Ok(ListNamespacesResponse {
+            namespaces: resp.namespaces,
+            page_token: resp.page_token,
+        })
+    }
+
+    #[napi(catch_unwind)]
+    /// Create a new namespace with optional properties.
+    pub async fn create_namespace(
+        &self,
+        namespace_path: Vec<String>,
+        mode: Option<String>,
+        properties: Option<HashMap<String, String>>,
+    ) -> napi::Result<CreateNamespaceResponse> {
+        let mode_str = mode
+            .map(|m| match m.to_lowercase().as_str() {
+                "create" => Ok("Create".to_string()),
+                "exist_ok" => Ok("ExistOk".to_string()),
+                "overwrite" => Ok("Overwrite".to_string()),
+                _ => Err(napi::Error::from_reason(format!(
+                    "Invalid mode '{}': expected one of 'create', 'exist_ok', 'overwrite'",
+                    m
+                ))),
+            })
+            .transpose()?;
+        let req = CreateNamespaceRequest {
+            id: Some(namespace_path),
+            mode: mode_str,
+            properties,
+            ..Default::default()
+        };
+        let resp = self
+            .get_inner()?
+            .create_namespace(req)
+            .await
+            .default_error()?;
+        Ok(CreateNamespaceResponse {
+            properties: resp.properties,
+            transaction_id: resp.transaction_id,
+        })
+    }
+
+    #[napi(catch_unwind)]
+    /// Drop a namespace.
+    pub async fn drop_namespace(
+        &self,
+        namespace_path: Vec<String>,
+        mode: Option<String>,
+        behavior: Option<String>,
+    ) -> napi::Result<DropNamespaceResponse> {
+        let mode_str = mode
+            .map(|m| match m.to_lowercase().as_str() {
+                "skip" => Ok("Skip".to_string()),
+                "fail" => Ok("Fail".to_string()),
+                _ => Err(napi::Error::from_reason(format!(
+                    "Invalid mode '{}': expected one of 'skip', 'fail'",
+                    m
+                ))),
+            })
+            .transpose()?;
+        let behavior_str = behavior
+            .map(|b| match b.to_lowercase().as_str() {
+                "restrict" => Ok("Restrict".to_string()),
+                "cascade" => Ok("Cascade".to_string()),
+                _ => Err(napi::Error::from_reason(format!(
+                    "Invalid behavior '{}': expected one of 'restrict', 'cascade'",
+                    b
+                ))),
+            })
+            .transpose()?;
+        let req = DropNamespaceRequest {
+            id: Some(namespace_path),
+            mode: mode_str,
+            behavior: behavior_str,
+            ..Default::default()
+        };
+        let resp = self
+            .get_inner()?
+            .drop_namespace(req)
+            .await
+            .default_error()?;
+        Ok(DropNamespaceResponse {
+            properties: resp.properties,
+            transaction_id: resp.transaction_id,
+        })
+    }
 }

nodejs/src/lib.rs

@@ -16,6 +16,7 @@ pub mod permutation;
 mod query;
 pub mod remote;
 mod rerankers;
+mod scannable;
 mod session;
 mod table;
 mod util;
@@ -35,8 +36,15 @@ pub struct ConnectionOptions {
     pub read_consistency_interval: Option<f64>,
     /// (For LanceDB OSS only): configuration for object storage.
     ///
-    /// The available options are described at https://lancedb.com/docs/storage/
+    /// The available options are described at https://docs.lancedb.com/storage/
     pub storage_options: Option<HashMap<String, String>>,
+    /// (For LanceDB OSS only): use directory namespace manifests as the source
+    /// of truth for table metadata. Existing directory-listed root tables are
+    /// migrated into the manifest on access.
+    pub manifest_enabled: Option<bool>,
+    /// (For LanceDB OSS only): extra properties for the backing namespace
+    /// client used by manifest-enabled native connections.
+    pub namespace_client_properties: Option<HashMap<String, String>>,
     /// (For LanceDB OSS only): the session to use for this connection. Holds
     /// shared caches and other session-specific state.
     pub session: Option<session::Session>,
@@ -60,6 +68,26 @@ pub struct OpenTableOptions {
     pub storage_options: Option<HashMap<String, String>>,
 }
 
+#[napi(object)]
+#[derive(Debug)]
+pub struct ConnectNamespaceOptions {
+    /// The interval, in seconds, at which to check for updates to the table
+    /// from other processes. If None, then consistency is not checked. For
+    /// performance reasons, this is the default. For strong consistency, set
+    /// this to zero seconds. Then every read will check for updates from other
+    /// processes. As a compromise, you can set this to a non-zero value for
+    /// eventual consistency.
+    pub read_consistency_interval: Option<f64>,
+    /// Configuration for object storage. The available options are described
+    /// at https://docs.lancedb.com/storage/
+    pub storage_options: Option<HashMap<String, String>>,
+    /// Extra properties for the backing namespace client.
+    pub namespace_client_properties: Option<HashMap<String, String>>,
+    /// The session to use for this connection. Holds shared caches and other
+    /// session-specific state.
+    pub session: Option<session::Session>,
+}
+
 #[napi_derive::module_init]
 fn init() {
     let env = Env::new()

nodejs/src/query.rs

@@ -3,6 +3,12 @@
 
 use std::sync::Arc;
 
+use crate::error::NapiErrorExt;
+use crate::error::convert_error;
+use crate::iterator::RecordBatchIterator;
+use crate::rerankers::RerankHybridCallbackArgs;
+use crate::rerankers::Reranker;
+use crate::util::{parse_distance_type, schema_to_buffer};
 use arrow_array::{
     Array, Float16Array as ArrowFloat16Array, Float32Array as ArrowFloat32Array,
     Float64Array as ArrowFloat64Array, UInt8Array as ArrowUInt8Array,
@@ -19,16 +25,27 @@ use lancedb::query::QueryBase;
 use lancedb::query::QueryExecutionOptions;
 use lancedb::query::Select;
 use lancedb::query::TakeQuery as LanceDbTakeQuery;
-use lancedb::query::VectorQuery as LanceDbVectorQuery;
+use lancedb::query::{ColumnOrdering as LanceDbColumnOrdering, VectorQuery as LanceDbVectorQuery};
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
 
-use crate::error::NapiErrorExt;
-use crate::error::convert_error;
-use crate::iterator::RecordBatchIterator;
-use crate::rerankers::RerankHybridCallbackArgs;
-use crate::rerankers::Reranker;
-use crate::util::{parse_distance_type, schema_to_buffer};
+#[napi(object)]
+pub struct ColumnOrdering {
+    pub ascending: bool,
+    pub nulls_first: bool,
+    pub column_name: String,
+}
+
+impl From<ColumnOrdering> for LanceDbColumnOrdering {
+    fn from(value: ColumnOrdering) -> Self {
+        match (value.ascending, value.nulls_first) {
+            (true, true) => Self::asc_nulls_first(value.column_name),
+            (true, false) => Self::asc_nulls_last(value.column_name),
+            (false, true) => Self::desc_nulls_first(value.column_name),
+            (false, false) => Self::desc_nulls_last(value.column_name),
+        }
+    }
+}
 
 fn bytes_to_arrow_array(data: Uint8Array, dtype: String) -> napi::Result<Arc<dyn Array>> {
     let buf = arrow_buffer::Buffer::from(data.to_vec());
@@ -128,6 +145,18 @@ impl Query {
         self.inner = self.inner.clone().with_row_id();
     }
 
+    #[napi]
+    pub fn order_by(&mut self, ordering: Option<Vec<ColumnOrdering>>) -> napi::Result<()> {
+        let ordering = ordering.map(|ordering| {
+            ordering
+                .into_iter()
+                .map(LanceDbColumnOrdering::from)
+                .collect()
+        });
+        self.inner = self.inner.clone().order_by(ordering);
+        Ok(())
+    }
+
     #[napi(catch_unwind)]
     pub async fn output_schema(&self) -> napi::Result<Buffer> {
         let schema = self.inner.output_schema().await.default_error()?;
@@ -328,6 +357,18 @@ impl VectorQuery {
         Ok(())
     }
 
+    #[napi]
+    pub fn order_by(&mut self, ordering: Option<Vec<ColumnOrdering>>) -> napi::Result<()> {
+        let ordering = ordering.map(|ordering| {
+            ordering
+                .into_iter()
+                .map(LanceDbColumnOrdering::from)
+                .collect()
+        });
+        self.inner = self.inner.clone().order_by(ordering);
+        Ok(())
+    }
+
     #[napi(catch_unwind)]
     pub async fn output_schema(&self) -> napi::Result<Buffer> {
         let schema = self.inner.output_schema().await.default_error()?;

nodejs/src/rerankers.rs

@@ -18,6 +18,7 @@ type RerankHybridFn = ThreadsafeFunction<
     RerankHybridCallbackArgs,
     Status,
     false,
+    true,
 >;
 
 /// Reranker implementation that "wraps" a NodeJS Reranker implementation.
@@ -32,7 +33,10 @@ impl Reranker {
     pub fn new(
         rerank_hybrid: Function<RerankHybridCallbackArgs, Promise<Buffer>>,
     ) -> napi::Result<Self> {
-        let rerank_hybrid = rerank_hybrid.build_threadsafe_function().build()?;
+        let rerank_hybrid = rerank_hybrid
+            .build_threadsafe_function()
+            .weak::<true>()
+            .build()?;
         Ok(Self { rerank_hybrid })
     }
 }

nodejs/src/scannable.rs

@@ -0,0 +1,253 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! NodeJS binding for the [`lancedb::data::scannable::Scannable`] trait.
+//!
+//! The JS side supplies a `getNextBatch(isStart)` callback that returns the
+//! next Arrow `RecordBatch` encoded as a self-contained Arrow IPC Stream
+//! message (schema message + record batch message + EOS marker) wrapped in a
+//! `Buffer`, or `null` when the stream is exhausted. The Rust side parses
+//! each buffer with `arrow_ipc::reader::StreamReader`, validates every
+//! standalone batch stream against the declared schema, and yields decoded
+//! `RecordBatch`es as a [`SendableRecordBatchStream`].
+//!
+//! `isStart` is `true` on the first `getNextBatch` call of each new
+//! `scan_as_stream` and `false` thereafter. JS uses it to drop any cached
+//! iterator and re-invoke its factory at scan boundaries, so retries
+//! triggered by mid-stream failures restart at batch 0.
+
+use std::io::Cursor;
+use std::sync::Arc;
+
+use arrow_array::RecordBatch;
+use arrow_ipc::reader::StreamReader;
+use arrow_schema::SchemaRef;
+use futures::stream::once;
+use lancedb::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream};
+use lancedb::data::scannable::Scannable as LanceScannable;
+use lancedb::ipc::ipc_file_to_schema;
+use lancedb::{Error, Result as LanceResult};
+use napi::bindgen_prelude::*;
+use napi::threadsafe_function::ThreadsafeFunction;
+use napi_derive::napi;
+
+/// Threadsafe handle to the JS `getNextBatch` callback. The callback takes a
+/// single boolean `isStart` (`true` on the first call of each new scan) and
+/// returns a Promise that resolves to a `Buffer` containing one IPC Stream
+/// message, or `null` at end-of-stream.
+type GetNextBatchFn = ThreadsafeFunction<bool, Promise<Option<Buffer>>, bool, Status, false>;
+
+/// A Rust-side view of a JS-constructed `Scannable`.
+///
+/// Held in JS as the return value of the `Scannable` class constructor. When
+/// passed to a consumer that accepts `impl lancedb::data::scannable::Scannable`,
+/// the consumer invokes `scan_as_stream()` to pull batches through the JS
+/// callback.
+#[napi]
+pub struct NapiScannable {
+    schema: SchemaRef,
+    num_rows: Option<usize>,
+    rescannable: bool,
+    // `ThreadsafeFunction` is not `Clone`; wrap in `Arc` so the stream
+    // returned by `scan_as_stream` can own a handle independent of `self`.
+    get_next_batch: Arc<GetNextBatchFn>,
+    // Tracks whether a scan has already started; used to enforce one-shot
+    // semantics on non-rescannable sources.
+    scanned: bool,
+}
+
+#[napi]
+impl NapiScannable {
+    /// Construct a new `NapiScannable`.
+    ///
+    /// - `schema_buf` — Arrow IPC File buffer carrying only the schema (no batches).
+    /// - `num_rows` — optional row count hint; not validated against the stream.
+    /// - `rescannable` — whether `get_next_batch` may be re-driven after the
+    ///   scan completes.
+    /// - `get_next_batch` -- JS callback that yields the next batch as an Arrow
+    ///   IPC Stream message wrapped in a `Buffer`, or `null` at EOF. The
+    ///   `isStart` argument is `true` on the first call of each new scan;
+    ///   JS uses it to discard any cached iterator before pulling.
+    #[napi(constructor)]
+    pub fn new(
+        schema_buf: Buffer,
+        num_rows: Option<i64>,
+        rescannable: bool,
+        get_next_batch: Function<bool, Promise<Option<Buffer>>>,
+    ) -> napi::Result<Self> {
+        let schema = ipc_file_to_schema(schema_buf.to_vec())
+            .map_err(|e| napi::Error::from_reason(format!("Invalid schema buffer: {}", e)))?;
+        let num_rows = num_rows
+            .map(|n| {
+                usize::try_from(n)
+                    .map_err(|_| napi::Error::from_reason("num_rows must be non-negative"))
+            })
+            .transpose()?;
+        let get_next_batch = Arc::new(get_next_batch.build_threadsafe_function().build()?);
+        Ok(Self {
+            schema,
+            num_rows,
+            rescannable,
+            get_next_batch,
+            scanned: false,
+        })
+    }
+}
+
+impl std::fmt::Debug for NapiScannable {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("NapiScannable")
+            .field("schema", &self.schema)
+            .field("num_rows", &self.num_rows)
+            .field("rescannable", &self.rescannable)
+            .finish()
+    }
+}
+
+impl LanceScannable for NapiScannable {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn scan_as_stream(&mut self) -> SendableRecordBatchStream {
+        let schema = self.schema.clone();
+
+        // One-shot enforcement for non-rescannable sources: return a stream
+        // whose first item is an error.
+        if self.scanned && !self.rescannable {
+            let err_stream = once(async {
+                Err(Error::InvalidInput {
+                    message: "Scannable has already been consumed (non-rescannable source)"
+                        .to_string(),
+                })
+            });
+            return Box::pin(SimpleRecordBatchStream::new(err_stream, schema));
+        }
+        self.scanned = true;
+
+        let tsfn = Arc::clone(&self.get_next_batch);
+        let declared_schema = schema.clone();
+
+        // State threaded through the unfold. `is_first_pull` starts true so
+        // the first call into JS signals a new-scan boundary; JS uses it to
+        // reset any cached iterator before factory()-ing a fresh one.
+        let initial = State {
+            tsfn,
+            batch_index: 0,
+            declared_schema,
+            errored: false,
+            is_first_pull: true,
+        };
+
+        let stream = futures::stream::unfold(initial, |mut state| async move {
+            if state.errored {
+                return None;
+            }
+
+            // Pull the next IPC Stream buffer from JS. `is_first_pull` is
+            // consumed here and cleared so subsequent pulls continue the
+            // same scan rather than restarting it.
+            let is_start = state.is_first_pull;
+            state.is_first_pull = false;
+            let buf = match pull_next(&state.tsfn, is_start).await {
+                Ok(Some(buf)) => buf,
+                Ok(None) => return None,
+                Err(e) => {
+                    state.errored = true;
+                    return Some((Err(e), state));
+                }
+            };
+
+            match decode_one_batch(buf.as_ref(), &state.declared_schema) {
+                Ok(batch) => {
+                    state.batch_index += 1;
+                    Some((Ok(batch), state))
+                }
+                Err(e) => {
+                    let tagged = Error::Runtime {
+                        message: format!(
+                            "[scannable/rust-bridge] failure at batch index {}: {}",
+                            state.batch_index, e
+                        ),
+                    };
+                    state.errored = true;
+                    Some((Err(tagged), state))
+                }
+            }
+        });
+
+        Box::pin(SimpleRecordBatchStream::new(stream, schema))
+    }
+
+    fn num_rows(&self) -> Option<usize> {
+        self.num_rows
+    }
+
+    fn rescannable(&self) -> bool {
+        self.rescannable
+    }
+}
+
+struct State {
+    tsfn: Arc<GetNextBatchFn>,
+    batch_index: usize,
+    declared_schema: SchemaRef,
+    errored: bool,
+    /// True for the very first pull of a new scan. Forwarded to JS so the
+    /// callback can drop any cached iterator and call its factory fresh,
+    /// which makes rescannable sources restart at batch 0 even when the
+    /// previous scan ended mid-stream.
+    is_first_pull: bool,
+}
+
+/// Invoke the JS callback and await its Promise. `is_start` is forwarded to
+/// the JS side as the `isStart` argument so it can reset its iterator at the
+/// scan boundary. Errors on the JS side surface here as rejected promises
+/// and are tunneled back as `lancedb::Error::Runtime`.
+async fn pull_next(tsfn: &GetNextBatchFn, is_start: bool) -> LanceResult<Option<Buffer>> {
+    let promise = tsfn
+        .call_async(is_start)
+        .await
+        .map_err(|e| Error::Runtime {
+            message: format!(
+                "[scannable/js-factory] napi error status={}, reason={}",
+                e.status, e.reason
+            ),
+        })?;
+    promise.await.map_err(|e| Error::Runtime {
+        message: format!(
+            "[scannable/js-iterator] napi error status={}, reason={}",
+            e.status, e.reason
+        ),
+    })
+}
+
+/// Decode one IPC Stream buffer (schema + batch + EOS) into a `RecordBatch`.
+/// Each buffer is a standalone IPC stream, so every decoded stream schema must
+/// match the one declared at construction.
+fn decode_one_batch(buf: &[u8], declared: &SchemaRef) -> LanceResult<RecordBatch> {
+    let reader = StreamReader::try_new(Cursor::new(buf), None).map_err(|e| Error::Runtime {
+        message: format!("failed to open IPC stream reader: {}", e),
+    })?;
+
+    let actual = reader.schema();
+    if actual.as_ref() != declared.as_ref() {
+        return Err(Error::InvalidInput {
+            message: format!(
+                "declared schema does not match stream schema: declared={:?} actual={:?}",
+                declared, actual
+            ),
+        });
+    }
+
+    let mut iter = reader;
+    let batch = iter
+        .next()
+        .ok_or_else(|| Error::Runtime {
+            message: "IPC stream contained schema but no record batch".to_string(),
+        })?
+        .map_err(|e| Error::Runtime {
+            message: format!("failed to decode record batch: {}", e),
+        })?;
+    Ok(batch)
+}

nodejs/src/table.rs

@@ -159,6 +159,14 @@ impl Table {
             .default_error()
     }
 
+    #[napi(catch_unwind)]
+    pub async fn prewarm_data(&self, columns: Option<Vec<String>>) -> napi::Result<()> {
+        self.inner_ref()?
+            .prewarm_data(columns)
+            .await
+            .default_error()
+    }
+
     #[napi(catch_unwind)]
     pub async fn wait_for_index(&self, index_names: Vec<String>, timeout_s: i64) -> Result<()> {
         let timeout = std::time::Duration::from_secs(timeout_s.try_into().unwrap());
@@ -336,6 +344,31 @@ impl Table {
         Ok(res.into())
     }
 
+    #[napi(catch_unwind)]
+    pub async fn set_unenforced_primary_key(&self, columns: Vec<String>) -> napi::Result<()> {
+        self.inner_ref()?
+            .set_unenforced_primary_key(columns)
+            .await
+            .default_error()
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> napi::Result<()> {
+        let native_spec = lancedb::table::LsmWriteSpec::try_from(spec)?;
+        self.inner_ref()?
+            .set_lsm_write_spec(native_spec)
+            .await
+            .default_error()
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn unset_lsm_write_spec(&self) -> napi::Result<()> {
+        self.inner_ref()?
+            .unset_lsm_write_spec()
+            .await
+            .default_error()
+    }
+
     #[napi(catch_unwind)]
     pub async fn version(&self) -> napi::Result<i64> {
         self.inner_ref()?
@@ -530,6 +563,63 @@ impl From<lancedb::index::IndexConfig> for IndexConfig {
     }
 }
 
+/// Specification selecting Lance's MemWAL LSM-style write path for
+/// `mergeInsert`.
+///
+/// `specType` must be `"bucket"`, `"identity"`, or `"unsharded"`. For
+/// `"bucket"`, `column` and `numBuckets` are required; for `"identity"`,
+/// `column` is required.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct LsmWriteSpec {
+    /// One of `"bucket"`, `"identity"`, or `"unsharded"`.
+    pub spec_type: String,
+    /// Bucket and identity variants: the sharding column.
+    pub column: Option<String>,
+    /// Bucket variant: the number of buckets, in `[1, 1024]`.
+    pub num_buckets: Option<u32>,
+    /// Names of indexes the MemWAL should keep up to date during writes.
+    pub maintained_indexes: Option<Vec<String>>,
+    /// Default `ShardWriter` configuration recorded in the MemWAL index.
+    pub writer_config_defaults: Option<HashMap<String, String>>,
+}
+
+impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
+    type Error = napi::Error;
+
+    fn try_from(value: LsmWriteSpec) -> napi::Result<Self> {
+        let maintained = value.maintained_indexes.unwrap_or_default();
+        let writer_config_defaults = value.writer_config_defaults.unwrap_or_default();
+        let spec = match value.spec_type.as_str() {
+            "bucket" => {
+                let column = value.column.ok_or_else(|| {
+                    napi::Error::from_reason("LsmWriteSpec bucket requires `column`")
+                })?;
+                let num_buckets = value.num_buckets.ok_or_else(|| {
+                    napi::Error::from_reason("LsmWriteSpec bucket requires `numBuckets`")
+                })?;
+                Self::bucket(column, num_buckets)
+            }
+            "identity" => {
+                let column = value.column.ok_or_else(|| {
+                    napi::Error::from_reason("LsmWriteSpec identity requires `column`")
+                })?;
+                Self::identity(column)
+            }
+            "unsharded" => Self::unsharded(),
+            other => {
+                return Err(napi::Error::from_reason(format!(
+                    "LsmWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'",
+                    other
+                )));
+            }
+        };
+        Ok(spec
+            .with_maintained_indexes(maintained)
+            .with_writer_config_defaults(writer_config_defaults))
+    }
+}
+
 /// Statistics about a compaction operation.
 #[napi(object)]
 #[derive(Clone, Debug)]

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.31.0-beta.3"
+current_version = "0.32.1-beta.0"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,6 +1,7 @@
 [package]
 name = "lancedb-python"
-version = "0.31.0-beta.3"
+version = "0.32.1-beta.0"
+publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true
@@ -14,18 +15,19 @@ name = "_lancedb"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "57.2", features = ["pyarrow"] }
+arrow = { version = "58.0.0", features = ["pyarrow"] }
 async-trait = "0.1"
 bytes = "1"
 lancedb = { path = "../rust/lancedb", default-features = false }
+datafusion-common.workspace = true
 lance-core.workspace = true
 lance-namespace.workspace = true
 lance-namespace-impls.workspace = true
 lance-io.workspace = true
 env_logger.workspace = true
 log.workspace = true
-pyo3 = { version = "0.26", features = ["extension-module", "abi3-py39"] }
-pyo3-async-runtimes = { version = "0.26", features = [
+pyo3 = { version = "0.28", features = ["extension-module", "abi3-py39"] }
+pyo3-async-runtimes = { version = "0.28", features = [
     "attributes",
     "tokio-runtime",
 ] }
@@ -34,10 +36,11 @@ futures.workspace = true
 serde = "1"
 serde_json = "1"
 snafu.workspace = true
-tokio = { version = "1.40", features = ["sync"] }
+tokio = { version = "1.40", features = ["sync", "rt-multi-thread"] }
+libc = "0.2"
 
 [build-dependencies]
-pyo3-build-config = { version = "0.26", features = [
+pyo3-build-config = { version = "0.28", features = [
     "extension-module",
     "abi3-py39",
 ] }

python/PYTHON_THIRD_PARTY_LICENSES.md

@@ -183,7 +183,6 @@
 | stack-data                     | 0.6.3           | MIT License                                                                                      | http://github.com/alexmojaki/stack_data                                                           |
 | sympy                          | 1.14.0          | BSD License                                                                                      | https://sympy.org                                                                                 |
 | tabulate                       | 0.9.0           | MIT License                                                                                      | https://github.com/astanin/python-tabulate                                                        |
-| tantivy                        | 0.25.1          | UNKNOWN                                                                                          | UNKNOWN                                                                                           |
 | threadpoolctl                  | 3.6.0           | BSD License                                                                                      | https://github.com/joblib/threadpoolctl                                                           |
 | timm                           | 1.0.24          | Apache Software License                                                                          | https://github.com/huggingface/pytorch-image-models                                               |
 | tinycss2                       | 1.4.0           | BSD License                                                                                      | https://www.courtbouillon.org/tinycss2                                                            |

python/pyproject.toml

@@ -57,7 +57,6 @@ tests = [
     "duckdb>=0.9.0",
     "pytz>=2023.3",
     "polars>=0.19, <=1.3.0",
-    "tantivy>=0.20.0",
     "pyarrow-stubs>=16.0",
     "pylance>=5.0.0b5",
     "requests>=2.31.0",

python/python/lancedb/__init__.py

@@ -7,7 +7,6 @@ import os
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
 from typing import Dict, Optional, Union, Any, List
-import warnings
 
 __version__ = importlib.metadata.version("lancedb")
 
@@ -73,6 +72,7 @@ def connect(
     client_config: Union[ClientConfig, Dict[str, Any], None] = None,
     storage_options: Optional[Dict[str, str]] = None,
     session: Optional[Session] = None,
+    manifest_enabled: bool = False,
     namespace_client_impl: Optional[str] = None,
     namespace_client_properties: Optional[Dict[str, str]] = None,
     namespace_client_pushdown_operations: Optional[List[str]] = None,
@@ -110,7 +110,11 @@ def connect(
         default configuration is used.
     storage_options: dict, optional
         Additional options for the storage backend. See available options at
-        <https://lancedb.com/docs/storage/>
+        <https://docs.lancedb.com/storage/>
+    manifest_enabled : bool, default False
+        When true for local/native connections, use directory namespace
+        manifests as the source of truth for table metadata. Existing
+        directory-listed root tables are migrated into the manifest on access.
     session: Session, optional
         (For LanceDB OSS only)
         A session to use for this connection. Sessions allow you to configure
@@ -158,11 +162,11 @@ def connect(
     conn : DBConnection
         A connection to a LanceDB database.
     """
-    if namespace_client_impl is not None or namespace_client_properties is not None:
-        if namespace_client_impl is None or namespace_client_properties is None:
+    if namespace_client_impl is not None:
+        if namespace_client_properties is None:
             raise ValueError(
-                "Both namespace_client_impl and "
-                "namespace_client_properties must be provided"
+                "namespace_client_properties must be provided when "
+                "namespace_client_impl is set"
             )
         if kwargs:
             raise ValueError(f"Unknown keyword arguments: {kwargs}")
@@ -175,6 +179,12 @@ def connect(
             namespace_client_pushdown_operations=namespace_client_pushdown_operations,
         )
 
+    if namespace_client_properties is not None and not manifest_enabled:
+        raise ValueError(
+            "namespace_client_impl must be provided when using "
+            "namespace_client_properties unless manifest_enabled=True"
+        )
+
     if namespace_client_pushdown_operations is not None:
         raise ValueError(
             "namespace_client_pushdown_operations is only valid when "
@@ -212,9 +222,92 @@ def connect(
         read_consistency_interval=read_consistency_interval,
         storage_options=storage_options,
         session=session,
+        manifest_enabled=manifest_enabled,
+        namespace_client_properties=namespace_client_properties,
     )
 
 
+WORKER_PROPERTY_PREFIX = "_lancedb_worker_"
+
+
+def _apply_worker_overrides(props: dict[str, str]) -> dict[str, str]:
+    """Apply worker property overrides.
+
+    Any key starting with ``_lancedb_worker_`` is extracted, the prefix
+    is stripped, and the resulting key-value pair is put back into the
+    map (overriding the existing value if present).  The original
+    prefixed key is removed.
+    """
+    worker_keys = [k for k in props if k.startswith(WORKER_PROPERTY_PREFIX)]
+    if not worker_keys:
+        return props
+    result = dict(props)
+    for key in worker_keys:
+        value = result.pop(key)
+        real_key = key[len(WORKER_PROPERTY_PREFIX) :]
+        result[real_key] = value
+    return result
+
+
+def deserialize_conn(
+    data: str,
+    *,
+    for_worker: bool = False,
+) -> DBConnection:
+    """Reconstruct a DBConnection from a serialized string.
+
+    The string must have been produced by
+    :meth:`DBConnection.serialize`.
+
+    Parameters
+    ----------
+    data : str
+        String produced by ``serialize()``.
+    for_worker : bool, default False
+        When ``True``, any namespace client property whose key starts
+        with ``_lancedb_worker_`` has that prefix stripped and the
+        value overrides the corresponding property.  For example,
+        ``_lancedb_worker_uri`` replaces ``uri``.
+
+    Returns
+    -------
+    DBConnection
+        A new connection matching the serialized state.
+    """
+    import json
+
+    parsed = json.loads(data)
+    connection_type = parsed.get("connection_type")
+
+    rci_secs = parsed.get("read_consistency_interval_seconds")
+    rci = timedelta(seconds=rci_secs) if rci_secs is not None else None
+    storage_options = parsed.get("storage_options")
+
+    if connection_type == "namespace":
+        props = dict(parsed.get("namespace_client_properties") or {})
+        if for_worker:
+            props = _apply_worker_overrides(props)
+        return connect_namespace(
+            namespace_client_impl=parsed["namespace_client_impl"],
+            namespace_client_properties=props,
+            read_consistency_interval=rci,
+            storage_options=storage_options,
+            namespace_client_pushdown_operations=parsed.get(
+                "namespace_client_pushdown_operations"
+            ),
+        )
+    elif connection_type == "local":
+        return LanceDBConnection(
+            parsed["uri"],
+            read_consistency_interval=rci,
+            storage_options=storage_options,
+            manifest_enabled=parsed.get("manifest_enabled", False),
+            namespace_client_properties=parsed.get("namespace_client_properties"),
+        )
+    else:
+        raise ValueError(f"Unknown connection_type: {connection_type}")
+
+
 async def connect_async(
     uri: URI,
     *,
@@ -225,6 +318,8 @@ async def connect_async(
     client_config: Optional[Union[ClientConfig, Dict[str, Any]]] = None,
     storage_options: Optional[Dict[str, str]] = None,
     session: Optional[Session] = None,
+    manifest_enabled: bool = False,
+    namespace_client_properties: Optional[Dict[str, str]] = None,
 ) -> AsyncConnection:
     """Connect to a LanceDB database.
 
@@ -257,13 +352,20 @@ async def connect_async(
         default configuration is used.
     storage_options: dict, optional
         Additional options for the storage backend. See available options at
-        <https://lancedb.com/docs/storage/>
+        <https://docs.lancedb.com/storage/>
     session: Session, optional
         (For LanceDB OSS only)
         A session to use for this connection. Sessions allow you to configure
         cache sizes for index and metadata caches, which can significantly
         impact memory use and performance. They can also be re-used across
         multiple connections to share the same cache state.
+    manifest_enabled : bool, default False
+        When true for local/native connections, use directory namespace
+        manifests as the source of truth for table metadata. Existing
+        directory-listed root tables are migrated into the manifest on access.
+    namespace_client_properties : dict, optional
+        Additional directory namespace client properties to use with
+        ``manifest_enabled=True``.
 
     Examples
     --------
@@ -306,6 +408,8 @@ async def connect_async(
             client_config,
             storage_options,
             session,
+            manifest_enabled,
+            namespace_client_properties,
         )
     )
 
@@ -333,13 +437,3 @@ __all__ = [
     "Table",
     "__version__",
 ]
-
-
-def __warn_on_fork():
-    warnings.warn(
-        "lance is not fork-safe. If you are using multiprocessing, use spawn instead.",
-    )
-
-
-if hasattr(os, "register_at_fork"):
-    os.register_at_fork(before=__warn_on_fork)  # type: ignore[attr-defined]

python/python/lancedb/_lancedb.pyi

@@ -12,6 +12,7 @@ from .index import (
     LabelList,
     HnswPq,
     HnswSq,
+    HnswFlat,
     FTS,
 )
 from lance_namespace import (
@@ -25,6 +26,7 @@ from .remote import ClientConfig
 
 IvfHnswPq: type[HnswPq] = HnswPq
 IvfHnswSq: type[HnswSq] = HnswSq
+IvfHnswFlat: type[HnswFlat] = HnswFlat
 
 class PyExpr:
     """A type-safe DataFusion expression node (Rust-side handle)."""
@@ -49,7 +51,7 @@ class PyExpr:
     def to_sql(self) -> str: ...
 
 def expr_col(name: str) -> PyExpr: ...
-def expr_lit(value: Union[bool, int, float, str]) -> PyExpr: ...
+def expr_lit(value: Union[bool, int, float, str, bytes]) -> PyExpr: ...
 def expr_func(name: str, args: List[PyExpr]) -> PyExpr: ...
 
 class Session:
@@ -180,6 +182,7 @@ class Table:
             IvfPq,
             HnswPq,
             HnswSq,
+            HnswFlat,
             BTree,
             Bitmap,
             LabelList,
@@ -214,6 +217,9 @@ class Table:
     async def uri(self) -> str: ...
     async def initial_storage_options(self) -> Optional[Dict[str, str]]: ...
     async def latest_storage_options(self) -> Optional[Dict[str, str]]: ...
+    async def set_unenforced_primary_key(self, columns: List[str]) -> None: ...
+    async def set_lsm_write_spec(self, spec: LsmWriteSpec) -> None: ...
+    async def unset_lsm_write_spec(self) -> None: ...
     @property
     def tags(self) -> Tags: ...
     def query(self) -> Query: ...
@@ -242,6 +248,8 @@ async def connect(
     client_config: Optional[Union[ClientConfig, Dict[str, Any]]],
     storage_options: Optional[Dict[str, str]],
     session: Optional[Session],
+    manifest_enabled: bool = False,
+    namespace_client_properties: Optional[Dict[str, str]] = None,
 ) -> Connection: ...
 
 class RecordBatchStream:
@@ -250,6 +258,11 @@ class RecordBatchStream:
     def __aiter__(self) -> "RecordBatchStream": ...
     async def __anext__(self) -> pa.RecordBatch: ...
 
+class ColumnOrdering(TypedDict):
+    column_name: str
+    ascending: bool
+    nulls_first: bool
+
 class Query:
     def where(self, filter: str): ...
     def where_expr(self, expr: PyExpr): ...
@@ -263,6 +276,7 @@ class Query:
     def postfilter(self): ...
     def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ...
     def nearest_to_text(self, query: dict) -> FTSQuery: ...
+    def order_by(self, ordering: Optional[List[ColumnOrdering]]): ...
     async def output_schema(self) -> pa.Schema: ...
     async def execute(
         self, max_batch_length: Optional[int], timeout: Optional[timedelta]
@@ -291,6 +305,7 @@ class FTSQuery:
     def get_query(self) -> str: ...
     def add_query_vector(self, query_vec: pa.Array) -> None: ...
     def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ...
+    def order_by(self, ordering: Optional[List[ColumnOrdering]]): ...
     async def output_schema(self) -> pa.Schema: ...
     async def execute(
         self, max_batch_length: Optional[int], timeout: Optional[timedelta]
@@ -316,6 +331,7 @@ class VectorQuery:
     def maximum_nprobes(self, maximum_nprobes: int): ...
     def bypass_vector_index(self): ...
     def nearest_to_text(self, query: dict) -> HybridQuery: ...
+    def order_by(self, ordering: Optional[List[ColumnOrdering]]): ...
     def to_query_request(self) -> PyQueryRequest: ...
 
 class HybridQuery:
@@ -334,6 +350,7 @@ class HybridQuery:
     def minimum_nprobes(self, minimum_nprobes: int): ...
     def maximum_nprobes(self, maximum_nprobes: int): ...
     def bypass_vector_index(self): ...
+    def order_by(self, ordering: Optional[List[ColumnOrdering]]): ...
     def to_vector_query(self) -> VectorQuery: ...
     def to_fts_query(self) -> FTSQuery: ...
     def get_limit(self) -> int: ...
@@ -363,6 +380,7 @@ class PyQueryRequest:
     bypass_vector_index: Optional[bool]
     postfilter: Optional[bool]
     norm: Optional[str]
+    order_by: Optional[List[ColumnOrdering]]
 
 class CompactionStats:
     fragments_removed: int
@@ -403,6 +421,37 @@ class MergeResult:
     num_deleted_rows: int
     num_attempts: int
 
+class LsmWriteSpec:
+    """Specification selecting Lance's MemWAL LSM-style write path for
+    `merge_insert`."""
+
+    @staticmethod
+    def bucket(column: str, num_buckets: int) -> "LsmWriteSpec": ...
+    @staticmethod
+    def identity(column: str) -> "LsmWriteSpec": ...
+    @staticmethod
+    def unsharded() -> "LsmWriteSpec": ...
+    def with_maintained_indexes(self, indexes: List[str]) -> "LsmWriteSpec":
+        """Return a copy of this spec asking the MemWAL to keep the named
+        indexes up to date as rows are appended."""
+        ...
+    def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "LsmWriteSpec":
+        """Return a copy of this spec recording the given default
+        `ShardWriter` configuration in the MemWAL index."""
+        ...
+    @property
+    def spec_type(self) -> str:
+        """One of 'bucket', 'identity', or 'unsharded'."""
+        ...
+    @property
+    def column(self) -> Optional[str]: ...
+    @property
+    def num_buckets(self) -> Optional[int]: ...
+    @property
+    def maintained_indexes(self) -> List[str]: ...
+    @property
+    def writer_config_defaults(self) -> Dict[str, str]: ...
+
 class AddColumnsResult:
     version: int
 
@@ -440,7 +489,7 @@ class AsyncPermutationBuilder:
     async def execute(self) -> Table: ...
 
 def async_permutation_builder(
-    table: Table, dest_table_name: str
+    table: Table,
 ) -> AsyncPermutationBuilder: ...
 def fts_query_to_json(query: Any) -> str: ...
 

python/python/lancedb/background_loop.py

@@ -2,7 +2,9 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import asyncio
+import os
 import threading
+import warnings
 
 
 class BackgroundEventLoop:
@@ -13,6 +15,9 @@ class BackgroundEventLoop:
     """
 
     def __init__(self):
+        self._start()
+
+    def _start(self):
         self.loop = asyncio.new_event_loop()
         self.thread = threading.Thread(
             target=self.loop.run_forever,
@@ -31,3 +36,30 @@ class BackgroundEventLoop:
 
 
 LOOP = BackgroundEventLoop()
+
+_FORK_WARNED = False
+
+
+def _reset_after_fork():
+    # Threads do not survive fork(), so the asyncio loop in LOOP.thread is
+    # dead in the child. Re-initialize the singleton in place so existing
+    # `from .background_loop import LOOP` references in other modules see
+    # the new state. The Rust-side tokio runtime is reset analogously by a
+    # pthread_atfork hook installed in the _lancedb extension.
+    LOOP._start()
+    global _FORK_WARNED
+    if not _FORK_WARNED:
+        _FORK_WARNED = True
+        warnings.warn(
+            "lancedb fork support is experimental: the internal async "
+            "runtime has been reset in the forked child, but a small chance "
+            "of deadlock remains if other state was mid-operation at fork "
+            "time. The 'forkserver' or 'spawn' multiprocessing start method "
+            "is likely a safer alternative.",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+
+
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(after_in_child=_reset_after_fork)

python/python/lancedb/common.py

@@ -96,7 +96,7 @@ def data_to_reader(
             f"Unknown data type {type(data)}. "
             "Supported types: list of dicts, pandas DataFrame, polars DataFrame, "
             "pyarrow Table/RecordBatch, or Pydantic models. "
-            "See https://lancedb.com/docs/tables/ for examples."
+            "See https://docs.lancedb.com/tables/ for examples."
         )
 
 

python/python/lancedb/db.py

@@ -282,7 +282,7 @@ class DBConnection(EnforceOverrides):
             Additional options for the storage backend. Options already set on the
             connection will be inherited by the table, but can be overridden here.
             See available options at
-            <https://lancedb.com/docs/storage/>
+            <https://docs.lancedb.com/storage/>
 
             To enable stable row IDs (row IDs remain stable after compaction,
             update, delete, and merges), set `new_table_enable_stable_row_ids`
@@ -433,7 +433,7 @@ class DBConnection(EnforceOverrides):
             Additional options for the storage backend. Options already set on the
             connection will be inherited by the table, but can be overridden here.
             See available options at
-            <https://lancedb.com/docs/storage/>
+            <https://docs.lancedb.com/storage/>
 
         Returns
         -------
@@ -529,6 +529,19 @@ class DBConnection(EnforceOverrides):
             "namespace_client is not supported for this connection type"
         )
 
+    def serialize(self) -> str:
+        """Serialize this connection for reconstruction.
+
+        The returned string can be passed to :func:`lancedb.deserialize_conn`
+        to recreate an equivalent connection, e.g. in a remote worker.
+
+        Returns
+        -------
+        str
+            Serialized representation of this connection.
+        """
+        raise NotImplementedError("serialize is not supported for this connection type")
+
 
 class LanceDBConnection(DBConnection):
     """
@@ -577,10 +590,16 @@ class LanceDBConnection(DBConnection):
         read_consistency_interval: Optional[timedelta] = None,
         storage_options: Optional[Dict[str, str]] = None,
         session: Optional[Session] = None,
+        manifest_enabled: bool = False,
+        namespace_client_properties: Optional[Dict[str, str]] = None,
         _inner: Optional[LanceDbConnection] = None,
     ):
+        self.storage_options = storage_options
+        self._manifest_enabled = manifest_enabled
+        self._namespace_client_properties = namespace_client_properties
         if _inner is not None:
             self._conn = _inner
+            self._cached_namespace_client = None
             return
 
         if not isinstance(uri, Path):
@@ -619,6 +638,8 @@ class LanceDBConnection(DBConnection):
                 None,
                 storage_options,
                 session,
+                manifest_enabled,
+                namespace_client_properties,
             )
 
         # TODO: It would be nice if we didn't store self.storage_options but it is
@@ -626,8 +647,8 @@ class LanceDBConnection(DBConnection):
         # work because some paths like LanceDBConnection.from_inner will lose the
         # storage_options.  Also, this class really shouldn't be holding any state
         # beyond _conn.
-        self.storage_options = storage_options
         self._conn = AsyncConnection(LOOP.run(do_connect()))
+        self._cached_namespace_client: Optional[LanceNamespace] = None
 
     @property
     def read_consistency_interval(self) -> Optional[timedelta]:
@@ -652,6 +673,24 @@ class LanceDBConnection(DBConnection):
         val += ")"
         return val
 
+    @override
+    def serialize(self) -> str:
+        import json
+
+        rci = self.read_consistency_interval
+        return json.dumps(
+            {
+                "connection_type": "local",
+                "uri": self.uri,
+                "storage_options": self.storage_options,
+                "manifest_enabled": self._manifest_enabled,
+                "namespace_client_properties": self._namespace_client_properties,
+                "read_consistency_interval_seconds": (
+                    rci.total_seconds() if rci else None
+                ),
+            }
+        )
+
     async def _async_get_table_names(self, start_after: Optional[str], limit: int):
         conn = AsyncConnection(await lancedb_connect(self.uri))
         return await conn.table_names(start_after=start_after, limit=limit)
@@ -687,10 +726,10 @@ class LanceDBConnection(DBConnection):
         """
         if namespace_path is None:
             namespace_path = []
-        return LOOP.run(
-            self._conn.list_namespaces(
-                namespace_path=namespace_path, page_token=page_token, limit=limit
-            )
+        return self._namespace_conn().list_namespaces(
+            namespace_path=namespace_path,
+            page_token=page_token,
+            limit=limit,
         )
 
     @override
@@ -700,27 +739,10 @@ class LanceDBConnection(DBConnection):
         mode: Optional[str] = None,
         properties: Optional[Dict[str, str]] = None,
     ) -> CreateNamespaceResponse:
-        """Create a new namespace.
-
-        Parameters
-        ----------
-        namespace_path: List[str]
-            The namespace identifier to create.
-        mode: str, optional
-            Creation mode - "create" (fail if exists), "exist_ok" (skip if exists),
-            or "overwrite" (replace if exists). Case insensitive.
-        properties: Dict[str, str], optional
-            Properties to set on the namespace.
-
-        Returns
-        -------
-        CreateNamespaceResponse
-            Response containing the properties of the created namespace.
-        """
-        return LOOP.run(
-            self._conn.create_namespace(
-                namespace_path=namespace_path, mode=mode, properties=properties
-            )
+        return self._namespace_conn().create_namespace(
+            namespace_path=namespace_path,
+            mode=mode,
+            properties=properties,
         )
 
     @override
@@ -730,46 +752,19 @@ class LanceDBConnection(DBConnection):
         mode: Optional[str] = None,
         behavior: Optional[str] = None,
     ) -> DropNamespaceResponse:
-        """Drop a namespace.
-
-        Parameters
-        ----------
-        namespace_path: List[str]
-            The namespace identifier to drop.
-        mode: str, optional
-            Whether to skip if not exists ("SKIP") or fail ("FAIL"). Case insensitive.
-        behavior: str, optional
-            Whether to restrict drop if not empty ("RESTRICT") or cascade ("CASCADE").
-            Case insensitive.
-
-        Returns
-        -------
-        DropNamespaceResponse
-            Response containing properties and transaction_id if applicable.
-        """
-        return LOOP.run(
-            self._conn.drop_namespace(
-                namespace_path=namespace_path, mode=mode, behavior=behavior
-            )
+        return self._namespace_conn().drop_namespace(
+            namespace_path=namespace_path,
+            mode=mode,
+            behavior=behavior,
         )
 
     @override
     def describe_namespace(
         self, namespace_path: List[str]
     ) -> DescribeNamespaceResponse:
-        """Describe a namespace.
-
-        Parameters
-        ----------
-        namespace_path: List[str]
-            The namespace identifier to describe.
-
-        Returns
-        -------
-        DescribeNamespaceResponse
-            Response containing the namespace properties.
-        """
-        return LOOP.run(self._conn.describe_namespace(namespace_path=namespace_path))
+        return self._namespace_conn().describe_namespace(
+            namespace_path=namespace_path,
+        )
 
     @override
     def list_tables(
@@ -798,6 +793,12 @@ class LanceDBConnection(DBConnection):
         """
         if namespace_path is None:
             namespace_path = []
+        if namespace_path:
+            return self._namespace_conn().list_tables(
+                namespace_path=namespace_path,
+                page_token=page_token,
+                limit=limit,
+            )
         return LOOP.run(
             self._conn.list_tables(
                 namespace_path=namespace_path, page_token=page_token, limit=limit
@@ -886,6 +887,22 @@ class LanceDBConnection(DBConnection):
             raise ValueError("mode must be either 'create' or 'overwrite'")
         validate_table_name(name)
 
+        if namespace_path:
+            return self._namespace_conn().create_table(
+                name,
+                data=data,
+                schema=schema,
+                mode=mode,
+                exist_ok=exist_ok,
+                on_bad_vectors=on_bad_vectors,
+                fill_value=fill_value,
+                embedding_functions=embedding_functions,
+                namespace_path=namespace_path,
+                storage_options=storage_options,
+                data_storage_version=data_storage_version,
+                enable_v2_manifest_paths=enable_v2_manifest_paths,
+            )
+
         tbl = LanceTable.create(
             self,
             name,
@@ -901,6 +918,19 @@ class LanceDBConnection(DBConnection):
         )
         return tbl
 
+    def _namespace_conn(self) -> DBConnection:
+        """Return a LanceNamespaceDBConnection backed by this connection's
+        directory namespace.  Used to delegate child-namespace operations."""
+        from lancedb.namespace import LanceNamespaceDBConnection
+
+        return LanceNamespaceDBConnection(
+            self.namespace_client(),
+            read_consistency_interval=self.read_consistency_interval,
+            storage_options=self.storage_options,
+            namespace_client_impl=None,
+            namespace_client_properties=None,
+        )
+
     @override
     def open_table(
         self,
@@ -917,7 +947,8 @@ class LanceDBConnection(DBConnection):
         name: str
             The name of the table.
         namespace_path: List[str], optional
-            The namespace to open the table from.
+            The namespace to open the table from.  When non-empty, the
+            table is resolved through the directory namespace client.
 
         Returns
         -------
@@ -936,6 +967,14 @@ class LanceDBConnection(DBConnection):
                 stacklevel=2,
             )
 
+        if namespace_path:
+            return self._namespace_conn().open_table(
+                name,
+                namespace_path=namespace_path,
+                storage_options=storage_options,
+                index_cache_size=index_cache_size,
+            )
+
         return LanceTable.open(
             self,
             name,
@@ -1020,6 +1059,9 @@ class LanceDBConnection(DBConnection):
         """
         if namespace_path is None:
             namespace_path = []
+        if namespace_path:
+            self._namespace_conn().drop_table(name, namespace_path=namespace_path)
+            return
         LOOP.run(
             self._conn.drop_table(
                 name, namespace_path=namespace_path, ignore_missing=ignore_missing
@@ -1071,14 +1113,17 @@ class LanceDBConnection(DBConnection):
         """Get the equivalent namespace client for this connection.
 
         Returns a DirectoryNamespace pointing to the same root with the
-        same storage options.
+        same storage options.  The result is cached for the lifetime of
+        this connection.
 
         Returns
         -------
         LanceNamespace
             The namespace client for this connection.
         """
-        return LOOP.run(self._conn.namespace_client())
+        if self._cached_namespace_client is None:
+            self._cached_namespace_client = LOOP.run(self._conn.namespace_client())
+        return self._cached_namespace_client
 
     @deprecation.deprecated(
         deprecated_in="0.15.1",
@@ -1353,6 +1398,7 @@ class AsyncConnection(object):
         namespace_path: Optional[List[str]] = None,
         embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
         location: Optional[str] = None,
+        namespace_client: Optional[Any] = None,
     ) -> AsyncTable:
         """Create an [AsyncTable][lancedb.table.AsyncTable] in the database.
 
@@ -1397,7 +1443,7 @@ class AsyncConnection(object):
             Additional options for the storage backend. Options already set on the
             connection will be inherited by the table, but can be overridden here.
             See available options at
-            <https://lancedb.com/docs/storage/>
+            <https://docs.lancedb.com/storage/>
 
             To enable stable row IDs (row IDs remain stable after compaction,
             update, delete, and merges), set `new_table_enable_stable_row_ids`
@@ -1550,6 +1596,7 @@ class AsyncConnection(object):
                 namespace_path=namespace_path,
                 storage_options=storage_options,
                 location=location,
+                namespace_client=namespace_client,
             )
         else:
             data = data_to_reader(data, schema)
@@ -1560,6 +1607,7 @@ class AsyncConnection(object):
                 namespace_path=namespace_path,
                 storage_options=storage_options,
                 location=location,
+                namespace_client=namespace_client,
             )
 
         return AsyncTable(new_table)
@@ -1588,7 +1636,7 @@ class AsyncConnection(object):
             Additional options for the storage backend. Options already set on the
             connection will be inherited by the table, but can be overridden here.
             See available options at
-            <https://lancedb.com/docs/storage/>
+            <https://docs.lancedb.com/storage/>
         index_cache_size: int, default 256
             **Deprecated**: Use session-level cache configuration instead.
             Create a Session with custom cache sizes and pass it to lancedb.connect().

python/python/lancedb/expr.py

@@ -63,7 +63,7 @@ def _coerce(value: "ExprLike") -> "Expr":
 
 
 # Type alias used in annotations.
-ExprLike = Union["Expr", bool, int, float, str]
+ExprLike = Union["Expr", bool, int, float, str, bytes]
 
 
 class Expr:
@@ -261,13 +261,13 @@ def col(name: str) -> Expr:
     return Expr(expr_col(name))
 
 
-def lit(value: Union[bool, int, float, str]) -> Expr:
+def lit(value: Union[bool, int, float, str, bytes]) -> Expr:
     """Create a literal (constant) value expression.
 
     Parameters
     ----------
     value:
-        A Python ``bool``, ``int``, ``float``, or ``str``.
+        A Python ``bool``, ``int``, ``float``, ``str``, or ``bytes``.
 
     Examples
     --------

python/python/lancedb/fts.py

@@ -1,201 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-"""Full text search index using tantivy-py"""
-
-import os
-from typing import List, Tuple, Optional
-
-import pyarrow as pa
-
-try:
-    import tantivy
-except ImportError:
-    raise ImportError(
-        "Please install tantivy-py `pip install tantivy` to use the full text search feature."  # noqa: E501
-    )
-
-from .table import LanceTable
-
-
-def create_index(
-    index_path: str,
-    text_fields: List[str],
-    ordering_fields: Optional[List[str]] = None,
-    tokenizer_name: str = "default",
-) -> tantivy.Index:
-    """
-    Create a new Index (not populated)
-
-    Parameters
-    ----------
-    index_path : str
-        Path to the index directory
-    text_fields : List[str]
-        List of text fields to index
-    ordering_fields: List[str]
-        List of unsigned type fields to order by at search time
-    tokenizer_name : str, default "default"
-        The tokenizer to use
-
-    Returns
-    -------
-    index : tantivy.Index
-        The index object (not yet populated)
-    """
-    if ordering_fields is None:
-        ordering_fields = []
-    # Declaring our schema.
-    schema_builder = tantivy.SchemaBuilder()
-    # special field that we'll populate with row_id
-    schema_builder.add_integer_field("doc_id", stored=True)
-    # data fields
-    for name in text_fields:
-        schema_builder.add_text_field(name, stored=True, tokenizer_name=tokenizer_name)
-    if ordering_fields:
-        for name in ordering_fields:
-            schema_builder.add_unsigned_field(name, fast=True)
-    schema = schema_builder.build()
-    os.makedirs(index_path, exist_ok=True)
-    index = tantivy.Index(schema, path=index_path)
-    return index
-
-
-def populate_index(
-    index: tantivy.Index,
-    table: LanceTable,
-    fields: List[str],
-    writer_heap_size: Optional[int] = None,
-    ordering_fields: Optional[List[str]] = None,
-) -> int:
-    """
-    Populate an index with data from a LanceTable
-
-    Parameters
-    ----------
-    index : tantivy.Index
-        The index object
-    table : LanceTable
-        The table to index
-    fields : List[str]
-        List of fields to index
-    writer_heap_size : int
-        The writer heap size in bytes, defaults to 1GB
-
-    Returns
-    -------
-    int
-        The number of rows indexed
-    """
-    if ordering_fields is None:
-        ordering_fields = []
-    writer_heap_size = writer_heap_size or 1024 * 1024 * 1024
-    # first check the fields exist and are string or large string type
-    nested = []
-
-    for name in fields:
-        try:
-            f = table.schema.field(name)  # raises KeyError if not found
-        except KeyError:
-            f = resolve_path(table.schema, name)
-            nested.append(name)
-
-        if not pa.types.is_string(f.type) and not pa.types.is_large_string(f.type):
-            raise TypeError(f"Field {name} is not a string type")
-
-    # create a tantivy writer
-    writer = index.writer(heap_size=writer_heap_size)
-    # write data into index
-    dataset = table.to_lance()
-    row_id = 0
-
-    max_nested_level = 0
-    if len(nested) > 0:
-        max_nested_level = max([len(name.split(".")) for name in nested])
-
-    for b in dataset.to_batches(columns=fields + ordering_fields):
-        if max_nested_level > 0:
-            b = pa.Table.from_batches([b])
-            for _ in range(max_nested_level - 1):
-                b = b.flatten()
-        for i in range(b.num_rows):
-            doc = tantivy.Document()
-            for name in fields:
-                value = b[name][i].as_py()
-                if value is not None:
-                    doc.add_text(name, value)
-            for name in ordering_fields:
-                value = b[name][i].as_py()
-                if value is not None:
-                    doc.add_unsigned(name, value)
-            if not doc.is_empty:
-                doc.add_integer("doc_id", row_id)
-                writer.add_document(doc)
-            row_id += 1
-    # commit changes
-    writer.commit()
-    return row_id
-
-
-def resolve_path(schema, field_name: str) -> pa.Field:
-    """
-    Resolve a nested field path to a list of field names
-
-    Parameters
-    ----------
-    field_name : str
-        The field name to resolve
-
-    Returns
-    -------
-    List[str]
-        The resolved path
-    """
-    path = field_name.split(".")
-    field = schema.field(path.pop(0))
-    for segment in path:
-        if pa.types.is_struct(field.type):
-            field = field.type.field(segment)
-        else:
-            raise KeyError(f"field {field_name} not found in schema {schema}")
-    return field
-
-
-def search_index(
-    index: tantivy.Index, query: str, limit: int = 10, ordering_field=None
-) -> Tuple[Tuple[int], Tuple[float]]:
-    """
-    Search an index for a query
-
-    Parameters
-    ----------
-    index : tantivy.Index
-        The index object
-    query : str
-        The query string
-    limit : int
-        The maximum number of results to return
-
-    Returns
-    -------
-    ids_and_score: list[tuple[int], tuple[float]]
-        A tuple of two tuples, the first containing the document ids
-        and the second containing the scores
-    """
-    searcher = index.searcher()
-    query = index.parse_query(query)
-    # get top results
-    if ordering_field:
-        results = searcher.search(query, limit, order_by_field=ordering_field)
-    else:
-        results = searcher.search(query, limit)
-    if results.count == 0:
-        return tuple(), tuple()
-    return tuple(
-        zip(
-            *[
-                (searcher.doc(doc_address)["doc_id"][0], score)
-                for score, doc_address in results.hits
-            ]
-        )
-    )

python/python/lancedb/index.py

@@ -7,6 +7,7 @@ from typing import Literal, Optional
 from ._lancedb import (
     IndexConfig,
 )
+from .types import BaseTokenizerType
 
 lang_mapping = {
     "ar": "Arabic",
@@ -111,8 +112,12 @@ class FTS:
         - "simple": Splits text by whitespace and punctuation.
         - "whitespace": Split text by whitespace, but not punctuation.
         - "raw": No tokenization. The entire text is treated as a single token.
+        - "ngram": N-gram tokenizer for substring-style matching.
+        - "jieba/*": Jieba tokenizer loaded from Lance's language model home.
+        - "lindera/*": Lindera tokenizer loaded from Lance's language model home.
     language : str, default "English"
-        The language to use for tokenization.
+        The language to use for stemming and stop-word removal. This is not the
+        primary way to enable CJK tokenization.
     max_token_length : int, default 40
         The maximum token length to index. Tokens longer than this length will be
         ignored.
@@ -127,10 +132,17 @@ class FTS:
     ascii_folding : bool, default True
         Whether to fold ASCII characters. This converts accented characters to
         their ASCII equivalent. For example, "café" would be converted to "cafe".
+
+    Notes
+    -----
+    Model-backed tokenizers such as ``jieba/default`` and ``lindera/ipadic``
+    require tokenizer models in Lance's language model home. Set
+    ``LANCE_LANGUAGE_MODEL_HOME`` to override the default platform data
+    directory under ``lance/language_models``.
     """
 
     with_position: bool = False
-    base_tokenizer: Literal["simple", "raw", "whitespace"] = "simple"
+    base_tokenizer: BaseTokenizerType = "simple"
     language: str = "English"
     max_token_length: Optional[int] = 40
     lower_case: bool = True
@@ -376,9 +388,98 @@ class HnswSq:
     target_partition_size: Optional[int] = None
 
 
+@dataclass
+class HnswFlat:
+    """Describe a HNSW-FLAT index configuration.
+
+    HNSW-FLAT stands for Hierarchical Navigable Small World without quantization.
+    It stores raw vectors in the HNSW graph, providing the highest recall among
+    the IVF_HNSW family at the cost of more memory and disk space compared to
+    :class:`HnswSq` or :class:`HnswPq`.
+
+    Parameters
+    ----------
+
+    distance_type: str, default "l2"
+
+        The distance metric used to train the index.
+
+        The following distance types are available:
+
+        "l2" - Euclidean distance. This is a very common distance metric that
+        accounts for both magnitude and direction when determining the distance
+        between vectors. l2 distance has a range of [0, ∞).
+
+        "cosine" - Cosine distance.  Cosine distance is a distance metric
+        calculated from the cosine similarity between two vectors. Cosine
+        similarity is a measure of similarity between two non-zero vectors of an
+        inner product space. It is defined to equal the cosine of the angle
+        between them.  Unlike l2, the cosine distance is not affected by the
+        magnitude of the vectors.  Cosine distance has a range of [0, 2].
+
+        "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+        distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+        l2 norm is 1), then dot distance is equivalent to the cosine distance.
+
+    num_partitions, default sqrt(num_rows)
+
+        The number of IVF partitions to create.
+
+        For HNSW, we recommend a small number of partitions. Setting this to 1
+        works well for most tables. For very large tables, training just one HNSW
+        graph will require too much memory. Each partition becomes its own HNSW
+        graph, so setting this value higher reduces the peak memory use of
+        training.
+
+    max_iterations, default 50
+
+        Max iterations to train kmeans.
+
+        When training an IVF index we use kmeans to calculate the partitions.
+        This parameter controls how many iterations of kmeans to run.
+
+    sample_rate, default 256
+
+        The rate used to calculate the number of training vectors for kmeans.
+
+    m, default 20
+
+        The number of neighbors to select for each vector in the HNSW graph.
+
+        This value controls the tradeoff between search speed and accuracy.
+        The higher the value the more accurate the search but the slower it
+        will be.
+
+    ef_construction, default 300
+
+        The number of candidates to evaluate during the construction of the HNSW
+        graph.
+
+        This value controls the tradeoff between build speed and accuracy.
+        The higher the value the more accurate the build but the slower it will
+        be.  150 to 300 is the typical range. 100 is a minimum for good quality
+        search results. In most cases, there is no benefit to setting this higher
+        than 500.  This value should be set to a value that is not less than `ef`
+        in the search phase.
+
+    target_partition_size, default is 1,048,576
+
+        The target size of each partition.
+    """
+
+    distance_type: Literal["l2", "cosine", "dot"] = "l2"
+    num_partitions: Optional[int] = None
+    max_iterations: int = 50
+    sample_rate: int = 256
+    m: int = 20
+    ef_construction: int = 300
+    target_partition_size: Optional[int] = None
+
+
 # Backwards-compatible aliases
 IvfHnswPq = HnswPq
 IvfHnswSq = HnswSq
+IvfHnswFlat = HnswFlat
 
 
 @dataclass
@@ -698,11 +799,13 @@ __all__ = [
     "IvfPq",
     "IvfHnswPq",
     "IvfHnswSq",
+    "IvfHnswFlat",
     "IvfSq",
     "IvfRq",
     "IvfFlat",
     "HnswPq",
     "HnswSq",
+    "HnswFlat",
     "IndexConfig",
     "FTS",
     "Bitmap",