docs(python): fix cross-references that resolved to the wrong page

`mkdocs build --strict` only catches references it cannot resolve. A bare
anchor such as `[limit][]` or `[vector search][search]` is matched by
autorefs against any heading on the site, so six of them silently linked
into the JavaScript reference instead. The relative links in
`permutation.py` and `remote/errors.py` pointed at in-page anchors and
paths that do not exist.

Targets that still exist here or in an imported inventory now use
mkdocstrings references; the guide pages deleted in #2770 use their
lancedb.com URLs.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

.agents/plugins/marketplace.json

@@ -0,0 +1,20 @@
+{
+  "name": "lancedb",
+  "interface": {
+    "displayName": "LanceDB"
+  },
+  "plugins": [
+    {
+      "name": "lancedb",
+      "source": {
+        "source": "local",
+        "path": "./plugins/lancedb"
+      },
+      "policy": {
+        "installation": "AVAILABLE",
+        "authentication": "ON_INSTALL"
+      },
+      "category": "Developer Tools"
+    }
+  ]
+}

.agents/skills/README.md

@@ -5,3 +5,7 @@ This directory contains repo-scoped code agent skills for the LanceDB project.
 Each skill is a folder that contains a required `SKILL.md` and optional bundled resources.
 
 Codex discovers skills from `.agents/skills` in the current working directory and parent directories.
+
+The `lancedb` skill lives in the `plugins/lancedb` plugin (see `plugins/lancedb/skills/lancedb`)
+so it can be installed via the plugin marketplaces (`.claude-plugin/marketplace.json` and
+`.agents/plugins/marketplace.json`); the `lancedb` entry here is a symlink into that plugin.

.agents/skills/lancedb

@@ -0,0 +1 @@
+../../plugins/lancedb/skills/lancedb

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.32.0-beta.2"
+current_version = "0.37.1-beta.0"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.
@@ -75,6 +75,13 @@ filename = "nodejs/Cargo.toml"
 replace = "\nversion = \"{new_version}\""
 search = "\nversion = \"{current_version}\""
 
+# The Python package takes its version from here (pyproject.toml declares
+# `dynamic = ["version"]`, so maturin reads it out of the crate manifest).
+[[tool.bumpversion.files]]
+filename = "python/Cargo.toml"
+replace = "\nversion = \"{new_version}\""
+search = "\nversion = \"{current_version}\""
+
 # Java documentation
 [[tool.bumpversion.files]]
 filename = "docs/src/java/java.md"

.claude-plugin/marketplace.json

@@ -0,0 +1,19 @@
+{
+  "name": "lancedb",
+  "owner": {
+    "name": "LanceDB"
+  },
+  "description": "LanceDB plugins for Claude Code.",
+  "plugins": [
+    {
+      "name": "lancedb",
+      "source": "./plugins/lancedb",
+      "description": "Write, review, debug, and document LanceDB pipelines in Python and TypeScript that work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables.",
+      "version": "0.1.0",
+      "author": {
+        "name": "LanceDB"
+      },
+      "category": "development"
+    }
+  ]
+}

.github/actions/create-failure-issue/action.yml

@@ -27,9 +27,20 @@ runs:
           # Extract failed job names
           FAILED_JOBS=$(echo "$JOB_RESULTS" | jq -r 'to_entries | map(select(.value.result == "failure")) | map(.key) | join(", ")')
 
-          # Create issue with workflow name, failed jobs, and run URL
+          TITLE="$WORKFLOW_NAME Failed ($FAILED_JOBS)"
+
+          # This action now also runs on nightly schedules, so a breakage that
+          # persists for a few days would otherwise file one issue per night.
+          # Comment on the open report instead when one already exists.
+          EXISTING=$(gh issue list --state open --label ci --limit 100 --json number,title \
+            | jq -r --arg title "$TITLE" 'map(select(.title == $title)) | .[0].number // empty')
+
+          if [ -n "$EXISTING" ]; then
+            gh issue comment "$EXISTING" --body "Failed again: $RUN_URL"
+            echo "Commented on existing issue #$EXISTING"
+          else
             gh issue create \
-            --title "$WORKFLOW_NAME Failed ($FAILED_JOBS)" \
+              --title "$TITLE" \
               --body "The workflow **$WORKFLOW_NAME** failed during execution.
 
         **Failed jobs:** $FAILED_JOBS
@@ -40,6 +51,7 @@ runs:
               --label "ci"
 
             echo "Issue created successfully"
+          fi
         else
           echo "No job failures detected, skipping issue creation"
         fi

.github/workflows/build_linux_wheel/action.yml

@@ -18,6 +18,14 @@ inputs:
     description: "The manylinux version to build for"
     required: false
     default: "2_17"
+  package-name:
+    description: "Override [project] name in python/pyproject.toml (e.g. 'lancedb-compat'). Default keeps 'lancedb'."
+    required: false
+    default: "lancedb"
+  rustflags:
+    description: "RUSTFLAGS for the build container, as a single whitespace-free token (e.g. '-Ctarget-cpu=x86-64-v2'). Empty leaves RUSTFLAGS unset, keeping the defaults from .cargo/config.toml."
+    required: false
+    default: ""
 runs:
   using: "composite"
   steps:
@@ -27,6 +35,18 @@ runs:
         ARM_BUILD: ${{ inputs.arm-build }}
       run: |
         echo "ARM BUILD: $ARM_BUILD"
+    - name: Patch package name for variant build
+      if: ${{ inputs.package-name != 'lancedb' }}
+      shell: bash
+      env:
+        PACKAGE_NAME: ${{ inputs.package-name }}
+      run: |
+        # Swap the [project] name so this build produces e.g. lancedb-compat
+        # wheels. The package still installs files under the lancedb/
+        # namespace -- import lancedb still works after pip install.
+        sed -i.bak 's/^name = "lancedb"$/name = "'"$PACKAGE_NAME"'"/' python/pyproject.toml
+        rm -f python/pyproject.toml.bak
+        grep '^name = ' python/pyproject.toml
     - name: Build x86_64 Manylinux wheel
       if: ${{ inputs.arm-build == 'false' }}
       uses: PyO3/maturin-action@v1
@@ -34,7 +54,7 @@ runs:
         maturin-version: "1.12.4"
         command: build
         working-directory: python
-        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc"
+        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
         target: x86_64-unknown-linux-gnu
         manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}
@@ -51,7 +71,7 @@ runs:
         maturin-version: "1.12.4"
         command: build
         working-directory: python
-        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc"
+        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
         target: aarch64-unknown-linux-gnu
         manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}

.github/workflows/cargo-publish.yml

@@ -6,7 +6,6 @@ on:
       # We don't publish pre-releases for Rust. Crates.io is just a source
       # distribution, so we don't need to publish pre-releases.
       - "v*-beta*"
-      - "*-v*" # for example, python-vX.Y.Z
 
 env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache

.github/workflows/gh-release.yml

@@ -0,0 +1,85 @@
+name: GitHub Release
+
+# All SDKs share one version, so a single `vX.Y.Z` tag produces a single GitHub
+# release covering all of them. The per-package publish workflows (PyPI, NPM,
+# Cargo, Maven) trigger off the same tag independently.
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  gh-release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Extract version
+        id: extract_version
+        env:
+          GITHUB_REF: ${{ github.ref }}
+        run: |
+          set -e
+          echo "Extracting tag and version from $GITHUB_REF"
+          if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
+            VERSION=${BASH_REMATCH[1]}
+            TAG=v$VERSION
+            echo "tag=$TAG" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
+          else
+            echo "Failed to extract version from $GITHUB_REF"
+            exit 1
+          fi
+          echo "Extracted version $VERSION from $GITHUB_REF"
+          if [[ $VERSION =~ beta ]]; then
+            echo "This is a beta release"
+            echo "prerelease=true" >> $GITHUB_OUTPUT
+
+            # Get last release (that is not this one)
+            FROM_TAG=$(git tag --sort='version:refname' \
+              | grep ^v \
+              | grep -vF "$TAG" \
+              | python ci/semver_sort.py v \
+              | tail -n 1)
+          else
+            echo "This is a stable release"
+            echo "prerelease=false" >> $GITHUB_OUTPUT
+            # Get last stable tag (ignore betas)
+            FROM_TAG=$(git tag --sort='version:refname' \
+              | grep ^v \
+              | grep -vF "$TAG" \
+              | grep -v beta \
+              | python ci/semver_sort.py v \
+              | tail -n 1)
+          fi
+          echo "Found from tag $FROM_TAG"
+          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
+      - name: Create Release Notes
+        id: release_notes
+        uses: mikepenz/release-changelog-builder-action@v4
+        with:
+          configuration: .github/release_notes.json
+          toTag: ${{ steps.extract_version.outputs.tag }}
+          fromTag: ${{ steps.extract_version.outputs.from_tag }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create GH release
+        uses: softprops/action-gh-release@v2
+        with:
+          # Marking betas as pre-releases keeps them from taking the "Latest"
+          # badge on the releases page.
+          prerelease: ${{ steps.extract_version.outputs.prerelease }}
+          make_latest: ${{ steps.extract_version.outputs.prerelease == 'false' }}
+          tag_name: ${{ steps.extract_version.outputs.tag }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          generate_release_notes: false
+          name: LanceDB v${{ steps.extract_version.outputs.version }}
+          body: ${{ steps.release_notes.outputs.changelog }}

.github/workflows/make-release-commit.yml

@@ -1,13 +1,14 @@
 name: Create release commit
 
-# This workflow increments versions, tags the version, and pushes it.
+# This workflow increments the version, tags it, and pushes it. All SDKs share
+# a single version, so one tag releases all of them.
 # When a tag is pushed, another workflow is triggered that creates a GH release
 # and uploads the binaries. This workflow is only for creating the tag.
 
 # This script will enforce that a minor version is incremented if there are any
-# breaking changes since the last minor increment. However, it isn't able to
+# breaking changes since the last minor increment. A breaking change in any SDK
-# differentiate between breaking changes in Node versus Python. If you wish to
+# bumps the minor version for all of them. If you wish to bypass this check, you
-# bypass this check, you can manually increment the version and push the tag.
+# can manually increment the version and push the tag.
 on:
   workflow_dispatch:
     inputs:
@@ -24,16 +25,6 @@ on:
         options:
           - preview
           - stable
-      python:
-        description: 'Make a Python release'
-        required: true
-        default: true
-        type: boolean
-      other:
-        description: 'Make a Node/Rust/Java release'
-        required: true
-        default: true
-        type: boolean
       bump-minor:
         description: 'Bump minor version'
         required: true
@@ -65,29 +56,16 @@ jobs:
         run: |
           git config user.name 'Lance Release'
           git config user.email 'lance-dev@lancedb.com'
-      - name: Bump Python version
+      - name: Bump version
-        if: ${{ inputs.python }}
-        working-directory: python
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          # Need to get the commit before bumping the version, so we can
-          # determine if there are breaking changes in the next step as well.
-          echo "COMMIT_BEFORE_BUMP=$(git rev-parse HEAD)" >> $GITHUB_ENV
-
-          pip install bump-my-version PyGithub packaging
-          bash ../ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} python-v
-      - name: Bump Node/Rust version
-        if: ${{ inputs.other }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           pip install bump-my-version PyGithub packaging
-          bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} v $COMMIT_BEFORE_BUMP
+          bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }}
           bash ci/update_lockfiles.sh --amend
       - name: Push new version tag
         if: ${{ !inputs.dry_run }}
-        uses: ad-m/github-push-action@master
+        uses: ad-m/github-push-action@881a6320fdb16eb5318c5054f31c218aec2b324c # v1.3.0
         with:
           # Need to use PAT here too to trigger next workflow. See comment above.
           github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/nodejs.yml

@@ -61,6 +61,11 @@ jobs:
         sudo apt update
         sudo apt install -y protobuf-compiler libssl-dev
     - uses: Swatinem/rust-cache@v2
+      with:
+        # Restore everywhere, but only save from main. Per-PR saves are
+        # unreadable outside their own branch anyway, since GitHub scopes
+        # caches to the creating ref.
+        save-if: ${{ github.ref == 'refs/heads/main' }}
     - name: Format Rust
       run: cargo fmt --all -- --check
     - name: Lint Rust
@@ -103,6 +108,11 @@ jobs:
         cache: 'pnpm'
         cache-dependency-path: nodejs/pnpm-lock.yaml
     - uses: Swatinem/rust-cache@v2
+      with:
+        # Restore everywhere, but only save from main. Per-PR saves are
+        # unreadable outside their own branch anyway, since GitHub scopes
+        # caches to the creating ref.
+        save-if: ${{ github.ref == 'refs/heads/main' }}
     - name: Install dependencies
       run: |
         sudo apt update
@@ -182,6 +192,11 @@ jobs:
         cache-dependency-path: nodejs/pnpm-lock.yaml
     - uses: dtolnay/rust-toolchain@stable
     - uses: Swatinem/rust-cache@v2
+      with:
+        # Restore everywhere, but only save from main. Per-PR saves are
+        # unreadable outside their own branch anyway, since GitHub scopes
+        # caches to the creating ref.
+        save-if: ${{ github.ref == 'refs/heads/main' }}
     - name: Install dependencies
       run: |
         brew install protobuf

.github/workflows/npm-publish.yml

@@ -10,10 +10,16 @@ permissions:
 
 on:
   push:
-    branches:
-      - main
     tags:
       - "v*"
+  # The cross-compiled targets (musl especially) break from toolchain and
+  # dependency changes that nothing else in CI catches, and discovering that
+  # mid-release is expensive. A nightly run keeps that signal while dropping
+  # the full 8-target release matrix from all ~90 pushes to main each month.
+  # `report-failure` files an issue when a nightly breaks.
+  schedule:
+    - cron: "0 8 * * *"
+  workflow_dispatch:
   pull_request:
     # This should trigger a dry run (we skip the final publish step)
     paths:
@@ -26,73 +32,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  gh-release:
-    if: startsWith(github.ref, 'refs/tags/v')
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Extract version
-        id: extract_version
-        env:
-          GITHUB_REF: ${{ github.ref }}
-        run: |
-          set -e
-          echo "Extracting tag and version from $GITHUB_REF"
-          if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
-            VERSION=${BASH_REMATCH[1]}
-            TAG=v$VERSION
-            echo "tag=$TAG" >> $GITHUB_OUTPUT
-            echo "version=$VERSION" >> $GITHUB_OUTPUT
-          else
-            echo "Failed to extract version from $GITHUB_REF"
-            exit 1
-          fi
-          echo "Extracted version $VERSION from $GITHUB_REF"
-          if [[ $VERSION =~ beta ]]; then
-            echo "This is a beta release"
-
-            # Get last release (that is not this one)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^v \
-              | grep -vF "$TAG" \
-              | python ci/semver_sort.py v \
-              | tail -n 1)
-          else
-            echo "This is a stable release"
-            # Get last stable tag (ignore betas)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^v \
-              | grep -vF "$TAG" \
-              | grep -v beta \
-              | python ci/semver_sort.py v \
-              | tail -n 1)
-          fi
-          echo "Found from tag $FROM_TAG"
-          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
-      - name: Create Release Notes
-        id: release_notes
-        uses: mikepenz/release-changelog-builder-action@v4
-        with:
-          configuration: .github/release_notes.json
-          toTag: ${{ steps.extract_version.outputs.tag }}
-          fromTag: ${{ steps.extract_version.outputs.from_tag }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Create GH release
-        uses: softprops/action-gh-release@v2
-        with:
-          prerelease: ${{ contains('beta', github.ref) }}
-          tag_name: ${{ steps.extract_version.outputs.tag }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          generate_release_notes: false
-          name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
-          body: ${{ steps.release_notes.outputs.changelog }}
-
   build-lancedb:
     strategy:
       fail-fast: false
@@ -101,9 +40,18 @@ jobs:
           - target: aarch64-apple-darwin
             host: macos-latest
             features: fp16kernels
-            pre_build: brew install protobuf
+            pre_build: |-
+              brew install protobuf
+              # Fat LTO (the workspace default in .cargo/config.toml) is
+              # single-threaded and is the peak-memory step of the build. On
+              # this runner it accounted for ~111 of the job's ~113 minutes,
+              # making it the critical path of the entire publish pipeline.
+              # ThinLTO parallelizes it across the runner's cores, for a few
+              # percent of runtime performance.
+              export CARGO_PROFILE_RELEASE_LTO=thin
+              export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
           - target: x86_64-pc-windows-msvc
-            host: windows-2025-8x-x64
+            host: windows-2025
             features: ","
             pre_build: |-
               choco install --no-progress protoc ninja nasm
@@ -111,19 +59,19 @@ jobs:
               # There is an issue where choco doesn't add nasm to the path
               export PATH="$PATH:/c/Program Files/NASM"
               nasm -v
-              # Fat LTO of the cdylib is single-threaded and the peak-memory
+              # See the ThinLTO note on aarch64-apple-darwin above. Keeping
-              # step of the build, and had started hitting rustc-LLVM OOM on the
+              # peak memory down is also what lets this run on the standard
-              # Windows runners. ThinLTO parallelizes it across the runner's
+              # 4-core runner: the 8-core larger runner was only needed to
-              # cores and keeps peak memory well under the limit.
+              # stop fat LTO from OOMing rustc-LLVM.
               export CARGO_PROFILE_RELEASE_LTO=thin
               export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
           - target: aarch64-pc-windows-msvc
-            host: windows-2025-8x-x64
+            host: windows-2025
             features: ","
             pre_build: |-
                 choco install --no-progress protoc
                 rustup target add aarch64-pc-windows-msvc
-                # See ThinLTO note on the x86_64-pc-windows-msvc target above.
+                # See the ThinLTO note on aarch64-apple-darwin above.
                 export CARGO_PROFILE_RELEASE_LTO=thin
                 export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
           - target: x86_64-unknown-linux-gnu
@@ -198,16 +146,49 @@ jobs:
         with:
           toolchain: stable
           targets: ${{ matrix.settings.target }}
-      - name: Cache cargo
+      # These builds were entirely uncached: the old key was static, so
-        uses: actions/cache@v5
+      # `actions/cache` (which only writes on a miss) could never refresh it,
+      # and the multi-GB whole-`target/` copy it tried to store never fit the
+      # repo's cache budget, so no entry was ever saved. rust-cache prunes
+      # `target/` to dependency artifacts and keys on Cargo.lock plus the rustc
+      # version, which both fixes the key and keeps entries a sane size.
+      #
+      # This caches dependency *compilation* only. The LTO link of the cdylib
+      # re-runs regardless, since the local crate changes every time, so the
+      # win is larger on the non-LTO jobs than here.
+      - name: Cache cargo (native builds)
+        uses: Swatinem/rust-cache@v2
+        if: ${{ !matrix.settings.docker }}
         with:
-          path: |
+          # The release profile and per-target dirs differ from what the test
-            ~/.cargo/registry/index/
+          # workflows cache, so these need to be separate entries.
-            ~/.cargo/registry/cache/
+          key: release-${{ matrix.settings.target }}
-            ~/.cargo/git/db/
+          # Only the nightly run on main writes, so tag and PR runs restore a
-            .cargo-cache
+          # warm entry without every dependabot PR writing its own (which would
-            target/
+          # be unreadable elsewhere anyway, since GitHub scopes caches to the
-          key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
+          # creating ref). The nightly cadence also keeps entries inside
+          # GitHub's 7-day eviction window, which a tag-only trigger would not.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      # Docker builds can use rust-cache too. `target/` already lives on the
+      # host because the whole workspace is bind-mounted into the container, and
+      # rust-cache's prune and save run host-side, so they can manage it -- which
+      # is what keeps the entry to dependency artifacts rather than a multi-GB
+      # copy of everything.
+      #
+      # Two differences from the native builds. The container's CARGO_HOME is
+      # bind-mounted from `.cargo-cache` rather than the host's ~/.cargo, so that
+      # has to be cached explicitly. And the key is derived from the *host* rustc
+      # version, which is not the compiler that produced these artifacts; that is
+      # safe because cargo fingerprints the real compiler and rebuilds on a
+      # mismatch, it just means a base-image toolchain bump costs one cold build
+      # instead of invalidating the key.
+      - name: Cache cargo (docker builds)
+        uses: Swatinem/rust-cache@v2
+        if: ${{ matrix.settings.docker }}
+        with:
+          key: docker-${{ matrix.settings.target }}
+          cache-directories: .cargo-cache
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
       - name: Install Zig
@@ -225,9 +206,13 @@ jobs:
         if: ${{ matrix.settings.docker }}
         with:
           image: ${{ matrix.settings.docker }}
+          # All three mounts must live under `.cargo-cache`, which is what the
+          # cache step above saves. Previously the registry mounts pointed at
+          # `.cargo/...`, a path nothing cached, so the container re-downloaded
+          # the whole crate registry on every run.
           options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
-                    -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \
+                    -v ${{ github.workspace }}/.cargo-cache/registry/cache:/usr/local/cargo/registry/cache \
-                    -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \
+                    -v ${{ github.workspace }}/.cargo-cache/registry/index:/usr/local/cargo/registry/index \
                     -v ${{ github.workspace }}:/build -w /build/nodejs"
           run: |
             set -e
@@ -239,6 +224,16 @@ jobs:
               --js ../lancedb/native.js \
               --strip \
               --output-dir dist/
+      # The container runs as root (`--user 0:0`), so everything it wrote to the
+      # mounted cache dirs is root-owned. rust-cache's post step runs as the
+      # runner user and has to both read these and delete from them while
+      # pruning, so hand them back before it runs.
+      - name: Take ownership of docker build output
+        if: ${{ matrix.settings.docker }}
+        run: |
+          sudo chown -R "$(id -u):$(id -g)" \
+            "${{ github.workspace }}/.cargo-cache" \
+            "${{ github.workspace }}/target"
       - name: Build
         run: |
           ${{ matrix.settings.pre_build }}
@@ -252,6 +247,15 @@ jobs:
               --output-dir dist/
         if: ${{ !matrix.settings.docker }}
         shell: bash
+      # The standard Windows runners have ~14 GB free, and a release `target/`
+      # for this workspace is a large fraction of that. Report the remaining
+      # headroom so a build that only just fits is visible before a dependency
+      # bump turns it into a failed release. `always()` so the numbers are
+      # still there when the build is what ran out of space.
+      - name: Report disk headroom
+        if: always()
+        run: df -h
+        shell: bash
       - name: Upload artifact
         uses: actions/upload-artifact@v7
         with:
@@ -402,7 +406,9 @@ jobs:
     name: Report Workflow Failure
     runs-on: ubuntu-latest
     needs: [build-lancedb, test-lancedb, publish]
-    if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
+    # Nightly runs are the only thing watching the cross-compiled targets now,
+    # so they have to report failures too or the signal is silently lost.
+    if: always() && failure() && (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'schedule')
     permissions:
       contents: read
       issues: write

.github/workflows/pypi-publish.yml

@@ -3,7 +3,7 @@ name: PyPI Publish
 on:
   push:
     tags:
-      - 'python-v*'
+      - 'v*'
   pull_request:
     # This should trigger a dry run (we skip the final publish step)
     paths:
@@ -20,9 +20,15 @@ env:
 permissions:
   contents: read
 
+# Without this, a force-push to a PR leaves the previous run going -- including
+# a ~74 minute Windows job and a billed arm64 wheel build.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   linux:
-    name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
+    name: Python ${{ matrix.config.package_name }} ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
     timeout-minutes: 60
     strategy:
       matrix:
@@ -31,11 +37,28 @@ jobs:
             manylinux: "2_28"
             extra_args: "--features fp16kernels"
             runner: ubuntu-22.04
+            package_name: "lancedb"
+            rustflags: ""
           # For successful fat LTO builds, we need a large runner to avoid OOM errors.
           - platform: aarch64
             manylinux: "2_28"
             extra_args: "--features fp16kernels"
             runner: ubuntu-2404-8x-arm64
+            package_name: "lancedb"
+            rustflags: ""
+          # `lancedb-compat`: pre-Haswell-friendly variant for x86_64 hosts
+          # without AVX2 (Sandy Bridge / Ivy Bridge / Westmere on Intel,
+          # Bulldozer / Piledriver / Steamroller on AMD). Compiled at the
+          # `x86-64-v2` baseline; runtime SIMD dispatch in lance-linalg
+          # picks the appropriate tier (scalar / AVX / AVX+FMA / AVX2+FMA
+          # / AVX-512) at load time. Same import as `lancedb` -- conflicts
+          # at install time, so users pick one.
+          - platform: x86_64
+            manylinux: "2_28"
+            extra_args: ""
+            runner: ubuntu-22.04
+            package_name: "lancedb-compat"
+            rustflags: "-Ctarget-cpu=x86-64-v2"
     runs-on: ${{ matrix.config.runner }}
     steps:
       - uses: actions/checkout@v6
@@ -52,11 +75,13 @@ jobs:
           args: "--release --strip ${{ matrix.config.extra_args }}"
           arm-build: ${{ matrix.config.platform == 'aarch64' }}
           manylinux: ${{ matrix.config.manylinux }}
+          package-name: ${{ matrix.config.package_name }}
+          rustflags: ${{ matrix.config.rustflags }}
       - uses: actions/upload-artifact@v7
-        if: startsWith(github.ref, 'refs/tags/python-v')
+        if: startsWith(github.ref, 'refs/tags/v')
         with:
-          name: wheels-linux-${{ matrix.config.platform }}-${{ matrix.config.manylinux }}
+          name: wheels-linux-${{ matrix.config.package_name }}-${{ matrix.config.platform }}-${{ matrix.config.manylinux }}
-          path: target/wheels/lancedb-*.whl
+          path: target/wheels/*.whl
           if-no-files-found: error
   mac:
     timeout-minutes: 90
@@ -82,7 +107,7 @@ jobs:
           python-minor-version: 10
           args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
       - uses: actions/upload-artifact@v7
-        if: startsWith(github.ref, 'refs/tags/python-v')
+        if: startsWith(github.ref, 'refs/tags/v')
         with:
           name: wheels-mac-${{ matrix.config.target }}
           path: target/wheels/lancedb-*.whl
@@ -103,19 +128,26 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: "3.13"
+      # NOTE: caching cargo here would be a no-op. This workflow only runs on
+      # tags and PRs, and GitHub only lets a run restore caches from its own ref
+      # or the default branch -- so with no run on main there is nothing that
+      # can populate an entry the release build would be allowed to read. Fixing
+      # this needs a main/nightly trigger (which would also catch wheel-build
+      # breakage before a release); the ~74 minutes here is otherwise dominated
+      # by the fat-LTO link, which no cache avoids.
       - uses: ./.github/workflows/build_windows_wheel
         with:
           python-minor-version: 10
           args: "--release --strip"
       - uses: actions/upload-artifact@v7
-        if: startsWith(github.ref, 'refs/tags/python-v')
+        if: startsWith(github.ref, 'refs/tags/v')
         with:
           name: wheels-windows
           path: target/wheels/lancedb-*.whl
           if-no-files-found: error
   publish:
     name: Publish wheels
-    if: startsWith(github.ref, 'refs/tags/python-v')
+    if: startsWith(github.ref, 'refs/tags/v')
     needs: [linux, mac, windows]
     runs-on: ubuntu-latest
     permissions:
@@ -145,7 +177,7 @@ jobs:
           FURY_TOKEN: ${{ secrets.FURY_TOKEN }}
         run: |
           shopt -s nullglob
-          WHEELS=(target/wheels/lancedb-*.whl)
+          WHEELS=(target/wheels/*.whl)
           if [[ ${#WHEELS[@]} -eq 0 ]]; then
             echo "No wheels found in target/wheels/" >&2
             exit 1
@@ -164,72 +196,6 @@ jobs:
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages-dir: target/wheels/
-  gh-release:
-    if: startsWith(github.ref, 'refs/tags/python-v')
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Extract version
-        id: extract_version
-        env:
-          GITHUB_REF: ${{ github.ref }}
-        run: |
-          set -e
-          echo "Extracting tag and version from $GITHUB_REF"
-          if [[ $GITHUB_REF =~ refs/tags/python-v(.*) ]]; then
-            VERSION=${BASH_REMATCH[1]}
-            TAG=python-v$VERSION
-            echo "tag=$TAG" >> $GITHUB_OUTPUT
-            echo "version=$VERSION" >> $GITHUB_OUTPUT
-          else
-            echo "Failed to extract version from $GITHUB_REF"
-            exit 1
-          fi
-          echo "Extracted version $VERSION from $GITHUB_REF"
-          if [[ $VERSION =~ beta ]]; then
-            echo "This is a beta release"
-
-            # Get last release (that is not this one)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^python-v \
-              | grep -vF "$TAG" \
-              | python ci/semver_sort.py python-v \
-              | tail -n 1)
-          else
-            echo "This is a stable release"
-            # Get last stable tag (ignore betas)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^python-v \
-              | grep -vF "$TAG" \
-              | grep -v beta \
-              | python ci/semver_sort.py python-v \
-              | tail -n 1)
-          fi
-          echo "Found from tag $FROM_TAG"
-          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
-      - name: Create Python Release Notes
-        id: python_release_notes
-        uses: mikepenz/release-changelog-builder-action@v4
-        with:
-          configuration: .github/release_notes.json
-          toTag: ${{ steps.extract_version.outputs.tag }}
-          fromTag: ${{ steps.extract_version.outputs.from_tag }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Create Python GH release
-        uses: softprops/action-gh-release@v2
-        with:
-          prerelease: ${{ contains('beta', github.ref) }}
-          tag_name: ${{ steps.extract_version.outputs.tag }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          generate_release_notes: false
-          name: Python LanceDB v${{ steps.extract_version.outputs.version }}
-          body: ${{ steps.python_release_notes.outputs.changelog }}
   report-failure:
     name: Report Workflow Failure
     runs-on: ubuntu-latest
@@ -237,7 +203,7 @@ jobs:
     permissions:
       contents: read
       issues: write
-    if: always() && failure() && startsWith(github.ref, 'refs/tags/python-v')
+    if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
     steps:
       - uses: actions/checkout@v6
       - uses: ./.github/actions/create-failure-issue

.github/workflows/python.yml

@@ -108,6 +108,15 @@ jobs:
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler
+      # `pip install -e .` builds the extension with maturin, which is most of
+      # this job's ~33 minutes. It had no Rust cache, so every dependency was
+      # recompiled from scratch on every run.
+      - uses: Swatinem/rust-cache@v2
+        with:
+          # Restore everywhere, but only save from main. Per-PR saves are
+          # unreadable outside their own branch anyway, since GitHub scopes
+          # caches to the creating ref.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Install
         run: |
           pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
@@ -168,6 +177,14 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: "3.13"
+      # maturin runs cargo natively on macOS (docker is Linux-only), so the host
+      # target dir is cacheable. This job had no Rust cache.
+      - uses: Swatinem/rust-cache@v2
+        with:
+          # Restore everywhere, but only save from main. Per-PR saves are
+          # unreadable outside their own branch anyway, since GitHub scopes
+          # caches to the creating ref.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - uses: ./.github/workflows/build_mac_wheel
         with:
           args: --profile ci
@@ -197,6 +214,14 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: "3.13"
+      # maturin runs cargo natively on Windows (docker is Linux-only), so the
+      # host target dir is cacheable. This job had no Rust cache at all and so
+      # rebuilt every dependency from scratch on every run.
+      - uses: Swatinem/rust-cache@v2
+        with:
+          # Restore everywhere, but only save from main. The repo sits at
+          # GitHub's cache cap, so per-PR saves just evict main's entries.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - uses: ./.github/workflows/build_windows_wheel
         with:
           args: --profile ci
@@ -224,6 +249,14 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: "3.10"
+      # As with Doctest, `pip install -e .` compiles the extension and this job
+      # had no Rust cache, which is most of its ~37 minutes.
+      - uses: Swatinem/rust-cache@v2
+        with:
+          # Restore everywhere, but only save from main. Per-PR saves are
+          # unreadable outside their own branch anyway, since GitHub scopes
+          # caches to the creating ref.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Install lancedb
         run: |
           pip install "pydantic<2"

.github/workflows/rust.yml

@@ -48,6 +48,11 @@ jobs:
         with:
           components: rustfmt, clippy
       - uses: Swatinem/rust-cache@v2
+        with:
+          # Restore everywhere, but only save from main. Per-PR saves are
+          # unreadable outside their own branch anyway, since GitHub scopes
+          # caches to the creating ref.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Install dependencies
         run: |
           sudo apt update
@@ -89,6 +94,11 @@ jobs:
         run: rm -f Cargo.lock
       - uses: rui314/setup-mold@v1
       - uses: Swatinem/rust-cache@v2
+        with:
+          # Restore everywhere, but only save from main. Per-PR saves are
+          # unreadable outside their own branch anyway, since GitHub scopes
+          # caches to the creating ref.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Install dependencies
         run: |
           sudo apt update
@@ -98,7 +108,7 @@ jobs:
           cargo build --profile ci --benches --all-features --tests
 
   linux:
-    timeout-minutes: 30
+    timeout-minutes: 60
     # To build all features, we need more disk space than is available
     # on the free OSS github runner. This is mostly due to the the
     # sentence-transformers feature.
@@ -118,6 +128,11 @@ jobs:
           fetch-depth: 0
           lfs: true
       - uses: Swatinem/rust-cache@v2
+        with:
+          # Restore everywhere, but only save from main. Per-PR saves are
+          # unreadable outside their own branch anyway, since GitHub scopes
+          # caches to the creating ref.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Install dependencies
         run: |
           sudo apt update
@@ -158,7 +173,7 @@ jobs:
         run: CARGO_ARGS="--profile ci" make -C ./lancedb remote-tests
 
   macos:
-    timeout-minutes: 30
+    timeout-minutes: 60
     strategy:
       matrix:
         mac-runner: ["macos-14", "macos-15"]
@@ -175,6 +190,11 @@ jobs:
       - name: CPU features
         run: sysctl -a | grep cpu
       - uses: Swatinem/rust-cache@v2
+        with:
+          # Restore everywhere, but only save from main. Per-PR saves are
+          # unreadable outside their own branch anyway, since GitHub scopes
+          # caches to the creating ref.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Install dependencies
         run: brew install protobuf
       - name: Run tests
@@ -187,12 +207,19 @@ jobs:
           cargo test --profile ci --features $ALL_FEATURES --locked
 
   windows:
-    runs-on: windows-2022
     strategy:
+      fail-fast: false
       matrix:
-        target:
+        include:
-          - x86_64-pc-windows-msvc
+          - target: x86_64-pc-windows-msvc
-          - aarch64-pc-windows-msvc
+            runner: windows-2022
+          # windows-11-arm is a standard runner, so it is free on public repos.
+          # Running natively lets the aarch64 tests actually execute -- this
+          # job used to cross-compile them and then skip the test step, paying
+          # full codegen and link cost for a compile check.
+          - target: aarch64-pc-windows-msvc
+            runner: windows-11-arm
+    runs-on: ${{ matrix.runner }}
     defaults:
       run:
         working-directory: rust/lancedb
@@ -201,6 +228,11 @@ jobs:
       - name: Set target
         run: rustup target add ${{ matrix.target }}
       - uses: Swatinem/rust-cache@v2
+        with:
+          # Restore everywhere, but only save from main. Per-PR saves are
+          # unreadable outside their own branch anyway, since GitHub scopes
+          # caches to the creating ref.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Install Protoc v21.12
         run: choco install --no-progress protoc
       - name: Build
@@ -208,11 +240,12 @@ jobs:
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
           cargo build --profile ci --features aws,remote --tests --locked --target ${{ matrix.target }}
       - name: Run tests
-        # Can only run tests when target matches host
-        if: ${{ matrix.target == 'x86_64-pc-windows-msvc' }}
         run: |
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo test --profile ci --features aws,remote --locked
+          # `--target` has to match the build step above. Without it cargo uses
+          # target/ci/ rather than target/<triple>/ci/ and rebuilds the entire
+          # dependency graph a second time.
+          cargo test --profile ci --features aws,remote --locked --target ${{ matrix.target }}
 
   msrv:
     # Check the minimum supported Rust version
@@ -238,6 +271,11 @@ jobs:
         with:
           toolchain: ${{ matrix.msrv }}
       - uses: Swatinem/rust-cache@v2
+        with:
+          # Restore everywhere, but only save from main. Per-PR saves are
+          # unreadable outside their own branch anyway, since GitHub scopes
+          # caches to the creating ref.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Downgrade  dependencies
         # These packages have newer requirements for MSRV
         run: |

AGENTS.md

@@ -92,6 +92,8 @@ Python bindings changes:
     * Should use `LOOP.run()` to call the corresponding `AsyncTable` method.
 6. Add concrete sync method to `RemoteTable` class in `python/python/lancedb/remote/table.py`.
 7. Add unit test in `python/tests/test_table.py`.
+8. If you added a new public class or module-level function (not just a method on an
+   existing class), expose it in the API reference. See "Python API reference" below.
 
 TypeScript bindings changes:
 
@@ -103,6 +105,33 @@ TypeScript bindings changes:
 5. Add test in `nodejs/__test__/table.test.ts`.
 6. Run `npm run docs` to generate TypeScript documentation.
 
+## Python API reference
+
+`docs/src/python/python.md` is the entire Python API reference. It is maintained by
+hand, and anything not listed there is not rendered at all, so new public classes and
+module-level functions have to be added explicitly. How depends on the module:
+
+* `lancedb.index`, `lancedb.embeddings`, `lancedb.remote`, and `lancedb.rerankers` are
+  rendered by a single directive each, driven by the module's `__all__`. Add the new
+  name to `__all__` and it appears; forget, and it is silently omitted.
+* Everything else (`lancedb`, `lancedb.table`, `lancedb.query`, `lancedb.db`, ...) is
+  listed symbol by symbol. Add a `::: lancedb.<module>.<Name>` line to the matching
+  section, and remember that the page separates synchronous and asynchronous APIs.
+
+Deliberately undocumented: concrete implementations reached through an abstract base
+(`LanceTable`, `LanceDBConnection`, `RemoteDBConnection`), query base classes already
+covered by `inherited_members`, and internal helpers.
+
+Cross-references in docstrings use mkdocstrings syntax, `[text][lancedb.table.Table]`.
+Plain relative links such as `[Table](Table)` do not resolve. To check your work:
+
+```shell
+pip install -r docs/requirements.txt
+cd docs && PYTHONPATH=. mkdocs build
+```
+
+The docs site only builds on pushes to `main`, so this is not covered by PR CI.
+
 ## Review Guidelines
 
 Please consider the following when reviewing code contributions.

Cargo.toml

@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=9.1.0-beta.2", default-features = false, "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=9.1.0-beta.2", "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=9.1.0-beta.2", "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=9.1.0-beta.2", "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=9.1.0-beta.2", default-features = false, "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=9.1.0-beta.2", "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=9.1.0-beta.2", "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=9.1.0-beta.2", "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=9.1.0-beta.2", default-features = false, "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=9.1.0-beta.2", "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=9.1.0-beta.2", "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=9.1.0-beta.2", "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=9.1.0-beta.2", "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=9.1.0-beta.2", "tag" = "v9.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }
@@ -64,7 +64,6 @@ snafu = "0.8"
 url = "2"
 num-traits = "0.2"
 regex = "1.10"
-lazy_static = "1"
 semver = "1.0.25"
 chrono = "0.4"
 

ci/bump_version.sh

@@ -2,9 +2,9 @@ set -e
 
 RELEASE_TYPE=${1:-"stable"}
 BUMP_MINOR=${2:-false}
-TAG_PREFIX=${3:-"v"} # Such as "python-v"
+HEAD_SHA=$(git rev-parse HEAD)
-HEAD_SHA=${4:-$(git rev-parse HEAD)}
 
+readonly TAG_PREFIX="v"
 readonly SELF_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 
 PREV_TAG=$(git tag --sort='version:refname' | grep ^$TAG_PREFIX | python $SELF_DIR/semver_sort.py $TAG_PREFIX | tail -n 1)

docs/mkdocs.yml

@@ -51,6 +51,11 @@ plugins:
           paths: [../python/python]
           options:
             docstring_style: numpy
+            docstring_options:
+              # Attributes documented in a `Parameters` section, and pydantic
+              # dataclasses whose `__init__` griffe cannot see statically, both
+              # trip this check. It reports nothing actionable here.
+              warn_unknown_params: false
             heading_level: 3
             show_signature_annotations: true
             show_root_heading: true

docs/openapi.yml

@@ -453,6 +453,16 @@ paths:
                     The metric type to use for the index. l2, Cosine, Dot are supported.
                 index_type:
                   type: string
+                custom_stop_words:
+                  type: [array, "null"]
+                  items:
+                    type: string
+                  description: |
+                    The custom stop-word list for an FTS index. A non-null
+                    array replaces the language's built-in stop-word list and is only
+                    applied when remove_stop_words is enabled. Null uses the built-in
+                    language list, while an empty array explicitly replaces it with no
+                    stop words.
       responses:
         "200":
           description: Index successfully created

docs/src/java/java.md

@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-core</artifactId>
-    <version>0.32.0-beta.2</version>
+    <version>0.37.1-beta.0</version>
 </dependency>
 ```
 
@@ -249,6 +249,57 @@ try (BufferAllocator allocator = new RootAllocator();
 }
 ```
 
+### Creating an Empty Table
+
+To create an empty table, send an Arrow IPC stream that contains the table schema and no record batches.
+The schema in the IPC stream becomes the table schema, and rows can be inserted later.
+
+```java
+import org.lance.namespace.model.CreateTableRequest;
+import org.lance.namespace.model.CreateTableResponse;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.ipc.ArrowStreamWriter;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.channels.Channels;
+import java.util.Arrays;
+
+Schema schema = new Schema(Arrays.asList(
+    new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null),
+    new Field("name", FieldType.nullable(new ArrowType.Utf8()), null),
+    new Field("embedding",
+        FieldType.nullable(new ArrowType.FixedSizeList(128)),
+        Arrays.asList(new Field("item",
+            FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
+            null)))
+));
+
+byte[] emptyTableData;
+try (BufferAllocator allocator = new RootAllocator();
+     VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
+    root.setRowCount(0);
+
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, Channels.newChannel(out))) {
+        writer.start();
+        writer.end();
+    }
+    emptyTableData = out.toByteArray();
+}
+
+CreateTableRequest request = new CreateTableRequest();
+request.setId(Arrays.asList("my_namespace", "empty_table"));
+
+CreateTableResponse response = namespaceClient.createTable(request, emptyTableData);
+```
+
 ### Insert
 
 ```java
@@ -431,9 +482,88 @@ query.setVector(vector);
 byte[] result = namespaceClient.queryTable(query);
 ```
 
-### Reading Query Results
+## Indexing
 
-Query results are returned in Apache Arrow IPC file format. Here's how to read them:
+The Java SDK exposes the REST namespace index operations through the same `LanceNamespace` client.
+Index creation runs asynchronously, so use `listTableIndices` or `describeTableIndexStats` to check progress.
+
+### Creating a Vector Index
+
+```java
+import org.lance.namespace.model.CreateTableIndexRequest;
+import org.lance.namespace.model.CreateTableIndexResponse;
+
+CreateTableIndexRequest request = new CreateTableIndexRequest();
+request.setId(Arrays.asList("my_namespace", "my_table"));
+request.setColumn("embedding");
+request.setIndexType("IVF_PQ");
+request.setDistanceType("cosine");
+request.setName("embedding_idx");
+
+CreateTableIndexResponse response = namespaceClient.createTableIndex(request);
+System.out.println("Index transaction: " + response.getTransactionId());
+```
+
+### Creating a Scalar Index
+
+```java
+import org.lance.namespace.model.CreateTableIndexRequest;
+import org.lance.namespace.model.CreateTableScalarIndexResponse;
+
+CreateTableIndexRequest request = new CreateTableIndexRequest();
+request.setId(Arrays.asList("my_namespace", "my_table"));
+request.setColumn("category");
+request.setIndexType("BTREE");
+request.setName("category_idx");
+
+CreateTableScalarIndexResponse response = namespaceClient.createTableScalarIndex(request);
+System.out.println("Index transaction: " + response.getTransactionId());
+```
+
+### Creating a Full Text Search Index
+
+```java
+import org.lance.namespace.model.CreateTableIndexRequest;
+import org.lance.namespace.model.CreateTableScalarIndexResponse;
+
+CreateTableIndexRequest request = new CreateTableIndexRequest();
+request.setId(Arrays.asList("my_namespace", "my_table"));
+request.setColumn("text_column");
+request.setIndexType("FTS");
+request.setName("text_idx");
+request.setBaseTokenizer("simple");
+request.setLowerCase(true);
+request.setWithPosition(true);
+
+CreateTableScalarIndexResponse response = namespaceClient.createTableScalarIndex(request);
+System.out.println("Index transaction: " + response.getTransactionId());
+```
+
+### Listing Indexes
+
+```java
+import org.lance.namespace.model.IndexContent;
+import org.lance.namespace.model.ListTableIndicesRequest;
+import org.lance.namespace.model.ListTableIndicesResponse;
+
+ListTableIndicesRequest request = new ListTableIndicesRequest();
+request.setId(Arrays.asList("my_namespace", "my_table"));
+
+ListTableIndicesResponse response = namespaceClient.listTableIndices(request);
+for (IndexContent index : response.getIndexes()) {
+    System.out.println(index.getIndexName() + ": " + index.getStatus());
+}
+```
+
+!!! note
+    The current Java namespace API exposes index type, index name, distance type, and full text search tokenizer options.
+    IVF training parameters such as `num_partitions` are not exposed by `CreateTableIndexRequest` yet.
+    To make those configurable from Java, the namespace API must add those fields first.
+
+## Reading Query Results
+
+Query results are returned as bytes in Apache Arrow IPC file format. Put the byte-channel
+adapter behind a small helper so query code can work with `ArrowFileReader` directly:
 
 ```java
 import org.apache.arrow.vector.ipc.ArrowFileReader;
@@ -441,16 +571,21 @@ import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.SeekableByteChannel;
 
-// Helper class to read Arrow data from byte array
+final class ArrowIpc {
-class ByteArraySeekableByteChannel implements SeekableByteChannel {
+    static ArrowFileReader openFileReader(byte[] data, BufferAllocator allocator) throws IOException {
+        return new ArrowFileReader(new ByteArraySeekableByteChannel(data), allocator);
+    }
+
+    private static final class ByteArraySeekableByteChannel implements SeekableByteChannel {
         private final byte[] data;
         private long position = 0;
         private boolean isOpen = true;
 
-    public ByteArraySeekableByteChannel(byte[] data) {
+        private ByteArraySeekableByteChannel(byte[] data) {
             this.data = data;
         }
 
@@ -473,13 +608,13 @@ class ByteArraySeekableByteChannel implements SeekableByteChannel {
         @Override public int write(ByteBuffer src) { throw new UnsupportedOperationException(); }
         @Override public SeekableByteChannel truncate(long size) { throw new UnsupportedOperationException(); }
     }
+}
 
 // Read query results
 byte[] queryResult = namespaceClient.queryTable(query);
 
 try (BufferAllocator allocator = new RootAllocator();
-     ArrowFileReader reader = new ArrowFileReader(
+     ArrowFileReader reader = ArrowIpc.openFileReader(queryResult, allocator)) {
-         new ByteArraySeekableByteChannel(queryResult), allocator)) {
 
     for (int i = 0; i < reader.getRecordBlocks().size(); i++) {
         reader.loadRecordBatch(reader.getRecordBlocks().get(i));

docs/src/js/_media/CONTRIBUTING.md

@@ -1,7 +1,7 @@
 # Contributing to LanceDB Typescript
 
 This document outlines the process for contributing to LanceDB Typescript.
-For general contribution guidelines, see [CONTRIBUTING.md](../CONTRIBUTING.md).
+For general contribution guidelines, see [CONTRIBUTING.md](https://github.com/lancedb/lancedb/blob/main/CONTRIBUTING.md).
 
 ## Project layout
 

docs/src/js/classes/Branches.md

@@ -83,6 +83,24 @@ Delete a branch.
 
 ***
 
+### diff()
+
+```ts
+diff(fromBranch): Promise<BranchDiff>
+```
+
+Compare a branch against main without modifying either branch.
+
+#### Parameters
+
+* **fromBranch**: `string`
+
+#### Returns
+
+`Promise`&lt;[`BranchDiff`](../interfaces/BranchDiff.md)&gt;
+
+***
+
 ### list()
 
 ```ts
@@ -94,3 +112,28 @@ List all branches, mapping name to branch metadata.
 #### Returns
 
 `Promise`&lt;`Record`&lt;`string`, [`BranchContents`](BranchContents.md)&gt;&gt;
+
+***
+
+### merge()
+
+```ts
+merge(fromBranch, dryRun): Promise<MergeBranchResult>
+```
+
+Merge a branch into main.
+
+Set `dryRun` to `true` to preview the merge. A rejected merge resolves
+with `status: "rejected"` instead of throwing.
+
+#### Parameters
+
+* **fromBranch**: `string`
+    Branch to merge from.
+
+* **dryRun**: `boolean` = `false`
+    When true, only preview the merge. Defaults to false.
+
+#### Returns
+
+`Promise`&lt;[`MergeBranchResult`](../interfaces/MergeBranchResult.md)&gt;

docs/src/js/classes/MergeInsertBuilder.md

@@ -76,24 +76,23 @@ the query optimizer chooses a suboptimal path.
 
 ***
 
-### useLsmWrite()
+### useLsm()
 
 ```ts
-useLsmWrite(useLsmWrite): MergeInsertBuilder
+useLsm(enable): MergeInsertBuilder
 ```
 
-Controls whether the merge uses the MemWAL LSM write path.
+Control MemWAL routing for this merge.
 
 By default (unset), a `mergeInsert` on a table with an LSM write spec is
-routed through Lance's MemWAL shard writer, and a table without one uses
+routed through Lance's MemWAL shard writer, and a table without one uses the
-the standard path. Pass `false` to force the standard path even when a
+standard path.
-spec is set. Pass `true` to require a spec — `mergeInsert` rejects if none
-is installed.
 
 #### Parameters
 
-* **useLsmWrite**: `boolean`
+* **enable**: `boolean`
-    Whether to use the LSM write path.
+    `true` forces MemWAL routing and errors if the table has no
+    LSM write spec. `false` forces the standard write path even when a spec is set.
 
 #### Returns
 

docs/src/js/classes/Query.md

@@ -497,6 +497,42 @@ ArrowTable.
 
 ***
 
+### useLsm()
+
+```ts
+useLsm(enable): this
+```
+
+Control MemWAL read routing for this query.
+
+By default (unset), when the table carries a MemWAL write spec (see
+[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
+they also return data written via the `mergeInsert` LSM path that has not yet
+been compacted into the base table (the active/frozen in-memory memtables and
+the flushed generations), deduplicated by primary key; a table without a spec
+reads the base table.
+
+#### Parameters
+
+* **enable**: `boolean`
+    `true` forces the LSM scanner and errors if the table has no
+    MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
+    even when a spec is present.
+    Note: the LSM scanner does not support every query shape (e.g. reranking,
+    hybrid search, `orderBy`). On a MemWAL table those shapes error unless
+    `useLsm(false)` is set, because a base-only read would silently exclude
+    un-compacted MemWAL data.
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+`StandardQueryBase.useLsm`
+
+***
+
 ### where()
 
 ```ts

docs/src/js/classes/TakeQuery.md

@@ -273,6 +273,29 @@ ArrowTable.
 
 ***
 
+### useLsm()
+
+```ts
+useLsm(enable): this
+```
+
+Control MemWAL read routing for this take query.
+
+`false` bypasses the MemWAL and reads the base table only — the escape hatch,
+since take-by-row-id/offset is not supported on the LSM scanner and, on a
+MemWAL table, auto-routes to it and errors otherwise.
+
+#### Parameters
+
+* **enable**: `boolean`
+    `false` reads the base table only.
+
+#### Returns
+
+`this`
+
+***
+
 ### withRowId()
 
 ```ts

docs/src/js/classes/VectorQuery.md

@@ -746,6 +746,42 @@ ArrowTable.
 
 ***
 
+### useLsm()
+
+```ts
+useLsm(enable): this
+```
+
+Control MemWAL read routing for this query.
+
+By default (unset), when the table carries a MemWAL write spec (see
+[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
+they also return data written via the `mergeInsert` LSM path that has not yet
+been compacted into the base table (the active/frozen in-memory memtables and
+the flushed generations), deduplicated by primary key; a table without a spec
+reads the base table.
+
+#### Parameters
+
+* **enable**: `boolean`
+    `true` forces the LSM scanner and errors if the table has no
+    MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
+    even when a spec is present.
+    Note: the LSM scanner does not support every query shape (e.g. reranking,
+    hybrid search, `orderBy`). On a MemWAL table those shapes error unless
+    `useLsm(false)` is set, because a base-only read would silently exclude
+    un-compacted MemWAL data.
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+`StandardQueryBase.useLsm`
+
+***
+
 ### where()
 
 ```ts

docs/src/js/globals.md

@@ -52,6 +52,11 @@
 - [AddDataOptions](interfaces/AddDataOptions.md)
 - [AddResult](interfaces/AddResult.md)
 - [AlterColumnsResult](interfaces/AlterColumnsResult.md)
+- [BranchColumnChange](interfaces/BranchColumnChange.md)
+- [BranchColumnSummary](interfaces/BranchColumnSummary.md)
+- [BranchDiff](interfaces/BranchDiff.md)
+- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
+- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
 - [ClientConfig](interfaces/ClientConfig.md)
 - [ColumnAlteration](interfaces/ColumnAlteration.md)
 - [ColumnOrdering](interfaces/ColumnOrdering.md)
@@ -86,6 +91,9 @@
 - [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
 - [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
 - [LsmWriteSpec](interfaces/LsmWriteSpec.md)
+- [MergeBlocker](interfaces/MergeBlocker.md)
+- [MergeBranchResult](interfaces/MergeBranchResult.md)
+- [MergePreview](interfaces/MergePreview.md)
 - [MergeResult](interfaces/MergeResult.md)
 - [NativeOAuthConfig](interfaces/NativeOAuthConfig.md)
 - [OAuthConfig](interfaces/OAuthConfig.md)

docs/src/js/interfaces/BranchColumnChange.md

@@ -0,0 +1,33 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / BranchColumnChange
+
+# Interface: BranchColumnChange
+
+A column whose definition differs between main and the branch.
+
+## Properties
+
+### branch
+
+```ts
+branch: BranchColumnSummary;
+```
+
+***
+
+### main
+
+```ts
+main: BranchColumnSummary;
+```
+
+***
+
+### name
+
+```ts
+name: string;
+```

docs/src/js/interfaces/BranchColumnSummary.md

@@ -0,0 +1,33 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / BranchColumnSummary
+
+# Interface: BranchColumnSummary
+
+Summary of a column in a branch diff.
+
+## Properties
+
+### dataType
+
+```ts
+dataType: string;
+```
+
+***
+
+### name
+
+```ts
+name: string;
+```
+
+***
+
+### nullable
+
+```ts
+nullable: boolean;
+```

docs/src/js/interfaces/BranchDiff.md

@@ -0,0 +1,129 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / BranchDiff
+
+# Interface: BranchDiff
+
+Read-only comparison of a branch against main.
+
+## Properties
+
+### addedColumns
+
+```ts
+addedColumns: BranchColumnSummary[];
+```
+
+***
+
+### addedIndexes
+
+```ts
+addedIndexes: BranchIndexSummary[];
+```
+
+***
+
+### baseMoved
+
+```ts
+baseMoved: boolean;
+```
+
+***
+
+### branchVersion
+
+```ts
+branchVersion: number;
+```
+
+***
+
+### changedColumns
+
+```ts
+changedColumns: BranchColumnChange[];
+```
+
+***
+
+### fromBranch
+
+```ts
+fromBranch: string;
+```
+
+***
+
+### mainVersion
+
+```ts
+mainVersion: number;
+```
+
+***
+
+### mergeBlockers
+
+```ts
+mergeBlockers: MergeBlocker[];
+```
+
+***
+
+### mergeable
+
+```ts
+mergeable: boolean;
+```
+
+***
+
+### parentVersion
+
+```ts
+parentVersion: number;
+```
+
+***
+
+### removedColumns
+
+```ts
+removedColumns: BranchColumnSummary[];
+```
+
+***
+
+### removedIndexes
+
+```ts
+removedIndexes: BranchIndexSummary[];
+```
+
+***
+
+### rowCountBranch
+
+```ts
+rowCountBranch: number;
+```
+
+***
+
+### rowCountMain
+
+```ts
+rowCountMain: number;
+```
+
+***
+
+### rowSummary
+
+```ts
+rowSummary: BranchRowCountSummary;
+```

docs/src/js/interfaces/BranchIndexSummary.md

@@ -0,0 +1,41 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / BranchIndexSummary
+
+# Interface: BranchIndexSummary
+
+Summary of an index in a branch diff.
+
+## Properties
+
+### columns
+
+```ts
+columns: string[];
+```
+
+***
+
+### indexName
+
+```ts
+indexName: string;
+```
+
+***
+
+### indexType?
+
+```ts
+optional indexType: string;
+```
+
+***
+
+### status
+
+```ts
+status: string;
+```

docs/src/js/interfaces/BranchRowCountSummary.md

@@ -0,0 +1,57 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / BranchRowCountSummary
+
+# Interface: BranchRowCountSummary
+
+Row-level comparison between main and the branch.
+
+## Properties
+
+### deltaAvailable
+
+```ts
+deltaAvailable: boolean;
+```
+
+***
+
+### inputsChanged
+
+```ts
+inputsChanged: number;
+```
+
+***
+
+### newOnBase
+
+```ts
+newOnBase: number;
+```
+
+***
+
+### newOnBranch
+
+```ts
+newOnBranch: number;
+```
+
+***
+
+### staleRecompute
+
+```ts
+staleRecompute: number;
+```
+
+***
+
+### unchanged
+
+```ts
+unchanged: number;
+```

docs/src/js/interfaces/FtsOptions.md

@@ -43,6 +43,34 @@ The following tokenizers are available:
 
 ***
 
+### blockSize?
+
+```ts
+optional blockSize: 128 | 256;
+```
+
+Number of documents per compressed posting block.
+
+The default is 128. Supported values are 128 and 256. A value of 256 uses
+the experimental FTS V3 format and may introduce breaking changes.
+
+***
+
+### customStopWords?
+
+```ts
+optional customStopWords: string[];
+```
+
+Custom stop words that replace the built-in list for `language`.
+
+This option only affects tokenization when `removeStopWords` is true.
+
+`undefined` keeps the built-in language list. An empty array explicitly
+replaces it with no stop words.
+
+***
+
 ### language?
 
 ```ts

docs/src/js/interfaces/MergeBlocker.md

@@ -0,0 +1,25 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MergeBlocker
+
+# Interface: MergeBlocker
+
+A reason why a branch cannot currently be merged.
+
+## Properties
+
+### code
+
+```ts
+code: string;
+```
+
+***
+
+### message
+
+```ts
+message: string;
+```

docs/src/js/interfaces/MergeBranchResult.md

@@ -0,0 +1,46 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MergeBranchResult
+
+# Interface: MergeBranchResult
+
+Result of previewing or attempting a branch merge.
+
+## Properties
+
+### diff
+
+```ts
+diff: BranchDiff;
+```
+
+***
+
+### mainVersionAfter?
+
+```ts
+optional mainVersionAfter: number;
+```
+
+***
+
+### preview
+
+```ts
+preview: MergePreview;
+```
+
+***
+
+### status
+
+```ts
+status:
+  | "unknown"
+  | "rejected"
+  | "ready"
+  | "notImplemented"
+  | "merged";
+```

docs/src/js/interfaces/MergePreview.md

@@ -0,0 +1,17 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MergePreview
+
+# Interface: MergePreview
+
+Changes that would be, or were, promoted by a branch merge.
+
+## Properties
+
+### promotedColumns
+
+```ts
+promotedColumns: string[];
+```

docs/src/js/interfaces/TokenizeOptions.md

@@ -30,6 +30,21 @@ The tokenizer to use. The default is "simple".
 
 ***
 
+### customStopWords?
+
+```ts
+optional customStopWords: string[];
+```
+
+Custom stop words that replace the built-in list for `language`.
+
+This option only affects tokenization when `removeStopWords` is true.
+
+`undefined` keeps the built-in language list. An empty array explicitly
+replaces it with no stop words.
+
+***
+
 ### language?
 
 ```ts

docs/src/python/python.md

@@ -26,6 +26,18 @@ is also an [asynchronous API client](#connections-asynchronous).
 
 ::: lancedb.db.DBConnection
 
+::: lancedb.Session
+
+## Namespaces (Synchronous)
+
+A namespace-backed connection resolves tables through a
+[Lance namespace](https://lancedb.github.io/lance-namespace/) service instead of
+listing a storage directory.
+
+::: lancedb.connect_namespace
+
+::: lancedb.namespace.LanceNamespaceDBConnection
+
 ## Tables (Synchronous)
 
 ::: lancedb.table.Table
@@ -34,8 +46,12 @@ is also an [asynchronous API client](#connections-asynchronous).
 
 ::: lancedb.table.FragmentSummaryStats
 
+::: lancedb.table.TableStatistics
+
 ::: lancedb.table.Tags
 
+::: lancedb.table.Branches
+
 ## Expressions
 
 Type-safe expression builder for filters and projections. Use these instead
@@ -62,29 +78,46 @@ of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and
 
 ::: lancedb.query.LanceHybridQueryBuilder
 
+::: lancedb.query.LanceEmptyQueryBuilder
+
+::: lancedb.query.LanceTakeQueryBuilder
+
+## Full text queries
+
+Structured full text queries can be passed to
+[Table.search][lancedb.table.Table.search] or
+[AsyncTable.search][lancedb.table.AsyncTable.search] in place of a query string,
+and combined with [BooleanQuery][lancedb.query.BooleanQuery].
+
+::: lancedb.query.FullTextQuery
+
+::: lancedb.query.MatchQuery
+
+::: lancedb.query.PhraseQuery
+
+::: lancedb.query.BoostQuery
+
+::: lancedb.query.MultiMatchQuery
+
+::: lancedb.query.BooleanQuery
+
+::: lancedb.query.FullTextOperator
+
+::: lancedb.query.Occur
+
 ## Embeddings
 
-::: lancedb.embeddings.registry.EmbeddingFunctionRegistry
+::: lancedb.embeddings
-
+    options:
-::: lancedb.embeddings.base.EmbeddingFunctionConfig
+      show_root_heading: false
-
+      show_root_toc_entry: false
-::: lancedb.embeddings.base.EmbeddingFunction
-
-::: lancedb.embeddings.base.TextEmbeddingFunction
-
-::: lancedb.embeddings.sentence_transformers.SentenceTransformerEmbeddings
-
-::: lancedb.embeddings.openai.OpenAIEmbeddings
-
-::: lancedb.embeddings.open_clip.OpenClipEmbeddings
 
 ## Remote configuration
 
-::: lancedb.remote.ClientConfig
+::: lancedb.remote
-
+    options:
-::: lancedb.remote.TimeoutConfig
+      show_root_heading: false
-
+      show_root_toc_entry: false
-::: lancedb.remote.RetryConfig
 
 ## Context
 
@@ -94,11 +127,50 @@ of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and
 
 ## Full text search
 
-Use [lancedb.table.Table.create_fts_index][] for the synchronous API or
+Pass `custom_stop_words` to [lancedb.index.FTS][]:
-[lancedb.table.AsyncTable.create_index][] with [lancedb.index.FTS][] for the
-asynchronous API.
 
-::: lancedb.index.FTS
+```python
+from lancedb.index import FTS
+
+table.create_index(
+    "text",
+    config=FTS(remove_stop_words=True, custom_stop_words=["acme", "internal"]),
+)
+```
+
+The list replaces the built-in stop words and is used only when
+`remove_stop_words=True`:
+
+- `custom_stop_words=None` uses the built-in list for `language`.
+- `custom_stop_words=[]` removes no words.
+- Values are passed through without trimming, lowercasing, or other rewriting.
+
+The same option is available on `lancedb.tokenize(...)` and the deprecated
+[lancedb.table.Table.create_fts_index][] compatibility helper:
+
+```python
+import lancedb
+
+tokens = list(lancedb.tokenize("acme makes searchable data",
+                               custom_stop_words=["acme"]))
+```
+
+::: lancedb.tokenize
+
+::: lancedb.FtsToken
+
+## Blobs
+
+Blob columns store large binary values out of line so they can be read lazily
+instead of being materialized with the rest of the row.
+
+::: lancedb.blob
+
+::: lancedb.BlobType
+
+::: lancedb._blob.BlobFile
+    options:
+      show_root_full_path: false
 
 ## Utilities
 
@@ -106,6 +178,14 @@ asynchronous API.
 
 ::: lancedb.merge.LanceMergeInsertBuilder
 
+::: lancedb.otel.instrument_lancedb_metrics
+
+## Exceptions
+
+::: lancedb.exceptions.MissingValueError
+
+::: lancedb.exceptions.MissingColumnError
+
 ## Integrations
 
 ## Pydantic
@@ -114,19 +194,30 @@ asynchronous API.
 
 ::: lancedb.pydantic.vector
 
+::: lancedb.pydantic.Vector
+
+::: lancedb.pydantic.MultiVector
+
 ::: lancedb.pydantic.LanceModel
 
+## PyTorch
+
+::: lancedb.streaming.StreamingDataset
+
+::: lancedb.permutation.permutation_builder
+
+::: lancedb.permutation.PermutationBuilder
+
+::: lancedb.permutation.Permutation
+
+::: lancedb.permutation.Transforms
+
 ## Reranking
 
-::: lancedb.rerankers.linear_combination.LinearCombinationReranker
+::: lancedb.rerankers
-
+    options:
-::: lancedb.rerankers.cohere.CohereReranker
+      show_root_heading: false
-
+      show_root_toc_entry: false
-::: lancedb.rerankers.colbert.ColbertReranker
-
-::: lancedb.rerankers.cross_encoder.CrossEncoderReranker
-
-::: lancedb.rerankers.openai.OpenaiReranker
 
 ## Connections (Asynchronous)
 
@@ -137,6 +228,12 @@ can be used to create, list, or open tables.
 
 ::: lancedb.db.AsyncConnection
 
+## Namespaces (Asynchronous)
+
+::: lancedb.connect_namespace_async
+
+::: lancedb.namespace.AsyncLanceNamespaceDBConnection
+
 ## Tables (Asynchronous)
 
 Table hold your actual data as a collection of records / rows.
@@ -145,32 +242,20 @@ Table hold your actual data as a collection of records / rows.
 
 ::: lancedb.table.AsyncTags
 
+::: lancedb.table.AsyncBranches
+
 ## Indices (Asynchronous)
 
 Indices can be created on a table to speed up queries. This section
 lists the indices that LanceDb supports.
 
-::: lancedb.index.BTree
+::: lancedb.index
-
+    options:
-::: lancedb.index.Bitmap
+      show_root_heading: false
-
+      show_root_toc_entry: false
-::: lancedb.index.LabelList
+      # `lang_mapping` is defined in the module rather than imported, so it is
-
+      # picked up despite not being in `__all__`. It is an internal lookup table.
-::: lancedb.index.FTS
+      filters: ["!^_", "!^lang_mapping$"]
-
-::: lancedb.index.IvfPq
-
-::: lancedb.index.HnswPq
-
-::: lancedb.index.HnswSq
-
-::: lancedb.index.IvfFlat
-
-::: lancedb.index.IvfSq
-
-::: lancedb.index.IvfRq
-
-::: lancedb.index.HnswFlat
 
 ::: lancedb.table.IndexStatistics
 
@@ -198,3 +283,7 @@ rows nearest to a query vector and can be created with the
 ::: lancedb.query.AsyncHybridQuery
     options:
       inherited_members: true
+
+::: lancedb.query.AsyncTakeQuery
+    options:
+      inherited_members: true

java/lancedb-core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
       <groupId>com.lancedb</groupId>
       <artifactId>lancedb-parent</artifactId>
-      <version>0.32.0-beta.2</version>
+      <version>0.37.1-beta.0</version>
       <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.32.0-beta.2</version>
+    <version>0.37.1-beta.0</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>9.1.0-beta.2</lance-core.version>
+        <lance-core.version>10.0.0-beta.5</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>

nodejs/CONTRIBUTING.md

@@ -1,7 +1,7 @@
 # Contributing to LanceDB Typescript
 
 This document outlines the process for contributing to LanceDB Typescript.
-For general contribution guidelines, see [CONTRIBUTING.md](../CONTRIBUTING.md).
+For general contribution guidelines, see [CONTRIBUTING.md](https://github.com/lancedb/lancedb/blob/main/CONTRIBUTING.md).
 
 ## Project layout
 

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.32.0-beta.2"
+version = "0.37.1-beta.0"
 publish = false
 license.workspace = true
 description.workspace = true

nodejs/__test__/arrow.test.ts

@@ -52,6 +52,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       Float64,
       Struct,
       List,
+      Map_,
       Int16,
       Int32,
       Int64,
@@ -69,6 +70,30 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
     type Schema = ApacheArrow["Schema"];
     type Table = ApacheArrow["Table"];
 
+    function expectValidMapField(
+      // biome-ignore lint/suspicious/noExplicitAny: Arrow Field types vary across supported versions
+      field: any,
+    ): void {
+      expect(DataType.isMap(field.type)).toBe(true);
+      expect(field.type.keysSorted).toBe(true);
+      expect(field.type.children).toHaveLength(1);
+
+      const entries = field.type.children[0];
+      expect(entries.name).toBe("entries");
+      expect(entries.nullable).toBe(false);
+      expect(DataType.isStruct(entries.type)).toBe(true);
+      expect(entries.type.children).toHaveLength(2);
+
+      const [key, value] = entries.type.children;
+      expect([key.name, value.name]).toEqual(["key", "value"]);
+      expect(key.nullable).toBe(false);
+      expect(DataType.isUtf8(key.type)).toBe(true);
+      expect(value.nullable).toBe(true);
+      expect(DataType.isInt(value.type)).toBe(true);
+      expect(value.type.bitWidth).toBe(32);
+      expect(value.type.isSigned).toBe(true);
+    }
+
     // Helper method to verify various ways to create a table
     async function checkTableCreation(
       tableCreationMethod: (
@@ -938,6 +963,65 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
           false,
         );
       });
+
+      it("will make an empty table with a Map field", async function () {
+        const schema = new Schema([
+          new Field(
+            "attributes",
+            new Map_(
+              new Field(
+                "entries",
+                new Struct([
+                  new Field("key", new Utf8(), false),
+                  new Field("value", new Int32(), true),
+                ]),
+                false,
+              ),
+              true,
+            ),
+          ),
+        ]);
+
+        const table = makeEmptyTable(schema);
+
+        expectValidMapField(table.schema.fields[0]);
+
+        const buffer = await fromTableToBuffer(table);
+        const roundTripped = tableFromIPC(buffer);
+
+        expectValidMapField(roundTripped.schema.fields[0]);
+      });
+
+      it("preserves string schema metadata", function () {
+        const metadata = new Map([["source", "fixture"]]);
+        const schema = new Schema(
+          [new Field("value", new Int32(), true)],
+          metadata,
+        );
+
+        expect(makeEmptyTable(schema).schema.metadata.get("source")).toBe(
+          "fixture",
+        );
+      });
+
+      it.each([
+        ["non-string keys", new Map<unknown, unknown>([[42, "fixture"]])],
+        ["non-string values", new Map<unknown, unknown>([["source", 42]])],
+        [
+          "non-string keys and values",
+          new Map<unknown, unknown>([[42, false]]),
+        ],
+      ])("rejects schema metadata with %s", function (_, metadataLike) {
+        const metadata = metadataLike as unknown as Map<string, string>;
+        const schema = new Schema(
+          [new Field("value", new Int32(), true)],
+          metadata,
+        );
+
+        expect(() => makeEmptyTable(schema)).toThrow(
+          "Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
+        );
+      });
     });
 
     describe("when using two versions of arrow", function () {

nodejs/__test__/remote.test.ts

@@ -15,6 +15,7 @@ import {
   OAuthHeaderProvider,
   StaticHeaderProvider,
 } from "../lancedb/header";
+import { Index } from "../lancedb/indices";
 
 // Test-only header providers
 class CustomProvider extends HeaderProvider {
@@ -225,6 +226,166 @@ describe("remote connection", () => {
     );
   });
 
+  it("sends FTS options to remote tables", async () => {
+    let createIndexBody: Record<string, unknown> | undefined;
+
+    await withMockDatabase(
+      (req, res) => {
+        const path = req.url ?? "";
+        if (path.endsWith("/describe/")) {
+          res.writeHead(200, { "Content-Type": "application/json" }).end(
+            JSON.stringify({
+              name: "t",
+              version: 1,
+              schema: {
+                fields: [
+                  { name: "text", type: { type: "string" }, nullable: false },
+                ],
+              },
+            }),
+          );
+          return;
+        }
+
+        if (path.endsWith("/create_index/")) {
+          let raw = "";
+          req.on("data", (chunk) => {
+            raw += chunk;
+          });
+          req.on("end", () => {
+            createIndexBody = JSON.parse(raw);
+            res.writeHead(200).end();
+          });
+          return;
+        }
+
+        res.writeHead(404).end();
+      },
+      async (db) => {
+        const table = await db.openTable("t");
+        await table.createIndex("text", {
+          config: Index.fts({
+            blockSize: 256,
+            removeStopWords: true,
+            customStopWords: ["the"],
+          }),
+        });
+      },
+    );
+
+    expect(createIndexBody?.["column"]).toBe("text");
+    expect(createIndexBody?.["index_type"]).toBe("FTS");
+    expect(createIndexBody?.["block_size"]).toBe(256);
+    expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]);
+  });
+
+  it("diffs and merges remote branches", async () => {
+    const sampleDiff = {
+      fromBranch: "exp",
+      parentVersion: 1,
+      mainVersion: 2,
+      branchVersion: 3,
+      baseMoved: false,
+      rowCountMain: 3,
+      rowCountBranch: 3,
+      rowSummary: {
+        unchanged: 3,
+        newOnBase: 0,
+        newOnBranch: 0,
+        staleRecompute: 0,
+        inputsChanged: 0,
+        deltaAvailable: false,
+      },
+      addedColumns: [{ name: "tag", dataType: "utf8", nullable: true }],
+      removedColumns: [],
+      changedColumns: [],
+      addedIndexes: [],
+      removedIndexes: [],
+      mergeable: true,
+      mergeBlockers: [],
+    };
+    const mergeBodies: Record<string, unknown>[] = [];
+
+    await withMockDatabase(
+      (req, res) => {
+        const path = req.url ?? "";
+        if (path.endsWith("/describe/")) {
+          res.writeHead(200, { "Content-Type": "application/json" }).end(
+            JSON.stringify({
+              name: "t",
+              version: 2,
+              schema: { fields: [] },
+            }),
+          );
+          return;
+        }
+
+        let raw = "";
+        req.on("data", (chunk) => {
+          raw += chunk;
+        });
+        req.on("end", () => {
+          const body = raw ? JSON.parse(raw) : {};
+          if (path.endsWith("/branches/diff/")) {
+            // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
+            expect(body).toEqual({ from_branch: "exp" });
+            res
+              .writeHead(200, { "Content-Type": "application/json" })
+              .end(JSON.stringify(sampleDiff));
+            return;
+          }
+          if (path.endsWith("/branches/merge/")) {
+            mergeBodies.push(body);
+            const dryRun = body["dry_run"] === true;
+            const response = {
+              status: dryRun ? "ready" : "rejected",
+              diff: dryRun
+                ? sampleDiff
+                : {
+                    ...sampleDiff,
+                    mergeable: false,
+                    mergeBlockers: [
+                      { code: "baseMoved", message: "main has advanced" },
+                    ],
+                  },
+              preview: { promotedColumns: dryRun ? ["tag"] : [] },
+            };
+            res
+              .writeHead(dryRun ? 200 : 409, {
+                "Content-Type": "application/json",
+              })
+              .end(JSON.stringify(response));
+            return;
+          }
+          res.writeHead(404).end();
+        });
+      },
+      async (db) => {
+        const table = await db.openTable("t");
+        const branches = await table.branches();
+
+        await expect(branches.diff("exp")).resolves.toEqual(sampleDiff);
+
+        const rejected = await branches.merge("exp");
+        expect(rejected.status).toBe("rejected");
+        expect(rejected.diff.mergeBlockers).toEqual([
+          { code: "baseMoved", message: "main has advanced" },
+        ]);
+
+        const preview = await branches.merge("exp", true);
+        expect(preview.status).toBe("ready");
+        expect(preview.preview.promotedColumns).toEqual(["tag"]);
+      },
+    );
+
+    expect(mergeBodies).toEqual([
+      // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
+      { from_branch: "exp", dry_run: false },
+      // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
+      { from_branch: "exp", dry_run: true },
+    ]);
+  });
+
   describe("TlsConfig", () => {
     it("should create TlsConfig with all fields", () => {
       const tlsConfig: TlsConfig = {

nodejs/__test__/sanitize.test.ts

@@ -2,7 +2,7 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import * as arrow from "../lancedb/arrow";
-import { sanitizeField, sanitizeType } from "../lancedb/sanitize";
+import { sanitizeField, sanitizeMap, sanitizeType } from "../lancedb/sanitize";
 
 describe("sanitize", function () {
   describe("sanitizeType function", function () {
@@ -181,4 +181,15 @@ describe("sanitize", function () {
       );
     });
   });
+
+  describe("sanitizeMap function", function () {
+    it.each([
+      ["no children", []],
+      ["two children", [{}, {}]],
+    ])("should reject a Map type with %s", function (_, children) {
+      expect(() => sanitizeMap({ children, keysSorted: false })).toThrow(
+        "Expected a Map type to have exactly one child",
+      );
+    });
+  });
 });

nodejs/__test__/table.test.ts

@@ -527,6 +527,14 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       );
     });
 
+    it("should expose useLsm on takeRowIds as the base-only escape hatch", async () => {
+      await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
+      // useLsm(false) is reachable on TakeQuery (the escape hatch for MemWAL tables,
+      // where take-by-row-id auto-routes to the LSM scanner and is rejected).
+      const res = await table.takeRowIds([0, 2]).useLsm(false).toArray();
+      expect(res.map((r) => r.id)).toEqual([1, 3]);
+    });
+
     it("should throw for negative number in takeRowIds", () => {
       expect(() => table.takeRowIds([-1])).toThrow("Row id cannot be negative");
       expect(() => table.takeRowIds([0, -5, 2])).toThrow(
@@ -2527,6 +2535,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(results3.length).toBe(1);
     });
 
+    test("full text search with custom posting block size", async () => {
+      const db = await connect(tmpDir.name);
+      const data = [
+        { text: "hello world", vector: [0.1, 0.2, 0.3] },
+        { text: "goodbye world", vector: [0.4, 0.5, 0.6] },
+      ];
+      const table = await db.createTable("test", data);
+      await table.createIndex("text", {
+        config: Index.fts({ blockSize: 256 }),
+      });
+
+      const index = (await table.listIndices()).find(
+        (index) => index.indexType === "FTS",
+      );
+      expect(index?.indexVersion).toBe(3);
+      expect(
+        (index?.indexDetails as Record<string, unknown>)["block_size"],
+      ).toBe(256);
+
+      const results = await table.search("hello").toArray();
+      expect(results[0].text).toBe(data[0].text);
+    });
+
+    test("rejects invalid full text posting block size", () => {
+      expect(() => Index.fts({ blockSize: 129 as 128 | 256 })).toThrow(
+        "128 or 256",
+      );
+    });
+
     test("full text search without lowercase", async () => {
       const db = await connect(tmpDir.name);
       const data = [
@@ -2732,6 +2769,15 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
   },
 );
 
+test("tokenize supports custom stop words", async () => {
+  const tokens = await tokenize("the lance data", {
+    stem: false,
+    removeStopWords: true,
+    customStopWords: ["lance"],
+  });
+  expect(tokens.map((token) => token.text)).toEqual(["the", "data"]);
+});
+
 describe("when calling explainPlan", () => {
   let tmpDir: tmp.DirResult;
   let table: Table;
@@ -3170,14 +3216,14 @@ describe("LSM merge insert", () => {
     await table.closeLsmWriters();
   });
 
-  it("falls back to the standard path with useLsmWrite(false)", async () => {
+  it("falls back to the standard path with useLsm(false)", async () => {
     const conn = await connect(tmpDir.name);
     const table = await bucketTable(conn);
 
     const res = await table
       .mergeInsert("id")
       .whenNotMatchedInsertAll()
-      .useLsmWrite(false)
+      .useLsm(false)
       .execute([
         { id: "b", value: 9 },
         { id: "e", value: 5 },
@@ -3211,4 +3257,36 @@ describe("LSM merge insert", () => {
         .execute([{ id: "g", value: 7 }]),
     ).rejects.toThrow();
   });
+
+  it("auto-routes reads through the MemWAL scanner", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await bucketTable(conn); // base ids "a", "b"
+
+    await table
+      .mergeInsert("id")
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .execute([{ id: "c", value: 3 }]);
+
+    // Default read auto-routes and includes the active memtable row.
+    const lsm = await table.query().toArray();
+    expect(lsm.map((r) => r.id).sort()).toEqual(["a", "b", "c"]);
+
+    // useLsm(false) bypasses the MemWAL and reads the base table only.
+    const baseOnly = await table.query().useLsm(false).toArray();
+    expect(baseOnly.map((r) => r.id).sort()).toEqual(["a", "b"]);
+  });
+
+  it("reads the base table when no LSM spec is installed", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await conn.createEmptyTable(
+      "plain",
+      new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
+    );
+    // No spec: default read and useLsm(false) both succeed against the base table.
+    await expect(table.query().toArray()).resolves.toBeDefined();
+    await expect(table.query().useLsm(false).toArray()).resolves.toBeDefined();
+    // useLsm(true) demands MemWAL routing; without a spec it errors.
+    await expect(table.query().useLsm(true).toArray()).rejects.toThrow();
+  });
 });

nodejs/examples/full_text_search.test.ts

@@ -29,8 +29,14 @@ test("full text search", async () => {
     const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
 
     await tbl.createIndex("doc", {
-      config: lancedb.Index.fts(),
+      config: lancedb.Index.fts({
+        stem: false,
+        removeStopWords: true,
+        customStopWords: ["banana"],
+      }),
     });
+    const tokens = await tbl.tokenize("apple banana", { column: "doc" });
+    expect(tokens.map((token) => token.text)).toEqual(["apple"]);
 
     // --8<-- [start:full_text_search]
     const result = await tbl

nodejs/lancedb/index.ts

@@ -124,6 +124,14 @@ export {
 export {
   Table,
   Branches,
+  BranchColumnSummary,
+  BranchColumnChange,
+  BranchIndexSummary,
+  BranchRowCountSummary,
+  MergeBlocker,
+  BranchDiff,
+  MergePreview,
+  MergeBranchResult,
   AddDataOptions,
   UpdateOptions,
   OptimizeOptions,
@@ -186,6 +194,16 @@ export interface TokenizeOptions {
   /** Whether to remove stop words. */
   removeStopWords?: boolean;
 
+  /**
+   * Custom stop words that replace the built-in list for `language`.
+   *
+   * This option only affects tokenization when `removeStopWords` is true.
+   *
+   * `undefined` keeps the built-in language list. An empty array explicitly
+   * replaces it with no stop words.
+   */
+  customStopWords?: string[];
+
   /** Whether to fold ASCII characters. */
   asciiFolding?: boolean;
 
@@ -217,6 +235,7 @@ export async function tokenize(
     options?.lowercase,
     options?.stem,
     options?.removeStopWords,
+    options?.customStopWords,
     options?.asciiFolding,
     options?.ngramMinLength,
     options?.ngramMaxLength,

nodejs/lancedb/indices.ts

@@ -553,6 +553,16 @@ export interface FtsOptions {
    */
   removeStopWords?: boolean;
 
+  /**
+   * Custom stop words that replace the built-in list for `language`.
+   *
+   * This option only affects tokenization when `removeStopWords` is true.
+   *
+   * `undefined` keeps the built-in language list. An empty array explicitly
+   * replaces it with no stop words.
+   */
+  customStopWords?: string[];
+
   /**
    * whether to remove punctuation
    */
@@ -572,6 +582,14 @@ export interface FtsOptions {
    * whether to only index the prefix of the token for ngram tokenizer
    */
   prefixOnly?: boolean;
+
+  /**
+   * Number of documents per compressed posting block.
+   *
+   * The default is 128. Supported values are 128 and 256. A value of 256 uses
+   * the experimental FTS V3 format and may introduce breaking changes.
+   */
+  blockSize?: 128 | 256;
 }
 
 export class Index {
@@ -747,10 +765,12 @@ export class Index {
         options?.lowercase,
         options?.stem,
         options?.removeStopWords,
+        options?.customStopWords,
         options?.asciiFolding,
         options?.ngramMinLength,
         options?.ngramMaxLength,
         options?.prefixOnly,
+        options?.blockSize,
       ),
     );
   }

nodejs/lancedb/merge.ts

@@ -88,21 +88,17 @@ export class MergeInsertBuilder {
     );
   }
   /**
-   * Controls whether the merge uses the MemWAL LSM write path.
+   * Control MemWAL routing for this merge.
    *
    * By default (unset), a `mergeInsert` on a table with an LSM write spec is
-   * routed through Lance's MemWAL shard writer, and a table without one uses
+   * routed through Lance's MemWAL shard writer, and a table without one uses the
-   * the standard path. Pass `false` to force the standard path even when a
+   * standard path.
-   * spec is set. Pass `true` to require a spec — `mergeInsert` rejects if none
-   * is installed.
    *
-   * @param useLsmWrite - Whether to use the LSM write path.
+   * @param enable - `true` forces MemWAL routing and errors if the table has no
+   * LSM write spec. `false` forces the standard write path even when a spec is set.
    */
-  useLsmWrite(useLsmWrite: boolean): MergeInsertBuilder {
+  useLsm(enable: boolean): MergeInsertBuilder {
-    return new MergeInsertBuilder(
+    return new MergeInsertBuilder(this.#native.useLsm(enable), this.#schema);
-      this.#native.useLsmWrite(useLsmWrite),
-      this.#schema,
-    );
   }
   /**
    * Controls how an LSM merge checks that its input targets a single shard.

nodejs/lancedb/query.ts

@@ -460,6 +460,30 @@ export class StandardQueryBase<
     this.doCall((inner: NativeQueryType) => inner.fastSearch());
     return this;
   }
+
+  /**
+   * Control MemWAL read routing for this query.
+   *
+   * By default (unset), when the table carries a MemWAL write spec (see
+   * {@link Table#setLsmWriteSpec}), reads are routed through the LSM scanner so
+   * they also return data written via the `mergeInsert` LSM path that has not yet
+   * been compacted into the base table (the active/frozen in-memory memtables and
+   * the flushed generations), deduplicated by primary key; a table without a spec
+   * reads the base table.
+   *
+   * @param enable - `true` forces the LSM scanner and errors if the table has no
+   * MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
+   * even when a spec is present.
+   *
+   * Note: the LSM scanner does not support every query shape (e.g. reranking,
+   * hybrid search, `orderBy`). On a MemWAL table those shapes error unless
+   * `useLsm(false)` is set, because a base-only read would silently exclude
+   * un-compacted MemWAL data.
+   */
+  useLsm(enable: boolean): this {
+    this.doCall((inner: NativeQueryType) => inner.useLsm(enable));
+    return this;
+  }
 }
 
 /**
@@ -748,6 +772,20 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
   constructor(inner: NativeTakeQuery) {
     super(inner);
   }
+
+  /**
+   * Control MemWAL read routing for this take query.
+   *
+   * `false` bypasses the MemWAL and reads the base table only — the escape hatch,
+   * since take-by-row-id/offset is not supported on the LSM scanner and, on a
+   * MemWAL table, auto-routes to it and errors otherwise.
+   *
+   * @param enable - `false` reads the base table only.
+   */
+  useLsm(enable: boolean): this {
+    this.doCall((inner: NativeTakeQuery) => inner.useLsm(enable));
+    return this;
+  }
 }
 
 /** A builder for LanceDB queries.

nodejs/lancedb/sanitize.ts

@@ -84,7 +84,7 @@ export function sanitizeMetadata(
     throw Error("Expected metadata, if present, to be a Map<string, string>");
   }
   for (const item of metadataLike) {
-    if (!(typeof item[0] === "string" || !(typeof item[1] === "string"))) {
+    if (typeof item[0] !== "string" || typeof item[1] !== "string") {
       throw Error(
         "Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
       );
@@ -288,12 +288,11 @@ export function sanitizeMap(typeLike: object) {
   if (!("keysSorted" in typeLike) || typeof typeLike.keysSorted !== "boolean") {
     throw Error("Expected a Map type to have a `keysSorted` property");
   }
+  if (typeLike.children.length !== 1) {
+    throw Error("Expected a Map type to have exactly one child");
+  }
 
-  return new Map_(
+  return new Map_(sanitizeField(typeLike.children[0]), typeLike.keysSorted);
-    // biome-ignore lint/suspicious/noExplicitAny: skip
-    typeLike.children.map((field) => sanitizeField(field)) as any,
-    typeLike.keysSorted,
-  );
 }
 
 export function sanitizeDuration(typeLike: object) {

nodejs/lancedb/table.ts

@@ -1329,6 +1329,76 @@ export interface FieldMetadataUpdate {
   replace?: boolean;
 }
 
+/** Summary of a column in a branch diff. */
+export interface BranchColumnSummary {
+  name: string;
+  dataType: string;
+  nullable: boolean;
+}
+
+/** A column whose definition differs between main and the branch. */
+export interface BranchColumnChange {
+  name: string;
+  main: BranchColumnSummary;
+  branch: BranchColumnSummary;
+}
+
+/** Summary of an index in a branch diff. */
+export interface BranchIndexSummary {
+  indexName: string;
+  columns: string[];
+  indexType?: string;
+  status: string;
+}
+
+/** Row-level comparison between main and the branch. */
+export interface BranchRowCountSummary {
+  unchanged: number;
+  newOnBase: number;
+  newOnBranch: number;
+  staleRecompute: number;
+  inputsChanged: number;
+  deltaAvailable: boolean;
+}
+
+/** A reason why a branch cannot currently be merged. */
+export interface MergeBlocker {
+  code: string;
+  message: string;
+}
+
+/** Read-only comparison of a branch against main. */
+export interface BranchDiff {
+  fromBranch: string;
+  parentVersion: number;
+  mainVersion: number;
+  branchVersion: number;
+  baseMoved: boolean;
+  rowCountMain: number;
+  rowCountBranch: number;
+  rowSummary: BranchRowCountSummary;
+  addedColumns: BranchColumnSummary[];
+  removedColumns: BranchColumnSummary[];
+  changedColumns: BranchColumnChange[];
+  addedIndexes: BranchIndexSummary[];
+  removedIndexes: BranchIndexSummary[];
+  mergeable: boolean;
+  mergeBlockers: MergeBlocker[];
+}
+
+/** Changes that would be, or were, promoted by a branch merge. */
+export interface MergePreview {
+  promotedColumns: string[];
+}
+
+/** Result of previewing or attempting a branch merge. */
+export interface MergeBranchResult {
+  status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown";
+  diff: BranchDiff;
+  preview: MergePreview;
+  mainVersionAfter?: number;
+}
+
 /**
  * Branch manager for a {@link Table}.
  *
@@ -1381,4 +1451,28 @@ export class Branches {
   async delete(name: string): Promise<void> {
     return await this.#inner.delete(name);
   }
+
+  /** Compare a branch against main without modifying either branch. */
+  async diff(fromBranch: string): Promise<BranchDiff> {
+    return (await this.#inner.diff(fromBranch)) as unknown as BranchDiff;
+  }
+
+  /**
+   * Merge a branch into main.
+   *
+   * Set `dryRun` to `true` to preview the merge. A rejected merge resolves
+   * with `status: "rejected"` instead of throwing.
+   *
+   * @param fromBranch Branch to merge from.
+   * @param dryRun When true, only preview the merge. Defaults to false.
+   */
+  async merge(
+    fromBranch: string,
+    dryRun: boolean = false,
+  ): Promise<MergeBranchResult> {
+    return (await this.#inner.merge(
+      fromBranch,
+      dryRun,
+    )) as unknown as MergeBranchResult;
+  }
 }

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.32.0-beta.2",
+	"version": "0.37.1-beta.0",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.32.0-beta.2",
+	"version": "0.37.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.32.0-beta.2",
+	"version": "0.37.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.32.0-beta.2",
+	"version": "0.37.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.32.0-beta.2",
+	"version": "0.37.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.32.0-beta.2",
+  "version": "0.37.1-beta.0",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.32.0-beta.2",
+	"version": "0.37.1-beta.0",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.32.0-beta.2",
+  "version": "0.37.1-beta.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.32.0-beta.2",
+      "version": "0.37.1-beta.0",
       "cpu": [
         "x64",
         "arm64"

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.32.0-beta.2",
+  "version": "0.37.1-beta.0",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",

nodejs/src/index.rs

@@ -43,6 +43,7 @@ pub fn tokenize(
     lower_case: Option<bool>,
     stem: Option<bool>,
     remove_stop_words: Option<bool>,
+    custom_stop_words: Option<Vec<String>>,
     ascii_folding: Option<bool>,
     ngram_min_length: Option<u32>,
     ngram_max_length: Option<u32>,
@@ -72,6 +73,7 @@ pub fn tokenize(
     if let Some(remove_stop_words) = remove_stop_words {
         opts = opts.remove_stop_words(remove_stop_words);
     }
+    opts = opts.custom_stop_words(custom_stop_words);
     if let Some(ascii_folding) = ascii_folding {
         opts = opts.ascii_folding(ascii_folding);
     }
@@ -222,11 +224,13 @@ impl Index {
         lower_case: Option<bool>,
         stem: Option<bool>,
         remove_stop_words: Option<bool>,
+        custom_stop_words: Option<Vec<String>>,
         ascii_folding: Option<bool>,
         ngram_min_length: Option<u32>,
         ngram_max_length: Option<u32>,
         prefix_only: Option<bool>,
-    ) -> Self {
+        block_size: Option<u32>,
+    ) -> napi::Result<Self> {
         let mut opts = FtsIndexBuilder::default();
         if let Some(with_position) = with_position {
             opts = opts.with_position(with_position);
@@ -249,6 +253,7 @@ impl Index {
         if let Some(remove_stop_words) = remove_stop_words {
             opts = opts.remove_stop_words(remove_stop_words);
         }
+        opts = opts.custom_stop_words(custom_stop_words);
         if let Some(ascii_folding) = ascii_folding {
             opts = opts.ascii_folding(ascii_folding);
         }
@@ -261,10 +266,15 @@ impl Index {
         if let Some(prefix_only) = prefix_only {
             opts = opts.ngram_prefix_only(prefix_only);
         }
-
+        if let Some(block_size) = block_size {
-        Self {
+            opts = opts
-            inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
+                .block_size(block_size as usize)
+                .map_err(|err| napi::Error::from_reason(err.to_string()))?;
         }
+
+        Ok(Self {
+            inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
+        })
     }
 
     #[napi(factory)]

nodejs/src/merge.rs

@@ -51,9 +51,9 @@ impl NativeMergeInsertBuilder {
     }
 
     #[napi]
-    pub fn use_lsm_write(&self, use_lsm_write: bool) -> Self {
+    pub fn use_lsm(&self, enable: bool) -> Self {
         let mut this = self.clone();
-        this.inner.use_lsm_write(use_lsm_write);
+        this.inner.use_lsm(enable);
         this
     }
 

nodejs/src/query.rs

@@ -168,6 +168,11 @@ impl Query {
         self.inner = self.inner.clone().with_row_id();
     }
 
+    #[napi]
+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_lsm(enable);
+    }
+
     #[napi]
     pub fn order_by(&mut self, ordering: Option<Vec<ColumnOrdering>>) -> napi::Result<()> {
         let ordering = ordering.map(|ordering| {
@@ -374,6 +379,11 @@ impl VectorQuery {
         self.inner = self.inner.clone().with_row_id();
     }
 
+    #[napi]
+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_lsm(enable);
+    }
+
     #[napi]
     pub fn rerank(
         &mut self,
@@ -479,6 +489,11 @@ impl TakeQuery {
         self.inner = self.inner.clone().with_row_id();
     }
 
+    #[napi]
+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_lsm(enable);
+    }
+
     #[napi(catch_unwind)]
     pub async fn output_schema(&self) -> napi::Result<Buffer> {
         let schema = self.inner.output_schema().await.default_error()?;

nodejs/src/remote.rs

@@ -232,6 +232,10 @@ impl From<ClientConfig> for lancedb::remote::ClientConfig {
             tls_config: config.tls_config.map(Into::into),
             header_provider: None, // the header provider is set separately later
             user_id: config.user_id,
+            // Resolved from LANCE_CLIENT_MAX_BYTES_PER_REQUEST or the default.
+            max_bytes_per_request: None,
+            // Resolved from LANCE_CLIENT_MAX_REQUEST_DURATION or the read timeout.
+            max_request_duration: None,
         }
     }
 }

nodejs/src/table.rs

@@ -1355,4 +1355,28 @@ impl Branches {
     pub async fn delete(&self, name: String) -> napi::Result<()> {
         self.inner.delete_branch(&name).await.default_error()
     }
+
+    #[napi(ts_return_type = "Promise<Record<string, unknown>>")]
+    pub async fn diff(&self, from_branch: String) -> napi::Result<serde_json::Value> {
+        let diff = self.inner.diff_branch(&from_branch).await.default_error()?;
+        serde_json::to_value(diff).map_err(|err| {
+            napi::Error::from_reason(format!("failed to serialize branch diff: {err}"))
+        })
+    }
+
+    #[napi(ts_return_type = "Promise<Record<string, unknown>>")]
+    pub async fn merge(
+        &self,
+        from_branch: String,
+        dry_run: Option<bool>,
+    ) -> napi::Result<serde_json::Value> {
+        let result = self
+            .inner
+            .merge_branch(&from_branch, dry_run.unwrap_or(false))
+            .await
+            .default_error()?;
+        serde_json::to_value(result).map_err(|err| {
+            napi::Error::from_reason(format!("failed to serialize branch merge result: {err}"))
+        })
+    }
 }

plugins/lancedb/.claude-plugin/plugin.json

@@ -0,0 +1,21 @@
+{
+  "name": "lancedb",
+  "description": "Write, review, debug, and document LanceDB pipelines in Python and TypeScript that work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables, with idiomatic query/search patterns and performance defaults for ingestion, indexing, filtering, and diagnostics.",
+  "version": "0.1.0",
+  "author": {
+    "name": "LanceDB"
+  },
+  "homepage": "https://www.lancedb.com",
+  "keywords": [
+    "lancedb",
+    "vector-search",
+    "full-text-search",
+    "hybrid-search",
+    "python",
+    "typescript",
+    "pipelines",
+    "ingestion",
+    "indexing",
+    "performance"
+  ]
+}

plugins/lancedb/.codex-plugin/plugin.json

@@ -0,0 +1,33 @@
+{
+  "name": "lancedb",
+  "version": "0.1.0",
+  "description": "Codex plugin for building LanceDB pipelines in Python and TypeScript.",
+  "author": {
+    "name": "LanceDB"
+  },
+  "keywords": [
+    "lancedb",
+    "vector-search",
+    "full-text-search",
+    "hybrid-search",
+    "python",
+    "typescript",
+    "pipelines"
+  ],
+  "skills": "./skills/",
+  "interface": {
+    "displayName": "LanceDB",
+    "shortDescription": "Build LanceDB pipelines in Python and TypeScript.",
+    "longDescription": "Write, review, debug, and document LanceDB pipelines in Python and TypeScript that work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables, with idiomatic query/search patterns and performance defaults for ingestion, indexing, filtering, and diagnostics.",
+    "developerName": "LanceDB",
+    "websiteURL": "https://www.lancedb.com",
+    "category": "Developer Tools",
+    "capabilities": [
+      "Developer Tools"
+    ],
+    "defaultPrompt": "Create a LanceDB table, embed sample text, and run a vector search.",
+    "composerIcon": "./assets/logo.png",
+    "logo": "./assets/logo.png",
+    "logoDark": "./assets/logo-dark.png"
+  }
+}

plugins/lancedb/skills/lancedb/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: lancedb
-description: Use when writing, reviewing, debugging, or documenting LanceDB pipelines in Python or TypeScript, especially code that should work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables. Helps avoid non-portable full-table materialization, choose idiomatic query/search patterns, and apply LanceDB performance defaults for ingestion, indexing, filtering, and diagnostics.
+description: Use when writing, reviewing, debugging, or documenting LanceDB pipelines in Python or TypeScript, especially code that should work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables. Helps avoid non-portable full-table materialization, choose idiomatic query/search patterns, apply LanceDB performance defaults for ingestion, indexing, filtering, and diagnostics, and resolve connections to the remote server for Enterprise-only operations such as jobs.
 ---
 
 # Building LanceDB Pipelines
@@ -19,7 +19,7 @@ Do NOT assume local-only table helpers exist on remote tables. If the user asks
 ## Workflow
 
 1. Identify the SDK: Python, TypeScript, or both.
-2. Identify the table mode: local/embedded OSS, remote Enterprise/Cloud, or portable across both. If the user says "LanceDB Enterprise", choose the remote table path.
+2. Identify the table mode: local/embedded OSS, remote Enterprise/Cloud, or portable across both. If the user says "LanceDB Enterprise", choose the remote table path. If the task involves jobs in any way (listing, inspecting, creating, or canceling jobs), it is always the remote path and requires a remote server connection — see "Connecting to the LanceDB remote server" below before doing anything else.
 3. Read the matching language branch before writing or changing code:
    - Python patterns: `references/python/patterns.md`
    - Python API quick reference: `references/python/api_reference.md`
@@ -29,7 +29,9 @@ Do NOT assume local-only table helpers exist on remote tables. If the user asks
    - TypeScript performance guidance: `references/typescript/performance.md`
    - Column metadata authoring (both SDKs): `references/column_metadata.md`
    - Branch operations (both SDKs): `references/branch_ops.md`
-4. Start with `patterns.md` for the selected SDK. Read `api_reference.md` when choosing method names or return collectors. Read `performance.md` when the task involves ingestion, indexing, filtering, query tuning, diagnostics, or large datasets. Read `column_metadata.md` when the task is documenting, tagging, classifying, or grouping table columns (field descriptions, `lancedb:tag:*` tags, logical column families). Read `branch_ops.md` when the task involves branch lifecycle (list/create/delete), writing to a non-main branch, or verifying a change stayed off main.
+   - Remote server connection resolution (jobs, raw REST): `references/remote_connect.md`
+   - Job operations REST API (list/describe/cancel/query_events): `references/remote_jobs.md`
+4. Start with `patterns.md` for the selected SDK. Read `api_reference.md` when choosing method names or return collectors. Read `performance.md` when the task involves ingestion, indexing, filtering, query tuning, diagnostics, or large datasets. Read `column_metadata.md` when the task is documenting, tagging, classifying, or grouping table columns (field descriptions, `lancedb:tag:*` tags, logical column families). Read `branch_ops.md` when the task involves branch lifecycle (list/create/delete), writing to a non-main branch, or verifying a change stayed off main. Read `remote_connect.md` when the task involves jobs or direct REST access to an Enterprise deployment, and `remote_jobs.md` for the job REST methods themselves (list, describe, cancel, query_events).
 5. For Python schemas, favor Pydantic models and validate records before writing. Use PyArrow schemas when Arrow-native, streaming, or highly dynamic data makes them materially better suited.
 6. Prefer `search()` or `query()` builders with explicit `select()` and `limit()` for reads.
 7. Avoid table-level full materialization in remote or portable code. This is the main local-vs-remote read pitfall.
@@ -70,6 +72,10 @@ Rules for portable Enterprise ingestion:
 
 This is Enterprise/Cloud-specific. Local/OSS tables have no separate data plane, so `mode="overwrite"` and immediate same-name reuse are fine there.
 
+## Connecting to the LanceDB remote server
+
+LanceDB Enterprise/Cloud deployments are served by a server implementing the lance-namespace OpenAPI spec (<https://github.com/lance-format/lance-namespace/blob/main/docs/src/spec.yaml>). Every remote (`db://...`) connection talks to such a server, and some operations exist only there. In particular, **all operations around jobs (listing, inspecting, creating, or canceling jobs) run server-side** — there is no local/OSS equivalent. Before any job work, or any direct REST call to an Enterprise deployment, read `references/remote_connect.md` to resolve the base URL, credentials, and database header and to validate the connection. Then use the four job REST methods documented in `references/remote_jobs.md` (list, describe, cancel, query_events).
+
 ## Script
 
 Run the scanner when reviewing or modifying an existing codebase:

plugins/lancedb/skills/lancedb/agents/openai.yaml

@@ -0,0 +1,6 @@
+interface:
+  display_name: "LanceDB"
+  short_description: "Build LanceDB pipelines in Python and TypeScript"
+  default_prompt: "Use $lancedb to create a table, embed sample text, and run a vector search."
+  icon_small: "./assets/icon.png"
+  icon_large: "./assets/icon.png"

plugins/lancedb/skills/lancedb/references/branch_ops.md

@@ -2,7 +2,7 @@
 
 Manage branches on a LanceDB table: list what exists, create new ones, delete stale ones, and direct read/write operations at a specific branch without touching main. Use for branch lifecycle tasks, experimental/isolated table versions, targeting an operation at a non-main branch, or confirming a mutation did not affect main.
 
-Works on local/OSS and remote Enterprise/Cloud tables.
+Works on local/OSS and remote Enterprise/Cloud tables, except merging a branch into main, which is Enterprise-only.
 
 ## The branch model (important)
 
@@ -101,6 +101,69 @@ assert b"lancedb:description" not in (table.schema.field("category").metadata or
 
 Two handles on the same branch see each other's writes (e.g. `table.branches.create("exp")` and `db.open_table(name, branch="exp")`); main stays isolated.
 
+## Merging a branch into main (Enterprise only)
+
+Merge is available through the SDKs (`table.branches.merge(...)`) on **Enterprise tables only** — it is not supported on Cloud or local/OSS tables, which raise `NotSupported`.
+
+`merge` takes the branch to merge **from** and a `dry_run` flag. Both the SDK method and the underlying REST endpoint **actually merge by default** (`dry_run=False`); pass `dry_run=True` to only preview. A rejected merge is **not an exception** — it returns a result with `status="rejected"` rather than raising, so inspect the return value. Use `branches.diff(from_branch)` to inspect a branch's pending diff without attempting a merge.
+
+```python
+exp = "experiment-reindex"
+
+# preview only — returns status="ready" if it would merge cleanly
+preview = table.branches.merge(exp, dry_run=True)
+
+# actually merge (default)
+result = table.branches.merge(exp)
+if result["status"] == "merged":
+    print("landed at", result["mainVersionAfter"])
+elif result["status"] == "rejected":
+    print(result["diff"]["mergeBlockers"])   # why it was refused
+
+# inspect a branch's pending diff without merging
+diff = table.branches.diff(exp)
+```
+
+Async: `await table.branches.merge(exp)`, `await table.branches.diff(exp)`.
+
+```typescript
+const branches = await table.branches();
+const exp = "experiment-reindex";
+
+// preview only (second arg is dryRun)
+const preview = await branches.merge(exp, true);
+
+// actually merge (default)
+const result = await branches.merge(exp);
+if (result.status === "merged") {
+  console.log("landed at", result.mainVersionAfter);
+} else if (result.status === "rejected") {
+  console.log(result.diff.mergeBlockers);
+}
+
+const diff = await branches.diff(exp);
+```
+
+The result is the wire JSON, containing `status` (`ready` on a passing dry run, `merged` on success, `rejected` when refused — also `notImplemented`/`unknown`), the branch `diff` (including `mergeBlockers` explaining any rejection), a `preview` of the columns that would be promoted, and — after a real merge — `mainVersionAfter`.
+
+### Merge preconditions
+
+Merge only **promotes newly added columns** onto main; it does not replay arbitrary commits. Practically, a branch is mergeable only if it has **exactly one commit since it was created, and that commit added a column**. The merge is rejected (`status: "rejected"`, with `mergeBlockers` set) if:
+
+- the branch was forked from another branch rather than directly from main
+- main has advanced since the branch was forked
+- the branch's rows changed since the fork (row counts must match main exactly)
+- the branch removed columns or changed a column's type/nullability
+- the branch added no columns (index-only changes are not merged)
+
+### Adding a column in a single commit
+
+Because the branch must contain just one column-adding commit, add the column with its values in one operation rather than add-then-backfill:
+
+1. **SQL transformation** — `add_columns` with a SQL expression computed from existing columns, so the column lands populated in one commit.
+2. **Precompute the values** — compute the column's values externally, then add the fully-populated column in a single operation (e.g. via `merge_insert`/`add_columns` with the data ready).
+3. **Lance-format-level data evolution (pylance)** — use Lance's data evolution with backfill, documented at <https://lance.org/guide/data_evolution/#with-data-backfill>.
+
 ## Quick reference
 
 | Goal | Python | TypeScript |
@@ -113,5 +176,7 @@ Two handles on the same branch see each other's writes (e.g. `table.branches.cre
 | Delete branch | `table.branches.delete(name)` | `await branches.delete(name)` |
 | Which branch is this handle on? | `table.current_branch()` (`None` = main) | `table.currentBranch()` (`null` = main) |
 | Target main | use the original (non-branch) handle | use the original (non-branch) handle |
+| Merge branch into main (Enterprise only) | `table.branches.merge(from_branch, dry_run=False)` | `await branches.merge(fromBranch, dryRun)` |
+| Preview a branch's pending diff (Enterprise only) | `table.branches.diff(from_branch)` | `await branches.diff(fromBranch)` |
 
 Branch names must be non-empty; empty names raise a validation error.

plugins/lancedb/skills/lancedb/references/python/api_reference.md

@@ -4,12 +4,19 @@ Quick method reference for Python LanceDB code. Cross-check source for non-trivi
 
 ## Connect
 
+If you're connecting to a remote database, use this:
 ```python
 import lancedb
 
-db = lancedb.connect("./camelot-db")         # local/OSS
+db = lancedb.connect("db://my-db", api_key=api_key, host_override=host_override) # remote
-db = lancedb.connect("db://my-db", api_key=api_key, region=region)  # remote
 ```
+(values may be found in LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, either in env vars or a .env file)
+
+If you're connecting to a local table using OSS LanceDB, use this:
+```python
+db = lancedb.connect("./camelot-db") # local/OSS
+```
+If you're not sure which, or if you can't find the api_key or host_override params, ask the user.
 
 **Place the local database directory next to the script/entrypoint that opens it** (i.e. resolve the path relative to the script, `Path(__file__).parent / "camelot-db"`), not buried under a shared `data/` folder. The Lance dataset is the database, not a data file — keeping it beside its code makes ownership obvious and paths stable regardless of the working directory the script is launched from.
 

plugins/lancedb/skills/lancedb/references/remote_connect.md

@@ -0,0 +1,45 @@
+# Connecting to a LanceDB remote server
+
+LanceDB Enterprise/Cloud deployments are served by a server implementing the
+lance-namespace OpenAPI spec
+(<https://github.com/lance-format/lance-namespace/blob/main/docs/src/spec.yaml>).
+Every remote (`db://...`) connection talks to such a server, and some operations
+exist only there. In particular, all operations around jobs (listing, inspecting,
+creating, or canceling jobs) run server-side — there is no local/OSS equivalent, so
+resolve a server connection before attempting any job work. The job REST methods
+themselves are documented in `references/remote_jobs.md`.
+
+Every request needs two things:
+
+1. **Base URL** — the server endpoint
+2. **Credentials** — an API key (`x-api-key` header over REST), and usually a database name (`x-lancedb-database` header)
+
+## Resolution steps
+
+1. If the user already gave a URL and API key (or said which environment they're working against), use that.
+2. Otherwise, look for credentials already available in the environment:
+   - Env vars like `LANCEDB_URI` / `LANCEDB_HOST` / `LANCEDB_API_KEY`
+   - A server endpoint already running or port-forwarded locally (the REST default port is 2333, i.e. `http://localhost:2333`)
+3. If you didn't find both pieces, ask the user directly: **"What's your LanceDB endpoint's URL, and what's your API key?"** Also ask which database to use if it isn't obvious. Don't guess or probe further — the user knows their deployment.
+
+## Validating the connection
+
+Make a cheap authenticated request and check the status before starting real work:
+
+```bash
+curl -s -w "\n%{http_code}" "{base_url}/v1/table/?limit=1" \
+  -H "x-api-key: <key>" \
+  -H "x-lancedb-database: <database>"
+```
+
+- `200` — connection, key, and database header all good
+- `401` — API key missing or wrong
+- `400` mentioning a database header — this deployment expects `x-lancedb-database`
+
+## Non-REST equivalents
+
+The same credentials work through the SDKs and CLI:
+
+- Python SDK: `lancedb.connect("db://<database>", api_key="<key>", host_override="<base_url>")`
+- TypeScript SDK: `await lancedb.connect("db://<database>", { apiKey: "<key>", hostOverride: "<base_url>" })`
+- `lancedb` CLI: a `[profiles.<name>]` entry in `~/.lancedb/config.toml` with `http_server_url`, `api_key`, `database`

plugins/lancedb/skills/lancedb/references/remote_jobs.md

@@ -0,0 +1,151 @@
+# Job operations over the LanceDB remote server REST API
+
+Jobs are server-side background operations on LanceDB Enterprise/Cloud — index builds,
+column backfills, materialized view refreshes, and similar async work. Endpoints that
+trigger async work (e.g. the column backfill or materialized view refresh endpoints)
+return a `job_id`; these four methods are how you track and manage those jobs.
+
+Resolve the connection first — see `references/remote_connect.md`. All four methods
+are **POST** requests under `{base_url}/v1/jobs/` with JSON bodies, and take the usual
+`x-api-key` / `x-lancedb-database` headers. If every job call returns `501`, job APIs
+are disabled on that deployment (the server has no job registry configured) — report
+that rather than retrying.
+
+## 1. List jobs — `POST /v1/jobs/list`
+
+The body is optional; an empty body lists everything. All fields are filters:
+
+```json
+{
+  "limit": 100,
+  "table_name": "my_table",
+  "job_type": "...",
+  "job_subtype": "...",
+  "state": "...",
+  "page_token": "..."
+}
+```
+
+```bash
+curl -s -X POST "{base_url}/v1/jobs/list" \
+  -H "x-api-key: <key>" -H "x-lancedb-database: <database>" \
+  -H "content-type: application/json" \
+  -d '{"table_name": "my_table"}'
+```
+
+Response:
+
+```json
+{
+  "jobs": [
+    {
+      "job_id": "...",
+      "table": "my_table",
+      "job_type": "...",
+      "job_subtype": "...",
+      "state": "done",
+      "created_at_millis": 1720000000000
+    }
+  ],
+  "page_token": "..."
+}
+```
+
+A `page_token` in the response means there are more results — pass it back in the next
+request to continue. Note list rows use a lowercase `state` string, while describe uses
+an uppercase `job_state`.
+
+## 2. Describe a job — `POST /v1/jobs/describe`
+
+Body: `{"job_id": "<id>"}`. Returns full detail for one job:
+
+```json
+{
+  "job_id": "...",
+  "job_type": "...",
+  "job_subtype": "...",
+  "job_state": "IN_PROGRESS",
+  "creation_ms": 1720000000000,
+  "spec": {},
+  "status": {}
+}
+```
+
+`job_state` is one of `IN_PROGRESS`, `CANCELLED`, `FAILED`, `DONE`. `spec` and `status`
+are job-type-specific JSON objects (the job's input specification and its current
+progress/status). Returns `404` for an unknown job id.
+
+## 3. Cancel a job — `POST /v1/jobs/cancel`
+
+Body: `{"job_id": "<id>"}`; response echoes `{"job_id": "<id>"}`. Cancellation is a
+service-level operation requiring the same administrative authorization as the
+`/admin` routes — a database-scoped API key that can list and describe jobs may still
+get a permission error here. Other errors: `404` unknown job, `409` state conflict
+(e.g. already in a terminal state), `429` too much write contention (safe to retry).
+
+## 4. Query job event history — `POST /v1/jobs/query_events`
+
+Returns the event history (state transitions, progress updates) for one or more jobs.
+Body: `{"job_id": "<id>"}` for one job, or `{"job_ids": ["<id>", ...]}` for a batch.
+Optional fields: `limit` (max event rows), `limit_per_job` (per job in a batch query),
+and `filter` — a SQL-like expression over the columns `state`, `updated_by`,
+`owner_component`, and `claim_entity`. (`full_text_search` is reserved and currently
+rejected as not implemented.)
+
+The response is **not JSON** — it is an Arrow IPC stream
+(`content-type: application/vnd.apache.arrow.stream`). Decode it, e.g. in Python:
+
+```python
+import pyarrow.ipc
+import requests
+
+resp = requests.post(
+    f"{base_url}/v1/jobs/query_events",
+    headers={"x-api-key": key, "x-lancedb-database": database},
+    json={"job_id": job_id},
+)
+resp.raise_for_status()
+events = pyarrow.ipc.open_stream(resp.content).read_all()
+```
+
+## Feature engineering (Geneva) jobs
+
+Feature engineering jobs — UDF column backfills and materialized view refreshes run
+through Geneva — are tracked **separately** from the `/v1/jobs` registry above. Their
+records live in a `geneva_jobs` table inside the database itself (in the `__system`
+namespace), and you access them through a Python `geneva` connection rather than the
+REST endpoints above:
+
+```python
+import geneva
+from geneva.jobs import JobStateManager
+
+# Same credentials as lancedb.connect / the REST API
+conn = geneva.connect("db://<database>", api_key="<key>", host_override="<base_url>")
+jsm = JobStateManager(conn)
+
+# List jobs. NOTE: status defaults to "RUNNING"; pass status=None for all jobs.
+# Statuses: PENDING | RUNNING | DONE | FAILED | CANCELLED
+jobs = jsm.list_jobs(table_name="my_table", status=None)
+
+# Fetch one job by id (returns a list of JobRecord)
+records = jsm.get("<job_id>")
+```
+
+Each `JobRecord` has `table_name`, `column_name`, `job_id`, `job_type`, `status`,
+`launched_at`, `completed_at`, `config`, `launched_by`, `manifest_id`, `cluster_name`,
+`metrics` (progress counters), `events` (human-readable history), and `updated_at`.
+For filters `list_jobs` doesn't support (e.g. time ranges), query the underlying table
+directly: `jsm.get_table(True).search().where("launched_at >= TIMESTAMP '...'")` —
+pass `True` to check out the latest version, since other processes update job state.
+
+Stale-status caveat: nothing reaps dead Geneva jobs, so a job can sit in
+`RUNNING`/`PENDING` forever if its worker died. Treat a job as effectively `FAILED`
+when it has been running longer than ~36 hours, or its `updated_at` is more than ~2
+hours old (this matches the heuristic the Geneva console UI applies on read).
+
+## Workflow tips
+
+- To wait for async work (a backfill, an index build), poll `describe` until
+  `job_state` leaves `IN_PROGRESS`; on `FAILED`, pull `status` and `query_events` for
+  the failure detail.

python/.bumpversion.toml

@@ -1,49 +0,0 @@
-[tool.bumpversion]
-current_version = "0.35.0-beta.2"
-parse = """(?x)
-    (?P<major>0|[1-9]\\d*)\\.
-    (?P<minor>0|[1-9]\\d*)\\.
-    (?P<patch>0|[1-9]\\d*)
-    (?:-(?P<pre_l>[a-zA-Z-]+)\\.(?P<pre_n>0|[1-9]\\d*))?
-"""
-serialize = [
-    "{major}.{minor}.{patch}-{pre_l}.{pre_n}",
-    "{major}.{minor}.{patch}",
-]
-search = "{current_version}"
-replace = "{new_version}"
-regex = false
-ignore_missing_version = false
-ignore_missing_files = false
-tag = true
-sign_tags = false
-tag_name = "python-v{new_version}"
-tag_message = "Bump version: {current_version} → {new_version}"
-allow_dirty = true
-commit = true
-message = "Bump version: {current_version} → {new_version}"
-commit_args = ""
-# bump-my-version >=1.4.0 rejects pre_commit_hooks containing shell syntax unless opted in.
-allow_shell_hooks = true
-
-# Update Cargo.lock after version bump
-pre_commit_hooks = [
-  """
-    cd python && cargo update -p lancedb-python
-    if git diff --quiet ../Cargo.lock; then
-        echo "Cargo.lock unchanged"
-    else
-        git add ../Cargo.lock
-        echo "Updated and staged Cargo.lock"
-    fi
-    """,
-]
-
-[tool.bumpversion.parts.pre_l]
-values = ["beta", "final"]
-optional_value = "final"
-
-[[tool.bumpversion.files]]
-filename = "Cargo.toml"
-search = "\nversion = \"{current_version}\""
-replace = "\nversion = \"{new_version}\""

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.35.0-beta.2"
+version = "0.37.1-beta.0"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"

python/README.md

@@ -8,6 +8,27 @@ A Python library for [LanceDB](https://github.com/lancedb/lancedb).
 pip install lancedb
 ```
 
+### Pre-Haswell x86_64 hosts: `lancedb-compat`
+
+The default `lancedb` wheel targets `x86-64-haswell` (AVX2 + FMA + F16C) for full performance on modern hardware. Pre-Haswell hosts — Intel Sandy Bridge / Ivy Bridge / Westmere; AMD Bulldozer / Piledriver / Steamroller — don't have AVX2 and crash with `Illegal instruction` at `import lancedb`.
+
+For those hosts, install the `lancedb-compat` package instead:
+
+```bash
+pip install lancedb-compat
+```
+
+Same Python API (`import lancedb` works as usual). The compat wheel is compiled at the `x86-64-v2` baseline (Nehalem-class) and uses runtime SIMD dispatch in the embedded lance crate to pick the right kernel tier (scalar / AVX / AVX+FMA / AVX2+FMA / AVX-512) at load time, so it still goes fast on modern hardware while running cleanly on the pre-Haswell silicon. Use `lance.simd_info()` from Python to verify which tier was selected.
+
+`lancedb` and `lancedb-compat` install to the same `lancedb/` namespace and conflict at install time. Pick one. To switch, `pip uninstall lancedb` first, then `pip install lancedb-compat` (or vice-versa).
+
+If you need a custom baseline (or `lancedb-compat` isn't yet published for your platform), build from source with the override:
+
+```bash
+RUSTFLAGS="-C target-cpu=x86-64-v2" maturin build --release
+pip install ./target/wheels/lancedb-*.whl
+```
+
 ### Preview Releases
 
 Stable releases are created about every 2 weeks. For the latest features and bug fixes, you can install the preview release. These releases receive the same level of testing as stable releases, but are not guaranteed to be available for more than 6 months after they are released. Once your application is stable, we recommend switching to stable releases.

python/python/lancedb/__init__.py

@@ -258,6 +258,7 @@ def tokenize(
     lower_case: bool = True,
     stem: bool = True,
     remove_stop_words: bool = True,
+    custom_stop_words: Optional[List[str]] = None,
     ascii_folding: bool = True,
     ngram_min_length: int = 3,
     ngram_max_length: int = 3,
@@ -265,9 +266,10 @@ def tokenize(
 ) -> Iterable[FtsToken]:
     """Tokenize a full-text search query using an explicit tokenizer.
 
-    This does not require a table or FTS index. The tokenizer options match
+    This does not require an FTS index. The tokenizer options match
-    :class:`lancedb.index.FTS`.
+    :class:`lancedb.index.FTS`. ``custom_stop_words`` accepts a list of strings.
     """
+
     return _tokenize(
         query,
         base_tokenizer=base_tokenizer,
@@ -276,6 +278,7 @@ def tokenize(
         lower_case=lower_case,
         stem=stem,
         remove_stop_words=remove_stop_words,
+        custom_stop_words=custom_stop_words,
         ascii_folding=ascii_folding,
         ngram_min_length=ngram_min_length,
         ngram_max_length=ngram_max_length,

python/python/lancedb/_lancedb.pyi

@@ -59,6 +59,7 @@ def tokenize(
     lower_case: bool = True,
     stem: bool = True,
     remove_stop_words: bool = True,
+    custom_stop_words: Optional[List[str]] = None,
     ascii_folding: bool = True,
     ngram_min_length: int = 3,
     ngram_max_length: int = 3,
@@ -219,6 +220,7 @@ class Table:
         data: pa.RecordBatchReader,
         mode: Literal["append", "overwrite"],
         progress: Optional[Any] = None,
+        write_parallelism: Optional[int] = None,
     ) -> AddResult: ...
     async def update(
         self, updates: Dict[str, str], where: Optional[str]
@@ -296,6 +298,11 @@ class Table:
     async def fetch_blobs(
         self, column: str, row_ids: list[int]
     ) -> pa.LargeBinaryArray: ...
+    async def fetch_blob_ranges(
+        self,
+        column: str,
+        requests: List[Tuple[int, int, int]],
+    ) -> pa.LargeBinaryArray: ...
     async def fetch_blob_files(
         self, column: str, row_ids: list[int]
     ) -> list[Optional[BlobFile]]: ...
@@ -318,6 +325,10 @@ class Branches:
     ) -> Table: ...
     async def checkout(self, name: str, version: Optional[int] = None) -> Table: ...
     async def delete(self, name: str) -> None: ...
+    async def diff(self, from_branch: str) -> Dict[str, Any]: ...
+    async def merge(
+        self, from_branch: str, dry_run: bool = False
+    ) -> Dict[str, Any]: ...
 
 class IndexConfig:
     name: str
@@ -386,6 +397,7 @@ class Query:
     def fast_search(self): ...
     def with_row_id(self): ...
     def postfilter(self): ...
+    def use_lsm(self, enable: bool): ...
     def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ...
     def nearest_to_text(self, query: dict) -> FTSQuery: ...
     def order_by(self, ordering: Optional[List[ColumnOrdering]]): ...
@@ -402,6 +414,7 @@ class Query:
 class TakeQuery:
     def select(self, columns: List[str]): ...
     def with_row_id(self): ...
+    def use_lsm(self, enable: bool): ...
     async def output_schema(self) -> pa.Schema: ...
     async def execute(self) -> RecordBatchStream: ...
     async def explain_plan(self, verbose: Optional[bool]) -> str: ...
@@ -420,6 +433,7 @@ class FTSQuery:
     def fast_search(self): ...
     def with_row_id(self): ...
     def postfilter(self): ...
+    def use_lsm(self, enable: bool): ...
     def get_query(self) -> str: ...
     def add_query_vector(self, query_vec: pa.Array) -> None: ...
     def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ...
@@ -447,6 +461,7 @@ class VectorQuery:
     def column(self, column: str): ...
     def distance_type(self, distance_type: str): ...
     def postfilter(self): ...
+    def use_lsm(self, enable: bool): ...
     def refine_factor(self, refine_factor: int): ...
     def nprobes(self, nprobes: int): ...
     def minimum_nprobes(self, minimum_nprobes: int): ...
@@ -470,6 +485,7 @@ class HybridQuery:
     def fast_search(self): ...
     def with_row_id(self): ...
     def postfilter(self): ...
+    def use_lsm(self, enable: bool): ...
     def distance_type(self, distance_type: str): ...
     def refine_factor(self, refine_factor: int): ...
     def nprobes(self, nprobes: int): ...
@@ -494,6 +510,7 @@ class PyQueryRequest:
     select: Optional[Union[str, List[str]]]
     fast_search: Optional[bool]
     with_row_id: Optional[bool]
+    use_lsm: Optional[bool]
     column: Optional[str]
     query_vector: Optional[List[pa.Array]]
     minimum_nprobes: Optional[int]

python/python/lancedb/db.py

@@ -41,6 +41,7 @@ from lance_namespace import (
     ListTablesResponse,
     connect as namespace_connect,
 )
+from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
 
 from . import __version__
 from ._lancedb import connect as lancedb_connect  # type: ignore
@@ -358,7 +359,7 @@ class DBConnection(EnforceOverrides):
 
         Data is converted to Arrow before being written to disk. For maximum
         control over how data is saved, either provide the PyArrow schema to
-        convert to or else provide a [PyArrow Table](pyarrow.Table) directly.
+        convert to or else provide a [PyArrow Table][pyarrow.Table] directly.
 
         >>> import pyarrow as pa
         >>> custom_schema = pa.schema([
@@ -746,11 +747,13 @@ class LanceDBConnection(DBConnection):
         """
         if namespace_path is None:
             namespace_path = []
-        return self._namespace_conn().list_namespaces(
+        return LOOP.run(
+            self._conn.list_namespaces(
                 namespace_path=namespace_path,
                 page_token=page_token,
                 limit=limit,
             )
+        )
 
     @override
     def create_namespace(
@@ -759,11 +762,13 @@ class LanceDBConnection(DBConnection):
         mode: Optional[str] = None,
         properties: Optional[Dict[str, str]] = None,
     ) -> CreateNamespaceResponse:
-        return self._namespace_conn().create_namespace(
+        return LOOP.run(
+            self._conn.create_namespace(
                 namespace_path=namespace_path,
                 mode=mode,
                 properties=properties,
             )
+        )
 
     @override
     def drop_namespace(
@@ -772,19 +777,24 @@ class LanceDBConnection(DBConnection):
         mode: Optional[str] = None,
         behavior: Optional[str] = None,
     ) -> DropNamespaceResponse:
-        return self._namespace_conn().drop_namespace(
+        try:
+            return LOOP.run(
+                self._conn.drop_namespace(
                     namespace_path=namespace_path,
                     mode=mode,
                     behavior=behavior,
                 )
+            )
+        except RuntimeError as e:
+            if "Namespace not empty" in str(e):
+                raise NamespaceNotEmptyError(str(e)) from e
+            raise
 
     @override
     def describe_namespace(
         self, namespace_path: List[str]
     ) -> DescribeNamespaceResponse:
-        return self._namespace_conn().describe_namespace(
+        return LOOP.run(self._conn.describe_namespace(namespace_path=namespace_path))
-            namespace_path=namespace_path,
-        )
 
     @override
     def list_tables(
@@ -813,12 +823,6 @@ class LanceDBConnection(DBConnection):
         """
         if namespace_path is None:
             namespace_path = []
-        if namespace_path:
-            return self._namespace_conn().list_tables(
-                namespace_path=namespace_path,
-                page_token=page_token,
-                limit=limit,
-            )
         return LOOP.run(
             self._conn.list_tables(
                 namespace_path=namespace_path, page_token=page_token, limit=limit
@@ -916,22 +920,6 @@ class LanceDBConnection(DBConnection):
             raise ValueError("mode must be either 'create' or 'overwrite'")
         validate_table_name(name)
 
-        if namespace_path:
-            return self._namespace_conn().create_table(
-                name,
-                data=data,
-                schema=schema,
-                mode=mode,
-                exist_ok=exist_ok,
-                on_bad_vectors=on_bad_vectors,
-                fill_value=fill_value,
-                embedding_functions=embedding_functions,
-                namespace_path=namespace_path,
-                storage_options=storage_options,
-                data_storage_version=data_storage_version,
-                enable_v2_manifest_paths=enable_v2_manifest_paths,
-            )
-
         tbl = LanceTable.create(
             self,
             name,
@@ -944,22 +932,11 @@ class LanceDBConnection(DBConnection):
             embedding_functions=embedding_functions,
             namespace_path=namespace_path,
             storage_options=storage_options,
+            data_storage_version=data_storage_version,
+            enable_v2_manifest_paths=enable_v2_manifest_paths,
         )
         return tbl
 
-    def _namespace_conn(self) -> DBConnection:
-        """Return a LanceNamespaceDBConnection backed by this connection's
-        directory namespace.  Used to delegate child-namespace operations."""
-        from lancedb.namespace import LanceNamespaceDBConnection
-
-        return LanceNamespaceDBConnection(
-            self.namespace_client(),
-            read_consistency_interval=self.read_consistency_interval,
-            storage_options=self.storage_options,
-            namespace_client_impl=None,
-            namespace_client_properties=None,
-        )
-
     @override
     def open_table(
         self,
@@ -1006,14 +983,7 @@ class LanceDBConnection(DBConnection):
                 stacklevel=2,
             )
 
-        if namespace_path:
+        try:
-            tbl = self._namespace_conn().open_table(
-                name,
-                namespace_path=namespace_path,
-                storage_options=storage_options,
-                index_cache_size=index_cache_size,
-            )
-        else:
             tbl = LanceTable.open(
                 self,
                 name,
@@ -1021,6 +991,15 @@ class LanceDBConnection(DBConnection):
                 storage_options=storage_options,
                 index_cache_size=index_cache_size,
             )
+        except (RuntimeError, ValueError) as e:
+            if namespace_path and (
+                "Table not found" in str(e) or "was not found" in str(e)
+            ):
+                table_id = namespace_path + [name]
+                raise TableNotFoundError(
+                    f"Table not found: {'$'.join(table_id)}"
+                ) from e
+            raise
 
         if branch is not None:
             tbl = tbl.branches.checkout(branch, version)
@@ -1104,9 +1083,6 @@ class LanceDBConnection(DBConnection):
         """
         if namespace_path is None:
             namespace_path = []
-        if namespace_path:
-            self._namespace_conn().drop_table(name, namespace_path=namespace_path)
-            return
         LOOP.run(
             self._conn.drop_table(
                 name, namespace_path=namespace_path, ignore_missing=ignore_missing
@@ -1553,7 +1529,7 @@ class AsyncConnection(object):
 
         Data is converted to Arrow before being written to disk. For maximum
         control over how data is saved, either provide the PyArrow schema to
-        convert to or else provide a [PyArrow Table](pyarrow.Table) directly.
+        convert to or else provide a [PyArrow Table][pyarrow.Table] directly.
 
         >>> import pyarrow as pa
         >>> custom_schema = pa.schema([

python/python/lancedb/embeddings/__init__.py

@@ -21,3 +21,32 @@ from .watsonx import WatsonxEmbeddings
 from .voyageai import VoyageAIEmbeddingFunction
 from .colpali import ColPaliEmbeddings
 from .siglip import SigLipEmbeddings
+
+# The API reference renders this package with a single mkdocstrings directive,
+# which only picks up names listed here. New embedding functions must be added
+# to both the imports above and this list, or they will silently go undocumented.
+__all__ = [
+    "EmbeddingFunction",
+    "EmbeddingFunctionConfig",
+    "TextEmbeddingFunction",
+    "EmbeddingFunctionRegistry",
+    "get_registry",
+    "register",
+    "SentenceTransformerEmbeddings",
+    "OpenAIEmbeddings",
+    "OpenClipEmbeddings",
+    "BedRockText",
+    "CohereEmbeddingFunction",
+    "GeminiText",
+    "GteEmbeddings",
+    "InstructorEmbeddingFunction",
+    "JinaEmbeddings",
+    "OllamaEmbeddings",
+    "TransformersEmbeddingFunction",
+    "ColbertEmbeddings",
+    "VoyageAIEmbeddingFunction",
+    "WatsonxEmbeddings",
+    "ColPaliEmbeddings",
+    "ImageBindEmbeddings",
+    "SigLipEmbeddings",
+]

python/python/lancedb/embeddings/cohere.py

@@ -39,6 +39,8 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
     query_input_type : str, default "search_query"
         The input type for the query column in the database
 
+    Notes
+    -----
     Cohere supports following input types:
 
     | Input Type               | Description                          |

python/python/lancedb/embeddings/watsonx.py

@@ -14,29 +14,76 @@ import numpy as np
 
 DEFAULT_WATSONX_URL = "https://us-south.ml.cloud.ibm.com"
 
-MODELS_DIMS = {
+# Models currently available on the watsonx.ai SaaS platform.
+# These are the IDs advertised to new users via model_names() and shown in
+# validation error messages.  Regional availability and withdrawal dates are
+# documented at:
+# https://www.ibm.com/docs/en/watsonx/saas?topic=models-supported-encoder
+CURRENT_MODELS: dict[str, int] = {
+    "ibm/granite-embedding-278m-multilingual": 768,
+    "ibm/slate-125m-english-rtrvr-v2": 768,
+    "ibm/slate-30m-english-rtrvr-v2": 384,
+    "intfloat/multilingual-e5-large": 1024,
+}
+
+# Full dimension map including legacy model IDs from earlier releases.
+# Kept so that existing tables whose stored metadata uses these names can still
+# resolve dimensions on load without raising an error.  These IDs are NOT
+# advertised to new users.
+MODELS_DIMS: dict[str, int] = {
+    **CURRENT_MODELS,
+    # Deprecated — withdrawal announced but still functional until the dates above.
+    "sentence-transformers/all-minilm-l6-v2": 384,
+    # Pre-v2 legacy names retained for metadata compatibility only.
     "ibm/slate-125m-english-rtrvr": 768,
     "ibm/slate-30m-english-rtrvr": 384,
     "sentence-transformers/all-minilm-l12-v2": 384,
-    "intfloat/multilingual-e5-large": 1024,
 }
 
 
 @register("watsonx")
 class WatsonxEmbeddings(TextEmbeddingFunction):
     """
+    An embedding function that uses the IBM watsonx.ai Embeddings API.
+
     API Docs:
-    ---------
         https://cloud.ibm.com/apidocs/watsonx-ai#text-embeddings
 
     Supported embedding models:
-    ---------------------------
         https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx
+
+    Parameters
+    ----------
+    name : str, default "ibm/slate-125m-english-rtrvr"
+        The ID of the embedding model to use.  For new tables,
+        ``"ibm/granite-embedding-278m-multilingual"`` is recommended.
+    api_key : str, optional
+        IBM Cloud API key.  Falls back to the ``WATSONX_API_KEY`` environment
+        variable when not provided.
+    project_id : str, optional
+        watsonx.ai project ID.  Explicit value takes precedence over the
+        ``WATSONX_PROJECT_ID`` environment variable.  Mutually exclusive with
+        ``space_id`` — exactly one must be supplied.
+    space_id : str, optional
+        watsonx.ai deployment space ID.  Explicit value takes precedence over
+        the ``WATSONX_SPACE_ID`` environment variable.  Mutually exclusive with
+        ``project_id`` — exactly one must be supplied.
+    url : str, optional
+        watsonx.ai service URL.  Defaults to
+        ``"https://us-south.ml.cloud.ibm.com"``.
+    params : dict, optional
+        Extra parameters forwarded verbatim to ``Embeddings`` (e.g.
+        ``{"truncate_input_tokens": 512}``).
     """
 
+    # Intentionally kept at the original pre-PR default so that existing tables
+    # whose stored metadata contains model:{} reload with the same model they
+    # were created with.  New users should pass name= explicitly, e.g.
+    # name="ibm/granite-embedding-278m-multilingual".
     name: str = "ibm/slate-125m-english-rtrvr"
     api_key: Optional[str] = None
     project_id: Optional[str] = None
+    space_id: Optional[str] = None
     url: Optional[str] = None
     params: Optional[Dict] = None
 
@@ -46,12 +93,13 @@ class WatsonxEmbeddings(TextEmbeddingFunction):
 
     @staticmethod
     def model_names():
-        return [
+        """Return the IDs of models currently available for new tables.
-            "ibm/slate-125m-english-rtrvr",
+
-            "ibm/slate-30m-english-rtrvr",
+        Legacy / deprecated IDs are intentionally excluded.  They remain
-            "sentence-transformers/all-minilm-l12-v2",
+        resolvable for dimension lookups on existing tables via ``MODELS_DIMS``,
-            "intfloat/multilingual-e5-large",
+        but should not be used when creating new tables.
-        ]
+        """
+        return list(CURRENT_MODELS.keys())
 
     def ndims(self):
         return self._ndims
@@ -59,7 +107,10 @@ class WatsonxEmbeddings(TextEmbeddingFunction):
     @cached_property
     def _ndims(self):
         if self.name not in MODELS_DIMS:
-            raise ValueError(f"Unknown model name {self.name}")
+            raise ValueError(
+                f"Unknown model '{self.name}'. "
+                f"Available models: {list(CURRENT_MODELS.keys())}"
+            )
         return MODELS_DIMS[self.name]
 
     def generate_embeddings(
@@ -81,27 +132,45 @@ class WatsonxEmbeddings(TextEmbeddingFunction):
             "ibm_watsonx_ai.foundation_models"
         )
 
-        kwargs = {"model_id": self.name}
+        # --- credentials ---
+        # Explicit field takes priority; env var is the fallback.
+        api_key = self.api_key or os.environ.get("WATSONX_API_KEY")
+        if not api_key:
+            raise ValueError(
+                "WATSONX_API_KEY not set. Either set it in your environment or "
+                "pass it as `api_key` argument to WatsonxEmbeddings."
+            )
+        credentials = ibm_watsonx_ai.Credentials(
+            api_key=api_key,
+            url=self.url or DEFAULT_WATSONX_URL,
+        )
+
+        # --- project_id / space_id (exactly one required) ---
+        # Explicit field always wins; env var is consulted only when the
+        # corresponding field was not set, so passing project_id= never
+        # conflicts with a stray WATSONX_SPACE_ID env var and vice-versa.
+        space_id, project_id = self.space_id, self.project_id
+
+        if project_id is None and space_id is None:
+            # Neither was passed explicitly — fall back to env vars.
+            project_id = os.environ.get("WATSONX_PROJECT_ID")
+            space_id = os.environ.get("WATSONX_SPACE_ID")
+
+        if project_id and space_id:
+            raise ValueError("Provide either `project_id` or `space_id`, not both.")
+        if not project_id and not space_id:
+            raise ValueError(
+                "Either WATSONX_PROJECT_ID or WATSONX_SPACE_ID must be set. "
+                "Pass one as an argument to WatsonxEmbeddings or set the "
+                "corresponding environment variable."
+            )
+
+        client_kwargs: Dict = dict(model_id=self.name, credentials=credentials)
         if self.params:
-            kwargs["params"] = self.params
+            client_kwargs["params"] = self.params
-        if self.project_id:
+        if project_id:
-            kwargs["project_id"] = self.project_id
+            client_kwargs["project_id"] = project_id
-        elif "WATSONX_PROJECT_ID" in os.environ:
-            kwargs["project_id"] = os.environ["WATSONX_PROJECT_ID"]
         else:
-            raise ValueError("WATSONX_PROJECT_ID must be set or passed")
+            client_kwargs["space_id"] = space_id
 
-        creds_kwargs = {}
+        return ibm_watsonx_ai_foundation_models.Embeddings(**client_kwargs)
-        if self.api_key:
-            creds_kwargs["api_key"] = self.api_key
-        elif "WATSONX_API_KEY" in os.environ:
-            creds_kwargs["api_key"] = os.environ["WATSONX_API_KEY"]
-        else:
-            raise ValueError("WATSONX_API_KEY must be set or passed")
-        if self.url:
-            creds_kwargs["url"] = self.url
-        else:
-            creds_kwargs["url"] = DEFAULT_WATSONX_URL
-        kwargs["credentials"] = ibm_watsonx_ai.Credentials(**creds_kwargs)
-
-        return ibm_watsonx_ai_foundation_models.Embeddings(**kwargs)

python/python/lancedb/index.py

@@ -2,7 +2,7 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 from dataclasses import dataclass
-from typing import Literal, Optional
+from typing import List, Literal, Optional
 
 from ._lancedb import (
     IndexConfig,
@@ -115,6 +115,12 @@ class FTS:
 
     For example, it works with `title`, `description`, `content`, etc.
 
+    Examples
+    --------
+    Create an index configuration that uses 256-document posting blocks:
+
+    >>> config = FTS(block_size=256)
+
     Attributes
     ----------
     with_position : bool, default False
@@ -145,9 +151,18 @@ class FTS:
     remove_stop_words : bool, default True
         Whether to remove stop words. Stop words are common words that are often
         removed from text before indexing. For example, in English "the" and "and".
+    custom_stop_words : list of str, optional
+        Custom words replace the built-in language stop words
+        and only take effect when ``remove_stop_words`` is True. ``None`` uses
+        the built-in language list, while an empty list explicitly uses no
+        stop words.
     ascii_folding : bool, default True
         Whether to fold ASCII characters. This converts accented characters to
         their ASCII equivalent. For example, "café" would be converted to "cafe".
+    block_size : int, default 128
+        The number of documents per compressed posting block. Supported values
+        are 128 and 256. A value of 256 uses the experimental FTS V3 format
+        and may introduce breaking changes.
 
     Notes
     -----
@@ -168,6 +183,8 @@ class FTS:
     ngram_min_length: int = 3
     ngram_max_length: int = 3
     prefix_only: bool = False
+    block_size: int = 128
+    custom_stop_words: Optional[List[str]] = None
 
 
 @dataclass
@@ -202,7 +219,7 @@ class HnswPq:
         distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
         l2 norm is 1), then dot distance is equivalent to the cosine distance.
 
-    num_partitions, default sqrt(num_rows)
+    num_partitions: int, default sqrt(num_rows)
 
         The number of IVF partitions to create.
 
@@ -211,7 +228,7 @@ class HnswPq:
         will require too much memory. Each partition becomes its own HNSW graph, so
         setting this value higher reduces the peak memory use of training.
 
-    num_sub_vectors, default is vector dimension / 16
+    num_sub_vectors: int, default is vector dimension / 16
 
         Number of sub-vectors of PQ.
 
@@ -233,7 +250,7 @@ class HnswPq:
         This value controls how much the sub-vectors are compressed.  The more bits
         the more accurate the index but the slower search. Only 4 and 8 are supported.
 
-    max_iterations, default 50
+    max_iterations: int, default 50
 
         Max iterations to train kmeans.
 
@@ -246,7 +263,7 @@ class HnswPq:
         those cases it is unlikely that setting this larger will lead to the index
         converging anyways.
 
-    sample_rate, default 256
+    sample_rate: int, default 256
 
         The rate used to calculate the number of training vectors for kmeans.
 
@@ -262,14 +279,14 @@ class HnswPq:
         Increasing this value might improve the quality of the index but in
         most cases the default should be sufficient.
 
-    m, default 20
+    m: int, default 20
 
         The number of neighbors to select for each vector in the HNSW graph.
 
         This value controls the tradeoff between search speed and accuracy.
         The higher the value the more accurate the search but the slower it will be.
 
-    ef_construction, default 300
+    ef_construction: int, default 300
 
         The number of candidates to evaluate during the construction of the HNSW graph.
 
@@ -280,7 +297,7 @@ class HnswPq:
         This value should be set to a value that is not less than `ef` in the
         search phase.
 
-    target_partition_size, default is 1,048,576
+    target_partition_size: int, default is 1,048,576
 
         The target size of each partition.
 
@@ -334,7 +351,7 @@ class HnswSq:
         distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
         l2 norm is 1), then dot distance is equivalent to the cosine distance.
 
-    num_partitions, default sqrt(num_rows)
+    num_partitions: int, default sqrt(num_rows)
 
         The number of IVF partitions to create.
 
@@ -343,7 +360,7 @@ class HnswSq:
         will require too much memory. Each partition becomes its own HNSW graph, so
         setting this value higher reduces the peak memory use of training.
 
-    max_iterations, default 50
+    max_iterations: int, default 50
 
         Max iterations to train kmeans.
 
@@ -356,7 +373,7 @@ class HnswSq:
         In those cases it is unlikely that setting this larger will lead to
         the index converging anyways.
 
-    sample_rate, default 256
+    sample_rate: int, default 256
 
         The rate used to calculate the number of training vectors for kmeans.
 
@@ -372,14 +389,14 @@ class HnswSq:
         Increasing this value might improve the quality of the index but in
         most cases the default should be sufficient.
 
-    m, default 20
+    m: int, default 20
 
         The number of neighbors to select for each vector in the HNSW graph.
 
         This value controls the tradeoff between search speed and accuracy.
         The higher the value the more accurate the search but the slower it will be.
 
-    ef_construction, default 300
+    ef_construction: int, default 300
 
         The number of candidates to evaluate during the construction of the HNSW graph.
 
@@ -390,7 +407,7 @@ class HnswSq:
         This value should be set to a value that is not less than `ef` in the search
         phase.
 
-    target_partition_size, default is 1,048,576
+    target_partition_size: int, default is 1,048,576
 
         The target size of each partition.
 
@@ -443,7 +460,7 @@ class HnswFlat:
         distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
         l2 norm is 1), then dot distance is equivalent to the cosine distance.
 
-    num_partitions, default sqrt(num_rows)
+    num_partitions: int, default sqrt(num_rows)
 
         The number of IVF partitions to create.
 
@@ -453,18 +470,18 @@ class HnswFlat:
         graph, so setting this value higher reduces the peak memory use of
         training.
 
-    max_iterations, default 50
+    max_iterations: int, default 50
 
         Max iterations to train kmeans.
 
         When training an IVF index we use kmeans to calculate the partitions.
         This parameter controls how many iterations of kmeans to run.
 
-    sample_rate, default 256
+    sample_rate: int, default 256
 
         The rate used to calculate the number of training vectors for kmeans.
 
-    m, default 20
+    m: int, default 20
 
         The number of neighbors to select for each vector in the HNSW graph.
 
@@ -472,7 +489,7 @@ class HnswFlat:
         The higher the value the more accurate the search but the slower it
         will be.
 
-    ef_construction, default 300
+    ef_construction: int, default 300
 
         The number of candidates to evaluate during the construction of the HNSW
         graph.
@@ -484,7 +501,7 @@ class HnswFlat:
         than 500.  This value should be set to a value that is not less than `ef`
         in the search phase.
 
-    target_partition_size, default is 1,048,576
+    target_partition_size: int, default is 1,048,576
 
         The target size of each partition.
     """
@@ -588,7 +605,7 @@ class IvfFlat:
 
         The default value is 256.
 
-    target_partition_size, default is 8192
+    target_partition_size: int, default is 8192
 
         The target size of each partition.
 
@@ -752,7 +769,7 @@ class IvfPq:
 
         The default value is 256.
 
-    target_partition_size, default is 8192
+    target_partition_size: int, default is 8192
 
         The target size of each partition.
 
@@ -813,7 +830,7 @@ class IvfRq:
     sample_rate: int, default 256
         Controls the number of training vectors: sample_rate * num_partitions.
 
-    target_partition_size, default is 8192
+    target_partition_size: int, default is 8192
         Target size of each partition.
     """
 
@@ -828,6 +845,9 @@ class IvfRq:
     accelerator: Optional[str] = None
 
 
+# The API reference renders this module with a single mkdocstrings directive,
+# which only picks up names listed here. New public names must be added to this
+# list, or they will silently go undocumented.
 __all__ = [
     "BTree",
     "IvfPq",

python/python/lancedb/merge.py

@@ -37,7 +37,7 @@ class LanceMergeInsertBuilder(object):
         self._when_not_matched_by_source_condition_expr = None
         self._timeout = None
         self._use_index = True
-        self._use_lsm_write = None
+        self._use_lsm = None
         self._validate_single_shard = None
 
     def when_matched_update_all(
@@ -113,22 +113,22 @@ class LanceMergeInsertBuilder(object):
         self._use_index = use_index
         return self
 
-    def use_lsm_write(self, use_lsm_write: bool) -> LanceMergeInsertBuilder:
+    def use_lsm(self, enable: bool) -> LanceMergeInsertBuilder:
         """
-        Controls whether the merge uses the MemWAL LSM write path.
+        Control MemWAL routing for this merge.
 
-        By default (unset), a `merge_insert` on a table with an LSM write spec
+        By default (unset), a `merge_insert` on a table with an LSM write spec is
-        is routed through Lance's MemWAL shard writer, and a table without one
+        routed through Lance's MemWAL shard writer, and a table without one uses
-        uses the standard path. Pass `False` to force the standard path even
+        the standard path.
-        when a spec is set. Pass `True` to require a spec — `merge_insert`
-        raises an error if none is installed.
 
         Parameters
         ----------
-        use_lsm_write: bool
+        enable: bool
-            Whether to use the LSM write path.
+            ``True`` forces MemWAL routing and errors if the table has no LSM
+            write spec. ``False`` forces the standard write path even when a spec
+            is set.
         """
-        self._use_lsm_write = use_lsm_write
+        self._use_lsm = enable
         return self
 
     def validate_single_shard(

python/python/lancedb/permutation.py

@@ -438,7 +438,8 @@ class Permutation:
         _reader: Optional[PermutationReader] = None,
     ):
         """
-        Internal constructor.  Use [from_tables](#from_tables) instead.
+        Internal constructor.  Use
+        [from_tables][lancedb.permutation.Permutation.from_tables] instead.
         """
         assert base_table is not None, "base_table is required"
         assert selection is not None, "selection is required"
@@ -885,7 +886,7 @@ class Permutation:
         This method refines the current selection, potentially removing columns.  It
         will not add back columns that were previously removed.
 
-        If any of the columns do not exist then an error will be raised
+        If any of the columns do not exist then an error will be raised.
 
         This does not introduce a post-processing step.  It simply reduces the amount
         of data we read.
@@ -898,6 +899,11 @@ class Permutation:
         for name in columns:
             value = self.selection.get(name, None)
             if value is None:
+                if name == "_rowid":
+                    # _rowid is a system column not in the default schema
+                    # but can be explicitly selected
+                    value = "_rowid"
+                else:
                     raise ValueError(
                         f"Cannot select column `{name}` because it does not exist"
                     )
@@ -980,8 +986,9 @@ class Permutation:
         types.  Conversion of strings, lists, and structs will require creating python
         objects and this is not zero-copy.
 
-        For custom formatting, use [with_transform](#with_transform) which overrides
+        For custom formatting, use
-        this method.
+        [with_transform][lancedb.permutation.Permutation.with_transform] which
+        overrides this method.
         """
         assert format is not None, "format is required"
         if format == "python":
@@ -1056,7 +1063,8 @@ class Permutation:
         Note: this method returns a new permutation and does not modify `self`
         It is provided for compatibility with the huggingface Dataset API.
 
-        Use [with_skip](#with_skip) instead to avoid confusion.
+        Use [with_skip][lancedb.permutation.Permutation.with_skip] instead to
+        avoid confusion.
         """
         return self.with_skip(skip)
 
@@ -1079,7 +1087,8 @@ class Permutation:
         Note: this method returns a new permutation and does not modify `self`
         It is provided for compatibility with the huggingface Dataset API.
 
-        Use [with_take](#with_take) instead to avoid confusion.
+        Use [with_take][lancedb.permutation.Permutation.with_take] instead to
+        avoid confusion.
         """
         return self.with_take(limit)
 
@@ -1102,7 +1111,8 @@ class Permutation:
         Note: this method returns a new permutation and does not modify `self`
         It is provided for compatibility with the huggingface Dataset API.
 
-        Use [with_repeat](#with_repeat) instead to avoid confusion.
+        Use [with_repeat][lancedb.permutation.Permutation.with_repeat] instead
+        to avoid confusion.
         """
         return self.with_repeat(times)
 

python/python/lancedb/query.py

@@ -651,7 +651,8 @@ class Query(pydantic.BaseModel):
     distance_type : Optional[str]
         the distance type to use for vector search
 
-        This can be l2 (default), cosine and dot.  See [metric definitions][search] for
+        This can be l2 (default), cosine and dot.  See
+        [metric definitions](https://lancedb.com/docs/search/vector-search/) for
         more details.
 
         If this is not a vector search this will be None.
@@ -664,8 +665,9 @@ class Query(pydantic.BaseModel):
 
         - A higher number makes search more accurate but also slower.
 
-        - See discussion in [Querying an ANN Index][querying-an-ann-index] for
+        - See discussion in
-          tuning advice.
+          [Querying an ANN Index](https://lancedb.com/docs/indexing/)
+          for tuning advice.
 
         Will be None if this is not a vector search.
     refine_factor : Optional[int]
@@ -673,8 +675,9 @@ class Query(pydantic.BaseModel):
 
         - A higher number makes search more accurate but also slower.
 
-        - See discussion in [Querying an ANN Index][querying-an-ann-index] for
+        - See discussion in
-          tuning advice.
+          [Querying an ANN Index](https://lancedb.com/docs/indexing/)
+          for tuning advice.
 
         Will be None if this is not a vector search.
     lower_bound : Optional[float]
@@ -778,6 +781,11 @@ class Query(pydantic.BaseModel):
     # if true, will only search the indexed data
     fast_search: Optional[bool] = None
 
+    # MemWAL LSM read routing: None auto-routes when the table carries a write
+    # spec, True forces the LSM scanner (errors without a spec), False reads the
+    # base table only
+    use_lsm: Optional[bool] = None
+
     # size of the nearest neighbor list maintained during HNSW search
     ef: Optional[int] = None
 
@@ -795,6 +803,9 @@ class Query(pydantic.BaseModel):
         query.full_text_query = req.full_text_search
         query.columns = req.select
         query.with_row_id = req.with_row_id
+        # use_lsm is a genuine tri-state (None / True / False); preserve it as-is
+        # so a round-tripped query keeps an explicit False.
+        query.use_lsm = req.use_lsm
         query.vector_column = req.column
         query.vector = req.query_vector
         query.distance_type = req.distance_type
@@ -967,6 +978,7 @@ class LanceQueryBuilder(ABC):
         self._with_row_address = None
         self._fragments = None
         self._fragment_ids = None
+        self._use_lsm = None
         self._vector = None
         self._text = None
         self._ef = None
@@ -1326,6 +1338,30 @@ class LanceQueryBuilder(ABC):
         self._fragment_ids = fragment_ids
         return self
 
+    def use_lsm(self, enable: bool) -> Self:
+        """Control MemWAL LSM read routing for this query.
+
+        By default (unset), a query against a table with an LSM write spec is
+        routed through the LSM scanner so it also returns data written via the
+        ``merge_insert`` LSM path that has not yet been compacted into the base
+        table (active/frozen memtables + flushed generations); a table without a
+        spec reads the base table.
+
+        Parameters
+        ----------
+        enable : bool
+            ``True`` forces the LSM scanner and errors if the table has no LSM
+            write spec. ``False`` bypasses the MemWAL and reads the base table
+            only, even when a spec is present.
+
+        Returns
+        -------
+        LanceQueryBuilder
+            The LanceQueryBuilder object.
+        """
+        self._use_lsm = enable
+        return self
+
     def explain_plan(self, verbose: Optional[bool] = False) -> str:
         """Return the execution plan for this query.
 
@@ -1618,8 +1654,8 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         Higher values will yield better recall (more likely to find vectors if
         they exist) at the expense of latency.
 
-        See discussion in [Querying an ANN Index][querying-an-ann-index] for
+        See discussion in [Querying an ANN Index](https://lancedb.com/docs/indexing/)
-        tuning advice.
+        for tuning advice.
 
         This method sets both the minimum and maximum number of probes to the same
         value. See `minimum_nprobes` and `maximum_nprobes` for more fine-grained
@@ -1719,8 +1755,8 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         As an example, a refine factor of 2 will sample 2x as many vectors as
         requested, re-ranks them, and returns the top half most relevant results.
 
-        See discussion in [Querying an ANN Index][querying-an-ann-index] for
+        See discussion in [Querying an ANN Index](https://lancedb.com/docs/indexing/)
-        tuning advice.
+        for tuning advice.
 
         Parameters
         ----------
@@ -1788,6 +1824,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
             with_row_address=self._with_row_address,
             fragments=self._fragments,
             fragment_ids=self._fragment_ids,
+            use_lsm=self._use_lsm,
             offset=self._offset,
             fast_search=self._fast_search,
             ef=self._ef,
@@ -2012,6 +2049,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
             with_row_address=self._with_row_address,
             fragments=self._fragments,
             fragment_ids=self._fragment_ids,
+            use_lsm=self._use_lsm,
             full_text_query=FullTextSearchQuery(
                 query=self._query_with_phrase_semantics(), columns=self._fts_columns
             ),
@@ -2078,6 +2116,7 @@ class LanceEmptyQueryBuilder(LanceQueryBuilder):
             with_row_address=self._with_row_address,
             fragments=self._fragments,
             fragment_ids=self._fragment_ids,
+            use_lsm=self._use_lsm,
             offset=self._offset,
             order_by=self._order_by,
         )
@@ -2655,6 +2694,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         if self._with_row_id:
             self._vector_query.with_row_id(True)
             self._fts_query.with_row_id(True)
+        if self._use_lsm is not None:
+            self._vector_query.use_lsm(self._use_lsm)
+            self._fts_query.use_lsm(self._use_lsm)
         if self._phrase_query:
             self._fts_query.phrase_query(True)
         if self._distance_type:
@@ -3002,7 +3044,7 @@ class AsyncQueryBase(object):
             if blob_mode == "bytes"
             else {}
         )
-        dataset = await self._table._to_lance()
+        dataset = await self._table.to_lance()
         scanner = dataset.scanner(
             **_scanner_kwargs_for_query(
                 query,
@@ -3231,6 +3273,27 @@ class AsyncStandardQuery(AsyncQueryBase):
         self._inner.fast_search()
         return self
 
+    def use_lsm(self, enable: bool) -> Self:
+        """
+        Control MemWAL LSM read routing for this query.
+
+        By default (unset), a query against a table with an LSM write spec (see
+        [AsyncTable.set_lsm_write_spec][lancedb.table.AsyncTable.set_lsm_write_spec])
+        is routed through the LSM scanner so it also returns data written via the
+        ``merge_insert`` LSM path that has not yet been compacted into the base
+        table (the active/frozen in-memory memtables and the flushed generations),
+        deduplicated by primary key; a table without a spec reads the base table.
+
+        Parameters
+        ----------
+        enable : bool
+            ``True`` forces the LSM scanner and errors if the table has no LSM
+            write spec. ``False`` bypasses the MemWAL and reads the base table
+            only, even when a spec is present.
+        """
+        self._inner.use_lsm(enable)
+        return self
+
     def postfilter(self) -> Self:
         """
         If this is called then filtering will happen after the search instead of
@@ -3319,8 +3382,9 @@ class AsyncQuery(AsyncStandardQuery):
         are various ANN search parameters that will let you fine tune your recall
         accuracy vs search latency.
 
-        Vector searches always have a [limit][].  If `limit` has not been called then
+        Vector searches always have a
-        a default `limit` of 10 will be used.
+        [limit][lancedb.query.AsyncVectorQuery.limit].  If `limit` has not been
+        called then a default `limit` of 10 will be used.
 
         Typically, a single vector is passed in as the query. However, you can also
         pass in multiple vectors. When multiple vectors are passed in, if the vector
@@ -3451,8 +3515,9 @@ class AsyncFTSQuery(AsyncStandardQuery):
         are various ANN search parameters that will let you fine tune your recall
         accuracy vs search latency.
 
-        Hybrid searches always have a [limit][].  If `limit` has not been called then
+        Hybrid searches always have a
-        a default `limit` of 10 will be used.
+        [limit][lancedb.query.AsyncHybridQuery.limit].  If `limit` has not been
+        called then a default `limit` of 10 will be used.
 
         Typically, a single vector is passed in as the query. However, you can also
         pass in multiple vectors.  This can be useful if you want to find the nearest
@@ -3875,16 +3940,14 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
         >>> asyncio.run(doctest_example()) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
         RRFReranker(K=60)
             ProjectionExec: expr=[vector@0 as vector, text@3 as text, _distance@2 as _distance]
-            Take: columns="vector, _rowid, _distance, (text)"
+              LanceRead: uri=..., projection=[text], source=stream(_rowid)
-              CoalesceBatchesExec: target_batch_size=1024
                 GlobalLimitExec: skip=0, fetch=10
                   FilterExec: _distance@2 IS NOT NULL
                     SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
                       KNNVectorDistance: metric=l2
                         LanceRead: uri=..., projection=[vector], ...
             ProjectionExec: expr=[vector@2 as vector, text@3 as text, _score@1 as _score]
-            Take: columns="_rowid, _score, (vector), (text)"
+              LanceRead: uri=..., projection=[vector, text], source=stream(_rowid)
-              CoalesceBatchesExec: target_batch_size=1024
                 GlobalLimitExec: skip=0, fetch=10
                   MatchQuery: column=text, query=[hello]
 
@@ -3946,6 +4009,15 @@ class AsyncTakeQuery(AsyncQueryBase):
     def __init__(self, inner: LanceTakeQuery, table: Optional["AsyncTable"] = None):
         super().__init__(inner, table)
 
+    def use_lsm(self, enable: bool) -> "AsyncTakeQuery":
+        """Control MemWAL LSM read routing for this take query.
+
+        ``False`` bypasses the MemWAL and reads the base table only — the escape
+        hatch, since take-by-row-id/offset is not supported on the LSM scanner.
+        """
+        self._inner.use_lsm(enable)
+        return self
+
     async def _plain_scan_to_pandas(
         self,
         blob_mode: BlobMode,
@@ -4004,6 +4076,16 @@ class BaseQueryBuilder(object):
         self._inner.with_row_id()
         return self
 
+    def use_lsm(self, enable: bool) -> Self:
+        """
+        Control MemWAL LSM read routing for this query.
+
+        ``False`` bypasses the MemWAL and reads the base table only, the escape
+        hatch for shapes the LSM scanner cannot honor (e.g. take-by-row-id).
+        """
+        self._inner.use_lsm(enable)
+        return self
+
     def with_row_address(self, with_row_address: bool = True) -> Self:
         """
         Include the _rowaddr column in scanner-backed plain query results.