Merge remote-tracking branch 'origin/main' into xuanwo/fetch-blob-ranges

# Conflicts: # rust/lancedb/src/blob.rs
2026-11-14 14:46:53 +00:00 · 2026-07-27 16:24:06 +08:00
parent d967e4c106 ba4558a64f
commit 58abe14c33
66 changed files with 3054 additions and 508 deletions
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.32.0-beta.2"
+current_version = "0.37.0-beta.0"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
@@ -75,6 +75,13 @@ filename = "nodejs/Cargo.toml"
 replace = "\nversion = \"{new_version}\""
 search = "\nversion = \"{current_version}\""

+# The Python package takes its version from here (pyproject.toml declares
+# `dynamic = ["version"]`, so maturin reads it out of the crate manifest).
+[[tool.bumpversion.files]]
+filename = "python/Cargo.toml"
+replace = "\nversion = \"{new_version}\""
+search = "\nversion = \"{current_version}\""
+
 # Java documentation
 [[tool.bumpversion.files]]
 filename = "docs/src/java/java.md"
@@ -6,7 +6,6 @@ on:
      # We don't publish pre-releases for Rust. Crates.io is just a source
      # distribution, so we don't need to publish pre-releases.
      - "v*-beta*"
-      - "*-v*" # for example, python-vX.Y.Z

 env:
  # This env var is used by Swatinem/rust-cache@v2 for the cache
@@ -0,0 +1,85 @@
+name: GitHub Release
+
+# All SDKs share one version, so a single `vX.Y.Z` tag produces a single GitHub
+# release covering all of them. The per-package publish workflows (PyPI, NPM,
+# Cargo, Maven) trigger off the same tag independently.
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  gh-release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Extract version
+        id: extract_version
+        env:
+          GITHUB_REF: ${{ github.ref }}
+        run: |
+          set -e
+          echo "Extracting tag and version from $GITHUB_REF"
+          if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
+            VERSION=${BASH_REMATCH[1]}
+            TAG=v$VERSION
+            echo "tag=$TAG" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
+          else
+            echo "Failed to extract version from $GITHUB_REF"
+            exit 1
+          fi
+          echo "Extracted version $VERSION from $GITHUB_REF"
+          if [[ $VERSION =~ beta ]]; then
+            echo "This is a beta release"
+            echo "prerelease=true" >> $GITHUB_OUTPUT
+
+            # Get last release (that is not this one)
+            FROM_TAG=$(git tag --sort='version:refname' \
+              | grep ^v \
+              | grep -vF "$TAG" \
+              | python ci/semver_sort.py v \
+              | tail -n 1)
+          else
+            echo "This is a stable release"
+            echo "prerelease=false" >> $GITHUB_OUTPUT
+            # Get last stable tag (ignore betas)
+            FROM_TAG=$(git tag --sort='version:refname' \
+              | grep ^v \
+              | grep -vF "$TAG" \
+              | grep -v beta \
+              | python ci/semver_sort.py v \
+              | tail -n 1)
+          fi
+          echo "Found from tag $FROM_TAG"
+          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
+      - name: Create Release Notes
+        id: release_notes
+        uses: mikepenz/release-changelog-builder-action@v4
+        with:
+          configuration: .github/release_notes.json
+          toTag: ${{ steps.extract_version.outputs.tag }}
+          fromTag: ${{ steps.extract_version.outputs.from_tag }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create GH release
+        uses: softprops/action-gh-release@v2
+        with:
+          # Marking betas as pre-releases keeps them from taking the "Latest"
+          # badge on the releases page.
+          prerelease: ${{ steps.extract_version.outputs.prerelease }}
+          make_latest: ${{ steps.extract_version.outputs.prerelease == 'false' }}
+          tag_name: ${{ steps.extract_version.outputs.tag }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          generate_release_notes: false
+          name: LanceDB v${{ steps.extract_version.outputs.version }}
+          body: ${{ steps.release_notes.outputs.changelog }}
@@ -1,13 +1,14 @@
 name: Create release commit

-# This workflow increments versions, tags the version, and pushes it.
+# This workflow increments the version, tags it, and pushes it. All SDKs share
+# a single version, so one tag releases all of them.
 # When a tag is pushed, another workflow is triggered that creates a GH release
 # and uploads the binaries. This workflow is only for creating the tag.

 # This script will enforce that a minor version is incremented if there are any
-# breaking changes since the last minor increment. However, it isn't able to
-# differentiate between breaking changes in Node versus Python. If you wish to
-# bypass this check, you can manually increment the version and push the tag.
+# breaking changes since the last minor increment. A breaking change in any SDK
+# bumps the minor version for all of them. If you wish to bypass this check, you
+# can manually increment the version and push the tag.
 on:
  workflow_dispatch:
    inputs:
@@ -24,16 +25,6 @@ on:
        options:
          - preview
          - stable
-      python:
-        description: 'Make a Python release'
-        required: true
-        default: true
-        type: boolean
-      other:
-        description: 'Make a Node/Rust/Java release'
-        required: true
-        default: true
-        type: boolean
      bump-minor:
        description: 'Bump minor version'
        required: true
@@ -65,25 +56,12 @@ jobs:
        run: |
          git config user.name 'Lance Release'
          git config user.email 'lance-dev@lancedb.com'
-      - name: Bump Python version
-        if: ${{ inputs.python }}
-        working-directory: python
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          # Need to get the commit before bumping the version, so we can
-          # determine if there are breaking changes in the next step as well.
-          echo "COMMIT_BEFORE_BUMP=$(git rev-parse HEAD)" >> $GITHUB_ENV
-
-          pip install bump-my-version PyGithub packaging
-          bash ../ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} python-v
-      - name: Bump Node/Rust version
-        if: ${{ inputs.other }}
+      - name: Bump version
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pip install bump-my-version PyGithub packaging
-          bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} v $COMMIT_BEFORE_BUMP
+          bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }}
          bash ci/update_lockfiles.sh --amend
      - name: Push new version tag
        if: ${{ !inputs.dry_run }}
@@ -26,73 +26,6 @@ concurrency:
  cancel-in-progress: true

 jobs:
-  gh-release:
-    if: startsWith(github.ref, 'refs/tags/v')
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Extract version
-        id: extract_version
-        env:
-          GITHUB_REF: ${{ github.ref }}
-        run: |
-          set -e
-          echo "Extracting tag and version from $GITHUB_REF"
-          if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
-            VERSION=${BASH_REMATCH[1]}
-            TAG=v$VERSION
-            echo "tag=$TAG" >> $GITHUB_OUTPUT
-            echo "version=$VERSION" >> $GITHUB_OUTPUT
-          else
-            echo "Failed to extract version from $GITHUB_REF"
-            exit 1
-          fi
-          echo "Extracted version $VERSION from $GITHUB_REF"
-          if [[ $VERSION =~ beta ]]; then
-            echo "This is a beta release"
-
-            # Get last release (that is not this one)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^v \
-              | grep -vF "$TAG" \
-              | python ci/semver_sort.py v \
-              | tail -n 1)
-          else
-            echo "This is a stable release"
-            # Get last stable tag (ignore betas)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^v \
-              | grep -vF "$TAG" \
-              | grep -v beta \
-              | python ci/semver_sort.py v \
-              | tail -n 1)
-          fi
-          echo "Found from tag $FROM_TAG"
-          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
-      - name: Create Release Notes
-        id: release_notes
-        uses: mikepenz/release-changelog-builder-action@v4
-        with:
-          configuration: .github/release_notes.json
-          toTag: ${{ steps.extract_version.outputs.tag }}
-          fromTag: ${{ steps.extract_version.outputs.from_tag }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Create GH release
-        uses: softprops/action-gh-release@v2
-        with:
-          prerelease: ${{ contains('beta', github.ref) }}
-          tag_name: ${{ steps.extract_version.outputs.tag }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          generate_release_notes: false
-          name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
-          body: ${{ steps.release_notes.outputs.changelog }}
-
  build-lancedb:
    strategy:
      fail-fast: false
@@ -3,7 +3,7 @@ name: PyPI Publish
 on:
  push:
    tags:
-      - 'python-v*'
+      - 'v*'
  pull_request:
    # This should trigger a dry run (we skip the final publish step)
    paths:
@@ -72,7 +72,7 @@ jobs:
          package-name: ${{ matrix.config.package_name }}
          rustflags: ${{ matrix.config.rustflags }}
      - uses: actions/upload-artifact@v7
-        if: startsWith(github.ref, 'refs/tags/python-v')
+        if: startsWith(github.ref, 'refs/tags/v')
        with:
          name: wheels-linux-${{ matrix.config.package_name }}-${{ matrix.config.platform }}-${{ matrix.config.manylinux }}
          path: target/wheels/*.whl
@@ -101,7 +101,7 @@ jobs:
          python-minor-version: 10
          args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
      - uses: actions/upload-artifact@v7
-        if: startsWith(github.ref, 'refs/tags/python-v')
+        if: startsWith(github.ref, 'refs/tags/v')
        with:
          name: wheels-mac-${{ matrix.config.target }}
          path: target/wheels/lancedb-*.whl
@@ -127,14 +127,14 @@ jobs:
          python-minor-version: 10
          args: "--release --strip"
      - uses: actions/upload-artifact@v7
-        if: startsWith(github.ref, 'refs/tags/python-v')
+        if: startsWith(github.ref, 'refs/tags/v')
        with:
          name: wheels-windows
          path: target/wheels/lancedb-*.whl
          if-no-files-found: error
  publish:
    name: Publish wheels
-    if: startsWith(github.ref, 'refs/tags/python-v')
+    if: startsWith(github.ref, 'refs/tags/v')
    needs: [linux, mac, windows]
    runs-on: ubuntu-latest
    permissions:
@@ -183,72 +183,6 @@ jobs:
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          packages-dir: target/wheels/
-  gh-release:
-    if: startsWith(github.ref, 'refs/tags/python-v')
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Extract version
-        id: extract_version
-        env:
-          GITHUB_REF: ${{ github.ref }}
-        run: |
-          set -e
-          echo "Extracting tag and version from $GITHUB_REF"
-          if [[ $GITHUB_REF =~ refs/tags/python-v(.*) ]]; then
-            VERSION=${BASH_REMATCH[1]}
-            TAG=python-v$VERSION
-            echo "tag=$TAG" >> $GITHUB_OUTPUT
-            echo "version=$VERSION" >> $GITHUB_OUTPUT
-          else
-            echo "Failed to extract version from $GITHUB_REF"
-            exit 1
-          fi
-          echo "Extracted version $VERSION from $GITHUB_REF"
-          if [[ $VERSION =~ beta ]]; then
-            echo "This is a beta release"
-
-            # Get last release (that is not this one)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^python-v \
-              | grep -vF "$TAG" \
-              | python ci/semver_sort.py python-v \
-              | tail -n 1)
-          else
-            echo "This is a stable release"
-            # Get last stable tag (ignore betas)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^python-v \
-              | grep -vF "$TAG" \
-              | grep -v beta \
-              | python ci/semver_sort.py python-v \
-              | tail -n 1)
-          fi
-          echo "Found from tag $FROM_TAG"
-          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
-      - name: Create Python Release Notes
-        id: python_release_notes
-        uses: mikepenz/release-changelog-builder-action@v4
-        with:
-          configuration: .github/release_notes.json
-          toTag: ${{ steps.extract_version.outputs.tag }}
-          fromTag: ${{ steps.extract_version.outputs.from_tag }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Create Python GH release
-        uses: softprops/action-gh-release@v2
-        with:
-          prerelease: ${{ contains('beta', github.ref) }}
-          tag_name: ${{ steps.extract_version.outputs.tag }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          generate_release_notes: false
-          name: Python LanceDB v${{ steps.extract_version.outputs.version }}
-          body: ${{ steps.python_release_notes.outputs.changelog }}
  report-failure:
    name: Report Workflow Failure
    runs-on: ubuntu-latest
@@ -256,7 +190,7 @@ jobs:
    permissions:
      contents: read
      issues: write
-    if: always() && failure() && startsWith(github.ref, 'refs/tags/python-v')
+    if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/create-failure-issue
@@ -157,9 +157,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.103"
+version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"

 [[package]]
 name = "approx"
@@ -535,13 +535,13 @@ dependencies = [

 [[package]]
 name = "async-trait"
-version = "0.1.89"
+version = "0.1.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
+checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.117",
+ "syn 3.0.3",
 ]

 [[package]]
@@ -1750,7 +1750,7 @@ version = "3.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -2288,9 +2288,9 @@ dependencies = [

 [[package]]
 name = "datafusion"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040"
+checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098"
 dependencies = [
 "arrow",
 "arrow-schema",
@@ -2335,9 +2335,9 @@ dependencies = [

 [[package]]
 name = "datafusion-catalog"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139"
+checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99"
 dependencies = [
 "arrow",
 "async-trait",
@@ -2360,9 +2360,9 @@ dependencies = [

 [[package]]
 name = "datafusion-catalog-listing"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891"
+checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02"
 dependencies = [
 "arrow",
 "async-trait",
@@ -2383,9 +2383,9 @@ dependencies = [

 [[package]]
 name = "datafusion-common"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65"
+checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -2407,9 +2407,9 @@ dependencies = [

 [[package]]
 name = "datafusion-common-runtime"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1"
+checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2"
 dependencies = [
 "futures",
 "log",
@@ -2418,9 +2418,9 @@ dependencies = [

 [[package]]
 name = "datafusion-datasource"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe"
+checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd"
 dependencies = [
 "arrow",
 "async-trait",
@@ -2448,9 +2448,9 @@ dependencies = [

 [[package]]
 name = "datafusion-datasource-arrow"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f"
+checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -2472,9 +2472,9 @@ dependencies = [

 [[package]]
 name = "datafusion-datasource-csv"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada"
+checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7"
 dependencies = [
 "arrow",
 "async-trait",
@@ -2495,9 +2495,9 @@ dependencies = [

 [[package]]
 name = "datafusion-datasource-json"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f"
+checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba"
 dependencies = [
 "arrow",
 "async-trait",
@@ -2518,15 +2518,15 @@ dependencies = [

 [[package]]
 name = "datafusion-doc"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e"
+checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891"

 [[package]]
 name = "datafusion-execution"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0"
+checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6"
 dependencies = [
 "arrow",
 "arrow-buffer",
@@ -2546,9 +2546,9 @@ dependencies = [

 [[package]]
 name = "datafusion-expr"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0"
+checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3"
 dependencies = [
 "arrow",
 "arrow-schema",
@@ -2568,9 +2568,9 @@ dependencies = [

 [[package]]
 name = "datafusion-expr-common"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2"
+checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356"
 dependencies = [
 "arrow",
 "datafusion-common",
@@ -2580,9 +2580,9 @@ dependencies = [

 [[package]]
 name = "datafusion-functions"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3"
+checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7"
 dependencies = [
 "arrow",
 "arrow-buffer",
@@ -2612,9 +2612,9 @@ dependencies = [

 [[package]]
 name = "datafusion-functions-aggregate"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09"
+checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b"
 dependencies = [
 "arrow",
 "datafusion-common",
@@ -2633,9 +2633,9 @@ dependencies = [

 [[package]]
 name = "datafusion-functions-aggregate-common"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c"
+checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545"
 dependencies = [
 "arrow",
 "datafusion-common",
@@ -2645,9 +2645,9 @@ dependencies = [

 [[package]]
 name = "datafusion-functions-nested"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5"
+checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f"
 dependencies = [
 "arrow",
 "arrow-ord",
@@ -2670,9 +2670,9 @@ dependencies = [

 [[package]]
 name = "datafusion-functions-table"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26"
+checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383"
 dependencies = [
 "arrow",
 "async-trait",
@@ -2686,9 +2686,9 @@ dependencies = [

 [[package]]
 name = "datafusion-functions-window"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78"
+checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d"
 dependencies = [
 "arrow",
 "datafusion-common",
@@ -2703,9 +2703,9 @@ dependencies = [

 [[package]]
 name = "datafusion-functions-window-common"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb"
+checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441"
 dependencies = [
 "datafusion-common",
 "datafusion-physical-expr-common",
@@ -2713,9 +2713,9 @@ dependencies = [

 [[package]]
 name = "datafusion-macros"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5"
+checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb"
 dependencies = [
 "datafusion-doc",
 "quote",
@@ -2724,9 +2724,9 @@ dependencies = [

 [[package]]
 name = "datafusion-optimizer"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281"
+checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179"
 dependencies = [
 "arrow",
 "chrono",
@@ -2743,9 +2743,9 @@ dependencies = [

 [[package]]
 name = "datafusion-physical-expr"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c"
+checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0"
 dependencies = [
 "arrow",
 "datafusion-common",
@@ -2764,9 +2764,9 @@ dependencies = [

 [[package]]
 name = "datafusion-physical-expr-adapter"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235"
+checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859"
 dependencies = [
 "arrow",
 "datafusion-common",
@@ -2779,9 +2779,9 @@ dependencies = [

 [[package]]
 name = "datafusion-physical-expr-common"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8"
+checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8"
 dependencies = [
 "arrow",
 "chrono",
@@ -2796,9 +2796,9 @@ dependencies = [

 [[package]]
 name = "datafusion-physical-optimizer"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36"
+checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183"
 dependencies = [
 "arrow",
 "datafusion-common",
@@ -2814,9 +2814,9 @@ dependencies = [

 [[package]]
 name = "datafusion-physical-plan"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc"
+checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1"
 dependencies = [
 "arrow",
 "arrow-data",
@@ -2847,9 +2847,9 @@ dependencies = [

 [[package]]
 name = "datafusion-pruning"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3"
+checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7"
 dependencies = [
 "arrow",
 "datafusion-common",
@@ -2863,9 +2863,9 @@ dependencies = [

 [[package]]
 name = "datafusion-session"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417"
+checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a"
 dependencies = [
 "async-trait",
 "datafusion-common",
@@ -2877,9 +2877,9 @@ dependencies = [

 [[package]]
 name = "datafusion-sql"
-version = "54.0.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099"
+checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69"
 dependencies = [
 "arrow",
 "bigdecimal",
@@ -3006,7 +3006,7 @@ dependencies = [
 "libc",
 "option-ext",
 "redox_users",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -3229,7 +3229,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
 "libc",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -4218,7 +4218,7 @@ dependencies = [
 "libc",
 "percent-encoding",
 "pin-project-lite",
- "socket2 0.5.10",
+ "socket2 0.6.3",
 "system-configuration",
 "tokio",
 "tower-service",
@@ -4517,7 +4517,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
 dependencies = [
 "hermit-abi",
 "libc",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -4611,7 +4611,7 @@ dependencies = [
 "portable-atomic-util",
 "serde_core",
 "wasm-bindgen",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -5379,7 +5379,7 @@ dependencies = [

 [[package]]
 name = "lancedb"
-version = "0.32.0-beta.2"
+version = "0.37.0-beta.0"
 dependencies = [
 "ahash",
 "anyhow",
@@ -5467,7 +5467,7 @@ dependencies = [

 [[package]]
 name = "lancedb-nodejs"
-version = "0.32.0-beta.2"
+version = "0.37.0-beta.0"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5492,7 +5492,7 @@ dependencies = [

 [[package]]
 name = "lancedb-python"
-version = "0.35.0-beta.2"
+version = "0.37.0-beta.0"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5592,9 +5592,9 @@ dependencies = [

 [[package]]
 name = "libc"
-version = "0.2.186"
+version = "0.2.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"

 [[package]]
 name = "libloading"
@@ -6065,9 +6065,9 @@ dependencies = [

 [[package]]
 name = "napi"
-version = "3.10.5"
+version = "3.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6826e5ddc15589b2d68c8ad5321c18e85d40488e93e32962f362e572669bccf6"
+checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941"
 dependencies = [
 "bitflags 2.11.1",
 "chrono",
@@ -6090,9 +6090,9 @@ checksum = "c9c366d2c8c60b86fa632df75f745509b52f9128f91a6bad4c796e44abb505e1"

 [[package]]
 name = "napi-derive"
-version = "3.5.10"
+version = "3.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0fe526e81c105d3640516fcde83909dd1afe757c0d7a15af58830b5bc0fb9a1"
+checksum = "a49c513341a61a16a10af6efcce46b30d0822ba2d4fb197d24d33dfc199c78d5"
 dependencies = [
 "convert_case",
 "ctor 1.0.5",
@@ -6104,9 +6104,9 @@ dependencies = [

 [[package]]
 name = "napi-derive-backend"
-version = "5.1.2"
+version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "514281397bcddd9ea9a876c7a21a57bff2374237a000ca9a64ea0211ec1993e2"
+checksum = "4747005fa3e2c9989ac45a723a514c5db2411238b72981a3cda4c701a9dfea17"
 dependencies = [
 "convert_case",
 "proc-macro2",
@@ -6117,9 +6117,9 @@ dependencies = [

 [[package]]
 name = "napi-sys"
-version = "3.2.3"
+version = "3.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73e43cf2eb0bd1bf95a43c07c076ebd2da5d1e015a71c3d201faeffffcc0ecac"
+checksum = "85fbf1fa9f1babfe396d74bbbf52b3643770243e8f5b0b46715d4caf7f0dfc9a"
 dependencies = [
 "libloading",
 ]
@@ -6208,7 +6208,7 @@ version = "0.50.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -7587,8 +7587,8 @@ version = "0.14.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
 dependencies = [
- "heck 0.4.1",
- "itertools 0.11.0",
+ "heck 0.5.0",
+ "itertools 0.14.0",
 "log",
 "multimap",
 "petgraph",
@@ -7607,7 +7607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
 dependencies = [
 "anyhow",
- "itertools 0.11.0",
+ "itertools 0.14.0",
 "proc-macro2",
 "quote",
 "syn 2.0.117",
@@ -7828,7 +7828,7 @@ dependencies = [
 "quinn-udp",
 "rustc-hash",
 "rustls 0.23.40",
- "socket2 0.5.10",
+ "socket2 0.6.3",
 "thiserror 2.0.18",
 "tokio",
 "tracing",
@@ -7866,9 +7866,9 @@ dependencies = [
 "cfg_aliases",
 "libc",
 "once_cell",
- "socket2 0.5.10",
+ "socket2 0.6.3",
 "tracing",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]

 [[package]]
@@ -8174,9 +8174,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.13.0"
+version = "1.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -8186,9 +8186,9 @@ dependencies = [

 [[package]]
 name = "regex-automata"
-version = "0.4.14"
+version = "0.4.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
+checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -8642,7 +8642,7 @@ dependencies = [
 "errno",
 "libc",
 "linux-raw-sys",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -8713,7 +8713,7 @@ dependencies = [
 "security-framework",
 "security-framework-sys",
 "webpki-root-certs",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -8918,9 +8918,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"

 [[package]]
 name = "serde"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
 dependencies = [
 "serde_core",
 "serde_derive",
@@ -8928,29 +8928,29 @@ dependencies = [

 [[package]]
 name = "serde_core"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.117",
+ "syn 3.0.3",
 ]

 [[package]]
 name = "serde_json"
-version = "1.0.150"
+version = "1.0.151"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
 dependencies = [
 "itoa",
 "memchr",
@@ -9281,7 +9281,7 @@ version = "0.8.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
 "proc-macro2",
 "quote",
 "syn 2.0.117",
@@ -9293,7 +9293,7 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
 "proc-macro2",
 "quote",
 "syn 2.0.117",
@@ -9609,6 +9609,17 @@ dependencies = [
 "unicode-ident",
 ]

+[[package]]
+name = "syn"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "sync_wrapper"
 version = "1.0.2"
@@ -9726,7 +9737,7 @@ dependencies = [
 "getrandom 0.4.2",
 "once_cell",
 "rustix",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -9929,9 +9940,9 @@ dependencies = [

 [[package]]
 name = "tokio"
-version = "1.52.3"
+version = "1.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
+checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
 dependencies = [
 "bytes",
 "libc",
@@ -10416,9 +10427,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"

 [[package]]
 name = "uuid"
-version = "1.23.5"
+version = "1.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a"
+checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
 dependencies = [
 "getrandom 0.4.2",
 "js-sys",
@@ -10703,7 +10714,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -2,9 +2,9 @@ set -e

 RELEASE_TYPE=${1:-"stable"}
 BUMP_MINOR=${2:-false}
-TAG_PREFIX=${3:-"v"} # Such as "python-v"
-HEAD_SHA=${4:-$(git rev-parse HEAD)}
+HEAD_SHA=$(git rev-parse HEAD)

+readonly TAG_PREFIX="v"
 readonly SELF_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )

 PREV_TAG=$(git tag --sort='version:refname' | grep ^$TAG_PREFIX | python $SELF_DIR/semver_sort.py $TAG_PREFIX | tail -n 1)
@@ -12,7 +12,7 @@ echo "Found previous tag $PREV_TAG"

 # Initially, we don't want to tag if we are doing stable, because we will bump
 # again later. See comment at end for why.
-if [[ "$RELEASE_TYPE" == 'stable' ]]; then 
+if [[ "$RELEASE_TYPE" == 'stable' ]]; then
  BUMP_ARGS="--no-tag"
 fi

@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-core</artifactId>
-    <version>0.32.0-beta.2</version>
+    <version>0.37.0-beta.0</version>
 </dependency>
 ```

@@ -249,6 +249,57 @@ try (BufferAllocator allocator = new RootAllocator();
 }
 ```

+### Creating an Empty Table
+
+To create an empty table, send an Arrow IPC stream that contains the table schema and no record batches.
+The schema in the IPC stream becomes the table schema, and rows can be inserted later.
+
+```java
+import org.lance.namespace.model.CreateTableRequest;
+import org.lance.namespace.model.CreateTableResponse;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.ipc.ArrowStreamWriter;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.channels.Channels;
+import java.util.Arrays;
+
+Schema schema = new Schema(Arrays.asList(
+    new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null),
+    new Field("name", FieldType.nullable(new ArrowType.Utf8()), null),
+    new Field("embedding",
+        FieldType.nullable(new ArrowType.FixedSizeList(128)),
+        Arrays.asList(new Field("item",
+            FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
+            null)))
+));
+
+byte[] emptyTableData;
+try (BufferAllocator allocator = new RootAllocator();
+     VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
+    root.setRowCount(0);
+
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, Channels.newChannel(out))) {
+        writer.start();
+        writer.end();
+    }
+    emptyTableData = out.toByteArray();
+}
+
+CreateTableRequest request = new CreateTableRequest();
+request.setId(Arrays.asList("my_namespace", "empty_table"));
+
+CreateTableResponse response = namespaceClient.createTable(request, emptyTableData);
+```
+
 ### Insert

 ```java
@@ -431,9 +482,88 @@ query.setVector(vector);
 byte[] result = namespaceClient.queryTable(query);
 ```

-### Reading Query Results
+## Indexing

-Query results are returned in Apache Arrow IPC file format. Here's how to read them:
+The Java SDK exposes the REST namespace index operations through the same `LanceNamespace` client.
+Index creation runs asynchronously, so use `listTableIndices` or `describeTableIndexStats` to check progress.
+
+### Creating a Vector Index
+
+```java
+import org.lance.namespace.model.CreateTableIndexRequest;
+import org.lance.namespace.model.CreateTableIndexResponse;
+
+CreateTableIndexRequest request = new CreateTableIndexRequest();
+request.setId(Arrays.asList("my_namespace", "my_table"));
+request.setColumn("embedding");
+request.setIndexType("IVF_PQ");
+request.setDistanceType("cosine");
+request.setName("embedding_idx");
+
+CreateTableIndexResponse response = namespaceClient.createTableIndex(request);
+System.out.println("Index transaction: " + response.getTransactionId());
+```
+
+### Creating a Scalar Index
+
+```java
+import org.lance.namespace.model.CreateTableIndexRequest;
+import org.lance.namespace.model.CreateTableScalarIndexResponse;
+
+CreateTableIndexRequest request = new CreateTableIndexRequest();
+request.setId(Arrays.asList("my_namespace", "my_table"));
+request.setColumn("category");
+request.setIndexType("BTREE");
+request.setName("category_idx");
+
+CreateTableScalarIndexResponse response = namespaceClient.createTableScalarIndex(request);
+System.out.println("Index transaction: " + response.getTransactionId());
+```
+
+### Creating a Full Text Search Index
+
+```java
+import org.lance.namespace.model.CreateTableIndexRequest;
+import org.lance.namespace.model.CreateTableScalarIndexResponse;
+
+CreateTableIndexRequest request = new CreateTableIndexRequest();
+request.setId(Arrays.asList("my_namespace", "my_table"));
+request.setColumn("text_column");
+request.setIndexType("FTS");
+request.setName("text_idx");
+request.setBaseTokenizer("simple");
+request.setLowerCase(true);
+request.setWithPosition(true);
+
+CreateTableScalarIndexResponse response = namespaceClient.createTableScalarIndex(request);
+System.out.println("Index transaction: " + response.getTransactionId());
+```
+
+### Listing Indexes
+
+```java
+import org.lance.namespace.model.IndexContent;
+import org.lance.namespace.model.ListTableIndicesRequest;
+import org.lance.namespace.model.ListTableIndicesResponse;
+
+ListTableIndicesRequest request = new ListTableIndicesRequest();
+request.setId(Arrays.asList("my_namespace", "my_table"));
+
+ListTableIndicesResponse response = namespaceClient.listTableIndices(request);
+for (IndexContent index : response.getIndexes()) {
+    System.out.println(index.getIndexName() + ": " + index.getStatus());
+}
+```
+
+!!! note
+    The current Java namespace API exposes index type, index name, distance type, and full text search tokenizer options.
+    IVF training parameters such as `num_partitions` are not exposed by `CreateTableIndexRequest` yet.
+    To make those configurable from Java, the namespace API must add those fields first.
+
+## Reading Query Results
+
+Query results are returned as bytes in Apache Arrow IPC file format. Put the byte-channel
+adapter behind a small helper so query code can work with `ArrowFileReader` directly:

 ```java
 import org.apache.arrow.vector.ipc.ArrowFileReader;
@@ -441,45 +571,50 @@ import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;

+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.SeekableByteChannel;

-// Helper class to read Arrow data from byte array
-class ByteArraySeekableByteChannel implements SeekableByteChannel {
-    private final byte[] data;
-    private long position = 0;
-    private boolean isOpen = true;
-
-    public ByteArraySeekableByteChannel(byte[] data) {
-        this.data = data;
+final class ArrowIpc {
+    static ArrowFileReader openFileReader(byte[] data, BufferAllocator allocator) throws IOException {
+        return new ArrowFileReader(new ByteArraySeekableByteChannel(data), allocator);
    }

-    @Override
-    public int read(ByteBuffer dst) {
-        int remaining = dst.remaining();
-        int available = (int) (data.length - position);
-        if (available <= 0) return -1;
-        int toRead = Math.min(remaining, available);
-        dst.put(data, (int) position, toRead);
-        position += toRead;
-        return toRead;
-    }
+    private static final class ByteArraySeekableByteChannel implements SeekableByteChannel {
+        private final byte[] data;
+        private long position = 0;
+        private boolean isOpen = true;

-    @Override public long position() { return position; }
-    @Override public SeekableByteChannel position(long newPosition) { position = newPosition; return this; }
-    @Override public long size() { return data.length; }
-    @Override public boolean isOpen() { return isOpen; }
-    @Override public void close() { isOpen = false; }
-    @Override public int write(ByteBuffer src) { throw new UnsupportedOperationException(); }
-    @Override public SeekableByteChannel truncate(long size) { throw new UnsupportedOperationException(); }
+        private ByteArraySeekableByteChannel(byte[] data) {
+            this.data = data;
+        }
+
+        @Override
+        public int read(ByteBuffer dst) {
+            int remaining = dst.remaining();
+            int available = (int) (data.length - position);
+            if (available <= 0) return -1;
+            int toRead = Math.min(remaining, available);
+            dst.put(data, (int) position, toRead);
+            position += toRead;
+            return toRead;
+        }
+
+        @Override public long position() { return position; }
+        @Override public SeekableByteChannel position(long newPosition) { position = newPosition; return this; }
+        @Override public long size() { return data.length; }
+        @Override public boolean isOpen() { return isOpen; }
+        @Override public void close() { isOpen = false; }
+        @Override public int write(ByteBuffer src) { throw new UnsupportedOperationException(); }
+        @Override public SeekableByteChannel truncate(long size) { throw new UnsupportedOperationException(); }
+    }
 }

 // Read query results
 byte[] queryResult = namespaceClient.queryTable(query);

 try (BufferAllocator allocator = new RootAllocator();
-     ArrowFileReader reader = new ArrowFileReader(
-         new ByteArraySeekableByteChannel(queryResult), allocator)) {
+     ArrowFileReader reader = ArrowIpc.openFileReader(queryResult, allocator)) {

    for (int i = 0; i < reader.getRecordBlocks().size(); i++) {
        reader.loadRecordBatch(reader.getRecordBlocks().get(i));
@@ -76,24 +76,23 @@ the query optimizer chooses a suboptimal path.

 ***

-### useLsmWrite()
+### useLsm()

 ```ts
-useLsmWrite(useLsmWrite): MergeInsertBuilder
+useLsm(enable): MergeInsertBuilder
 ```

-Controls whether the merge uses the MemWAL LSM write path.
+Control MemWAL routing for this merge.

 By default (unset), a `mergeInsert` on a table with an LSM write spec is
-routed through Lance's MemWAL shard writer, and a table without one uses
-the standard path. Pass `false` to force the standard path even when a
-spec is set. Pass `true` to require a spec — `mergeInsert` rejects if none
-is installed.
+routed through Lance's MemWAL shard writer, and a table without one uses the
+standard path.

 #### Parameters

-* **useLsmWrite**: `boolean`
-    Whether to use the LSM write path.
+* **enable**: `boolean`
+    `true` forces MemWAL routing and errors if the table has no
+    LSM write spec. `false` forces the standard write path even when a spec is set.

 #### Returns

@@ -497,6 +497,42 @@ ArrowTable.

 ***

+### useLsm()
+
+```ts
+useLsm(enable): this
+```
+
+Control MemWAL read routing for this query.
+
+By default (unset), when the table carries a MemWAL write spec (see
+[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
+they also return data written via the `mergeInsert` LSM path that has not yet
+been compacted into the base table (the active/frozen in-memory memtables and
+the flushed generations), deduplicated by primary key; a table without a spec
+reads the base table.
+
+#### Parameters
+
+* **enable**: `boolean`
+    `true` forces the LSM scanner and errors if the table has no
+    MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
+    even when a spec is present.
+    Note: the LSM scanner does not support every query shape (e.g. reranking,
+    hybrid search, `orderBy`). On a MemWAL table those shapes error unless
+    `useLsm(false)` is set, because a base-only read would silently exclude
+    un-compacted MemWAL data.
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+`StandardQueryBase.useLsm`
+
+***
+
 ### where()

 ```ts
@@ -273,6 +273,29 @@ ArrowTable.

 ***

+### useLsm()
+
+```ts
+useLsm(enable): this
+```
+
+Control MemWAL read routing for this take query.
+
+`false` bypasses the MemWAL and reads the base table only — the escape hatch,
+since take-by-row-id/offset is not supported on the LSM scanner and, on a
+MemWAL table, auto-routes to it and errors otherwise.
+
+#### Parameters
+
+* **enable**: `boolean`
+    `false` reads the base table only.
+
+#### Returns
+
+`this`
+
+***
+
 ### withRowId()

 ```ts
@@ -746,6 +746,42 @@ ArrowTable.

 ***

+### useLsm()
+
+```ts
+useLsm(enable): this
+```
+
+Control MemWAL read routing for this query.
+
+By default (unset), when the table carries a MemWAL write spec (see
+[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
+they also return data written via the `mergeInsert` LSM path that has not yet
+been compacted into the base table (the active/frozen in-memory memtables and
+the flushed generations), deduplicated by primary key; a table without a spec
+reads the base table.
+
+#### Parameters
+
+* **enable**: `boolean`
+    `true` forces the LSM scanner and errors if the table has no
+    MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
+    even when a spec is present.
+    Note: the LSM scanner does not support every query shape (e.g. reranking,
+    hybrid search, `orderBy`). On a MemWAL table those shapes error unless
+    `useLsm(false)` is set, because a base-only read would silently exclude
+    un-compacted MemWAL data.
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+`StandardQueryBase.useLsm`
+
+***
+
 ### where()

 ```ts
@@ -43,6 +43,19 @@ The following tokenizers are available:

 ***

+### blockSize?
+
+```ts
+optional blockSize: 128 | 256;
+```
+
+Number of documents per compressed posting block.
+
+The default is 128. Supported values are 128 and 256. A value of 256 uses
+the experimental FTS V3 format and may introduce breaking changes.
+
+***
+
 ### language?

 ```ts
@@ -8,7 +8,7 @@
    <parent>
      <groupId>com.lancedb</groupId>
      <artifactId>lancedb-parent</artifactId>
-      <version>0.32.0-beta.2</version>
+      <version>0.37.0-beta.0</version>
      <relativePath>../pom.xml</relativePath>
    </parent>

@@ -6,7 +6,7 @@

    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-parent</artifactId>
-    <version>0.32.0-beta.2</version>
+    <version>0.37.0-beta.0</version>
    <packaging>pom</packaging>
    <name>${project.artifactId}</name>
    <description>LanceDB Java SDK Parent POM</description>
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.32.0-beta.2"
+version = "0.37.0-beta.0"
 publish = false
 license.workspace = true
 description.workspace = true
@@ -15,6 +15,7 @@ import {
  OAuthHeaderProvider,
  StaticHeaderProvider,
 } from "../lancedb/header";
+import { Index } from "../lancedb/indices";

 // Test-only header providers
 class CustomProvider extends HeaderProvider {
@@ -225,6 +226,54 @@ describe("remote connection", () => {
    );
  });

+  it("sends the FTS posting block size to remote tables", async () => {
+    let createIndexBody: Record<string, unknown> | undefined;
+
+    await withMockDatabase(
+      (req, res) => {
+        const path = req.url ?? "";
+        if (path.endsWith("/describe/")) {
+          res.writeHead(200, { "Content-Type": "application/json" }).end(
+            JSON.stringify({
+              name: "t",
+              version: 1,
+              schema: {
+                fields: [
+                  { name: "text", type: { type: "string" }, nullable: false },
+                ],
+              },
+            }),
+          );
+          return;
+        }
+
+        if (path.endsWith("/create_index/")) {
+          let raw = "";
+          req.on("data", (chunk) => {
+            raw += chunk;
+          });
+          req.on("end", () => {
+            createIndexBody = JSON.parse(raw);
+            res.writeHead(200).end();
+          });
+          return;
+        }
+
+        res.writeHead(404).end();
+      },
+      async (db) => {
+        const table = await db.openTable("t");
+        await table.createIndex("text", {
+          config: Index.fts({ blockSize: 256 }),
+        });
+      },
+    );
+
+    expect(createIndexBody?.["column"]).toBe("text");
+    expect(createIndexBody?.["index_type"]).toBe("FTS");
+    expect(createIndexBody?.["block_size"]).toBe(256);
+  });
+
  it("diffs and merges remote branches", async () => {
    const sampleDiff = {
      fromBranch: "exp",
@@ -527,6 +527,14 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
      );
    });

+    it("should expose useLsm on takeRowIds as the base-only escape hatch", async () => {
+      await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
+      // useLsm(false) is reachable on TakeQuery (the escape hatch for MemWAL tables,
+      // where take-by-row-id auto-routes to the LSM scanner and is rejected).
+      const res = await table.takeRowIds([0, 2]).useLsm(false).toArray();
+      expect(res.map((r) => r.id)).toEqual([1, 3]);
+    });
+
    it("should throw for negative number in takeRowIds", () => {
      expect(() => table.takeRowIds([-1])).toThrow("Row id cannot be negative");
      expect(() => table.takeRowIds([0, -5, 2])).toThrow(
@@ -2527,6 +2535,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
      expect(results3.length).toBe(1);
    });

+    test("full text search with custom posting block size", async () => {
+      const db = await connect(tmpDir.name);
+      const data = [
+        { text: "hello world", vector: [0.1, 0.2, 0.3] },
+        { text: "goodbye world", vector: [0.4, 0.5, 0.6] },
+      ];
+      const table = await db.createTable("test", data);
+      await table.createIndex("text", {
+        config: Index.fts({ blockSize: 256 }),
+      });
+
+      const index = (await table.listIndices()).find(
+        (index) => index.indexType === "FTS",
+      );
+      expect(index?.indexVersion).toBe(3);
+      expect(
+        (index?.indexDetails as Record<string, unknown>)["block_size"],
+      ).toBe(256);
+
+      const results = await table.search("hello").toArray();
+      expect(results[0].text).toBe(data[0].text);
+    });
+
+    test("rejects invalid full text posting block size", () => {
+      expect(() => Index.fts({ blockSize: 129 as 128 | 256 })).toThrow(
+        "128 or 256",
+      );
+    });
+
    test("full text search without lowercase", async () => {
      const db = await connect(tmpDir.name);
      const data = [
@@ -3170,14 +3207,14 @@ describe("LSM merge insert", () => {
    await table.closeLsmWriters();
  });

-  it("falls back to the standard path with useLsmWrite(false)", async () => {
+  it("falls back to the standard path with useLsm(false)", async () => {
    const conn = await connect(tmpDir.name);
    const table = await bucketTable(conn);

    const res = await table
      .mergeInsert("id")
      .whenNotMatchedInsertAll()
-      .useLsmWrite(false)
+      .useLsm(false)
      .execute([
        { id: "b", value: 9 },
        { id: "e", value: 5 },
@@ -3211,4 +3248,36 @@ describe("LSM merge insert", () => {
        .execute([{ id: "g", value: 7 }]),
    ).rejects.toThrow();
  });
+
+  it("auto-routes reads through the MemWAL scanner", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await bucketTable(conn); // base ids "a", "b"
+
+    await table
+      .mergeInsert("id")
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .execute([{ id: "c", value: 3 }]);
+
+    // Default read auto-routes and includes the active memtable row.
+    const lsm = await table.query().toArray();
+    expect(lsm.map((r) => r.id).sort()).toEqual(["a", "b", "c"]);
+
+    // useLsm(false) bypasses the MemWAL and reads the base table only.
+    const baseOnly = await table.query().useLsm(false).toArray();
+    expect(baseOnly.map((r) => r.id).sort()).toEqual(["a", "b"]);
+  });
+
+  it("reads the base table when no LSM spec is installed", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await conn.createEmptyTable(
+      "plain",
+      new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
+    );
+    // No spec: default read and useLsm(false) both succeed against the base table.
+    await expect(table.query().toArray()).resolves.toBeDefined();
+    await expect(table.query().useLsm(false).toArray()).resolves.toBeDefined();
+    // useLsm(true) demands MemWAL routing; without a spec it errors.
+    await expect(table.query().useLsm(true).toArray()).rejects.toThrow();
+  });
 });
@@ -572,6 +572,14 @@ export interface FtsOptions {
   * whether to only index the prefix of the token for ngram tokenizer
   */
  prefixOnly?: boolean;
+
+  /**
+   * Number of documents per compressed posting block.
+   *
+   * The default is 128. Supported values are 128 and 256. A value of 256 uses
+   * the experimental FTS V3 format and may introduce breaking changes.
+   */
+  blockSize?: 128 | 256;
 }

 export class Index {
@@ -751,6 +759,7 @@ export class Index {
        options?.ngramMinLength,
        options?.ngramMaxLength,
        options?.prefixOnly,
+        options?.blockSize,
      ),
    );
  }
@@ -88,21 +88,17 @@ export class MergeInsertBuilder {
    );
  }
  /**
-   * Controls whether the merge uses the MemWAL LSM write path.
+   * Control MemWAL routing for this merge.
   *
   * By default (unset), a `mergeInsert` on a table with an LSM write spec is
-   * routed through Lance's MemWAL shard writer, and a table without one uses
-   * the standard path. Pass `false` to force the standard path even when a
-   * spec is set. Pass `true` to require a spec — `mergeInsert` rejects if none
-   * is installed.
+   * routed through Lance's MemWAL shard writer, and a table without one uses the
+   * standard path.
   *
-   * @param useLsmWrite - Whether to use the LSM write path.
+   * @param enable - `true` forces MemWAL routing and errors if the table has no
+   * LSM write spec. `false` forces the standard write path even when a spec is set.
   */
-  useLsmWrite(useLsmWrite: boolean): MergeInsertBuilder {
-    return new MergeInsertBuilder(
-      this.#native.useLsmWrite(useLsmWrite),
-      this.#schema,
-    );
+  useLsm(enable: boolean): MergeInsertBuilder {
+    return new MergeInsertBuilder(this.#native.useLsm(enable), this.#schema);
  }
  /**
   * Controls how an LSM merge checks that its input targets a single shard.
@@ -460,6 +460,30 @@ export class StandardQueryBase<
    this.doCall((inner: NativeQueryType) => inner.fastSearch());
    return this;
  }
+
+  /**
+   * Control MemWAL read routing for this query.
+   *
+   * By default (unset), when the table carries a MemWAL write spec (see
+   * {@link Table#setLsmWriteSpec}), reads are routed through the LSM scanner so
+   * they also return data written via the `mergeInsert` LSM path that has not yet
+   * been compacted into the base table (the active/frozen in-memory memtables and
+   * the flushed generations), deduplicated by primary key; a table without a spec
+   * reads the base table.
+   *
+   * @param enable - `true` forces the LSM scanner and errors if the table has no
+   * MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
+   * even when a spec is present.
+   *
+   * Note: the LSM scanner does not support every query shape (e.g. reranking,
+   * hybrid search, `orderBy`). On a MemWAL table those shapes error unless
+   * `useLsm(false)` is set, because a base-only read would silently exclude
+   * un-compacted MemWAL data.
+   */
+  useLsm(enable: boolean): this {
+    this.doCall((inner: NativeQueryType) => inner.useLsm(enable));
+    return this;
+  }
 }

 /**
@@ -748,6 +772,20 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
  constructor(inner: NativeTakeQuery) {
    super(inner);
  }
+
+  /**
+   * Control MemWAL read routing for this take query.
+   *
+   * `false` bypasses the MemWAL and reads the base table only — the escape hatch,
+   * since take-by-row-id/offset is not supported on the LSM scanner and, on a
+   * MemWAL table, auto-routes to it and errors otherwise.
+   *
+   * @param enable - `false` reads the base table only.
+   */
+  useLsm(enable: boolean): this {
+    this.doCall((inner: NativeTakeQuery) => inner.useLsm(enable));
+    return this;
+  }
 }

 /** A builder for LanceDB queries.
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.32.0-beta.2",
+	"version": "0.37.0-beta.0",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.32.0-beta.2",
+	"version": "0.37.0-beta.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.32.0-beta.2",
+	"version": "0.37.0-beta.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.32.0-beta.2",
+	"version": "0.37.0-beta.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.32.0-beta.2",
+	"version": "0.37.0-beta.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",
@@ -1,6 +1,6 @@
 {
  "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.32.0-beta.2",
+  "version": "0.37.0-beta.0",
  "os": [
    "win32"
  ],
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.32.0-beta.2",
+	"version": "0.37.0-beta.0",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
@@ -1,12 +1,12 @@
 {
  "name": "@lancedb/lancedb",
-  "version": "0.32.0-beta.2",
+  "version": "0.37.0-beta.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "@lancedb/lancedb",
-      "version": "0.32.0-beta.2",
+      "version": "0.37.0-beta.0",
      "cpu": [
        "x64",
        "arm64"
@@ -11,7 +11,7 @@
    "ann"
  ],
  "private": false,
-  "version": "0.32.0-beta.2",
+  "version": "0.37.0-beta.0",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
@@ -226,7 +226,8 @@ impl Index {
        ngram_min_length: Option<u32>,
        ngram_max_length: Option<u32>,
        prefix_only: Option<bool>,
-    ) -> Self {
+        block_size: Option<u32>,
+    ) -> napi::Result<Self> {
        let mut opts = FtsIndexBuilder::default();
        if let Some(with_position) = with_position {
            opts = opts.with_position(with_position);
@@ -261,10 +262,15 @@ impl Index {
        if let Some(prefix_only) = prefix_only {
            opts = opts.ngram_prefix_only(prefix_only);
        }
-
-        Self {
-            inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
+        if let Some(block_size) = block_size {
+            opts = opts
+                .block_size(block_size as usize)
+                .map_err(|err| napi::Error::from_reason(err.to_string()))?;
        }
+
+        Ok(Self {
+            inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
+        })
    }

    #[napi(factory)]
@@ -51,9 +51,9 @@ impl NativeMergeInsertBuilder {
    }

    #[napi]
-    pub fn use_lsm_write(&self, use_lsm_write: bool) -> Self {
+    pub fn use_lsm(&self, enable: bool) -> Self {
        let mut this = self.clone();
-        this.inner.use_lsm_write(use_lsm_write);
+        this.inner.use_lsm(enable);
        this
    }

@@ -168,6 +168,11 @@ impl Query {
        self.inner = self.inner.clone().with_row_id();
    }

+    #[napi]
+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_lsm(enable);
+    }
+
    #[napi]
    pub fn order_by(&mut self, ordering: Option<Vec<ColumnOrdering>>) -> napi::Result<()> {
        let ordering = ordering.map(|ordering| {
@@ -374,6 +379,11 @@ impl VectorQuery {
        self.inner = self.inner.clone().with_row_id();
    }

+    #[napi]
+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_lsm(enable);
+    }
+
    #[napi]
    pub fn rerank(
        &mut self,
@@ -479,6 +489,11 @@ impl TakeQuery {
        self.inner = self.inner.clone().with_row_id();
    }

+    #[napi]
+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_lsm(enable);
+    }
+
    #[napi(catch_unwind)]
    pub async fn output_schema(&self) -> napi::Result<Buffer> {
        let schema = self.inner.output_schema().await.default_error()?;
@@ -2,7 +2,7 @@

 Manage branches on a LanceDB table: list what exists, create new ones, delete stale ones, and direct read/write operations at a specific branch without touching main. Use for branch lifecycle tasks, experimental/isolated table versions, targeting an operation at a non-main branch, or confirming a mutation did not affect main.

-Works on local/OSS and remote Enterprise/Cloud tables.
+Works on local/OSS and remote Enterprise/Cloud tables, except merging a branch into main, which is Enterprise-only.

 ## The branch model (important)

@@ -101,6 +101,69 @@ assert b"lancedb:description" not in (table.schema.field("category").metadata or

 Two handles on the same branch see each other's writes (e.g. `table.branches.create("exp")` and `db.open_table(name, branch="exp")`); main stays isolated.

+## Merging a branch into main (Enterprise only)
+
+Merge is available through the SDKs (`table.branches.merge(...)`) on **Enterprise tables only** — it is not supported on Cloud or local/OSS tables, which raise `NotSupported`.
+
+`merge` takes the branch to merge **from** and a `dry_run` flag. Both the SDK method and the underlying REST endpoint **actually merge by default** (`dry_run=False`); pass `dry_run=True` to only preview. A rejected merge is **not an exception** — it returns a result with `status="rejected"` rather than raising, so inspect the return value. Use `branches.diff(from_branch)` to inspect a branch's pending diff without attempting a merge.
+
+```python
+exp = "experiment-reindex"
+
+# preview only — returns status="ready" if it would merge cleanly
+preview = table.branches.merge(exp, dry_run=True)
+
+# actually merge (default)
+result = table.branches.merge(exp)
+if result["status"] == "merged":
+    print("landed at", result["mainVersionAfter"])
+elif result["status"] == "rejected":
+    print(result["diff"]["mergeBlockers"])   # why it was refused
+
+# inspect a branch's pending diff without merging
+diff = table.branches.diff(exp)
+```
+
+Async: `await table.branches.merge(exp)`, `await table.branches.diff(exp)`.
+
+```typescript
+const branches = await table.branches();
+const exp = "experiment-reindex";
+
+// preview only (second arg is dryRun)
+const preview = await branches.merge(exp, true);
+
+// actually merge (default)
+const result = await branches.merge(exp);
+if (result.status === "merged") {
+  console.log("landed at", result.mainVersionAfter);
+} else if (result.status === "rejected") {
+  console.log(result.diff.mergeBlockers);
+}
+
+const diff = await branches.diff(exp);
+```
+
+The result is the wire JSON, containing `status` (`ready` on a passing dry run, `merged` on success, `rejected` when refused — also `notImplemented`/`unknown`), the branch `diff` (including `mergeBlockers` explaining any rejection), a `preview` of the columns that would be promoted, and — after a real merge — `mainVersionAfter`.
+
+### Merge preconditions
+
+Merge only **promotes newly added columns** onto main; it does not replay arbitrary commits. Practically, a branch is mergeable only if it has **exactly one commit since it was created, and that commit added a column**. The merge is rejected (`status: "rejected"`, with `mergeBlockers` set) if:
+
+- the branch was forked from another branch rather than directly from main
+- main has advanced since the branch was forked
+- the branch's rows changed since the fork (row counts must match main exactly)
+- the branch removed columns or changed a column's type/nullability
+- the branch added no columns (index-only changes are not merged)
+
+### Adding a column in a single commit
+
+Because the branch must contain just one column-adding commit, add the column with its values in one operation rather than add-then-backfill:
+
+1. **SQL transformation** — `add_columns` with a SQL expression computed from existing columns, so the column lands populated in one commit.
+2. **Precompute the values** — compute the column's values externally, then add the fully-populated column in a single operation (e.g. via `merge_insert`/`add_columns` with the data ready).
+3. **Lance-format-level data evolution (pylance)** — use Lance's data evolution with backfill, documented at <https://lance.org/guide/data_evolution/#with-data-backfill>.
+
 ## Quick reference

 | Goal | Python | TypeScript |
@@ -113,5 +176,7 @@ Two handles on the same branch see each other's writes (e.g. `table.branches.cre
 | Delete branch | `table.branches.delete(name)` | `await branches.delete(name)` |
 | Which branch is this handle on? | `table.current_branch()` (`None` = main) | `table.currentBranch()` (`null` = main) |
 | Target main | use the original (non-branch) handle | use the original (non-branch) handle |
+| Merge branch into main (Enterprise only) | `table.branches.merge(from_branch, dry_run=False)` | `await branches.merge(fromBranch, dryRun)` |
+| Preview a branch's pending diff (Enterprise only) | `table.branches.diff(from_branch)` | `await branches.diff(fromBranch)` |

 Branch names must be non-empty; empty names raise a validation error.
@@ -4,12 +4,19 @@ Quick method reference for Python LanceDB code. Cross-check source for non-trivi

 ## Connect

+If you're connecting to a remote database, use this:
 ```python
 import lancedb

-db = lancedb.connect("./camelot-db")         # local/OSS
-db = lancedb.connect("db://my-db", api_key=api_key, region=region)  # remote
+db = lancedb.connect("db://my-db", api_key=api_key, host_override=host_override) # remote
 ```
+(values may be found in LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, either in env vars or a .env file)
+
+If you're connecting to a local table using OSS LanceDB, use this:
+```python
+db = lancedb.connect("./camelot-db") # local/OSS
+```
+If you're not sure which, or if you can't find the api_key or host_override params, ask the user.

 **Place the local database directory next to the script/entrypoint that opens it** (i.e. resolve the path relative to the script, `Path(__file__).parent / "camelot-db"`), not buried under a shared `data/` folder. The Lance dataset is the database, not a data file — keeping it beside its code makes ownership obvious and paths stable regardless of the working directory the script is launched from.

@@ -1,49 +0,0 @@
-[tool.bumpversion]
-current_version = "0.35.0-beta.2"
-parse = """(?x)
-    (?P<major>0|[1-9]\\d*)\\.
-    (?P<minor>0|[1-9]\\d*)\\.
-    (?P<patch>0|[1-9]\\d*)
-    (?:-(?P<pre_l>[a-zA-Z-]+)\\.(?P<pre_n>0|[1-9]\\d*))?
-"""
-serialize = [
-    "{major}.{minor}.{patch}-{pre_l}.{pre_n}",
-    "{major}.{minor}.{patch}",
-]
-search = "{current_version}"
-replace = "{new_version}"
-regex = false
-ignore_missing_version = false
-ignore_missing_files = false
-tag = true
-sign_tags = false
-tag_name = "python-v{new_version}"
-tag_message = "Bump version: {current_version} → {new_version}"
-allow_dirty = true
-commit = true
-message = "Bump version: {current_version} → {new_version}"
-commit_args = ""
-# bump-my-version >=1.4.0 rejects pre_commit_hooks containing shell syntax unless opted in.
-allow_shell_hooks = true
-
-# Update Cargo.lock after version bump
-pre_commit_hooks = [
-  """
-    cd python && cargo update -p lancedb-python
-    if git diff --quiet ../Cargo.lock; then
-        echo "Cargo.lock unchanged"
-    else
-        git add ../Cargo.lock
-        echo "Updated and staged Cargo.lock"
-    fi
-    """,
-]
-
-[tool.bumpversion.parts.pre_l]
-values = ["beta", "final"]
-optional_value = "final"
-
-[[tool.bumpversion.files]]
-filename = "Cargo.toml"
-search = "\nversion = \"{current_version}\""
-replace = "\nversion = \"{new_version}\""
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.35.0-beta.2"
+version = "0.37.0-beta.0"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -396,6 +396,7 @@ class Query:
    def fast_search(self): ...
    def with_row_id(self): ...
    def postfilter(self): ...
+    def use_lsm(self, enable: bool): ...
    def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ...
    def nearest_to_text(self, query: dict) -> FTSQuery: ...
    def order_by(self, ordering: Optional[List[ColumnOrdering]]): ...
@@ -412,6 +413,7 @@ class Query:
 class TakeQuery:
    def select(self, columns: List[str]): ...
    def with_row_id(self): ...
+    def use_lsm(self, enable: bool): ...
    async def output_schema(self) -> pa.Schema: ...
    async def execute(self) -> RecordBatchStream: ...
    async def explain_plan(self, verbose: Optional[bool]) -> str: ...
@@ -430,6 +432,7 @@ class FTSQuery:
    def fast_search(self): ...
    def with_row_id(self): ...
    def postfilter(self): ...
+    def use_lsm(self, enable: bool): ...
    def get_query(self) -> str: ...
    def add_query_vector(self, query_vec: pa.Array) -> None: ...
    def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ...
@@ -457,6 +460,7 @@ class VectorQuery:
    def column(self, column: str): ...
    def distance_type(self, distance_type: str): ...
    def postfilter(self): ...
+    def use_lsm(self, enable: bool): ...
    def refine_factor(self, refine_factor: int): ...
    def nprobes(self, nprobes: int): ...
    def minimum_nprobes(self, minimum_nprobes: int): ...
@@ -480,6 +484,7 @@ class HybridQuery:
    def fast_search(self): ...
    def with_row_id(self): ...
    def postfilter(self): ...
+    def use_lsm(self, enable: bool): ...
    def distance_type(self, distance_type: str): ...
    def refine_factor(self, refine_factor: int): ...
    def nprobes(self, nprobes: int): ...
@@ -504,6 +509,7 @@ class PyQueryRequest:
    select: Optional[Union[str, List[str]]]
    fast_search: Optional[bool]
    with_row_id: Optional[bool]
+    use_lsm: Optional[bool]
    column: Optional[str]
    query_vector: Optional[List[pa.Array]]
    minimum_nprobes: Optional[int]
@@ -115,6 +115,12 @@ class FTS:

    For example, it works with `title`, `description`, `content`, etc.

+    Examples
+    --------
+    Create an index configuration that uses 256-document posting blocks:
+
+    >>> config = FTS(block_size=256)
+
    Attributes
    ----------
    with_position : bool, default False
@@ -148,6 +154,10 @@ class FTS:
    ascii_folding : bool, default True
        Whether to fold ASCII characters. This converts accented characters to
        their ASCII equivalent. For example, "café" would be converted to "cafe".
+    block_size : int, default 128
+        The number of documents per compressed posting block. Supported values
+        are 128 and 256. A value of 256 uses the experimental FTS V3 format
+        and may introduce breaking changes.

    Notes
    -----
@@ -168,6 +178,7 @@ class FTS:
    ngram_min_length: int = 3
    ngram_max_length: int = 3
    prefix_only: bool = False
+    block_size: int = 128


@dataclass
@@ -37,7 +37,7 @@ class LanceMergeInsertBuilder(object):
        self._when_not_matched_by_source_condition_expr = None
        self._timeout = None
        self._use_index = True
-        self._use_lsm_write = None
+        self._use_lsm = None
        self._validate_single_shard = None

    def when_matched_update_all(
@@ -113,22 +113,22 @@ class LanceMergeInsertBuilder(object):
        self._use_index = use_index
        return self

-    def use_lsm_write(self, use_lsm_write: bool) -> LanceMergeInsertBuilder:
+    def use_lsm(self, enable: bool) -> LanceMergeInsertBuilder:
        """
-        Controls whether the merge uses the MemWAL LSM write path.
+        Control MemWAL routing for this merge.

-        By default (unset), a `merge_insert` on a table with an LSM write spec
-        is routed through Lance's MemWAL shard writer, and a table without one
-        uses the standard path. Pass `False` to force the standard path even
-        when a spec is set. Pass `True` to require a spec — `merge_insert`
-        raises an error if none is installed.
+        By default (unset), a `merge_insert` on a table with an LSM write spec is
+        routed through Lance's MemWAL shard writer, and a table without one uses
+        the standard path.

        Parameters
        ----------
-        use_lsm_write: bool
-            Whether to use the LSM write path.
+        enable: bool
+            ``True`` forces MemWAL routing and errors if the table has no LSM
+            write spec. ``False`` forces the standard write path even when a spec
+            is set.
        """
-        self._use_lsm_write = use_lsm_write
+        self._use_lsm = enable
        return self

    def validate_single_shard(
@@ -885,7 +885,7 @@ class Permutation:
        This method refines the current selection, potentially removing columns.  It
        will not add back columns that were previously removed.

-        If any of the columns do not exist then an error will be raised
+        If any of the columns do not exist then an error will be raised.

        This does not introduce a post-processing step.  It simply reduces the amount
        of data we read.
@@ -898,9 +898,14 @@ class Permutation:
        for name in columns:
            value = self.selection.get(name, None)
            if value is None:
-                raise ValueError(
-                    f"Cannot select column `{name}` because it does not exist"
-                )
+                if name == "_rowid":
+                    # _rowid is a system column not in the default schema
+                    # but can be explicitly selected
+                    value = "_rowid"
+                else:
+                    raise ValueError(
+                        f"Cannot select column `{name}` because it does not exist"
+                    )
            new_selection[name] = value
        return self._with_selection(new_selection)

@@ -778,6 +778,11 @@ class Query(pydantic.BaseModel):
    # if true, will only search the indexed data
    fast_search: Optional[bool] = None

+    # MemWAL LSM read routing: None auto-routes when the table carries a write
+    # spec, True forces the LSM scanner (errors without a spec), False reads the
+    # base table only
+    use_lsm: Optional[bool] = None
+
    # size of the nearest neighbor list maintained during HNSW search
    ef: Optional[int] = None

@@ -795,6 +800,9 @@ class Query(pydantic.BaseModel):
        query.full_text_query = req.full_text_search
        query.columns = req.select
        query.with_row_id = req.with_row_id
+        # use_lsm is a genuine tri-state (None / True / False); preserve it as-is
+        # so a round-tripped query keeps an explicit False.
+        query.use_lsm = req.use_lsm
        query.vector_column = req.column
        query.vector = req.query_vector
        query.distance_type = req.distance_type
@@ -967,6 +975,7 @@ class LanceQueryBuilder(ABC):
        self._with_row_address = None
        self._fragments = None
        self._fragment_ids = None
+        self._use_lsm = None
        self._vector = None
        self._text = None
        self._ef = None
@@ -1326,6 +1335,30 @@ class LanceQueryBuilder(ABC):
        self._fragment_ids = fragment_ids
        return self

+    def use_lsm(self, enable: bool) -> Self:
+        """Control MemWAL LSM read routing for this query.
+
+        By default (unset), a query against a table with an LSM write spec is
+        routed through the LSM scanner so it also returns data written via the
+        ``merge_insert`` LSM path that has not yet been compacted into the base
+        table (active/frozen memtables + flushed generations); a table without a
+        spec reads the base table.
+
+        Parameters
+        ----------
+        enable : bool
+            ``True`` forces the LSM scanner and errors if the table has no LSM
+            write spec. ``False`` bypasses the MemWAL and reads the base table
+            only, even when a spec is present.
+
+        Returns
+        -------
+        LanceQueryBuilder
+            The LanceQueryBuilder object.
+        """
+        self._use_lsm = enable
+        return self
+
    def explain_plan(self, verbose: Optional[bool] = False) -> str:
        """Return the execution plan for this query.

@@ -1788,6 +1821,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
            with_row_address=self._with_row_address,
            fragments=self._fragments,
            fragment_ids=self._fragment_ids,
+            use_lsm=self._use_lsm,
            offset=self._offset,
            fast_search=self._fast_search,
            ef=self._ef,
@@ -2012,6 +2046,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
            with_row_address=self._with_row_address,
            fragments=self._fragments,
            fragment_ids=self._fragment_ids,
+            use_lsm=self._use_lsm,
            full_text_query=FullTextSearchQuery(
                query=self._query_with_phrase_semantics(), columns=self._fts_columns
            ),
@@ -2078,6 +2113,7 @@ class LanceEmptyQueryBuilder(LanceQueryBuilder):
            with_row_address=self._with_row_address,
            fragments=self._fragments,
            fragment_ids=self._fragment_ids,
+            use_lsm=self._use_lsm,
            offset=self._offset,
            order_by=self._order_by,
        )
@@ -2655,6 +2691,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
        if self._with_row_id:
            self._vector_query.with_row_id(True)
            self._fts_query.with_row_id(True)
+        if self._use_lsm is not None:
+            self._vector_query.use_lsm(self._use_lsm)
+            self._fts_query.use_lsm(self._use_lsm)
        if self._phrase_query:
            self._fts_query.phrase_query(True)
        if self._distance_type:
@@ -3231,6 +3270,27 @@ class AsyncStandardQuery(AsyncQueryBase):
        self._inner.fast_search()
        return self

+    def use_lsm(self, enable: bool) -> Self:
+        """
+        Control MemWAL LSM read routing for this query.
+
+        By default (unset), a query against a table with an LSM write spec (see
+        [AsyncTable.set_lsm_write_spec][lancedb.table.AsyncTable.set_lsm_write_spec])
+        is routed through the LSM scanner so it also returns data written via the
+        ``merge_insert`` LSM path that has not yet been compacted into the base
+        table (the active/frozen in-memory memtables and the flushed generations),
+        deduplicated by primary key; a table without a spec reads the base table.
+
+        Parameters
+        ----------
+        enable : bool
+            ``True`` forces the LSM scanner and errors if the table has no LSM
+            write spec. ``False`` bypasses the MemWAL and reads the base table
+            only, even when a spec is present.
+        """
+        self._inner.use_lsm(enable)
+        return self
+
    def postfilter(self) -> Self:
        """
        If this is called then filtering will happen after the search instead of
@@ -3944,6 +4004,15 @@ class AsyncTakeQuery(AsyncQueryBase):
    def __init__(self, inner: LanceTakeQuery, table: Optional["AsyncTable"] = None):
        super().__init__(inner, table)

+    def use_lsm(self, enable: bool) -> "AsyncTakeQuery":
+        """Control MemWAL LSM read routing for this take query.
+
+        ``False`` bypasses the MemWAL and reads the base table only — the escape
+        hatch, since take-by-row-id/offset is not supported on the LSM scanner.
+        """
+        self._inner.use_lsm(enable)
+        return self
+
    async def _plain_scan_to_pandas(
        self,
        blob_mode: BlobMode,
@@ -4002,6 +4071,16 @@ class BaseQueryBuilder(object):
        self._inner.with_row_id()
        return self

+    def use_lsm(self, enable: bool) -> Self:
+        """
+        Control MemWAL LSM read routing for this query.
+
+        ``False`` bypasses the MemWAL and reads the base table only, the escape
+        hatch for shapes the LSM scanner cannot honor (e.g. take-by-row-id).
+        """
+        self._inner.use_lsm(enable)
+        return self
+
    def with_row_address(self, with_row_address: bool = True) -> Self:
        """
        Include the _rowaddr column in scanner-backed plain query results.
@@ -344,6 +344,7 @@ class RemoteTable(Table):
        ngram_min_length: int = 3,
        ngram_max_length: int = 3,
        prefix_only: bool = False,
+        block_size: int = 128,
        name: Optional[str] = None,
    ):
        """Create a full-text search index on a column.
@@ -364,6 +365,7 @@ class RemoteTable(Table):
            ngram_min_length=ngram_min_length,
            ngram_max_length=ngram_max_length,
            prefix_only=prefix_only,
+            block_size=block_size,
        )
        LOOP.run(
            self._table.create_index(
@@ -23,7 +23,7 @@ class AnswerdotaiRerankers(Reranker):
    column : str, default "text"
        The name of the column to use as input to the cross encoder model.
    return_score : str, default "relevance"
-        options are "relevance" or "all". Only "relevance" is supported for now.
+        options are "relevance" or "all".
    **kwargs
        Additional keyword arguments to pass to the model. For example, 'device'.
        See AnswerDotAI/rerankers for more information.
@@ -77,12 +77,13 @@ class AnswerdotaiRerankers(Reranker):
        vector_results: pa.Table,
        fts_results: pa.Table,
    ):
-        combined_results = self.merge_results(vector_results, fts_results)
+        if self.score == "all":
+            combined_results = self._merge_and_keep_scores(vector_results, fts_results)
+        else:
+            combined_results = self.merge_results(vector_results, fts_results)
        combined_results = self._rerank(combined_results, query)
        if self.score == "relevance":
            combined_results = self._keep_relevance_score(combined_results)
-        elif self.score == "all":
-            combined_results = self._merge_and_keep_scores(vector_results, fts_results)
        combined_results = combined_results.sort_by(
            [("_relevance_score", "descending")]
        )
@@ -16,7 +16,7 @@ class ColbertReranker(AnswerdotaiRerankers):
    column : str, default "text"
        The name of the column to use as input to the cross encoder model.
    return_score : str, default "relevance"
-        options are "relevance" or "all". Only "relevance" is supported for now.
+        options are "relevance" or "all".
    **kwargs
        Additional keyword arguments to pass to the model, for example, 'device'.
        See AnswerDotAI/rerankers for more information.
@@ -1107,6 +1107,7 @@ class Table(ABC):
        ngram_min_length: int = 3,
        ngram_max_length: int = 3,
        prefix_only: bool = False,
+        block_size: int = 128,
        wait_timeout: Optional[timedelta] = None,
        name: Optional[str] = None,
    ):
@@ -1178,6 +1179,10 @@ class Table(ABC):
            The maximum length of an n-gram.
        prefix_only: bool, default False
            Whether to only index the prefix of the token for ngram tokenizer.
+        block_size: int, default 128
+            The number of documents per compressed posting block. Must be 128
+            or 256. A value of 256 uses the experimental FTS V3 format and
+            may introduce breaking changes.
        wait_timeout: timedelta, optional
            The timeout to wait if indexing is asynchronous.
        name: str, optional
@@ -3054,6 +3059,7 @@ class LanceTable(Table):
        ngram_min_length: int = 3,
        ngram_max_length: int = 3,
        prefix_only: bool = False,
+        block_size: int = 128,
        name: Optional[str] = None,
    ):
        """Create a full-text search index on a column.
@@ -3103,9 +3109,7 @@ class LanceTable(Table):
        else:
            tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)

-        config = FTS(
-            **tokenizer_configs,
-        )
+        config = FTS(block_size=block_size, **tokenizer_configs)

        try:
            LOOP.run(
@@ -5387,6 +5391,8 @@ class AsyncTable:
            async_query = async_query.where(query.filter)
        if query.fast_search:
            async_query = async_query.fast_search()
+        if query.use_lsm is not None:
+            async_query = async_query.use_lsm(query.use_lsm)
        if query.with_row_id:
            async_query = async_query.with_row_id()
        if query.order_by:
@@ -5507,7 +5513,7 @@ class AsyncTable:
                when_not_matched_by_source_condition_expr=merge._when_not_matched_by_source_condition_expr,
                timeout=merge._timeout,
                use_index=merge._use_index,
-                use_lsm_write=merge._use_lsm_write,
+                use_lsm=merge._use_lsm,
                validate_single_shard=merge._validate_single_shard,
            ),
        )
@@ -226,6 +226,23 @@ def test_create_inverted_index(table, with_position):
    assert any(i.name == "custom_fts_index" for i in fts_indices)


+@pytest.mark.parametrize("block_size", [128, 256])
+def test_create_inverted_index_block_size(table, block_size):
+    table.create_index("text", config=FTS(block_size=block_size))
+
+    index = next(index for index in table.list_indices() if index.index_type == "FTS")
+    assert index.index_details["block_size"] == block_size
+    assert index.index_version == (2 if block_size == 128 else 3)
+
+    results = table.search("puppy").limit(5).to_list()
+    assert len(results) == 5
+
+
+def test_create_inverted_index_rejects_invalid_block_size(table):
+    with pytest.raises(ValueError, match="128 or 256"):
+        table.create_index("text", config=FTS(block_size=129))
+
+
 def test_search_fts(table):
    table.create_fts_index("text")
    results = table.search("puppy").select(["id", "text"]).limit(5).to_list()
@@ -9,6 +9,7 @@ import lancedb
 import pyarrow as pa
 import pytest
 from lancedb._lancedb import LsmWriteSpec
+from lancedb.index import FTS, IvfPq

 SCHEMA = pa.schema(
    [
@@ -102,19 +103,35 @@ def test_lsm_merge_insert_identity(tmp_path):
    assert result.num_rows == 2


-def test_lsm_merge_insert_use_lsm_write_false(tmp_path):
+def test_lsm_merge_insert_use_lsm_false(tmp_path):
    table = _bucket_table(tmp_path)  # rows id = 1, 2, 3
-    # use_lsm_write(False) opts out: the standard path runs and commits.
+    # use_lsm(False) opts out: the standard path runs and commits even with a spec.
    result = (
        table.merge_insert("id")
        .when_not_matched_insert_all()
-        .use_lsm_write(False)
+        .use_lsm(False)
        .execute(_reader([3, 4, 5]))
    )
    assert result.num_inserted_rows == 2
    assert table.count_rows() == 5


+def test_lsm_merge_insert_use_lsm_true_without_spec_errors(tmp_path):
+    # A table with a primary key but no LSM write spec installed.
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    table = db.create_table("t", _reader([1, 2, 3]))
+    table.set_unenforced_primary_key("id")
+    # use_lsm(True) demands MemWAL routing; without a spec it errors.
+    with pytest.raises(Exception, match="use_lsm"):
+        (
+            table.merge_insert("id")
+            .when_matched_update_all()
+            .when_not_matched_insert_all()
+            .use_lsm(True)
+            .execute(_reader([3, 4, 5]))
+        )
+
+
 def test_lsm_merge_insert_validate_single_shard_off(tmp_path):
    table = _bucket_table(tmp_path)
    result = (
@@ -127,19 +144,20 @@ def test_lsm_merge_insert_validate_single_shard_off(tmp_path):
    assert result.num_rows == 3


-def test_lsm_merge_insert_use_lsm_write_true_requires_spec(tmp_path):
+def test_lsm_merge_insert_no_spec_uses_standard_path(tmp_path):
    # A table with a primary key but no LSM write spec installed.
    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
    table = db.create_table("t", _reader([1, 2, 3]))
    table.set_unenforced_primary_key("id")
-    with pytest.raises(Exception, match="use_lsm_write"):
-        (
-            table.merge_insert("id")
-            .when_matched_update_all()
-            .when_not_matched_insert_all()
-            .use_lsm_write(True)
-            .execute(_reader([4]))
-        )
+    # With no spec, a default merge_insert uses the standard path and commits.
+    result = (
+        table.merge_insert("id")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute(_reader([3, 4, 5]))
+    )
+    assert result.num_inserted_rows == 2
+    assert table.count_rows() == 5


 def test_lsm_merge_insert_rejects_on_not_primary_key(tmp_path):
@@ -194,3 +212,445 @@ async def test_async_lsm_merge_insert(tmp_path):
    result = await builder.execute(_reader([3, 4, 5]))
    assert result.num_rows == 3
    await table.close_lsm_writers()
+
+
+def _lsm_upsert(table, ids):
+    """Upsert ``ids`` (value = 0..n) through the LSM merge_insert path."""
+    (
+        table.merge_insert([])
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute(_reader(ids))
+    )
+
+
+def test_lsm_read_sees_active_memtable(tmp_path):
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    table = db.create_table("t", _reader([1, 2, 3]))  # base ids 1,2,3
+    table.set_unenforced_primary_key("id")
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+
+    _lsm_upsert(table, [4, 5])  # active memtable only, not committed to base
+
+    # Default read auto-routes through the LSM scanner: base ∪ active memtable.
+    lsm = table.search().to_arrow()
+    assert sorted(lsm["id"].to_pylist()) == [1, 2, 3, 4, 5]
+
+    # use_lsm(False) bypasses the MemWAL and reads the base table only.
+    base_only = table.search().use_lsm(False).to_arrow()
+    assert sorted(base_only["id"].to_pylist()) == [1, 2, 3]
+
+
+def test_lsm_read_dedup_newest_wins(tmp_path):
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    table = db.create_table("t", _reader([1, 2, 3]))  # id 2 -> value 1
+    table.set_unenforced_primary_key("id")
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+
+    _lsm_upsert(table, [2, 3, 4])  # ids 2,3,4 -> values 0,1,2
+
+    lsm = table.search().to_arrow().sort_by("id")
+    assert lsm["id"].to_pylist() == [1, 2, 3, 4]
+    # id 1 from base (value 0); 2,3,4 from memtable (values 0,1,2).
+    assert lsm["value"].to_pylist() == [0, 0, 1, 2]
+
+
+def test_lsm_read_without_spec_reads_base(tmp_path):
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    table = db.create_table("t", _reader([1, 2, 3]))
+    table.set_unenforced_primary_key("id")  # no LSM write spec
+
+    # No spec: default read and use_lsm(False) both read the base table, no error.
+    assert sorted(table.search().to_arrow()["id"].to_pylist()) == [1, 2, 3]
+    assert sorted(table.search().use_lsm(False).to_arrow()["id"].to_pylist()) == [
+        1,
+        2,
+        3,
+    ]
+
+
+def test_lsm_read_unsupported_shape_errors_without_use_lsm_false(tmp_path):
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    table = db.create_table("t", _reader([1, 2, 3]))
+    table.set_unenforced_primary_key("id")
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    _lsm_upsert(table, [4])
+
+    # with_row_id is unsupported by the LSM scanner; on a MemWAL table the default
+    # (auto-routed) read hard-errors instead of silently reading a stale base.
+    with pytest.raises(Exception):
+        table.search().with_row_id(True).to_arrow()
+
+    # use_lsm(False) is the escape hatch: it reads the base table only.
+    base = table.search().with_row_id(True).use_lsm(False).to_arrow()
+    assert sorted(base["id"].to_pylist()) == [1, 2, 3]
+
+
+@pytest.mark.asyncio
+async def test_async_lsm_read(tmp_path):
+    db = await lancedb.connect_async(
+        tmp_path, read_consistency_interval=timedelta(seconds=0)
+    )
+    table = await db.create_table("t", _reader([1, 2, 3]))
+    await table.set_unenforced_primary_key("id")
+    await table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+
+    builder = (
+        table.merge_insert([]).when_matched_update_all().when_not_matched_insert_all()
+    )
+    await builder.execute(_reader([4, 5]))
+
+    arrow = await table.query().to_arrow()
+    assert sorted(arrow["id"].to_pylist()) == [1, 2, 3, 4, 5]
+
+
+VECTOR_DIM = 8
+
+VECTOR_SCHEMA = pa.schema(
+    [
+        pa.field("id", pa.int64(), nullable=False),
+        pa.field("category", pa.utf8(), nullable=False),
+        pa.field("vector", pa.list_(pa.float32(), VECTOR_DIM), nullable=False),
+    ]
+)
+
+
+def _vector_reader(rows):
+    """Rows are ``(id, category, [f32; VECTOR_DIM])`` tuples."""
+    batch = pa.RecordBatch.from_arrays(
+        [
+            pa.array([row[0] for row in rows], type=pa.int64()),
+            pa.array([row[1] for row in rows], type=pa.utf8()),
+            pa.array([row[2] for row in rows], type=pa.list_(pa.float32(), VECTOR_DIM)),
+        ],
+        schema=VECTOR_SCHEMA,
+    )
+    return pa.RecordBatchReader.from_batches(VECTOR_SCHEMA, [batch])
+
+
+def _vector_table(tmp_path):
+    """Base table whose vector column is indexed so its rows are visible to the LSM
+    vector scanner (the base arm uses ``fast_search`` — indexed data only), plus an
+    unsharded LSM spec that maintains that index for the memtable.
+
+    Rows 1,2 are category ``a``, row 3 is ``b``, and 4..60 are filler ``c`` that
+    give the tiny IVF index enough data to train.
+    """
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    rows = [
+        (
+            i,
+            "a" if i in (1, 2) else "b" if i == 3 else "c",
+            [float((i * 7 + j) % 13) for j in range(VECTOR_DIM)],
+        )
+        for i in range(1, 61)
+    ]
+    table = db.create_table("t", _vector_reader(rows))
+    table.set_unenforced_primary_key("id")
+    # num_partitions=1 makes the search exhaustive within the single partition
+    # (deterministic); num_bits=4 keeps PQ training viable on a tiny dataset.
+    table.create_index(
+        "vector", config=IvfPq(num_partitions=1, num_sub_vectors=2, num_bits=4)
+    )
+    index_name = table.list_indices()[0].name
+    table.set_lsm_write_spec(
+        LsmWriteSpec.unsharded().with_maintained_indexes([index_name])
+    )
+    return table
+
+
+def _vector_upsert(table, rows):
+    (
+        table.merge_insert([])
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute(_vector_reader(rows))
+    )
+
+
+def test_lsm_read_vector_sees_memtable(tmp_path):
+    table = _vector_table(tmp_path)
+    # id 1000 lands in the active memtable, not committed to the base table.
+    _vector_upsert(table, [(1000, "a", [1.0] * VECTOR_DIM)])
+
+    query = [1.0] * VECTOR_DIM
+    # Vector search auto-routes through the LSM scanner: indexed base ∪ memtable.
+    ids = set(table.search(query).limit(100).to_arrow()["id"].to_pylist())
+    assert {1, 2, 3} <= ids  # indexed base rows
+    assert 1000 in ids  # in-flight memtable row
+
+    # use_lsm(False) bypasses the MemWAL, so the in-flight row is not visible.
+    base_ids = set(
+        table.search(query).use_lsm(False).limit(100).to_arrow()["id"].to_pylist()
+    )
+    assert {1, 2, 3} <= base_ids
+    assert 1000 not in base_ids
+
+
+def test_lsm_read_vector_prefilter(tmp_path):
+    table = _vector_table(tmp_path)
+    # in-flight rows in both categories.
+    _vector_upsert(
+        table, [(1000, "a", [1.0] * VECTOR_DIM), (1001, "b", [1.0] * VECTOR_DIM)]
+    )
+
+    query = [1.0] * VECTOR_DIM
+    # The `where` predicate must apply as a prefilter across base ∪ memtable —
+    # regression test for the vector arm silently dropping the filter.
+    rows = table.search(query).where("category = 'a'").limit(100).to_arrow()
+    assert set(rows["id"].to_pylist()) == {1, 2, 1000}
+    assert set(rows["category"].to_pylist()) == {"a"}
+
+    # Sanity: without the filter, other categories are returned too.
+    unfiltered = set(table.search(query).limit(100).to_arrow()["category"].to_pylist())
+    assert unfiltered != {"a"}
+
+
+def test_lsm_read_plain_prefilter(tmp_path):
+    table = _vector_table(tmp_path)
+    _vector_upsert(
+        table, [(1000, "a", [1.0] * VECTOR_DIM), (1001, "b", [1.0] * VECTOR_DIM)]
+    )
+
+    # Plain scan + filter over base ∪ memtable: base 'a' rows 1,2 and memtable 1000.
+    rows = table.search().where("category = 'a'").to_arrow()
+    assert set(rows["id"].to_pylist()) == {1, 2, 1000}
+
+
+FTS_SCHEMA = pa.schema(
+    [
+        pa.field("id", pa.int64(), nullable=False),
+        pa.field("text", pa.utf8(), nullable=False),
+    ]
+)
+
+
+def _fts_reader(rows):
+    """Rows are ``(id, text)`` tuples."""
+    batch = pa.RecordBatch.from_arrays(
+        [
+            pa.array([row[0] for row in rows], type=pa.int64()),
+            pa.array([row[1] for row in rows], type=pa.utf8()),
+        ],
+        schema=FTS_SCHEMA,
+    )
+    return pa.RecordBatchReader.from_batches(FTS_SCHEMA, [batch])
+
+
+def test_lsm_read_fts_sees_memtable(tmp_path):
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    table = db.create_table(
+        "t",
+        _fts_reader(
+            [
+                (1, "the quick brown fox"),
+                (2, "lazy dog sleeps"),
+                (3, "quick red fox"),
+            ]
+        ),
+    )
+    table.set_unenforced_primary_key("id")
+    # Native FTS index (tantivy is not compatible with the LSM memtable index).
+    table.create_index("text", config=FTS())
+    index_name = table.list_indices()[0].name
+    table.set_lsm_write_spec(
+        LsmWriteSpec.unsharded().with_maintained_indexes([index_name])
+    )
+
+    # in-flight doc 4 lands in the memtable's maintained FTS index.
+    (
+        table.merge_insert([])
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute(_fts_reader([(4, "brown fox jumps")]))
+    )
+
+    # Full-text search auto-routes through the LSM scanner: base ∪ memtable.
+    ids = set(
+        table.search("fox", query_type="fts", fts_columns="text")
+        .limit(10)
+        .to_arrow()["id"]
+        .to_pylist()
+    )
+    assert ids == {1, 3, 4}
+
+    # Prefilter restricts the FTS results across both tiers.
+    filtered = set(
+        table.search("fox", query_type="fts", fts_columns="text")
+        .where("id > 1")
+        .limit(10)
+        .to_arrow()["id"]
+        .to_pylist()
+    )
+    assert filtered == {3, 4}
+
+
+def test_lsm_read_vector_unsupported_knobs_error(tmp_path):
+    table = _vector_table(tmp_path)
+    _vector_upsert(table, [(1000, "a", [1.0] * VECTOR_DIM)])
+    query = [1.0] * VECTOR_DIM
+
+    # distance_range and use_index(False) change the vector result set/mode, which
+    # the LSM scanner can't honor, so it hard-errors instead of silently returning
+    # wrong results (matching the prefilter / unsupported-shape contract).
+    with pytest.raises(Exception, match="distance_range"):
+        table.search(query).distance_range(0.0, 0.5).to_arrow()
+    with pytest.raises(Exception, match="use_index"):
+        table.search(query).bypass_vector_index().to_arrow()
+
+    # use_lsm(False) is the escape hatch: the base-only standard path honors them.
+    base = table.search(query).distance_range(0.0, 100.0).use_lsm(False).to_arrow()
+    assert 1000 not in set(base["id"].to_pylist())
+
+
+def test_lsm_read_vector_limit_offset(tmp_path):
+    table = _vector_table(tmp_path)
+    _vector_upsert(table, [(1000, "a", [1.0] * VECTOR_DIM)])
+    query = [1.0] * VECTOR_DIM
+    # Lance's plan_vector over-fetches k + offset internally, so paging is correct:
+    # the second page is a full page (not truncated) and disjoint from the first.
+    page1 = table.search(query).limit(3).offset(0).to_arrow()["id"].to_pylist()
+    page2 = table.search(query).limit(3).offset(3).to_arrow()["id"].to_pylist()
+    assert len(page1) == 3
+    # If k ignored offset, page2 would be empty (limit - offset = 0); a full second
+    # page that differs from the first proves offset widens the candidate pool.
+    assert len(page2) == 3
+    assert set(page1) != set(page2)
+
+
+def test_lsm_read_vector_postfilter_errors(tmp_path):
+    table = _vector_table(tmp_path)
+    _vector_upsert(table, [(1000, "a", [1.0] * VECTOR_DIM)])
+    query = [1.0] * VECTOR_DIM
+    # The LSM scanner always prefilters; a requested postfilter changes results, so
+    # it hard-errors rather than silently prefiltering.
+    with pytest.raises(Exception, match="postfilter"):
+        table.search(query).where("category = 'a'").postfilter().to_arrow()
+
+
+def test_lsm_read_projection_excludes_pk(tmp_path):
+    table = _vector_table(tmp_path)
+    _vector_upsert(table, [(1000, "a", [1.0] * VECTOR_DIM)])
+    # Selecting only 'category' must not leak the 'id' primary key Lance appends
+    # internally for dedup.
+    rows = table.search().select(["category"]).where("category = 'a'").to_arrow()
+    assert rows.column_names == ["category"]
+
+
+def test_lsm_read_fts_unmaintained_index_errors(tmp_path):
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    table = db.create_table("t", _fts_reader([(1, "quick fox"), (2, "lazy dog")]))
+    table.set_unenforced_primary_key("id")
+    table.create_index("text", config=FTS())
+    # No maintained indexes: the active memtable FTS arm cannot serve un-compacted
+    # docs, so the search would silently omit them — reject instead.
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    with pytest.raises(Exception, match="maintained"):
+        table.search("fox", query_type="fts", fts_columns="text").to_arrow()
+
+
+def test_lsm_read_time_travel_errors(tmp_path):
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    table = db.create_table("t", _reader([1, 2, 3]))
+    table.set_unenforced_primary_key("id")
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    pinned = table.version
+    table.add(_reader([4, 5]))  # standard add commits a newer version
+    table.checkout(pinned)  # detached head at the historical version
+
+    # The WAL/manifest expose current live state, so an LSM read at a pinned
+    # historical version is rejected.
+    with pytest.raises(Exception, match="time-travel"):
+        table.search().to_arrow()
+    # use_lsm(False) reads the base table at the pinned version.
+    base = table.search().use_lsm(False).to_arrow()
+    assert sorted(base["id"].to_pylist()) == [1, 2, 3]
+
+
+def test_lsm_read_take_row_ids_errors(tmp_path):
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    table = db.create_table("t", _reader([1, 2, 3]))
+    table.set_unenforced_primary_key("id")
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    _lsm_upsert(table, [4])
+    # take-by-row-id auto-routes through the LSM scanner, which has no stable _rowid,
+    # so it hard-errors instead of failing with an opaque column-not-found error.
+    with pytest.raises(Exception, match="row id"):
+        table.take_row_ids([0, 1]).to_arrow()
+    # use_lsm(False) is the escape hatch: it reads the base table.
+    base = table.take_row_ids([0, 1]).use_lsm(False).to_arrow()
+    assert base.num_rows == 2
+
+
+def test_lsm_read_fts_postfilter_errors(tmp_path):
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    table = db.create_table("t", _fts_reader([(1, "quick fox"), (2, "lazy dog")]))
+    table.set_unenforced_primary_key("id")
+    table.create_index("text", config=FTS())
+    index_name = table.list_indices()[0].name
+    table.set_lsm_write_spec(
+        LsmWriteSpec.unsharded().with_maintained_indexes([index_name])
+    )
+    # The LSM scanner always prefilters; postfilter on FTS changes result semantics,
+    # so it hard-errors (previously only the vector arm rejected it).
+    with pytest.raises(Exception, match="postfilter"):
+        (
+            table.search("fox", query_type="fts", fts_columns="text")
+            .where("id > 0")
+            .postfilter()
+            .to_arrow()
+        )
+
+
+def test_lsm_read_fts_multiple_same_type_indexes_errors(tmp_path):
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    table = db.create_table("t", _fts_reader([(1, "quick fox"), (2, "lazy dog")]))
+    table.set_unenforced_primary_key("id")
+    table.create_index("text", config=FTS(), name="fts_a")
+    table.create_index("text", config=FTS(), name="fts_b", replace=False)
+    table.set_lsm_write_spec(
+        LsmWriteSpec.unsharded().with_maintained_indexes(["fts_a"])
+    )
+    # Two FTS indexes on the column: the base planner's chosen index is ambiguous, so
+    # the scanner can't pick a catch-up watermark and rejects rather than risk
+    # dropping rows the actually-used index has not caught up to.
+    with pytest.raises(Exception, match="multiple"):
+        table.search("fox", query_type="fts", fts_columns="text").to_arrow()
+
+
+def test_lsm_read_vector_unmaintained_index_errors(tmp_path):
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    rows = [
+        (i, "a", [float((i * 7 + j) % 13) for j in range(VECTOR_DIM)])
+        for i in range(1, 61)
+    ]
+    table = db.create_table("t", _vector_reader(rows))
+    table.set_unenforced_primary_key("id")
+    table.create_index(
+        "vector", config=IvfPq(num_partitions=1, num_sub_vectors=2, num_bits=4)
+    )
+    # Spec with NO maintained indexes: the base vector index's catch-up is untracked,
+    # so the scanner rejects rather than risk dropping compacted-but-unindexed rows.
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    with pytest.raises(Exception, match="maintained"):
+        table.search([1.0] * VECTOR_DIM).to_arrow()
+
+
+def test_lsm_read_fts_optimized_index_not_rejected(tmp_path):
+    db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
+    table = db.create_table("t", _fts_reader([(i, "quick fox") for i in range(1, 6)]))
+    table.set_unenforced_primary_key("id")
+    table.create_index("text", config=FTS())
+    table.add(_fts_reader([(i, "lazy fox") for i in range(6, 11)]))
+    table.optimize()  # may split the FTS index into multiple physical segments
+    name = table.list_indices()[0].name
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([name]))
+    # Multiple physical segments of one logical index must not be miscounted as
+    # multiple indexes and rejected.
+    ids = set(
+        table.search("fox", query_type="fts", fts_columns="text")
+        .limit(20)
+        .to_arrow()["id"]
+        .to_pylist()
+    )
+    assert ids == set(range(1, 11))
@@ -1136,3 +1136,61 @@ def test_take_offsets_empty_permutation(some_permutation: Permutation):
    result = some_permutation.take_offsets([])

    assert result == []
+
+
+def test_select_rowid(some_permutation: Permutation):
+    """Test that _rowid can be selected alongside regular columns."""
+    perm_with_rowid = some_permutation.select_columns(["_rowid", "id"])
+    assert "_rowid" in perm_with_rowid.column_names
+    batches = list(perm_with_rowid.iter(100, skip_last_batch=False))
+    for batch in batches:
+        assert "_rowid" in batch[0]
+
+
+def test_select_rowid_only(some_permutation: Permutation):
+    """Test that _rowid can be selected as the sole column."""
+    perm_rowid_only = some_permutation.select_columns(["_rowid"])
+    assert perm_rowid_only.column_names == ["_rowid"]
+    batches = list(perm_rowid_only.iter(100, skip_last_batch=False))
+    assert len(batches) > 0
+    for batch in batches:
+        assert list(batch[0].keys()) == ["_rowid"]
+
+
+def test_select_rowid_not_in_default(some_permutation: Permutation):
+    """Test that _rowid is NOT in the default column_names or schema."""
+    assert "_rowid" not in some_permutation.column_names
+    assert "_rowid" not in some_permutation.schema.names
+
+
+def test_select_rowid_identity_permutation(mem_db):
+    """Test that _rowid works with an identity permutation."""
+    tbl = mem_db.create_table(
+        "test_rowid_identity", pa.table({"id": range(10), "value": range(10)})
+    )
+    perm = Permutation.identity(tbl)
+    perm_with_rowid = perm.select_columns(["_rowid", "id"])
+    batches = list(perm_with_rowid.iter(10, skip_last_batch=False))
+    assert len(batches) == 1
+    assert "_rowid" in batches[0][0]
+
+
+def test_rename_rowid(some_permutation: Permutation):
+    """Test that _rowid can be selected and then renamed."""
+    perm_with_rowid = some_permutation.select_columns(["_rowid", "id"])
+    renamed = perm_with_rowid.rename_column("_rowid", "my_row_id")
+    assert "my_row_id" in renamed.column_names
+    assert "_rowid" not in renamed.column_names
+    batches = list(renamed.iter(100, skip_last_batch=False))
+    for batch in batches:
+        assert "my_row_id" in batch[0]
+        assert "_rowid" not in batch[0]
+
+
+def test_remove_rowid_after_select(some_permutation: Permutation):
+    """Test that _rowid can be selected and then removed."""
+    perm_with_rowid = some_permutation.select_columns(["_rowid", "id"])
+    assert "_rowid" in perm_with_rowid.column_names
+    perm_without_rowid = perm_with_rowid.remove_columns(["_rowid"])
+    assert "_rowid" not in perm_without_rowid.column_names
+    assert perm_without_rowid.column_names == ["id"]
@@ -768,7 +768,10 @@ def test_table_create_indices():
        # Test create_fts_index with custom name (legacy method)
        with pytest.warns(DeprecationWarning, match="create_fts_index"):
            table.create_fts_index(
-                "text", wait_timeout=timedelta(seconds=2), name="custom_fts_idx"
+                "text",
+                wait_timeout=timedelta(seconds=2),
+                block_size=256,
+                name="custom_fts_idx",
            )

        # Test create_index with custom name (legacy form: vector_column_name kwarg)
@@ -791,6 +794,7 @@ def test_table_create_indices():
        fts_req = received_requests[1]
        assert "name" in fts_req
        assert fts_req["name"] == "custom_fts_idx"
+        assert fts_req["block_size"] == 256

        # Check vector index request has custom name
        vector_req = received_requests[2]
@@ -876,7 +880,7 @@ def test_remote_create_index_new_api():
            _warnings.simplefilter("error", DeprecationWarning)
            table.create_index("vector", config=IvfPq(distance_type="l2"))
            table.create_index("category", config=BTree())
-            table.create_index("text", config=FTS())
+            table.create_index("text", config=FTS(block_size=256))
            # IvfRq via new API
            table.create_index("vector", config=IvfRq(distance_type="l2"))

@@ -896,6 +900,7 @@ def test_remote_create_index_new_api():
            "vector",
            "vector",
        ]
+        assert received_requests[2]["block_size"] == 256


 def test_table_wait_for_index_timeout():
@@ -644,6 +644,21 @@ def test_cross_encoder_reranker_return_all(tmp_path):
    assert "_distance" in result.column_names


+def test_answerdotai_reranker_return_all(tmp_path):
+    pytest.importorskip("rerankers")
+    reranker = AnswerdotaiRerankers(return_score="all")
+    table, schema = get_test_table(tmp_path)
+    query = "single player experience"
+    result = (
+        table.search(query, query_type="hybrid", vector_column_name="vector")
+        .rerank(reranker=reranker)
+        .to_arrow()
+    )
+    assert "_relevance_score" in result.column_names
+    assert "_score" in result.column_names
+    assert "_distance" in result.column_names
+
+
 # ---------------------------------------------------------------------------
 # Regression tests for LinearCombinationReranker scoring bugs (issue #3154)
 # ---------------------------------------------------------------------------
@@ -60,6 +60,9 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                    .ngram_min_length(params.ngram_min_length)
                    .ngram_max_length(params.ngram_max_length)
                    .ngram_prefix_only(params.prefix_only);
+                let inner_opts = inner_opts
+                    .block_size(params.block_size)
+                    .map_err(|err| PyValueError::new_err(err.to_string()))?;
                Ok(LanceDbIndex::FTS(inner_opts))
            }
            "IvfFlat" => {
@@ -207,6 +210,7 @@ struct FtsParams {
    ngram_min_length: u32,
    ngram_max_length: u32,
    prefix_only: bool,
+    block_size: usize,
 }

 #[derive(FromPyObject)]
@@ -294,6 +294,7 @@ pub struct PyQueryRequest {
    pub select: PySelect,
    pub fast_search: Option<bool>,
    pub with_row_id: Option<bool>,
+    pub use_lsm: Option<bool>,
    pub column: Option<String>,
    pub query_vector: Option<PyQueryVectors>,
    pub minimum_nprobes: Option<usize>,
@@ -324,6 +325,7 @@ impl From<AnyQuery> for PyQueryRequest {
                select: PySelect(query_request.select),
                fast_search: Some(query_request.fast_search),
                with_row_id: Some(query_request.with_row_id),
+                use_lsm: query_request.use_lsm,
                column: None,
                query_vector: None,
                minimum_nprobes: None,
@@ -348,6 +350,7 @@ impl From<AnyQuery> for PyQueryRequest {
                select: PySelect(vector_query.base.select),
                fast_search: Some(vector_query.base.fast_search),
                with_row_id: Some(vector_query.base.with_row_id),
+                use_lsm: vector_query.base.use_lsm,
                column: vector_query.column,
                query_vector: Some(PyQueryVectors(vector_query.query_vector)),
                minimum_nprobes: Some(vector_query.minimum_nprobes),
@@ -474,6 +477,10 @@ impl Query {
        self.inner = self.inner.clone().fast_search();
    }

+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_lsm(enable);
+    }
+
    pub fn with_row_id(&mut self) {
        self.inner = self.inner.clone().with_row_id();
    }
@@ -636,6 +643,10 @@ impl TakeQuery {
        self.inner = self.inner.clone().with_row_id();
    }

+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_lsm(enable);
+    }
+
    #[pyo3(signature = ())]
    pub fn output_schema(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.inner.clone();
@@ -745,6 +756,10 @@ impl FTSQuery {
        self.inner = self.inner.clone().fast_search();
    }

+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_lsm(enable);
+    }
+
    pub fn with_row_id(&mut self) {
        self.inner = self.inner.clone().with_row_id();
    }
@@ -892,6 +907,10 @@ impl VectorQuery {
        self.inner = self.inner.clone().fast_search();
    }

+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_lsm(enable);
+    }
+
    pub fn with_row_id(&mut self) {
        self.inner = self.inner.clone().with_row_id();
    }
@@ -1086,6 +1105,11 @@ impl HybridQuery {
        self.inner_fts.postfilter();
    }

+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner_vec.use_lsm(enable);
+        self.inner_fts.use_lsm(enable);
+    }
+
    pub fn add_query_vector(&mut self, vector: Bound<'_, PyAny>) -> PyResult<()> {
        self.inner_vec.add_query_vector(vector)
    }
@@ -1236,8 +1236,8 @@ impl Table {
        if let Some(use_index) = parameters.use_index {
            builder.use_index(use_index);
        }
-        if let Some(use_lsm_write) = parameters.use_lsm_write {
-            builder.use_lsm_write(use_lsm_write);
+        if let Some(use_lsm) = parameters.use_lsm {
+            builder.use_lsm(use_lsm);
        }
        if let Some(validate_single_shard) = parameters.validate_single_shard {
            builder.validate_single_shard(validate_single_shard);
@@ -1475,7 +1475,7 @@ pub struct MergeInsertParams {
    when_not_matched_by_source_condition_expr: Option<PyExpr>,
    timeout: Option<std::time::Duration>,
    use_index: Option<bool>,
-    use_lsm_write: Option<bool>,
+    use_lsm: Option<bool>,
    validate_single_shard: Option<bool>,
 }

@@ -1,12 +1,17 @@
 # Release process

-There are five total packages we release. Four are the `lancedb` packages
-for Python, Rust, Java, and Node.js. The other one is the legacy `vectordb`
-package node.js.
+We release four `lancedb` packages: Python, Rust, Java, and Node.js.

-The Python package is versioned and released separately from the Rust, Java, and Node.js
-ones. For Node.js the release process is shared between `lancedb` and
-`vectordb` for now.
+All four share a single version number, defined by `current_version` in
+`.bumpversion.toml`. One `vX.Y.Z` tag releases all of them, so a breaking change
+in any SDK bumps the minor version for every SDK.
+
+> [!NOTE]
+> Python used to be versioned separately, under `python-vX.Y.Z` tags. It ran
+> three minor versions ahead of the other SDKs, which made the two numbers hard
+> to reason about. Both tracks were merged at `v0.37.0`: the Python line went
+> `0.36` → `0.37` as usual, while Rust, Java, and Node.js jumped `0.33` → `0.37`
+> to catch up. Tags before `v0.37.0` follow the old split scheme.

 ## Preview releases

@@ -27,20 +32,21 @@ The release process uses a handful of GitHub actions to automate the process.
  ┌─────────────────────┐
  │Create Release Commit│
  └─┬───────────────────┘
-    │                           ┌────────────┐ ┌──►Python GH Release
-    ├──►(tag) python-vX.Y.Z ───►│PyPI Publish├─┤
-    │                           └────────────┘ └──►Python Wheels
-    │
-    │                           ┌───────────┐
-    └──►(tag) vX.Y.Z ───┬──────►│NPM Publish├──┬──►Rust/Node GH Release
-                        │       └───────────┘  │
-                        │                      └──►NPM Packages
-                        │       ┌─────────────┐
-                        ├──────►│Cargo Publish├───►Cargo Release
-                        │       └─────────────┘
-                        │       ┌─────────────┐
-                        └──────►│Maven Publish├───►Java Maven Repo Release
-                                └─────────────┘
+    │                    ┌──────────────┐
+    └──►(tag) vX.Y.Z ─┬─►│GitHub Release├───►GH Release
+                      │  └──────────────┘
+                      │  ┌────────────┐
+                      ├─►│PyPI Publish├─────►Python Wheels
+                      │  └────────────┘
+                      │  ┌───────────┐
+                      ├─►│NPM Publish├──────►NPM Packages
+                      │  └───────────┘
+                      │  ┌─────────────┐
+                      ├─►│Cargo Publish├────►Cargo Release
+                      │  └─────────────┘
+                      │  ┌─────────────┐
+                      └─►│Maven Publish├────►Java Maven Repo Release
+                         └─────────────┘
 ```

 To start a release, trigger a `Create Release Commit` action from
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.32.0-beta.2"
+version = "0.37.0-beta.0"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -54,7 +54,26 @@ pub enum Index {
    /// substrings of the raw bytes, unlike the tokenized [`Index::FTS`] index.
    Fm(FmIndexBuilder),

-    /// Full text search index using bm25.
+    /// Full text search index using BM25.
+    ///
+    /// The posting block size defaults to 128. Supported values are 128 and 256;
+    /// a value of 256 uses the experimental FTS V3 format and may introduce
+    /// breaking changes.
+    ///
+    /// ```
+    /// use lancedb::index::{Index, scalar::FtsIndexBuilder};
+    ///
+    /// # async fn create_fts_index(
+    /// #     table: &lancedb::Table,
+    /// # ) -> Result<(), Box<dyn std::error::Error>> {
+    /// let params = FtsIndexBuilder::default().block_size(256)?;
+    /// table
+    ///     .create_index(&["text"], Index::FTS(params))
+    ///     .execute()
+    ///     .await?;
+    /// # Ok(())
+    /// # }
+    /// ```
    FTS(FtsIndexBuilder),

    /// IVF index
@@ -167,6 +167,11 @@
 //! # }
 //! ```

+// The MemWAL LSM read path (`table::query::lsm`) deepens the `create_plan` future's
+// type graph enough to overflow the default trait-recursion limit while evaluating
+// auto-traits (`Send`) through the Linux io_uring build's moka cache. Raise it.
+#![recursion_limit = "256"]
+
 pub mod arrow;
 pub mod blob;
 pub mod connection;
@@ -523,6 +523,26 @@ pub trait QueryBase {
    ///
    /// This allows ordering query results by one or more columns in either ascending or descending order.
    fn order_by(self, ordering: Option<Vec<ColumnOrdering>>) -> Self;
+
+    /// Control MemWAL read routing for this query.
+    ///
+    /// By default (unset), when the table carries a MemWAL write spec (see
+    /// [`crate::Table::set_lsm_write_spec`]), reads are routed through the LSM
+    /// scanner so they also return data written via the `merge_insert` LSM path
+    /// that has not yet been compacted into the base table (active/frozen
+    /// memtables and flushed generations); a table without a spec reads the base
+    /// table.
+    ///
+    /// - `use_lsm(true)` forces LSM routing and errors if the table has no
+    ///   MemWAL write spec.
+    /// - `use_lsm(false)` bypasses the MemWAL and reads the base table only,
+    ///   even when a spec is present.
+    ///
+    /// Note: the LSM scanner does not support every query shape (e.g. reranking,
+    /// hybrid search, `order_by`). On a MemWAL table those shapes error unless
+    /// `use_lsm(false)` is set, because a base-only read would silently
+    /// exclude un-compacted MemWAL data.
+    fn use_lsm(self, enable: bool) -> Self;
 }

 pub trait HasQuery {
@@ -593,6 +613,11 @@ impl<T: HasQuery> QueryBase for T {
        self.mut_query().order_by = ordering;
        self
    }
+
+    fn use_lsm(mut self, enable: bool) -> Self {
+        self.mut_query().use_lsm = Some(enable);
+        self
+    }
 }

 /// Options for controlling the execution of a query
@@ -844,6 +869,20 @@ pub struct QueryRequest {
    ///
    /// This allows ordering query results by one or more columns in either ascending or descending order.
    pub order_by: Option<Vec<ColumnOrdering>>,
+
+    /// Controls MemWAL read routing. When unset (the default), a query against a
+    /// table that carries a MemWAL write spec (see
+    /// [`crate::Table::set_lsm_write_spec`]) is routed through the LSM scanner so
+    /// it also sees data written via the `merge_insert` LSM path that has not yet
+    /// been compacted into the base table — the active and frozen in-memory
+    /// memtables and the flushed (L0) generations, deduplicated by primary key
+    /// against the base table (newest generation wins); a table without a spec
+    /// reads the base table.
+    ///
+    /// - `Some(true)` forces LSM routing and errors if the table has no MemWAL
+    ///   write spec.
+    /// - `Some(false)` reads only the base table, bypassing the MemWAL.
+    pub use_lsm: Option<bool>,
 }

 impl Default for QueryRequest {
@@ -862,6 +901,7 @@ impl Default for QueryRequest {
            norm: None,
            disable_scoring_autoprojection: false,
            order_by: None,
+            use_lsm: None,
        }
    }
 }
@@ -617,6 +617,11 @@ impl<S: HttpSend> RemoteTable<S> {
    ) -> Result<()> {
        params.check_filter()?;
        body["prefilter"] = params.prefilter.into();
+        // Only forward use_lsm when explicitly set; a server that predates it
+        // ignores the field and routes as it would by default.
+        if let Some(use_lsm) = params.use_lsm {
+            body["use_lsm"] = serde_json::Value::Bool(use_lsm);
+        }
        if let Some(offset) = params.offset {
            body["offset"] = serde_json::Value::Number(serde_json::Number::from(offset));
        }
@@ -2843,6 +2848,10 @@ struct MergeInsertRequest {
    // (the default is true)
    #[serde(skip_serializing_if = "is_true")]
    use_index: bool,
+    // Only serialize use_lsm when explicitly set (Some); a server that predates
+    // it ignores the field and routes as it would by default.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    use_lsm: Option<bool>,
 }

 fn is_true(b: &bool) -> bool {
@@ -2894,6 +2903,7 @@ impl TryFrom<MergeInsertBuilder> for MergeInsertRequest {
            when_not_matched_by_source_delete_filt,
            // Only serialize use_index when it's false for backwards compatibility
            use_index: value.use_index,
+            use_lsm: value.use_lsm,
        })
    }
 }
@@ -4496,6 +4506,15 @@ mod tests {
                serde_json::to_value(InvertedIndexParams::default()).unwrap(),
                Index::FTS(Default::default()),
            ),
+            (
+                "FTS",
+                {
+                    let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
+                    body["block_size"] = 256.into();
+                    body
+                },
+                Index::FTS(InvertedIndexParams::default().block_size(256).unwrap()),
+            ),
        ];

        for (index_type, expected_body, index) in cases {
@@ -382,7 +382,9 @@ mod tests {
    use crate::connect;
    use crate::connection::ConnectBuilder;
    use crate::index::Index;
-    use crate::index::scalar::{BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder};
+    use crate::index::scalar::{
+        BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, FtsIndexBuilder,
+    };
    use crate::index::vector::{
        IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder,
    };
@@ -1362,7 +1364,10 @@ mod tests {
            .unwrap();

        table
-            .create_index(&["text"], Index::FTS(Default::default()))
+            .create_index(
+                &["text"],
+                Index::FTS(FtsIndexBuilder::default().block_size(256).unwrap()),
+            )
            .execute()
            .await
            .unwrap();
@@ -1372,6 +1377,10 @@ mod tests {
        assert_eq!(index.index_type, crate::index::IndexType::FTS);
        assert_eq!(index.columns, vec!["text".to_string()]);
        assert_eq!(index.name, "text_idx");
+        assert_eq!(index.index_version, Some(3));
+        let index_params: FtsIndexBuilder =
+            serde_json::from_str(index.index_details.as_deref().unwrap()).unwrap();
+        assert_eq!(index_params.posting_block_size(), 256);

        let num_rows = 120;
        let stats = table.index_stats("text_idx").await.unwrap().unwrap();
@@ -73,7 +73,7 @@ pub struct MergeInsertBuilder {
    pub(crate) when_not_matched_by_source_delete_filt: Option<MergeFilter>,
    pub(crate) timeout: Option<Duration>,
    pub(crate) use_index: bool,
-    pub(crate) use_lsm_write: Option<bool>,
+    pub(crate) use_lsm: Option<bool>,
    pub(crate) validate_single_shard: bool,
 }

@@ -89,7 +89,7 @@ impl MergeInsertBuilder {
            when_not_matched_by_source_delete_filt: None,
            timeout: None,
            use_index: true,
-            use_lsm_write: None,
+            use_lsm: None,
            validate_single_shard: true,
        }
    }
@@ -187,16 +187,17 @@ impl MergeInsertBuilder {
        self
    }

-    /// Controls whether `merge_insert` uses the MemWAL LSM write path.
+    /// Control MemWAL routing for this `merge_insert`.
    ///
    /// By default (unset), a `merge_insert` on a table with an
-    /// [`LsmWriteSpec`](super::LsmWriteSpec) installed is routed through
-    /// Lance's MemWAL shard writer, and a table without one uses the standard
-    /// path. Calling this with `false` forces the standard path even when a
-    /// spec is set. Calling it with `true` requires a spec — `merge_insert`
-    /// errors if none is installed.
-    pub fn use_lsm_write(&mut self, use_lsm_write: bool) -> &mut Self {
-        self.use_lsm_write = Some(use_lsm_write);
+    /// [`LsmWriteSpec`](super::LsmWriteSpec) installed is routed through Lance's
+    /// MemWAL shard writer; a table without one uses the standard path.
+    ///
+    /// - `use_lsm(true)` forces MemWAL routing and errors if the table has no
+    ///   LSM write spec.
+    /// - `use_lsm(false)` forces the standard write path even when a spec is set.
+    pub fn use_lsm(&mut self, enable: bool) -> &mut Self {
+        self.use_lsm = Some(enable);
        self
    }

@@ -626,7 +627,7 @@ mod lsm_tests {
    }

    #[tokio::test]
-    async fn lsm_merge_insert_use_lsm_write_false_falls_back() {
+    async fn lsm_merge_insert_use_lsm_false_falls_back() {
        let dir = tempdir().unwrap();
        let table = id_value_table(&dir).await;
        table
@@ -634,9 +635,10 @@ mod lsm_tests {
            .await
            .unwrap();

-        // use_lsm_write(false) opts out: the standard path runs and commits.
+        // use_lsm(false) opts out: the standard path runs and commits even though
+        // a spec is installed.
        let mut builder = table.merge_insert(&["id"]);
-        builder.when_not_matched_insert_all().use_lsm_write(false);
+        builder.when_not_matched_insert_all().use_lsm(false);
        let result = builder
            .execute(id_value_reader(vec![3, 4, 5]))
            .await
@@ -646,6 +648,25 @@ mod lsm_tests {
        assert_eq!(table.count_rows(None).await.unwrap(), 5);
    }

+    #[tokio::test]
+    async fn lsm_merge_insert_use_lsm_true_without_spec_errors() {
+        let dir = tempdir().unwrap();
+        let table = id_value_table(&dir).await;
+
+        // use_lsm(true) demands MemWAL routing; without a write spec it errors
+        // rather than silently falling back to the standard path.
+        let mut builder = table.merge_insert(&["id"]);
+        builder
+            .when_matched_update_all(None)
+            .when_not_matched_insert_all()
+            .use_lsm(true);
+        let err = builder
+            .execute(id_value_reader(vec![3, 4, 5]))
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidInput { .. }), "got {err:?}");
+    }
+
    #[tokio::test]
    async fn lsm_merge_insert_rejects_on_not_primary_key() {
        let dir = tempdir().unwrap();
@@ -754,18 +775,23 @@ mod lsm_tests {
    }

    #[tokio::test]
-    async fn lsm_merge_insert_use_lsm_write_true_requires_spec() {
+    async fn lsm_merge_insert_no_spec_uses_standard_path() {
        let dir = tempdir().unwrap();
        // id_value_table sets a primary key but no LSM write spec.
        let table = id_value_table(&dir).await;

+        // Without a spec, a default merge_insert (use_lsm unset) simply uses
+        // the standard path and commits — no opt-out required, no error.
        let mut builder = table.merge_insert(&["id"]);
        builder
            .when_matched_update_all(None)
-            .when_not_matched_insert_all()
-            .use_lsm_write(true);
-        let err = builder.execute(id_value_reader(vec![4])).await.unwrap_err();
-        assert!(matches!(err, Error::InvalidInput { .. }), "got {err:?}");
+            .when_not_matched_insert_all();
+        let result = builder
+            .execute(id_value_reader(vec![3, 4, 5]))
+            .await
+            .unwrap();
+        assert_eq!(result.num_inserted_rows, 2);
+        assert_eq!(table.count_rows(None).await.unwrap(), 5);
    }

    #[tokio::test]
@@ -818,4 +844,387 @@ mod lsm_tests {
            .await
            .unwrap();
    }
+
+    // ---------------------------------------------------------------------
+    // LSM read path
+    // ---------------------------------------------------------------------
+
+    use crate::arrow::SendableRecordBatchStream;
+    use crate::query::{ExecutableQuery, QueryBase};
+    use arrow::array::AsArray;
+    use arrow::datatypes::Int64Type;
+    use futures::TryStreamExt;
+
+    /// Collect `(id, value)` pairs from a result stream, sorted by id.
+    async fn collect_id_value(stream: SendableRecordBatchStream) -> Vec<(i64, i64)> {
+        let batches: Vec<_> = stream.try_collect().await.unwrap();
+        let mut rows = Vec::new();
+        for batch in &batches {
+            let ids = batch
+                .column_by_name("id")
+                .unwrap()
+                .as_primitive::<Int64Type>();
+            let values = batch
+                .column_by_name("value")
+                .unwrap()
+                .as_primitive::<Int64Type>();
+            for i in 0..batch.num_rows() {
+                rows.push((ids.value(i), values.value(i)));
+            }
+        }
+        rows.sort();
+        rows
+    }
+
+    /// Upsert `ids` (value = 0..n) through the LSM `merge_insert` path.
+    async fn lsm_upsert(table: &Table, ids: Vec<i64>) {
+        let mut builder = table.merge_insert(&[]);
+        builder
+            .when_matched_update_all(None)
+            .when_not_matched_insert_all();
+        builder.execute(id_value_reader(ids)).await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn lsm_read_sees_active_memtable() {
+        let dir = tempdir().unwrap();
+        let table = id_value_table(&dir).await; // base: ids 1,2,3 (value 0,1,2)
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+
+        // Insert ids 4,5 into the active memtable (not committed to base).
+        lsm_upsert(&table, vec![4, 5]).await;
+
+        // Default read auto-routes through the LSM scanner: base ∪ active memtable.
+        let lsm = table.query().execute().await.unwrap();
+        let rows = collect_id_value(lsm).await;
+        assert_eq!(
+            rows.iter().map(|(id, _)| *id).collect::<Vec<_>>(),
+            vec![1, 2, 3, 4, 5]
+        );
+
+        // use_lsm(false) bypasses the MemWAL and reads the base table only.
+        let base_only = table.query().use_lsm(false).execute().await.unwrap();
+        let rows = collect_id_value(base_only).await;
+        assert_eq!(
+            rows.iter().map(|(id, _)| *id).collect::<Vec<_>>(),
+            vec![1, 2, 3]
+        );
+    }
+
+    #[tokio::test]
+    async fn lsm_read_dedup_newest_wins() {
+        let dir = tempdir().unwrap();
+        let table = id_value_table(&dir).await; // base: id 2 -> value 1
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+
+        // Upsert ids 2,3,4 with values 0,1,2. id 2 and 3 shadow the base rows.
+        lsm_upsert(&table, vec![2, 3, 4]).await;
+
+        let lsm = table.query().execute().await.unwrap();
+        let rows = collect_id_value(lsm).await;
+        // id 1 from base (value 0); ids 2,3,4 from memtable (values 0,1,2).
+        assert_eq!(rows, vec![(1, 0), (2, 0), (3, 1), (4, 2)]);
+    }
+
+    #[tokio::test]
+    async fn lsm_read_point_lookup_filter() {
+        let dir = tempdir().unwrap();
+        let table = id_value_table(&dir).await;
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+        lsm_upsert(&table, vec![2, 3, 4]).await; // id 2 -> value 0 (shadows base)
+
+        let lsm = table.query().only_if("id = 2").execute().await.unwrap();
+        let rows = collect_id_value(lsm).await;
+        assert_eq!(rows, vec![(2, 0)]);
+    }
+
+    #[tokio::test]
+    async fn lsm_read_multi_shard() {
+        let dir = tempdir().unwrap();
+        let table = id_value_table(&dir).await;
+        table
+            .set_lsm_write_spec(LsmWriteSpec::bucket("id", 8))
+            .await
+            .unwrap();
+
+        // Two single-row upserts that route to (likely) different buckets; each
+        // closes the writer so the next opens a fresh shard.
+        lsm_upsert(&table, vec![10]).await;
+        table.close_lsm_writers().await.unwrap();
+        lsm_upsert(&table, vec![11]).await;
+
+        let lsm = table.query().execute().await.unwrap();
+        let rows = collect_id_value(lsm).await;
+        let ids: Vec<i64> = rows.iter().map(|(id, _)| *id).collect();
+        // Base 1,2,3 + flushed/active shards for 10 and 11.
+        assert_eq!(ids, vec![1, 2, 3, 10, 11]);
+    }
+
+    #[tokio::test]
+    async fn lsm_read_after_close_sees_flushed() {
+        let dir = tempdir().unwrap();
+        let table = id_value_table(&dir).await;
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+        lsm_upsert(&table, vec![4, 5]).await;
+        // close flushes the active memtable to an on-disk generation and drops
+        // the cached writer; the read must still see those rows via the shard
+        // manifest snapshot.
+        table.close_lsm_writers().await.unwrap();
+
+        let lsm = table.query().execute().await.unwrap();
+        let ids: Vec<i64> = collect_id_value(lsm)
+            .await
+            .iter()
+            .map(|(id, _)| *id)
+            .collect();
+        assert_eq!(ids, vec![1, 2, 3, 4, 5]);
+    }
+
+    #[tokio::test]
+    async fn lsm_read_without_spec_reads_base() {
+        let dir = tempdir().unwrap();
+        let table = id_value_table(&dir).await; // no LSM write spec
+
+        // With no spec installed there is nothing to route: the default read and
+        // an explicit use_lsm(false) both read the base table without error.
+        for query in [table.query(), table.query().use_lsm(false)] {
+            let rows = collect_id_value(query.execute().await.unwrap()).await;
+            assert_eq!(
+                rows.iter().map(|(id, _)| *id).collect::<Vec<_>>(),
+                vec![1, 2, 3]
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn lsm_read_unsupported_shape_errors_without_use_lsm_false() {
+        let dir = tempdir().unwrap();
+        let table = id_value_table(&dir).await;
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+        lsm_upsert(&table, vec![4]).await;
+
+        // `with_row_id` is a shape the LSM scanner cannot honor. On a MemWAL
+        // table the default (auto-routed) read hard-errors rather than silently
+        // reading a stale base-only result that would exclude un-compacted row 4.
+        let err = table
+            .query()
+            .with_row_id()
+            .execute()
+            .await
+            .err()
+            .expect("unsupported shape on a MemWAL table must error");
+        assert!(matches!(err, Error::NotSupported { .. }), "got {err:?}");
+
+        // use_lsm(false) is the escape hatch: it reads the base table only.
+        let rows = collect_id_value(
+            table
+                .query()
+                .with_row_id()
+                .use_lsm(false)
+                .execute()
+                .await
+                .unwrap(),
+        )
+        .await;
+        assert_eq!(
+            rows.iter().map(|(id, _)| *id).collect::<Vec<_>>(),
+            vec![1, 2, 3]
+        );
+    }
+
+    /// A reader of `[id: Int64, text: Utf8]` rows.
+    fn id_text_reader(rows: Vec<(i64, &str)>) -> Box<dyn RecordBatchReader + Send> {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("text", DataType::Utf8, false),
+        ]));
+        let ids: Vec<i64> = rows.iter().map(|(id, _)| *id).collect();
+        let texts: Vec<&str> = rows.iter().map(|(_, t)| *t).collect();
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int64Array::from(ids)),
+                Arc::new(StringArray::from(texts)),
+            ],
+        )
+        .unwrap();
+        Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema))
+    }
+
+    #[tokio::test]
+    async fn lsm_read_full_text_search() {
+        use crate::index::Index;
+        use lance_index::scalar::FullTextSearchQuery;
+
+        let dir = tempdir().unwrap();
+        let conn = connect(dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+        let table = conn
+            .create_table(
+                "t",
+                id_text_reader(vec![(1, "alpha"), (2, "beta"), (3, "gamma")]),
+            )
+            .execute()
+            .await
+            .unwrap();
+        table.set_unenforced_primary_key(["id"]).await.unwrap();
+        table
+            .create_index(&["text"], Index::FTS(Default::default()))
+            .execute()
+            .await
+            .unwrap();
+        let fts_index = table.list_indices().await.unwrap()[0].name.clone();
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([fts_index]))
+            .await
+            .unwrap();
+
+        // Insert a row whose term ("zebra") exists in no base row.
+        let mut builder = table.merge_insert(&[]);
+        builder
+            .when_matched_update_all(None)
+            .when_not_matched_insert_all();
+        builder
+            .execute(id_text_reader(vec![(99, "zebra")]))
+            .await
+            .unwrap();
+
+        let search = |term: &str| {
+            let q = FullTextSearchQuery::new(term.to_string())
+                .with_column("text".to_string())
+                .unwrap();
+            table.query().full_text_search(q)
+        };
+
+        // "zebra" lives only in the active memtable; LSM read finds it.
+        let stream = search("zebra").execute().await.unwrap();
+        let batches: Vec<_> = stream.try_collect().await.unwrap();
+        let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(rows, 1, "LSM FTS must surface the memtable row");
+
+        // A base-only term still matches the base table through the LSM scan.
+        let stream = search("alpha").execute().await.unwrap();
+        let batches: Vec<_> = stream.try_collect().await.unwrap();
+        let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(rows, 1, "LSM FTS must still see base rows");
+    }
+
+    #[tokio::test]
+    async fn lsm_read_vector_search() {
+        use crate::index::Index;
+        use crate::index::vector::IvfPqIndexBuilder;
+        use arrow::array::{FixedSizeListBuilder, Float32Builder};
+        use arrow::datatypes::Int64Type;
+
+        const DIM: i32 = 8;
+        const N: i64 = 256;
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new(
+                "vec",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
+                false,
+            ),
+        ]));
+        let make_batch = |rows: Vec<(i64, f32)>| -> RecordBatch {
+            let ids: Vec<i64> = rows.iter().map(|(id, _)| *id).collect();
+            let mut vb = FixedSizeListBuilder::new(Float32Builder::new(), DIM);
+            for (_, fill) in &rows {
+                for _ in 0..DIM {
+                    vb.values().append_value(*fill);
+                }
+                vb.append(true);
+            }
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(Int64Array::from(ids)), Arc::new(vb.finish())],
+            )
+            .unwrap()
+        };
+
+        let dir = tempdir().unwrap();
+        let conn = connect(dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+        // Base rows fill each vector with its own id (0..256); all far from 1000.
+        let base = make_batch((0..N).map(|i| (i, i as f32)).collect());
+        let base_reader: Box<dyn RecordBatchReader + Send> =
+            Box::new(RecordBatchIterator::new(vec![Ok(base)], schema.clone()));
+        let table = conn.create_table("t", base_reader).execute().await.unwrap();
+        table.set_unenforced_primary_key(["id"]).await.unwrap();
+        table
+            .create_index(
+                &["vec"],
+                Index::IvfPq(
+                    IvfPqIndexBuilder::default()
+                        .num_partitions(1)
+                        .num_sub_vectors(2),
+                ),
+            )
+            .execute()
+            .await
+            .unwrap();
+        let vec_index = table.list_indices().await.unwrap()[0].name.clone();
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([vec_index]))
+            .await
+            .unwrap();
+
+        // Insert a vector (filled with 1000) that is nearest to the query.
+        let mut builder = table.merge_insert(&[]);
+        builder
+            .when_matched_update_all(None)
+            .when_not_matched_insert_all();
+        let insert_reader: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
+            vec![Ok(make_batch(vec![(9999, 1000.0)]))],
+            schema.clone(),
+        ));
+        builder.execute(insert_reader).await.unwrap();
+
+        // KNN near [1000; DIM]: the default (auto-routed) read surfaces the
+        // memtable row.
+        let stream = table
+            .query()
+            .nearest_to(&[1000.0_f32; 8])
+            .unwrap()
+            .limit(1)
+            .execute()
+            .await
+            .unwrap();
+        let batches: Vec<_> = stream.try_collect().await.unwrap();
+        let ids: Vec<i64> = batches
+            .iter()
+            .flat_map(|b| {
+                b.column_by_name("id")
+                    .unwrap()
+                    .as_primitive::<Int64Type>()
+                    .values()
+                    .to_vec()
+            })
+            .collect();
+        assert_eq!(
+            ids,
+            vec![9999],
+            "LSM vector search must rank the memtable row first"
+        );
+    }
 }
@@ -306,6 +306,39 @@ impl ShardWriterEntry {
        }
        Ok(())
    }
+
+    /// The cached writer's latest in-memory manifest (current generation +
+    /// flushed generations). `Ok(None)` if the writer was already closed.
+    /// Used by the LSM read path to snapshot this shard authoritatively
+    /// without re-reading the on-disk manifest.
+    async fn manifest(&self) -> Result<Option<lance_index::mem_wal::ShardManifest>> {
+        let guard = self.inner.read().await;
+        let Some(writer) = guard.as_ref() else {
+            return Ok(None);
+        };
+        writer.manifest().await.map_err(|e| Error::Runtime {
+            message: format!("read: shard writer manifest read failed: {}", e),
+        })
+    }
+
+    /// Atomically capture the cached writer's active + frozen-awaiting-flush
+    /// memtables for unified LSM scanning. `Ok(None)` if the writer was
+    /// already closed.
+    async fn in_memory_memtable_refs(
+        &self,
+    ) -> Result<Option<lance::dataset::mem_wal::scanner::InMemoryMemTables>> {
+        let guard = self.inner.read().await;
+        let Some(writer) = guard.as_ref() else {
+            return Ok(None);
+        };
+        writer
+            .in_memory_memtable_refs()
+            .await
+            .map(Some)
+            .map_err(|e| Error::Runtime {
+                message: format!("read: shard writer memtable capture failed: {}", e),
+            })
+    }
 }

 impl ShardWriterCache {
@@ -345,6 +378,36 @@ impl ShardWriterCache {
        Ok(entry)
    }

+    /// Snapshot the cached writer's shard for the LSM read path: its shard id,
+    /// authoritative in-memory manifest, and active + frozen memtable refs.
+    /// Returns `None` when no writer is currently cached (e.g. nothing has been
+    /// written this session, or the writer was closed).
+    #[allow(clippy::redundant_pub_crate)]
+    pub(crate) async fn read_snapshot(
+        &self,
+    ) -> Result<
+        Option<(
+            Uuid,
+            Option<lance_index::mem_wal::ShardManifest>,
+            Option<lance::dataset::mem_wal::scanner::InMemoryMemTables>,
+        )>,
+    > {
+        let cached = {
+            let guard = self.slot.read().await;
+            guard.as_ref().map(|(id, entry)| (*id, entry.clone()))
+        };
+        let Some((shard_id, entry)) = cached else {
+            return Ok(None);
+        };
+        // Capture memtables before the manifest. If a flush interleaves, dedup
+        // tolerates the same rows appearing in both a memtable and a freshly
+        // flushed generation, but would drop rows present in neither. Manifest
+        // last guarantees any generation flushed mid-capture is still covered.
+        let memtables = entry.in_memory_memtable_refs().await?;
+        let manifest = entry.manifest().await?;
+        Ok(Some((shard_id, manifest, memtables)))
+    }
+
    /// Close the cached writer, if any, and clear the slot.
    #[allow(clippy::redundant_pub_crate)]
    pub(crate) async fn drain_and_close(&self) -> Result<()> {
@@ -408,19 +471,20 @@ pub(crate) async fn lsm_dispatch_decision(
    table: &NativeTable,
    params: &MergeInsertBuilder,
 ) -> Result<LsmDispatch> {
-    // `Some(false)` is an explicit opt-out: use the standard path.
-    if params.use_lsm_write == Some(false) {
+    // Explicit opt-out: use the standard path regardless of any installed spec.
+    if params.use_lsm == Some(false) {
        return Ok(LsmDispatch::Standard);
    }

    let dataset = table.dataset.get().await?;
    let Some(details) = dataset.mem_wal_index_details().await? else {
-        // No LSM write spec installed. `Some(true)` explicitly asked for the
-        // LSM path, which is meaningless without a spec; `None` (the default)
-        // just falls back to the standard path.
-        if params.use_lsm_write == Some(true) {
+        // No write spec installed. `use_lsm(true)` demanded MemWAL routing, so
+        // that is an error; otherwise fall back to the standard path.
+        if params.use_lsm == Some(true) {
            return Err(Error::InvalidInput {
-                message: "merge_insert: use_lsm_write(true) requires an LSM write spec on the table; call set_lsm_write_spec first".to_string(),
+                message: "use_lsm(true) was set but the table has no MemWAL write spec; \
+                    install one with set_lsm_write_spec or leave use_lsm unset"
+                    .to_string(),
            });
        }
        return Ok(LsmDispatch::Standard);
@@ -449,7 +513,7 @@ pub(crate) async fn lsm_dispatch_decision(

    if !is_upsert_only(params) {
        return Err(Error::InvalidInput {
-            message: "merge_insert: when an LSM write spec is set, only the upsert form (when_matched_update_all without a filter + when_not_matched_insert_all, no by-source delete) is supported; call use_lsm_write(false) to use the standard merge_insert path".to_string(),
+            message: "merge_insert: when an LSM write spec is set, only the upsert form (when_matched_update_all without a filter + when_not_matched_insert_all, no by-source delete) is supported; call use_lsm(false) to use the standard merge_insert path".to_string(),
        });
    }

@@ -3,6 +3,8 @@

 use std::sync::Arc;

+mod lsm;
+
 use super::NativeTable;
 use crate::connection::NamespaceClientPushdownOperation;
 use crate::error::{Error, Result};
@@ -20,6 +22,7 @@ use datafusion_physical_plan::projection::ProjectionExec;
 use datafusion_physical_plan::repartition::RepartitionExec;
 use datafusion_physical_plan::union::UnionExec;
 use futures::future::try_join_all;
+use lance::dataset::mem_wal::DatasetMemWalExt;
 use lance::dataset::scanner::DatasetRecordBatchStream;
 use lance::dataset::scanner::Scanner;
 use lance_datafusion::exec::{analyze_plan as lance_analyze_plan, execute_plan};
@@ -53,7 +56,7 @@ pub async fn execute_query(
    // QueryTable pushdown runs the query server-side, but only on the main
    // branch: the namespace request carries no branch yet, so a branch handle
    // must fall through to local execution.
-    if can_execute_namespace_query(table, query)
+    if can_execute_namespace_query(table, query).await?
        && let Some(ref namespace_client) = table.namespace_client
    {
        return execute_namespace_query(table, namespace_client.clone(), query, options).await;
@@ -61,18 +64,35 @@ pub async fn execute_query(
    execute_generic_query(table, query, options).await
 }

-fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> bool {
-    table
+async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> Result<bool> {
+    if !(table
        .pushdown_operations
        .contains(&NamespaceClientPushdownOperation::QueryTable)
        && table.namespace_client.is_some()
        && table.dataset.current_branch().is_none()
-        && !requires_local_namespace_execution(query)
+        && !requires_local_namespace_execution(query))
+    {
+        return Ok(false);
+    }
+    // A MemWAL write spec means reads auto-route through the LSM scanner in
+    // `create_plan` even when `use_lsm` is unset. The namespace request has no
+    // use_lsm field, so pushing the default query down would silently omit
+    // un-compacted rows — force local execution whenever a spec is installed.
+    let dataset = table.dataset.get().await?;
+    if dataset.mem_wal_index_details().await?.is_some() {
+        return Ok(false);
+    }
+    Ok(true)
 }

 fn requires_local_namespace_execution(query: &AnyQuery) -> bool {
-    // The namespace QueryTable request has no approx_mode field yet, so
-    // pushing this query down would silently ignore the user's setting.
+    // The namespace QueryTable request has no approx_mode or use_lsm field yet, so
+    // pushing these down would silently ignore the user's setting. For use_lsm that
+    // is worse than a tuning miss: MemWAL read routing lives only in `create_plan`,
+    // so a pushed-down query would return stale base-only data with no error.
+    if query.base().use_lsm.is_some() {
+        return true;
+    }
    matches!(
        query,
        AnyQuery::VectorQuery(VectorQueryRequest {
@@ -120,6 +140,29 @@ pub async fn create_plan(
    query.base.check_filter()?;

    let ds_ref = table.dataset.get().await?;
+
+    // MemWAL read routing driven by `use_lsm`:
+    //   * unset  — route through the LSM scanner iff the table carries a write spec
+    //   * Some(true)  — force LSM routing; error if the table has no write spec
+    //   * Some(false) — read the base table only, bypassing the MemWAL
+    // The LSM scanner surfaces in-flight `merge_insert` data (active/frozen
+    // memtables + flushed generations); validation and dispatch live in `lsm`.
+    let has_spec = ds_ref.mem_wal_index_details().await?.is_some();
+    let use_lsm = match query.base.use_lsm {
+        Some(true) if !has_spec => {
+            return Err(Error::InvalidInput {
+                message: "use_lsm(true) was set but the table has no MemWAL write spec; \
+                    install one with set_lsm_write_spec or leave use_lsm unset"
+                    .to_string(),
+            });
+        }
+        Some(enable) => enable,
+        None => has_spec,
+    };
+    if use_lsm {
+        return lsm::create_lsm_plan(table, ds_ref, query).await;
+    }
+
    let schema = ds_ref.schema();
    let mut column = query.column.clone();

@@ -904,6 +947,65 @@ mod tests {
        assert_eq!(namespace_client.query_table_calls.load(Ordering::SeqCst), 0);
    }

+    #[tokio::test]
+    async fn test_execute_query_use_lsm_with_namespace_pushdown_runs_locally() {
+        use crate::connect;
+        use crate::table::query::execute_query;
+        use arrow_array::{Int32Array, RecordBatch};
+        use arrow_schema::{DataType, Field, Schema};
+
+        let conn = connect("memory://").execute().await.unwrap();
+
+        let vectors = Arc::new(fixed_size_list_array(
+            vec![0.0, 0.0, 10.0, 10.0, 20.0, 20.0],
+            2,
+        ));
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("vector", vectors.data_type().clone(), false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3])), vectors],
+        )
+        .unwrap();
+
+        let table = conn
+            .create_table("test_use_lsm_namespace_fallback", batch)
+            .execute()
+            .await
+            .unwrap();
+        let namespace_client = Arc::new(CountingNamespaceClient::default());
+        let mut native_table = table.as_native().unwrap().clone();
+        native_table.namespace_client = Some(namespace_client.clone());
+        native_table
+            .pushdown_operations
+            .insert(NamespaceClientPushdownOperation::QueryTable);
+
+        // `use_lsm` set (even to false) must force local execution — the namespace
+        // request has no use_lsm field, so a pushdown would silently ignore it.
+        let query_vector = Arc::new(Float32Array::from(vec![0.0, 0.0]));
+        let query = AnyQuery::VectorQuery(VectorQueryRequest {
+            base: QueryRequest {
+                limit: Some(1),
+                use_lsm: Some(false),
+                ..Default::default()
+            },
+            column: Some("vector".to_string()),
+            query_vector: vec![query_vector as ArrayRef],
+            ..Default::default()
+        });
+
+        let stream = execute_query(&native_table, &query, QueryExecutionOptions::default())
+            .await
+            .unwrap();
+        let batches = stream.try_collect::<Vec<_>>().await.unwrap();
+        let count: usize = batches.iter().map(|b| b.num_rows()).sum();
+
+        assert_eq!(count, 1);
+        assert_eq!(namespace_client.query_table_calls.load(Ordering::SeqCst), 0);
+    }
+
    #[tokio::test]
    async fn test_create_plan_multivector_structure() {
        use arrow_array::{Float32Array, RecordBatch};
@@ -0,0 +1,786 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! MemWAL LSM read path.
+//!
+//! When a table has an LSM write spec installed (see [`set_lsm_write_spec`]),
+//! reads are routed through Lance's [`LsmScanner`] instead of the plain
+//! base-table scan unless the query sets
+//! [`use_lsm(false)`](crate::query::QueryBase::use_lsm). This makes data
+//! written via the LSM `merge_insert` path — which lives in the active/frozen
+//! in-memory memtables and the flushed SSTable generations until an external
+//! compaction merges it into the base table — visible to queries, deduplicated by
+//! primary key (newest generation wins).
+//!
+//! Three query shapes are supported, mirroring the standard scan: a plain scan
+//! (filter / projection / limit), full-text search, and vector (ANN) search. All
+//! three run through a single [`LsmScanner`], so a `where` filter is honored as a
+//! prefilter uniformly — including for vector search, where `LsmScanner` threads
+//! it into the vector planner's prefilter. Shapes the LSM path cannot honor are
+//! rejected with [`Error::NotSupported`]; the caller must set `use_lsm(false)` to
+//! run those against the base table.
+//!
+//! [`set_lsm_write_spec`]: crate::Table::set_lsm_write_spec
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use arrow_array::Array;
+use arrow_schema::{DataType, Schema as ArrowSchema};
+use datafusion_physical_plan::expressions::Column;
+use datafusion_physical_plan::projection::ProjectionExec;
+use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr};
+use lance::Dataset;
+use lance::dataset::mem_wal::scanner::InMemoryMemTables;
+use lance::dataset::mem_wal::{
+    DatasetMemWalExt, LsmScanner, ShardManifestStore, ShardSnapshot, ShardWriterConfig,
+};
+use lance_index::mem_wal::{MemWalIndexDetails, ShardManifest};
+use uuid::Uuid;
+
+use super::NativeTable;
+use crate::DistanceType;
+use crate::error::{Error, Result};
+use crate::query::{DEFAULT_TOP_K, QueryFilter, Select, VectorQueryRequest};
+use crate::utils::default_vector_column;
+
+/// Over-fetch factor for the LSM vector/FTS arms. With the default of `1.0` a
+/// source blocked by cross-generation PK dedup fetches exactly `k` and can return
+/// fewer than `k` live rows; upserts routinely create such blocked candidates, so
+/// widen the per-source fetch to keep result pages filled.
+const LSM_OVERFETCH_FACTOR: f64 = 2.0;
+
+/// Build the LSM read plan for a MemWAL-routed query.
+///
+/// The caller guarantees `ds_ref` carries a MemWAL write spec (routing is decided
+/// in [`create_plan`](super::create_plan)). Errors with [`Error::NotSupported`]
+/// for query shapes the LSM scanner cannot honor — the caller must set
+/// `use_lsm(false)` to run those against the base table.
+pub(super) async fn create_lsm_plan(
+    table: &NativeTable,
+    ds_ref: Arc<Dataset>,
+    query: VectorQueryRequest,
+) -> Result<Arc<dyn ExecutionPlan>> {
+    reject_unsupported(&query)?;
+
+    // A time-traveled (checked-out) handle pins an older dataset version, but the
+    // WAL manifests and cached writer expose current live state — mixing them would
+    // surface WAL rows written after the requested version. `use_lsm(false)` reads
+    // the base table at the pinned version.
+    if table.dataset.time_travel_version().is_some() {
+        return Err(Error::NotSupported {
+            message: "the MemWAL LSM scanner cannot read from a time-traveled dataset version; set use_lsm(false) to read the base table at this version".to_string(),
+        });
+    }
+
+    // Routing guarantees a write spec is installed (see `create_plan`).
+    let details = ds_ref
+        .mem_wal_index_details()
+        .await?
+        .ok_or_else(|| Error::Runtime {
+            message: "the MemWAL LSM write spec disappeared during read planning".to_string(),
+        })?;
+
+    let pk_columns = pk_columns(&ds_ref)?;
+    // The base index an indexed arm relies on may lag compaction; resolve it so the
+    // snapshot retains SSTables the index has not yet caught up to.
+    let arm_index = arm_maintained_index_name(&ds_ref, &query, &details).await?;
+    let (snapshots, in_memory) =
+        build_read_context(table, &ds_ref, &details, arm_index.as_deref()).await?;
+
+    let limit = query.base.limit;
+    let offset = query.base.offset;
+
+    let plan = if !query.query_vector.is_empty() {
+        vector_plan(
+            &ds_ref,
+            &query,
+            &details,
+            pk_columns.clone(),
+            snapshots,
+            in_memory,
+            limit,
+            offset,
+        )
+        .await?
+    } else if let Some(fts) = &query.base.full_text_search {
+        fts_plan(
+            &ds_ref,
+            fts.clone(),
+            &query,
+            &details,
+            pk_columns.clone(),
+            snapshots,
+            in_memory,
+            limit,
+            offset,
+        )
+        .await?
+    } else {
+        plain_plan(
+            &ds_ref,
+            &query,
+            pk_columns.clone(),
+            snapshots,
+            in_memory,
+            limit,
+            offset,
+        )
+        .await?
+    };
+
+    // Lance appends the primary-key columns internally for dedup and keeps them in
+    // the output; drop the ones the user did not request so the projection matches.
+    restore_projection(plan, &query, &pk_columns)
+}
+
+/// Reject query shapes the LSM read path does not implement. On a MemWAL table
+/// reads route through the LSM scanner by default, so an unsupported shape is a
+/// hard error rather than a silent fallback to the base-only scan — which would
+/// exclude un-compacted MemWAL data. The caller must set `use_lsm(false)` to
+/// run these against the base table, accepting that the results omit un-compacted
+/// MemWAL data.
+///
+/// A `where` filter is intentionally *not* rejected: every arm routes through
+/// [`LsmScanner`], which applies it as a prefilter (see [`base_scanner`]).
+fn reject_unsupported(query: &VectorQueryRequest) -> Result<()> {
+    let unsupported = |what: &str| {
+        Err(Error::NotSupported {
+            message: format!(
+                "the MemWAL LSM scanner does not support {what}; set use_lsm(false) to read the base table only (results will exclude un-compacted MemWAL data)"
+            ),
+        })
+    };
+    if query.query_vector.len() > 1 {
+        return unsupported("multiple query vectors");
+    }
+    if !query.query_vector.is_empty() && query.base.full_text_search.is_some() {
+        return unsupported("hybrid (vector + full-text) search");
+    }
+    if query.base.with_row_id {
+        return unsupported("with_row_id (the LSM scanner exposes _rowaddr, not a stable _rowid)");
+    }
+    if query.base.reranker.is_some() {
+        return unsupported("reranking / hybrid search");
+    }
+    if query.base.order_by.is_some() {
+        return unsupported("order_by");
+    }
+    // Vector-only knobs the LSM scanner cannot honor. Both change results rather
+    // than just recall, so error instead of silently ignoring them: distance_range
+    // would return rows outside the bound, and use_index(false) asks for a
+    // brute-force search the index-only base arm can't do. (ef / approx_mode /
+    // maximum_nprobes are recall/speed knobs and are left to no-op — and
+    // maximum_nprobes defaults to Some, so it cannot be rejected on presence.)
+    if !query.query_vector.is_empty() {
+        if query.lower_bound.is_some() || query.upper_bound.is_some() {
+            return unsupported("distance_range on vector search");
+        }
+        if !query.use_index {
+            return unsupported(
+                "use_index(false) / brute-force vector search (the LSM base arm is index-only)",
+            );
+        }
+    }
+    // Postfilter changes result semantics for both vector and full-text search, and
+    // the LSM scanner always prefilters — reject a requested postfilter for either.
+    if (!query.query_vector.is_empty() || query.base.full_text_search.is_some())
+        && !query.base.prefilter
+    {
+        return unsupported(
+            "postfilter on vector or full-text search (the LSM scanner always prefilters)",
+        );
+    }
+    match &query.base.select {
+        Select::All | Select::Columns(_) => {}
+        Select::Dynamic(_) | Select::Expr(_) => return unsupported("dynamic column projection"),
+    }
+    if let Some(QueryFilter::Substrait(_)) = &query.base.filter {
+        return unsupported("Substrait filters");
+    }
+    // Take-by-row-id / row-offset queries carry a `_rowid` / `_rowoffset` filter,
+    // columns the LSM scanner never exposes (only `_rowaddr`); reject with guidance
+    // rather than failing deep in datafusion with a column-not-found error.
+    if let Some(QueryFilter::Datafusion(expr)) = &query.base.filter
+        && expr
+            .column_refs()
+            .iter()
+            .any(|c| c.name == "_rowid" || c.name == "_rowoffset")
+    {
+        return unsupported(
+            "take by row id or row offset (the LSM scanner has no stable _rowid / _rowoffset)",
+        );
+    }
+    Ok(())
+}
+
+/// Primary-key column names from the dataset's unenforced primary key.
+fn pk_columns(dataset: &Dataset) -> Result<Vec<String>> {
+    let pk: Vec<String> = dataset
+        .schema()
+        .unenforced_primary_key()
+        .iter()
+        .map(|f| f.name.clone())
+        .collect();
+    if pk.is_empty() {
+        return Err(Error::InvalidInput {
+            message:
+                "the MemWAL LSM scanner requires an unenforced primary key, but the table has none"
+                    .to_string(),
+        });
+    }
+    Ok(pk)
+}
+
+/// Per-shard SSTable exclusion watermark: the generation at or below which SSTables
+/// are safe to drop for this arm. A generation is droppable only once it is
+/// compacted into the base table AND covered by `index_name`'s catch-up (for an
+/// indexed arm); a plain scan (`index_name == None`) uses the compaction watermark
+/// alone. Capping at the index catch-up keeps rows the base index has not yet
+/// indexed visible through their SSTable. First occurrence per shard mirrors Lance's
+/// `compacted_generation_for_shard`.
+fn exclusion_watermarks(
+    details: &MemWalIndexDetails,
+    index_name: Option<&str>,
+) -> HashMap<Uuid, u64> {
+    let mut exclude: HashMap<Uuid, u64> = HashMap::new();
+    for entry in &details.compacted_sstables {
+        let mut watermark = entry.generation;
+        if let Some(name) = index_name
+            && let Some(caught_up) = details
+                .index_catchup
+                .iter()
+                .find(|icp| icp.index_name == name)
+                .and_then(|icp| icp.caught_up_generation_for_shard(&entry.shard_id))
+        {
+            watermark = watermark.min(caught_up);
+        }
+        exclude.entry(entry.shard_id).or_insert(watermark);
+    }
+    exclude
+}
+
+/// Assemble the per-shard snapshots (flushed SSTable generations) and the
+/// in-memory memtables (active + frozen) for the table.
+///
+/// Snapshots for all shards come from their on-disk manifests; for the shard
+/// with a live cached `ShardWriter` (this session's in-flight writes) the
+/// writer's authoritative in-memory manifest and memtables override the
+/// on-disk view so a read sees data not yet flushed.
+async fn build_read_context(
+    table: &NativeTable,
+    dataset: &Dataset,
+    details: &MemWalIndexDetails,
+    index_name: Option<&str>,
+) -> Result<(Vec<ShardSnapshot>, HashMap<Uuid, InMemoryMemTables>)> {
+    let exclude = exclusion_watermarks(details, index_name);
+
+    let shard_ids = dataset.list_mem_wal_latest_shard_ids().await?;
+    // Use the dataset's own object store (not `ObjectStore::from_uri`, which
+    // builds a fresh registry and would miss `memory://` and custom-registered
+    // stores). The base path matches `list_mem_wal_latest_shard_ids`.
+    let store = dataset.object_store(None).await?;
+    let base_path = dataset.branch_location().path;
+    let scan_batch_size = ShardWriterConfig::default().manifest_scan_batch_size;
+
+    let mut snapshots: Vec<ShardSnapshot> = Vec::new();
+    for shard_id in shard_ids {
+        let manifest_store =
+            ShardManifestStore::new(store.clone(), &base_path, shard_id, scan_batch_size);
+        if let Some(manifest) = manifest_store.read_latest().await? {
+            snapshots.push(snapshot_from_manifest(shard_id, &manifest, &exclude));
+        }
+    }
+
+    // WAL-only writers (enable_memtable=false) keep no in-memory memtable, and
+    // `in_memory_memtable_refs` errors in that mode; the on-disk manifests above
+    // already cover their flushed SSTables, so skip the live-writer snapshot. (Lance
+    // forbids maintained indexes in WAL-only mode, so only plain scans reach here.)
+    let wal_only = details
+        .writer_config_defaults
+        .get("enable_memtable")
+        .map(|v| v == "false")
+        .unwrap_or(false);
+
+    // Override the active shard with the cached writer's in-memory view.
+    let mut in_memory: HashMap<Uuid, InMemoryMemTables> = HashMap::new();
+    if !wal_only
+        && let Some((shard_id, manifest, memtables)) =
+            table.dataset.shard_writer().read_snapshot().await?
+    {
+        if let Some(manifest) = manifest {
+            let snapshot = snapshot_from_manifest(shard_id, &manifest, &exclude);
+            match snapshots.iter_mut().find(|s| s.shard_id == shard_id) {
+                Some(existing) => *existing = snapshot,
+                None => snapshots.push(snapshot),
+            }
+        }
+        if let Some(memtables) = memtables {
+            in_memory.insert(shard_id, memtables);
+        }
+    }
+
+    Ok((snapshots, in_memory))
+}
+
+/// Convert a shard manifest into a read snapshot (current + not-yet-compacted
+/// flushed SSTables). SSTable generations at or below the shard's compaction
+/// watermark are already in the base table and are skipped.
+fn snapshot_from_manifest(
+    shard_id: Uuid,
+    manifest: &ShardManifest,
+    compacted: &HashMap<Uuid, u64>,
+) -> ShardSnapshot {
+    let mut snapshot = ShardSnapshot::new(shard_id)
+        .with_spec_id(manifest.shard_spec_id)
+        .with_current_generation(manifest.current_generation);
+    let watermark = compacted.get(&shard_id).copied();
+    for sstable in &manifest.sstables {
+        if watermark.is_some_and(|w| sstable.generation <= w) {
+            continue;
+        }
+        snapshot = snapshot.with_sstable(sstable.generation, sstable.path.clone());
+    }
+    snapshot
+}
+
+/// Columns selected by the query, if an explicit projection was requested.
+fn selected_columns(query: &VectorQueryRequest) -> Option<Vec<String>> {
+    match &query.base.select {
+        Select::Columns(columns) => Some(columns.clone()),
+        _ => None,
+    }
+}
+
+/// Non-negative `Option<usize>` limit/offset as the `Option<i64>` the scanner
+/// expects.
+fn as_i64(value: Option<usize>) -> Option<i64> {
+    value.map(|v| v as i64)
+}
+
+/// Build a base `LsmScanner` configured with sources, filter, and projection.
+///
+/// The filter set here is applied as a prefilter across every arm — plain scan,
+/// full-text search, and vector search — since all three terminate on this
+/// scanner's `create_plan`.
+fn base_scanner(
+    dataset: &Dataset,
+    query: &VectorQueryRequest,
+    pk_columns: Vec<String>,
+    snapshots: Vec<ShardSnapshot>,
+    in_memory: HashMap<Uuid, InMemoryMemTables>,
+) -> Result<LsmScanner> {
+    let mut scanner = LsmScanner::new(Arc::new(dataset.clone()), snapshots, pk_columns);
+    for (shard_id, memtables) in in_memory {
+        scanner = scanner.with_in_memory_memtables(shard_id, memtables);
+    }
+    if let Some(columns) = selected_columns(query) {
+        let refs: Vec<&str> = columns.iter().map(String::as_str).collect();
+        scanner = scanner.project(&refs)?;
+    }
+    if let Some(filter) = &query.base.filter {
+        scanner = match filter {
+            QueryFilter::Sql(sql) => scanner.filter(sql)?,
+            QueryFilter::Datafusion(expr) => scanner.filter_expr(expr.clone()),
+            QueryFilter::Substrait(_) => {
+                return Err(Error::NotSupported {
+                    message: "the MemWAL LSM scanner does not support Substrait filters; set use_lsm(false) to read the base table only".to_string(),
+                });
+            }
+        };
+    }
+    Ok(scanner)
+}
+
+/// Plain scan: filter / projection / limit over base ∪ SSTables ∪ in-memory.
+/// The plain scan applies limit and offset inside the planner.
+async fn plain_plan(
+    dataset: &Dataset,
+    query: &VectorQueryRequest,
+    pk_columns: Vec<String>,
+    snapshots: Vec<ShardSnapshot>,
+    in_memory: HashMap<Uuid, InMemoryMemTables>,
+    limit: Option<usize>,
+    offset: Option<usize>,
+) -> Result<Arc<dyn ExecutionPlan>> {
+    let scanner = base_scanner(dataset, query, pk_columns, snapshots, in_memory)?
+        .limit(as_i64(limit), as_i64(offset))?;
+    Ok(scanner.create_plan().await?)
+}
+
+/// Full-text search over base ∪ SSTables ∪ in-memory, merged by local BM25 score.
+/// The scanner threads the query filter in as a prefilter and pages via limit/offset.
+#[allow(clippy::too_many_arguments)]
+async fn fts_plan(
+    dataset: &Dataset,
+    fts: lance_index::scalar::FullTextSearchQuery,
+    query: &VectorQueryRequest,
+    details: &MemWalIndexDetails,
+    pk_columns: Vec<String>,
+    snapshots: Vec<ShardSnapshot>,
+    in_memory: HashMap<Uuid, InMemoryMemTables>,
+    limit: Option<usize>,
+    offset: Option<usize>,
+) -> Result<Arc<dyn ExecutionPlan>> {
+    // Pre-check for a lancedb-flavored error; `LsmScanner` also validates the
+    // single-column requirement, but without the `use_lsm(false)` guidance.
+    let columns: Vec<String> = fts.columns().into_iter().collect();
+    if columns.len() > 1 {
+        return Err(Error::NotSupported {
+            message: "the MemWAL LSM scanner full-text search supports a single column; set use_lsm(false) to read the base table only".to_string(),
+        });
+    }
+    let column = columns.first().ok_or_else(|| Error::NotSupported {
+        message: "the MemWAL LSM scanner full-text search requires an explicit FTS column"
+            .to_string(),
+    })?;
+
+    // Without a maintained in-memory FTS index for this column, the active memtable
+    // arm produces an empty plan (`active_source_can_execute_fts` returns false), so
+    // the search silently omits un-compacted documents. Reject rather than mislead.
+    if !index_maintained(
+        dataset,
+        column,
+        &details.maintained_indexes,
+        "InvertedIndexDetails",
+    )
+    .await?
+    {
+        return Err(Error::NotSupported {
+            message: format!(
+                "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise un-compacted documents are omitted. set use_lsm(false) to read the base table only"
+            ),
+        });
+    }
+
+    let scanner = base_scanner(dataset, query, pk_columns, snapshots, in_memory)?
+        .with_overfetch_factor(LSM_OVERFETCH_FACTOR)
+        .full_text_search(fts)?
+        .limit(as_i64(limit), as_i64(offset))?;
+    Ok(scanner.create_plan().await?)
+}
+
+/// Whether an index of `type_url_suffix` covering `column` is in the MemWAL spec's
+/// maintained set. Only a maintained index has its catch-up tracked (so exclusion is
+/// gated correctly) and its in-memory arm kept current; an unmaintained base index
+/// falls back to the compaction watermark and can drop rows it has not re-indexed.
+/// The type must match specifically — a maintained BTree on the same column is not
+/// the FTS/vector index the arm relies on.
+async fn index_maintained(
+    dataset: &Dataset,
+    column: &str,
+    maintained: &[String],
+    type_url_suffix: &str,
+) -> Result<bool> {
+    use lance::index::DatasetIndexExt;
+    let Some(field) = dataset.schema().field(column) else {
+        return Ok(false);
+    };
+    let indices = dataset.load_indices().await?;
+    Ok(indices.iter().any(|idx| {
+        idx.fields.contains(&field.id)
+            && maintained.iter().any(|m| m == &idx.name)
+            && idx
+                .index_details
+                .as_ref()
+                .is_some_and(|d| d.type_url.ends_with(type_url_suffix))
+    }))
+}
+
+/// The maintained base index the query's arm relies on (vector index for ANN, FTS
+/// index for full-text), used to gate SSTable compaction exclusion by index catch-up.
+/// `None` for a plain scan or when no maintained index covers the searched column.
+async fn arm_maintained_index_name(
+    dataset: &Dataset,
+    query: &VectorQueryRequest,
+    details: &MemWalIndexDetails,
+) -> Result<Option<String>> {
+    use lance::index::DatasetIndexExt;
+    // Resolve the arm's searched column, the index-detail type it relies on, and a
+    // label for diagnostics — catch-up is taken from the vector/FTS index
+    // specifically, not a BTree on the same column.
+    let (column, type_url_suffix, arm) = if !query.query_vector.is_empty() {
+        let arrow_schema = ArrowSchema::from(dataset.schema());
+        let column = match &query.column {
+            Some(column) => column.clone(),
+            None => {
+                let dim = query.query_vector.first().map(|v| v.len() as i32);
+                default_vector_column(&arrow_schema, dim)?
+            }
+        };
+        (column, "VectorIndexDetails", "vector")
+    } else if let Some(fts) = &query.base.full_text_search {
+        match fts.columns().into_iter().next() {
+            Some(column) => (column, "InvertedIndexDetails", "full-text"),
+            None => return Ok(None),
+        }
+    } else {
+        return Ok(None);
+    };
+    let Some(field) = dataset.schema().field(&column) else {
+        return Ok(None);
+    };
+    let indices = dataset.load_indices().await?;
+    let segment_names: Vec<String> = indices
+        .iter()
+        .filter(|idx| {
+            idx.fields.contains(&field.id)
+                && idx
+                    .index_details
+                    .as_ref()
+                    .is_some_and(|d| d.type_url.ends_with(type_url_suffix))
+        })
+        .map(|idx| idx.name.clone())
+        .collect();
+    resolve_single_index(segment_names, &details.maintained_indexes, arm, &column)
+}
+
+/// Resolve the single logical index from the names of its matching physical
+/// segments. `load_indices` returns one entry per segment, so one logical index can
+/// appear multiple times (same name); dedupe by name before counting. Errors when
+/// more than one *distinct* index covers the field — the base planner's choice is
+/// ambiguous and their catch-up watermarks can diverge, so gating exclusion on the
+/// wrong one could drop SSTables the used index has not caught up to. Otherwise
+/// returns the name only when it is maintained (else the caller falls back to the
+/// compaction watermark).
+fn resolve_single_index(
+    mut names: Vec<String>,
+    maintained: &[String],
+    arm: &str,
+    column: &str,
+) -> Result<Option<String>> {
+    names.sort();
+    names.dedup();
+    if names.len() > 1 {
+        return Err(Error::NotSupported {
+            message: format!(
+                "the MemWAL LSM scanner cannot resolve the {arm} index catch-up watermark for '{column}': it has multiple {arm} indexes; set use_lsm(false) to read the base table only"
+            ),
+        });
+    }
+    Ok(names
+        .into_iter()
+        .next()
+        .filter(|name| maintained.contains(name)))
+}
+
+/// Drop the primary-key columns Lance appends internally for dedup when the user's
+/// explicit projection did not request them, restoring the requested output schema.
+/// `Select::All` legitimately includes the pk columns and is left untouched.
+fn restore_projection(
+    plan: Arc<dyn ExecutionPlan>,
+    query: &VectorQueryRequest,
+    pk_columns: &[String],
+) -> Result<Arc<dyn ExecutionPlan>> {
+    let Select::Columns(selected) = &query.base.select else {
+        return Ok(plan);
+    };
+    let schema = plan.schema();
+    // Keep a column unless it is a pk column the user did not select (this preserves
+    // user columns and score columns like `_distance`, dropping only leaked pk).
+    let keep: Vec<(Arc<dyn PhysicalExpr>, String)> = schema
+        .fields()
+        .iter()
+        .enumerate()
+        .filter(|(_, f)| {
+            selected.iter().any(|c| c == f.name()) || !pk_columns.iter().any(|pk| pk == f.name())
+        })
+        .map(|(i, f)| {
+            (
+                Arc::new(Column::new(f.name(), i)) as Arc<dyn PhysicalExpr>,
+                f.name().clone(),
+            )
+        })
+        .collect();
+    if keep.len() == schema.fields().len() {
+        return Ok(plan);
+    }
+    Ok(Arc::new(ProjectionExec::try_new(keep, plan)?))
+}
+
+/// Vector (ANN) search over base ∪ SSTables ∪ in-memory, routed through the same
+/// [`LsmScanner`] as the other arms so the query filter is applied as a prefilter.
+///
+/// Note: the base and SSTable arms use `fast_search` (indexed data only), so a
+/// base-table row not covered by a vector index is invisible here — it surfaces
+/// only via the memtable or `use_lsm(false)`.
+#[allow(clippy::too_many_arguments)]
+async fn vector_plan(
+    dataset: &Dataset,
+    query: &VectorQueryRequest,
+    details: &MemWalIndexDetails,
+    pk_columns: Vec<String>,
+    snapshots: Vec<ShardSnapshot>,
+    in_memory: HashMap<Uuid, InMemoryMemTables>,
+    limit: Option<usize>,
+    offset: Option<usize>,
+) -> Result<Arc<dyn ExecutionPlan>> {
+    let query_vector = query
+        .query_vector
+        .first()
+        .cloned()
+        .ok_or_else(|| Error::InvalidInput {
+            message: "vector search requires a query vector".to_string(),
+        })?;
+
+    let arrow_schema = ArrowSchema::from(dataset.schema());
+    let column = match &query.column {
+        Some(column) => column.clone(),
+        None => default_vector_column(&arrow_schema, Some(query_vector.len() as i32))?,
+    };
+
+    // The base arm relies on the column's vector index (`fast_search`). Unless it is
+    // maintained, its catch-up is untracked and exclusion falls back to the
+    // compaction watermark — dropping compacted SSTables the (lagging) base index has
+    // not re-indexed. Reject rather than silently omit rows, mirroring the FTS arm.
+    if !index_maintained(
+        dataset,
+        &column,
+        &details.maintained_indexes,
+        "VectorIndexDetails",
+    )
+    .await?
+    {
+        return Err(Error::NotSupported {
+            message: format!(
+                "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only"
+            ),
+        });
+    }
+
+    // The LSM vector planner is Float32-only; reject binary (uint8) vectors with a
+    // clear error rather than failing deep in the planner.
+    if is_binary_vector_column(&arrow_schema, &column) {
+        return Err(Error::NotSupported {
+            message: "the MemWAL LSM scanner does not support binary (uint8) vector search; set use_lsm(false) to read the base table only".to_string(),
+        });
+    }
+
+    let distance_type = resolve_distance_type(dataset, query, &column).await?;
+
+    // `nearest` takes a flat query vector and builds the fixed-size list itself.
+    // Guard `k` to at least 1 so a degenerate `limit(0)` is trimmed by `limit`
+    // below rather than rejected by `nearest`.
+    let k = limit.unwrap_or(DEFAULT_TOP_K).max(1);
+    let mut scanner = base_scanner(dataset, query, pk_columns, snapshots, in_memory)?
+        .with_overfetch_factor(LSM_OVERFETCH_FACTOR)
+        .nearest(&column, query_vector.as_ref(), k)?
+        .nprobes(query.minimum_nprobes)
+        .distance_metric(distance_type.into());
+    if let Some(refine_factor) = query.refine_factor {
+        scanner = scanner.refine(refine_factor);
+    }
+    scanner = scanner.limit(as_i64(limit), as_i64(offset))?;
+    Ok(scanner.create_plan().await?)
+}
+
+/// Whether `column` stores binary (uint8) vectors, which the LSM vector planner
+/// does not support.
+fn is_binary_vector_column(schema: &ArrowSchema, column: &str) -> bool {
+    matches!(
+        schema.field_with_name(column).map(|f| f.data_type()),
+        Ok(DataType::FixedSizeList(field, _)) if matches!(field.data_type(), DataType::UInt8)
+    )
+}
+
+/// Resolve the distance metric for the vector arm: the explicit query metric if
+/// set, else the metric of the column's vector index, else L2.
+async fn resolve_distance_type(
+    dataset: &Dataset,
+    query: &VectorQueryRequest,
+    column: &str,
+) -> Result<DistanceType> {
+    if let Some(dt) = query.distance_type {
+        return Ok(dt);
+    }
+    // Inherit the column's vector-index metric so cross-source distances match
+    // the metric the maintained memtable index was built with.
+    use lance::index::{DatasetIndexExt, DatasetIndexInternalExt};
+    use lance_index::metrics::NoOpMetricsCollector;
+    let field = dataset.schema().field(column);
+    if let Some(field) = field {
+        let indices = dataset.load_indices().await?;
+        for index in indices.iter() {
+            if index.fields.contains(&field.id)
+                && let Ok(vector_index) = dataset
+                    .open_vector_index(column, &index.uuid, &NoOpMetricsCollector)
+                    .await
+            {
+                return Ok(vector_index.metric_type().into());
+            }
+        }
+    }
+    Ok(DistanceType::L2)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use lance_index::mem_wal::{CompactedSsTable, IndexCatchupProgress};
+
+    #[test]
+    fn exclusion_watermark_gates_on_lagging_index_catchup() {
+        let shard = Uuid::from_u128(1);
+        let details = MemWalIndexDetails {
+            // Compaction has drained generations through 5 into the base table...
+            compacted_sstables: vec![CompactedSsTable::new(shard, 5)],
+            // ...but the FTS index has only caught up through generation 2.
+            index_catchup: vec![IndexCatchupProgress::new(
+                "fts_idx".to_string(),
+                vec![CompactedSsTable::new(shard, 2)],
+            )],
+            maintained_indexes: vec!["fts_idx".to_string()],
+            ..Default::default()
+        };
+
+        // Plain scan: drop every compacted generation (through 5).
+        assert_eq!(exclusion_watermarks(&details, None).get(&shard), Some(&5));
+
+        // FTS arm with a lagging index: exclusion is capped at the index catch-up
+        // (2), so SSTable generations 3..=5 are retained until the index covers
+        // them — otherwise those documents would silently vanish from FTS results.
+        assert_eq!(
+            exclusion_watermarks(&details, Some("fts_idx")).get(&shard),
+            Some(&2)
+        );
+
+        // A caught-up index — or one untracked in index_catchup — falls back to the
+        // compaction watermark.
+        assert_eq!(
+            exclusion_watermarks(&details, Some("caught_up_idx")).get(&shard),
+            Some(&5)
+        );
+    }
+
+    #[test]
+    fn resolve_single_index_dedupes_segments() {
+        let maintained = vec!["fts_idx".to_string()];
+        // Two physical segments of ONE logical index must not count as "multiple".
+        assert_eq!(
+            resolve_single_index(
+                vec!["fts_idx".to_string(), "fts_idx".to_string()],
+                &maintained,
+                "full-text",
+                "text"
+            )
+            .unwrap(),
+            Some("fts_idx".to_string())
+        );
+        // Two distinct indexes on the field are ambiguous → error.
+        assert!(
+            resolve_single_index(
+                vec!["fts_a".to_string(), "fts_b".to_string()],
+                &maintained,
+                "full-text",
+                "text"
+            )
+            .is_err()
+        );
+        // A single unmaintained index resolves to None (compaction-watermark fallback).
+        assert_eq!(
+            resolve_single_index(vec!["other".to_string()], &maintained, "full-text", "text")
+                .unwrap(),
+            None
+        );
+    }
+}