feat(query): add use_lsm to read MemWAL LSM data (#3489)

## What

MemWAL LSM **read** support. When a table has an LSM write spec
(`set_lsm_write_spec`), `merge_insert` upserts live in the MemWAL
active/frozen memtables and flushed SSTables until an external
compaction merges them into the base table, so a normal scan returns
**stale** data. This routes reads through Lance's `LsmScanner` so
queries also surface that in-flight data, deduplicated by primary key
(newest generation wins).

## How

- Adds a **`use_lsm: Option<bool>`** query flag, symmetric with the
`merge_insert` flag:
- **unset** — auto-route through the LSM scanner when the table carries
a write spec
- **`use_lsm(true)`** — force the LSM path; error if there is no spec
    - **`use_lsm(false)`** — read the base table only (the escape hatch)
- Plain scan, single-column full-text search, and single-vector ANN all
run through one `LsmScanner` (assembled from on-disk shard manifests
plus the cached writer's in-memory memtables), so a `where` predicate is
honored as a **prefilter** uniformly — including for vector search.
- **Compaction-aware snapshots:** an SSTable generation is dropped only
once it is both compacted into the base table and covered by the arm's
base-index catch-up (`index_catchup`); plain scans use the compaction
watermark alone.
- Query shapes the scanner cannot honor hard-error with guidance to set
`use_lsm(false)`: hybrid, multi/binary vectors, `with_row_id`,
reranking, `order_by`, dynamic/Substrait projection or filters,
`distance_range`, `use_index(false)`, postfilter, take-by-row-id/offset,
reads from a time-traveled version, and an unmaintained or ambiguous
(multiple) FTS/vector index. Namespace-pushdown queries fall back to
local execution when a spec is present; WAL-only writers are handled.
- Exposed across the Rust core and the Python (`use_lsm`) and TypeScript
(`useLsm`) bindings, including `TakeQuery`.

Rebased from Lance `7.2.0-beta.3` to `10.0.0-beta.3`.

nodejs/__test__/table.test.ts

@@ -527,6 +527,14 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       );
     });
 
+    it("should expose useLsm on takeRowIds as the base-only escape hatch", async () => {
+      await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
+      // useLsm(false) is reachable on TakeQuery (the escape hatch for MemWAL tables,
+      // where take-by-row-id auto-routes to the LSM scanner and is rejected).
+      const res = await table.takeRowIds([0, 2]).useLsm(false).toArray();
+      expect(res.map((r) => r.id)).toEqual([1, 3]);
+    });
+
     it("should throw for negative number in takeRowIds", () => {
       expect(() => table.takeRowIds([-1])).toThrow("Row id cannot be negative");
       expect(() => table.takeRowIds([0, -5, 2])).toThrow(
@@ -3199,14 +3207,14 @@ describe("LSM merge insert", () => {
     await table.closeLsmWriters();
   });
 
-  it("falls back to the standard path with useLsmWrite(false)", async () => {
+  it("falls back to the standard path with useLsm(false)", async () => {
     const conn = await connect(tmpDir.name);
     const table = await bucketTable(conn);
 
     const res = await table
       .mergeInsert("id")
       .whenNotMatchedInsertAll()
-      .useLsmWrite(false)
+      .useLsm(false)
       .execute([
         { id: "b", value: 9 },
         { id: "e", value: 5 },
@@ -3240,4 +3248,36 @@ describe("LSM merge insert", () => {
         .execute([{ id: "g", value: 7 }]),
     ).rejects.toThrow();
   });
+
+  it("auto-routes reads through the MemWAL scanner", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await bucketTable(conn); // base ids "a", "b"
+
+    await table
+      .mergeInsert("id")
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .execute([{ id: "c", value: 3 }]);
+
+    // Default read auto-routes and includes the active memtable row.
+    const lsm = await table.query().toArray();
+    expect(lsm.map((r) => r.id).sort()).toEqual(["a", "b", "c"]);
+
+    // useLsm(false) bypasses the MemWAL and reads the base table only.
+    const baseOnly = await table.query().useLsm(false).toArray();
+    expect(baseOnly.map((r) => r.id).sort()).toEqual(["a", "b"]);
+  });
+
+  it("reads the base table when no LSM spec is installed", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await conn.createEmptyTable(
+      "plain",
+      new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
+    );
+    // No spec: default read and useLsm(false) both succeed against the base table.
+    await expect(table.query().toArray()).resolves.toBeDefined();
+    await expect(table.query().useLsm(false).toArray()).resolves.toBeDefined();
+    // useLsm(true) demands MemWAL routing; without a spec it errors.
+    await expect(table.query().useLsm(true).toArray()).rejects.toThrow();
+  });
 });

nodejs/lancedb/merge.ts

@@ -88,21 +88,17 @@ export class MergeInsertBuilder {
     );
   }
   /**
-   * Controls whether the merge uses the MemWAL LSM write path.
+   * Control MemWAL routing for this merge.
    *
    * By default (unset), a `mergeInsert` on a table with an LSM write spec is
-   * routed through Lance's MemWAL shard writer, and a table without one uses
-   * the standard path. Pass `false` to force the standard path even when a
-   * spec is set. Pass `true` to require a spec — `mergeInsert` rejects if none
-   * is installed.
+   * routed through Lance's MemWAL shard writer, and a table without one uses the
+   * standard path.
    *
-   * @param useLsmWrite - Whether to use the LSM write path.
+   * @param enable - `true` forces MemWAL routing and errors if the table has no
+   * LSM write spec. `false` forces the standard write path even when a spec is set.
    */
-  useLsmWrite(useLsmWrite: boolean): MergeInsertBuilder {
-    return new MergeInsertBuilder(
-      this.#native.useLsmWrite(useLsmWrite),
-      this.#schema,
-    );
+  useLsm(enable: boolean): MergeInsertBuilder {
+    return new MergeInsertBuilder(this.#native.useLsm(enable), this.#schema);
   }
   /**
    * Controls how an LSM merge checks that its input targets a single shard.

nodejs/lancedb/query.ts

@@ -460,6 +460,30 @@ export class StandardQueryBase<
     this.doCall((inner: NativeQueryType) => inner.fastSearch());
     return this;
   }
+
+  /**
+   * Control MemWAL read routing for this query.
+   *
+   * By default (unset), when the table carries a MemWAL write spec (see
+   * {@link Table#setLsmWriteSpec}), reads are routed through the LSM scanner so
+   * they also return data written via the `mergeInsert` LSM path that has not yet
+   * been compacted into the base table (the active/frozen in-memory memtables and
+   * the flushed generations), deduplicated by primary key; a table without a spec
+   * reads the base table.
+   *
+   * @param enable - `true` forces the LSM scanner and errors if the table has no
+   * MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
+   * even when a spec is present.
+   *
+   * Note: the LSM scanner does not support every query shape (e.g. reranking,
+   * hybrid search, `orderBy`). On a MemWAL table those shapes error unless
+   * `useLsm(false)` is set, because a base-only read would silently exclude
+   * un-compacted MemWAL data.
+   */
+  useLsm(enable: boolean): this {
+    this.doCall((inner: NativeQueryType) => inner.useLsm(enable));
+    return this;
+  }
 }
 
 /**
@@ -748,6 +772,20 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
   constructor(inner: NativeTakeQuery) {
     super(inner);
   }
+
+  /**
+   * Control MemWAL read routing for this take query.
+   *
+   * `false` bypasses the MemWAL and reads the base table only — the escape hatch,
+   * since take-by-row-id/offset is not supported on the LSM scanner and, on a
+   * MemWAL table, auto-routes to it and errors otherwise.
+   *
+   * @param enable - `false` reads the base table only.
+   */
+  useLsm(enable: boolean): this {
+    this.doCall((inner: NativeTakeQuery) => inner.useLsm(enable));
+    return this;
+  }
 }
 
 /** A builder for LanceDB queries.

nodejs/src/merge.rs

@@ -51,9 +51,9 @@ impl NativeMergeInsertBuilder {
     }
 
     #[napi]
-    pub fn use_lsm_write(&self, use_lsm_write: bool) -> Self {
+    pub fn use_lsm(&self, enable: bool) -> Self {
         let mut this = self.clone();
-        this.inner.use_lsm_write(use_lsm_write);
+        this.inner.use_lsm(enable);
         this
     }
 

nodejs/src/query.rs

@@ -168,6 +168,11 @@ impl Query {
         self.inner = self.inner.clone().with_row_id();
     }
 
+    #[napi]
+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_lsm(enable);
+    }
+
     #[napi]
     pub fn order_by(&mut self, ordering: Option<Vec<ColumnOrdering>>) -> napi::Result<()> {
         let ordering = ordering.map(|ordering| {
@@ -374,6 +379,11 @@ impl VectorQuery {
         self.inner = self.inner.clone().with_row_id();
     }
 
+    #[napi]
+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_lsm(enable);
+    }
+
     #[napi]
     pub fn rerank(
         &mut self,
@@ -479,6 +489,11 @@ impl TakeQuery {
         self.inner = self.inner.clone().with_row_id();
     }
 
+    #[napi]
+    pub fn use_lsm(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_lsm(enable);
+    }
+
     #[napi(catch_unwind)]
     pub async fn output_schema(&self) -> napi::Result<Buffer> {
         let schema = self.inner.output_schema().await.default_error()?;