feat: bring the MemWAL LSM surface to parity across the SDKs

Four of the eight LSM methods are remote-only in the core: `impl BaseTable
for NativeTable` implements only set/unset/get_lsm_write_spec and
close_lsm_writers, while flush_lsm, compact_lsm and get_lsm_stats fall
through to trait defaults returning NotSupported. That is why Node had
bound the four that work locally and stopped, and why the remaining four
had no binding-level coverage anywhere.

Node: add napi bindings for flush_lsm, compact_lsm, checkpoint_lsm and
get_lsm_stats, with typed LsmStats/BucketStats/GenerationStats/
MemtableStats objects mirroring the existing LsmWriteSpec object in the
same file. Tests assert each binding reaches the core and surfaces
NotSupported locally; behavior against a real endpoint stays covered by
the mocked-endpoint tests in rust/lancedb/src/remote/table.rs.

Python: LsmWriteSpec was importable only from the private lancedb._lancedb
-- it appeared in table.py solely under `if TYPE_CHECKING:`. Export it as
lancedb.LsmWriteSpec, add it to __all__, and list it in the API reference,
which had no mention of it and so rendered it nowhere.

Java: add the LSM routes to lancedb-core. Java reaches LanceDB purely over
REST through the generated namespace client, and these routes are not in
the Lance Namespace spec, so they are issued through a small dedicated
client. LsmWriteSpec is deliberately not org.lance.memwal.
InitializeMemWalParams: that type defaults to maintaining no indexes where
a spec here defaults to maintaining every index, and it cannot express the
null that asks the server to resolve the set. checkpointLsm is ported from
rust/lancedb/src/table/checkpoint.rs with its constants and status
semantics intact -- 429/503 retried in place, 421 restarting from flush.

Note: `mvnw spotless:apply` cannot run on JDK 21 (google-java-format 1.7,
pinned in java/pom.xml, predates JDK 16's compiler API change). This is
pre-existing and reproduces on a pristine main checkout; the Java sources
here were formatted by hand to the checkstyle rules.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

nodejs/__test__/table.test.ts

@@ -3341,6 +3341,59 @@ describe("LSM merge insert", () => {
   });
 });
 
+describe("LSM convergence and stats", () => {
+  let tmpDir: tmp.DirResult;
+
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  async function lsmTable(conn: Connection): Promise<Table> {
+    const table = await conn.createEmptyTable(
+      "t",
+      new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
+    );
+    await table.setUnenforcedPrimaryKey("id");
+    await table.setLsmWriteSpec({ specType: "unsharded" });
+    return table;
+  }
+
+  // These four route through the server that owns the MemWAL, so a local table
+  // rejects them rather than answering. What is asserted here is that the
+  // bindings reach the core at all; the behavior against a real endpoint is
+  // covered by the mocked endpoint tests in rust/lancedb/src/remote/table.rs.
+  it("rejects flushLsm on a local table", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await lsmTable(conn);
+
+    await expect(table.flushLsm()).rejects.toThrow(/not supported/i);
+  });
+
+  it("rejects compactLsm on a local table", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await lsmTable(conn);
+
+    await expect(table.compactLsm()).rejects.toThrow(/not supported/i);
+  });
+
+  it("rejects getLsmStats on a local table", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await lsmTable(conn);
+
+    await expect(table.getLsmStats()).rejects.toThrow(/not supported/i);
+    await expect(table.getLsmStats(true)).rejects.toThrow(/not supported/i);
+  });
+
+  it("rejects checkpointLsm on a local table", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await lsmTable(conn);
+
+    // checkpointLsm seals first, so it surfaces flushLsm's rejection.
+    await expect(table.checkpointLsm()).rejects.toThrow(/not supported/i);
+  });
+});
+
 describe("computed columns", () => {
   let tmpDir: tmp.DirResult;
   beforeEach(() => {

nodejs/lancedb/index.ts

@@ -147,6 +147,10 @@ export {
   FtsToken,
   TokenizeTableOptions,
   LsmWriteSpec,
+  LsmStats,
+  BucketStats,
+  GenerationStats,
+  MemtableStats,
   ColumnAlteration,
   FieldMetadataUpdate,
 } from "./table";

nodejs/lancedb/table.ts

@@ -31,6 +31,7 @@ import {
   IndexConfig,
   IndexStatistics,
   Job,
+  LsmStats,
   Branches as NativeBranches,
   OptimizeStats,
   RefreshColumnResult,
@@ -50,6 +51,12 @@ import {
 import { sanitizeType } from "./sanitize";
 import { IntoSql, toSQL } from "./util";
 export { IndexConfig } from "./native";
+export {
+  BucketStats,
+  GenerationStats,
+  LsmStats,
+  MemtableStats,
+} from "./native";
 
 /**
  * Progress snapshot for a write operation, delivered to the `progress`
@@ -706,6 +713,59 @@ export abstract class Table {
    * @returns {Promise<void>}
    */
   abstract closeLsmWriters(): Promise<void>;
+  /**
+   * Seal every bucket's active memtable into a new L0 generation.
+   *
+   * Returns once the seal is committed. Sealing an empty memtable is a no-op,
+   * so this is safe to call repeatedly.
+   * @returns {Promise<void>}
+   */
+  abstract flushLsm(): Promise<void>;
+  /**
+   * Trigger a background L0 → base compaction pass per bucket.
+   *
+   * Returns once the passes are *dispatched*, not once they finish — watch
+   * {@link Table#getLsmStats} for progress, or use
+   * {@link Table#checkpointLsm} to wait for convergence.
+   * @returns {Promise<void>}
+   */
+  abstract compactLsm(): Promise<void>;
+  /**
+   * Converge this table's LSM write path into its base table.
+   *
+   * Seals once, then triggers compaction and polls until the L0 that existed
+   * at the start is gone. The target set is fixed at the start, so
+   * generations created *during* the checkpoint are ignored — that is what
+   * lets it terminate under write load, and what makes it best-effort: it
+   * converges the fresh tier as of some instant. Idempotent, abandonable at
+   * any point, and safe to run on a cadence.
+   *
+   * There is no liveness bound — the compactor pool is shared across tables,
+   * so a checkpoint queued behind unrelated work looks exactly like one that
+   * is merging. The caller owns the deadline.
+   * @returns {Promise<void>}
+   * @example
+   * ```ts
+   * const before = await table.getLsmStats();
+   * await table.checkpointLsm();
+   * const after = await table.getLsmStats();
+   * ```
+   */
+  abstract checkpointLsm(): Promise<void>;
+  /**
+   * Read live per-bucket LSM state.
+   *
+   * Answers "how far behind is my fresh tier", "which bucket is hot", and
+   * "why is my fresh-tier vector search brute-force". Mutates no table state.
+   *
+   * Resolves to `undefined` only when the LSM write path is not enabled.
+   * @param {boolean} includeGenerationRows Also count rows per L0 generation.
+   *   Off by default because each count opens an uncached Lance dataset.
+   * @returns {Promise<LsmStats | undefined>}
+   */
+  abstract getLsmStats(
+    includeGenerationRows?: boolean,
+  ): Promise<LsmStats | undefined>;
   /** Retrieve the version of the table */
 
   abstract version(): Promise<number>;
@@ -1266,6 +1326,24 @@ export class LocalTable extends Table {
     return await this.inner.closeLsmWriters();
   }
 
+  async flushLsm(): Promise<void> {
+    return await this.inner.flushLsm();
+  }
+
+  async compactLsm(): Promise<void> {
+    return await this.inner.compactLsm();
+  }
+
+  async checkpointLsm(): Promise<void> {
+    return await this.inner.checkpointLsm();
+  }
+
+  async getLsmStats(
+    includeGenerationRows: boolean = false,
+  ): Promise<LsmStats | undefined> {
+    return (await this.inner.getLsmStats(includeGenerationRows)) ?? undefined;
+  }
+
   async version(): Promise<number> {
     return await this.inner.version();
   }

nodejs/src/table.rs

@@ -497,6 +497,34 @@ impl Table {
         self.inner_ref()?.close_lsm_writers().await.default_error()
     }
 
+    #[napi(catch_unwind)]
+    pub async fn flush_lsm(&self) -> napi::Result<()> {
+        self.inner_ref()?.flush_lsm().await.default_error()
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn compact_lsm(&self) -> napi::Result<()> {
+        self.inner_ref()?.compact_lsm().await.default_error()
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn checkpoint_lsm(&self) -> napi::Result<()> {
+        self.inner_ref()?.checkpoint_lsm().await.default_error()
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn get_lsm_stats(
+        &self,
+        include_generation_rows: bool,
+    ) -> napi::Result<Option<LsmStats>> {
+        let stats = self
+            .inner_ref()?
+            .get_lsm_stats(include_generation_rows)
+            .await
+            .default_error()?;
+        Ok(stats.map(LsmStats::from))
+    }
+
     #[napi(catch_unwind)]
     pub async fn version(&self) -> napi::Result<i64> {
         self.inner_ref()?
@@ -889,6 +917,129 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
     }
 }
 
+/// One flushed L0 generation.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct GenerationStats {
+    /// The generation number. Increases as memtables are sealed into L0.
+    pub generation: i64,
+    /// On-disk size of the generation.
+    pub bytes: i64,
+    /// Present only when `includeGenerationRows` was requested. Off by default
+    /// because each count opens an uncached Lance dataset.
+    pub rows: Option<i64>,
+}
+
+impl From<lancedb::table::GenerationStats> for GenerationStats {
+    fn from(g: lancedb::table::GenerationStats) -> Self {
+        Self {
+            generation: g.generation as i64,
+            bytes: g.bytes as i64,
+            rows: g.rows.map(|r| r as i64),
+        }
+    }
+}
+
+/// One in-memory memtable.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct MemtableStats {
+    /// The generation this memtable will become once sealed.
+    pub generation: i64,
+    /// Rows currently buffered.
+    pub rows: i64,
+    /// Estimated in-memory size.
+    pub bytes: i64,
+    /// Record batches currently buffered.
+    pub batches: i64,
+    /// Names of the indexes this memtable carries. An absent name is the whole
+    /// answer to "why is my fresh-tier search on that column brute-force".
+    pub indexes: Vec<String>,
+}
+
+impl From<lancedb::table::MemtableStats> for MemtableStats {
+    fn from(m: lancedb::table::MemtableStats) -> Self {
+        Self {
+            generation: m.generation as i64,
+            rows: m.rows as i64,
+            bytes: m.bytes as i64,
+            batches: m.batches as i64,
+            indexes: m.indexes,
+        }
+    }
+}
+
+/// Live state of one bucket. A table is N buckets on one node; flattening to a
+/// single number hides the one hot bucket that is usually why someone opened
+/// this endpoint.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct BucketStats {
+    /// The shard this bucket writes.
+    pub shard_id: String,
+    /// `"Active"` or `"Sealed"` (drop-table 2PC in flight).
+    pub status: String,
+    /// Epoch of the writer that currently owns the shard.
+    pub writer_epoch: i64,
+    /// Version of the shard manifest these numbers were read from.
+    pub manifest_version: i64,
+    /// The generation the active memtable will become.
+    pub current_generation: i64,
+    /// WAL position replay resumes from.
+    pub replay_after_wal_entry_position: i64,
+    /// Highest WAL position the writer has seen. The difference against
+    /// `replayAfterWalEntryPosition` is the WAL lag.
+    pub wal_entry_position_last_seen: i64,
+    /// Flushed L0 generations not yet merged into the base table.
+    pub generations: Vec<GenerationStats>,
+    /// Whether a pass owns this bucket's compaction latch right now. Says *a*
+    /// driver is running, not *whose*, and the latch is held from dispatch —
+    /// including while the pass queues for a pod-wide compactor permit. Read it
+    /// as "do not pile on", never as "mine is progressing".
+    pub compacting: bool,
+    /// Oldest first, active last. Absent for a `"Sealed"` bucket, whose
+    /// in-memory state is torn down.
+    pub memtables: Option<Vec<MemtableStats>>,
+}
+
+impl From<lancedb::table::BucketStats> for BucketStats {
+    fn from(b: lancedb::table::BucketStats) -> Self {
+        Self {
+            shard_id: b.shard_id,
+            status: b.status,
+            writer_epoch: b.writer_epoch as i64,
+            manifest_version: b.manifest_version as i64,
+            current_generation: b.current_generation as i64,
+            replay_after_wal_entry_position: b.replay_after_wal_entry_position as i64,
+            wal_entry_position_last_seen: b.wal_entry_position_last_seen as i64,
+            generations: b.generations.into_iter().map(Into::into).collect(),
+            compacting: b.compacting,
+            memtables: b
+                .memtables
+                .map(|ms| ms.into_iter().map(Into::into).collect()),
+        }
+    }
+}
+
+/// Live per-bucket LSM state, as returned by `Table#getLsmStats`.
+///
+/// Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
+/// the caller's to compute.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct LsmStats {
+    /// One entry per bucket backing this table.
+    pub buckets: Vec<BucketStats>,
+}
+
+impl From<lancedb::table::LsmStats> for LsmStats {
+    fn from(stats: lancedb::table::LsmStats) -> Self {
+        Self {
+            buckets: stats.buckets.into_iter().map(Into::into).collect(),
+        }
+    }
+}
+
 /// Statistics about a compaction operation.
 #[napi(object)]
 #[derive(Clone, Debug)]