refactor(table): shrink LsmStats to measured per-bucket fields

`get_lsm_stats` reported four table-level rollups, three per-bucket
derived fields, and two pieces of in-process state. All of it is either
arithmetic the caller can do on the per-bucket list (`bucket_count`,
`generations_total`, `l0_bytes_total`, `memtable_rows_total`, `l0_bytes`,
`wal_lag`) or a value that is true only for the microsecond it was read
(`fenced`, `compaction_in_progress`). Drop them.

What is left is measured: manifest metadata, one entry per flushed L0
generation, and one entry per in-memory memtable. Three shape changes
follow:

* `active_memtable` + `frozen_memtables` collapse into `memtables`,
  ordered oldest first with the active one last. Absent — not empty — for
  a `Sealed` bucket, whose writer is torn down.
* Generation `rows` is unconditional, so `LsmStatsOptions` and its
  `include_generation_rows` flag are gone. A row count that had to be
  asked for was routinely absent from the one report that needed it.
* `memtable_indexes` becomes a list of names on each memtable, since each
  carries its own index store and a frozen one predates an index created
  after it froze.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

python/python/lancedb/_lancedb.pyi

@@ -292,7 +292,7 @@ class Table:
     ) -> dict: ...
     async def flush_lsm(self) -> dict: ...
     async def compact_lsm(self, max_generations_per_bucket: Optional[int]) -> dict: ...
-    async def get_lsm_stats(self, include_generation_rows: bool) -> Optional[dict]: ...
+    async def get_lsm_stats(self) -> Optional[dict]: ...
     async def close_lsm_writers(self) -> None: ...
     @property
     def tags(self) -> Tags: ...

python/python/lancedb/table.py

@@ -3903,12 +3903,10 @@ class LanceTable(Table):
             )
         )
 
-    def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]:
+    def get_lsm_stats(self) -> Optional[dict]:
         """Synchronous version of
         [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
-        return LOOP.run(
-            self._table.get_lsm_stats(include_generation_rows=include_generation_rows)
-        )
+        return LOOP.run(self._table.get_lsm_stats())
 
     def close_lsm_writers(self) -> None:
         """Close cached MemWAL shard writers. See
@@ -4676,10 +4674,8 @@ class AsyncTable:
         """
         return await self._inner.compact_lsm(max_generations_per_bucket)
 
-    async def get_lsm_stats(
-        self, *, include_generation_rows: bool = False
-    ) -> Optional[dict]:
-        """Read live LSM state: per-bucket detail plus a table-level aggregate.
+    async def get_lsm_stats(self) -> Optional[dict]:
+        """Read live per-bucket LSM state.
 
         Answers "how far behind is my fresh tier", "which bucket is hot", and
         "why is my fresh-tier vector search brute-force". Mutates no table
@@ -4688,14 +4684,8 @@ class AsyncTable:
 
         Returns ``None`` — and only — when the LSM write path is not enabled
         for the table.
-
-        Parameters
-        ----------
-        include_generation_rows
-            Report row counts per L0 generation. Off by default: each costs
-            one Lance manifest read.
         """
-        return await self._inner.get_lsm_stats(include_generation_rows)
+        return await self._inner.get_lsm_stats()
 
     async def close_lsm_writers(self) -> None:
         """Drain and close any cached MemWAL shard writers for this table.

python/src/table.rs

@@ -57,58 +57,30 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult
             "wal_entry_position_last_seen",
             b.wal_entry_position_last_seen,
         )?;
-        e.set_item("wal_lag", b.wal_lag)?;
-        e.set_item("l0_bytes", b.l0_bytes)?;
-        e.set_item("fenced", b.fenced)?;
-        e.set_item("compaction_in_progress", b.compaction_in_progress)?;
 
         let generations = PyList::empty(py);
         for g in &b.generations {
             let ge = PyDict::new(py);
             ge.set_item("generation", g.generation)?;
-            ge.set_item("path", &g.path)?;
             ge.set_item("bytes", g.bytes)?;
             ge.set_item("rows", g.rows)?;
             generations.append(ge)?;
         }
         e.set_item("generations", generations)?;
 
-        let memtable = |m: &lancedb::table::MemtableStats| -> PyResult<Py<PyDict>> {
-            let d = PyDict::new(py);
-            d.set_item("generation", m.generation)?;
-            d.set_item("rows", m.rows)?;
-            d.set_item("bytes", m.bytes)?;
-            d.set_item("batches", m.batches)?;
-            Ok(d.unbind())
-        };
         e.set_item(
-            "active_memtable",
-            b.active_memtable.as_ref().map(memtable).transpose()?,
-        )?;
-        e.set_item(
-            "frozen_memtables",
-            b.frozen_memtables
+            "memtables",
+            b.memtables
                 .as_ref()
-                .map(|fs| {
+                .map(|ms| {
                     let l = PyList::empty(py);
-                    for m in fs {
-                        l.append(memtable(m)?)?;
-                    }
-                    PyResult::Ok(l.unbind())
-                })
-                .transpose()?,
-        )?;
-        e.set_item(
-            "memtable_indexes",
-            b.memtable_indexes
-                .as_ref()
-                .map(|ixs| {
-                    let l = PyList::empty(py);
-                    for i in ixs {
+                    for m in ms {
                         let d = PyDict::new(py);
-                        d.set_item("name", &i.name)?;
-                        d.set_item("kind", &i.kind)?;
-                        d.set_item("column", &i.column)?;
+                        d.set_item("generation", m.generation)?;
+                        d.set_item("rows", m.rows)?;
+                        d.set_item("bytes", m.bytes)?;
+                        d.set_item("batches", m.batches)?;
+                        d.set_item("indexes", m.indexes.clone())?;
                         l.append(d)?;
                     }
                     PyResult::Ok(l.unbind())
@@ -118,10 +90,6 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult
         buckets.append(e)?;
     }
     out.set_item("buckets", buckets)?;
-    out.set_item("bucket_count", stats.bucket_count)?;
-    out.set_item("generations_total", stats.generations_total)?;
-    out.set_item("l0_bytes_total", stats.l0_bytes_total)?;
-    out.set_item("memtable_rows_total", stats.memtable_rows_total)?;
     Ok(out.unbind())
 }
 
@@ -1473,17 +1441,10 @@ impl Table {
     }
 
     /// Live LSM state, or `None` when the LSM write path is not enabled.
-    #[pyo3(signature = (include_generation_rows=false))]
-    pub fn get_lsm_stats(
-        self_: PyRef<'_, Self>,
-        include_generation_rows: bool,
-    ) -> PyResult<Bound<'_, PyAny>> {
+    pub fn get_lsm_stats(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
-        let opts = lancedb::table::LsmStatsOptions {
-            include_generation_rows,
-        };
         future_into_py(self_.py(), async move {
-            let stats = inner.get_lsm_stats(opts).await.infer_error()?;
+            let stats = inner.get_lsm_stats().await.infer_error()?;
             Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose())
         })
     }