mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-15 00:02:26 +00:00
refactor(lsm): rename LSM stats to SSTable and table shard
Aligns the LSM stats surface with the MemWAL naming settled upstream in lance-format/lance#7943 and #7957, where the persisted unit became an SSTable. Two terms in this API predate that pass. **Generation -> SSTable.** A `GenerationStats` describes one flushed MemTable, which is an SSTable. The generation *number* is kept — an SSTable is identified by its generation — so only the noun moved. **Bucket -> table shard.** Each entry is one MemWAL shard. "Bucket" names only the hash sharding transform, so it was wrong for `identity` and `year` sharding, which produce shards and no buckets at all. | Before | After | |---|---| | `GenerationStats` | `SsTableStats` | | `BucketStats` | `TableShardStats` | | `LsmStats.buckets` | `LsmStats.table_shards` | | `BucketStats.generations` | `TableShardStats.sstables` | | `include_generation_rows` | `include_sstable_rows` | | `newest_generation` | `newest_sstable_generation` | | `outstanding_generations` | `outstanding_sstables` | Applied across Rust, Python, TypeScript, and Java, including the `get_lsm_stats` JSON field names. `LsmWriteSpec::Bucket` and `num_buckets` are unchanged — those name the sharding transform, not the shard. ## Compatibility Breaking for the MemWAL LSM stats API, which is experimental and paired with a server that renames the same fields. The JSON keys `table_shards` and `sstables` must roll out together with the WAL server change. ## Validation - `cargo check -p lancedb --all-features`, `cargo fmt --all` - `ruff format --check` and `ruff check` on the touched Python - `biome check` on the touched TypeScript The typedoc markdown under `docs/src/js` was updated by hand, not regenerated: `npm run docs` needs the napi-built `./native` types. Worth running `npm run docs` on this branch to confirm the generator agrees. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HgVf5C2yRbwK6pe1aMYkmg
This commit is contained in:
co-authored by
Claude Opus 5
parent
9d3962686e
commit
ab74aa620c
+16
-16
@@ -33,16 +33,16 @@ use pyo3::{
|
||||
|
||||
mod scannable;
|
||||
|
||||
/// Convert `LsmStats` to a Python dict, preserving the per-bucket list.
|
||||
/// Convert `LsmStats` to a Python dict, preserving the per-table-shard list.
|
||||
///
|
||||
/// Deliberately not flattened to a table-level summary: a table is N
|
||||
/// buckets on one node, and the per-bucket detail is the reason the
|
||||
/// endpoint exists — flattening hides the single hot bucket someone opened
|
||||
/// table shards on one node, and the per-shard detail is the reason the
|
||||
/// endpoint exists — flattening hides the single hot table shard someone opened
|
||||
/// it to find.
|
||||
fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult<Py<PyDict>> {
|
||||
let out = PyDict::new(py);
|
||||
let buckets = PyList::empty(py);
|
||||
for b in &stats.buckets {
|
||||
let table_shards = PyList::empty(py);
|
||||
for b in &stats.table_shards {
|
||||
let e = PyDict::new(py);
|
||||
e.set_item("shard_id", &b.shard_id)?;
|
||||
e.set_item("status", &b.status)?;
|
||||
@@ -58,15 +58,15 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult
|
||||
b.wal_entry_position_last_seen,
|
||||
)?;
|
||||
|
||||
let generations = PyList::empty(py);
|
||||
for g in &b.generations {
|
||||
let sstables = PyList::empty(py);
|
||||
for g in &b.sstables {
|
||||
let ge = PyDict::new(py);
|
||||
ge.set_item("generation", g.generation)?;
|
||||
ge.set_item("bytes", g.bytes)?;
|
||||
ge.set_item("rows", g.rows)?;
|
||||
generations.append(ge)?;
|
||||
sstables.append(ge)?;
|
||||
}
|
||||
e.set_item("generations", generations)?;
|
||||
e.set_item("sstables", sstables)?;
|
||||
e.set_item("compacting", b.compacting)?;
|
||||
|
||||
e.set_item(
|
||||
@@ -88,9 +88,9 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult
|
||||
})
|
||||
.transpose()?,
|
||||
)?;
|
||||
buckets.append(e)?;
|
||||
table_shards.append(e)?;
|
||||
}
|
||||
out.set_item("buckets", buckets)?;
|
||||
out.set_item("table_shards", table_shards)?;
|
||||
Ok(out.unbind())
|
||||
}
|
||||
|
||||
@@ -1492,7 +1492,7 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
/// Seal every bucket's active memtable into L0.
|
||||
/// Freeze every table shard's active memtable into an SSTable.
|
||||
pub fn flush_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(
|
||||
@@ -1501,7 +1501,7 @@ impl Table {
|
||||
)
|
||||
}
|
||||
|
||||
/// Trigger a background L0 → base pass per bucket. Returns once the
|
||||
/// Trigger a background SSTable compaction pass per table shard. Returns once the
|
||||
/// passes are dispatched, not once they finish — watch `get_lsm_stats`.
|
||||
pub fn compact_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
@@ -1511,15 +1511,15 @@ impl Table {
|
||||
}
|
||||
|
||||
/// Live LSM state, or `None` when the LSM write path is not enabled.
|
||||
#[pyo3(signature = (include_generation_rows=false))]
|
||||
#[pyo3(signature = (include_sstable_rows=false))]
|
||||
pub fn get_lsm_stats(
|
||||
self_: PyRef<'_, Self>,
|
||||
include_generation_rows: bool,
|
||||
include_sstable_rows: bool,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let stats = inner
|
||||
.get_lsm_stats(include_generation_rows)
|
||||
.get_lsm_stats(include_sstable_rows)
|
||||
.await
|
||||
.infer_error()?;
|
||||
Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose())
|
||||
|
||||
Reference in New Issue
Block a user