mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-14 14:46:53 +00:00
feat(table): checkpoint_lsm, flush_lsm, compact_lsm, get_lsm_stats
Converge a table's LSM write path into its base table, and inspect it. `checkpoint_lsm` is `flush` then `compact`, repeated until the fresh tier is empty — and the loop runs **client-side**. Putting it on the server would mean a background task, which means a single-flight intent, an intent that leaks on panic, a bounded-iteration policy, an "is it done" observable, and a story for every way a client can vanish mid-operation. None of that exists in this shape: each request does a bounded unit of work and reports what is left, so completion is *carried in the responses* rather than inferred from a shared counter that cannot distinguish "converged" from "hasn't started yet". Best-effort by construction. Nothing is frozen, so `converged` means L0 was empty as of the last pass. It is idempotent, abandonable at any point with zero consequence, and safe to run on a cadence — an already- converged table costs one round trip and zero compaction passes, because `flush` reports `generations_remaining` and the loop is never entered. The failure taxonomy is the load-bearing part. Five distinct conditions used to arrive at a client as one 503, so `Error::LsmRoute` carries a classification read from the response body's namespace error code *at the point of receipt* — before any generic helper folds the body into a string and keeps only the status. Contention (429) retries; a draining node (503 / `InvalidTableState`) stops, because the drain gate is a one-way latch and retrying it spins until the deadline to report a failure that was knowable on the first response. Transport retry is disabled on these routes for the same reason: it treats every 503 alike. `get_lsm_stats` returns `Option<LsmStats>`, matching `get_lsm_write_spec` — `None` only when the table has no LSM write path, since a struct of zeros would read as measurements. Python bindings mirror all four, preserving the per-bucket detail rather than flattening to a table-level summary. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Daniel Rammer
co-authored by
Claude Opus 5
Claude Opus 5
parent
5eedeea120
commit
d26139417e
@@ -284,6 +284,15 @@ class Table:
|
||||
async def set_lsm_write_spec(self, spec: LsmWriteSpec) -> None: ...
|
||||
async def unset_lsm_write_spec(self) -> None: ...
|
||||
async def get_lsm_write_spec(self) -> Optional[LsmWriteSpec]: ...
|
||||
async def checkpoint_lsm(
|
||||
self,
|
||||
deadline_secs: Optional[float],
|
||||
max_generations_per_bucket: Optional[int],
|
||||
include_stats: bool,
|
||||
) -> dict: ...
|
||||
async def flush_lsm(self) -> dict: ...
|
||||
async def compact_lsm(self, max_generations_per_bucket: Optional[int]) -> dict: ...
|
||||
async def get_lsm_stats(self, include_generation_rows: bool) -> Optional[dict]: ...
|
||||
async def close_lsm_writers(self) -> None: ...
|
||||
@property
|
||||
def tags(self) -> Tags: ...
|
||||
|
||||
@@ -3872,6 +3872,44 @@ class LanceTable(Table):
|
||||
[`AsyncTable.get_lsm_write_spec`][lancedb.AsyncTable.get_lsm_write_spec]."""
|
||||
return LOOP.run(self._table.get_lsm_write_spec())
|
||||
|
||||
def checkpoint_lsm(
|
||||
self,
|
||||
*,
|
||||
deadline_secs: Optional[float] = 300.0,
|
||||
max_generations_per_bucket: Optional[int] = None,
|
||||
include_stats: bool = False,
|
||||
) -> dict:
|
||||
"""Synchronous version of
|
||||
[`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm]."""
|
||||
return LOOP.run(
|
||||
self._table.checkpoint_lsm(
|
||||
deadline_secs=deadline_secs,
|
||||
max_generations_per_bucket=max_generations_per_bucket,
|
||||
include_stats=include_stats,
|
||||
)
|
||||
)
|
||||
|
||||
def flush_lsm(self) -> dict:
|
||||
"""Synchronous version of
|
||||
[`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm]."""
|
||||
return LOOP.run(self._table.flush_lsm())
|
||||
|
||||
def compact_lsm(self, *, max_generations_per_bucket: Optional[int] = None) -> dict:
|
||||
"""Synchronous version of
|
||||
[`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
|
||||
return LOOP.run(
|
||||
self._table.compact_lsm(
|
||||
max_generations_per_bucket=max_generations_per_bucket
|
||||
)
|
||||
)
|
||||
|
||||
def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]:
|
||||
"""Synchronous version of
|
||||
[`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
|
||||
return LOOP.run(
|
||||
self._table.get_lsm_stats(include_generation_rows=include_generation_rows)
|
||||
)
|
||||
|
||||
def close_lsm_writers(self) -> None:
|
||||
"""Close cached MemWAL shard writers. See
|
||||
[`AsyncTable.close_lsm_writers`][lancedb.AsyncTable.close_lsm_writers]."""
|
||||
@@ -4582,6 +4620,83 @@ class AsyncTable:
|
||||
"""
|
||||
return await self._inner.get_lsm_write_spec()
|
||||
|
||||
async def checkpoint_lsm(
|
||||
self,
|
||||
*,
|
||||
deadline_secs: Optional[float] = 300.0,
|
||||
max_generations_per_bucket: Optional[int] = None,
|
||||
include_stats: bool = False,
|
||||
) -> dict:
|
||||
"""Converge this table's LSM write path into its base table.
|
||||
|
||||
One flush (sealing every memtable into L0), then bounded compaction
|
||||
passes until L0 is empty. The loop runs client-side, so each request
|
||||
does a bounded unit of work and reports what is left.
|
||||
|
||||
Best-effort: nothing is frozen, so with writes flowing there may be
|
||||
new rows by the time it returns — ``converged`` means L0 was empty as
|
||||
of the last pass. Idempotent and safe on a cadence; an
|
||||
already-converged table costs one round trip and zero compaction
|
||||
passes.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
deadline_secs
|
||||
Stop and report ``converged=False`` after this much wall clock.
|
||||
``None`` runs until the table converges — safe only when you know
|
||||
it is not under write load.
|
||||
max_generations_per_bucket
|
||||
Bound on generations merged **per bucket** per request. An
|
||||
N-bucket table admits ``N * max_generations_per_bucket`` merges
|
||||
per call.
|
||||
include_stats
|
||||
Attach a ``get_lsm_stats`` block to the report.
|
||||
"""
|
||||
return await self._inner.checkpoint_lsm(
|
||||
deadline_secs, max_generations_per_bucket, include_stats
|
||||
)
|
||||
|
||||
async def flush_lsm(self) -> dict:
|
||||
"""Seal every bucket's active memtable into L0.
|
||||
|
||||
Does not touch the base table — moving L0 into base is
|
||||
`compact_lsm`. On a node that has not claimed this table, this claims
|
||||
it and replays its WAL log first.
|
||||
"""
|
||||
return await self._inner.flush_lsm()
|
||||
|
||||
async def compact_lsm(
|
||||
self, *, max_generations_per_bucket: Optional[int] = None
|
||||
) -> dict:
|
||||
"""Run one bounded L0 to base compaction pass per bucket.
|
||||
|
||||
One pass, not convergence: that is what bounds each request's cost
|
||||
and gives a caller driving its own cadence a progress signal per
|
||||
round trip. Use `checkpoint_lsm` to loop to convergence.
|
||||
"""
|
||||
return await self._inner.compact_lsm(max_generations_per_bucket)
|
||||
|
||||
async def get_lsm_stats(
|
||||
self, *, include_generation_rows: bool = False
|
||||
) -> Optional[dict]:
|
||||
"""Read live LSM state: per-bucket detail plus a table-level aggregate.
|
||||
|
||||
Answers "how far behind is my fresh tier", "which bucket is hot", and
|
||||
"why is my fresh-tier vector search brute-force". Mutates no table
|
||||
state, though on a node that has not claimed this table it claims it,
|
||||
exactly as a read would.
|
||||
|
||||
Returns ``None`` — and only — when the LSM write path is not enabled
|
||||
for the table.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
include_generation_rows
|
||||
Report row counts per L0 generation. Off by default: each costs
|
||||
one Lance manifest read.
|
||||
"""
|
||||
return await self._inner.get_lsm_stats(include_generation_rows)
|
||||
|
||||
async def close_lsm_writers(self) -> None:
|
||||
"""Drain and close any cached MemWAL shard writers for this table.
|
||||
|
||||
|
||||
+222
-1
@@ -28,11 +28,103 @@ use pyo3::{
|
||||
Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
pyclass, pyfunction, pymethods,
|
||||
types::{IntoPyDict, PyAnyMethods, PyBytes, PyDict, PyDictMethods},
|
||||
types::{IntoPyDict, PyAnyMethods, PyBytes, PyDict, PyDictMethods, PyList, PyListMethods},
|
||||
};
|
||||
|
||||
mod scannable;
|
||||
|
||||
/// Convert `LsmStats` to a Python dict, preserving the per-bucket list.
|
||||
///
|
||||
/// Deliberately not flattened to a table-level summary: a table is N
|
||||
/// buckets on one node, and the per-bucket detail is the reason the
|
||||
/// endpoint exists — flattening hides the single hot bucket someone opened
|
||||
/// it to find.
|
||||
fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult<Py<PyDict>> {
|
||||
let out = PyDict::new(py);
|
||||
let buckets = PyList::empty(py);
|
||||
for b in &stats.buckets {
|
||||
let e = PyDict::new(py);
|
||||
e.set_item("shard_id", &b.shard_id)?;
|
||||
e.set_item("status", &b.status)?;
|
||||
e.set_item("writer_epoch", b.writer_epoch)?;
|
||||
e.set_item("manifest_version", b.manifest_version)?;
|
||||
e.set_item("current_generation", b.current_generation)?;
|
||||
e.set_item(
|
||||
"replay_after_wal_entry_position",
|
||||
b.replay_after_wal_entry_position,
|
||||
)?;
|
||||
e.set_item(
|
||||
"wal_entry_position_last_seen",
|
||||
b.wal_entry_position_last_seen,
|
||||
)?;
|
||||
e.set_item("wal_lag", b.wal_lag)?;
|
||||
e.set_item("l0_bytes", b.l0_bytes)?;
|
||||
e.set_item("fenced", b.fenced)?;
|
||||
e.set_item("compaction_in_progress", b.compaction_in_progress)?;
|
||||
|
||||
let generations = PyList::empty(py);
|
||||
for g in &b.generations {
|
||||
let ge = PyDict::new(py);
|
||||
ge.set_item("generation", g.generation)?;
|
||||
ge.set_item("path", &g.path)?;
|
||||
ge.set_item("bytes", g.bytes)?;
|
||||
ge.set_item("rows", g.rows)?;
|
||||
generations.append(ge)?;
|
||||
}
|
||||
e.set_item("generations", generations)?;
|
||||
|
||||
let memtable = |m: &lancedb::table::MemtableStats| -> PyResult<Py<PyDict>> {
|
||||
let d = PyDict::new(py);
|
||||
d.set_item("generation", m.generation)?;
|
||||
d.set_item("rows", m.rows)?;
|
||||
d.set_item("bytes", m.bytes)?;
|
||||
d.set_item("batches", m.batches)?;
|
||||
Ok(d.unbind())
|
||||
};
|
||||
e.set_item(
|
||||
"active_memtable",
|
||||
b.active_memtable.as_ref().map(memtable).transpose()?,
|
||||
)?;
|
||||
e.set_item(
|
||||
"frozen_memtables",
|
||||
b.frozen_memtables
|
||||
.as_ref()
|
||||
.map(|fs| {
|
||||
let l = PyList::empty(py);
|
||||
for m in fs {
|
||||
l.append(memtable(m)?)?;
|
||||
}
|
||||
PyResult::Ok(l.unbind())
|
||||
})
|
||||
.transpose()?,
|
||||
)?;
|
||||
e.set_item(
|
||||
"memtable_indexes",
|
||||
b.memtable_indexes
|
||||
.as_ref()
|
||||
.map(|ixs| {
|
||||
let l = PyList::empty(py);
|
||||
for i in ixs {
|
||||
let d = PyDict::new(py);
|
||||
d.set_item("name", &i.name)?;
|
||||
d.set_item("kind", &i.kind)?;
|
||||
d.set_item("column", &i.column)?;
|
||||
l.append(d)?;
|
||||
}
|
||||
PyResult::Ok(l.unbind())
|
||||
})
|
||||
.transpose()?,
|
||||
)?;
|
||||
buckets.append(e)?;
|
||||
}
|
||||
out.set_item("buckets", buckets)?;
|
||||
out.set_item("bucket_count", stats.bucket_count)?;
|
||||
out.set_item("generations_total", stats.generations_total)?;
|
||||
out.set_item("l0_bytes_total", stats.l0_bytes_total)?;
|
||||
out.set_item("memtable_rows_total", stats.memtable_rows_total)?;
|
||||
Ok(out.unbind())
|
||||
}
|
||||
|
||||
#[derive(FromPyObject)]
|
||||
enum PredicateArg {
|
||||
Expr(PyExpr),
|
||||
@@ -1267,6 +1359,135 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
/// Converge the table's LSM write path into its base table.
|
||||
///
|
||||
/// Returns a dict mirroring `CheckpointReport`. Best-effort: with
|
||||
/// writes flowing, new rows may land after the last pass.
|
||||
#[pyo3(signature = (deadline_secs=None, max_generations_per_bucket=None, include_stats=false))]
|
||||
pub fn checkpoint_lsm(
|
||||
self_: PyRef<'_, Self>,
|
||||
deadline_secs: Option<f64>,
|
||||
max_generations_per_bucket: Option<usize>,
|
||||
include_stats: bool,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
let opts = lancedb::table::CheckpointOptions {
|
||||
deadline: deadline_secs.map(std::time::Duration::from_secs_f64),
|
||||
max_generations_per_bucket,
|
||||
include_stats,
|
||||
};
|
||||
future_into_py(self_.py(), async move {
|
||||
let report = inner.checkpoint_lsm(opts).await.infer_error()?;
|
||||
Python::attach(|py| {
|
||||
let d = PyDict::new(py);
|
||||
d.set_item("converged", report.converged)?;
|
||||
d.set_item(
|
||||
"stop_reason",
|
||||
report.stop_reason.map(|r| match r {
|
||||
lancedb::table::CheckpointStopReason::DeadlineExceeded => {
|
||||
"deadline_exceeded"
|
||||
}
|
||||
lancedb::table::CheckpointStopReason::NodeDraining => "node_draining",
|
||||
lancedb::table::CheckpointStopReason::RepeatedlyUnclaimed => {
|
||||
"repeatedly_unclaimed"
|
||||
}
|
||||
}),
|
||||
)?;
|
||||
d.set_item("rows_sealed", report.rows_sealed)?;
|
||||
d.set_item("generations_compacted", report.generations_compacted)?;
|
||||
d.set_item("rows_merged", report.rows_merged)?;
|
||||
d.set_item("generations_remaining", report.generations_remaining)?;
|
||||
d.set_item("compact_calls", report.compact_calls)?;
|
||||
d.set_item(
|
||||
"stats",
|
||||
report.stats.map(|s| lsm_stats_to_py(py, &s)).transpose()?,
|
||||
)?;
|
||||
Ok(d.unbind())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Seal every bucket's active memtable into L0. Returns a dict
|
||||
/// mirroring `FlushReport`.
|
||||
pub fn flush_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let report = inner.flush_lsm().await.infer_error()?;
|
||||
Python::attach(|py| {
|
||||
let d = PyDict::new(py);
|
||||
let buckets = PyList::empty(py);
|
||||
// Per-bucket detail is preserved rather than flattened: a
|
||||
// table is N buckets on one node, and the one hot bucket is
|
||||
// usually why someone is looking.
|
||||
for b in &report.buckets {
|
||||
let e = PyDict::new(py);
|
||||
e.set_item("shard_id", &b.shard_id)?;
|
||||
e.set_item("sealed_generation", b.sealed_generation)?;
|
||||
e.set_item("rows_sealed", b.rows_sealed)?;
|
||||
e.set_item("generations_remaining", b.generations_remaining)?;
|
||||
buckets.append(e)?;
|
||||
}
|
||||
d.set_item("buckets", buckets)?;
|
||||
d.set_item("rows_sealed", report.rows_sealed)?;
|
||||
d.set_item("generations_remaining", report.generations_remaining)?;
|
||||
Ok(d.unbind())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Run one bounded L0 → base pass per bucket. Returns a dict mirroring
|
||||
/// `CompactReport`.
|
||||
#[pyo3(signature = (max_generations_per_bucket=None))]
|
||||
pub fn compact_lsm(
|
||||
self_: PyRef<'_, Self>,
|
||||
max_generations_per_bucket: Option<usize>,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
let opts = lancedb::table::CompactOptions {
|
||||
max_generations_per_bucket,
|
||||
};
|
||||
future_into_py(self_.py(), async move {
|
||||
let report = inner.compact_lsm(opts).await.infer_error()?;
|
||||
Python::attach(|py| {
|
||||
let d = PyDict::new(py);
|
||||
let buckets = PyList::empty(py);
|
||||
for b in &report.buckets {
|
||||
let e = PyDict::new(py);
|
||||
e.set_item("shard_id", &b.shard_id)?;
|
||||
e.set_item("generations_consumed", b.generations_consumed)?;
|
||||
e.set_item("rows_merged", b.rows_merged)?;
|
||||
e.set_item("generations_remaining", b.generations_remaining)?;
|
||||
buckets.append(e)?;
|
||||
}
|
||||
d.set_item("buckets", buckets)?;
|
||||
d.set_item("generations_consumed", report.generations_consumed)?;
|
||||
d.set_item("rows_merged", report.rows_merged)?;
|
||||
d.set_item("generations_remaining", report.generations_remaining)?;
|
||||
d.set_item(
|
||||
"highest_wal_entry_position",
|
||||
report.highest_wal_entry_position,
|
||||
)?;
|
||||
Ok(d.unbind())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Live LSM state, or `None` when the LSM write path is not enabled.
|
||||
#[pyo3(signature = (include_generation_rows=false))]
|
||||
pub fn get_lsm_stats(
|
||||
self_: PyRef<'_, Self>,
|
||||
include_generation_rows: bool,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
let opts = lancedb::table::LsmStatsOptions {
|
||||
include_generation_rows,
|
||||
};
|
||||
future_into_py(self_.py(), async move {
|
||||
let stats = inner.get_lsm_stats(opts).await.infer_error()?;
|
||||
Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn close_lsm_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
|
||||
Reference in New Issue
Block a user