mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-14 15:52:26 +00:00
refactor(lsm): finish SSTable and table shard naming
Applies the SSTable / table shard / freeze vocabulary from the previous commit to code that came in from main, and to spots that commit missed: - `get_lsm_stats` test fixtures in Rust and Python still sent the old `buckets` / `generations` keys, so the Rust test target no longer compiled and the Python tests would fail against `table_shards`. - The Java javadoc example called a nonexistent `LsmWriteSpec.tableShard`, an over-rename of `LsmWriteSpec.bucket`; the README used the old `buckets()` / `generations()` accessors. - Prose that still said bucket (for a shard), seal, L0, or flushed generations, and identifier-style "table_shard" / "tableShard" in prose. `LsmWriteSpec::bucket`, `num_buckets`, the hash-bucket values in the merge-insert router, and the `"Sealed"` status wire value are unchanged: those name the sharding transform or a server value, not the shard. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
d5544cd5d9
commit
06d7f82fa2
@@ -1386,7 +1386,7 @@ class LanceQueryBuilder(ABC):
|
||||
By default (unset), a query against a table with an LSM write spec is
|
||||
routed through the LSM scanner so it also returns data written via the
|
||||
``merge_insert`` LSM path that has not yet been compacted into the base
|
||||
table (active/frozen memtables + flushed generations); a table without a
|
||||
table (active/frozen memtables + SSTables); a table without a
|
||||
spec reads the base table.
|
||||
|
||||
Parameters
|
||||
@@ -3461,7 +3461,7 @@ class AsyncStandardQuery(AsyncQueryBase):
|
||||
[AsyncTable.set_lsm_write_spec][lancedb.table.AsyncTable.set_lsm_write_spec])
|
||||
is routed through the LSM scanner so it also returns data written via the
|
||||
``merge_insert`` LSM path that has not yet been compacted into the base
|
||||
table (the active/frozen in-memory memtables and the flushed generations),
|
||||
table (the active/frozen in-memory memtables and the SSTables),
|
||||
deduplicated by primary key; a table without a spec reads the base table.
|
||||
|
||||
Parameters
|
||||
|
||||
@@ -5477,7 +5477,7 @@ class AsyncTable:
|
||||
"""Converge this table's LSM write path into its base table.
|
||||
|
||||
One flush, freezing every memtable into an SSTable, then compaction triggers
|
||||
until every generation that existed at that moment has reached base.
|
||||
until every SSTable that existed at that moment has reached base.
|
||||
The loop runs client-side, reading progress from ``get_lsm_stats``.
|
||||
|
||||
Best-effort: SSTables created *while* it runs are deliberately not
|
||||
@@ -5516,9 +5516,9 @@ class AsyncTable:
|
||||
async def get_lsm_stats(
|
||||
self, *, include_sstable_rows: bool = False
|
||||
) -> Optional[dict]:
|
||||
"""Read live per-bucket LSM state.
|
||||
"""Read live per-table-shard LSM state.
|
||||
|
||||
Answers "how far behind is my fresh tier", "which bucket is hot", and
|
||||
Answers "how far behind is my fresh tier", "which table shard is hot", and
|
||||
"why is my fresh-tier vector search brute-force". Mutates no table
|
||||
state, though on a node that has not claimed this table it claims it,
|
||||
exactly as a read would.
|
||||
|
||||
@@ -1882,7 +1882,7 @@ def send_json(request, payload, status=200):
|
||||
|
||||
def test_get_lsm_stats_sync():
|
||||
"""The sync wrapper round-trips the server payload into a dict."""
|
||||
bucket = {
|
||||
table_shard = {
|
||||
"shard_id": "b0",
|
||||
"status": "Active",
|
||||
"writer_epoch": 3,
|
||||
@@ -1890,7 +1890,7 @@ def test_get_lsm_stats_sync():
|
||||
"current_generation": 6,
|
||||
"replay_after_wal_entry_position": 40,
|
||||
"wal_entry_position_last_seen": 42,
|
||||
"generations": [{"generation": 5, "bytes": 1024, "rows": 7}],
|
||||
"sstables": [{"generation": 5, "bytes": 1024, "rows": 7}],
|
||||
"compacting": False,
|
||||
"memtables": [
|
||||
{
|
||||
@@ -1907,10 +1907,10 @@ def test_get_lsm_stats_sync():
|
||||
def lsm_handler(request, route):
|
||||
assert route == "get_lsm_stats"
|
||||
seen_bodies.append(read_json_body(request))
|
||||
send_json(request, {"lsm_stats": {"buckets": [bucket]}})
|
||||
send_json(request, {"lsm_stats": {"table_shards": [table_shard]}})
|
||||
|
||||
with lsm_test_table(lsm_handler) as table:
|
||||
assert table.get_lsm_stats() == {"buckets": [bucket]}
|
||||
assert table.get_lsm_stats() == {"table_shards": [table_shard]}
|
||||
# Off by default, and forwarded when asked for.
|
||||
assert seen_bodies == [{"include_sstable_rows": False}]
|
||||
table.get_lsm_stats(include_sstable_rows=True)
|
||||
@@ -1954,8 +1954,8 @@ def test_checkpoint_lsm_sync():
|
||||
called.append(route)
|
||||
if route == "get_lsm_stats":
|
||||
# An empty SSTable tier yields no target watermark, so the loop is done
|
||||
# after the seal without ever polling compaction.
|
||||
send_json(request, {"lsm_stats": {"buckets": []}})
|
||||
# after the freeze without ever polling compaction.
|
||||
send_json(request, {"lsm_stats": {"table_shards": []}})
|
||||
else:
|
||||
request.send_response(202)
|
||||
request.end_headers()
|
||||
|
||||
Reference in New Issue
Block a user