refactor(lsm): finish SSTable and table shard naming

Applies the SSTable / table shard / freeze vocabulary from the previous
commit to code that came in from main, and to spots that commit missed:

- `get_lsm_stats` test fixtures in Rust and Python still sent the old
  `buckets` / `generations` keys, so the Rust test target no longer
  compiled and the Python tests would fail against `table_shards`.
- The Java javadoc example called a nonexistent `LsmWriteSpec.tableShard`,
  an over-rename of `LsmWriteSpec.bucket`; the README used the old
  `buckets()` / `generations()` accessors.
- Prose that still said bucket (for a shard), seal, L0, or flushed
  generations, and identifier-style "table_shard" / "tableShard" in prose.

`LsmWriteSpec::bucket`, `num_buckets`, the hash-bucket values in the
merge-insert router, and the `"Sealed"` status wire value are unchanged:
those name the sharding transform or a server value, not the shard.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Daniel Rammer
2026-10-12 15:30:06 -05:00
co-authored by Claude Opus 5.5
parent d5544cd5d9
commit 06d7f82fa2
23 changed files with 125 additions and 125 deletions
+3 -3
View File
@@ -55,9 +55,9 @@ lsm.setLsmWriteSpec(LsmWriteSpec.bucket("id", 16));
// Converge the fresh tier into the base table.
lsm.checkpointLsm();
// Inspect live per-bucket state.
lsm.getLsmStats().ifPresent(stats -> stats.buckets().forEach(bucket ->
System.out.println(bucket.shardId() + ": " + bucket.generations().size() + " L0 generations")));
// Inspect live per-table-shard state.
lsm.getLsmStats().ifPresent(stats -> stats.tableShards().forEach(tableShard ->
System.out.println(tableShard.shardId() + ": " + tableShard.sstables().size() + " SSTables")));
client.close();
```
@@ -25,7 +25,7 @@ import com.fasterxml.jackson.databind.JsonNode;
*
* <p>The alternative — Jackson's {@code path()}, which yields a missing node that reads as an empty
* array or a zero — is unsafe here because {@link LanceDbTableLsm#checkpointLsm()} decides
* convergence from these numbers. A defaulted {@code generations} array is indistinguishable from a
* convergence from these numbers. A defaulted {@code sstables} array is indistinguishable from a
* drained one, so a malformed response would report a checkpoint that never happened.
*/
final class JsonFields {
@@ -38,7 +38,7 @@ import java.util.OptionalLong;
* .buildRestClient();
*
* LanceDbTableLsm lsm = new LanceDbTableLsm(client, "my_table");
* lsm.setLsmWriteSpec(LsmWriteSpec.tableShard("id", 16));
* lsm.setLsmWriteSpec(LsmWriteSpec.bucket("id", 16));
* // ... merge_insert traffic ...
* lsm.checkpointLsm();
* }</pre>
@@ -94,7 +94,7 @@ public class LanceDbTableLsm {
* Install an {@link LsmWriteSpec} on this table, selecting the MemWAL LSM write path for future
* {@code mergeInsert} calls.
*
* <p>All variants require the table to have an unenforced primary key; tableShard sharding
* <p>All variants require the table to have an unenforced primary key; bucket sharding
* additionally requires it to be the single column being bucketed.
*/
public void setLsmWriteSpec(LsmWriteSpec spec) {
@@ -136,7 +136,7 @@ public class LanceDbTableLsm {
/**
* Freeze every table shard's active memtable into a new SSTable.
*
* <p>Returns once the seal is committed. Sealing an empty memtable is a no-op, so this is safe to
* <p>Returns once the freeze is committed. Freezing an empty memtable is a no-op, so this is safe to
* call repeatedly.
*/
public void flushLsm() {
@@ -154,9 +154,9 @@ public class LanceDbTableLsm {
}
/**
* Read live per-tableShard LSM state.
* Read live per-table-shard LSM state.
*
* <p>Answers "how far behind is my fresh tier", "which tableShard is hot", and "why is my fresh-tier
* <p>Answers "how far behind is my fresh tier", "which table shard is hot", and "why is my fresh-tier
* vector search brute-force". Mutates no table state.
*
* <p>Empty only when the LSM write path is not enabled — that is, when the server sends an absent
@@ -206,8 +206,8 @@ public class LanceDbTableLsm {
*/
public void checkpointLsm() {
for (int reissue = 0; reissue <= MAX_REISSUES; reissue++) {
// The seal turns everything written before this call into a generation, so the
// watermark has to be read after it. Idempotent: sealing an empty memtable is a
// The freeze turns everything written before this call into an SSTable, so the
// watermark has to be read after it. Idempotent: freezing an empty memtable is a
// no-op, so a re-issue does not churn empty sstables.
if (issueVoid(this::flushLsm)) {
backoff(reissue);
@@ -240,7 +240,7 @@ public class LanceDbTableLsm {
}
/**
* Trigger and poll until no tableShard holds a generation at or below its target.
* Trigger and poll until no table shard holds an SSTable generation at or below its target.
*
* @return true when the drain finished, false when the table needs re-claiming from flush.
*/
@@ -254,9 +254,9 @@ public class LanceDbTableLsm {
return true;
}
// `compacting` is the tableShard's compaction latch, held from dispatch until the pass
// `compacting` is the table shard's compaction latch, held from dispatch until the pass
// ends — including while it waits on a pod-wide permit. So it answers one question
// only: do not pile on. Buckets with nothing outstanding are skipped, not counted
// only: do not pile on. Table shards with nothing outstanding are skipped, not counted
// as idle.
long outstanding = 0;
boolean allCompacting = true;
@@ -285,7 +285,7 @@ public class LanceDbTableLsm {
if (!isRetryable(e)) {
throw e;
}
// A 429 here means the server could latch no tableShard at all, which the poll
// A 429 here means the server could latch no table shard at all, which the poll
// above already handles. Not retried in place: the latch it would contend for
// is the one doing the work, so fall through and re-read — POLL_INTERVAL_MS is
// the backoff.
@@ -295,7 +295,7 @@ public class LanceDbTableLsm {
}
}
/** The newest generation held by each tableShard, skipping tableShards holding none. */
/** The newest SSTable generation held by each table shard, skipping table shards holding none. */
private static Map<String, Long> newestSstableGenerations(LsmStats stats) {
Map<String, Long> targets = new HashMap<String, Long>();
for (TableShardStats tableShard : stats.tableShards()) {
@@ -20,7 +20,7 @@ import java.util.Collections;
import java.util.List;
/**
* Live per-tableShard LSM state, as returned by {@link LanceDbTableLsm#getLsmStats()}.
* Live per-table-shard LSM state, as returned by {@link LanceDbTableLsm#getLsmStats()}.
*
* <p>Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are the caller's to
* compute. There is no "LSM is off" shape — that case is an empty {@link java.util.Optional},
@@ -35,7 +35,7 @@ public class LsmStats {
this.tableShards = Collections.unmodifiableList(tableShards);
}
/** One entry per tableShard. */
/** One entry per table shard. */
public List<TableShardStats> tableShards() {
return tableShards;
}
@@ -37,7 +37,7 @@ public class MemtableStats {
this.indexes = Collections.unmodifiableList(indexes);
}
/** The generation this memtable will become once sealed. */
/** The generation this memtable will become once frozen. */
public long generation() {
return generation;
}
@@ -22,8 +22,8 @@ import java.util.Optional;
import java.util.OptionalLong;
/**
* Live state of one tableShard. A table is N tableShards on one node; flattening to a single number hides
* the one hot tableShard that is usually why someone opened this endpoint.
* Live state of one table shard. A table is N table shards on one node; flattening to a single number hides
* the one hot table shard that is usually why someone opened this endpoint.
*/
public class TableShardStats {
private static final String CONTEXT = "tableShard stats";
@@ -62,7 +62,7 @@ public class TableShardStats {
this.memtables = memtables == null ? null : Collections.unmodifiableList(memtables);
}
/** The shard this tableShard writes. */
/** The id of this table shard. */
public String shardId() {
return shardId;
}
@@ -106,7 +106,7 @@ public class TableShardStats {
}
/**
* Whether a pass owns this tableShard's compaction latch right now. Says <em>a</em> driver is
* Whether a pass owns this table shard's compaction latch right now. Says <em>a</em> driver is
* running, not <em>whose</em>, and the latch is held from dispatch — including while the pass
* queues for a pod-wide compactor permit. Read it as "do not pile on", never as "mine is
* progressing".
@@ -115,7 +115,7 @@ public class TableShardStats {
return compacting;
}
/** Oldest first, active last. Empty for a {@code "Sealed"} tableShard, whose state is torn down. */
/** Oldest first, active last. Empty for a {@code "Sealed"} table shard, whose state is torn down. */
public Optional<List<MemtableStats>> memtables() {
return Optional.ofNullable(memtables);
}