mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-13 23:32:27 +00:00
refactor(lsm): finish SSTable and table shard naming
Applies the SSTable / table shard / freeze vocabulary from the previous commit to code that came in from main, and to spots that commit missed: - `get_lsm_stats` test fixtures in Rust and Python still sent the old `buckets` / `generations` keys, so the Rust test target no longer compiled and the Python tests would fail against `table_shards`. - The Java javadoc example called a nonexistent `LsmWriteSpec.tableShard`, an over-rename of `LsmWriteSpec.bucket`; the README used the old `buckets()` / `generations()` accessors. - Prose that still said bucket (for a shard), seal, L0, or flushed generations, and identifier-style "table_shard" / "tableShard" in prose. `LsmWriteSpec::bucket`, `num_buckets`, the hash-bucket values in the merge-insert router, and the `"Sealed"` status wire value are unchanged: those name the sharding transform or a server value, not the shard. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
d5544cd5d9
commit
06d7f82fa2
+3
-3
@@ -55,9 +55,9 @@ lsm.setLsmWriteSpec(LsmWriteSpec.bucket("id", 16));
|
||||
// Converge the fresh tier into the base table.
|
||||
lsm.checkpointLsm();
|
||||
|
||||
// Inspect live per-bucket state.
|
||||
lsm.getLsmStats().ifPresent(stats -> stats.buckets().forEach(bucket ->
|
||||
System.out.println(bucket.shardId() + ": " + bucket.generations().size() + " L0 generations")));
|
||||
// Inspect live per-table-shard state.
|
||||
lsm.getLsmStats().ifPresent(stats -> stats.tableShards().forEach(tableShard ->
|
||||
System.out.println(tableShard.shardId() + ": " + tableShard.sstables().size() + " SSTables")));
|
||||
|
||||
client.close();
|
||||
```
|
||||
|
||||
@@ -25,7 +25,7 @@ import com.fasterxml.jackson.databind.JsonNode;
|
||||
*
|
||||
* <p>The alternative — Jackson's {@code path()}, which yields a missing node that reads as an empty
|
||||
* array or a zero — is unsafe here because {@link LanceDbTableLsm#checkpointLsm()} decides
|
||||
* convergence from these numbers. A defaulted {@code generations} array is indistinguishable from a
|
||||
* convergence from these numbers. A defaulted {@code sstables} array is indistinguishable from a
|
||||
* drained one, so a malformed response would report a checkpoint that never happened.
|
||||
*/
|
||||
final class JsonFields {
|
||||
|
||||
@@ -38,7 +38,7 @@ import java.util.OptionalLong;
|
||||
* .buildRestClient();
|
||||
*
|
||||
* LanceDbTableLsm lsm = new LanceDbTableLsm(client, "my_table");
|
||||
* lsm.setLsmWriteSpec(LsmWriteSpec.tableShard("id", 16));
|
||||
* lsm.setLsmWriteSpec(LsmWriteSpec.bucket("id", 16));
|
||||
* // ... merge_insert traffic ...
|
||||
* lsm.checkpointLsm();
|
||||
* }</pre>
|
||||
@@ -94,7 +94,7 @@ public class LanceDbTableLsm {
|
||||
* Install an {@link LsmWriteSpec} on this table, selecting the MemWAL LSM write path for future
|
||||
* {@code mergeInsert} calls.
|
||||
*
|
||||
* <p>All variants require the table to have an unenforced primary key; tableShard sharding
|
||||
* <p>All variants require the table to have an unenforced primary key; bucket sharding
|
||||
* additionally requires it to be the single column being bucketed.
|
||||
*/
|
||||
public void setLsmWriteSpec(LsmWriteSpec spec) {
|
||||
@@ -136,7 +136,7 @@ public class LanceDbTableLsm {
|
||||
/**
|
||||
* Freeze every table shard's active memtable into a new SSTable.
|
||||
*
|
||||
* <p>Returns once the seal is committed. Sealing an empty memtable is a no-op, so this is safe to
|
||||
* <p>Returns once the freeze is committed. Freezing an empty memtable is a no-op, so this is safe to
|
||||
* call repeatedly.
|
||||
*/
|
||||
public void flushLsm() {
|
||||
@@ -154,9 +154,9 @@ public class LanceDbTableLsm {
|
||||
}
|
||||
|
||||
/**
|
||||
* Read live per-tableShard LSM state.
|
||||
* Read live per-table-shard LSM state.
|
||||
*
|
||||
* <p>Answers "how far behind is my fresh tier", "which tableShard is hot", and "why is my fresh-tier
|
||||
* <p>Answers "how far behind is my fresh tier", "which table shard is hot", and "why is my fresh-tier
|
||||
* vector search brute-force". Mutates no table state.
|
||||
*
|
||||
* <p>Empty only when the LSM write path is not enabled — that is, when the server sends an absent
|
||||
@@ -206,8 +206,8 @@ public class LanceDbTableLsm {
|
||||
*/
|
||||
public void checkpointLsm() {
|
||||
for (int reissue = 0; reissue <= MAX_REISSUES; reissue++) {
|
||||
// The seal turns everything written before this call into a generation, so the
|
||||
// watermark has to be read after it. Idempotent: sealing an empty memtable is a
|
||||
// The freeze turns everything written before this call into an SSTable, so the
|
||||
// watermark has to be read after it. Idempotent: freezing an empty memtable is a
|
||||
// no-op, so a re-issue does not churn empty sstables.
|
||||
if (issueVoid(this::flushLsm)) {
|
||||
backoff(reissue);
|
||||
@@ -240,7 +240,7 @@ public class LanceDbTableLsm {
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger and poll until no tableShard holds a generation at or below its target.
|
||||
* Trigger and poll until no table shard holds an SSTable generation at or below its target.
|
||||
*
|
||||
* @return true when the drain finished, false when the table needs re-claiming from flush.
|
||||
*/
|
||||
@@ -254,9 +254,9 @@ public class LanceDbTableLsm {
|
||||
return true;
|
||||
}
|
||||
|
||||
// `compacting` is the tableShard's compaction latch, held from dispatch until the pass
|
||||
// `compacting` is the table shard's compaction latch, held from dispatch until the pass
|
||||
// ends — including while it waits on a pod-wide permit. So it answers one question
|
||||
// only: do not pile on. Buckets with nothing outstanding are skipped, not counted
|
||||
// only: do not pile on. Table shards with nothing outstanding are skipped, not counted
|
||||
// as idle.
|
||||
long outstanding = 0;
|
||||
boolean allCompacting = true;
|
||||
@@ -285,7 +285,7 @@ public class LanceDbTableLsm {
|
||||
if (!isRetryable(e)) {
|
||||
throw e;
|
||||
}
|
||||
// A 429 here means the server could latch no tableShard at all, which the poll
|
||||
// A 429 here means the server could latch no table shard at all, which the poll
|
||||
// above already handles. Not retried in place: the latch it would contend for
|
||||
// is the one doing the work, so fall through and re-read — POLL_INTERVAL_MS is
|
||||
// the backoff.
|
||||
@@ -295,7 +295,7 @@ public class LanceDbTableLsm {
|
||||
}
|
||||
}
|
||||
|
||||
/** The newest generation held by each tableShard, skipping tableShards holding none. */
|
||||
/** The newest SSTable generation held by each table shard, skipping table shards holding none. */
|
||||
private static Map<String, Long> newestSstableGenerations(LsmStats stats) {
|
||||
Map<String, Long> targets = new HashMap<String, Long>();
|
||||
for (TableShardStats tableShard : stats.tableShards()) {
|
||||
|
||||
@@ -20,7 +20,7 @@ import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Live per-tableShard LSM state, as returned by {@link LanceDbTableLsm#getLsmStats()}.
|
||||
* Live per-table-shard LSM state, as returned by {@link LanceDbTableLsm#getLsmStats()}.
|
||||
*
|
||||
* <p>Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are the caller's to
|
||||
* compute. There is no "LSM is off" shape — that case is an empty {@link java.util.Optional},
|
||||
@@ -35,7 +35,7 @@ public class LsmStats {
|
||||
this.tableShards = Collections.unmodifiableList(tableShards);
|
||||
}
|
||||
|
||||
/** One entry per tableShard. */
|
||||
/** One entry per table shard. */
|
||||
public List<TableShardStats> tableShards() {
|
||||
return tableShards;
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ public class MemtableStats {
|
||||
this.indexes = Collections.unmodifiableList(indexes);
|
||||
}
|
||||
|
||||
/** The generation this memtable will become once sealed. */
|
||||
/** The generation this memtable will become once frozen. */
|
||||
public long generation() {
|
||||
return generation;
|
||||
}
|
||||
|
||||
@@ -22,8 +22,8 @@ import java.util.Optional;
|
||||
import java.util.OptionalLong;
|
||||
|
||||
/**
|
||||
* Live state of one tableShard. A table is N tableShards on one node; flattening to a single number hides
|
||||
* the one hot tableShard that is usually why someone opened this endpoint.
|
||||
* Live state of one table shard. A table is N table shards on one node; flattening to a single number hides
|
||||
* the one hot table shard that is usually why someone opened this endpoint.
|
||||
*/
|
||||
public class TableShardStats {
|
||||
private static final String CONTEXT = "tableShard stats";
|
||||
@@ -62,7 +62,7 @@ public class TableShardStats {
|
||||
this.memtables = memtables == null ? null : Collections.unmodifiableList(memtables);
|
||||
}
|
||||
|
||||
/** The shard this tableShard writes. */
|
||||
/** The id of this table shard. */
|
||||
public String shardId() {
|
||||
return shardId;
|
||||
}
|
||||
@@ -106,7 +106,7 @@ public class TableShardStats {
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a pass owns this tableShard's compaction latch right now. Says <em>a</em> driver is
|
||||
* Whether a pass owns this table shard's compaction latch right now. Says <em>a</em> driver is
|
||||
* running, not <em>whose</em>, and the latch is held from dispatch — including while the pass
|
||||
* queues for a pod-wide compactor permit. Read it as "do not pile on", never as "mine is
|
||||
* progressing".
|
||||
@@ -115,7 +115,7 @@ public class TableShardStats {
|
||||
return compacting;
|
||||
}
|
||||
|
||||
/** Oldest first, active last. Empty for a {@code "Sealed"} tableShard, whose state is torn down. */
|
||||
/** Oldest first, active last. Empty for a {@code "Sealed"} table shard, whose state is torn down. */
|
||||
public Optional<List<MemtableStats>> memtables() {
|
||||
return Optional.ofNullable(memtables);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user