From ab74aa620cef4a84fc2f01abf1c7dc0f17cb160c Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Thu, 27 Aug 2026 15:14:08 -0500 Subject: [PATCH] refactor(lsm): rename LSM stats to SSTable and table shard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aligns the LSM stats surface with the MemWAL naming settled upstream in lance-format/lance#7943 and #7957, where the persisted unit became an SSTable. Two terms in this API predate that pass. **Generation -> SSTable.** A `GenerationStats` describes one flushed MemTable, which is an SSTable. The generation *number* is kept — an SSTable is identified by its generation — so only the noun moved. **Bucket -> table shard.** Each entry is one MemWAL shard. "Bucket" names only the hash sharding transform, so it was wrong for `identity` and `year` sharding, which produce shards and no buckets at all. | Before | After | |---|---| | `GenerationStats` | `SsTableStats` | | `BucketStats` | `TableShardStats` | | `LsmStats.buckets` | `LsmStats.table_shards` | | `BucketStats.generations` | `TableShardStats.sstables` | | `include_generation_rows` | `include_sstable_rows` | | `newest_generation` | `newest_sstable_generation` | | `outstanding_generations` | `outstanding_sstables` | Applied across Rust, Python, TypeScript, and Java, including the `get_lsm_stats` JSON field names. `LsmWriteSpec::Bucket` and `num_buckets` are unchanged — those name the sharding transform, not the shard. ## Compatibility Breaking for the MemWAL LSM stats API, which is experimental and paired with a server that renames the same fields. The JSON keys `table_shards` and `sstables` must roll out together with the WAL server change. ## Validation - `cargo check -p lancedb --all-features`, `cargo fmt --all` - `ruff format --check` and `ruff check` on the touched Python - `biome check` on the touched TypeScript The typedoc markdown under `docs/src/js` was updated by hand, not regenerated: `npm run docs` needs the napi-built `./native` types. Worth running `npm run docs` on this branch to confirm the generator agrees. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgVf5C2yRbwK6pe1aMYkmg --- docs/src/js/classes/Table.md | 14 ++-- docs/src/js/globals.md | 4 +- docs/src/js/interfaces/GenerationStats.md | 40 --------- docs/src/js/interfaces/LsmStats.md | 10 +-- docs/src/js/interfaces/SsTableStats.md | 40 +++++++++ .../{BucketStats.md => TableShardStats.md} | 20 ++--- .../java/com/lancedb/LanceDbTableLsm.java | 52 ++++++------ .../src/main/java/com/lancedb/LsmStats.java | 26 +++--- ...GenerationStats.java => SsTableStats.java} | 16 ++-- ...{BucketStats.java => TableShardStats.java} | 58 ++++++------- .../java/com/lancedb/LanceDbTableLsmTest.java | 80 +++++++++--------- nodejs/lancedb/index.ts | 4 +- nodejs/lancedb/table.ts | 20 ++--- nodejs/src/table.rs | 52 ++++++------ python/python/lancedb/_lancedb.pyi | 2 +- python/python/lancedb/remote/table.py | 4 +- python/python/lancedb/table.py | 26 +++--- python/python/tests/test_remote_db.py | 10 +-- python/src/table.rs | 32 ++++---- rust/lancedb/src/query.rs | 2 +- rust/lancedb/src/remote/table.rs | 12 +-- rust/lancedb/src/table.rs | 22 ++--- rust/lancedb/src/table/checkpoint.rs | 12 +-- rust/lancedb/src/table/lsm_stats.rs | 82 ++++++++++--------- 24 files changed, 323 insertions(+), 317 deletions(-) delete mode 100644 docs/src/js/interfaces/GenerationStats.md create mode 100644 docs/src/js/interfaces/SsTableStats.md rename docs/src/js/interfaces/{BucketStats.md => TableShardStats.md} (71%) rename java/lancedb-core/src/main/java/com/lancedb/{GenerationStats.java => SsTableStats.java} (75%) rename java/lancedb-core/src/main/java/com/lancedb/{BucketStats.java => TableShardStats.java} (76%) diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 159348450..ec7b50113 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -221,7 +221,7 @@ abstract checkpointLsm(): Promise Converge this table's LSM write path into its base table. -Seals once, then triggers compaction and polls until the L0 that existed +Freezes once, then triggers compaction and polls until the SSTables that existed at the start is gone. The target set is fixed at the start, so generations created *during* the checkpoint are ignored — that is what lets it terminate under write load, and what makes it best-effort: it @@ -289,7 +289,7 @@ It is a no-op when no writers are cached. abstract compactLsm(): Promise ``` -Trigger a background L0 → base compaction pass per bucket. +Trigger a background SSTable compaction pass per table shard. Returns once the passes are *dispatched*, not once they finish — watch [Table#getLsmStats](Table.md#getlsmstats) for progress, or use @@ -505,7 +505,7 @@ Drop an index from the table. abstract flushLsm(): Promise ``` -Seal every bucket's active memtable into a new L0 generation. +Freeze every table shard's active memtable into a new SSTable. Returns once the seal is committed. Sealing an empty memtable is a no-op, so this is safe to call repeatedly. @@ -519,10 +519,10 @@ so this is safe to call repeatedly. ### getLsmStats() ```ts -abstract getLsmStats(includeGenerationRows?): Promise +abstract getLsmStats(includeSstableRows?): Promise ``` -Read live per-bucket LSM state. +Read live per-table-shard LSM state. Answers "how far behind is my fresh tier", "which bucket is hot", and "why is my fresh-tier vector search brute-force". Mutates no table state. @@ -531,8 +531,8 @@ Resolves to `undefined` only when the LSM write path is not enabled. #### Parameters -* **includeGenerationRows?**: `boolean` - Also count rows per L0 generation. +* **includeSstableRows?**: `boolean` + Also count rows per SSTable. Off by default because each count opens an uncached Lance dataset. #### Returns diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index beb9cbeff..355e388e8 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -60,7 +60,6 @@ - [BranchDiff](interfaces/BranchDiff.md) - [BranchIndexSummary](interfaces/BranchIndexSummary.md) - [BranchRowCountSummary](interfaces/BranchRowCountSummary.md) -- [BucketStats](interfaces/BucketStats.md) - [CherryPickError](interfaces/CherryPickError.md) - [CherryPickPreview](interfaces/CherryPickPreview.md) - [CherryPickResult](interfaces/CherryPickResult.md) @@ -87,7 +86,6 @@ - [FtsToken](interfaces/FtsToken.md) - [FullTextQuery](interfaces/FullTextQuery.md) - [FullTextSearchOptions](interfaces/FullTextSearchOptions.md) -- [GenerationStats](interfaces/GenerationStats.md) - [HnswPqOptions](interfaces/HnswPqOptions.md) - [HnswSqOptions](interfaces/HnswSqOptions.md) - [IndexConfig](interfaces/IndexConfig.md) @@ -126,7 +124,9 @@ - [SplitHashOptions](interfaces/SplitHashOptions.md) - [SplitRandomOptions](interfaces/SplitRandomOptions.md) - [SplitSequentialOptions](interfaces/SplitSequentialOptions.md) +- [SsTableStats](interfaces/SsTableStats.md) - [TableNamesOptions](interfaces/TableNamesOptions.md) +- [TableShardStats](interfaces/TableShardStats.md) - [TableStatistics](interfaces/TableStatistics.md) - [TimeoutConfig](interfaces/TimeoutConfig.md) - [TlsConfig](interfaces/TlsConfig.md) diff --git a/docs/src/js/interfaces/GenerationStats.md b/docs/src/js/interfaces/GenerationStats.md deleted file mode 100644 index 19dd2afda..000000000 --- a/docs/src/js/interfaces/GenerationStats.md +++ /dev/null @@ -1,40 +0,0 @@ -[**@lancedb/lancedb**](../README.md) • **Docs** - -*** - -[@lancedb/lancedb](../globals.md) / GenerationStats - -# Interface: GenerationStats - -One flushed L0 generation. - -## Properties - -### bytes - -```ts -bytes: number; -``` - -On-disk size of the generation. - -*** - -### generation - -```ts -generation: number; -``` - -The generation number. Increases as memtables are sealed into L0. - -*** - -### rows? - -```ts -optional rows: number; -``` - -Present only when `includeGenerationRows` was requested. Off by default -because each count opens an uncached Lance dataset. diff --git a/docs/src/js/interfaces/LsmStats.md b/docs/src/js/interfaces/LsmStats.md index 76a2f50db..380511651 100644 --- a/docs/src/js/interfaces/LsmStats.md +++ b/docs/src/js/interfaces/LsmStats.md @@ -6,17 +6,17 @@ # Interface: LsmStats -Live per-bucket LSM state, as returned by `Table#getLsmStats`. +Live per-table-shard LSM state, as returned by `Table#getLsmStats`. -Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are +Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are the caller's to compute. ## Properties -### buckets +### tableShards ```ts -buckets: BucketStats[]; +tableShards: TableShardStats[]; ``` -One entry per bucket backing this table. +One entry per table shard backing this table. diff --git a/docs/src/js/interfaces/SsTableStats.md b/docs/src/js/interfaces/SsTableStats.md new file mode 100644 index 000000000..7b01a46f2 --- /dev/null +++ b/docs/src/js/interfaces/SsTableStats.md @@ -0,0 +1,40 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / SsTableStats + +# Interface: SsTableStats + +One SSTable. + +## Properties + +### bytes + +```ts +bytes: number; +``` + +On-disk size of the SSTable. + +*** + +### generation + +```ts +generation: number; +``` + +The generation number. Increases as memtables are frozen into SSTables. + +*** + +### rows? + +```ts +optional rows: number; +``` + +Present only when `includeSstableRows` was requested. Off by default +because each count opens an uncached Lance dataset. diff --git a/docs/src/js/interfaces/BucketStats.md b/docs/src/js/interfaces/TableShardStats.md similarity index 71% rename from docs/src/js/interfaces/BucketStats.md rename to docs/src/js/interfaces/TableShardStats.md index 3f5095672..e7a883e87 100644 --- a/docs/src/js/interfaces/BucketStats.md +++ b/docs/src/js/interfaces/TableShardStats.md @@ -2,12 +2,12 @@ *** -[@lancedb/lancedb](../globals.md) / BucketStats +[@lancedb/lancedb](../globals.md) / TableShardStats -# Interface: BucketStats +# Interface: TableShardStats -Live state of one bucket. A table is N buckets on one node; flattening to a -single number hides the one hot bucket that is usually why someone opened +Live state of one table shard. A table is N table shards on one node; flattening to a +single number hides the one hot table shard that is usually why someone opened this endpoint. ## Properties @@ -18,7 +18,7 @@ this endpoint. compacting: boolean; ``` -Whether a pass owns this bucket's compaction latch right now. Says *a* +Whether a pass owns this table shard's compaction latch right now. Says *a* driver is running, not *whose*, and the latch is held from dispatch — including while the pass queues for a pod-wide compactor permit. Read it as "do not pile on", never as "mine is progressing". @@ -35,13 +35,13 @@ The generation the active memtable will become. *** -### generations +### sstables ```ts -generations: GenerationStats[]; +sstables: SsTableStats[]; ``` -Flushed L0 generations not yet merged into the base table. +SSTables not yet merged into the base table. *** @@ -61,7 +61,7 @@ Version of the shard manifest these numbers were read from. optional memtables: MemtableStats[]; ``` -Oldest first, active last. Absent for a `"Sealed"` bucket, whose +Oldest first, active last. Absent for a `"Sealed"` table shard, whose in-memory state is torn down. *** @@ -82,7 +82,7 @@ WAL position replay resumes from. shardId: string; ``` -The shard this bucket writes. +The shard this table shard writes. *** diff --git a/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java b/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java index 23b18199e..3c0c042b4 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java +++ b/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java @@ -26,7 +26,7 @@ import java.util.OptionalLong; * *

Installing an {@link LsmWriteSpec} routes {@code mergeInsert} upserts through Lance's MemWAL — * an LSM-style append — instead of the standard merge path. Rows land in an in-memory memtable, - * seal into L0 generations, and are merged into the base table by compaction. + * freeze into SSTables, and are merged into the base table by compaction. * *

These routes are not part of the Lance Namespace specification, so they are issued directly * rather than through {@link org.lance.namespace.LanceNamespace}. @@ -38,7 +38,7 @@ import java.util.OptionalLong; * .buildRestClient(); * * LanceDbTableLsm lsm = new LanceDbTableLsm(client, "my_table"); - * lsm.setLsmWriteSpec(LsmWriteSpec.bucket("id", 16)); + * lsm.setLsmWriteSpec(LsmWriteSpec.tableShard("id", 16)); * // ... merge_insert traffic ... * lsm.checkpointLsm(); * } @@ -94,7 +94,7 @@ public class LanceDbTableLsm { * Install an {@link LsmWriteSpec} on this table, selecting the MemWAL LSM write path for future * {@code mergeInsert} calls. * - *

All variants require the table to have an unenforced primary key; bucket sharding + *

All variants require the table to have an unenforced primary key; tableShard sharding * additionally requires it to be the single column being bucketed. */ public void setLsmWriteSpec(LsmWriteSpec spec) { @@ -130,7 +130,7 @@ public class LanceDbTableLsm { } /** - * Seal every bucket's active memtable into a new L0 generation. + * Freeze every table shard's active memtable into a new SSTable. * *

Returns once the seal is committed. Sealing an empty memtable is a no-op, so this is safe to * call repeatedly. @@ -140,7 +140,7 @@ public class LanceDbTableLsm { } /** - * Trigger a background L0 → base compaction pass per bucket. + * Trigger a background SSTable compaction pass per table shard. * *

Returns once the passes are dispatched, not once they finish — watch {@link * #getLsmStats}, or use {@link #checkpointLsm} to wait for convergence. @@ -150,9 +150,9 @@ public class LanceDbTableLsm { } /** - * Read live per-bucket LSM state. + * Read live per-tableShard LSM state. * - *

Answers "how far behind is my fresh tier", "which bucket is hot", and "why is my fresh-tier + *

Answers "how far behind is my fresh tier", "which tableShard is hot", and "why is my fresh-tier * vector search brute-force". Mutates no table state. * *

Empty only when the LSM write path is not enabled — that is, when the server sends an absent @@ -160,13 +160,13 @@ public class LanceDbTableLsm { * one throws rather than decoding to something empty, because {@link #checkpointLsm} reads * convergence out of these numbers and cannot tell a defaulted array from a drained one. * - * @param includeGenerationRows Also count rows per L0 generation. Off by default because each + * @param includeSstableRows Also count rows per SSTable. Off by default because each * count opens an uncached Lance dataset. * @throws IllegalStateException if the response is absent or does not decode. */ - public Optional getLsmStats(boolean includeGenerationRows) { + public Optional getLsmStats(boolean includeSstableRows) { Map body = new LinkedHashMap(); - body.put("include_generation_rows", includeGenerationRows); + body.put("include_sstable_rows", includeSstableRows); JsonNode response = client.post(route("get_lsm_stats"), body); if (response == null) { throw new IllegalStateException("get_lsm_stats returned an empty response body"); @@ -186,8 +186,8 @@ public class LanceDbTableLsm { /** * Converge this table's LSM write path into its base table. * - *

Seals once, fixes a target watermark from the resulting L0, then triggers compaction and - * polls until that L0 is gone. The target set is fixed at the start, so generations created + *

Freezes once, fixes a target watermark from the resulting SSTables, then triggers compaction and + * polls until those SSTables are gone. The target set is fixed at the start, so sstables created * during the checkpoint are ignored — that is what lets it terminate under write load, * and what makes it best-effort: it converges the fresh tier as of some instant. Idempotent, * abandonable at any point, safe on a cadence. @@ -204,7 +204,7 @@ public class LanceDbTableLsm { for (int reissue = 0; reissue <= MAX_REISSUES; reissue++) { // The seal turns everything written before this call into a generation, so the // watermark has to be read after it. Idempotent: sealing an empty memtable is a - // no-op, so a re-issue does not churn empty generations. + // no-op, so a re-issue does not churn empty sstables. if (issueVoid(this::flushLsm)) { backoff(reissue); continue; @@ -220,7 +220,7 @@ public class LanceDbTableLsm { return; } - Map targets = newestGenerations(stats.value.get()); + Map targets = newestSstableGenerations(stats.value.get()); if (targets.isEmpty()) { return; } @@ -236,7 +236,7 @@ public class LanceDbTableLsm { } /** - * Trigger and poll until no bucket holds a generation at or below its target. + * Trigger and poll until no tableShard holds a generation at or below its target. * * @return true when the drain finished, false when the table needs re-claiming from flush. */ @@ -250,21 +250,21 @@ public class LanceDbTableLsm { return true; } - // `compacting` is the bucket's compaction latch, held from dispatch until the pass + // `compacting` is the tableShard's compaction latch, held from dispatch until the pass // ends — including while it waits on a pod-wide permit. So it answers one question // only: do not pile on. Buckets with nothing outstanding are skipped, not counted // as idle. long outstanding = 0; boolean allCompacting = true; - for (BucketStats bucket : stats.value.get().buckets()) { - Long target = targets.get(bucket.shardId()); + for (TableShardStats tableShard : stats.value.get().tableShards()) { + Long target = targets.get(tableShard.shardId()); if (target == null) { continue; } - long remaining = bucket.outstandingGenerations(target); + long remaining = tableShard.outstandingSstables(target); if (remaining > 0) { outstanding += remaining; - allCompacting &= bucket.compacting(); + allCompacting &= tableShard.compacting(); } } if (outstanding == 0) { @@ -281,7 +281,7 @@ public class LanceDbTableLsm { if (!isRetryable(e)) { throw e; } - // A 429 here means the server could latch no bucket at all, which the poll + // A 429 here means the server could latch no tableShard at all, which the poll // above already handles. Not retried in place: the latch it would contend for // is the one doing the work, so fall through and re-read — POLL_INTERVAL_MS is // the backoff. @@ -291,13 +291,13 @@ public class LanceDbTableLsm { } } - /** The newest generation held by each bucket, skipping buckets holding none. */ - private static Map newestGenerations(LsmStats stats) { + /** The newest generation held by each tableShard, skipping tableShards holding none. */ + private static Map newestSstableGenerations(LsmStats stats) { Map targets = new HashMap(); - for (BucketStats bucket : stats.buckets()) { - OptionalLong newest = bucket.newestGeneration(); + for (TableShardStats tableShard : stats.tableShards()) { + OptionalLong newest = tableShard.newestSstableGeneration(); if (newest.isPresent()) { - targets.put(bucket.shardId(), newest.getAsLong()); + targets.put(tableShard.shardId(), newest.getAsLong()); } } return targets; diff --git a/java/lancedb-core/src/main/java/com/lancedb/LsmStats.java b/java/lancedb-core/src/main/java/com/lancedb/LsmStats.java index 3496ebc96..b1cf3c67f 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/LsmStats.java +++ b/java/lancedb-core/src/main/java/com/lancedb/LsmStats.java @@ -20,37 +20,37 @@ import java.util.Collections; import java.util.List; /** - * Live per-bucket LSM state, as returned by {@link LanceDbTableLsm#getLsmStats()}. + * Live per-tableShard LSM state, as returned by {@link LanceDbTableLsm#getLsmStats()}. * - *

Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are the caller's to + *

Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are the caller's to * compute. There is no "LSM is off" shape — that case is an empty {@link java.util.Optional}, * because a stats object of zeros would read as measurements. */ public class LsmStats { private static final String CONTEXT = "lsm stats"; - private final List buckets; + private final List tableShards; - LsmStats(List buckets) { - this.buckets = Collections.unmodifiableList(buckets); + LsmStats(List tableShards) { + this.tableShards = Collections.unmodifiableList(tableShards); } - /** One entry per bucket. */ - public List buckets() { - return buckets; + /** One entry per tableShard. */ + public List tableShards() { + return tableShards; } static LsmStats fromJson(JsonNode node) { JsonFields.requiredObject(node, CONTEXT); - List buckets = new ArrayList(); - for (JsonNode bucket : JsonFields.requiredArray(node, "buckets", CONTEXT)) { - buckets.add(BucketStats.fromJson(bucket)); + List tableShards = new ArrayList(); + for (JsonNode tableShard : JsonFields.requiredArray(node, "table_shards", CONTEXT)) { + tableShards.add(TableShardStats.fromJson(tableShard)); } - return new LsmStats(buckets); + return new LsmStats(tableShards); } @Override public String toString() { - return "LsmStats{buckets=" + buckets + "}"; + return "LsmStats{tableShards=" + tableShards + "}"; } } diff --git a/java/lancedb-core/src/main/java/com/lancedb/GenerationStats.java b/java/lancedb-core/src/main/java/com/lancedb/SsTableStats.java similarity index 75% rename from java/lancedb-core/src/main/java/com/lancedb/GenerationStats.java rename to java/lancedb-core/src/main/java/com/lancedb/SsTableStats.java index 12222407c..6e57c1ddf 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/GenerationStats.java +++ b/java/lancedb-core/src/main/java/com/lancedb/SsTableStats.java @@ -17,21 +17,21 @@ import com.fasterxml.jackson.databind.JsonNode; import java.util.OptionalLong; -/** One flushed L0 generation. */ -public class GenerationStats { +/** One SSTable. */ +public class SsTableStats { private static final String CONTEXT = "generation stats"; private final long generation; private final long bytes; private final Long rows; - GenerationStats(long generation, long bytes, Long rows) { + SsTableStats(long generation, long bytes, Long rows) { this.generation = generation; this.bytes = bytes; this.rows = rows; } - /** The generation number. Increases as memtables are sealed into L0. */ + /** The generation number. Increases as memtables are frozen into SSTables. */ public long generation() { return generation; } @@ -42,16 +42,16 @@ public class GenerationStats { } /** - * Rows in this generation, present only when {@code includeGenerationRows} was requested. Off by + * Rows in this generation, present only when {@code includeSstableRows} was requested. Off by * default because each count opens an uncached Lance dataset. */ public OptionalLong rows() { return rows == null ? OptionalLong.empty() : OptionalLong.of(rows); } - static GenerationStats fromJson(JsonNode node) { + static SsTableStats fromJson(JsonNode node) { JsonFields.requiredObject(node, CONTEXT); - return new GenerationStats( + return new SsTableStats( JsonFields.requiredLong(node, "generation", CONTEXT), JsonFields.requiredLong(node, "bytes", CONTEXT), JsonFields.optionalLong(node, "rows", CONTEXT)); @@ -59,6 +59,6 @@ public class GenerationStats { @Override public String toString() { - return "GenerationStats{generation=" + generation + ", bytes=" + bytes + ", rows=" + rows + "}"; + return "SsTableStats{generation=" + generation + ", bytes=" + bytes + ", rows=" + rows + "}"; } } diff --git a/java/lancedb-core/src/main/java/com/lancedb/BucketStats.java b/java/lancedb-core/src/main/java/com/lancedb/TableShardStats.java similarity index 76% rename from java/lancedb-core/src/main/java/com/lancedb/BucketStats.java rename to java/lancedb-core/src/main/java/com/lancedb/TableShardStats.java index 2a8060c5d..6b6c7063a 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/BucketStats.java +++ b/java/lancedb-core/src/main/java/com/lancedb/TableShardStats.java @@ -22,11 +22,11 @@ import java.util.Optional; import java.util.OptionalLong; /** - * Live state of one bucket. A table is N buckets on one node; flattening to a single number hides - * the one hot bucket that is usually why someone opened this endpoint. + * Live state of one tableShard. A table is N tableShards on one node; flattening to a single number hides + * the one hot tableShard that is usually why someone opened this endpoint. */ -public class BucketStats { - private static final String CONTEXT = "bucket stats"; +public class TableShardStats { + private static final String CONTEXT = "tableShard stats"; private final String shardId; private final String status; @@ -35,11 +35,11 @@ public class BucketStats { private final long currentGeneration; private final long replayAfterWalEntryPosition; private final long walEntryPositionLastSeen; - private final List generations; + private final List sstables; private final boolean compacting; private final List memtables; - BucketStats( + TableShardStats( String shardId, String status, long writerEpoch, @@ -47,7 +47,7 @@ public class BucketStats { long currentGeneration, long replayAfterWalEntryPosition, long walEntryPositionLastSeen, - List generations, + List sstables, boolean compacting, List memtables) { this.shardId = shardId; @@ -57,12 +57,12 @@ public class BucketStats { this.currentGeneration = currentGeneration; this.replayAfterWalEntryPosition = replayAfterWalEntryPosition; this.walEntryPositionLastSeen = walEntryPositionLastSeen; - this.generations = Collections.unmodifiableList(generations); + this.sstables = Collections.unmodifiableList(sstables); this.compacting = compacting; this.memtables = memtables == null ? null : Collections.unmodifiableList(memtables); } - /** The shard this bucket writes. */ + /** The shard this tableShard writes. */ public String shardId() { return shardId; } @@ -100,13 +100,13 @@ public class BucketStats { return walEntryPositionLastSeen; } - /** Flushed L0 generations not yet merged into the base table. */ - public List generations() { - return generations; + /** SSTables not yet merged into the base table. */ + public List sstables() { + return sstables; } /** - * Whether a pass owns this bucket's compaction latch right now. Says a driver is + * Whether a pass owns this tableShard's compaction latch right now. Says a driver is * running, not whose, and the latch is held from dispatch — including while the pass * queues for a pod-wide compactor permit. Read it as "do not pile on", never as "mine is * progressing". @@ -115,15 +115,15 @@ public class BucketStats { return compacting; } - /** Oldest first, active last. Empty for a {@code "Sealed"} bucket, whose state is torn down. */ + /** Oldest first, active last. Empty for a {@code "Sealed"} tableShard, whose state is torn down. */ public Optional> memtables() { return Optional.ofNullable(memtables); } - /** The newest flushed generation, or empty when L0 is empty. */ - OptionalLong newestGeneration() { + /** The newest SSTable generation, or empty when the tier is empty. */ + OptionalLong newestSstableGeneration() { OptionalLong newest = OptionalLong.empty(); - for (GenerationStats generation : generations) { + for (SsTableStats generation : sstables) { if (!newest.isPresent() || generation.generation() > newest.getAsLong()) { newest = OptionalLong.of(generation.generation()); } @@ -132,15 +132,15 @@ public class BucketStats { } /** - * How many generations at or below {@code target} are still in L0. + * How many SSTables at or below {@code target} are still uncompacted. * *

A count, not a boolean: one pass drains a bounded prefix rather than the whole target set, * so a boolean would read as "no progress" for every pass but the last. Compaction drains * oldest-first, so this decreases monotonically. */ - long outstandingGenerations(long target) { + long outstandingSstables(long target) { long count = 0; - for (GenerationStats generation : generations) { + for (SsTableStats generation : sstables) { if (generation.generation() <= target) { count++; } @@ -148,11 +148,11 @@ public class BucketStats { return count; } - static BucketStats fromJson(JsonNode node) { + static TableShardStats fromJson(JsonNode node) { JsonFields.requiredObject(node, CONTEXT); - List generations = new ArrayList(); - for (JsonNode generation : JsonFields.requiredArray(node, "generations", CONTEXT)) { - generations.add(GenerationStats.fromJson(generation)); + List sstables = new ArrayList(); + for (JsonNode generation : JsonFields.requiredArray(node, "sstables", CONTEXT)) { + sstables.add(SsTableStats.fromJson(generation)); } JsonNode memtablesNode = JsonFields.optionalArray(node, "memtables", CONTEXT); @@ -164,7 +164,7 @@ public class BucketStats { } } - return new BucketStats( + return new TableShardStats( JsonFields.requiredText(node, "shard_id", CONTEXT), JsonFields.requiredText(node, "status", CONTEXT), JsonFields.requiredLong(node, "writer_epoch", CONTEXT), @@ -172,21 +172,21 @@ public class BucketStats { JsonFields.requiredLong(node, "current_generation", CONTEXT), JsonFields.requiredLong(node, "replay_after_wal_entry_position", CONTEXT), JsonFields.requiredLong(node, "wal_entry_position_last_seen", CONTEXT), - generations, + sstables, JsonFields.requiredBoolean(node, "compacting", CONTEXT), memtables); } @Override public String toString() { - return "BucketStats{shardId=" + return "TableShardStats{shardId=" + shardId + ", status=" + status + ", currentGeneration=" + currentGeneration - + ", generations=" - + generations + + ", sstables=" + + sstables + ", compacting=" + compacting + "}"; diff --git a/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java b/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java index e84fa5421..167edcd10 100644 --- a/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java +++ b/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java @@ -132,10 +132,10 @@ public class LanceDbTableLsmTest { enqueue("set_lsm_write_spec", 200, ""); lsm.setLsmWriteSpec( - LsmWriteSpec.bucket("id", 16).withMaintainedIndexes(Arrays.asList("id_idx"))); + LsmWriteSpec.tableShard("id", 16).withMaintainedIndexes(Arrays.asList("id_idx"))); JsonNode body = MAPPER.readTree(requestBodies.get(0)); - assertEquals("bucket", body.get("sharding").get("mode").asText()); + assertEquals("tableShard", body.get("sharding").get("mode").asText()); assertEquals("id", body.get("sharding").get("column").asText()); assertEquals(16, body.get("sharding").get("num_buckets").asInt()); assertEquals(1, body.get("maintained_indexes").size()); @@ -201,7 +201,7 @@ public class LanceDbTableLsmTest { enqueue( "get_lsm_write_spec", 200, - "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"bucket\",\"column\":\"id\"," + "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"tableShard\",\"column\":\"id\"," + "\"num_buckets\":16},\"maintained_indexes\":[\"id_idx\"]," + "\"writer_config_defaults\":{\"durable_write\":\"true\"}}}"); @@ -228,14 +228,14 @@ public class LanceDbTableLsmTest { @Test public void testGetLsmStats() throws Exception { - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L))); + enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); Optional got = lsm.getLsmStats(true); assertEquals("/v1/table/my_table/get_lsm_stats/", requestPaths.get(0)); - assertTrue(MAPPER.readTree(requestBodies.get(0)).get("include_generation_rows").asBoolean()); + assertTrue(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean()); assertTrue(got.isPresent()); - BucketStats decoded = got.get().buckets().get(0); + TableShardStats decoded = got.get().tableShards().get(0); assertEquals("shard-0", decoded.shardId()); assertEquals("Active", decoded.status()); assertEquals(1, decoded.writerEpoch()); @@ -243,8 +243,8 @@ public class LanceDbTableLsmTest { assertEquals(9, decoded.currentGeneration()); assertFalse(decoded.compacting()); assertEquals(Arrays.asList(7L, 8L), generationNumbers(decoded)); - assertEquals(1024, decoded.generations().get(0).bytes()); - assertFalse(decoded.generations().get(0).rows().isPresent(), "rows absent unless requested"); + assertEquals(1024, decoded.sstables().get(0).bytes()); + assertFalse(decoded.sstables().get(0).rows().isPresent(), "rows absent unless requested"); assertFalse(decoded.memtables().isPresent(), "absent memtables stay absent"); } @@ -254,19 +254,19 @@ public class LanceDbTableLsmTest { enqueue( "get_lsm_stats", 200, - "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "{\"lsm_stats\":{\"tableShards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9," + "\"replay_after_wal_entry_position\":3,\"wal_entry_position_last_seen\":11," - + "\"generations\":[{\"generation\":7,\"bytes\":1024,\"rows\":42}]," + + "\"sstables\":[{\"generation\":7,\"bytes\":1024,\"rows\":42}]," + "\"compacting\":true,\"memtables\":[{\"generation\":8,\"rows\":5," + "\"bytes\":64,\"batches\":2,\"indexes\":[\"id_idx\"]}]}]}}"); - BucketStats decoded = lsm.getLsmStats(true).get().buckets().get(0); + TableShardStats decoded = lsm.getLsmStats(true).get().tableShards().get(0); assertEquals(3, decoded.replayAfterWalEntryPosition()); assertEquals(11, decoded.walEntryPositionLastSeen()); assertTrue(decoded.compacting()); - assertEquals(42, decoded.generations().get(0).rows().getAsLong()); + assertEquals(42, decoded.sstables().get(0).rows().getAsLong()); assertTrue(decoded.memtables().isPresent()); MemtableStats memtable = decoded.memtables().get().get(0); assertEquals(8, memtable.generation()); @@ -289,7 +289,7 @@ public class LanceDbTableLsmTest { lsm.getLsmStats(); - assertFalse(MAPPER.readTree(requestBodies.get(0)).get("include_generation_rows").asBoolean()); + assertFalse(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean()); } // =========================================================================== @@ -334,8 +334,8 @@ public class LanceDbTableLsmTest { @Test public void testCheckpointReturnsWhenNoGenerationsOutstanding() { enqueue("flush_lsm", 200, ""); - // A bucket with no L0 generations yields no target, so the drain never starts. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false))); + // A table shard with no SSTables yields no target, so the drain never starts. + enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false))); lsm.checkpointLsm(); @@ -345,12 +345,12 @@ public class LanceDbTableLsmTest { @Test public void testCheckpointConvergesOnceTargetGenerationsAreGone() { enqueue("flush_lsm", 200, ""); - // Watermark read: shard-0 holds generations 7 and 8, so target = 8. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L))); + // Watermark read: shard-0 holds sstables 7 and 8, so target = 8. + enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); // First drain poll: both still outstanding, nothing compacting -> dispatch a pass. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L))); + enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); // Second drain poll: drained past the target -> done. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 9L))); + enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 9L))); enqueue("compact_lsm", 200, ""); lsm.checkpointLsm(); @@ -362,14 +362,14 @@ public class LanceDbTableLsmTest { @Test public void testCheckpointDoesNotPileOnWhileEveryTargetBucketIsCompacting() { enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", true, 4L))); + enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", true, 4L))); // Still compacting on the first poll, so no pass is dispatched; then it drains. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", true, 4L))); - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 5L))); + enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", true, 4L))); + enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 5L))); lsm.checkpointLsm(); - assertEquals(0, countCalls("compact_lsm"), "a latched bucket is left alone"); + assertEquals(0, countCalls("compact_lsm"), "a latched tableShard is left alone"); } @Test @@ -378,7 +378,7 @@ public class LanceDbTableLsmTest { // from flush rather than retrying the read in place. enqueue("flush_lsm", 200, ""); enqueue("get_lsm_stats", 421, "no claim"); - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false))); + enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false))); lsm.checkpointLsm(); @@ -389,7 +389,7 @@ public class LanceDbTableLsmTest { public void testCheckpointRetriesRetryableStatusInPlace() { enqueue("flush_lsm", 429, "latch held"); enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false))); + enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false))); lsm.checkpointLsm(); @@ -421,27 +421,27 @@ public class LanceDbTableLsmTest { /** * A stats payload that does not decode must fail closed. Every one of these bodies used to be - * read as "no buckets", which is indistinguishable from a drained table, so {@code checkpointLsm} + * read as "no tableShards", which is indistinguishable from a drained table, so {@code checkpointLsm} * reported convergence for a checkpoint that never ran. */ @Test public void testCheckpointRejectsMalformedStats() { Map malformed = new LinkedHashMap(); malformed.put("no response body at all", ""); - malformed.put("stats object with no buckets", "{\"lsm_stats\":{}}"); - malformed.put("bucket missing its required fields", "{\"lsm_stats\":{\"buckets\":[{}]}}"); + malformed.put("stats object with no tableShards", "{\"lsm_stats\":{}}"); + malformed.put("tableShard missing its required fields", "{\"lsm_stats\":{\"tableShards\":[{}]}}"); malformed.put( - "bucket missing generations", - "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "tableShard missing sstables", + "{\"lsm_stats\":{\"tableShards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9," + "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0," + "\"compacting\":false}]}}"); malformed.put( "generation with a non-numeric generation number", - "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "{\"lsm_stats\":{\"tableShards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9," + "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0," - + "\"generations\":[{\"generation\":\"7\",\"bytes\":1024}]," + + "\"sstables\":[{\"generation\":\"7\",\"bytes\":1024}]," + "\"compacting\":false}]}}"); for (Map.Entry each : malformed.entrySet()) { @@ -492,22 +492,22 @@ public class LanceDbTableLsmTest { // harness // =========================================================================== - private static List generationNumbers(BucketStats bucket) { + private static List generationNumbers(TableShardStats tableShard) { List numbers = new ArrayList(); - for (GenerationStats generation : bucket.generations()) { + for (SsTableStats generation : tableShard.sstables()) { numbers.add(generation.generation()); } return numbers; } - /** Build an {@code lsm_stats} response body from bucket fragments. */ - private static String stats(String... buckets) { - return "{\"lsm_stats\":{\"buckets\":[" + String.join(",", buckets) + "]}}"; + /** Build an {@code lsm_stats} response body from tableShard fragments. */ + private static String stats(String... tableShards) { + return "{\"lsm_stats\":{\"tableShards\":[" + String.join(",", tableShards) + "]}}"; } - private static String bucket(String shardId, boolean compacting, Long... generations) { + private static String tableShard(String shardId, boolean compacting, Long... sstables) { StringBuilder gens = new StringBuilder(); - for (Long generation : generations) { + for (Long generation : sstables) { if (gens.length() > 0) { gens.append(","); } @@ -517,7 +517,7 @@ public class LanceDbTableLsmTest { + shardId + "\",\"status\":\"Active\",\"writer_epoch\":1,\"manifest_version\":2," + "\"current_generation\":9,\"replay_after_wal_entry_position\":0," - + "\"wal_entry_position_last_seen\":0,\"generations\":[" + + "\"wal_entry_position_last_seen\":0,\"sstables\":[" + gens + "],\"compacting\":" + compacting diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 34d7ce4d9..31d9b51a6 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -157,8 +157,8 @@ export { TokenizeTableOptions, LsmWriteSpec, LsmStats, - BucketStats, - GenerationStats, + TableShardStats, + SsTableStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index dc062e337..af7176209 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -55,8 +55,8 @@ import { sanitizeType } from "./sanitize"; import { IntoSql, toSQL } from "./util"; export { IndexConfig } from "./native"; export { - BucketStats, - GenerationStats, + TableShardStats, + SsTableStats, LsmStats, MemtableStats, } from "./native"; @@ -741,7 +741,7 @@ export abstract class Table { */ abstract closeLsmWriters(): Promise; /** - * Seal every bucket's active memtable into a new L0 generation. + * Freeze every table shard's active memtable into a new SSTable. * * Returns once the seal is committed. Sealing an empty memtable is a no-op, * so this is safe to call repeatedly. @@ -749,7 +749,7 @@ export abstract class Table { */ abstract flushLsm(): Promise; /** - * Trigger a background L0 → base compaction pass per bucket. + * Trigger a background SSTable compaction pass per table shard. * * Returns once the passes are *dispatched*, not once they finish — watch * {@link Table#getLsmStats} for progress, or use @@ -760,9 +760,9 @@ export abstract class Table { /** * Converge this table's LSM write path into its base table. * - * Seals once, then triggers compaction and polls until the L0 that existed + * Freezes once, then triggers compaction and polls until the SSTables that existed * at the start is gone. The target set is fixed at the start, so - * generations created *during* the checkpoint are ignored — that is what + * SSTables created *during* the checkpoint are ignored — that is what * lets it terminate under write load, and what makes it best-effort: it * converges the fresh tier as of some instant. Idempotent, abandonable at * any point, and safe to run on a cadence. @@ -786,12 +786,12 @@ export abstract class Table { * "why is my fresh-tier vector search brute-force". Mutates no table state. * * Resolves to `undefined` only when the LSM write path is not enabled. - * @param {boolean} includeGenerationRows Also count rows per L0 generation. + * @param {boolean} includeSstableRows Also count rows per SSTable. * Off by default because each count opens an uncached Lance dataset. * @returns {Promise} */ abstract getLsmStats( - includeGenerationRows?: boolean, + includeSstableRows?: boolean, ): Promise; /** Retrieve the version of the table */ @@ -1388,9 +1388,9 @@ export class LocalTable extends Table { } async getLsmStats( - includeGenerationRows: boolean = false, + includeSstableRows: boolean = false, ): Promise { - return (await this.inner.getLsmStats(includeGenerationRows)) ?? undefined; + return (await this.inner.getLsmStats(includeSstableRows)) ?? undefined; } async version(): Promise { diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index db74d38fa..183427aa3 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -542,11 +542,11 @@ impl Table { #[napi(catch_unwind)] pub async fn get_lsm_stats( &self, - include_generation_rows: bool, + include_sstable_rows: bool, ) -> napi::Result> { let stats = self .inner_ref()? - .get_lsm_stats(include_generation_rows) + .get_lsm_stats(include_sstable_rows) .await .default_error()?; Ok(stats.map(LsmStats::from)) @@ -950,21 +950,21 @@ impl From for LsmWriteSpec { } } -/// One flushed L0 generation. +/// One SSTable. #[napi(object)] #[derive(Clone, Debug)] -pub struct GenerationStats { - /// The generation number. Increases as memtables are sealed into L0. +pub struct SsTableStats { + /// The generation number. Increases as memtables are frozen into SSTables. pub generation: i64, - /// On-disk size of the generation. + /// On-disk size of the SSTable. pub bytes: i64, - /// Present only when `includeGenerationRows` was requested. Off by default + /// Present only when `includeSstableRows` was requested. Off by default /// because each count opens an uncached Lance dataset. pub rows: Option, } -impl From for GenerationStats { - fn from(g: lancedb::table::GenerationStats) -> Self { +impl From for SsTableStats { + fn from(g: lancedb::table::SsTableStats) -> Self { Self { generation: g.generation as i64, bytes: g.bytes as i64, @@ -977,7 +977,7 @@ impl From for GenerationStats { #[napi(object)] #[derive(Clone, Debug)] pub struct MemtableStats { - /// The generation this memtable will become once sealed. + /// The generation this memtable will become once frozen. pub generation: i64, /// Rows currently buffered. pub rows: i64, @@ -1002,13 +1002,13 @@ impl From for MemtableStats { } } -/// Live state of one bucket. A table is N buckets on one node; flattening to a -/// single number hides the one hot bucket that is usually why someone opened +/// Live state of one table shard. A table is N table shards on one node; flattening to a +/// single number hides the one hot table shard that is usually why someone opened /// this endpoint. #[napi(object)] #[derive(Clone, Debug)] -pub struct BucketStats { - /// The shard this bucket writes. +pub struct TableShardStats { + /// The shard this table shard writes. pub shard_id: String, /// `"Active"` or `"Sealed"` (drop-table 2PC in flight). pub status: String, @@ -1023,20 +1023,20 @@ pub struct BucketStats { /// Highest WAL position the writer has seen. The difference against /// `replayAfterWalEntryPosition` is the WAL lag. pub wal_entry_position_last_seen: i64, - /// Flushed L0 generations not yet merged into the base table. - pub generations: Vec, - /// Whether a pass owns this bucket's compaction latch right now. Says *a* + /// SSTables not yet merged into the base table. + pub sstables: Vec, + /// Whether a pass owns this table shard's compaction latch right now. Says *a* /// driver is running, not *whose*, and the latch is held from dispatch — /// including while the pass queues for a pod-wide compactor permit. Read it /// as "do not pile on", never as "mine is progressing". pub compacting: bool, - /// Oldest first, active last. Absent for a `"Sealed"` bucket, whose + /// Oldest first, active last. Absent for a `"Sealed"` table shard, whose /// in-memory state is torn down. pub memtables: Option>, } -impl From for BucketStats { - fn from(b: lancedb::table::BucketStats) -> Self { +impl From for TableShardStats { + fn from(b: lancedb::table::TableShardStats) -> Self { Self { shard_id: b.shard_id, status: b.status, @@ -1045,7 +1045,7 @@ impl From for BucketStats { current_generation: b.current_generation as i64, replay_after_wal_entry_position: b.replay_after_wal_entry_position as i64, wal_entry_position_last_seen: b.wal_entry_position_last_seen as i64, - generations: b.generations.into_iter().map(Into::into).collect(), + sstables: b.sstables.into_iter().map(Into::into).collect(), compacting: b.compacting, memtables: b .memtables @@ -1054,21 +1054,21 @@ impl From for BucketStats { } } -/// Live per-bucket LSM state, as returned by `Table#getLsmStats`. +/// Live per-table-shard LSM state, as returned by `Table#getLsmStats`. /// -/// Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are +/// Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are /// the caller's to compute. #[napi(object)] #[derive(Clone, Debug)] pub struct LsmStats { - /// One entry per bucket backing this table. - pub buckets: Vec, + /// One entry per table shard backing this table. + pub table_shards: Vec, } impl From for LsmStats { fn from(stats: lancedb::table::LsmStats) -> Self { Self { - buckets: stats.buckets.into_iter().map(Into::into).collect(), + table_shards: stats.table_shards.into_iter().map(Into::into).collect(), } } } diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 7d7ca7f2a..12fc48bfb 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -385,7 +385,7 @@ class Table: async def checkpoint_lsm(self) -> None: ... async def flush_lsm(self) -> None: ... async def compact_lsm(self) -> None: ... - async def get_lsm_stats(self, include_generation_rows: bool) -> Optional[dict]: ... + async def get_lsm_stats(self, include_sstable_rows: bool) -> Optional[dict]: ... async def close_lsm_writers(self) -> None: ... @property def tags(self) -> Tags: ... diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 02748b9bc..3e482a4ce 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -1029,11 +1029,11 @@ class RemoteTable(Table): [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm].""" return LOOP.run(self._table.compact_lsm()) - def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]: + def get_lsm_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]: """Synchronous version of [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats].""" return LOOP.run( - self._table.get_lsm_stats(include_generation_rows=include_generation_rows) + self._table.get_lsm_stats(include_sstable_rows=include_sstable_rows) ) def close_lsm_writers(self) -> None: diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 765b7fa14..8d7ce905c 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -4189,11 +4189,11 @@ class LanceTable(Table): [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm].""" return LOOP.run(self._table.compact_lsm()) - def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]: + def get_lsm_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]: """Synchronous version of [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats].""" return LOOP.run( - self._table.get_lsm_stats(include_generation_rows=include_generation_rows) + self._table.get_lsm_stats(include_sstable_rows=include_sstable_rows) ) def close_lsm_writers(self) -> None: @@ -4916,16 +4916,16 @@ class AsyncTable: async def checkpoint_lsm(self) -> None: """Converge this table's LSM write path into its base table. - One flush, sealing every memtable into L0, then compaction triggers + One flush, freezing every memtable into an SSTable, then compaction triggers until every generation that existed at that moment has reached base. The loop runs client-side, reading progress from ``get_lsm_stats``. - Best-effort: generations created *while* it runs are deliberately not + Best-effort: SSTables created *while* it runs are deliberately not waited on, which is what lets it terminate on a table taking writes. Idempotent and safe on a cadence. There is no deadline, and the caller owns that. It returns when the - target generations are gone, raises on a terminal server fault, and + target SSTables are gone, raises on a terminal server fault, and otherwise waits however long the server takes. A slow table and a stuck one are the same picture from the client: the compactor pool is shared across every table on the node, so a checkpoint queued behind @@ -4936,25 +4936,25 @@ class AsyncTable: await self._inner.checkpoint_lsm() async def flush_lsm(self) -> None: - """Seal every bucket's active memtable into L0. + """Freeze every table shard's active memtable into an SSTable. - Does not touch the base table — moving L0 into base is + Does not touch the base table — compacting SSTables into base is `compact_lsm`. On a node that has not claimed this table, this claims it and replays its WAL log first. """ await self._inner.flush_lsm() async def compact_lsm(self) -> None: - """Trigger a background L0 to base compaction pass per bucket. + """Trigger a background SSTable compaction pass per table shard. Returns once the passes are dispatched, not once they finish: watch ``get_lsm_stats`` for progress, or use ``checkpoint_lsm`` to loop - until the current L0 has reached base. + until the current SSTables have reached base. """ await self._inner.compact_lsm() async def get_lsm_stats( - self, *, include_generation_rows: bool = False + self, *, include_sstable_rows: bool = False ) -> Optional[dict]: """Read live per-bucket LSM state. @@ -4967,12 +4967,12 @@ class AsyncTable: Parameters ---------- - include_generation_rows - Report a row count per L0 generation. Off by default: each count + include_sstable_rows + Report a row count per SSTable. Off by default: each count opens an uncached Lance dataset, and ``checkpoint_lsm`` polls this needing only generation numbers. """ - return await self._inner.get_lsm_stats(include_generation_rows) + return await self._inner.get_lsm_stats(include_sstable_rows) async def close_lsm_writers(self) -> None: """Drain and close any cached MemWAL shard writers for this table. diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index ab0df386d..492a21dcc 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1278,9 +1278,9 @@ def test_get_lsm_stats_sync(): with lsm_test_table(lsm_handler) as table: assert table.get_lsm_stats() == {"buckets": [bucket]} # Off by default, and forwarded when asked for. - assert seen_bodies == [{"include_generation_rows": False}] - table.get_lsm_stats(include_generation_rows=True) - assert seen_bodies[-1] == {"include_generation_rows": True} + assert seen_bodies == [{"include_sstable_rows": False}] + table.get_lsm_stats(include_sstable_rows=True) + assert seen_bodies[-1] == {"include_sstable_rows": True} def test_get_lsm_stats_sync_returns_none_when_lsm_disabled(): @@ -1309,7 +1309,7 @@ def test_flush_and_compact_lsm_sync(): def test_checkpoint_lsm_sync(): - """Seal, read the watermark, and return once L0 holds nothing. + """Freeze, read the watermark, and return once no SSTables remain. The convergence loop itself is covered in Rust; this pins the sync binding to the endpoints it drives. @@ -1319,7 +1319,7 @@ def test_checkpoint_lsm_sync(): def lsm_handler(request, route): called.append(route) if route == "get_lsm_stats": - # An empty L0 yields no target watermark, so the loop is done + # An empty SSTable tier yields no target watermark, so the loop is done # after the seal without ever polling compaction. send_json(request, {"lsm_stats": {"buckets": []}}) else: diff --git a/python/src/table.rs b/python/src/table.rs index 784d29136..a9c45e8de 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -33,16 +33,16 @@ use pyo3::{ mod scannable; -/// Convert `LsmStats` to a Python dict, preserving the per-bucket list. +/// Convert `LsmStats` to a Python dict, preserving the per-table-shard list. /// /// Deliberately not flattened to a table-level summary: a table is N -/// buckets on one node, and the per-bucket detail is the reason the -/// endpoint exists — flattening hides the single hot bucket someone opened +/// table shards on one node, and the per-shard detail is the reason the +/// endpoint exists — flattening hides the single hot table shard someone opened /// it to find. fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult> { let out = PyDict::new(py); - let buckets = PyList::empty(py); - for b in &stats.buckets { + let table_shards = PyList::empty(py); + for b in &stats.table_shards { let e = PyDict::new(py); e.set_item("shard_id", &b.shard_id)?; e.set_item("status", &b.status)?; @@ -58,15 +58,15 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult b.wal_entry_position_last_seen, )?; - let generations = PyList::empty(py); - for g in &b.generations { + let sstables = PyList::empty(py); + for g in &b.sstables { let ge = PyDict::new(py); ge.set_item("generation", g.generation)?; ge.set_item("bytes", g.bytes)?; ge.set_item("rows", g.rows)?; - generations.append(ge)?; + sstables.append(ge)?; } - e.set_item("generations", generations)?; + e.set_item("sstables", sstables)?; e.set_item("compacting", b.compacting)?; e.set_item( @@ -88,9 +88,9 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult }) .transpose()?, )?; - buckets.append(e)?; + table_shards.append(e)?; } - out.set_item("buckets", buckets)?; + out.set_item("table_shards", table_shards)?; Ok(out.unbind()) } @@ -1492,7 +1492,7 @@ impl Table { }) } - /// Seal every bucket's active memtable into L0. + /// Freeze every table shard's active memtable into an SSTable. pub fn flush_lsm(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py( @@ -1501,7 +1501,7 @@ impl Table { ) } - /// Trigger a background L0 → base pass per bucket. Returns once the + /// Trigger a background SSTable compaction pass per table shard. Returns once the /// passes are dispatched, not once they finish — watch `get_lsm_stats`. pub fn compact_lsm(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); @@ -1511,15 +1511,15 @@ impl Table { } /// Live LSM state, or `None` when the LSM write path is not enabled. - #[pyo3(signature = (include_generation_rows=false))] + #[pyo3(signature = (include_sstable_rows=false))] pub fn get_lsm_stats( self_: PyRef<'_, Self>, - include_generation_rows: bool, + include_sstable_rows: bool, ) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { let stats = inner - .get_lsm_stats(include_generation_rows) + .get_lsm_stats(include_sstable_rows) .await .infer_error()?; Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose()) diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index cd346f42e..57d9b8162 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -878,7 +878,7 @@ pub struct QueryRequest { /// [`crate::Table::set_lsm_write_spec`]) is routed through the LSM scanner so /// it also sees data written via the `merge_insert` LSM path that has not yet /// been compacted into the base table — the active and frozen in-memory - /// memtables and the flushed (L0) generations, deduplicated by primary key + /// memtables and the SSTables, deduplicated by primary key /// against the base table (newest generation wins); a table without a spec /// reads the base table. /// diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 1afc2615a..eda768883 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2951,13 +2951,13 @@ impl BaseTable for RemoteTable { Ok(()) } - async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result> { + async fn get_lsm_stats(&self, include_sstable_rows: bool) -> Result> { // Read-semantics POST, like `get_lsm_write_spec`. let request = self .client .post(&format!("/v1/table/{}/get_lsm_stats/", self.identifier)) .json(&serde_json::json!({ - "include_generation_rows": include_generation_rows, + "include_sstable_rows": include_sstable_rows, })); let (request_id, response) = self.send_lsm_route(request).await?; let body = response.text().await.err_to_http(request_id.clone())?; @@ -8260,7 +8260,7 @@ mod tests { http::Response::builder().status(200).body(body).unwrap() } - /// A flush landing in an empty L0 finishes on the opening stats read + /// A flush landing in an empty SSTable tier finishes on the opening stats read /// alone. Asserting zero compacts is the point: "it returned Ok" is also /// true of a loop that ran a pointless pass. #[tokio::test(start_paused = true)] @@ -8314,7 +8314,7 @@ mod tests { } /// Generations created *during* the checkpoint are not waited on, which - /// is what lets the loop terminate on a table taking writes where "L0 is + /// is what lets the loop terminate on a table taking writes where "the SSTable tier is /// empty" never becomes true. #[tokio::test(start_paused = true)] async fn test_checkpoint_ignores_generations_created_while_it_runs() { @@ -8593,7 +8593,7 @@ mod tests { } /// WAL off ⇒ `None`; WAL on ⇒ a fully populated `Some` with no field - /// defaulting to a zero it did not measure. `include_generation_rows` + /// defaulting to a zero it did not measure. `include_sstable_rows` /// rides in the body and is off unless asked for. #[tokio::test] async fn test_get_lsm_stats_round_trip() { @@ -8602,7 +8602,7 @@ mod tests { let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!( - body["include_generation_rows"], true, + body["include_sstable_rows"], true, "the flag must reach the server, not be silently dropped" ); let response = serde_json::json!({ diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 8436657ca..7364a5898 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -102,7 +102,7 @@ use futures::future::join_all; pub use lance::dataset::refs::{BranchContents, Ref, TagContents, Tags as LanceTags}; pub use lance::dataset::scanner::DatasetRecordBatchStream; pub use lance_index::optimize::OptimizeOptions; -pub use lsm_stats::{BucketStats, GenerationStats, LsmStats, MemtableStats}; +pub use lsm_stats::{LsmStats, MemtableStats, SsTableStats, TableShardStats}; pub use optimize::{CompactionOptions, OptimizeAction, OptimizeStats}; pub use refresh::RefreshColumnResult; pub use schema_evolution::{ @@ -673,7 +673,7 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { message: "get_lsm_write_spec is not supported on this table type".into(), }) } - /// Seal every bucket's active memtable into L0. + /// Freeze every table shard's active memtable into an SSTable. /// /// The default implementation returns `NotSupported`. async fn flush_lsm(&self) -> Result<()> { @@ -681,7 +681,7 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { message: "flush_lsm is not supported on this table type".into(), }) } - /// Trigger a background L0 → base compaction pass per bucket. + /// Trigger a background SSTable compaction pass per table shard. /// /// The default implementation returns `NotSupported`. async fn compact_lsm(&self) -> Result<()> { @@ -693,7 +693,7 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { /// enabled for this table. /// /// The default implementation returns `NotSupported`. - async fn get_lsm_stats(&self, _include_generation_rows: bool) -> Result> { + async fn get_lsm_stats(&self, _include_sstable_rows: bool) -> Result> { Err(Error::NotSupported { message: "get_lsm_stats is not supported on this table type".into(), }) @@ -1897,7 +1897,7 @@ impl Table { /// Converge this table's LSM write path into its base table. /// - /// One `flush` to seal every memtable into L0, then compaction triggers + /// One `flush` to freeze every memtable into an SSTable, then compaction triggers /// until every generation that existed at that moment has reached base. /// The loop runs client-side, reading progress from `get_lsm_stats`, so /// there is no held socket and nothing to reconcile if you drop this @@ -1932,10 +1932,10 @@ impl Table { checkpoint::checkpoint_lsm(self).await } - /// Seal every bucket's active memtable into L0 without touching the + /// Freeze every table shard's active memtable into an SSTable without touching the /// base table. /// - /// Independently useful: flushing makes memtable rows readable from L0 at + /// Independently useful: flushing makes memtable rows readable from an SSTable at /// a lower per-query cost. On a node that has not claimed this table it /// claims it and replays the WAL log first — reporting "nothing to flush" /// without replaying would lie about durable data. @@ -1943,7 +1943,7 @@ impl Table { self.inner.flush_lsm().await } - /// Run one bounded L0 → base compaction pass per bucket, reporting what + /// Run one bounded SSTable compaction pass per table shard, reporting what /// it merged and what is left. /// /// One pass, not convergence: that bounds each request's cost and gives a @@ -1959,7 +1959,7 @@ impl Table { /// state, though on a node that has not claimed this table it claims it, /// exactly as a read would. /// - /// `include_generation_rows` reports a row count per L0 generation. Off by + /// `include_sstable_rows` reports a row count per SSTable. Off by /// default: each count opens an uncached Lance dataset, and /// `checkpoint_lsm` polls this needing only generation numbers. /// @@ -1970,8 +1970,8 @@ impl Table { /// /// Do not build a checkpoint's termination on this: the completion /// predicate lives in the `flush` and `compact` responses. - pub async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result> { - self.inner.get_lsm_stats(include_generation_rows).await + pub async fn get_lsm_stats(&self, include_sstable_rows: bool) -> Result> { + self.inner.get_lsm_stats(include_sstable_rows).await } /// Drain and close any cached MemWAL shard writers held for this table. diff --git a/rust/lancedb/src/table/checkpoint.rs b/rust/lancedb/src/table/checkpoint.rs index bb76604ed..1f53023da 100644 --- a/rust/lancedb/src/table/checkpoint.rs +++ b/rust/lancedb/src/table/checkpoint.rs @@ -4,7 +4,7 @@ //! Converging a table's LSM write path into its base table. //! //! `checkpoint_lsm` seals once, then triggers compaction and watches -//! generation numbers until the L0 that existed at the start is gone. +//! generation numbers until the SSTables that existed at the start are gone. //! //! The loop runs in the client, not the server: `compact_lsm` dispatches a //! pass and returns, so nothing holds a socket and a client can vanish @@ -150,7 +150,7 @@ where } /// Drive [`Table::checkpoint_lsm`]: seal once, fix the target watermark -/// from the resulting L0, then trigger and poll until it drains. +/// from the resulting SSTables, then trigger and poll until they drain. pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> { for reissue in 0..=MAX_REISSUES { // The seal turns everything written before this call into a @@ -177,9 +177,9 @@ pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> { return Ok(()); }; let targets: HashMap = stats - .buckets + .table_shards .iter() - .filter_map(|b| Some((b.shard_id.clone(), b.newest_generation()?))) + .filter_map(|b| Some((b.shard_id.clone(), b.newest_sstable_generation()?))) .collect(); if targets.is_empty() { return Ok(()); @@ -226,11 +226,11 @@ async fn drain_to_targets( // with nothing outstanding are skipped, not counted as idle. let mut outstanding = 0; let mut all_compacting = true; - for b in &stats.buckets { + for b in &stats.table_shards { let Some(target) = targets.get(&b.shard_id) else { continue; }; - let n = b.outstanding_generations(*target); + let n = b.outstanding_sstables(*target); if n > 0 { outstanding += n; all_compacting &= b.compacting; diff --git a/rust/lancedb/src/table/lsm_stats.rs b/rust/lancedb/src/table/lsm_stats.rs index 953aea90f..a9af93d07 100644 --- a/rust/lancedb/src/table/lsm_stats.rs +++ b/rust/lancedb/src/table/lsm_stats.rs @@ -1,21 +1,21 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Live per-bucket LSM state — the shape [`crate::Table::get_lsm_stats`] +//! Live per-table_shard LSM state — the shape [`crate::Table::get_lsm_stats`] //! returns and [`super::checkpoint`] polls. //! -//! Nothing here is derived: sums and differences (total L0 bytes, WAL lag) +//! Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) //! are the caller's to compute. There is no "WAL is off" shape — that case is //! `None`, because a struct of zeros would read as measurements. use serde::Deserialize; -/// One flushed L0 generation. +/// One SSTable. #[derive(Debug, Clone, Deserialize)] -pub struct GenerationStats { +pub struct SsTableStats { pub generation: u64, pub bytes: u64, - /// Present only when `include_generation_rows` was requested. Off by + /// Present only when `include_sstable_rows` was requested. Off by /// default because each count opens an uncached Lance dataset, and the /// checkpoint loop polls this route needing only generation numbers. #[serde(default)] @@ -34,11 +34,11 @@ pub struct MemtableStats { pub indexes: Vec, } -/// Live state of one bucket. A table is N buckets on one node; flattening to -/// a single number hides the one hot bucket that is usually why someone +/// Live state of one table_shard. A table is N table_shards on one node; flattening to +/// a single number hides the one hot table_shard that is usually why someone /// opened this endpoint. #[derive(Debug, Clone, Deserialize)] -pub struct BucketStats { +pub struct TableShardStats { pub shard_id: String, /// `Active` | `Sealed` (drop-table 2PC in flight). pub status: String, @@ -47,42 +47,42 @@ pub struct BucketStats { pub current_generation: u64, pub replay_after_wal_entry_position: u64, pub wal_entry_position_last_seen: u64, - pub generations: Vec, - /// Whether a pass owns this bucket's compaction latch right now. Says *a* + pub sstables: Vec, + /// Whether a pass owns this table_shard's compaction latch right now. Says *a* /// driver is running, not *whose*, and the latch is held from dispatch — /// including while the pass queues for a pod-wide compactor permit. Read /// it as "do not pile on", never as "mine is progressing". pub compacting: bool, - /// Oldest first, active last. Absent for a `Sealed` bucket, whose + /// Oldest first, active last. Absent for a `Sealed` table_shard, whose /// in-memory state is torn down. #[serde(default)] pub memtables: Option>, } -impl BucketStats { - /// The newest flushed generation, or `None` when L0 is empty. - pub(crate) fn newest_generation(&self) -> Option { - self.generations.iter().map(|g| g.generation).max() +impl TableShardStats { + /// The newest SSTable generation, or `None` when the tier is empty. + pub(crate) fn newest_sstable_generation(&self) -> Option { + self.sstables.iter().map(|g| g.generation).max() } - /// How many generations at or below `target` are still in L0. + /// How many SSTables at or below `target` are still uncompacted. /// /// A count, not a boolean: one pass drains a bounded prefix rather than /// the whole target set, so a boolean would read as "no progress" for /// every pass but the last. Compaction drains oldest-first, so this /// decreases monotonically. - pub(crate) fn outstanding_generations(&self, target: u64) -> usize { - self.generations + pub(crate) fn outstanding_sstables(&self, target: u64) -> usize { + self.sstables .iter() .filter(|g| g.generation <= target) .count() } } -/// Live LSM state, one entry per bucket. +/// Live LSM state, one entry per table_shard. #[derive(Debug, Clone, Deserialize)] pub struct LsmStats { - pub buckets: Vec, + pub table_shards: Vec, } /// Server-side JSON envelope for `get_lsm_stats`. `lsm_stats` is null when @@ -97,18 +97,18 @@ pub(crate) struct GetLsmStatsResponse { mod tests { use super::*; - fn bucket(shard: &str, generations: &[u64], compacting: bool) -> BucketStats { - BucketStats { + fn table_shard(shard: &str, sstables: &[u64], compacting: bool) -> TableShardStats { + TableShardStats { shard_id: shard.into(), status: "Active".into(), writer_epoch: 1, manifest_version: 1, - current_generation: generations.iter().max().copied().unwrap_or(0) + 1, + current_generation: sstables.iter().max().copied().unwrap_or(0) + 1, replay_after_wal_entry_position: 0, wal_entry_position_last_seen: 0, - generations: generations + sstables: sstables .iter() - .map(|g| GenerationStats { + .map(|g| SsTableStats { generation: *g, bytes: 1, rows: None, @@ -123,40 +123,46 @@ mod tests { /// generation created after it must not hold the loop open — that is why /// the predicate terminates under write load. #[test] - fn newer_generations_do_not_extend_the_target() { - let start = bucket("b0", &[7, 8], false); - let target = start.newest_generation().expect("L0 is non-empty"); + fn newer_sstables_do_not_extend_the_target() { + let start = table_shard("b0", &[7, 8], false); + let target = start + .newest_sstable_generation() + .expect("the SSTable tier is non-empty"); assert_eq!(target, 8); // Compaction drained 7 and 8; 9 and 10 arrived while it ran. - let later = bucket("b0", &[9, 10], false); + let later = table_shard("b0", &[9, 10], false); assert_eq!( - later.outstanding_generations(target), + later.outstanding_sstables(target), 0, - "generations above the target are somebody else's problem" + "sstables above the target are somebody else's problem" ); // Still holding 8 means still outstanding. assert_eq!( - bucket("b0", &[8, 9], false).outstanding_generations(target), + table_shard("b0", &[8, 9], false).outstanding_sstables(target), 1 ); } - /// The metric counts generations, not buckets: a pass drains a bounded - /// prefix, so one bucket going 3 → 2 → 1 → 0 is three steps. + /// The metric counts SSTables, not table shards: a pass drains a bounded + /// prefix, so one table_shard going 3 → 2 → 1 → 0 is three steps. #[test] - fn progress_is_measured_in_generations() { + fn progress_is_measured_in_sstables() { let target = 3; let counts: Vec = [&[1u64, 2, 3][..], &[2, 3][..], &[3][..], &[][..]] .iter() - .map(|gens| bucket("b0", gens, false).outstanding_generations(target)) + .map(|gens| table_shard("b0", gens, false).outstanding_sstables(target)) .collect(); assert_eq!(counts, vec![3, 2, 1, 0]); } #[test] - fn empty_l0_has_no_target() { - assert!(bucket("b0", &[], false).newest_generation().is_none()); + fn an_empty_sstable_tier_has_no_target() { + assert!( + table_shard("b0", &[], false) + .newest_sstable_generation() + .is_none() + ); } }