mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-11 15:52:17 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ab74aa620c |
@@ -221,7 +221,7 @@ abstract checkpointLsm(): Promise<void>
|
|||||||
|
|
||||||
Converge this table's LSM write path into its base table.
|
Converge this table's LSM write path into its base table.
|
||||||
|
|
||||||
Seals once, then triggers compaction and polls until the L0 that existed
|
Freezes once, then triggers compaction and polls until the SSTables that existed
|
||||||
at the start is gone. The target set is fixed at the start, so
|
at the start is gone. The target set is fixed at the start, so
|
||||||
generations created *during* the checkpoint are ignored — that is what
|
generations created *during* the checkpoint are ignored — that is what
|
||||||
lets it terminate under write load, and what makes it best-effort: it
|
lets it terminate under write load, and what makes it best-effort: it
|
||||||
@@ -289,7 +289,7 @@ It is a no-op when no writers are cached.
|
|||||||
abstract compactLsm(): Promise<void>
|
abstract compactLsm(): Promise<void>
|
||||||
```
|
```
|
||||||
|
|
||||||
Trigger a background L0 → base compaction pass per bucket.
|
Trigger a background SSTable compaction pass per table shard.
|
||||||
|
|
||||||
Returns once the passes are *dispatched*, not once they finish — watch
|
Returns once the passes are *dispatched*, not once they finish — watch
|
||||||
[Table#getLsmStats](Table.md#getlsmstats) for progress, or use
|
[Table#getLsmStats](Table.md#getlsmstats) for progress, or use
|
||||||
@@ -505,7 +505,7 @@ Drop an index from the table.
|
|||||||
abstract flushLsm(): Promise<void>
|
abstract flushLsm(): Promise<void>
|
||||||
```
|
```
|
||||||
|
|
||||||
Seal every bucket's active memtable into a new L0 generation.
|
Freeze every table shard's active memtable into a new SSTable.
|
||||||
|
|
||||||
Returns once the seal is committed. Sealing an empty memtable is a no-op,
|
Returns once the seal is committed. Sealing an empty memtable is a no-op,
|
||||||
so this is safe to call repeatedly.
|
so this is safe to call repeatedly.
|
||||||
@@ -519,10 +519,10 @@ so this is safe to call repeatedly.
|
|||||||
### getLsmStats()
|
### getLsmStats()
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
abstract getLsmStats(includeGenerationRows?): Promise<undefined | LsmStats>
|
abstract getLsmStats(includeSstableRows?): Promise<undefined | LsmStats>
|
||||||
```
|
```
|
||||||
|
|
||||||
Read live per-bucket LSM state.
|
Read live per-table-shard LSM state.
|
||||||
|
|
||||||
Answers "how far behind is my fresh tier", "which bucket is hot", and
|
Answers "how far behind is my fresh tier", "which bucket is hot", and
|
||||||
"why is my fresh-tier vector search brute-force". Mutates no table state.
|
"why is my fresh-tier vector search brute-force". Mutates no table state.
|
||||||
@@ -531,8 +531,8 @@ Resolves to `undefined` only when the LSM write path is not enabled.
|
|||||||
|
|
||||||
#### Parameters
|
#### Parameters
|
||||||
|
|
||||||
* **includeGenerationRows?**: `boolean`
|
* **includeSstableRows?**: `boolean`
|
||||||
Also count rows per L0 generation.
|
Also count rows per SSTable.
|
||||||
Off by default because each count opens an uncached Lance dataset.
|
Off by default because each count opens an uncached Lance dataset.
|
||||||
|
|
||||||
#### Returns
|
#### Returns
|
||||||
|
|||||||
@@ -60,7 +60,6 @@
|
|||||||
- [BranchDiff](interfaces/BranchDiff.md)
|
- [BranchDiff](interfaces/BranchDiff.md)
|
||||||
- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
|
- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
|
||||||
- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
|
- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
|
||||||
- [BucketStats](interfaces/BucketStats.md)
|
|
||||||
- [CherryPickError](interfaces/CherryPickError.md)
|
- [CherryPickError](interfaces/CherryPickError.md)
|
||||||
- [CherryPickPreview](interfaces/CherryPickPreview.md)
|
- [CherryPickPreview](interfaces/CherryPickPreview.md)
|
||||||
- [CherryPickResult](interfaces/CherryPickResult.md)
|
- [CherryPickResult](interfaces/CherryPickResult.md)
|
||||||
@@ -87,7 +86,6 @@
|
|||||||
- [FtsToken](interfaces/FtsToken.md)
|
- [FtsToken](interfaces/FtsToken.md)
|
||||||
- [FullTextQuery](interfaces/FullTextQuery.md)
|
- [FullTextQuery](interfaces/FullTextQuery.md)
|
||||||
- [FullTextSearchOptions](interfaces/FullTextSearchOptions.md)
|
- [FullTextSearchOptions](interfaces/FullTextSearchOptions.md)
|
||||||
- [GenerationStats](interfaces/GenerationStats.md)
|
|
||||||
- [HnswPqOptions](interfaces/HnswPqOptions.md)
|
- [HnswPqOptions](interfaces/HnswPqOptions.md)
|
||||||
- [HnswSqOptions](interfaces/HnswSqOptions.md)
|
- [HnswSqOptions](interfaces/HnswSqOptions.md)
|
||||||
- [IndexConfig](interfaces/IndexConfig.md)
|
- [IndexConfig](interfaces/IndexConfig.md)
|
||||||
@@ -126,7 +124,9 @@
|
|||||||
- [SplitHashOptions](interfaces/SplitHashOptions.md)
|
- [SplitHashOptions](interfaces/SplitHashOptions.md)
|
||||||
- [SplitRandomOptions](interfaces/SplitRandomOptions.md)
|
- [SplitRandomOptions](interfaces/SplitRandomOptions.md)
|
||||||
- [SplitSequentialOptions](interfaces/SplitSequentialOptions.md)
|
- [SplitSequentialOptions](interfaces/SplitSequentialOptions.md)
|
||||||
|
- [SsTableStats](interfaces/SsTableStats.md)
|
||||||
- [TableNamesOptions](interfaces/TableNamesOptions.md)
|
- [TableNamesOptions](interfaces/TableNamesOptions.md)
|
||||||
|
- [TableShardStats](interfaces/TableShardStats.md)
|
||||||
- [TableStatistics](interfaces/TableStatistics.md)
|
- [TableStatistics](interfaces/TableStatistics.md)
|
||||||
- [TimeoutConfig](interfaces/TimeoutConfig.md)
|
- [TimeoutConfig](interfaces/TimeoutConfig.md)
|
||||||
- [TlsConfig](interfaces/TlsConfig.md)
|
- [TlsConfig](interfaces/TlsConfig.md)
|
||||||
|
|||||||
@@ -1,40 +0,0 @@
|
|||||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
|
||||||
|
|
||||||
***
|
|
||||||
|
|
||||||
[@lancedb/lancedb](../globals.md) / GenerationStats
|
|
||||||
|
|
||||||
# Interface: GenerationStats
|
|
||||||
|
|
||||||
One flushed L0 generation.
|
|
||||||
|
|
||||||
## Properties
|
|
||||||
|
|
||||||
### bytes
|
|
||||||
|
|
||||||
```ts
|
|
||||||
bytes: number;
|
|
||||||
```
|
|
||||||
|
|
||||||
On-disk size of the generation.
|
|
||||||
|
|
||||||
***
|
|
||||||
|
|
||||||
### generation
|
|
||||||
|
|
||||||
```ts
|
|
||||||
generation: number;
|
|
||||||
```
|
|
||||||
|
|
||||||
The generation number. Increases as memtables are sealed into L0.
|
|
||||||
|
|
||||||
***
|
|
||||||
|
|
||||||
### rows?
|
|
||||||
|
|
||||||
```ts
|
|
||||||
optional rows: number;
|
|
||||||
```
|
|
||||||
|
|
||||||
Present only when `includeGenerationRows` was requested. Off by default
|
|
||||||
because each count opens an uncached Lance dataset.
|
|
||||||
@@ -6,17 +6,17 @@
|
|||||||
|
|
||||||
# Interface: LsmStats
|
# Interface: LsmStats
|
||||||
|
|
||||||
Live per-bucket LSM state, as returned by `Table#getLsmStats`.
|
Live per-table-shard LSM state, as returned by `Table#getLsmStats`.
|
||||||
|
|
||||||
Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
|
Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
|
||||||
the caller's to compute.
|
the caller's to compute.
|
||||||
|
|
||||||
## Properties
|
## Properties
|
||||||
|
|
||||||
### buckets
|
### tableShards
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
buckets: BucketStats[];
|
tableShards: TableShardStats[];
|
||||||
```
|
```
|
||||||
|
|
||||||
One entry per bucket backing this table.
|
One entry per table shard backing this table.
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
[@lancedb/lancedb](../globals.md) / SsTableStats
|
||||||
|
|
||||||
|
# Interface: SsTableStats
|
||||||
|
|
||||||
|
One SSTable.
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
|
||||||
|
### bytes
|
||||||
|
|
||||||
|
```ts
|
||||||
|
bytes: number;
|
||||||
|
```
|
||||||
|
|
||||||
|
On-disk size of the SSTable.
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### generation
|
||||||
|
|
||||||
|
```ts
|
||||||
|
generation: number;
|
||||||
|
```
|
||||||
|
|
||||||
|
The generation number. Increases as memtables are frozen into SSTables.
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### rows?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional rows: number;
|
||||||
|
```
|
||||||
|
|
||||||
|
Present only when `includeSstableRows` was requested. Off by default
|
||||||
|
because each count opens an uncached Lance dataset.
|
||||||
@@ -2,12 +2,12 @@
|
|||||||
|
|
||||||
***
|
***
|
||||||
|
|
||||||
[@lancedb/lancedb](../globals.md) / BucketStats
|
[@lancedb/lancedb](../globals.md) / TableShardStats
|
||||||
|
|
||||||
# Interface: BucketStats
|
# Interface: TableShardStats
|
||||||
|
|
||||||
Live state of one bucket. A table is N buckets on one node; flattening to a
|
Live state of one table shard. A table is N table shards on one node; flattening to a
|
||||||
single number hides the one hot bucket that is usually why someone opened
|
single number hides the one hot table shard that is usually why someone opened
|
||||||
this endpoint.
|
this endpoint.
|
||||||
|
|
||||||
## Properties
|
## Properties
|
||||||
@@ -18,7 +18,7 @@ this endpoint.
|
|||||||
compacting: boolean;
|
compacting: boolean;
|
||||||
```
|
```
|
||||||
|
|
||||||
Whether a pass owns this bucket's compaction latch right now. Says *a*
|
Whether a pass owns this table shard's compaction latch right now. Says *a*
|
||||||
driver is running, not *whose*, and the latch is held from dispatch —
|
driver is running, not *whose*, and the latch is held from dispatch —
|
||||||
including while the pass queues for a pod-wide compactor permit. Read it
|
including while the pass queues for a pod-wide compactor permit. Read it
|
||||||
as "do not pile on", never as "mine is progressing".
|
as "do not pile on", never as "mine is progressing".
|
||||||
@@ -35,13 +35,13 @@ The generation the active memtable will become.
|
|||||||
|
|
||||||
***
|
***
|
||||||
|
|
||||||
### generations
|
### sstables
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
generations: GenerationStats[];
|
sstables: SsTableStats[];
|
||||||
```
|
```
|
||||||
|
|
||||||
Flushed L0 generations not yet merged into the base table.
|
SSTables not yet merged into the base table.
|
||||||
|
|
||||||
***
|
***
|
||||||
|
|
||||||
@@ -61,7 +61,7 @@ Version of the shard manifest these numbers were read from.
|
|||||||
optional memtables: MemtableStats[];
|
optional memtables: MemtableStats[];
|
||||||
```
|
```
|
||||||
|
|
||||||
Oldest first, active last. Absent for a `"Sealed"` bucket, whose
|
Oldest first, active last. Absent for a `"Sealed"` table shard, whose
|
||||||
in-memory state is torn down.
|
in-memory state is torn down.
|
||||||
|
|
||||||
***
|
***
|
||||||
@@ -82,7 +82,7 @@ WAL position replay resumes from.
|
|||||||
shardId: string;
|
shardId: string;
|
||||||
```
|
```
|
||||||
|
|
||||||
The shard this bucket writes.
|
The shard this table shard writes.
|
||||||
|
|
||||||
***
|
***
|
||||||
|
|
||||||
@@ -26,7 +26,7 @@ import java.util.OptionalLong;
|
|||||||
*
|
*
|
||||||
* <p>Installing an {@link LsmWriteSpec} routes {@code mergeInsert} upserts through Lance's MemWAL —
|
* <p>Installing an {@link LsmWriteSpec} routes {@code mergeInsert} upserts through Lance's MemWAL —
|
||||||
* an LSM-style append — instead of the standard merge path. Rows land in an in-memory memtable,
|
* an LSM-style append — instead of the standard merge path. Rows land in an in-memory memtable,
|
||||||
* seal into L0 generations, and are merged into the base table by compaction.
|
* freeze into SSTables, and are merged into the base table by compaction.
|
||||||
*
|
*
|
||||||
* <p>These routes are not part of the Lance Namespace specification, so they are issued directly
|
* <p>These routes are not part of the Lance Namespace specification, so they are issued directly
|
||||||
* rather than through {@link org.lance.namespace.LanceNamespace}.
|
* rather than through {@link org.lance.namespace.LanceNamespace}.
|
||||||
@@ -38,7 +38,7 @@ import java.util.OptionalLong;
|
|||||||
* .buildRestClient();
|
* .buildRestClient();
|
||||||
*
|
*
|
||||||
* LanceDbTableLsm lsm = new LanceDbTableLsm(client, "my_table");
|
* LanceDbTableLsm lsm = new LanceDbTableLsm(client, "my_table");
|
||||||
* lsm.setLsmWriteSpec(LsmWriteSpec.bucket("id", 16));
|
* lsm.setLsmWriteSpec(LsmWriteSpec.tableShard("id", 16));
|
||||||
* // ... merge_insert traffic ...
|
* // ... merge_insert traffic ...
|
||||||
* lsm.checkpointLsm();
|
* lsm.checkpointLsm();
|
||||||
* }</pre>
|
* }</pre>
|
||||||
@@ -94,7 +94,7 @@ public class LanceDbTableLsm {
|
|||||||
* Install an {@link LsmWriteSpec} on this table, selecting the MemWAL LSM write path for future
|
* Install an {@link LsmWriteSpec} on this table, selecting the MemWAL LSM write path for future
|
||||||
* {@code mergeInsert} calls.
|
* {@code mergeInsert} calls.
|
||||||
*
|
*
|
||||||
* <p>All variants require the table to have an unenforced primary key; bucket sharding
|
* <p>All variants require the table to have an unenforced primary key; tableShard sharding
|
||||||
* additionally requires it to be the single column being bucketed.
|
* additionally requires it to be the single column being bucketed.
|
||||||
*/
|
*/
|
||||||
public void setLsmWriteSpec(LsmWriteSpec spec) {
|
public void setLsmWriteSpec(LsmWriteSpec spec) {
|
||||||
@@ -130,7 +130,7 @@ public class LanceDbTableLsm {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Seal every bucket's active memtable into a new L0 generation.
|
* Freeze every table shard's active memtable into a new SSTable.
|
||||||
*
|
*
|
||||||
* <p>Returns once the seal is committed. Sealing an empty memtable is a no-op, so this is safe to
|
* <p>Returns once the seal is committed. Sealing an empty memtable is a no-op, so this is safe to
|
||||||
* call repeatedly.
|
* call repeatedly.
|
||||||
@@ -140,7 +140,7 @@ public class LanceDbTableLsm {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Trigger a background L0 → base compaction pass per bucket.
|
* Trigger a background SSTable compaction pass per table shard.
|
||||||
*
|
*
|
||||||
* <p>Returns once the passes are <em>dispatched</em>, not once they finish — watch {@link
|
* <p>Returns once the passes are <em>dispatched</em>, not once they finish — watch {@link
|
||||||
* #getLsmStats}, or use {@link #checkpointLsm} to wait for convergence.
|
* #getLsmStats}, or use {@link #checkpointLsm} to wait for convergence.
|
||||||
@@ -150,9 +150,9 @@ public class LanceDbTableLsm {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Read live per-bucket LSM state.
|
* Read live per-tableShard LSM state.
|
||||||
*
|
*
|
||||||
* <p>Answers "how far behind is my fresh tier", "which bucket is hot", and "why is my fresh-tier
|
* <p>Answers "how far behind is my fresh tier", "which tableShard is hot", and "why is my fresh-tier
|
||||||
* vector search brute-force". Mutates no table state.
|
* vector search brute-force". Mutates no table state.
|
||||||
*
|
*
|
||||||
* <p>Empty only when the LSM write path is not enabled — that is, when the server sends an absent
|
* <p>Empty only when the LSM write path is not enabled — that is, when the server sends an absent
|
||||||
@@ -160,13 +160,13 @@ public class LanceDbTableLsm {
|
|||||||
* one throws rather than decoding to something empty, because {@link #checkpointLsm} reads
|
* one throws rather than decoding to something empty, because {@link #checkpointLsm} reads
|
||||||
* convergence out of these numbers and cannot tell a defaulted array from a drained one.
|
* convergence out of these numbers and cannot tell a defaulted array from a drained one.
|
||||||
*
|
*
|
||||||
* @param includeGenerationRows Also count rows per L0 generation. Off by default because each
|
* @param includeSstableRows Also count rows per SSTable. Off by default because each
|
||||||
* count opens an uncached Lance dataset.
|
* count opens an uncached Lance dataset.
|
||||||
* @throws IllegalStateException if the response is absent or does not decode.
|
* @throws IllegalStateException if the response is absent or does not decode.
|
||||||
*/
|
*/
|
||||||
public Optional<LsmStats> getLsmStats(boolean includeGenerationRows) {
|
public Optional<LsmStats> getLsmStats(boolean includeSstableRows) {
|
||||||
Map<String, Object> body = new LinkedHashMap<String, Object>();
|
Map<String, Object> body = new LinkedHashMap<String, Object>();
|
||||||
body.put("include_generation_rows", includeGenerationRows);
|
body.put("include_sstable_rows", includeSstableRows);
|
||||||
JsonNode response = client.post(route("get_lsm_stats"), body);
|
JsonNode response = client.post(route("get_lsm_stats"), body);
|
||||||
if (response == null) {
|
if (response == null) {
|
||||||
throw new IllegalStateException("get_lsm_stats returned an empty response body");
|
throw new IllegalStateException("get_lsm_stats returned an empty response body");
|
||||||
@@ -186,8 +186,8 @@ public class LanceDbTableLsm {
|
|||||||
/**
|
/**
|
||||||
* Converge this table's LSM write path into its base table.
|
* Converge this table's LSM write path into its base table.
|
||||||
*
|
*
|
||||||
* <p>Seals once, fixes a target watermark from the resulting L0, then triggers compaction and
|
* <p>Freezes once, fixes a target watermark from the resulting SSTables, then triggers compaction and
|
||||||
* polls until that L0 is gone. The target set is fixed at the start, so generations created
|
* polls until those SSTables are gone. The target set is fixed at the start, so sstables created
|
||||||
* <em>during</em> the checkpoint are ignored — that is what lets it terminate under write load,
|
* <em>during</em> the checkpoint are ignored — that is what lets it terminate under write load,
|
||||||
* and what makes it best-effort: it converges the fresh tier as of some instant. Idempotent,
|
* and what makes it best-effort: it converges the fresh tier as of some instant. Idempotent,
|
||||||
* abandonable at any point, safe on a cadence.
|
* abandonable at any point, safe on a cadence.
|
||||||
@@ -204,7 +204,7 @@ public class LanceDbTableLsm {
|
|||||||
for (int reissue = 0; reissue <= MAX_REISSUES; reissue++) {
|
for (int reissue = 0; reissue <= MAX_REISSUES; reissue++) {
|
||||||
// The seal turns everything written before this call into a generation, so the
|
// The seal turns everything written before this call into a generation, so the
|
||||||
// watermark has to be read after it. Idempotent: sealing an empty memtable is a
|
// watermark has to be read after it. Idempotent: sealing an empty memtable is a
|
||||||
// no-op, so a re-issue does not churn empty generations.
|
// no-op, so a re-issue does not churn empty sstables.
|
||||||
if (issueVoid(this::flushLsm)) {
|
if (issueVoid(this::flushLsm)) {
|
||||||
backoff(reissue);
|
backoff(reissue);
|
||||||
continue;
|
continue;
|
||||||
@@ -220,7 +220,7 @@ public class LanceDbTableLsm {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String, Long> targets = newestGenerations(stats.value.get());
|
Map<String, Long> targets = newestSstableGenerations(stats.value.get());
|
||||||
if (targets.isEmpty()) {
|
if (targets.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -236,7 +236,7 @@ public class LanceDbTableLsm {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Trigger and poll until no bucket holds a generation at or below its target.
|
* Trigger and poll until no tableShard holds a generation at or below its target.
|
||||||
*
|
*
|
||||||
* @return true when the drain finished, false when the table needs re-claiming from flush.
|
* @return true when the drain finished, false when the table needs re-claiming from flush.
|
||||||
*/
|
*/
|
||||||
@@ -250,21 +250,21 @@ public class LanceDbTableLsm {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// `compacting` is the bucket's compaction latch, held from dispatch until the pass
|
// `compacting` is the tableShard's compaction latch, held from dispatch until the pass
|
||||||
// ends — including while it waits on a pod-wide permit. So it answers one question
|
// ends — including while it waits on a pod-wide permit. So it answers one question
|
||||||
// only: do not pile on. Buckets with nothing outstanding are skipped, not counted
|
// only: do not pile on. Buckets with nothing outstanding are skipped, not counted
|
||||||
// as idle.
|
// as idle.
|
||||||
long outstanding = 0;
|
long outstanding = 0;
|
||||||
boolean allCompacting = true;
|
boolean allCompacting = true;
|
||||||
for (BucketStats bucket : stats.value.get().buckets()) {
|
for (TableShardStats tableShard : stats.value.get().tableShards()) {
|
||||||
Long target = targets.get(bucket.shardId());
|
Long target = targets.get(tableShard.shardId());
|
||||||
if (target == null) {
|
if (target == null) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
long remaining = bucket.outstandingGenerations(target);
|
long remaining = tableShard.outstandingSstables(target);
|
||||||
if (remaining > 0) {
|
if (remaining > 0) {
|
||||||
outstanding += remaining;
|
outstanding += remaining;
|
||||||
allCompacting &= bucket.compacting();
|
allCompacting &= tableShard.compacting();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (outstanding == 0) {
|
if (outstanding == 0) {
|
||||||
@@ -281,7 +281,7 @@ public class LanceDbTableLsm {
|
|||||||
if (!isRetryable(e)) {
|
if (!isRetryable(e)) {
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
// A 429 here means the server could latch no bucket at all, which the poll
|
// A 429 here means the server could latch no tableShard at all, which the poll
|
||||||
// above already handles. Not retried in place: the latch it would contend for
|
// above already handles. Not retried in place: the latch it would contend for
|
||||||
// is the one doing the work, so fall through and re-read — POLL_INTERVAL_MS is
|
// is the one doing the work, so fall through and re-read — POLL_INTERVAL_MS is
|
||||||
// the backoff.
|
// the backoff.
|
||||||
@@ -291,13 +291,13 @@ public class LanceDbTableLsm {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The newest generation held by each bucket, skipping buckets holding none. */
|
/** The newest generation held by each tableShard, skipping tableShards holding none. */
|
||||||
private static Map<String, Long> newestGenerations(LsmStats stats) {
|
private static Map<String, Long> newestSstableGenerations(LsmStats stats) {
|
||||||
Map<String, Long> targets = new HashMap<String, Long>();
|
Map<String, Long> targets = new HashMap<String, Long>();
|
||||||
for (BucketStats bucket : stats.buckets()) {
|
for (TableShardStats tableShard : stats.tableShards()) {
|
||||||
OptionalLong newest = bucket.newestGeneration();
|
OptionalLong newest = tableShard.newestSstableGeneration();
|
||||||
if (newest.isPresent()) {
|
if (newest.isPresent()) {
|
||||||
targets.put(bucket.shardId(), newest.getAsLong());
|
targets.put(tableShard.shardId(), newest.getAsLong());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return targets;
|
return targets;
|
||||||
|
|||||||
@@ -20,37 +20,37 @@ import java.util.Collections;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Live per-bucket LSM state, as returned by {@link LanceDbTableLsm#getLsmStats()}.
|
* Live per-tableShard LSM state, as returned by {@link LanceDbTableLsm#getLsmStats()}.
|
||||||
*
|
*
|
||||||
* <p>Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are the caller's to
|
* <p>Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are the caller's to
|
||||||
* compute. There is no "LSM is off" shape — that case is an empty {@link java.util.Optional},
|
* compute. There is no "LSM is off" shape — that case is an empty {@link java.util.Optional},
|
||||||
* because a stats object of zeros would read as measurements.
|
* because a stats object of zeros would read as measurements.
|
||||||
*/
|
*/
|
||||||
public class LsmStats {
|
public class LsmStats {
|
||||||
private static final String CONTEXT = "lsm stats";
|
private static final String CONTEXT = "lsm stats";
|
||||||
|
|
||||||
private final List<BucketStats> buckets;
|
private final List<TableShardStats> tableShards;
|
||||||
|
|
||||||
LsmStats(List<BucketStats> buckets) {
|
LsmStats(List<TableShardStats> tableShards) {
|
||||||
this.buckets = Collections.unmodifiableList(buckets);
|
this.tableShards = Collections.unmodifiableList(tableShards);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** One entry per bucket. */
|
/** One entry per tableShard. */
|
||||||
public List<BucketStats> buckets() {
|
public List<TableShardStats> tableShards() {
|
||||||
return buckets;
|
return tableShards;
|
||||||
}
|
}
|
||||||
|
|
||||||
static LsmStats fromJson(JsonNode node) {
|
static LsmStats fromJson(JsonNode node) {
|
||||||
JsonFields.requiredObject(node, CONTEXT);
|
JsonFields.requiredObject(node, CONTEXT);
|
||||||
List<BucketStats> buckets = new ArrayList<BucketStats>();
|
List<TableShardStats> tableShards = new ArrayList<TableShardStats>();
|
||||||
for (JsonNode bucket : JsonFields.requiredArray(node, "buckets", CONTEXT)) {
|
for (JsonNode tableShard : JsonFields.requiredArray(node, "table_shards", CONTEXT)) {
|
||||||
buckets.add(BucketStats.fromJson(bucket));
|
tableShards.add(TableShardStats.fromJson(tableShard));
|
||||||
}
|
}
|
||||||
return new LsmStats(buckets);
|
return new LsmStats(tableShards);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "LsmStats{buckets=" + buckets + "}";
|
return "LsmStats{tableShards=" + tableShards + "}";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+8
-8
@@ -17,21 +17,21 @@ import com.fasterxml.jackson.databind.JsonNode;
|
|||||||
|
|
||||||
import java.util.OptionalLong;
|
import java.util.OptionalLong;
|
||||||
|
|
||||||
/** One flushed L0 generation. */
|
/** One SSTable. */
|
||||||
public class GenerationStats {
|
public class SsTableStats {
|
||||||
private static final String CONTEXT = "generation stats";
|
private static final String CONTEXT = "generation stats";
|
||||||
|
|
||||||
private final long generation;
|
private final long generation;
|
||||||
private final long bytes;
|
private final long bytes;
|
||||||
private final Long rows;
|
private final Long rows;
|
||||||
|
|
||||||
GenerationStats(long generation, long bytes, Long rows) {
|
SsTableStats(long generation, long bytes, Long rows) {
|
||||||
this.generation = generation;
|
this.generation = generation;
|
||||||
this.bytes = bytes;
|
this.bytes = bytes;
|
||||||
this.rows = rows;
|
this.rows = rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The generation number. Increases as memtables are sealed into L0. */
|
/** The generation number. Increases as memtables are frozen into SSTables. */
|
||||||
public long generation() {
|
public long generation() {
|
||||||
return generation;
|
return generation;
|
||||||
}
|
}
|
||||||
@@ -42,16 +42,16 @@ public class GenerationStats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Rows in this generation, present only when {@code includeGenerationRows} was requested. Off by
|
* Rows in this generation, present only when {@code includeSstableRows} was requested. Off by
|
||||||
* default because each count opens an uncached Lance dataset.
|
* default because each count opens an uncached Lance dataset.
|
||||||
*/
|
*/
|
||||||
public OptionalLong rows() {
|
public OptionalLong rows() {
|
||||||
return rows == null ? OptionalLong.empty() : OptionalLong.of(rows);
|
return rows == null ? OptionalLong.empty() : OptionalLong.of(rows);
|
||||||
}
|
}
|
||||||
|
|
||||||
static GenerationStats fromJson(JsonNode node) {
|
static SsTableStats fromJson(JsonNode node) {
|
||||||
JsonFields.requiredObject(node, CONTEXT);
|
JsonFields.requiredObject(node, CONTEXT);
|
||||||
return new GenerationStats(
|
return new SsTableStats(
|
||||||
JsonFields.requiredLong(node, "generation", CONTEXT),
|
JsonFields.requiredLong(node, "generation", CONTEXT),
|
||||||
JsonFields.requiredLong(node, "bytes", CONTEXT),
|
JsonFields.requiredLong(node, "bytes", CONTEXT),
|
||||||
JsonFields.optionalLong(node, "rows", CONTEXT));
|
JsonFields.optionalLong(node, "rows", CONTEXT));
|
||||||
@@ -59,6 +59,6 @@ public class GenerationStats {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "GenerationStats{generation=" + generation + ", bytes=" + bytes + ", rows=" + rows + "}";
|
return "SsTableStats{generation=" + generation + ", bytes=" + bytes + ", rows=" + rows + "}";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+29
-29
@@ -22,11 +22,11 @@ import java.util.Optional;
|
|||||||
import java.util.OptionalLong;
|
import java.util.OptionalLong;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Live state of one bucket. A table is N buckets on one node; flattening to a single number hides
|
* Live state of one tableShard. A table is N tableShards on one node; flattening to a single number hides
|
||||||
* the one hot bucket that is usually why someone opened this endpoint.
|
* the one hot tableShard that is usually why someone opened this endpoint.
|
||||||
*/
|
*/
|
||||||
public class BucketStats {
|
public class TableShardStats {
|
||||||
private static final String CONTEXT = "bucket stats";
|
private static final String CONTEXT = "tableShard stats";
|
||||||
|
|
||||||
private final String shardId;
|
private final String shardId;
|
||||||
private final String status;
|
private final String status;
|
||||||
@@ -35,11 +35,11 @@ public class BucketStats {
|
|||||||
private final long currentGeneration;
|
private final long currentGeneration;
|
||||||
private final long replayAfterWalEntryPosition;
|
private final long replayAfterWalEntryPosition;
|
||||||
private final long walEntryPositionLastSeen;
|
private final long walEntryPositionLastSeen;
|
||||||
private final List<GenerationStats> generations;
|
private final List<SsTableStats> sstables;
|
||||||
private final boolean compacting;
|
private final boolean compacting;
|
||||||
private final List<MemtableStats> memtables;
|
private final List<MemtableStats> memtables;
|
||||||
|
|
||||||
BucketStats(
|
TableShardStats(
|
||||||
String shardId,
|
String shardId,
|
||||||
String status,
|
String status,
|
||||||
long writerEpoch,
|
long writerEpoch,
|
||||||
@@ -47,7 +47,7 @@ public class BucketStats {
|
|||||||
long currentGeneration,
|
long currentGeneration,
|
||||||
long replayAfterWalEntryPosition,
|
long replayAfterWalEntryPosition,
|
||||||
long walEntryPositionLastSeen,
|
long walEntryPositionLastSeen,
|
||||||
List<GenerationStats> generations,
|
List<SsTableStats> sstables,
|
||||||
boolean compacting,
|
boolean compacting,
|
||||||
List<MemtableStats> memtables) {
|
List<MemtableStats> memtables) {
|
||||||
this.shardId = shardId;
|
this.shardId = shardId;
|
||||||
@@ -57,12 +57,12 @@ public class BucketStats {
|
|||||||
this.currentGeneration = currentGeneration;
|
this.currentGeneration = currentGeneration;
|
||||||
this.replayAfterWalEntryPosition = replayAfterWalEntryPosition;
|
this.replayAfterWalEntryPosition = replayAfterWalEntryPosition;
|
||||||
this.walEntryPositionLastSeen = walEntryPositionLastSeen;
|
this.walEntryPositionLastSeen = walEntryPositionLastSeen;
|
||||||
this.generations = Collections.unmodifiableList(generations);
|
this.sstables = Collections.unmodifiableList(sstables);
|
||||||
this.compacting = compacting;
|
this.compacting = compacting;
|
||||||
this.memtables = memtables == null ? null : Collections.unmodifiableList(memtables);
|
this.memtables = memtables == null ? null : Collections.unmodifiableList(memtables);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The shard this bucket writes. */
|
/** The shard this tableShard writes. */
|
||||||
public String shardId() {
|
public String shardId() {
|
||||||
return shardId;
|
return shardId;
|
||||||
}
|
}
|
||||||
@@ -100,13 +100,13 @@ public class BucketStats {
|
|||||||
return walEntryPositionLastSeen;
|
return walEntryPositionLastSeen;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Flushed L0 generations not yet merged into the base table. */
|
/** SSTables not yet merged into the base table. */
|
||||||
public List<GenerationStats> generations() {
|
public List<SsTableStats> sstables() {
|
||||||
return generations;
|
return sstables;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Whether a pass owns this bucket's compaction latch right now. Says <em>a</em> driver is
|
* Whether a pass owns this tableShard's compaction latch right now. Says <em>a</em> driver is
|
||||||
* running, not <em>whose</em>, and the latch is held from dispatch — including while the pass
|
* running, not <em>whose</em>, and the latch is held from dispatch — including while the pass
|
||||||
* queues for a pod-wide compactor permit. Read it as "do not pile on", never as "mine is
|
* queues for a pod-wide compactor permit. Read it as "do not pile on", never as "mine is
|
||||||
* progressing".
|
* progressing".
|
||||||
@@ -115,15 +115,15 @@ public class BucketStats {
|
|||||||
return compacting;
|
return compacting;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Oldest first, active last. Empty for a {@code "Sealed"} bucket, whose state is torn down. */
|
/** Oldest first, active last. Empty for a {@code "Sealed"} tableShard, whose state is torn down. */
|
||||||
public Optional<List<MemtableStats>> memtables() {
|
public Optional<List<MemtableStats>> memtables() {
|
||||||
return Optional.ofNullable(memtables);
|
return Optional.ofNullable(memtables);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The newest flushed generation, or empty when L0 is empty. */
|
/** The newest SSTable generation, or empty when the tier is empty. */
|
||||||
OptionalLong newestGeneration() {
|
OptionalLong newestSstableGeneration() {
|
||||||
OptionalLong newest = OptionalLong.empty();
|
OptionalLong newest = OptionalLong.empty();
|
||||||
for (GenerationStats generation : generations) {
|
for (SsTableStats generation : sstables) {
|
||||||
if (!newest.isPresent() || generation.generation() > newest.getAsLong()) {
|
if (!newest.isPresent() || generation.generation() > newest.getAsLong()) {
|
||||||
newest = OptionalLong.of(generation.generation());
|
newest = OptionalLong.of(generation.generation());
|
||||||
}
|
}
|
||||||
@@ -132,15 +132,15 @@ public class BucketStats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* How many generations at or below {@code target} are still in L0.
|
* How many SSTables at or below {@code target} are still uncompacted.
|
||||||
*
|
*
|
||||||
* <p>A count, not a boolean: one pass drains a bounded prefix rather than the whole target set,
|
* <p>A count, not a boolean: one pass drains a bounded prefix rather than the whole target set,
|
||||||
* so a boolean would read as "no progress" for every pass but the last. Compaction drains
|
* so a boolean would read as "no progress" for every pass but the last. Compaction drains
|
||||||
* oldest-first, so this decreases monotonically.
|
* oldest-first, so this decreases monotonically.
|
||||||
*/
|
*/
|
||||||
long outstandingGenerations(long target) {
|
long outstandingSstables(long target) {
|
||||||
long count = 0;
|
long count = 0;
|
||||||
for (GenerationStats generation : generations) {
|
for (SsTableStats generation : sstables) {
|
||||||
if (generation.generation() <= target) {
|
if (generation.generation() <= target) {
|
||||||
count++;
|
count++;
|
||||||
}
|
}
|
||||||
@@ -148,11 +148,11 @@ public class BucketStats {
|
|||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
static BucketStats fromJson(JsonNode node) {
|
static TableShardStats fromJson(JsonNode node) {
|
||||||
JsonFields.requiredObject(node, CONTEXT);
|
JsonFields.requiredObject(node, CONTEXT);
|
||||||
List<GenerationStats> generations = new ArrayList<GenerationStats>();
|
List<SsTableStats> sstables = new ArrayList<SsTableStats>();
|
||||||
for (JsonNode generation : JsonFields.requiredArray(node, "generations", CONTEXT)) {
|
for (JsonNode generation : JsonFields.requiredArray(node, "sstables", CONTEXT)) {
|
||||||
generations.add(GenerationStats.fromJson(generation));
|
sstables.add(SsTableStats.fromJson(generation));
|
||||||
}
|
}
|
||||||
|
|
||||||
JsonNode memtablesNode = JsonFields.optionalArray(node, "memtables", CONTEXT);
|
JsonNode memtablesNode = JsonFields.optionalArray(node, "memtables", CONTEXT);
|
||||||
@@ -164,7 +164,7 @@ public class BucketStats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return new BucketStats(
|
return new TableShardStats(
|
||||||
JsonFields.requiredText(node, "shard_id", CONTEXT),
|
JsonFields.requiredText(node, "shard_id", CONTEXT),
|
||||||
JsonFields.requiredText(node, "status", CONTEXT),
|
JsonFields.requiredText(node, "status", CONTEXT),
|
||||||
JsonFields.requiredLong(node, "writer_epoch", CONTEXT),
|
JsonFields.requiredLong(node, "writer_epoch", CONTEXT),
|
||||||
@@ -172,21 +172,21 @@ public class BucketStats {
|
|||||||
JsonFields.requiredLong(node, "current_generation", CONTEXT),
|
JsonFields.requiredLong(node, "current_generation", CONTEXT),
|
||||||
JsonFields.requiredLong(node, "replay_after_wal_entry_position", CONTEXT),
|
JsonFields.requiredLong(node, "replay_after_wal_entry_position", CONTEXT),
|
||||||
JsonFields.requiredLong(node, "wal_entry_position_last_seen", CONTEXT),
|
JsonFields.requiredLong(node, "wal_entry_position_last_seen", CONTEXT),
|
||||||
generations,
|
sstables,
|
||||||
JsonFields.requiredBoolean(node, "compacting", CONTEXT),
|
JsonFields.requiredBoolean(node, "compacting", CONTEXT),
|
||||||
memtables);
|
memtables);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "BucketStats{shardId="
|
return "TableShardStats{shardId="
|
||||||
+ shardId
|
+ shardId
|
||||||
+ ", status="
|
+ ", status="
|
||||||
+ status
|
+ status
|
||||||
+ ", currentGeneration="
|
+ ", currentGeneration="
|
||||||
+ currentGeneration
|
+ currentGeneration
|
||||||
+ ", generations="
|
+ ", sstables="
|
||||||
+ generations
|
+ sstables
|
||||||
+ ", compacting="
|
+ ", compacting="
|
||||||
+ compacting
|
+ compacting
|
||||||
+ "}";
|
+ "}";
|
||||||
@@ -132,10 +132,10 @@ public class LanceDbTableLsmTest {
|
|||||||
enqueue("set_lsm_write_spec", 200, "");
|
enqueue("set_lsm_write_spec", 200, "");
|
||||||
|
|
||||||
lsm.setLsmWriteSpec(
|
lsm.setLsmWriteSpec(
|
||||||
LsmWriteSpec.bucket("id", 16).withMaintainedIndexes(Arrays.asList("id_idx")));
|
LsmWriteSpec.tableShard("id", 16).withMaintainedIndexes(Arrays.asList("id_idx")));
|
||||||
|
|
||||||
JsonNode body = MAPPER.readTree(requestBodies.get(0));
|
JsonNode body = MAPPER.readTree(requestBodies.get(0));
|
||||||
assertEquals("bucket", body.get("sharding").get("mode").asText());
|
assertEquals("tableShard", body.get("sharding").get("mode").asText());
|
||||||
assertEquals("id", body.get("sharding").get("column").asText());
|
assertEquals("id", body.get("sharding").get("column").asText());
|
||||||
assertEquals(16, body.get("sharding").get("num_buckets").asInt());
|
assertEquals(16, body.get("sharding").get("num_buckets").asInt());
|
||||||
assertEquals(1, body.get("maintained_indexes").size());
|
assertEquals(1, body.get("maintained_indexes").size());
|
||||||
@@ -201,7 +201,7 @@ public class LanceDbTableLsmTest {
|
|||||||
enqueue(
|
enqueue(
|
||||||
"get_lsm_write_spec",
|
"get_lsm_write_spec",
|
||||||
200,
|
200,
|
||||||
"{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"bucket\",\"column\":\"id\","
|
"{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"tableShard\",\"column\":\"id\","
|
||||||
+ "\"num_buckets\":16},\"maintained_indexes\":[\"id_idx\"],"
|
+ "\"num_buckets\":16},\"maintained_indexes\":[\"id_idx\"],"
|
||||||
+ "\"writer_config_defaults\":{\"durable_write\":\"true\"}}}");
|
+ "\"writer_config_defaults\":{\"durable_write\":\"true\"}}}");
|
||||||
|
|
||||||
@@ -228,14 +228,14 @@ public class LanceDbTableLsmTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testGetLsmStats() throws Exception {
|
public void testGetLsmStats() throws Exception {
|
||||||
enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L)));
|
enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 7L, 8L)));
|
||||||
|
|
||||||
Optional<LsmStats> got = lsm.getLsmStats(true);
|
Optional<LsmStats> got = lsm.getLsmStats(true);
|
||||||
|
|
||||||
assertEquals("/v1/table/my_table/get_lsm_stats/", requestPaths.get(0));
|
assertEquals("/v1/table/my_table/get_lsm_stats/", requestPaths.get(0));
|
||||||
assertTrue(MAPPER.readTree(requestBodies.get(0)).get("include_generation_rows").asBoolean());
|
assertTrue(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean());
|
||||||
assertTrue(got.isPresent());
|
assertTrue(got.isPresent());
|
||||||
BucketStats decoded = got.get().buckets().get(0);
|
TableShardStats decoded = got.get().tableShards().get(0);
|
||||||
assertEquals("shard-0", decoded.shardId());
|
assertEquals("shard-0", decoded.shardId());
|
||||||
assertEquals("Active", decoded.status());
|
assertEquals("Active", decoded.status());
|
||||||
assertEquals(1, decoded.writerEpoch());
|
assertEquals(1, decoded.writerEpoch());
|
||||||
@@ -243,8 +243,8 @@ public class LanceDbTableLsmTest {
|
|||||||
assertEquals(9, decoded.currentGeneration());
|
assertEquals(9, decoded.currentGeneration());
|
||||||
assertFalse(decoded.compacting());
|
assertFalse(decoded.compacting());
|
||||||
assertEquals(Arrays.asList(7L, 8L), generationNumbers(decoded));
|
assertEquals(Arrays.asList(7L, 8L), generationNumbers(decoded));
|
||||||
assertEquals(1024, decoded.generations().get(0).bytes());
|
assertEquals(1024, decoded.sstables().get(0).bytes());
|
||||||
assertFalse(decoded.generations().get(0).rows().isPresent(), "rows absent unless requested");
|
assertFalse(decoded.sstables().get(0).rows().isPresent(), "rows absent unless requested");
|
||||||
assertFalse(decoded.memtables().isPresent(), "absent memtables stay absent");
|
assertFalse(decoded.memtables().isPresent(), "absent memtables stay absent");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -254,19 +254,19 @@ public class LanceDbTableLsmTest {
|
|||||||
enqueue(
|
enqueue(
|
||||||
"get_lsm_stats",
|
"get_lsm_stats",
|
||||||
200,
|
200,
|
||||||
"{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
|
"{\"lsm_stats\":{\"tableShards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
|
||||||
+ "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
|
+ "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
|
||||||
+ "\"replay_after_wal_entry_position\":3,\"wal_entry_position_last_seen\":11,"
|
+ "\"replay_after_wal_entry_position\":3,\"wal_entry_position_last_seen\":11,"
|
||||||
+ "\"generations\":[{\"generation\":7,\"bytes\":1024,\"rows\":42}],"
|
+ "\"sstables\":[{\"generation\":7,\"bytes\":1024,\"rows\":42}],"
|
||||||
+ "\"compacting\":true,\"memtables\":[{\"generation\":8,\"rows\":5,"
|
+ "\"compacting\":true,\"memtables\":[{\"generation\":8,\"rows\":5,"
|
||||||
+ "\"bytes\":64,\"batches\":2,\"indexes\":[\"id_idx\"]}]}]}}");
|
+ "\"bytes\":64,\"batches\":2,\"indexes\":[\"id_idx\"]}]}]}}");
|
||||||
|
|
||||||
BucketStats decoded = lsm.getLsmStats(true).get().buckets().get(0);
|
TableShardStats decoded = lsm.getLsmStats(true).get().tableShards().get(0);
|
||||||
|
|
||||||
assertEquals(3, decoded.replayAfterWalEntryPosition());
|
assertEquals(3, decoded.replayAfterWalEntryPosition());
|
||||||
assertEquals(11, decoded.walEntryPositionLastSeen());
|
assertEquals(11, decoded.walEntryPositionLastSeen());
|
||||||
assertTrue(decoded.compacting());
|
assertTrue(decoded.compacting());
|
||||||
assertEquals(42, decoded.generations().get(0).rows().getAsLong());
|
assertEquals(42, decoded.sstables().get(0).rows().getAsLong());
|
||||||
assertTrue(decoded.memtables().isPresent());
|
assertTrue(decoded.memtables().isPresent());
|
||||||
MemtableStats memtable = decoded.memtables().get().get(0);
|
MemtableStats memtable = decoded.memtables().get().get(0);
|
||||||
assertEquals(8, memtable.generation());
|
assertEquals(8, memtable.generation());
|
||||||
@@ -289,7 +289,7 @@ public class LanceDbTableLsmTest {
|
|||||||
|
|
||||||
lsm.getLsmStats();
|
lsm.getLsmStats();
|
||||||
|
|
||||||
assertFalse(MAPPER.readTree(requestBodies.get(0)).get("include_generation_rows").asBoolean());
|
assertFalse(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean());
|
||||||
}
|
}
|
||||||
|
|
||||||
// ===========================================================================
|
// ===========================================================================
|
||||||
@@ -334,8 +334,8 @@ public class LanceDbTableLsmTest {
|
|||||||
@Test
|
@Test
|
||||||
public void testCheckpointReturnsWhenNoGenerationsOutstanding() {
|
public void testCheckpointReturnsWhenNoGenerationsOutstanding() {
|
||||||
enqueue("flush_lsm", 200, "");
|
enqueue("flush_lsm", 200, "");
|
||||||
// A bucket with no L0 generations yields no target, so the drain never starts.
|
// A table shard with no SSTables yields no target, so the drain never starts.
|
||||||
enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false)));
|
enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false)));
|
||||||
|
|
||||||
lsm.checkpointLsm();
|
lsm.checkpointLsm();
|
||||||
|
|
||||||
@@ -345,12 +345,12 @@ public class LanceDbTableLsmTest {
|
|||||||
@Test
|
@Test
|
||||||
public void testCheckpointConvergesOnceTargetGenerationsAreGone() {
|
public void testCheckpointConvergesOnceTargetGenerationsAreGone() {
|
||||||
enqueue("flush_lsm", 200, "");
|
enqueue("flush_lsm", 200, "");
|
||||||
// Watermark read: shard-0 holds generations 7 and 8, so target = 8.
|
// Watermark read: shard-0 holds sstables 7 and 8, so target = 8.
|
||||||
enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L)));
|
enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 7L, 8L)));
|
||||||
// First drain poll: both still outstanding, nothing compacting -> dispatch a pass.
|
// First drain poll: both still outstanding, nothing compacting -> dispatch a pass.
|
||||||
enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L)));
|
enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 7L, 8L)));
|
||||||
// Second drain poll: drained past the target -> done.
|
// Second drain poll: drained past the target -> done.
|
||||||
enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 9L)));
|
enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 9L)));
|
||||||
enqueue("compact_lsm", 200, "");
|
enqueue("compact_lsm", 200, "");
|
||||||
|
|
||||||
lsm.checkpointLsm();
|
lsm.checkpointLsm();
|
||||||
@@ -362,14 +362,14 @@ public class LanceDbTableLsmTest {
|
|||||||
@Test
|
@Test
|
||||||
public void testCheckpointDoesNotPileOnWhileEveryTargetBucketIsCompacting() {
|
public void testCheckpointDoesNotPileOnWhileEveryTargetBucketIsCompacting() {
|
||||||
enqueue("flush_lsm", 200, "");
|
enqueue("flush_lsm", 200, "");
|
||||||
enqueue("get_lsm_stats", 200, stats(bucket("shard-0", true, 4L)));
|
enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", true, 4L)));
|
||||||
// Still compacting on the first poll, so no pass is dispatched; then it drains.
|
// Still compacting on the first poll, so no pass is dispatched; then it drains.
|
||||||
enqueue("get_lsm_stats", 200, stats(bucket("shard-0", true, 4L)));
|
enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", true, 4L)));
|
||||||
enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 5L)));
|
enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 5L)));
|
||||||
|
|
||||||
lsm.checkpointLsm();
|
lsm.checkpointLsm();
|
||||||
|
|
||||||
assertEquals(0, countCalls("compact_lsm"), "a latched bucket is left alone");
|
assertEquals(0, countCalls("compact_lsm"), "a latched tableShard is left alone");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -378,7 +378,7 @@ public class LanceDbTableLsmTest {
|
|||||||
// from flush rather than retrying the read in place.
|
// from flush rather than retrying the read in place.
|
||||||
enqueue("flush_lsm", 200, "");
|
enqueue("flush_lsm", 200, "");
|
||||||
enqueue("get_lsm_stats", 421, "no claim");
|
enqueue("get_lsm_stats", 421, "no claim");
|
||||||
enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false)));
|
enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false)));
|
||||||
|
|
||||||
lsm.checkpointLsm();
|
lsm.checkpointLsm();
|
||||||
|
|
||||||
@@ -389,7 +389,7 @@ public class LanceDbTableLsmTest {
|
|||||||
public void testCheckpointRetriesRetryableStatusInPlace() {
|
public void testCheckpointRetriesRetryableStatusInPlace() {
|
||||||
enqueue("flush_lsm", 429, "latch held");
|
enqueue("flush_lsm", 429, "latch held");
|
||||||
enqueue("flush_lsm", 200, "");
|
enqueue("flush_lsm", 200, "");
|
||||||
enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false)));
|
enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false)));
|
||||||
|
|
||||||
lsm.checkpointLsm();
|
lsm.checkpointLsm();
|
||||||
|
|
||||||
@@ -421,27 +421,27 @@ public class LanceDbTableLsmTest {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* A stats payload that does not decode must fail closed. Every one of these bodies used to be
|
* A stats payload that does not decode must fail closed. Every one of these bodies used to be
|
||||||
* read as "no buckets", which is indistinguishable from a drained table, so {@code checkpointLsm}
|
* read as "no tableShards", which is indistinguishable from a drained table, so {@code checkpointLsm}
|
||||||
* reported convergence for a checkpoint that never ran.
|
* reported convergence for a checkpoint that never ran.
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void testCheckpointRejectsMalformedStats() {
|
public void testCheckpointRejectsMalformedStats() {
|
||||||
Map<String, String> malformed = new LinkedHashMap<String, String>();
|
Map<String, String> malformed = new LinkedHashMap<String, String>();
|
||||||
malformed.put("no response body at all", "");
|
malformed.put("no response body at all", "");
|
||||||
malformed.put("stats object with no buckets", "{\"lsm_stats\":{}}");
|
malformed.put("stats object with no tableShards", "{\"lsm_stats\":{}}");
|
||||||
malformed.put("bucket missing its required fields", "{\"lsm_stats\":{\"buckets\":[{}]}}");
|
malformed.put("tableShard missing its required fields", "{\"lsm_stats\":{\"tableShards\":[{}]}}");
|
||||||
malformed.put(
|
malformed.put(
|
||||||
"bucket missing generations",
|
"tableShard missing sstables",
|
||||||
"{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
|
"{\"lsm_stats\":{\"tableShards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
|
||||||
+ "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
|
+ "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
|
||||||
+ "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0,"
|
+ "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0,"
|
||||||
+ "\"compacting\":false}]}}");
|
+ "\"compacting\":false}]}}");
|
||||||
malformed.put(
|
malformed.put(
|
||||||
"generation with a non-numeric generation number",
|
"generation with a non-numeric generation number",
|
||||||
"{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
|
"{\"lsm_stats\":{\"tableShards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
|
||||||
+ "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
|
+ "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
|
||||||
+ "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0,"
|
+ "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0,"
|
||||||
+ "\"generations\":[{\"generation\":\"7\",\"bytes\":1024}],"
|
+ "\"sstables\":[{\"generation\":\"7\",\"bytes\":1024}],"
|
||||||
+ "\"compacting\":false}]}}");
|
+ "\"compacting\":false}]}}");
|
||||||
|
|
||||||
for (Map.Entry<String, String> each : malformed.entrySet()) {
|
for (Map.Entry<String, String> each : malformed.entrySet()) {
|
||||||
@@ -492,22 +492,22 @@ public class LanceDbTableLsmTest {
|
|||||||
// harness
|
// harness
|
||||||
// ===========================================================================
|
// ===========================================================================
|
||||||
|
|
||||||
private static List<Long> generationNumbers(BucketStats bucket) {
|
private static List<Long> generationNumbers(TableShardStats tableShard) {
|
||||||
List<Long> numbers = new ArrayList<Long>();
|
List<Long> numbers = new ArrayList<Long>();
|
||||||
for (GenerationStats generation : bucket.generations()) {
|
for (SsTableStats generation : tableShard.sstables()) {
|
||||||
numbers.add(generation.generation());
|
numbers.add(generation.generation());
|
||||||
}
|
}
|
||||||
return numbers;
|
return numbers;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Build an {@code lsm_stats} response body from bucket fragments. */
|
/** Build an {@code lsm_stats} response body from tableShard fragments. */
|
||||||
private static String stats(String... buckets) {
|
private static String stats(String... tableShards) {
|
||||||
return "{\"lsm_stats\":{\"buckets\":[" + String.join(",", buckets) + "]}}";
|
return "{\"lsm_stats\":{\"tableShards\":[" + String.join(",", tableShards) + "]}}";
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String bucket(String shardId, boolean compacting, Long... generations) {
|
private static String tableShard(String shardId, boolean compacting, Long... sstables) {
|
||||||
StringBuilder gens = new StringBuilder();
|
StringBuilder gens = new StringBuilder();
|
||||||
for (Long generation : generations) {
|
for (Long generation : sstables) {
|
||||||
if (gens.length() > 0) {
|
if (gens.length() > 0) {
|
||||||
gens.append(",");
|
gens.append(",");
|
||||||
}
|
}
|
||||||
@@ -517,7 +517,7 @@ public class LanceDbTableLsmTest {
|
|||||||
+ shardId
|
+ shardId
|
||||||
+ "\",\"status\":\"Active\",\"writer_epoch\":1,\"manifest_version\":2,"
|
+ "\",\"status\":\"Active\",\"writer_epoch\":1,\"manifest_version\":2,"
|
||||||
+ "\"current_generation\":9,\"replay_after_wal_entry_position\":0,"
|
+ "\"current_generation\":9,\"replay_after_wal_entry_position\":0,"
|
||||||
+ "\"wal_entry_position_last_seen\":0,\"generations\":["
|
+ "\"wal_entry_position_last_seen\":0,\"sstables\":["
|
||||||
+ gens
|
+ gens
|
||||||
+ "],\"compacting\":"
|
+ "],\"compacting\":"
|
||||||
+ compacting
|
+ compacting
|
||||||
|
|||||||
@@ -157,8 +157,8 @@ export {
|
|||||||
TokenizeTableOptions,
|
TokenizeTableOptions,
|
||||||
LsmWriteSpec,
|
LsmWriteSpec,
|
||||||
LsmStats,
|
LsmStats,
|
||||||
BucketStats,
|
TableShardStats,
|
||||||
GenerationStats,
|
SsTableStats,
|
||||||
MemtableStats,
|
MemtableStats,
|
||||||
ColumnAlteration,
|
ColumnAlteration,
|
||||||
FieldMetadataUpdate,
|
FieldMetadataUpdate,
|
||||||
|
|||||||
+10
-10
@@ -55,8 +55,8 @@ import { sanitizeType } from "./sanitize";
|
|||||||
import { IntoSql, toSQL } from "./util";
|
import { IntoSql, toSQL } from "./util";
|
||||||
export { IndexConfig } from "./native";
|
export { IndexConfig } from "./native";
|
||||||
export {
|
export {
|
||||||
BucketStats,
|
TableShardStats,
|
||||||
GenerationStats,
|
SsTableStats,
|
||||||
LsmStats,
|
LsmStats,
|
||||||
MemtableStats,
|
MemtableStats,
|
||||||
} from "./native";
|
} from "./native";
|
||||||
@@ -741,7 +741,7 @@ export abstract class Table {
|
|||||||
*/
|
*/
|
||||||
abstract closeLsmWriters(): Promise<void>;
|
abstract closeLsmWriters(): Promise<void>;
|
||||||
/**
|
/**
|
||||||
* Seal every bucket's active memtable into a new L0 generation.
|
* Freeze every table shard's active memtable into a new SSTable.
|
||||||
*
|
*
|
||||||
* Returns once the seal is committed. Sealing an empty memtable is a no-op,
|
* Returns once the seal is committed. Sealing an empty memtable is a no-op,
|
||||||
* so this is safe to call repeatedly.
|
* so this is safe to call repeatedly.
|
||||||
@@ -749,7 +749,7 @@ export abstract class Table {
|
|||||||
*/
|
*/
|
||||||
abstract flushLsm(): Promise<void>;
|
abstract flushLsm(): Promise<void>;
|
||||||
/**
|
/**
|
||||||
* Trigger a background L0 → base compaction pass per bucket.
|
* Trigger a background SSTable compaction pass per table shard.
|
||||||
*
|
*
|
||||||
* Returns once the passes are *dispatched*, not once they finish — watch
|
* Returns once the passes are *dispatched*, not once they finish — watch
|
||||||
* {@link Table#getLsmStats} for progress, or use
|
* {@link Table#getLsmStats} for progress, or use
|
||||||
@@ -760,9 +760,9 @@ export abstract class Table {
|
|||||||
/**
|
/**
|
||||||
* Converge this table's LSM write path into its base table.
|
* Converge this table's LSM write path into its base table.
|
||||||
*
|
*
|
||||||
* Seals once, then triggers compaction and polls until the L0 that existed
|
* Freezes once, then triggers compaction and polls until the SSTables that existed
|
||||||
* at the start is gone. The target set is fixed at the start, so
|
* at the start is gone. The target set is fixed at the start, so
|
||||||
* generations created *during* the checkpoint are ignored — that is what
|
* SSTables created *during* the checkpoint are ignored — that is what
|
||||||
* lets it terminate under write load, and what makes it best-effort: it
|
* lets it terminate under write load, and what makes it best-effort: it
|
||||||
* converges the fresh tier as of some instant. Idempotent, abandonable at
|
* converges the fresh tier as of some instant. Idempotent, abandonable at
|
||||||
* any point, and safe to run on a cadence.
|
* any point, and safe to run on a cadence.
|
||||||
@@ -786,12 +786,12 @@ export abstract class Table {
|
|||||||
* "why is my fresh-tier vector search brute-force". Mutates no table state.
|
* "why is my fresh-tier vector search brute-force". Mutates no table state.
|
||||||
*
|
*
|
||||||
* Resolves to `undefined` only when the LSM write path is not enabled.
|
* Resolves to `undefined` only when the LSM write path is not enabled.
|
||||||
* @param {boolean} includeGenerationRows Also count rows per L0 generation.
|
* @param {boolean} includeSstableRows Also count rows per SSTable.
|
||||||
* Off by default because each count opens an uncached Lance dataset.
|
* Off by default because each count opens an uncached Lance dataset.
|
||||||
* @returns {Promise<LsmStats | undefined>}
|
* @returns {Promise<LsmStats | undefined>}
|
||||||
*/
|
*/
|
||||||
abstract getLsmStats(
|
abstract getLsmStats(
|
||||||
includeGenerationRows?: boolean,
|
includeSstableRows?: boolean,
|
||||||
): Promise<LsmStats | undefined>;
|
): Promise<LsmStats | undefined>;
|
||||||
/** Retrieve the version of the table */
|
/** Retrieve the version of the table */
|
||||||
|
|
||||||
@@ -1388,9 +1388,9 @@ export class LocalTable extends Table {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async getLsmStats(
|
async getLsmStats(
|
||||||
includeGenerationRows: boolean = false,
|
includeSstableRows: boolean = false,
|
||||||
): Promise<LsmStats | undefined> {
|
): Promise<LsmStats | undefined> {
|
||||||
return (await this.inner.getLsmStats(includeGenerationRows)) ?? undefined;
|
return (await this.inner.getLsmStats(includeSstableRows)) ?? undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
async version(): Promise<number> {
|
async version(): Promise<number> {
|
||||||
|
|||||||
+26
-26
@@ -542,11 +542,11 @@ impl Table {
|
|||||||
#[napi(catch_unwind)]
|
#[napi(catch_unwind)]
|
||||||
pub async fn get_lsm_stats(
|
pub async fn get_lsm_stats(
|
||||||
&self,
|
&self,
|
||||||
include_generation_rows: bool,
|
include_sstable_rows: bool,
|
||||||
) -> napi::Result<Option<LsmStats>> {
|
) -> napi::Result<Option<LsmStats>> {
|
||||||
let stats = self
|
let stats = self
|
||||||
.inner_ref()?
|
.inner_ref()?
|
||||||
.get_lsm_stats(include_generation_rows)
|
.get_lsm_stats(include_sstable_rows)
|
||||||
.await
|
.await
|
||||||
.default_error()?;
|
.default_error()?;
|
||||||
Ok(stats.map(LsmStats::from))
|
Ok(stats.map(LsmStats::from))
|
||||||
@@ -950,21 +950,21 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One flushed L0 generation.
|
/// One SSTable.
|
||||||
#[napi(object)]
|
#[napi(object)]
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct GenerationStats {
|
pub struct SsTableStats {
|
||||||
/// The generation number. Increases as memtables are sealed into L0.
|
/// The generation number. Increases as memtables are frozen into SSTables.
|
||||||
pub generation: i64,
|
pub generation: i64,
|
||||||
/// On-disk size of the generation.
|
/// On-disk size of the SSTable.
|
||||||
pub bytes: i64,
|
pub bytes: i64,
|
||||||
/// Present only when `includeGenerationRows` was requested. Off by default
|
/// Present only when `includeSstableRows` was requested. Off by default
|
||||||
/// because each count opens an uncached Lance dataset.
|
/// because each count opens an uncached Lance dataset.
|
||||||
pub rows: Option<i64>,
|
pub rows: Option<i64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<lancedb::table::GenerationStats> for GenerationStats {
|
impl From<lancedb::table::SsTableStats> for SsTableStats {
|
||||||
fn from(g: lancedb::table::GenerationStats) -> Self {
|
fn from(g: lancedb::table::SsTableStats) -> Self {
|
||||||
Self {
|
Self {
|
||||||
generation: g.generation as i64,
|
generation: g.generation as i64,
|
||||||
bytes: g.bytes as i64,
|
bytes: g.bytes as i64,
|
||||||
@@ -977,7 +977,7 @@ impl From<lancedb::table::GenerationStats> for GenerationStats {
|
|||||||
#[napi(object)]
|
#[napi(object)]
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct MemtableStats {
|
pub struct MemtableStats {
|
||||||
/// The generation this memtable will become once sealed.
|
/// The generation this memtable will become once frozen.
|
||||||
pub generation: i64,
|
pub generation: i64,
|
||||||
/// Rows currently buffered.
|
/// Rows currently buffered.
|
||||||
pub rows: i64,
|
pub rows: i64,
|
||||||
@@ -1002,13 +1002,13 @@ impl From<lancedb::table::MemtableStats> for MemtableStats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Live state of one bucket. A table is N buckets on one node; flattening to a
|
/// Live state of one table shard. A table is N table shards on one node; flattening to a
|
||||||
/// single number hides the one hot bucket that is usually why someone opened
|
/// single number hides the one hot table shard that is usually why someone opened
|
||||||
/// this endpoint.
|
/// this endpoint.
|
||||||
#[napi(object)]
|
#[napi(object)]
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct BucketStats {
|
pub struct TableShardStats {
|
||||||
/// The shard this bucket writes.
|
/// The shard this table shard writes.
|
||||||
pub shard_id: String,
|
pub shard_id: String,
|
||||||
/// `"Active"` or `"Sealed"` (drop-table 2PC in flight).
|
/// `"Active"` or `"Sealed"` (drop-table 2PC in flight).
|
||||||
pub status: String,
|
pub status: String,
|
||||||
@@ -1023,20 +1023,20 @@ pub struct BucketStats {
|
|||||||
/// Highest WAL position the writer has seen. The difference against
|
/// Highest WAL position the writer has seen. The difference against
|
||||||
/// `replayAfterWalEntryPosition` is the WAL lag.
|
/// `replayAfterWalEntryPosition` is the WAL lag.
|
||||||
pub wal_entry_position_last_seen: i64,
|
pub wal_entry_position_last_seen: i64,
|
||||||
/// Flushed L0 generations not yet merged into the base table.
|
/// SSTables not yet merged into the base table.
|
||||||
pub generations: Vec<GenerationStats>,
|
pub sstables: Vec<SsTableStats>,
|
||||||
/// Whether a pass owns this bucket's compaction latch right now. Says *a*
|
/// Whether a pass owns this table shard's compaction latch right now. Says *a*
|
||||||
/// driver is running, not *whose*, and the latch is held from dispatch —
|
/// driver is running, not *whose*, and the latch is held from dispatch —
|
||||||
/// including while the pass queues for a pod-wide compactor permit. Read it
|
/// including while the pass queues for a pod-wide compactor permit. Read it
|
||||||
/// as "do not pile on", never as "mine is progressing".
|
/// as "do not pile on", never as "mine is progressing".
|
||||||
pub compacting: bool,
|
pub compacting: bool,
|
||||||
/// Oldest first, active last. Absent for a `"Sealed"` bucket, whose
|
/// Oldest first, active last. Absent for a `"Sealed"` table shard, whose
|
||||||
/// in-memory state is torn down.
|
/// in-memory state is torn down.
|
||||||
pub memtables: Option<Vec<MemtableStats>>,
|
pub memtables: Option<Vec<MemtableStats>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<lancedb::table::BucketStats> for BucketStats {
|
impl From<lancedb::table::TableShardStats> for TableShardStats {
|
||||||
fn from(b: lancedb::table::BucketStats) -> Self {
|
fn from(b: lancedb::table::TableShardStats) -> Self {
|
||||||
Self {
|
Self {
|
||||||
shard_id: b.shard_id,
|
shard_id: b.shard_id,
|
||||||
status: b.status,
|
status: b.status,
|
||||||
@@ -1045,7 +1045,7 @@ impl From<lancedb::table::BucketStats> for BucketStats {
|
|||||||
current_generation: b.current_generation as i64,
|
current_generation: b.current_generation as i64,
|
||||||
replay_after_wal_entry_position: b.replay_after_wal_entry_position as i64,
|
replay_after_wal_entry_position: b.replay_after_wal_entry_position as i64,
|
||||||
wal_entry_position_last_seen: b.wal_entry_position_last_seen as i64,
|
wal_entry_position_last_seen: b.wal_entry_position_last_seen as i64,
|
||||||
generations: b.generations.into_iter().map(Into::into).collect(),
|
sstables: b.sstables.into_iter().map(Into::into).collect(),
|
||||||
compacting: b.compacting,
|
compacting: b.compacting,
|
||||||
memtables: b
|
memtables: b
|
||||||
.memtables
|
.memtables
|
||||||
@@ -1054,21 +1054,21 @@ impl From<lancedb::table::BucketStats> for BucketStats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Live per-bucket LSM state, as returned by `Table#getLsmStats`.
|
/// Live per-table-shard LSM state, as returned by `Table#getLsmStats`.
|
||||||
///
|
///
|
||||||
/// Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
|
/// Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
|
||||||
/// the caller's to compute.
|
/// the caller's to compute.
|
||||||
#[napi(object)]
|
#[napi(object)]
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct LsmStats {
|
pub struct LsmStats {
|
||||||
/// One entry per bucket backing this table.
|
/// One entry per table shard backing this table.
|
||||||
pub buckets: Vec<BucketStats>,
|
pub table_shards: Vec<TableShardStats>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<lancedb::table::LsmStats> for LsmStats {
|
impl From<lancedb::table::LsmStats> for LsmStats {
|
||||||
fn from(stats: lancedb::table::LsmStats) -> Self {
|
fn from(stats: lancedb::table::LsmStats) -> Self {
|
||||||
Self {
|
Self {
|
||||||
buckets: stats.buckets.into_iter().map(Into::into).collect(),
|
table_shards: stats.table_shards.into_iter().map(Into::into).collect(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -385,7 +385,7 @@ class Table:
|
|||||||
async def checkpoint_lsm(self) -> None: ...
|
async def checkpoint_lsm(self) -> None: ...
|
||||||
async def flush_lsm(self) -> None: ...
|
async def flush_lsm(self) -> None: ...
|
||||||
async def compact_lsm(self) -> None: ...
|
async def compact_lsm(self) -> None: ...
|
||||||
async def get_lsm_stats(self, include_generation_rows: bool) -> Optional[dict]: ...
|
async def get_lsm_stats(self, include_sstable_rows: bool) -> Optional[dict]: ...
|
||||||
async def close_lsm_writers(self) -> None: ...
|
async def close_lsm_writers(self) -> None: ...
|
||||||
@property
|
@property
|
||||||
def tags(self) -> Tags: ...
|
def tags(self) -> Tags: ...
|
||||||
|
|||||||
@@ -1029,11 +1029,11 @@ class RemoteTable(Table):
|
|||||||
[`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
|
[`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
|
||||||
return LOOP.run(self._table.compact_lsm())
|
return LOOP.run(self._table.compact_lsm())
|
||||||
|
|
||||||
def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]:
|
def get_lsm_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]:
|
||||||
"""Synchronous version of
|
"""Synchronous version of
|
||||||
[`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
|
[`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
|
||||||
return LOOP.run(
|
return LOOP.run(
|
||||||
self._table.get_lsm_stats(include_generation_rows=include_generation_rows)
|
self._table.get_lsm_stats(include_sstable_rows=include_sstable_rows)
|
||||||
)
|
)
|
||||||
|
|
||||||
def close_lsm_writers(self) -> None:
|
def close_lsm_writers(self) -> None:
|
||||||
|
|||||||
@@ -4189,11 +4189,11 @@ class LanceTable(Table):
|
|||||||
[`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
|
[`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
|
||||||
return LOOP.run(self._table.compact_lsm())
|
return LOOP.run(self._table.compact_lsm())
|
||||||
|
|
||||||
def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]:
|
def get_lsm_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]:
|
||||||
"""Synchronous version of
|
"""Synchronous version of
|
||||||
[`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
|
[`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
|
||||||
return LOOP.run(
|
return LOOP.run(
|
||||||
self._table.get_lsm_stats(include_generation_rows=include_generation_rows)
|
self._table.get_lsm_stats(include_sstable_rows=include_sstable_rows)
|
||||||
)
|
)
|
||||||
|
|
||||||
def close_lsm_writers(self) -> None:
|
def close_lsm_writers(self) -> None:
|
||||||
@@ -4916,16 +4916,16 @@ class AsyncTable:
|
|||||||
async def checkpoint_lsm(self) -> None:
|
async def checkpoint_lsm(self) -> None:
|
||||||
"""Converge this table's LSM write path into its base table.
|
"""Converge this table's LSM write path into its base table.
|
||||||
|
|
||||||
One flush, sealing every memtable into L0, then compaction triggers
|
One flush, freezing every memtable into an SSTable, then compaction triggers
|
||||||
until every generation that existed at that moment has reached base.
|
until every generation that existed at that moment has reached base.
|
||||||
The loop runs client-side, reading progress from ``get_lsm_stats``.
|
The loop runs client-side, reading progress from ``get_lsm_stats``.
|
||||||
|
|
||||||
Best-effort: generations created *while* it runs are deliberately not
|
Best-effort: SSTables created *while* it runs are deliberately not
|
||||||
waited on, which is what lets it terminate on a table taking writes.
|
waited on, which is what lets it terminate on a table taking writes.
|
||||||
Idempotent and safe on a cadence.
|
Idempotent and safe on a cadence.
|
||||||
|
|
||||||
There is no deadline, and the caller owns that. It returns when the
|
There is no deadline, and the caller owns that. It returns when the
|
||||||
target generations are gone, raises on a terminal server fault, and
|
target SSTables are gone, raises on a terminal server fault, and
|
||||||
otherwise waits however long the server takes. A slow table and a
|
otherwise waits however long the server takes. A slow table and a
|
||||||
stuck one are the same picture from the client: the compactor pool is
|
stuck one are the same picture from the client: the compactor pool is
|
||||||
shared across every table on the node, so a checkpoint queued behind
|
shared across every table on the node, so a checkpoint queued behind
|
||||||
@@ -4936,25 +4936,25 @@ class AsyncTable:
|
|||||||
await self._inner.checkpoint_lsm()
|
await self._inner.checkpoint_lsm()
|
||||||
|
|
||||||
async def flush_lsm(self) -> None:
|
async def flush_lsm(self) -> None:
|
||||||
"""Seal every bucket's active memtable into L0.
|
"""Freeze every table shard's active memtable into an SSTable.
|
||||||
|
|
||||||
Does not touch the base table — moving L0 into base is
|
Does not touch the base table — compacting SSTables into base is
|
||||||
`compact_lsm`. On a node that has not claimed this table, this claims
|
`compact_lsm`. On a node that has not claimed this table, this claims
|
||||||
it and replays its WAL log first.
|
it and replays its WAL log first.
|
||||||
"""
|
"""
|
||||||
await self._inner.flush_lsm()
|
await self._inner.flush_lsm()
|
||||||
|
|
||||||
async def compact_lsm(self) -> None:
|
async def compact_lsm(self) -> None:
|
||||||
"""Trigger a background L0 to base compaction pass per bucket.
|
"""Trigger a background SSTable compaction pass per table shard.
|
||||||
|
|
||||||
Returns once the passes are dispatched, not once they finish: watch
|
Returns once the passes are dispatched, not once they finish: watch
|
||||||
``get_lsm_stats`` for progress, or use ``checkpoint_lsm`` to loop
|
``get_lsm_stats`` for progress, or use ``checkpoint_lsm`` to loop
|
||||||
until the current L0 has reached base.
|
until the current SSTables have reached base.
|
||||||
"""
|
"""
|
||||||
await self._inner.compact_lsm()
|
await self._inner.compact_lsm()
|
||||||
|
|
||||||
async def get_lsm_stats(
|
async def get_lsm_stats(
|
||||||
self, *, include_generation_rows: bool = False
|
self, *, include_sstable_rows: bool = False
|
||||||
) -> Optional[dict]:
|
) -> Optional[dict]:
|
||||||
"""Read live per-bucket LSM state.
|
"""Read live per-bucket LSM state.
|
||||||
|
|
||||||
@@ -4967,12 +4967,12 @@ class AsyncTable:
|
|||||||
|
|
||||||
Parameters
|
Parameters
|
||||||
----------
|
----------
|
||||||
include_generation_rows
|
include_sstable_rows
|
||||||
Report a row count per L0 generation. Off by default: each count
|
Report a row count per SSTable. Off by default: each count
|
||||||
opens an uncached Lance dataset, and ``checkpoint_lsm`` polls this
|
opens an uncached Lance dataset, and ``checkpoint_lsm`` polls this
|
||||||
needing only generation numbers.
|
needing only generation numbers.
|
||||||
"""
|
"""
|
||||||
return await self._inner.get_lsm_stats(include_generation_rows)
|
return await self._inner.get_lsm_stats(include_sstable_rows)
|
||||||
|
|
||||||
async def close_lsm_writers(self) -> None:
|
async def close_lsm_writers(self) -> None:
|
||||||
"""Drain and close any cached MemWAL shard writers for this table.
|
"""Drain and close any cached MemWAL shard writers for this table.
|
||||||
|
|||||||
@@ -1278,9 +1278,9 @@ def test_get_lsm_stats_sync():
|
|||||||
with lsm_test_table(lsm_handler) as table:
|
with lsm_test_table(lsm_handler) as table:
|
||||||
assert table.get_lsm_stats() == {"buckets": [bucket]}
|
assert table.get_lsm_stats() == {"buckets": [bucket]}
|
||||||
# Off by default, and forwarded when asked for.
|
# Off by default, and forwarded when asked for.
|
||||||
assert seen_bodies == [{"include_generation_rows": False}]
|
assert seen_bodies == [{"include_sstable_rows": False}]
|
||||||
table.get_lsm_stats(include_generation_rows=True)
|
table.get_lsm_stats(include_sstable_rows=True)
|
||||||
assert seen_bodies[-1] == {"include_generation_rows": True}
|
assert seen_bodies[-1] == {"include_sstable_rows": True}
|
||||||
|
|
||||||
|
|
||||||
def test_get_lsm_stats_sync_returns_none_when_lsm_disabled():
|
def test_get_lsm_stats_sync_returns_none_when_lsm_disabled():
|
||||||
@@ -1309,7 +1309,7 @@ def test_flush_and_compact_lsm_sync():
|
|||||||
|
|
||||||
|
|
||||||
def test_checkpoint_lsm_sync():
|
def test_checkpoint_lsm_sync():
|
||||||
"""Seal, read the watermark, and return once L0 holds nothing.
|
"""Freeze, read the watermark, and return once no SSTables remain.
|
||||||
|
|
||||||
The convergence loop itself is covered in Rust; this pins the sync
|
The convergence loop itself is covered in Rust; this pins the sync
|
||||||
binding to the endpoints it drives.
|
binding to the endpoints it drives.
|
||||||
@@ -1319,7 +1319,7 @@ def test_checkpoint_lsm_sync():
|
|||||||
def lsm_handler(request, route):
|
def lsm_handler(request, route):
|
||||||
called.append(route)
|
called.append(route)
|
||||||
if route == "get_lsm_stats":
|
if route == "get_lsm_stats":
|
||||||
# An empty L0 yields no target watermark, so the loop is done
|
# An empty SSTable tier yields no target watermark, so the loop is done
|
||||||
# after the seal without ever polling compaction.
|
# after the seal without ever polling compaction.
|
||||||
send_json(request, {"lsm_stats": {"buckets": []}})
|
send_json(request, {"lsm_stats": {"buckets": []}})
|
||||||
else:
|
else:
|
||||||
|
|||||||
+16
-16
@@ -33,16 +33,16 @@ use pyo3::{
|
|||||||
|
|
||||||
mod scannable;
|
mod scannable;
|
||||||
|
|
||||||
/// Convert `LsmStats` to a Python dict, preserving the per-bucket list.
|
/// Convert `LsmStats` to a Python dict, preserving the per-table-shard list.
|
||||||
///
|
///
|
||||||
/// Deliberately not flattened to a table-level summary: a table is N
|
/// Deliberately not flattened to a table-level summary: a table is N
|
||||||
/// buckets on one node, and the per-bucket detail is the reason the
|
/// table shards on one node, and the per-shard detail is the reason the
|
||||||
/// endpoint exists — flattening hides the single hot bucket someone opened
|
/// endpoint exists — flattening hides the single hot table shard someone opened
|
||||||
/// it to find.
|
/// it to find.
|
||||||
fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult<Py<PyDict>> {
|
fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult<Py<PyDict>> {
|
||||||
let out = PyDict::new(py);
|
let out = PyDict::new(py);
|
||||||
let buckets = PyList::empty(py);
|
let table_shards = PyList::empty(py);
|
||||||
for b in &stats.buckets {
|
for b in &stats.table_shards {
|
||||||
let e = PyDict::new(py);
|
let e = PyDict::new(py);
|
||||||
e.set_item("shard_id", &b.shard_id)?;
|
e.set_item("shard_id", &b.shard_id)?;
|
||||||
e.set_item("status", &b.status)?;
|
e.set_item("status", &b.status)?;
|
||||||
@@ -58,15 +58,15 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult
|
|||||||
b.wal_entry_position_last_seen,
|
b.wal_entry_position_last_seen,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
let generations = PyList::empty(py);
|
let sstables = PyList::empty(py);
|
||||||
for g in &b.generations {
|
for g in &b.sstables {
|
||||||
let ge = PyDict::new(py);
|
let ge = PyDict::new(py);
|
||||||
ge.set_item("generation", g.generation)?;
|
ge.set_item("generation", g.generation)?;
|
||||||
ge.set_item("bytes", g.bytes)?;
|
ge.set_item("bytes", g.bytes)?;
|
||||||
ge.set_item("rows", g.rows)?;
|
ge.set_item("rows", g.rows)?;
|
||||||
generations.append(ge)?;
|
sstables.append(ge)?;
|
||||||
}
|
}
|
||||||
e.set_item("generations", generations)?;
|
e.set_item("sstables", sstables)?;
|
||||||
e.set_item("compacting", b.compacting)?;
|
e.set_item("compacting", b.compacting)?;
|
||||||
|
|
||||||
e.set_item(
|
e.set_item(
|
||||||
@@ -88,9 +88,9 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult
|
|||||||
})
|
})
|
||||||
.transpose()?,
|
.transpose()?,
|
||||||
)?;
|
)?;
|
||||||
buckets.append(e)?;
|
table_shards.append(e)?;
|
||||||
}
|
}
|
||||||
out.set_item("buckets", buckets)?;
|
out.set_item("table_shards", table_shards)?;
|
||||||
Ok(out.unbind())
|
Ok(out.unbind())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1492,7 +1492,7 @@ impl Table {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Seal every bucket's active memtable into L0.
|
/// Freeze every table shard's active memtable into an SSTable.
|
||||||
pub fn flush_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
pub fn flush_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||||
let inner = self_.inner_ref()?.clone();
|
let inner = self_.inner_ref()?.clone();
|
||||||
future_into_py(
|
future_into_py(
|
||||||
@@ -1501,7 +1501,7 @@ impl Table {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Trigger a background L0 → base pass per bucket. Returns once the
|
/// Trigger a background SSTable compaction pass per table shard. Returns once the
|
||||||
/// passes are dispatched, not once they finish — watch `get_lsm_stats`.
|
/// passes are dispatched, not once they finish — watch `get_lsm_stats`.
|
||||||
pub fn compact_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
pub fn compact_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||||
let inner = self_.inner_ref()?.clone();
|
let inner = self_.inner_ref()?.clone();
|
||||||
@@ -1511,15 +1511,15 @@ impl Table {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Live LSM state, or `None` when the LSM write path is not enabled.
|
/// Live LSM state, or `None` when the LSM write path is not enabled.
|
||||||
#[pyo3(signature = (include_generation_rows=false))]
|
#[pyo3(signature = (include_sstable_rows=false))]
|
||||||
pub fn get_lsm_stats(
|
pub fn get_lsm_stats(
|
||||||
self_: PyRef<'_, Self>,
|
self_: PyRef<'_, Self>,
|
||||||
include_generation_rows: bool,
|
include_sstable_rows: bool,
|
||||||
) -> PyResult<Bound<'_, PyAny>> {
|
) -> PyResult<Bound<'_, PyAny>> {
|
||||||
let inner = self_.inner_ref()?.clone();
|
let inner = self_.inner_ref()?.clone();
|
||||||
future_into_py(self_.py(), async move {
|
future_into_py(self_.py(), async move {
|
||||||
let stats = inner
|
let stats = inner
|
||||||
.get_lsm_stats(include_generation_rows)
|
.get_lsm_stats(include_sstable_rows)
|
||||||
.await
|
.await
|
||||||
.infer_error()?;
|
.infer_error()?;
|
||||||
Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose())
|
Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose())
|
||||||
|
|||||||
@@ -878,7 +878,7 @@ pub struct QueryRequest {
|
|||||||
/// [`crate::Table::set_lsm_write_spec`]) is routed through the LSM scanner so
|
/// [`crate::Table::set_lsm_write_spec`]) is routed through the LSM scanner so
|
||||||
/// it also sees data written via the `merge_insert` LSM path that has not yet
|
/// it also sees data written via the `merge_insert` LSM path that has not yet
|
||||||
/// been compacted into the base table — the active and frozen in-memory
|
/// been compacted into the base table — the active and frozen in-memory
|
||||||
/// memtables and the flushed (L0) generations, deduplicated by primary key
|
/// memtables and the SSTables, deduplicated by primary key
|
||||||
/// against the base table (newest generation wins); a table without a spec
|
/// against the base table (newest generation wins); a table without a spec
|
||||||
/// reads the base table.
|
/// reads the base table.
|
||||||
///
|
///
|
||||||
|
|||||||
@@ -2951,13 +2951,13 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result<Option<LsmStats>> {
|
async fn get_lsm_stats(&self, include_sstable_rows: bool) -> Result<Option<LsmStats>> {
|
||||||
// Read-semantics POST, like `get_lsm_write_spec`.
|
// Read-semantics POST, like `get_lsm_write_spec`.
|
||||||
let request = self
|
let request = self
|
||||||
.client
|
.client
|
||||||
.post(&format!("/v1/table/{}/get_lsm_stats/", self.identifier))
|
.post(&format!("/v1/table/{}/get_lsm_stats/", self.identifier))
|
||||||
.json(&serde_json::json!({
|
.json(&serde_json::json!({
|
||||||
"include_generation_rows": include_generation_rows,
|
"include_sstable_rows": include_sstable_rows,
|
||||||
}));
|
}));
|
||||||
let (request_id, response) = self.send_lsm_route(request).await?;
|
let (request_id, response) = self.send_lsm_route(request).await?;
|
||||||
let body = response.text().await.err_to_http(request_id.clone())?;
|
let body = response.text().await.err_to_http(request_id.clone())?;
|
||||||
@@ -8260,7 +8260,7 @@ mod tests {
|
|||||||
http::Response::builder().status(200).body(body).unwrap()
|
http::Response::builder().status(200).body(body).unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A flush landing in an empty L0 finishes on the opening stats read
|
/// A flush landing in an empty SSTable tier finishes on the opening stats read
|
||||||
/// alone. Asserting zero compacts is the point: "it returned Ok" is also
|
/// alone. Asserting zero compacts is the point: "it returned Ok" is also
|
||||||
/// true of a loop that ran a pointless pass.
|
/// true of a loop that ran a pointless pass.
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
@@ -8314,7 +8314,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Generations created *during* the checkpoint are not waited on, which
|
/// Generations created *during* the checkpoint are not waited on, which
|
||||||
/// is what lets the loop terminate on a table taking writes where "L0 is
|
/// is what lets the loop terminate on a table taking writes where "the SSTable tier is
|
||||||
/// empty" never becomes true.
|
/// empty" never becomes true.
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
async fn test_checkpoint_ignores_generations_created_while_it_runs() {
|
async fn test_checkpoint_ignores_generations_created_while_it_runs() {
|
||||||
@@ -8593,7 +8593,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// WAL off ⇒ `None`; WAL on ⇒ a fully populated `Some` with no field
|
/// WAL off ⇒ `None`; WAL on ⇒ a fully populated `Some` with no field
|
||||||
/// defaulting to a zero it did not measure. `include_generation_rows`
|
/// defaulting to a zero it did not measure. `include_sstable_rows`
|
||||||
/// rides in the body and is off unless asked for.
|
/// rides in the body and is off unless asked for.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_get_lsm_stats_round_trip() {
|
async fn test_get_lsm_stats_round_trip() {
|
||||||
@@ -8602,7 +8602,7 @@ mod tests {
|
|||||||
let body = request.body().unwrap().as_bytes().unwrap();
|
let body = request.body().unwrap().as_bytes().unwrap();
|
||||||
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
|
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
body["include_generation_rows"], true,
|
body["include_sstable_rows"], true,
|
||||||
"the flag must reach the server, not be silently dropped"
|
"the flag must reach the server, not be silently dropped"
|
||||||
);
|
);
|
||||||
let response = serde_json::json!({
|
let response = serde_json::json!({
|
||||||
|
|||||||
+11
-11
@@ -102,7 +102,7 @@ use futures::future::join_all;
|
|||||||
pub use lance::dataset::refs::{BranchContents, Ref, TagContents, Tags as LanceTags};
|
pub use lance::dataset::refs::{BranchContents, Ref, TagContents, Tags as LanceTags};
|
||||||
pub use lance::dataset::scanner::DatasetRecordBatchStream;
|
pub use lance::dataset::scanner::DatasetRecordBatchStream;
|
||||||
pub use lance_index::optimize::OptimizeOptions;
|
pub use lance_index::optimize::OptimizeOptions;
|
||||||
pub use lsm_stats::{BucketStats, GenerationStats, LsmStats, MemtableStats};
|
pub use lsm_stats::{LsmStats, MemtableStats, SsTableStats, TableShardStats};
|
||||||
pub use optimize::{CompactionOptions, OptimizeAction, OptimizeStats};
|
pub use optimize::{CompactionOptions, OptimizeAction, OptimizeStats};
|
||||||
pub use refresh::RefreshColumnResult;
|
pub use refresh::RefreshColumnResult;
|
||||||
pub use schema_evolution::{
|
pub use schema_evolution::{
|
||||||
@@ -673,7 +673,7 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
|
|||||||
message: "get_lsm_write_spec is not supported on this table type".into(),
|
message: "get_lsm_write_spec is not supported on this table type".into(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
/// Seal every bucket's active memtable into L0.
|
/// Freeze every table shard's active memtable into an SSTable.
|
||||||
///
|
///
|
||||||
/// The default implementation returns `NotSupported`.
|
/// The default implementation returns `NotSupported`.
|
||||||
async fn flush_lsm(&self) -> Result<()> {
|
async fn flush_lsm(&self) -> Result<()> {
|
||||||
@@ -681,7 +681,7 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
|
|||||||
message: "flush_lsm is not supported on this table type".into(),
|
message: "flush_lsm is not supported on this table type".into(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
/// Trigger a background L0 → base compaction pass per bucket.
|
/// Trigger a background SSTable compaction pass per table shard.
|
||||||
///
|
///
|
||||||
/// The default implementation returns `NotSupported`.
|
/// The default implementation returns `NotSupported`.
|
||||||
async fn compact_lsm(&self) -> Result<()> {
|
async fn compact_lsm(&self) -> Result<()> {
|
||||||
@@ -693,7 +693,7 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
|
|||||||
/// enabled for this table.
|
/// enabled for this table.
|
||||||
///
|
///
|
||||||
/// The default implementation returns `NotSupported`.
|
/// The default implementation returns `NotSupported`.
|
||||||
async fn get_lsm_stats(&self, _include_generation_rows: bool) -> Result<Option<LsmStats>> {
|
async fn get_lsm_stats(&self, _include_sstable_rows: bool) -> Result<Option<LsmStats>> {
|
||||||
Err(Error::NotSupported {
|
Err(Error::NotSupported {
|
||||||
message: "get_lsm_stats is not supported on this table type".into(),
|
message: "get_lsm_stats is not supported on this table type".into(),
|
||||||
})
|
})
|
||||||
@@ -1897,7 +1897,7 @@ impl Table {
|
|||||||
|
|
||||||
/// Converge this table's LSM write path into its base table.
|
/// Converge this table's LSM write path into its base table.
|
||||||
///
|
///
|
||||||
/// One `flush` to seal every memtable into L0, then compaction triggers
|
/// One `flush` to freeze every memtable into an SSTable, then compaction triggers
|
||||||
/// until every generation that existed at that moment has reached base.
|
/// until every generation that existed at that moment has reached base.
|
||||||
/// The loop runs client-side, reading progress from `get_lsm_stats`, so
|
/// The loop runs client-side, reading progress from `get_lsm_stats`, so
|
||||||
/// there is no held socket and nothing to reconcile if you drop this
|
/// there is no held socket and nothing to reconcile if you drop this
|
||||||
@@ -1932,10 +1932,10 @@ impl Table {
|
|||||||
checkpoint::checkpoint_lsm(self).await
|
checkpoint::checkpoint_lsm(self).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Seal every bucket's active memtable into L0 without touching the
|
/// Freeze every table shard's active memtable into an SSTable without touching the
|
||||||
/// base table.
|
/// base table.
|
||||||
///
|
///
|
||||||
/// Independently useful: flushing makes memtable rows readable from L0 at
|
/// Independently useful: flushing makes memtable rows readable from an SSTable at
|
||||||
/// a lower per-query cost. On a node that has not claimed this table it
|
/// a lower per-query cost. On a node that has not claimed this table it
|
||||||
/// claims it and replays the WAL log first — reporting "nothing to flush"
|
/// claims it and replays the WAL log first — reporting "nothing to flush"
|
||||||
/// without replaying would lie about durable data.
|
/// without replaying would lie about durable data.
|
||||||
@@ -1943,7 +1943,7 @@ impl Table {
|
|||||||
self.inner.flush_lsm().await
|
self.inner.flush_lsm().await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run one bounded L0 → base compaction pass per bucket, reporting what
|
/// Run one bounded SSTable compaction pass per table shard, reporting what
|
||||||
/// it merged and what is left.
|
/// it merged and what is left.
|
||||||
///
|
///
|
||||||
/// One pass, not convergence: that bounds each request's cost and gives a
|
/// One pass, not convergence: that bounds each request's cost and gives a
|
||||||
@@ -1959,7 +1959,7 @@ impl Table {
|
|||||||
/// state, though on a node that has not claimed this table it claims it,
|
/// state, though on a node that has not claimed this table it claims it,
|
||||||
/// exactly as a read would.
|
/// exactly as a read would.
|
||||||
///
|
///
|
||||||
/// `include_generation_rows` reports a row count per L0 generation. Off by
|
/// `include_sstable_rows` reports a row count per SSTable. Off by
|
||||||
/// default: each count opens an uncached Lance dataset, and
|
/// default: each count opens an uncached Lance dataset, and
|
||||||
/// `checkpoint_lsm` polls this needing only generation numbers.
|
/// `checkpoint_lsm` polls this needing only generation numbers.
|
||||||
///
|
///
|
||||||
@@ -1970,8 +1970,8 @@ impl Table {
|
|||||||
///
|
///
|
||||||
/// Do not build a checkpoint's termination on this: the completion
|
/// Do not build a checkpoint's termination on this: the completion
|
||||||
/// predicate lives in the `flush` and `compact` responses.
|
/// predicate lives in the `flush` and `compact` responses.
|
||||||
pub async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result<Option<LsmStats>> {
|
pub async fn get_lsm_stats(&self, include_sstable_rows: bool) -> Result<Option<LsmStats>> {
|
||||||
self.inner.get_lsm_stats(include_generation_rows).await
|
self.inner.get_lsm_stats(include_sstable_rows).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Drain and close any cached MemWAL shard writers held for this table.
|
/// Drain and close any cached MemWAL shard writers held for this table.
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
//! Converging a table's LSM write path into its base table.
|
//! Converging a table's LSM write path into its base table.
|
||||||
//!
|
//!
|
||||||
//! `checkpoint_lsm` seals once, then triggers compaction and watches
|
//! `checkpoint_lsm` seals once, then triggers compaction and watches
|
||||||
//! generation numbers until the L0 that existed at the start is gone.
|
//! generation numbers until the SSTables that existed at the start are gone.
|
||||||
//!
|
//!
|
||||||
//! The loop runs in the client, not the server: `compact_lsm` dispatches a
|
//! The loop runs in the client, not the server: `compact_lsm` dispatches a
|
||||||
//! pass and returns, so nothing holds a socket and a client can vanish
|
//! pass and returns, so nothing holds a socket and a client can vanish
|
||||||
@@ -150,7 +150,7 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Drive [`Table::checkpoint_lsm`]: seal once, fix the target watermark
|
/// Drive [`Table::checkpoint_lsm`]: seal once, fix the target watermark
|
||||||
/// from the resulting L0, then trigger and poll until it drains.
|
/// from the resulting SSTables, then trigger and poll until they drain.
|
||||||
pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> {
|
pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> {
|
||||||
for reissue in 0..=MAX_REISSUES {
|
for reissue in 0..=MAX_REISSUES {
|
||||||
// The seal turns everything written before this call into a
|
// The seal turns everything written before this call into a
|
||||||
@@ -177,9 +177,9 @@ pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> {
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
let targets: HashMap<String, u64> = stats
|
let targets: HashMap<String, u64> = stats
|
||||||
.buckets
|
.table_shards
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|b| Some((b.shard_id.clone(), b.newest_generation()?)))
|
.filter_map(|b| Some((b.shard_id.clone(), b.newest_sstable_generation()?)))
|
||||||
.collect();
|
.collect();
|
||||||
if targets.is_empty() {
|
if targets.is_empty() {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -226,11 +226,11 @@ async fn drain_to_targets(
|
|||||||
// with nothing outstanding are skipped, not counted as idle.
|
// with nothing outstanding are skipped, not counted as idle.
|
||||||
let mut outstanding = 0;
|
let mut outstanding = 0;
|
||||||
let mut all_compacting = true;
|
let mut all_compacting = true;
|
||||||
for b in &stats.buckets {
|
for b in &stats.table_shards {
|
||||||
let Some(target) = targets.get(&b.shard_id) else {
|
let Some(target) = targets.get(&b.shard_id) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let n = b.outstanding_generations(*target);
|
let n = b.outstanding_sstables(*target);
|
||||||
if n > 0 {
|
if n > 0 {
|
||||||
outstanding += n;
|
outstanding += n;
|
||||||
all_compacting &= b.compacting;
|
all_compacting &= b.compacting;
|
||||||
|
|||||||
@@ -1,21 +1,21 @@
|
|||||||
// SPDX-License-Identifier: Apache-2.0
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||||
|
|
||||||
//! Live per-bucket LSM state — the shape [`crate::Table::get_lsm_stats`]
|
//! Live per-table_shard LSM state — the shape [`crate::Table::get_lsm_stats`]
|
||||||
//! returns and [`super::checkpoint`] polls.
|
//! returns and [`super::checkpoint`] polls.
|
||||||
//!
|
//!
|
||||||
//! Nothing here is derived: sums and differences (total L0 bytes, WAL lag)
|
//! Nothing here is derived: sums and differences (total SSTable bytes, WAL lag)
|
||||||
//! are the caller's to compute. There is no "WAL is off" shape — that case is
|
//! are the caller's to compute. There is no "WAL is off" shape — that case is
|
||||||
//! `None`, because a struct of zeros would read as measurements.
|
//! `None`, because a struct of zeros would read as measurements.
|
||||||
|
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
|
|
||||||
/// One flushed L0 generation.
|
/// One SSTable.
|
||||||
#[derive(Debug, Clone, Deserialize)]
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
pub struct GenerationStats {
|
pub struct SsTableStats {
|
||||||
pub generation: u64,
|
pub generation: u64,
|
||||||
pub bytes: u64,
|
pub bytes: u64,
|
||||||
/// Present only when `include_generation_rows` was requested. Off by
|
/// Present only when `include_sstable_rows` was requested. Off by
|
||||||
/// default because each count opens an uncached Lance dataset, and the
|
/// default because each count opens an uncached Lance dataset, and the
|
||||||
/// checkpoint loop polls this route needing only generation numbers.
|
/// checkpoint loop polls this route needing only generation numbers.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -34,11 +34,11 @@ pub struct MemtableStats {
|
|||||||
pub indexes: Vec<String>,
|
pub indexes: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Live state of one bucket. A table is N buckets on one node; flattening to
|
/// Live state of one table_shard. A table is N table_shards on one node; flattening to
|
||||||
/// a single number hides the one hot bucket that is usually why someone
|
/// a single number hides the one hot table_shard that is usually why someone
|
||||||
/// opened this endpoint.
|
/// opened this endpoint.
|
||||||
#[derive(Debug, Clone, Deserialize)]
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
pub struct BucketStats {
|
pub struct TableShardStats {
|
||||||
pub shard_id: String,
|
pub shard_id: String,
|
||||||
/// `Active` | `Sealed` (drop-table 2PC in flight).
|
/// `Active` | `Sealed` (drop-table 2PC in flight).
|
||||||
pub status: String,
|
pub status: String,
|
||||||
@@ -47,42 +47,42 @@ pub struct BucketStats {
|
|||||||
pub current_generation: u64,
|
pub current_generation: u64,
|
||||||
pub replay_after_wal_entry_position: u64,
|
pub replay_after_wal_entry_position: u64,
|
||||||
pub wal_entry_position_last_seen: u64,
|
pub wal_entry_position_last_seen: u64,
|
||||||
pub generations: Vec<GenerationStats>,
|
pub sstables: Vec<SsTableStats>,
|
||||||
/// Whether a pass owns this bucket's compaction latch right now. Says *a*
|
/// Whether a pass owns this table_shard's compaction latch right now. Says *a*
|
||||||
/// driver is running, not *whose*, and the latch is held from dispatch —
|
/// driver is running, not *whose*, and the latch is held from dispatch —
|
||||||
/// including while the pass queues for a pod-wide compactor permit. Read
|
/// including while the pass queues for a pod-wide compactor permit. Read
|
||||||
/// it as "do not pile on", never as "mine is progressing".
|
/// it as "do not pile on", never as "mine is progressing".
|
||||||
pub compacting: bool,
|
pub compacting: bool,
|
||||||
/// Oldest first, active last. Absent for a `Sealed` bucket, whose
|
/// Oldest first, active last. Absent for a `Sealed` table_shard, whose
|
||||||
/// in-memory state is torn down.
|
/// in-memory state is torn down.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub memtables: Option<Vec<MemtableStats>>,
|
pub memtables: Option<Vec<MemtableStats>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BucketStats {
|
impl TableShardStats {
|
||||||
/// The newest flushed generation, or `None` when L0 is empty.
|
/// The newest SSTable generation, or `None` when the tier is empty.
|
||||||
pub(crate) fn newest_generation(&self) -> Option<u64> {
|
pub(crate) fn newest_sstable_generation(&self) -> Option<u64> {
|
||||||
self.generations.iter().map(|g| g.generation).max()
|
self.sstables.iter().map(|g| g.generation).max()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// How many generations at or below `target` are still in L0.
|
/// How many SSTables at or below `target` are still uncompacted.
|
||||||
///
|
///
|
||||||
/// A count, not a boolean: one pass drains a bounded prefix rather than
|
/// A count, not a boolean: one pass drains a bounded prefix rather than
|
||||||
/// the whole target set, so a boolean would read as "no progress" for
|
/// the whole target set, so a boolean would read as "no progress" for
|
||||||
/// every pass but the last. Compaction drains oldest-first, so this
|
/// every pass but the last. Compaction drains oldest-first, so this
|
||||||
/// decreases monotonically.
|
/// decreases monotonically.
|
||||||
pub(crate) fn outstanding_generations(&self, target: u64) -> usize {
|
pub(crate) fn outstanding_sstables(&self, target: u64) -> usize {
|
||||||
self.generations
|
self.sstables
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|g| g.generation <= target)
|
.filter(|g| g.generation <= target)
|
||||||
.count()
|
.count()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Live LSM state, one entry per bucket.
|
/// Live LSM state, one entry per table_shard.
|
||||||
#[derive(Debug, Clone, Deserialize)]
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
pub struct LsmStats {
|
pub struct LsmStats {
|
||||||
pub buckets: Vec<BucketStats>,
|
pub table_shards: Vec<TableShardStats>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Server-side JSON envelope for `get_lsm_stats`. `lsm_stats` is null when
|
/// Server-side JSON envelope for `get_lsm_stats`. `lsm_stats` is null when
|
||||||
@@ -97,18 +97,18 @@ pub(crate) struct GetLsmStatsResponse {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
fn bucket(shard: &str, generations: &[u64], compacting: bool) -> BucketStats {
|
fn table_shard(shard: &str, sstables: &[u64], compacting: bool) -> TableShardStats {
|
||||||
BucketStats {
|
TableShardStats {
|
||||||
shard_id: shard.into(),
|
shard_id: shard.into(),
|
||||||
status: "Active".into(),
|
status: "Active".into(),
|
||||||
writer_epoch: 1,
|
writer_epoch: 1,
|
||||||
manifest_version: 1,
|
manifest_version: 1,
|
||||||
current_generation: generations.iter().max().copied().unwrap_or(0) + 1,
|
current_generation: sstables.iter().max().copied().unwrap_or(0) + 1,
|
||||||
replay_after_wal_entry_position: 0,
|
replay_after_wal_entry_position: 0,
|
||||||
wal_entry_position_last_seen: 0,
|
wal_entry_position_last_seen: 0,
|
||||||
generations: generations
|
sstables: sstables
|
||||||
.iter()
|
.iter()
|
||||||
.map(|g| GenerationStats {
|
.map(|g| SsTableStats {
|
||||||
generation: *g,
|
generation: *g,
|
||||||
bytes: 1,
|
bytes: 1,
|
||||||
rows: None,
|
rows: None,
|
||||||
@@ -123,40 +123,46 @@ mod tests {
|
|||||||
/// generation created after it must not hold the loop open — that is why
|
/// generation created after it must not hold the loop open — that is why
|
||||||
/// the predicate terminates under write load.
|
/// the predicate terminates under write load.
|
||||||
#[test]
|
#[test]
|
||||||
fn newer_generations_do_not_extend_the_target() {
|
fn newer_sstables_do_not_extend_the_target() {
|
||||||
let start = bucket("b0", &[7, 8], false);
|
let start = table_shard("b0", &[7, 8], false);
|
||||||
let target = start.newest_generation().expect("L0 is non-empty");
|
let target = start
|
||||||
|
.newest_sstable_generation()
|
||||||
|
.expect("the SSTable tier is non-empty");
|
||||||
assert_eq!(target, 8);
|
assert_eq!(target, 8);
|
||||||
|
|
||||||
// Compaction drained 7 and 8; 9 and 10 arrived while it ran.
|
// Compaction drained 7 and 8; 9 and 10 arrived while it ran.
|
||||||
let later = bucket("b0", &[9, 10], false);
|
let later = table_shard("b0", &[9, 10], false);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
later.outstanding_generations(target),
|
later.outstanding_sstables(target),
|
||||||
0,
|
0,
|
||||||
"generations above the target are somebody else's problem"
|
"sstables above the target are somebody else's problem"
|
||||||
);
|
);
|
||||||
|
|
||||||
// Still holding 8 means still outstanding.
|
// Still holding 8 means still outstanding.
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
bucket("b0", &[8, 9], false).outstanding_generations(target),
|
table_shard("b0", &[8, 9], false).outstanding_sstables(target),
|
||||||
1
|
1
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The metric counts generations, not buckets: a pass drains a bounded
|
/// The metric counts SSTables, not table shards: a pass drains a bounded
|
||||||
/// prefix, so one bucket going 3 → 2 → 1 → 0 is three steps.
|
/// prefix, so one table_shard going 3 → 2 → 1 → 0 is three steps.
|
||||||
#[test]
|
#[test]
|
||||||
fn progress_is_measured_in_generations() {
|
fn progress_is_measured_in_sstables() {
|
||||||
let target = 3;
|
let target = 3;
|
||||||
let counts: Vec<usize> = [&[1u64, 2, 3][..], &[2, 3][..], &[3][..], &[][..]]
|
let counts: Vec<usize> = [&[1u64, 2, 3][..], &[2, 3][..], &[3][..], &[][..]]
|
||||||
.iter()
|
.iter()
|
||||||
.map(|gens| bucket("b0", gens, false).outstanding_generations(target))
|
.map(|gens| table_shard("b0", gens, false).outstanding_sstables(target))
|
||||||
.collect();
|
.collect();
|
||||||
assert_eq!(counts, vec![3, 2, 1, 0]);
|
assert_eq!(counts, vec![3, 2, 1, 0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn empty_l0_has_no_target() {
|
fn an_empty_sstable_tier_has_no_target() {
|
||||||
assert!(bucket("b0", &[], false).newest_generation().is_none());
|
assert!(
|
||||||
|
table_shard("b0", &[], false)
|
||||||
|
.newest_sstable_generation()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user