From fe34eb354977f53e01855ddec4fba6b3a4df1333 Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Fri, 16 Oct 2026 13:33:57 -0500 Subject: [PATCH] refactor!: rename LSM APIs to MemWAL and align SSTable/table shard naming (#4075) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Renames the MemWAL surface in two passes, across Rust, Python, TypeScript, and Java, including the wire format. ### 1. MemWAL APIs drop "LSM" The write path these APIs drive is Lance's MemWAL; "LSM" described its shape rather than naming it. Turning it on and off now reads as enabling a feature rather than installing a spec. | Before | After | |---|---| | `set_lsm_write_spec` | `enable_memwal` | | `unset_lsm_write_spec` | `disable_memwal` | | `get_lsm_write_spec` | `get_memwal_spec` | | `flush_lsm` / `compact_lsm` / `checkpoint_lsm` | `flush_memwal` / `compact_memwal` / `checkpoint_memwal` | | `get_lsm_stats` | `get_memwal_stats` | | `close_lsm_writers` | `close_memwal_writers` | | `use_lsm` (query, merge insert) | `use_memwal` | | `LsmWriteSpec` / `LsmStats` | `MemWalWriteSpec` / `MemWalStats` | | Java `LanceDbTableLsm` | `LanceDbTableMemWal` | TypeScript and Java use camelCase (`enableMemWal`, `getMemWalStats`, ...); the napi methods set `js_name` so the generated names keep the `MemWal` casing. flush/compact/checkpoint keep a `_memwal` suffix so `compact_memwal` is not confused with `compact_files` / `optimize`, which compact the base table. Internal names (`LsmMode`, `LsmScanner`, the `merge::lsm` / `query::lsm` modules) are unchanged. ### 2. Stats use SSTable and table shard Aligns the stats surface with the MemWAL naming settled upstream in lance-format/lance#7943 and #7957, where the persisted unit became an **SSTable**. **Generation → SSTable.** A `GenerationStats` describes one flushed MemTable, which is an SSTable. The generation *number* is kept — an SSTable is identified by its generation — so only the noun moved. **Bucket → table shard.** Each entry is one MemWAL shard. "Bucket" names only the hash sharding *transform*, so it was already wrong for `identity` and `year` sharding, which produce shards and no buckets at all. | Before | After | |---|---| | `GenerationStats` | `SsTableStats` | | `BucketStats` | `TableShardStats` | | `stats.buckets` | `stats.table_shards` | | `BucketStats.generations` | `TableShardStats.sstables` | | `include_generation_rows` | `include_sstable_rows` | | `newest_generation` | `newest_sstable_generation` | | `outstanding_generations` | `outstanding_sstables` | Docs and comments follow suit: "seal" → "freeze", "L0" / "flushed generations" → SSTables, "bucket" (meaning a shard) → table shard. `MemWalWriteSpec::bucket`, `num_buckets`, and the `"Sealed"` shard status value are **unchanged** — those name the sharding transform or a server value, not the shard. ## Compatibility Breaking, for an experimental API. The wire format changes with the SDKs, so this must roll out together with the matching server change: - Routes: `/set_lsm_write_spec/` → `/enable_memwal/`, `/unset_lsm_write_spec/` → `/disable_memwal/`, `/get_lsm_write_spec/` → `/get_memwal_spec/`, `/flush_lsm/` → `/flush_memwal/`, `/compact_lsm/` → `/compact_memwal/`, `/get_lsm_stats/` → `/get_memwal_stats/` - Response keys: `lsm_write_spec` → `memwal_spec`, `lsm_stats` → `memwal_stats`, `buckets` → `table_shards`, `generations` → `sstables` - Request fields: `include_generation_rows` → `include_sstable_rows`; `use_lsm` → `use_memwal` on query and merge-insert bodies ## Validation - Rust: `cargo test --features remote -p lancedb --lib --tests` — 1467 lib tests and all integration test targets pass; `cargo clippy` clean - Python: rebuilt the extension; `test_remote_db`, `test_memwal_spec`, `test_merge_insert_lsm`, `test_query`, `test_permutation`, `test_table` — 655 passed; `ruff format` / `ruff check` clean - TypeScript: `pnpm build`; `table`, `merge`, `query` tests — 593 passed; `biome check` clean; `docs/src/js` regenerated with `npm run docs` - Java: `mvn test` — 37 passed (Spotless applied with JDK 11; the pinned google-java-format does not run on JDK 21) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 (1M context) --- docs/src/js/classes/AutoQuery.md | 12 +- docs/src/js/classes/MergeInsertBuilder.md | 12 +- docs/src/js/classes/Query.md | 12 +- docs/src/js/classes/Table.md | 232 ++++++------- docs/src/js/classes/TakeQuery.md | 4 +- docs/src/js/classes/VectorQuery.md | 12 +- docs/src/js/globals.md | 8 +- docs/src/js/interfaces/GenerationStats.md | 40 --- docs/src/js/interfaces/LsmStats.md | 22 -- docs/src/js/interfaces/MemWalStats.md | 22 ++ .../{LsmWriteSpec.md => MemWalWriteSpec.md} | 4 +- docs/src/js/interfaces/MemtableStats.md | 2 +- docs/src/js/interfaces/SsTableStats.md | 40 +++ .../{BucketStats.md => TableShardStats.md} | 34 +- docs/src/python/python.md | 2 +- .../materialized-views.mdx | 2 +- docs/web/maintenance/background-jobs.mdx | 2 +- docs/web/sql/ddl/table.mdx | 8 +- java/README.md | 18 +- .../src/main/java/com/lancedb/JsonFields.java | 6 +- .../LanceDbNamespaceClientBuilder.java | 4 +- .../java/com/lancedb/LanceDbRestClient.java | 13 +- ...bTableLsm.java => LanceDbTableMemWal.java} | 180 +++++----- .../src/main/java/com/lancedb/LsmStats.java | 56 ---- .../main/java/com/lancedb/MemWalStats.java | 56 ++++ ...LsmWriteSpec.java => MemWalWriteSpec.java} | 41 +-- .../main/java/com/lancedb/MemtableStats.java | 2 +- ...GenerationStats.java => SsTableStats.java} | 16 +- ...{BucketStats.java => TableShardStats.java} | 60 ++-- ...mTest.java => LanceDbTableMemWalTest.java} | 299 ++++++++--------- nodejs/__test__/table.test.ts | 98 +++--- nodejs/lancedb/index.ts | 8 +- nodejs/lancedb/merge.ts | 12 +- nodejs/lancedb/query.ts | 14 +- nodejs/lancedb/table.ts | 120 +++---- nodejs/src/merge.rs | 6 +- nodejs/src/query.rs | 18 +- nodejs/src/table.rs | 144 ++++---- python/python/lancedb/__init__.py | 4 +- python/python/lancedb/_lancedb.pyi | 44 +-- python/python/lancedb/_wal_hybrid.py | 8 +- python/python/lancedb/merge.py | 14 +- python/python/lancedb/permutation.py | 2 +- python/python/lancedb/query.py | 54 +-- python/python/lancedb/remote/table.py | 48 +-- python/python/lancedb/table.py | 162 ++++----- ..._lsm_write_spec.py => test_memwal_spec.py} | 108 +++--- python/python/tests/test_merge_insert_lsm.py | 102 +++--- python/python/tests/test_remote_db.py | 54 +-- python/python/tests/test_wal_hybrid.py | 10 +- python/python/tests/utils.py | 6 +- python/src/lib.rs | 4 +- python/src/query.rs | 28 +- python/src/table.rs | 141 ++++---- .../src/dataloader/permutation/builder.rs | 18 +- rust/lancedb/src/lib.rs | 2 +- rust/lancedb/src/materialized_view/refresh.rs | 6 +- rust/lancedb/src/query.rs | 26 +- rust/lancedb/src/query/wal_fusion.rs | 4 +- rust/lancedb/src/remote/table.rs | 312 ++++++++---------- rust/lancedb/src/remote/table/blobs.rs | 4 +- rust/lancedb/src/remote/table/insert.rs | 2 +- rust/lancedb/src/table.rs | 294 +++++++++-------- rust/lancedb/src/table/checkpoint.rs | 58 ++-- rust/lancedb/src/table/computed_columns.rs | 24 +- rust/lancedb/src/table/datafusion.rs | 14 +- .../table/{lsm_stats.rs => memwal_stats.rs} | 92 +++--- rust/lancedb/src/table/merge.rs | 197 +++++------ rust/lancedb/src/table/merge/lsm.rs | 118 +++---- rust/lancedb/src/table/query.rs | 36 +- rust/lancedb/src/table/query/lsm.rs | 46 +-- rust/lancedb/src/table/refresh.rs | 22 +- rust/lancedb/src/table/schema_evolution.rs | 4 +- 73 files changed, 1853 insertions(+), 1856 deletions(-) delete mode 100644 docs/src/js/interfaces/GenerationStats.md delete mode 100644 docs/src/js/interfaces/LsmStats.md create mode 100644 docs/src/js/interfaces/MemWalStats.md rename docs/src/js/interfaces/{LsmWriteSpec.md => MemWalWriteSpec.md} (93%) create mode 100644 docs/src/js/interfaces/SsTableStats.md rename docs/src/js/interfaces/{BucketStats.md => TableShardStats.md} (71%) rename java/lancedb-core/src/main/java/com/lancedb/{LanceDbTableLsm.java => LanceDbTableMemWal.java} (57%) delete mode 100644 java/lancedb-core/src/main/java/com/lancedb/LsmStats.java create mode 100644 java/lancedb-core/src/main/java/com/lancedb/MemWalStats.java rename java/lancedb-core/src/main/java/com/lancedb/{LsmWriteSpec.java => MemWalWriteSpec.java} (87%) rename java/lancedb-core/src/main/java/com/lancedb/{GenerationStats.java => SsTableStats.java} (75%) rename java/lancedb-core/src/main/java/com/lancedb/{BucketStats.java => TableShardStats.java} (76%) rename java/lancedb-core/src/test/java/com/lancedb/{LanceDbTableLsmTest.java => LanceDbTableMemWalTest.java} (63%) rename python/python/tests/{test_lsm_write_spec.py => test_memwal_spec.py} (65%) rename rust/lancedb/src/table/{lsm_stats.rs => memwal_stats.rs} (56%) diff --git a/docs/src/js/classes/AutoQuery.md b/docs/src/js/classes/AutoQuery.md index 12f517dbd..1e9859215 100644 --- a/docs/src/js/classes/AutoQuery.md +++ b/docs/src/js/classes/AutoQuery.md @@ -425,19 +425,19 @@ Empty results retain the query's output schema. *** -### useLsm() +### useMemWal() ```ts -useLsm(enable): this +useMemWal(enable): this ``` Control MemWAL read routing for this query. By default (unset), when the table carries a MemWAL write spec (see -[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so +[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so they also return data written via the `mergeInsert` LSM path that has not yet been compacted into the base table (the active/frozen in-memory memtables and -the flushed generations), deduplicated by primary key; a table without a spec +the SSTables), deduplicated by primary key; a table without a spec reads the base table. #### Parameters @@ -448,7 +448,7 @@ reads the base table. even when a spec is present. Note: the LSM scanner does not support every query shape (e.g. reranking, hybrid search, `orderBy`). On a MemWAL table those shapes error unless - `useLsm(false)` is set, because a base-only read would silently exclude + `useMemWal(false)` is set, because a base-only read would silently exclude un-compacted MemWAL data. #### Returns @@ -457,7 +457,7 @@ reads the base table. #### Inherited from -`StandardQueryBase.useLsm` +`StandardQueryBase.useMemWal` *** diff --git a/docs/src/js/classes/MergeInsertBuilder.md b/docs/src/js/classes/MergeInsertBuilder.md index 81349a0bc..cf40837d3 100644 --- a/docs/src/js/classes/MergeInsertBuilder.md +++ b/docs/src/js/classes/MergeInsertBuilder.md @@ -76,15 +76,15 @@ the query optimizer chooses a suboptimal path. *** -### useLsm() +### useMemWal() ```ts -useLsm(enable): MergeInsertBuilder +useMemWal(enable): MergeInsertBuilder ``` Control MemWAL routing for this merge. -By default (unset), a `mergeInsert` on a table with an LSM write spec is +By default (unset), a `mergeInsert` on a table with a MemWAL write spec is routed through Lance's MemWAL shard writer, and a table without one uses the standard path. @@ -92,7 +92,7 @@ standard path. * **enable**: `boolean` `true` forces MemWAL routing and errors if the table has no - LSM write spec. `false` forces the standard write path even when a spec is set. + MemWAL write spec. `false` forces the standard write path even when a spec is set. #### Returns @@ -108,12 +108,12 @@ validateSingleShard(validateSingleShard): MergeInsertBuilder Controls how an LSM merge checks that its input targets a single shard. -When a table has an LSM write spec, every row in a `mergeInsert` call must +When a table has a MemWAL write spec, every row in a `mergeInsert` call must route to the same shard. When `true` (the default), every row is inspected to verify this. When `false`, only the first row is inspected and the shard it routes to is used for the whole input — a faster path for callers that have already pre-sharded their input. Has no effect on tables without -an LSM write spec. +a MemWAL write spec. #### Parameters diff --git a/docs/src/js/classes/Query.md b/docs/src/js/classes/Query.md index 3038bf2b3..9d83404f4 100644 --- a/docs/src/js/classes/Query.md +++ b/docs/src/js/classes/Query.md @@ -499,19 +499,19 @@ Empty results retain the query's output schema. *** -### useLsm() +### useMemWal() ```ts -useLsm(enable): this +useMemWal(enable): this ``` Control MemWAL read routing for this query. By default (unset), when the table carries a MemWAL write spec (see -[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so +[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so they also return data written via the `mergeInsert` LSM path that has not yet been compacted into the base table (the active/frozen in-memory memtables and -the flushed generations), deduplicated by primary key; a table without a spec +the SSTables), deduplicated by primary key; a table without a spec reads the base table. #### Parameters @@ -522,7 +522,7 @@ reads the base table. even when a spec is present. Note: the LSM scanner does not support every query shape (e.g. reranking, hybrid search, `orderBy`). On a MemWAL table those shapes error unless - `useLsm(false)` is set, because a base-only read would silently exclude + `useMemWal(false)` is set, because a base-only read would silently exclude un-compacted MemWAL data. #### Returns @@ -531,7 +531,7 @@ reads the base table. #### Inherited from -`StandardQueryBase.useLsm` +`StandardQueryBase.useMemWal` *** diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 59c3209c8..3eaf9c9ca 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -228,17 +228,17 @@ version of the table. *** -### checkpointLsm() +### checkpointMemWal() ```ts -abstract checkpointLsm(): Promise +abstract checkpointMemWal(): Promise ``` -Converge this table's LSM write path into its base table. +Converge this table's MemWAL write path into its base table. -Seals once, then triggers compaction and polls until the L0 that existed +Freezes once, then triggers compaction and polls until the SSTables that existed at the start is gone. The target set is fixed at the start, so -generations created *during* the checkpoint are ignored — that is what +SSTables created *during* the checkpoint are ignored — that is what lets it terminate under write load, and what makes it best-effort: it converges the fresh tier as of some instant. Idempotent, abandonable at any point, and safe to run on a cadence. @@ -254,9 +254,9 @@ is merging. The caller owns the deadline. #### Example ```ts -const before = await table.getLsmStats(); -await table.checkpointLsm(); -const after = await table.getLsmStats(); +const before = await table.getMemWalStats(); +await table.checkpointMemWal(); +const after = await table.getMemWalStats(); ``` *** @@ -279,15 +279,15 @@ Any attempt to use the table after it is closed will result in an error. *** -### closeLsmWriters() +### closeMemWalWriters() ```ts -abstract closeLsmWriters(): Promise +abstract closeMemWalWriters(): Promise ``` Drain and close any cached MemWAL shard writers held for this table. -When an [LsmWriteSpec](../interfaces/LsmWriteSpec.md) is installed, `mergeInsert` opens MemWAL +When an [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) is installed, `mergeInsert` opens MemWAL shard writers and caches them for reuse across calls. This closes them, flushing pending data; writers reopen lazily on the next `mergeInsert`. It is a no-op when no writers are cached. @@ -298,17 +298,17 @@ It is a no-op when no writers are cached. *** -### compactLsm() +### compactMemWal() ```ts -abstract compactLsm(): Promise +abstract compactMemWal(): Promise ``` -Trigger a background L0 → base compaction pass per bucket. +Trigger a background SSTable compaction pass per table shard. Returns once the passes are *dispatched*, not once they finish — watch -[Table#getLsmStats](Table.md#getlsmstats) for progress, or use -[Table#checkpointLsm](Table.md#checkpointlsm) to wait for convergence. +[Table#getMemWalStats](Table.md#getmemwalstats) for progress, or use +[Table#checkpointMemWal](Table.md#checkpointmemwal) to wait for convergence. #### Returns @@ -449,6 +449,23 @@ containing the new version number of the table *** +### disableMemWal() + +```ts +abstract disableMemWal(): Promise +``` + +Remove the [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) from this table, reverting to the standard +`mergeInsert` write path. + +Errors if no spec is currently set. + +#### Returns + +`Promise`<`void`> + +*** + ### display() ```ts @@ -514,6 +531,54 @@ Drop an index from the table. *** +### enableMemWal() + +```ts +abstract enableMemWal(spec): Promise +``` + +Install an [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) on this table, selecting Lance's MemWAL +LSM-style write path for future `mergeInsert` calls. + +`MemWalWriteSpec` chooses one of three sharding strategies via `specType`: + +- `"bucket"` — hash-bucket writes by the single-column unenforced primary + key (`column` and `numBuckets` required). +- `"identity"` — shard by the raw value of a scalar `column`. +- `"unsharded"` — route every write to a single shard. + +All variants require the table to have an unenforced primary key +([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally +requires it to be the single column being bucketed. + +Omitting `maintainedIndexes` maintains every index the table has, +including ones created later, and skips a kind the MemWAL cannot maintain. +Naming them pins an exact set, and a still-building index is rejected +rather than quietly omitted. + +#### Parameters + +* **spec**: [`MemWalWriteSpec`](../interfaces/MemWalWriteSpec.md) + The sharding spec to install. + +#### Returns + +`Promise`<`void`> + +#### Example + +```ts +await table.setUnenforcedPrimaryKey("id"); +await table.enableMemWal({ + specType: "bucket", + column: "id", + numBuckets: 16, + maintainedIndexes: ["id_idx"], +}); +``` + +*** + ### fetchBlobFiles() ```ts @@ -582,15 +647,15 @@ memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values. *** -### flushLsm() +### flushMemWal() ```ts -abstract flushLsm(): Promise +abstract flushMemWal(): Promise ``` -Seal every bucket's active memtable into a new L0 generation. +Freeze every table shard's active memtable into a new SSTable. -Returns once the seal is committed. Sealing an empty memtable is a no-op, +Returns once the freeze is committed. Freezing an empty memtable is a no-op, so this is safe to call repeatedly. #### Returns @@ -634,48 +699,48 @@ const { records, truncated } = await table.functionErrors({ column: "embedding" *** -### getLsmStats() +### getMemWalSpec() ```ts -abstract getLsmStats(includeGenerationRows?): Promise +abstract getMemWalSpec(): Promise ``` -Read live per-bucket LSM state. +Read the [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) currently installed on this table. -Answers "how far behind is my fresh tier", "which bucket is hot", and -"why is my fresh-tier vector search brute-force". Mutates no table state. - -Resolves to `undefined` only when the LSM write path is not enabled. - -#### Parameters - -* **includeGenerationRows?**: `boolean` - Also count rows per L0 generation. - Off by default because each count opens an uncached Lance dataset. - -#### Returns - -`Promise`<`undefined` \| [`LsmStats`](../interfaces/LsmStats.md)> - -*** - -### getLsmWriteSpec() - -```ts -abstract getLsmWriteSpec(): Promise -``` - -Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on this table. - -Resolves to `undefined` when the MemWAL LSM write path is not enabled (no -spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)). +Resolves to `undefined` when the MemWAL write path is not enabled (no +spec has been set, or it was removed with [Table#disableMemWal](Table.md#disablememwal)). The spec is the one installed, including its maintained-index selection: an absent `maintainedIndexes` for every index the table has, an empty array for none. #### Returns -`Promise`<`undefined` \| [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)> +`Promise`<`undefined` \| [`MemWalWriteSpec`](../interfaces/MemWalWriteSpec.md)> + +*** + +### getMemWalStats() + +```ts +abstract getMemWalStats(includeSstableRows?): Promise +``` + +Read live per-table-shard MemWAL state. + +Answers "how far behind is my fresh tier", "which table shard is hot", and +"why is my fresh-tier vector search brute-force". Mutates no table state. + +Resolves to `undefined` only when the MemWAL write path is not enabled. + +#### Parameters + +* **includeSstableRows?**: `boolean` + Also count rows per SSTable. + Off by default because each count opens an uncached Lance dataset. + +#### Returns + +`Promise`<`undefined` \| [`MemWalStats`](../interfaces/MemWalStats.md)> *** @@ -1093,54 +1158,6 @@ of the given query *** -### setLsmWriteSpec() - -```ts -abstract setLsmWriteSpec(spec): Promise -``` - -Install an [LsmWriteSpec](../interfaces/LsmWriteSpec.md) on this table, selecting Lance's MemWAL -LSM-style write path for future `mergeInsert` calls. - -`LsmWriteSpec` chooses one of three sharding strategies via `specType`: - -- `"bucket"` — hash-bucket writes by the single-column unenforced primary - key (`column` and `numBuckets` required). -- `"identity"` — shard by the raw value of a scalar `column`. -- `"unsharded"` — route every write to a single shard. - -All variants require the table to have an unenforced primary key -([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally -requires it to be the single column being bucketed. - -Omitting `maintainedIndexes` maintains every index the table has, -including ones created later, and skips a kind the MemWAL cannot maintain. -Naming them pins an exact set, and a still-building index is rejected -rather than quietly omitted. - -#### Parameters - -* **spec**: [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md) - The sharding spec to install. - -#### Returns - -`Promise`<`void`> - -#### Example - -```ts -await table.setUnenforcedPrimaryKey("id"); -await table.setLsmWriteSpec({ - specType: "bucket", - column: "id", - numBuckets: 16, - maintainedIndexes: ["id_idx"], -}); -``` - -*** - ### setUnenforcedPrimaryKey() ```ts @@ -1295,23 +1312,6 @@ same tokenizer model files must also exist locally. *** -### unsetLsmWriteSpec() - -```ts -abstract unsetLsmWriteSpec(): Promise -``` - -Remove the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) from this table, reverting to the standard -`mergeInsert` write path. - -Errors if no spec is currently set. - -#### Returns - -`Promise`<`void`> - -*** - ### update() #### update(opts) diff --git a/docs/src/js/classes/TakeQuery.md b/docs/src/js/classes/TakeQuery.md index f3d59a2a3..9e8d03149 100644 --- a/docs/src/js/classes/TakeQuery.md +++ b/docs/src/js/classes/TakeQuery.md @@ -275,10 +275,10 @@ Empty results retain the query's output schema. *** -### useLsm() +### useMemWal() ```ts -useLsm(enable): this +useMemWal(enable): this ``` Control MemWAL read routing for this take query. diff --git a/docs/src/js/classes/VectorQuery.md b/docs/src/js/classes/VectorQuery.md index 4809ebefd..e329cb254 100644 --- a/docs/src/js/classes/VectorQuery.md +++ b/docs/src/js/classes/VectorQuery.md @@ -781,19 +781,19 @@ Empty results retain the query's output schema. *** -### useLsm() +### useMemWal() ```ts -useLsm(enable): this +useMemWal(enable): this ``` Control MemWAL read routing for this query. By default (unset), when the table carries a MemWAL write spec (see -[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so +[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so they also return data written via the `mergeInsert` LSM path that has not yet been compacted into the base table (the active/frozen in-memory memtables and -the flushed generations), deduplicated by primary key; a table without a spec +the SSTables), deduplicated by primary key; a table without a spec reads the base table. #### Parameters @@ -804,7 +804,7 @@ reads the base table. even when a spec is present. Note: the LSM scanner does not support every query shape (e.g. reranking, hybrid search, `orderBy`). On a MemWAL table those shapes error unless - `useLsm(false)` is set, because a base-only read would silently exclude + `useMemWal(false)` is set, because a base-only read would silently exclude un-compacted MemWAL data. #### Returns @@ -813,7 +813,7 @@ reads the base table. #### Inherited from -`StandardQueryBase.useLsm` +`StandardQueryBase.useMemWal` *** diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 9cf3136af..739b9033a 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -66,7 +66,6 @@ - [BranchDiff](interfaces/BranchDiff.md) - [BranchIndexSummary](interfaces/BranchIndexSummary.md) - [BranchRowCountSummary](interfaces/BranchRowCountSummary.md) -- [BucketStats](interfaces/BucketStats.md) - [CatalogOptions](interfaces/CatalogOptions.md) - [CherryPickError](interfaces/CherryPickError.md) - [CherryPickPreview](interfaces/CherryPickPreview.md) @@ -98,7 +97,6 @@ - [FunctionErrorRecord](interfaces/FunctionErrorRecord.md) - [FunctionErrors](interfaces/FunctionErrors.md) - [FunctionErrorsOptions](interfaces/FunctionErrorsOptions.md) -- [GenerationStats](interfaces/GenerationStats.md) - [HnswPqOptions](interfaces/HnswPqOptions.md) - [HnswSqOptions](interfaces/HnswSqOptions.md) - [IndexConfig](interfaces/IndexConfig.md) @@ -115,9 +113,9 @@ - [ListTablesOptions](interfaces/ListTablesOptions.md) - [ListTablesResponse](interfaces/ListTablesResponse.md) - [ListingOptions](interfaces/ListingOptions.md) -- [LsmStats](interfaces/LsmStats.md) -- [LsmWriteSpec](interfaces/LsmWriteSpec.md) - [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md) +- [MemWalStats](interfaces/MemWalStats.md) +- [MemWalWriteSpec](interfaces/MemWalWriteSpec.md) - [MemtableStats](interfaces/MemtableStats.md) - [MergeResult](interfaces/MergeResult.md) - [NativeOAuthConfig](interfaces/NativeOAuthConfig.md) @@ -140,7 +138,9 @@ - [SplitHashOptions](interfaces/SplitHashOptions.md) - [SplitRandomOptions](interfaces/SplitRandomOptions.md) - [SplitSequentialOptions](interfaces/SplitSequentialOptions.md) +- [SsTableStats](interfaces/SsTableStats.md) - [TableNamesOptions](interfaces/TableNamesOptions.md) +- [TableShardStats](interfaces/TableShardStats.md) - [TableStatistics](interfaces/TableStatistics.md) - [TimeoutConfig](interfaces/TimeoutConfig.md) - [TlsConfig](interfaces/TlsConfig.md) diff --git a/docs/src/js/interfaces/GenerationStats.md b/docs/src/js/interfaces/GenerationStats.md deleted file mode 100644 index 19dd2afda..000000000 --- a/docs/src/js/interfaces/GenerationStats.md +++ /dev/null @@ -1,40 +0,0 @@ -[**@lancedb/lancedb**](../README.md) • **Docs** - -*** - -[@lancedb/lancedb](../globals.md) / GenerationStats - -# Interface: GenerationStats - -One flushed L0 generation. - -## Properties - -### bytes - -```ts -bytes: number; -``` - -On-disk size of the generation. - -*** - -### generation - -```ts -generation: number; -``` - -The generation number. Increases as memtables are sealed into L0. - -*** - -### rows? - -```ts -optional rows: number; -``` - -Present only when `includeGenerationRows` was requested. Off by default -because each count opens an uncached Lance dataset. diff --git a/docs/src/js/interfaces/LsmStats.md b/docs/src/js/interfaces/LsmStats.md deleted file mode 100644 index 76a2f50db..000000000 --- a/docs/src/js/interfaces/LsmStats.md +++ /dev/null @@ -1,22 +0,0 @@ -[**@lancedb/lancedb**](../README.md) • **Docs** - -*** - -[@lancedb/lancedb](../globals.md) / LsmStats - -# Interface: LsmStats - -Live per-bucket LSM state, as returned by `Table#getLsmStats`. - -Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are -the caller's to compute. - -## Properties - -### buckets - -```ts -buckets: BucketStats[]; -``` - -One entry per bucket backing this table. diff --git a/docs/src/js/interfaces/MemWalStats.md b/docs/src/js/interfaces/MemWalStats.md new file mode 100644 index 000000000..4221867fc --- /dev/null +++ b/docs/src/js/interfaces/MemWalStats.md @@ -0,0 +1,22 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MemWalStats + +# Interface: MemWalStats + +Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`. + +Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are +the caller's to compute. + +## Properties + +### tableShards + +```ts +tableShards: TableShardStats[]; +``` + +One entry per table shard backing this table. diff --git a/docs/src/js/interfaces/LsmWriteSpec.md b/docs/src/js/interfaces/MemWalWriteSpec.md similarity index 93% rename from docs/src/js/interfaces/LsmWriteSpec.md rename to docs/src/js/interfaces/MemWalWriteSpec.md index 693542dbd..50b74da41 100644 --- a/docs/src/js/interfaces/LsmWriteSpec.md +++ b/docs/src/js/interfaces/MemWalWriteSpec.md @@ -2,9 +2,9 @@ *** -[@lancedb/lancedb](../globals.md) / LsmWriteSpec +[@lancedb/lancedb](../globals.md) / MemWalWriteSpec -# Interface: LsmWriteSpec +# Interface: MemWalWriteSpec Specification selecting Lance's MemWAL LSM-style write path for `mergeInsert`. diff --git a/docs/src/js/interfaces/MemtableStats.md b/docs/src/js/interfaces/MemtableStats.md index fdc1e4467..63b8eace8 100644 --- a/docs/src/js/interfaces/MemtableStats.md +++ b/docs/src/js/interfaces/MemtableStats.md @@ -36,7 +36,7 @@ Estimated in-memory size. generation: number; ``` -The generation this memtable will become once sealed. +The generation this memtable will become once frozen. *** diff --git a/docs/src/js/interfaces/SsTableStats.md b/docs/src/js/interfaces/SsTableStats.md new file mode 100644 index 000000000..7b01a46f2 --- /dev/null +++ b/docs/src/js/interfaces/SsTableStats.md @@ -0,0 +1,40 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / SsTableStats + +# Interface: SsTableStats + +One SSTable. + +## Properties + +### bytes + +```ts +bytes: number; +``` + +On-disk size of the SSTable. + +*** + +### generation + +```ts +generation: number; +``` + +The generation number. Increases as memtables are frozen into SSTables. + +*** + +### rows? + +```ts +optional rows: number; +``` + +Present only when `includeSstableRows` was requested. Off by default +because each count opens an uncached Lance dataset. diff --git a/docs/src/js/interfaces/BucketStats.md b/docs/src/js/interfaces/TableShardStats.md similarity index 71% rename from docs/src/js/interfaces/BucketStats.md rename to docs/src/js/interfaces/TableShardStats.md index 3f5095672..64be5e863 100644 --- a/docs/src/js/interfaces/BucketStats.md +++ b/docs/src/js/interfaces/TableShardStats.md @@ -2,12 +2,12 @@ *** -[@lancedb/lancedb](../globals.md) / BucketStats +[@lancedb/lancedb](../globals.md) / TableShardStats -# Interface: BucketStats +# Interface: TableShardStats -Live state of one bucket. A table is N buckets on one node; flattening to a -single number hides the one hot bucket that is usually why someone opened +Live state of one table shard. A table is N table shards on one node; flattening to a +single number hides the one hot table shard that is usually why someone opened this endpoint. ## Properties @@ -18,7 +18,7 @@ this endpoint. compacting: boolean; ``` -Whether a pass owns this bucket's compaction latch right now. Says *a* +Whether a pass owns this table shard's compaction latch right now. Says *a* driver is running, not *whose*, and the latch is held from dispatch — including while the pass queues for a pod-wide compactor permit. Read it as "do not pile on", never as "mine is progressing". @@ -35,16 +35,6 @@ The generation the active memtable will become. *** -### generations - -```ts -generations: GenerationStats[]; -``` - -Flushed L0 generations not yet merged into the base table. - -*** - ### manifestVersion ```ts @@ -61,7 +51,7 @@ Version of the shard manifest these numbers were read from. optional memtables: MemtableStats[]; ``` -Oldest first, active last. Absent for a `"Sealed"` bucket, whose +Oldest first, active last. Absent for a `"Sealed"` table shard, whose in-memory state is torn down. *** @@ -82,7 +72,17 @@ WAL position replay resumes from. shardId: string; ``` -The shard this bucket writes. +The id of this table shard. + +*** + +### sstables + +```ts +sstables: SsTableStats[]; +``` + +SSTables not yet merged into the base table. *** diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 24372a83d..8405852f2 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -297,7 +297,7 @@ listing a storage directory. ::: lancedb.table.Branches -::: lancedb.LsmWriteSpec +::: lancedb.MemWalWriteSpec ## Functions and Jobs diff --git a/docs/web/feature-engineering/materialized-views.mdx b/docs/web/feature-engineering/materialized-views.mdx index ab949adb1..ffe4fc959 100644 --- a/docs/web/feature-engineering/materialized-views.mdx +++ b/docs/web/feature-engineering/materialized-views.mdx @@ -52,7 +52,7 @@ In Python SDK 0.40.0, create the source with above does. Stable row IDs let the view track source rows across updates and compaction. Set this option when creating the table; it cannot be added later. -A source with an LSM write specification or retained, uncompacted write-ahead-log +A source with a MemWAL write spec or retained, uncompacted write-ahead-log rows is not accepted by the local materialized-view implementation. ## Refresh behavior {#incremental} diff --git a/docs/web/maintenance/background-jobs.mdx b/docs/web/maintenance/background-jobs.mdx index 151828e5d..2fe5d549c 100644 --- a/docs/web/maintenance/background-jobs.mdx +++ b/docs/web/maintenance/background-jobs.mdx @@ -32,7 +32,7 @@ log from growing without bound: | Job type | What it does | | --- | --- | -| `sstable_compaction` | Folds a WAL bucket's SSTables into the base table by primary key. Distinct from `compact`, which merges base-table fragments -- the two are unrelated. | +| `sstable_compaction` | Folds a table shard's SSTables into the base table by primary key. Distinct from `compact`, which merges base-table fragments -- the two are unrelated. | | `wal_index_catchup` | Extends a base-table index over the rows `sstable_compaction` just folded in, and records how far the index has caught up. Nothing else advances that record. | ## Index maintenance {#index} diff --git a/docs/web/sql/ddl/table.mdx b/docs/web/sql/ddl/table.mdx index 390c24b65..05406e677 100644 --- a/docs/web/sql/ddl/table.mdx +++ b/docs/web/sql/ddl/table.mdx @@ -102,10 +102,10 @@ duplicates impossible. It tells the planner what to assume. The column must be declared `NOT NULL`. ```sql -ALTER TABLE t SET WAL -ALTER TABLE t SET WAL WITH (sharding = 'bucket', column = 'id', buckets = 4) -ALTER TABLE t SET WAL WITH (maintained_indexes = '') -ALTER TABLE t UNSET WAL +ALTER TABLE t SET MEMWAL +ALTER TABLE t SET MEMWAL WITH (sharding = 'bucket', column = 'id', buckets = 4) +ALTER TABLE t SET MEMWAL WITH (maintained_indexes = '') +ALTER TABLE t UNSET MEMWAL ``` A write-ahead log makes small frequent writes cheap by batching them out of the diff --git a/java/README.md b/java/README.md index 4abe5f50e..0e54d20e4 100644 --- a/java/README.md +++ b/java/README.md @@ -29,7 +29,7 @@ LanceNamespace namespaceClient = LanceDbNamespaceClientBuilder.newBuilder() .build(); ``` -## MemWAL LSM write path +## MemWAL write path Most table operations reach LanceDB through the `LanceNamespace` above, which is generated from the Lance Namespace specification. The MemWAL LSM routes are not part @@ -37,27 +37,27 @@ of that specification, so they are issued through a separate client: ```java import com.lancedb.LanceDbRestClient; -import com.lancedb.LanceDbTableLsm; -import com.lancedb.LsmWriteSpec; +import com.lancedb.LanceDbTableMemWal; +import com.lancedb.MemWalWriteSpec; LanceDbRestClient client = LanceDbNamespaceClientBuilder.newBuilder() .apiKey("your_lancedb_cloud_api_key") .database("your_database_name") .buildRestClient(); -LanceDbTableLsm lsm = new LanceDbTableLsm(client, "my_table"); +LanceDbTableMemWal memWal = new LanceDbTableMemWal(client, "my_table"); // Route future merge_insert upserts through the MemWAL, hash-bucketed by `id`. -lsm.setLsmWriteSpec(LsmWriteSpec.bucket("id", 16)); +memWal.enableMemWal(MemWalWriteSpec.bucket("id", 16)); // ... merge_insert traffic ... // Converge the fresh tier into the base table. -lsm.checkpointLsm(); +memWal.checkpointMemWal(); -// Inspect live per-bucket state. -lsm.getLsmStats().ifPresent(stats -> stats.buckets().forEach(bucket -> - System.out.println(bucket.shardId() + ": " + bucket.generations().size() + " L0 generations"))); +// Inspect live per-table-shard state. +memWal.getMemWalStats().ifPresent(stats -> stats.tableShards().forEach(tableShard -> + System.out.println(tableShard.shardId() + ": " + tableShard.sstables().size() + " SSTables"))); client.close(); ``` diff --git a/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java b/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java index b78e2411a..e0a288eaf 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java +++ b/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java @@ -20,12 +20,12 @@ import com.fasterxml.jackson.databind.JsonNode; * *

Every reader fails closed: a missing, null, or wrong-typed field throws rather than * defaulting. That mirrors the serde decoding the Rust client applies to the same payloads in - * {@code rust/lancedb/src/table/lsm_stats.rs}, where a required field has no default and a + * {@code rust/lancedb/src/table/memwal_stats.rs}, where a required field has no default and a * malformed response is an error rather than a zero. * *

The alternative — Jackson's {@code path()}, which yields a missing node that reads as an empty - * array or a zero — is unsafe here because {@link LanceDbTableLsm#checkpointLsm()} decides - * convergence from these numbers. A defaulted {@code generations} array is indistinguishable from a + * array or a zero — is unsafe here because {@link LanceDbTableMemWal#checkpointMemWal()} decides + * convergence from these numbers. A defaulted {@code sstables} array is indistinguishable from a * drained one, so a malformed response would report a checkpoint that never happened. */ final class JsonFields { diff --git a/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java b/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java index da241dfd5..05aed8c74 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java +++ b/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java @@ -151,8 +151,8 @@ public class LanceDbNamespaceClientBuilder { * Build a {@link LanceDbRestClient} for the same endpoint. * *

Needed only for LanceDB routes that the Lance Namespace specification does not cover — the - * MemWAL LSM write path, reached through {@link LanceDbTableLsm}. Every other table operation - * belongs on the {@link LanceNamespace} from {@link #build()}. + * MemWAL write path, reached through {@link LanceDbTableMemWal}. Every other table + * operation belongs on the {@link LanceNamespace} from {@link #build()}. * *

The returned client owns an HTTP connection pool; close it when you are done with it. * diff --git a/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java b/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java index baafbb9df..5275c8346 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java +++ b/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java @@ -31,9 +31,9 @@ import java.io.UncheckedIOException; * specification does not cover. * *

Most table operations reach LanceDB through {@link org.lance.namespace.LanceNamespace}, which - * is generated from the namespace spec. A handful of routes — the MemWAL LSM write path in + * is generated from the namespace spec. A handful of routes — the MemWAL write path in * particular — are served by the same endpoint but are not part of that spec, so they are issued - * directly here. See {@link LanceDbTableLsm}. + * directly here. See {@link LanceDbTableMemWal}. * *

Obtain one from {@link LanceDbNamespaceClientBuilder#buildRestClient()}. */ @@ -50,8 +50,8 @@ public class LanceDbRestClient implements Closeable { this.apiKey = apiKey; this.database = database; // Automatic retries off, deliberately. The default strategy retries 429 and 503 — - // exactly the two statuses LanceDbTableLsm.checkpointLsm() acts on — which would - // silently double its explicit retry budget and would also retry compact_lsm in + // exactly the two statuses LanceDbTableMemWal.checkpointMemWal() acts on — which would + // silently double its explicit retry budget and would also retry compact_memwal in // place, where the loop is designed to fall through to a fresh stats poll instead. // The checkpoint loop owns the 421/429/503 transitions; the transport must not. this.http = HttpClients.custom().disableAutomaticRetries().build(); @@ -98,8 +98,9 @@ public class LanceDbRestClient implements Closeable { /** * A non-2xx response. * - *

The status is exposed because callers act on it: {@link LanceDbTableLsm#checkpointLsm()} - * treats 429 and 503 as retryable and 421 as a lost node claim. + *

The status is exposed because callers act on it: {@link + * LanceDbTableMemWal#checkpointMemWal()} treats 429 and 503 as retryable and 421 as a lost node + * claim. */ public static class HttpException extends RuntimeException { private static final long serialVersionUID = 1L; diff --git a/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java b/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableMemWal.java similarity index 57% rename from java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java rename to java/lancedb-core/src/main/java/com/lancedb/LanceDbTableMemWal.java index bb1467ecd..75c6c33b1 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java +++ b/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableMemWal.java @@ -22,11 +22,11 @@ import java.util.Optional; import java.util.OptionalLong; /** - * The MemWAL LSM write path for one LanceDB Cloud or Enterprise table. + * The MemWAL write path for one LanceDB Cloud or Enterprise table. * - *

Installing an {@link LsmWriteSpec} routes {@code mergeInsert} upserts through Lance's MemWAL — - * an LSM-style append — instead of the standard merge path. Rows land in an in-memory memtable, - * seal into L0 generations, and are merged into the base table by compaction. + *

Installing an {@link MemWalWriteSpec} routes {@code mergeInsert} upserts through Lance's + * MemWAL — an LSM-style append — instead of the standard merge path. Rows land in an in-memory + * memtable, freeze into SSTables, and are merged into the base table by compaction. * *

These routes are not part of the Lance Namespace specification, so they are issued directly * rather than through {@link org.lance.namespace.LanceNamespace}. @@ -37,23 +37,23 @@ import java.util.OptionalLong; * .database("your_database_name") * .buildRestClient(); * - * LanceDbTableLsm lsm = new LanceDbTableLsm(client, "my_table"); - * lsm.setLsmWriteSpec(LsmWriteSpec.bucket("id", 16)); + * LanceDbTableMemWal memWal = new LanceDbTableMemWal(client, "my_table"); + * memWal.enableMemWal(MemWalWriteSpec.bucket("id", 16)); * // ... merge_insert traffic ... - * lsm.checkpointLsm(); + * memWal.checkpointMemWal(); * } */ -public class LanceDbTableLsm { +public class LanceDbTableMemWal { /** - * Interval between {@code get_lsm_stats} polls during a checkpoint. One interval is roughly one - * compaction pass, the granularity at which the answer can change. + * Interval between {@code get_memwal_stats} polls during a checkpoint. One interval is roughly + * one compaction pass, the granularity at which the answer can change. */ private static final long POLL_INTERVAL_MS = 5_000L; /** - * Cap on re-issues from {@code flushLsm} after a 421, so a crash-looping node cannot turn flush → - * compact → 421 → flush into a spin. + * Cap on re-issues from {@code flushMemWal} after a 421, so a crash-looping node cannot turn + * flush → compact → 421 → flush into a spin. * *

Deliberately not shared with {@link #MAX_RETRIES}: a claim that keeps evaporating is a * broken node, while contention is routine and wants a real budget. @@ -79,7 +79,7 @@ public class LanceDbTableLsm { * @param tableIdentifier The table's full identifier, {@code $}-delimited when it sits inside a * namespace, such as {@code analytics$events}. */ - public LanceDbTableLsm(LanceDbRestClient client, String tableIdentifier) { + public LanceDbTableMemWal(LanceDbRestClient client, String tableIdentifier) { if (client == null) { throw new IllegalArgumentException("Client cannot be null"); } @@ -91,140 +91,140 @@ public class LanceDbTableLsm { } /** - * Install an {@link LsmWriteSpec} on this table, selecting the MemWAL LSM write path for future - * {@code mergeInsert} calls. + * Install an {@link MemWalWriteSpec} on this table, selecting the MemWAL write path for + * future {@code mergeInsert} calls. * *

All variants require the table to have an unenforced primary key; bucket sharding * additionally requires it to be the single column being bucketed. */ - public void setLsmWriteSpec(LsmWriteSpec spec) { + public void enableMemWal(MemWalWriteSpec spec) { if (spec == null) { throw new IllegalArgumentException("Spec cannot be null"); } - client.post(route("set_lsm_write_spec"), spec.toRequestBody()); + client.post(route("enable_memwal"), spec.toRequestBody()); } /** - * Remove the {@link LsmWriteSpec} from this table, reverting to the standard {@code mergeInsert} - * write path. + * Remove the {@link MemWalWriteSpec} from this table, reverting to the standard {@code + * mergeInsert} write path. * *

Errors if no spec is currently set. */ - public void unsetLsmWriteSpec() { - client.post(route("unset_lsm_write_spec"), null); + public void disableMemWal() { + client.post(route("disable_memwal"), null); } /** - * Read the {@link LsmWriteSpec} currently installed on this table. + * Read the {@link MemWalWriteSpec} currently installed on this table. * - *

Empty when the LSM write path is not enabled at all — no spec has been set, or one was - * removed with {@link #unsetLsmWriteSpec()}. That is a different answer from a present spec - * whose {@link LsmWriteSpec#maintainedIndexes()} is null, which selects indexes automatically. + *

Empty when the MemWAL write path is not enabled at all — no spec has been set, or one was + * removed with {@link #disableMemWal()}. That is a different answer from a present spec whose + * {@link MemWalWriteSpec#maintainedIndexes()} is null, which selects indexes automatically. * *

The spec read back is the one that was installed, selection included: null maintains every - * supported index the table has now or gains later, an empty list maintains none, and a - * non-empty list maintains exactly those. All three round-trip. + * supported index the table has now or gains later, an empty list maintains none, and a non-empty + * list maintains exactly those. All three round-trip. */ - public Optional getLsmWriteSpec() { - JsonNode response = client.post(route("get_lsm_write_spec"), null); - if (response == null || !response.hasNonNull("lsm_write_spec")) { + public Optional getMemWalSpec() { + JsonNode response = client.post(route("get_memwal_spec"), null); + if (response == null || !response.hasNonNull("memwal_spec")) { return Optional.empty(); } - return Optional.of(LsmWriteSpec.fromJson(response.get("lsm_write_spec"))); + return Optional.of(MemWalWriteSpec.fromJson(response.get("memwal_spec"))); } /** - * Seal every bucket's active memtable into a new L0 generation. + * Freeze every table shard's active memtable into a new SSTable. * - *

Returns once the seal is committed. Sealing an empty memtable is a no-op, so this is safe to - * call repeatedly. + *

Returns once the freeze is committed. Freezing an empty memtable is a no-op, so this is safe + * to call repeatedly. */ - public void flushLsm() { - client.post(route("flush_lsm"), null); + public void flushMemWal() { + client.post(route("flush_memwal"), null); } /** - * Trigger a background L0 → base compaction pass per bucket. + * Trigger a background SSTable compaction pass per table shard. * *

Returns once the passes are dispatched, not once they finish — watch {@link - * #getLsmStats}, or use {@link #checkpointLsm} to wait for convergence. + * #getMemWalStats}, or use {@link #checkpointMemWal} to wait for convergence. */ - public void compactLsm() { - client.post(route("compact_lsm"), null); + public void compactMemWal() { + client.post(route("compact_memwal"), null); } /** - * Read live per-bucket LSM state. + * Read live per-table-shard MemWAL state. * - *

Answers "how far behind is my fresh tier", "which bucket is hot", and "why is my fresh-tier - * vector search brute-force". Mutates no table state. + *

Answers "how far behind is my fresh tier", "which table shard is hot", and "why is my + * fresh-tier vector search brute-force". Mutates no table state. * - *

Empty only when the LSM write path is not enabled — that is, when the server sends an absent - * or null {@code lsm_stats}. A stats object that is present is decoded strictly, and a malformed - * one throws rather than decoding to something empty, because {@link #checkpointLsm} reads - * convergence out of these numbers and cannot tell a defaulted array from a drained one. + *

Empty only when the MemWAL write path is not enabled — that is, when the server sends an + * absent or null {@code memwal_stats}. A stats object that is present is decoded strictly, and a + * malformed one throws rather than decoding to something empty, because {@link #checkpointMemWal} + * reads convergence out of these numbers and cannot tell a defaulted array from a drained one. * - * @param includeGenerationRows Also count rows per L0 generation. Off by default because each - * count opens an uncached Lance dataset. + * @param includeSstableRows Also count rows per SSTable. Off by default because each count opens + * an uncached Lance dataset. * @throws IllegalStateException if the response is absent or does not decode. */ - public Optional getLsmStats(boolean includeGenerationRows) { + public Optional getMemWalStats(boolean includeSstableRows) { Map body = new LinkedHashMap(); - body.put("include_generation_rows", includeGenerationRows); - JsonNode response = client.post(route("get_lsm_stats"), body); + body.put("include_sstable_rows", includeSstableRows); + JsonNode response = client.post(route("get_memwal_stats"), body); if (response == null) { - throw new IllegalStateException("get_lsm_stats returned an empty response body"); + throw new IllegalStateException("get_memwal_stats returned an empty response body"); } - JsonNode stats = response.get("lsm_stats"); + JsonNode stats = response.get("memwal_stats"); if (stats == null || stats.isNull()) { return Optional.empty(); } - return Optional.of(LsmStats.fromJson(stats)); + return Optional.of(MemWalStats.fromJson(stats)); } - /** Equivalent to {@code getLsmStats(false)}. */ - public Optional getLsmStats() { - return getLsmStats(false); + /** Equivalent to {@code getMemWalStats(false)}. */ + public Optional getMemWalStats() { + return getMemWalStats(false); } /** - * Converge this table's LSM write path into its base table. + * Converge this table's MemWAL write path into its base table. * - *

Seals once, fixes a target watermark from the resulting L0, then triggers compaction and - * polls until that L0 is gone. The target set is fixed at the start, so generations created - * during the checkpoint are ignored — that is what lets it terminate under write load, - * and what makes it best-effort: it converges the fresh tier as of some instant. Idempotent, - * abandonable at any point, safe on a cadence. + *

Freezes once, fixes a target watermark from the resulting SSTables, then triggers compaction + * and polls until those SSTables are gone. The target set is fixed at the start, so sstables + * created during the checkpoint are ignored — that is what lets it terminate under write + * load, and what makes it best-effort: it converges the fresh tier as of some instant. + * Idempotent, abandonable at any point, safe on a cadence. * - *

The loop runs here, not on the server: {@link #compactLsm} dispatches a pass and returns, so - * nothing holds a socket and a client can vanish mid-operation with nothing to reconcile. + *

The loop runs here, not on the server: {@link #compactMemWal} dispatches a pass and returns, + * so nothing holds a socket and a client can vanish mid-operation with nothing to reconcile. * Completion is read from generation numbers in the shard manifest — durable state, unlike a * count in a compact response, which a concurrent write invalidates. * *

No liveness bound — the caller owns the deadline. The compactor pool is shared across * tables, so a checkpoint queued behind unrelated work looks exactly like one that is merging. */ - public void checkpointLsm() { + public void checkpointMemWal() { for (int reissue = 0; reissue <= MAX_REISSUES; reissue++) { - // The seal turns everything written before this call into a generation, so the - // watermark has to be read after it. Idempotent: sealing an empty memtable is a - // no-op, so a re-issue does not churn empty generations. - if (issueVoid(this::flushLsm)) { + // The freeze turns everything written before this call into an SSTable, so the + // watermark has to be read after it. Idempotent: freezing an empty memtable is a + // no-op, so a re-issue does not churn empty sstables. + if (issueVoid(this::flushMemWal)) { backoff(reissue); continue; } - Attempt> stats = issue(() -> getLsmStats(false)); + Attempt> stats = issue(() -> getMemWalStats(false)); if (stats.lostClaim) { backoff(reissue); continue; } if (!stats.value.isPresent()) { - // Not WAL-backed; flushLsm would have errored first but for a race. + // Not WAL-backed; flushMemWal would have errored first but for a race. return; } - Map targets = newestGenerations(stats.value.get()); + Map targets = newestSstableGenerations(stats.value.get()); if (targets.isEmpty()) { return; } @@ -235,18 +235,18 @@ public class LanceDbTableLsm { backoff(reissue); } throw new IllegalStateException( - "checkpointLsm: the owning node kept losing its claim; re-issued from flush the maximum " + "checkpointMemWal: the owning node kept losing its claim; re-issued from flush the maximum " + "number of times"); } /** - * Trigger and poll until no bucket holds a generation at or below its target. + * Trigger and poll until no table shard holds an SSTable generation at or below its target. * * @return true when the drain finished, false when the table needs re-claiming from flush. */ private boolean drainToTargets(Map targets) { while (true) { - Attempt> stats = issue(() -> getLsmStats(false)); + Attempt> stats = issue(() -> getMemWalStats(false)); if (stats.lostClaim) { return false; } @@ -254,21 +254,21 @@ public class LanceDbTableLsm { return true; } - // `compacting` is the bucket's compaction latch, held from dispatch until the pass + // `compacting` is the table shard's compaction latch, held from dispatch until the pass // ends — including while it waits on a pod-wide permit. So it answers one question - // only: do not pile on. Buckets with nothing outstanding are skipped, not counted + // only: do not pile on. Table shards with nothing outstanding are skipped, not counted // as idle. long outstanding = 0; boolean allCompacting = true; - for (BucketStats bucket : stats.value.get().buckets()) { - Long target = targets.get(bucket.shardId()); + for (TableShardStats tableShard : stats.value.get().tableShards()) { + Long target = targets.get(tableShard.shardId()); if (target == null) { continue; } - long remaining = bucket.outstandingGenerations(target); + long remaining = tableShard.outstandingSstables(target); if (remaining > 0) { outstanding += remaining; - allCompacting &= bucket.compacting(); + allCompacting &= tableShard.compacting(); } } if (outstanding == 0) { @@ -277,7 +277,7 @@ public class LanceDbTableLsm { if (!allCompacting) { try { - compactLsm(); + compactMemWal(); } catch (LanceDbRestClient.HttpException e) { if (isLostClaim(e)) { return false; @@ -285,7 +285,7 @@ public class LanceDbTableLsm { if (!isRetryable(e)) { throw e; } - // A 429 here means the server could latch no bucket at all, which the poll + // A 429 here means the server could latch no table shard at all, which the poll // above already handles. Not retried in place: the latch it would contend for // is the one doing the work, so fall through and re-read — POLL_INTERVAL_MS is // the backoff. @@ -295,13 +295,13 @@ public class LanceDbTableLsm { } } - /** The newest generation held by each bucket, skipping buckets holding none. */ - private static Map newestGenerations(LsmStats stats) { + /** The newest SSTable generation held by each table shard, skipping table shards holding none. */ + private static Map newestSstableGenerations(MemWalStats stats) { Map targets = new HashMap(); - for (BucketStats bucket : stats.buckets()) { - OptionalLong newest = bucket.newestGeneration(); + for (TableShardStats tableShard : stats.tableShards()) { + OptionalLong newest = tableShard.newestSstableGeneration(); if (newest.isPresent()) { - targets.put(bucket.shardId(), newest.getAsLong()); + targets.put(tableShard.shardId(), newest.getAsLong()); } } return targets; diff --git a/java/lancedb-core/src/main/java/com/lancedb/LsmStats.java b/java/lancedb-core/src/main/java/com/lancedb/LsmStats.java deleted file mode 100644 index 3496ebc96..000000000 --- a/java/lancedb-core/src/main/java/com/lancedb/LsmStats.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.lancedb; - -import com.fasterxml.jackson.databind.JsonNode; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Live per-bucket LSM state, as returned by {@link LanceDbTableLsm#getLsmStats()}. - * - *

Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are the caller's to - * compute. There is no "LSM is off" shape — that case is an empty {@link java.util.Optional}, - * because a stats object of zeros would read as measurements. - */ -public class LsmStats { - private static final String CONTEXT = "lsm stats"; - - private final List buckets; - - LsmStats(List buckets) { - this.buckets = Collections.unmodifiableList(buckets); - } - - /** One entry per bucket. */ - public List buckets() { - return buckets; - } - - static LsmStats fromJson(JsonNode node) { - JsonFields.requiredObject(node, CONTEXT); - List buckets = new ArrayList(); - for (JsonNode bucket : JsonFields.requiredArray(node, "buckets", CONTEXT)) { - buckets.add(BucketStats.fromJson(bucket)); - } - return new LsmStats(buckets); - } - - @Override - public String toString() { - return "LsmStats{buckets=" + buckets + "}"; - } -} diff --git a/java/lancedb-core/src/main/java/com/lancedb/MemWalStats.java b/java/lancedb-core/src/main/java/com/lancedb/MemWalStats.java new file mode 100644 index 000000000..c84220508 --- /dev/null +++ b/java/lancedb-core/src/main/java/com/lancedb/MemWalStats.java @@ -0,0 +1,56 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb; + +import com.fasterxml.jackson.databind.JsonNode; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Live per-table-shard MemWAL state, as returned by {@link LanceDbTableMemWal#getMemWalStats()}. + * + *

Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are the caller's + * to compute. There is no "LSM is off" shape — that case is an empty {@link java.util.Optional}, + * because a stats object of zeros would read as measurements. + */ +public class MemWalStats { + private static final String CONTEXT = "memwal stats"; + + private final List tableShards; + + MemWalStats(List tableShards) { + this.tableShards = Collections.unmodifiableList(tableShards); + } + + /** One entry per table shard. */ + public List tableShards() { + return tableShards; + } + + static MemWalStats fromJson(JsonNode node) { + JsonFields.requiredObject(node, CONTEXT); + List tableShards = new ArrayList(); + for (JsonNode tableShard : JsonFields.requiredArray(node, "table_shards", CONTEXT)) { + tableShards.add(TableShardStats.fromJson(tableShard)); + } + return new MemWalStats(tableShards); + } + + @Override + public String toString() { + return "MemWalStats{tableShards=" + tableShards + "}"; + } +} diff --git a/java/lancedb-core/src/main/java/com/lancedb/LsmWriteSpec.java b/java/lancedb-core/src/main/java/com/lancedb/MemWalWriteSpec.java similarity index 87% rename from java/lancedb-core/src/main/java/com/lancedb/LsmWriteSpec.java rename to java/lancedb-core/src/main/java/com/lancedb/MemWalWriteSpec.java index c02298c45..95f541214 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/LsmWriteSpec.java +++ b/java/lancedb-core/src/main/java/com/lancedb/MemWalWriteSpec.java @@ -27,16 +27,16 @@ import java.util.Map; * *

Construct via {@link #bucket}, {@link #identity}, or {@link #unsharded}, then optionally chain * {@link #withMaintainedIndexes} and {@link #withWriterConfigDefaults}. Install it with {@link - * LanceDbTableLsm#setLsmWriteSpec} and remove it with {@link LanceDbTableLsm#unsetLsmWriteSpec}. + * LanceDbTableMemWal#enableMemWal} and remove it with {@link LanceDbTableMemWal#disableMemWal}. * *

This is deliberately not {@code org.lance.memwal.InitializeMemWalParams}. That type is Lance's * own, and its maintained-index default is the opposite of this one: it defaults to maintaining * nothing, while a fresh spec here maintains every index. It also cannot express * the null that asks the server to resolve the set. */ -public class LsmWriteSpec { +public class MemWalWriteSpec { - private static final String CONTEXT = "get_lsm_write_spec response"; + private static final String CONTEXT = "get_memwal_spec response"; /** How writes are routed to MemWAL shards. */ public enum Sharding { @@ -73,7 +73,7 @@ public class LsmWriteSpec { private final List maintainedIndexes; private final Map writerConfigDefaults; - private LsmWriteSpec( + private MemWalWriteSpec( Sharding sharding, String column, Integer numBuckets, @@ -95,11 +95,11 @@ public class LsmWriteSpec { * @param column A non-nested column with a supported scalar type. * @param numBuckets The number of buckets, in {@code [1, 1024]}. */ - public static LsmWriteSpec bucket(String column, int numBuckets) { + public static MemWalWriteSpec bucket(String column, int numBuckets) { if (column == null || column.trim().isEmpty()) { throw new IllegalArgumentException("Column cannot be null or empty"); } - return new LsmWriteSpec( + return new MemWalWriteSpec( Sharding.BUCKET, column, numBuckets, null, new HashMap()); } @@ -111,16 +111,17 @@ public class LsmWriteSpec { * with a given primary key must always produce the same {@code column} value, or upserts of that * key can land in different shards and a stale version can win. */ - public static LsmWriteSpec identity(String column) { + public static MemWalWriteSpec identity(String column) { if (column == null || column.trim().isEmpty()) { throw new IllegalArgumentException("Column cannot be null or empty"); } - return new LsmWriteSpec(Sharding.IDENTITY, column, null, null, new HashMap()); + return new MemWalWriteSpec( + Sharding.IDENTITY, column, null, null, new HashMap()); } /** No sharding — every write goes to a single MemWAL shard — maintaining every index. */ - public static LsmWriteSpec unsharded() { - return new LsmWriteSpec(Sharding.UNSHARDED, null, null, null, new HashMap()); + public static MemWalWriteSpec unsharded() { + return new MemWalWriteSpec(Sharding.UNSHARDED, null, null, null, new HashMap()); } /** @@ -131,8 +132,8 @@ public class LsmWriteSpec { * *

Note that {@code null} and the empty list mean opposite things here. */ - public LsmWriteSpec withMaintainedIndexes(List maintainedIndexes) { - return new LsmWriteSpec( + public MemWalWriteSpec withMaintainedIndexes(List maintainedIndexes) { + return new MemWalWriteSpec( sharding, column, numBuckets, @@ -149,11 +150,11 @@ public class LsmWriteSpec { * max_unflushed_memtable_bytes}, and {@code enable_memtable}. Duration knobs carry an {@code _ms} * suffix, such as {@code max_wal_flush_interval_ms}. */ - public LsmWriteSpec withWriterConfigDefaults(Map writerConfigDefaults) { + public MemWalWriteSpec withWriterConfigDefaults(Map writerConfigDefaults) { if (writerConfigDefaults == null) { throw new IllegalArgumentException("writerConfigDefaults cannot be null"); } - return new LsmWriteSpec( + return new MemWalWriteSpec( sharding, column, numBuckets, @@ -191,7 +192,7 @@ public class LsmWriteSpec { return Collections.unmodifiableMap(writerConfigDefaults); } - /** Render this spec as the {@code set_lsm_write_spec} request body. */ + /** Render this spec as the {@code enable_memwal} request body. */ Map toRequestBody() { Map shardingBody = new LinkedHashMap(); shardingBody.put("mode", sharding.wireName()); @@ -211,15 +212,15 @@ public class LsmWriteSpec { } /** - * Rebuild a spec from a {@code get_lsm_write_spec} response body. + * Rebuild a spec from a {@code get_memwal_spec} response body. * *

A null {@code maintained_indexes} selects every index the table has; an empty list selects * none. */ - static LsmWriteSpec fromJson(JsonNode node) { + static MemWalWriteSpec fromJson(JsonNode node) { JsonNode shardingNode = node.get("sharding"); if (shardingNode == null || shardingNode.get("mode") == null) { - throw new IllegalStateException("get_lsm_write_spec response has no sharding mode"); + throw new IllegalStateException("get_memwal_spec response has no sharding mode"); } Sharding sharding = Sharding.fromWireName(shardingNode.get("mode").asText()); @@ -252,12 +253,12 @@ public class LsmWriteSpec { .forEachRemaining(name -> defaults.put(name, defaultsNode.get(name).asText())); } - return new LsmWriteSpec(sharding, column, numBuckets, maintainedIndexes, defaults); + return new MemWalWriteSpec(sharding, column, numBuckets, maintainedIndexes, defaults); } @Override public String toString() { - return "LsmWriteSpec{sharding=" + return "MemWalWriteSpec{sharding=" + sharding + ", column=" + column diff --git a/java/lancedb-core/src/main/java/com/lancedb/MemtableStats.java b/java/lancedb-core/src/main/java/com/lancedb/MemtableStats.java index 777e915aa..b9ef5a89d 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/MemtableStats.java +++ b/java/lancedb-core/src/main/java/com/lancedb/MemtableStats.java @@ -37,7 +37,7 @@ public class MemtableStats { this.indexes = Collections.unmodifiableList(indexes); } - /** The generation this memtable will become once sealed. */ + /** The generation this memtable will become once frozen. */ public long generation() { return generation; } diff --git a/java/lancedb-core/src/main/java/com/lancedb/GenerationStats.java b/java/lancedb-core/src/main/java/com/lancedb/SsTableStats.java similarity index 75% rename from java/lancedb-core/src/main/java/com/lancedb/GenerationStats.java rename to java/lancedb-core/src/main/java/com/lancedb/SsTableStats.java index 12222407c..6e57c1ddf 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/GenerationStats.java +++ b/java/lancedb-core/src/main/java/com/lancedb/SsTableStats.java @@ -17,21 +17,21 @@ import com.fasterxml.jackson.databind.JsonNode; import java.util.OptionalLong; -/** One flushed L0 generation. */ -public class GenerationStats { +/** One SSTable. */ +public class SsTableStats { private static final String CONTEXT = "generation stats"; private final long generation; private final long bytes; private final Long rows; - GenerationStats(long generation, long bytes, Long rows) { + SsTableStats(long generation, long bytes, Long rows) { this.generation = generation; this.bytes = bytes; this.rows = rows; } - /** The generation number. Increases as memtables are sealed into L0. */ + /** The generation number. Increases as memtables are frozen into SSTables. */ public long generation() { return generation; } @@ -42,16 +42,16 @@ public class GenerationStats { } /** - * Rows in this generation, present only when {@code includeGenerationRows} was requested. Off by + * Rows in this generation, present only when {@code includeSstableRows} was requested. Off by * default because each count opens an uncached Lance dataset. */ public OptionalLong rows() { return rows == null ? OptionalLong.empty() : OptionalLong.of(rows); } - static GenerationStats fromJson(JsonNode node) { + static SsTableStats fromJson(JsonNode node) { JsonFields.requiredObject(node, CONTEXT); - return new GenerationStats( + return new SsTableStats( JsonFields.requiredLong(node, "generation", CONTEXT), JsonFields.requiredLong(node, "bytes", CONTEXT), JsonFields.optionalLong(node, "rows", CONTEXT)); @@ -59,6 +59,6 @@ public class GenerationStats { @Override public String toString() { - return "GenerationStats{generation=" + generation + ", bytes=" + bytes + ", rows=" + rows + "}"; + return "SsTableStats{generation=" + generation + ", bytes=" + bytes + ", rows=" + rows + "}"; } } diff --git a/java/lancedb-core/src/main/java/com/lancedb/BucketStats.java b/java/lancedb-core/src/main/java/com/lancedb/TableShardStats.java similarity index 76% rename from java/lancedb-core/src/main/java/com/lancedb/BucketStats.java rename to java/lancedb-core/src/main/java/com/lancedb/TableShardStats.java index 2a8060c5d..dcbc4fa93 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/BucketStats.java +++ b/java/lancedb-core/src/main/java/com/lancedb/TableShardStats.java @@ -22,11 +22,11 @@ import java.util.Optional; import java.util.OptionalLong; /** - * Live state of one bucket. A table is N buckets on one node; flattening to a single number hides - * the one hot bucket that is usually why someone opened this endpoint. + * Live state of one table shard. A table is N table shards on one node; flattening to a single + * number hides the one hot table shard that is usually why someone opened this endpoint. */ -public class BucketStats { - private static final String CONTEXT = "bucket stats"; +public class TableShardStats { + private static final String CONTEXT = "tableShard stats"; private final String shardId; private final String status; @@ -35,11 +35,11 @@ public class BucketStats { private final long currentGeneration; private final long replayAfterWalEntryPosition; private final long walEntryPositionLastSeen; - private final List generations; + private final List sstables; private final boolean compacting; private final List memtables; - BucketStats( + TableShardStats( String shardId, String status, long writerEpoch, @@ -47,7 +47,7 @@ public class BucketStats { long currentGeneration, long replayAfterWalEntryPosition, long walEntryPositionLastSeen, - List generations, + List sstables, boolean compacting, List memtables) { this.shardId = shardId; @@ -57,12 +57,12 @@ public class BucketStats { this.currentGeneration = currentGeneration; this.replayAfterWalEntryPosition = replayAfterWalEntryPosition; this.walEntryPositionLastSeen = walEntryPositionLastSeen; - this.generations = Collections.unmodifiableList(generations); + this.sstables = Collections.unmodifiableList(sstables); this.compacting = compacting; this.memtables = memtables == null ? null : Collections.unmodifiableList(memtables); } - /** The shard this bucket writes. */ + /** The id of this table shard. */ public String shardId() { return shardId; } @@ -100,13 +100,13 @@ public class BucketStats { return walEntryPositionLastSeen; } - /** Flushed L0 generations not yet merged into the base table. */ - public List generations() { - return generations; + /** SSTables not yet merged into the base table. */ + public List sstables() { + return sstables; } /** - * Whether a pass owns this bucket's compaction latch right now. Says a driver is + * Whether a pass owns this table shard's compaction latch right now. Says a driver is * running, not whose, and the latch is held from dispatch — including while the pass * queues for a pod-wide compactor permit. Read it as "do not pile on", never as "mine is * progressing". @@ -115,15 +115,17 @@ public class BucketStats { return compacting; } - /** Oldest first, active last. Empty for a {@code "Sealed"} bucket, whose state is torn down. */ + /** + * Oldest first, active last. Empty for a {@code "Sealed"} table shard, whose state is torn down. + */ public Optional> memtables() { return Optional.ofNullable(memtables); } - /** The newest flushed generation, or empty when L0 is empty. */ - OptionalLong newestGeneration() { + /** The newest SSTable generation, or empty when the tier is empty. */ + OptionalLong newestSstableGeneration() { OptionalLong newest = OptionalLong.empty(); - for (GenerationStats generation : generations) { + for (SsTableStats generation : sstables) { if (!newest.isPresent() || generation.generation() > newest.getAsLong()) { newest = OptionalLong.of(generation.generation()); } @@ -132,15 +134,15 @@ public class BucketStats { } /** - * How many generations at or below {@code target} are still in L0. + * How many SSTables at or below {@code target} are still uncompacted. * *

A count, not a boolean: one pass drains a bounded prefix rather than the whole target set, * so a boolean would read as "no progress" for every pass but the last. Compaction drains * oldest-first, so this decreases monotonically. */ - long outstandingGenerations(long target) { + long outstandingSstables(long target) { long count = 0; - for (GenerationStats generation : generations) { + for (SsTableStats generation : sstables) { if (generation.generation() <= target) { count++; } @@ -148,11 +150,11 @@ public class BucketStats { return count; } - static BucketStats fromJson(JsonNode node) { + static TableShardStats fromJson(JsonNode node) { JsonFields.requiredObject(node, CONTEXT); - List generations = new ArrayList(); - for (JsonNode generation : JsonFields.requiredArray(node, "generations", CONTEXT)) { - generations.add(GenerationStats.fromJson(generation)); + List sstables = new ArrayList(); + for (JsonNode generation : JsonFields.requiredArray(node, "sstables", CONTEXT)) { + sstables.add(SsTableStats.fromJson(generation)); } JsonNode memtablesNode = JsonFields.optionalArray(node, "memtables", CONTEXT); @@ -164,7 +166,7 @@ public class BucketStats { } } - return new BucketStats( + return new TableShardStats( JsonFields.requiredText(node, "shard_id", CONTEXT), JsonFields.requiredText(node, "status", CONTEXT), JsonFields.requiredLong(node, "writer_epoch", CONTEXT), @@ -172,21 +174,21 @@ public class BucketStats { JsonFields.requiredLong(node, "current_generation", CONTEXT), JsonFields.requiredLong(node, "replay_after_wal_entry_position", CONTEXT), JsonFields.requiredLong(node, "wal_entry_position_last_seen", CONTEXT), - generations, + sstables, JsonFields.requiredBoolean(node, "compacting", CONTEXT), memtables); } @Override public String toString() { - return "BucketStats{shardId=" + return "TableShardStats{shardId=" + shardId + ", status=" + status + ", currentGeneration=" + currentGeneration - + ", generations=" - + generations + + ", sstables=" + + sstables + ", compacting=" + compacting + "}"; diff --git a/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java b/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableMemWalTest.java similarity index 63% rename from java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java rename to java/lancedb-core/src/test/java/com/lancedb/LanceDbTableMemWalTest.java index 66ce140b6..349479988 100644 --- a/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java +++ b/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableMemWalTest.java @@ -46,12 +46,12 @@ import static org.junit.jupiter.api.Assertions.*; *

The wire assertions mirror the Rust mocked-endpoint tests in {@code * rust/lancedb/src/remote/table.rs}, which are the contract these routes have to match. */ -public class LanceDbTableLsmTest { +public class LanceDbTableMemWalTest { private static final ObjectMapper MAPPER = new ObjectMapper(); private HttpServer server; private LanceDbRestClient client; - private LanceDbTableLsm lsm; + private LanceDbTableMemWal memWal; private final List requestPaths = Collections.synchronizedList(new ArrayList()); private final List requestBodies = Collections.synchronizedList(new ArrayList()); @@ -101,7 +101,7 @@ public class LanceDbTableLsmTest { .database("test-db") .endpoint("http://127.0.0.1:" + server.getAddress().getPort()) .buildRestClient(); - lsm = new LanceDbTableLsm(client, "my_table"); + memWal = new LanceDbTableMemWal(client, "my_table"); } @AfterEach @@ -115,12 +115,12 @@ public class LanceDbTableLsmTest { // =========================================================================== @Test - public void testSetLsmWriteSpecUnsharded() throws Exception { - enqueue("set_lsm_write_spec", 200, ""); + public void testEnableMemWalUnsharded() throws Exception { + enqueue("enable_memwal", 200, ""); - lsm.setLsmWriteSpec(LsmWriteSpec.unsharded()); + memWal.enableMemWal(MemWalWriteSpec.unsharded()); - assertEquals("/v1/table/my_table/set_lsm_write_spec/", requestPaths.get(0)); + assertEquals("/v1/table/my_table/enable_memwal/", requestPaths.get(0)); JsonNode body = MAPPER.readTree(requestBodies.get(0)); assertEquals("unsharded", body.get("sharding").get("mode").asText()); assertFalse(body.get("sharding").has("column")); @@ -128,11 +128,11 @@ public class LanceDbTableLsmTest { } @Test - public void testSetLsmWriteSpecBucket() throws Exception { - enqueue("set_lsm_write_spec", 200, ""); + public void testEnableMemWalBucket() throws Exception { + enqueue("enable_memwal", 200, ""); - lsm.setLsmWriteSpec( - LsmWriteSpec.bucket("id", 16).withMaintainedIndexes(Arrays.asList("id_idx"))); + memWal.enableMemWal( + MemWalWriteSpec.bucket("id", 16).withMaintainedIndexes(Arrays.asList("id_idx"))); JsonNode body = MAPPER.readTree(requestBodies.get(0)); assertEquals("bucket", body.get("sharding").get("mode").asText()); @@ -143,10 +143,10 @@ public class LanceDbTableLsmTest { } @Test - public void testSetLsmWriteSpecIdentity() throws Exception { - enqueue("set_lsm_write_spec", 200, ""); + public void testEnableMemWalIdentity() throws Exception { + enqueue("enable_memwal", 200, ""); - lsm.setLsmWriteSpec(LsmWriteSpec.identity("tenant")); + memWal.enableMemWal(MemWalWriteSpec.identity("tenant")); JsonNode body = MAPPER.readTree(requestBodies.get(0)); assertEquals("identity", body.get("sharding").get("mode").asText()); @@ -160,55 +160,55 @@ public class LanceDbTableLsmTest { */ @Test public void testMaintainedIndexesNullAndEmptyAreDistinctOnTheWire() throws Exception { - enqueue("set_lsm_write_spec", 200, ""); + enqueue("enable_memwal", 200, ""); - lsm.setLsmWriteSpec(LsmWriteSpec.unsharded()); + memWal.enableMemWal(MemWalWriteSpec.unsharded()); JsonNode fresh = MAPPER.readTree(requestBodies.get(0)); assertTrue(fresh.has("maintained_indexes"), "the key must be present"); assertTrue(fresh.get("maintained_indexes").isNull(), "a fresh spec sends null, not []"); - lsm.setLsmWriteSpec( - LsmWriteSpec.unsharded().withMaintainedIndexes(Collections.emptyList())); + memWal.enableMemWal( + MemWalWriteSpec.unsharded().withMaintainedIndexes(Collections.emptyList())); JsonNode none = MAPPER.readTree(requestBodies.get(1)); assertTrue(none.get("maintained_indexes").isArray()); assertEquals(0, none.get("maintained_indexes").size()); } @Test - public void testSetLsmWriteSpecWriterConfigDefaults() throws Exception { - enqueue("set_lsm_write_spec", 200, ""); + public void testEnableMemWalWriterConfigDefaults() throws Exception { + enqueue("enable_memwal", 200, ""); Map defaults = new HashMap(); defaults.put("max_memtable_rows", "50000"); - lsm.setLsmWriteSpec(LsmWriteSpec.unsharded().withWriterConfigDefaults(defaults)); + memWal.enableMemWal(MemWalWriteSpec.unsharded().withWriterConfigDefaults(defaults)); JsonNode body = MAPPER.readTree(requestBodies.get(0)); assertEquals("50000", body.get("writer_config_defaults").get("max_memtable_rows").asText()); } @Test - public void testUnsetLsmWriteSpec() { - enqueue("unset_lsm_write_spec", 200, ""); + public void testDisableMemWal() { + enqueue("disable_memwal", 200, ""); - lsm.unsetLsmWriteSpec(); + memWal.disableMemWal(); - assertEquals("/v1/table/my_table/unset_lsm_write_spec/", requestPaths.get(0)); + assertEquals("/v1/table/my_table/disable_memwal/", requestPaths.get(0)); assertEquals("", requestBodies.get(0)); } @Test - public void testGetLsmWriteSpec() { + public void testGetMemWalSpec() { enqueue( - "get_lsm_write_spec", + "get_memwal_spec", 200, - "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"bucket\",\"column\":\"id\"," + "{\"memwal_spec\":{\"sharding\":{\"mode\":\"bucket\",\"column\":\"id\"," + "\"num_buckets\":16},\"maintained_indexes\":[\"id_idx\"]," + "\"writer_config_defaults\":{\"durable_write\":\"true\"}}}"); - Optional spec = lsm.getLsmWriteSpec(); + Optional spec = memWal.getMemWalSpec(); assertTrue(spec.isPresent()); - assertEquals(LsmWriteSpec.Sharding.BUCKET, spec.get().sharding()); + assertEquals(MemWalWriteSpec.Sharding.BUCKET, spec.get().sharding()); assertEquals("id", spec.get().column()); assertEquals(Integer.valueOf(16), spec.get().numBuckets()); assertEquals(Arrays.asList("id_idx"), spec.get().maintainedIndexes()); @@ -217,28 +217,28 @@ public class LanceDbTableLsmTest { /** A null selection, every index the table has, reads back as null rather than an empty list. */ @Test - public void testGetLsmWriteSpecRoundTripsEveryIndexSelection() { + public void testGetMemWalSpecRoundTripsEveryIndexSelection() { enqueue( - "get_lsm_write_spec", + "get_memwal_spec", 200, - "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "\"maintained_indexes\":null,\"writer_config_defaults\":{}}}"); - Optional spec = lsm.getLsmWriteSpec(); + Optional spec = memWal.getMemWalSpec(); assertTrue(spec.isPresent()); assertNull(spec.get().maintainedIndexes()); } @Test - public void testGetLsmWriteSpecRoundTripsNoIndexSelection() { + public void testGetMemWalSpecRoundTripsNoIndexSelection() { enqueue( - "get_lsm_write_spec", + "get_memwal_spec", 200, - "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "\"maintained_indexes\":[],\"writer_config_defaults\":{}}}"); - Optional spec = lsm.getLsmWriteSpec(); + Optional spec = memWal.getMemWalSpec(); assertTrue(spec.isPresent()); assertEquals(Collections.emptyList(), spec.get().maintainedIndexes()); @@ -249,32 +249,32 @@ public class LanceDbTableLsmTest { * rejected: reading a string or an object as null would turn a malformed response into a policy. */ @Test - public void testGetLsmWriteSpecRejectsANonArrayMaintainedIndexes() { + public void testGetMemWalSpecRejectsANonArrayMaintainedIndexes() { enqueue( - "get_lsm_write_spec", + "get_memwal_spec", 200, - "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "\"maintained_indexes\":\"id_idx\",\"writer_config_defaults\":{}}}"); - assertThrows(IllegalStateException.class, () -> lsm.getLsmWriteSpec()); + assertThrows(IllegalStateException.class, () -> memWal.getMemWalSpec()); } @Test - public void testGetLsmWriteSpecRejectsNonStringIndexNames() { + public void testGetMemWalSpecRejectsNonStringIndexNames() { enqueue( - "get_lsm_write_spec", + "get_memwal_spec", 200, - "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "\"maintained_indexes\":[7],\"writer_config_defaults\":{}}}"); - assertThrows(IllegalStateException.class, () -> lsm.getLsmWriteSpec()); + assertThrows(IllegalStateException.class, () -> memWal.getMemWalSpec()); } @Test - public void testGetLsmWriteSpecAbsent() { - enqueue("get_lsm_write_spec", 200, "{\"lsm_write_spec\":null}"); + public void testGetMemWalSpecAbsent() { + enqueue("get_memwal_spec", 200, "{\"memwal_spec\":null}"); - assertFalse(lsm.getLsmWriteSpec().isPresent()); + assertFalse(memWal.getMemWalSpec().isPresent()); } // =========================================================================== @@ -282,15 +282,15 @@ public class LanceDbTableLsmTest { // =========================================================================== @Test - public void testGetLsmStats() throws Exception { - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L))); + public void testGetMemWalStats() throws Exception { + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); - Optional got = lsm.getLsmStats(true); + Optional got = memWal.getMemWalStats(true); - assertEquals("/v1/table/my_table/get_lsm_stats/", requestPaths.get(0)); - assertTrue(MAPPER.readTree(requestBodies.get(0)).get("include_generation_rows").asBoolean()); + assertEquals("/v1/table/my_table/get_memwal_stats/", requestPaths.get(0)); + assertTrue(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean()); assertTrue(got.isPresent()); - BucketStats decoded = got.get().buckets().get(0); + TableShardStats decoded = got.get().tableShards().get(0); assertEquals("shard-0", decoded.shardId()); assertEquals("Active", decoded.status()); assertEquals(1, decoded.writerEpoch()); @@ -298,30 +298,30 @@ public class LanceDbTableLsmTest { assertEquals(9, decoded.currentGeneration()); assertFalse(decoded.compacting()); assertEquals(Arrays.asList(7L, 8L), generationNumbers(decoded)); - assertEquals(1024, decoded.generations().get(0).bytes()); - assertFalse(decoded.generations().get(0).rows().isPresent(), "rows absent unless requested"); + assertEquals(1024, decoded.sstables().get(0).bytes()); + assertFalse(decoded.sstables().get(0).rows().isPresent(), "rows absent unless requested"); assertFalse(decoded.memtables().isPresent(), "absent memtables stay absent"); } /** The optional fields decode when the server does send them. */ @Test - public void testGetLsmStatsDecodesOptionalFields() { + public void testGetMemWalStatsDecodesOptionalFields() { enqueue( - "get_lsm_stats", + "get_memwal_stats", 200, - "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "{\"memwal_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9," + "\"replay_after_wal_entry_position\":3,\"wal_entry_position_last_seen\":11," - + "\"generations\":[{\"generation\":7,\"bytes\":1024,\"rows\":42}]," + + "\"sstables\":[{\"generation\":7,\"bytes\":1024,\"rows\":42}]," + "\"compacting\":true,\"memtables\":[{\"generation\":8,\"rows\":5," + "\"bytes\":64,\"batches\":2,\"indexes\":[\"id_idx\"]}]}]}}"); - BucketStats decoded = lsm.getLsmStats(true).get().buckets().get(0); + TableShardStats decoded = memWal.getMemWalStats(true).get().tableShards().get(0); assertEquals(3, decoded.replayAfterWalEntryPosition()); assertEquals(11, decoded.walEntryPositionLastSeen()); assertTrue(decoded.compacting()); - assertEquals(42, decoded.generations().get(0).rows().getAsLong()); + assertEquals(42, decoded.sstables().get(0).rows().getAsLong()); assertTrue(decoded.memtables().isPresent()); MemtableStats memtable = decoded.memtables().get().get(0); assertEquals(8, memtable.generation()); @@ -332,19 +332,19 @@ public class LanceDbTableLsmTest { } @Test - public void testGetLsmStatsAbsentWhenLsmDisabled() { - enqueue("get_lsm_stats", 200, "{\"lsm_stats\":null}"); + public void testGetMemWalStatsAbsentWhenMemWalDisabled() { + enqueue("get_memwal_stats", 200, "{\"memwal_stats\":null}"); - assertFalse(lsm.getLsmStats().isPresent()); + assertFalse(memWal.getMemWalStats().isPresent()); } @Test - public void testGetLsmStatsDefaultsToExcludingGenerationRows() throws Exception { - enqueue("get_lsm_stats", 200, stats()); + public void testGetMemWalStatsDefaultsToExcludingGenerationRows() throws Exception { + enqueue("get_memwal_stats", 200, stats()); - lsm.getLsmStats(); + memWal.getMemWalStats(); - assertFalse(MAPPER.readTree(requestBodies.get(0)).get("include_generation_rows").asBoolean()); + assertFalse(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean()); } // =========================================================================== @@ -353,22 +353,22 @@ public class LanceDbTableLsmTest { @Test public void testFlushAndCompactRoutes() { - enqueue("flush_lsm", 200, ""); - enqueue("compact_lsm", 200, ""); + enqueue("flush_memwal", 200, ""); + enqueue("compact_memwal", 200, ""); - lsm.flushLsm(); - lsm.compactLsm(); + memWal.flushMemWal(); + memWal.compactMemWal(); - assertEquals("/v1/table/my_table/flush_lsm/", requestPaths.get(0)); - assertEquals("/v1/table/my_table/compact_lsm/", requestPaths.get(1)); + assertEquals("/v1/table/my_table/flush_memwal/", requestPaths.get(0)); + assertEquals("/v1/table/my_table/compact_memwal/", requestPaths.get(1)); } @Test public void testHttpErrorCarriesStatus() { - enqueue("flush_lsm", 404, "no such table"); + enqueue("flush_memwal", 404, "no such table"); LanceDbRestClient.HttpException e = - assertThrows(LanceDbRestClient.HttpException.class, () -> lsm.flushLsm()); + assertThrows(LanceDbRestClient.HttpException.class, () -> memWal.flushMemWal()); assertEquals(404, e.statusCode()); } @@ -377,97 +377,98 @@ public class LanceDbTableLsmTest { // =========================================================================== @Test - public void testCheckpointReturnsWhenLsmDisabled() { - enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 200, "{\"lsm_stats\":null}"); + public void testCheckpointReturnsWhenMemWalDisabled() { + enqueue("flush_memwal", 200, ""); + enqueue("get_memwal_stats", 200, "{\"memwal_stats\":null}"); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(0, countCalls("compact_lsm"), "nothing to compact when the LSM path is off"); + assertEquals(0, countCalls("compact_memwal"), "nothing to compact when the LSM path is off"); } @Test public void testCheckpointReturnsWhenNoGenerationsOutstanding() { - enqueue("flush_lsm", 200, ""); - // A bucket with no L0 generations yields no target, so the drain never starts. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false))); + enqueue("flush_memwal", 200, ""); + // A table shard with no SSTables yields no target, so the drain never starts. + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false))); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(0, countCalls("compact_lsm")); + assertEquals(0, countCalls("compact_memwal")); } @Test public void testCheckpointConvergesOnceTargetGenerationsAreGone() { - enqueue("flush_lsm", 200, ""); - // Watermark read: shard-0 holds generations 7 and 8, so target = 8. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L))); + enqueue("flush_memwal", 200, ""); + // Watermark read: shard-0 holds sstables 7 and 8, so target = 8. + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); // First drain poll: both still outstanding, nothing compacting -> dispatch a pass. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L))); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); // Second drain poll: drained past the target -> done. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 9L))); - enqueue("compact_lsm", 200, ""); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 9L))); + enqueue("compact_memwal", 200, ""); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(1, countCalls("compact_lsm"), "one pass dispatched"); - assertEquals(3, countCalls("get_lsm_stats"), "watermark read plus two drain polls"); + assertEquals(1, countCalls("compact_memwal"), "one pass dispatched"); + assertEquals(3, countCalls("get_memwal_stats"), "watermark read plus two drain polls"); } @Test public void testCheckpointDoesNotPileOnWhileEveryTargetBucketIsCompacting() { - enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", true, 4L))); + enqueue("flush_memwal", 200, ""); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", true, 4L))); // Still compacting on the first poll, so no pass is dispatched; then it drains. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", true, 4L))); - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 5L))); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", true, 4L))); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 5L))); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(0, countCalls("compact_lsm"), "a latched bucket is left alone"); + assertEquals(0, countCalls("compact_memwal"), "a latched table shard is left alone"); } @Test public void testCheckpointRetriesFromFlushAfterLostClaim() { // 421 on the watermark read: the node lost its claim, so the whole thing restarts // from flush rather than retrying the read in place. - enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 421, "no claim"); - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false))); + enqueue("flush_memwal", 200, ""); + enqueue("get_memwal_stats", 421, "no claim"); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false))); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(2, countCalls("flush_lsm"), "re-issued from flush"); + assertEquals(2, countCalls("flush_memwal"), "re-issued from flush"); } @Test public void testCheckpointRetriesRetryableStatusInPlace() { - enqueue("flush_lsm", 429, "latch held"); - enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false))); + enqueue("flush_memwal", 429, "latch held"); + enqueue("flush_memwal", 200, ""); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false))); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(2, countCalls("flush_lsm"), "429 retried in place, not re-issued"); + assertEquals(2, countCalls("flush_memwal"), "429 retried in place, not re-issued"); } @Test public void testCheckpointPropagatesTerminalStatus() { - enqueue("flush_lsm", 400, "bad request"); + enqueue("flush_memwal", 400, "bad request"); LanceDbRestClient.HttpException e = - assertThrows(LanceDbRestClient.HttpException.class, () -> lsm.checkpointLsm()); + assertThrows(LanceDbRestClient.HttpException.class, () -> memWal.checkpointMemWal()); assertEquals(400, e.statusCode()); - assertEquals(1, countCalls("flush_lsm"), "a terminal status is not retried"); + assertEquals(1, countCalls("flush_memwal"), "a terminal status is not retried"); } @Test public void testCheckpointGivesUpAfterRepeatedLostClaims() { - enqueue("flush_lsm", 421, "no claim"); + enqueue("flush_memwal", 421, "no claim"); - IllegalStateException e = assertThrows(IllegalStateException.class, () -> lsm.checkpointLsm()); + IllegalStateException e = + assertThrows(IllegalStateException.class, () -> memWal.checkpointMemWal()); assertTrue(e.getMessage().contains("kept losing its claim"), e.getMessage()); - assertEquals(4, countCalls("flush_lsm"), "the initial attempt plus MAX_REISSUES"); + assertEquals(4, countCalls("flush_memwal"), "the initial attempt plus MAX_REISSUES"); } // =========================================================================== @@ -476,50 +477,51 @@ public class LanceDbTableLsmTest { /** * A stats payload that does not decode must fail closed. Every one of these bodies used to be - * read as "no buckets", which is indistinguishable from a drained table, so {@code checkpointLsm} - * reported convergence for a checkpoint that never ran. + * read as "no table shards", which is indistinguishable from a drained table, so {@code + * checkpointMemWal} reported convergence for a checkpoint that never ran. */ @Test public void testCheckpointRejectsMalformedStats() { Map malformed = new LinkedHashMap(); malformed.put("no response body at all", ""); - malformed.put("stats object with no buckets", "{\"lsm_stats\":{}}"); - malformed.put("bucket missing its required fields", "{\"lsm_stats\":{\"buckets\":[{}]}}"); + malformed.put("stats object with no table shards", "{\"memwal_stats\":{}}"); malformed.put( - "bucket missing generations", - "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "table shard missing its required fields", "{\"memwal_stats\":{\"table_shards\":[{}]}}"); + malformed.put( + "table shard missing sstables", + "{\"memwal_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9," + "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0," + "\"compacting\":false}]}}"); malformed.put( "generation with a non-numeric generation number", - "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "{\"memwal_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9," + "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0," - + "\"generations\":[{\"generation\":\"7\",\"bytes\":1024}]," + + "\"sstables\":[{\"generation\":\"7\",\"bytes\":1024}]," + "\"compacting\":false}]}}"); for (Map.Entry each : malformed.entrySet()) { setUpFresh(); - enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 200, each.getValue()); + enqueue("flush_memwal", 200, ""); + enqueue("get_memwal_stats", 200, each.getValue()); assertThrows( IllegalStateException.class, - () -> lsm.checkpointLsm(), + () -> memWal.checkpointMemWal(), each.getKey() + " must not report convergence"); } } - /** The one shape that legitimately means "this table has no LSM write path". */ + /** The one shape that legitimately means "this table has no MemWAL write path". */ @Test public void testCheckpointTreatsNullStatsAsNotWalBacked() { - enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 200, "{\"lsm_stats\":null}"); + enqueue("flush_memwal", 200, ""); + enqueue("get_memwal_stats", 200, "{\"memwal_stats\":null}"); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(1, countCalls("get_lsm_stats")); + assertEquals(1, countCalls("get_memwal_stats")); } // =========================================================================== @@ -529,40 +531,41 @@ public class LanceDbTableLsmTest { /** * The transport must not retry on the checkpoint loop's behalf. Apache HttpClient's default * strategy retries exactly 429 and 503 — the two statuses {@code isRetryable} owns — which - * doubled every budget here and also retried {@code compact_lsm} in place, where the loop is + * doubled every budget here and also retried {@code compact_memwal} in place, where the loop is * built to fall through to a fresh stats poll instead. */ @Test public void testCheckpointRetryBudgetIsNotDoubledByTheTransport() { - enqueue("flush_lsm", 429, "latch held"); + enqueue("flush_memwal", 429, "latch held"); LanceDbRestClient.HttpException e = - assertThrows(LanceDbRestClient.HttpException.class, () -> lsm.checkpointLsm()); + assertThrows(LanceDbRestClient.HttpException.class, () -> memWal.checkpointMemWal()); assertEquals(429, e.statusCode(), "the exhausted budget propagates the last error as itself"); - assertEquals(9, countCalls("flush_lsm"), "the initial request plus MAX_RETRIES, and no more"); + assertEquals( + 9, countCalls("flush_memwal"), "the initial request plus MAX_RETRIES, and no more"); } // =========================================================================== // harness // =========================================================================== - private static List generationNumbers(BucketStats bucket) { + private static List generationNumbers(TableShardStats tableShard) { List numbers = new ArrayList(); - for (GenerationStats generation : bucket.generations()) { + for (SsTableStats generation : tableShard.sstables()) { numbers.add(generation.generation()); } return numbers; } - /** Build an {@code lsm_stats} response body from bucket fragments. */ - private static String stats(String... buckets) { - return "{\"lsm_stats\":{\"buckets\":[" + String.join(",", buckets) + "]}}"; + /** Build a {@code memwal_stats} response body from table shard fragments. */ + private static String stats(String... tableShards) { + return "{\"memwal_stats\":{\"table_shards\":[" + String.join(",", tableShards) + "]}}"; } - private static String bucket(String shardId, boolean compacting, Long... generations) { + private static String tableShard(String shardId, boolean compacting, Long... sstables) { StringBuilder gens = new StringBuilder(); - for (Long generation : generations) { + for (Long generation : sstables) { if (gens.length() > 0) { gens.append(","); } @@ -572,7 +575,7 @@ public class LanceDbTableLsmTest { + shardId + "\",\"status\":\"Active\",\"writer_epoch\":1,\"manifest_version\":2," + "\"current_generation\":9,\"replay_after_wal_entry_position\":0," - + "\"wal_entry_position_last_seen\":0,\"generations\":[" + + "\"wal_entry_position_last_seen\":0,\"sstables\":[" + gens + "],\"compacting\":" + compacting @@ -597,7 +600,7 @@ public class LanceDbTableLsmTest { return requestPaths.stream().filter(path -> operationOf(path).equals(operation)).count(); } - /** {@code /v1/table/my_table/flush_lsm/} -> {@code flush_lsm}. */ + /** {@code /v1/table/my_table/flush_memwal/} -> {@code flush_memwal}. */ private static String operationOf(String path) { String[] segments = path.split("/"); return segments.length == 0 ? "" : segments[segments.length - 1]; diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 6c770494b..f53f859b8 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -640,11 +640,11 @@ describe.each(arrowVersions)( ); }); - it("should expose useLsm on takeRowIds as the base-only escape hatch", async () => { + it("should expose useMemWal on takeRowIds as the base-only escape hatch", async () => { await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]); - // useLsm(false) is reachable on TakeQuery (the escape hatch for MemWAL tables, + // useMemWal(false) is reachable on TakeQuery (the escape hatch for MemWAL tables, // where take-by-row-id auto-routes to the LSM scanner and is rejected). - const res = await table.takeRowIds([0, 2]).useLsm(false).toArray(); + const res = await table.takeRowIds([0, 2]).useMemWal(false).toArray(); expect(res.map((r) => r.id)).toEqual([1, 3]); }); @@ -4628,7 +4628,7 @@ describe("setUnenforcedPrimaryKey", () => { }); }); -describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { +describe("enableMemWal / disableMemWal", () => { let tmpDir: tmp.DirResult; beforeEach(() => { @@ -4648,16 +4648,16 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { const table = await makeTable(conn); await table.setUnenforcedPrimaryKey("id"); - await table.setLsmWriteSpec({ + await table.enableMemWal({ specType: "bucket", column: "id", numBuckets: 4, }); - await table.unsetLsmWriteSpec(); + await table.disableMemWal(); // A second unset errors — there is no spec left to remove. - await expect(table.unsetLsmWriteSpec()).rejects.toThrow(); + await expect(table.disableMemWal()).rejects.toThrow(); // A fresh spec can be installed after unset. - await table.setLsmWriteSpec({ + await table.enableMemWal({ specType: "bucket", column: "id", numBuckets: 8, @@ -4669,8 +4669,8 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { const table = await makeTable(conn); await table.setUnenforcedPrimaryKey("id"); - await table.setLsmWriteSpec({ specType: "unsharded" }); - await table.unsetLsmWriteSpec(); + await table.enableMemWal({ specType: "unsharded" }); + await table.disableMemWal(); }); it("installs an identity spec", async () => { @@ -4678,8 +4678,8 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { const table = await makeTable(conn); await table.setUnenforcedPrimaryKey("id"); - await table.setLsmWriteSpec({ specType: "identity", column: "id" }); - await table.unsetLsmWriteSpec(); + await table.enableMemWal({ specType: "identity", column: "id" }); + await table.disableMemWal(); }); it("rejects an invalid spec", async () => { @@ -4689,7 +4689,7 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { await table.setUnenforcedPrimaryKey("id"); // num_buckets out of range. await expect( - table.setLsmWriteSpec({ + table.enableMemWal({ specType: "bucket", column: "id", numBuckets: 0, @@ -4697,7 +4697,7 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { ).rejects.toThrow(); // Column mismatch. await expect( - table.setLsmWriteSpec({ + table.enableMemWal({ specType: "bucket", column: "missing", numBuckets: 4, @@ -4705,13 +4705,13 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { ).rejects.toThrow(); }); - it("reads back the installed spec via getLsmWriteSpec", async () => { + it("reads back the installed spec via getMemWalSpec", async () => { const conn = await connect(tmpDir.name); const table = await makeTable(conn); await table.setUnenforcedPrimaryKey("id"); // Nothing installed yet. - expect(await table.getLsmWriteSpec()).toBeUndefined(); + expect(await table.getMemWalSpec()).toBeUndefined(); // A real scalar index is needed to name it as a maintained index. await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]); @@ -4722,14 +4722,14 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { // defaults. Lance writer-config keys are canonically snake_case. // biome-ignore lint/style/useNamingConvention: Lance writer-config keys are snake_case const writerConfigDefaults = { durable_write: "false" }; - await table.setLsmWriteSpec({ + await table.enableMemWal({ specType: "bucket", column: "id", numBuckets: 4, maintainedIndexes: [indexName], writerConfigDefaults, }); - const spec = await table.getLsmWriteSpec(); + const spec = await table.getMemWalSpec(); expect(spec).toBeDefined(); expect(spec?.specType).toBe("bucket"); expect(spec?.column).toBe("id"); @@ -4738,19 +4738,19 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { expect(spec?.writerConfigDefaults).toEqual(writerConfigDefaults); // After unset, undefined again. - await table.unsetLsmWriteSpec(); - expect(await table.getLsmWriteSpec()).toBeUndefined(); + await table.disableMemWal(); + expect(await table.getMemWalSpec()).toBeUndefined(); // Identity round-trips (column recovered from the schema). - await table.setLsmWriteSpec({ specType: "identity", column: "id" }); - const identity = await table.getLsmWriteSpec(); + await table.enableMemWal({ specType: "identity", column: "id" }); + const identity = await table.getMemWalSpec(); expect(identity?.specType).toBe("identity"); expect(identity?.column).toBe("id"); - await table.unsetLsmWriteSpec(); + await table.disableMemWal(); // Unsharded round-trips (no routing column). - await table.setLsmWriteSpec({ specType: "unsharded" }); - const unsharded = await table.getLsmWriteSpec(); + await table.enableMemWal({ specType: "unsharded" }); + const unsharded = await table.getMemWalSpec(); expect(unsharded?.specType).toBe("unsharded"); expect(unsharded?.column).toBeFalsy(); }); @@ -4779,7 +4779,7 @@ describe("LSM merge insert", () => { ]); await table.setUnenforcedPrimaryKey("id"); // numBuckets = 1: every row routes to the single bucket. - await table.setLsmWriteSpec({ + await table.enableMemWal({ specType: "bucket", column: "id", numBuckets: 1, @@ -4804,17 +4804,17 @@ describe("LSM merge insert", () => { expect(res.version).toBe(0); expect(res.numInsertedRows).toBe(0); - await table.closeLsmWriters(); + await table.closeMemWalWriters(); }); - it("falls back to the standard path with useLsm(false)", async () => { + it("falls back to the standard path with useMemWal(false)", async () => { const conn = await connect(tmpDir.name); const table = await bucketTable(conn); const res = await table .mergeInsert("id") .whenNotMatchedInsertAll() - .useLsm(false) + .useMemWal(false) .execute([ { id: "b", value: 9 }, { id: "e", value: 5 }, @@ -4837,7 +4837,7 @@ describe("LSM merge insert", () => { expect(res.numRows).toBe(1); }); - it("rejects a non-upsert merge under an LSM spec", async () => { + it("rejects a non-upsert merge under a MemWAL spec", async () => { const conn = await connect(tmpDir.name); const table = await bucketTable(conn); @@ -4863,22 +4863,24 @@ describe("LSM merge insert", () => { const lsm = await table.query().toArray(); expect(lsm.map((r) => r.id).sort()).toEqual(["a", "b", "c"]); - // useLsm(false) bypasses the MemWAL and reads the base table only. - const baseOnly = await table.query().useLsm(false).toArray(); + // useMemWal(false) bypasses the MemWAL and reads the base table only. + const baseOnly = await table.query().useMemWal(false).toArray(); expect(baseOnly.map((r) => r.id).sort()).toEqual(["a", "b"]); }); - it("reads the base table when no LSM spec is installed", async () => { + it("reads the base table when no MemWAL spec is installed", async () => { const conn = await connect(tmpDir.name); const table = await conn.createEmptyTable( "plain", new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]), ); - // No spec: default read and useLsm(false) both succeed against the base table. + // No spec: default read and useMemWal(false) both succeed against the base table. await expect(table.query().toArray()).resolves.toBeDefined(); - await expect(table.query().useLsm(false).toArray()).resolves.toBeDefined(); - // useLsm(true) demands MemWAL routing; without a spec it errors. - await expect(table.query().useLsm(true).toArray()).rejects.toThrow(); + await expect( + table.query().useMemWal(false).toArray(), + ).resolves.toBeDefined(); + // useMemWal(true) demands MemWAL routing; without a spec it errors. + await expect(table.query().useMemWal(true).toArray()).rejects.toThrow(); }); }); @@ -4896,7 +4898,7 @@ describe("LSM convergence and stats", () => { new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]), ); await table.setUnenforcedPrimaryKey("id"); - await table.setLsmWriteSpec({ specType: "unsharded" }); + await table.enableMemWal({ specType: "unsharded" }); return table; } @@ -4904,34 +4906,34 @@ describe("LSM convergence and stats", () => { // rejects them rather than answering. What is asserted here is that the // bindings reach the core at all; the behavior against a real endpoint is // covered by the mocked endpoint tests in rust/lancedb/src/remote/table.rs. - it("rejects flushLsm on a local table", async () => { + it("rejects flushMemWal on a local table", async () => { const conn = await connect(tmpDir.name); const table = await lsmTable(conn); - await expect(table.flushLsm()).rejects.toThrow(/not supported/i); + await expect(table.flushMemWal()).rejects.toThrow(/not supported/i); }); - it("rejects compactLsm on a local table", async () => { + it("rejects compactMemWal on a local table", async () => { const conn = await connect(tmpDir.name); const table = await lsmTable(conn); - await expect(table.compactLsm()).rejects.toThrow(/not supported/i); + await expect(table.compactMemWal()).rejects.toThrow(/not supported/i); }); - it("rejects getLsmStats on a local table", async () => { + it("rejects getMemWalStats on a local table", async () => { const conn = await connect(tmpDir.name); const table = await lsmTable(conn); - await expect(table.getLsmStats()).rejects.toThrow(/not supported/i); - await expect(table.getLsmStats(true)).rejects.toThrow(/not supported/i); + await expect(table.getMemWalStats()).rejects.toThrow(/not supported/i); + await expect(table.getMemWalStats(true)).rejects.toThrow(/not supported/i); }); - it("rejects checkpointLsm on a local table", async () => { + it("rejects checkpointMemWal on a local table", async () => { const conn = await connect(tmpDir.name); const table = await lsmTable(conn); - // checkpointLsm seals first, so it surfaces flushLsm's rejection. - await expect(table.checkpointLsm()).rejects.toThrow(/not supported/i); + // checkpointMemWal freezes first, so it surfaces flushMemWal's rejection. + await expect(table.checkpointMemWal()).rejects.toThrow(/not supported/i); }); }); diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index be46e23dc..3254a0742 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -167,10 +167,10 @@ export { WriteProgress, FtsToken, TokenizeTableOptions, - LsmWriteSpec, - LsmStats, - BucketStats, - GenerationStats, + MemWalWriteSpec, + MemWalStats, + TableShardStats, + SsTableStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, diff --git a/nodejs/lancedb/merge.ts b/nodejs/lancedb/merge.ts index 5f1d8704a..0c6cdaa29 100644 --- a/nodejs/lancedb/merge.ts +++ b/nodejs/lancedb/merge.ts @@ -90,25 +90,25 @@ export class MergeInsertBuilder { /** * Control MemWAL routing for this merge. * - * By default (unset), a `mergeInsert` on a table with an LSM write spec is + * By default (unset), a `mergeInsert` on a table with a MemWAL write spec is * routed through Lance's MemWAL shard writer, and a table without one uses the * standard path. * * @param enable - `true` forces MemWAL routing and errors if the table has no - * LSM write spec. `false` forces the standard write path even when a spec is set. + * MemWAL write spec. `false` forces the standard write path even when a spec is set. */ - useLsm(enable: boolean): MergeInsertBuilder { - return new MergeInsertBuilder(this.#native.useLsm(enable), this.#schema); + useMemWal(enable: boolean): MergeInsertBuilder { + return new MergeInsertBuilder(this.#native.useMemWal(enable), this.#schema); } /** * Controls how an LSM merge checks that its input targets a single shard. * - * When a table has an LSM write spec, every row in a `mergeInsert` call must + * When a table has a MemWAL write spec, every row in a `mergeInsert` call must * route to the same shard. When `true` (the default), every row is inspected * to verify this. When `false`, only the first row is inspected and the * shard it routes to is used for the whole input — a faster path for callers * that have already pre-sharded their input. Has no effect on tables without - * an LSM write spec. + * a MemWAL write spec. * * @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`. */ diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index cce505c97..3583e5992 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -464,10 +464,10 @@ export class StandardQueryBase< * Control MemWAL read routing for this query. * * By default (unset), when the table carries a MemWAL write spec (see - * {@link Table#setLsmWriteSpec}), reads are routed through the LSM scanner so + * {@link Table#enableMemWal}), reads are routed through the LSM scanner so * they also return data written via the `mergeInsert` LSM path that has not yet * been compacted into the base table (the active/frozen in-memory memtables and - * the flushed generations), deduplicated by primary key; a table without a spec + * the SSTables), deduplicated by primary key; a table without a spec * reads the base table. * * @param enable - `true` forces the LSM scanner and errors if the table has no @@ -476,11 +476,11 @@ export class StandardQueryBase< * * Note: the LSM scanner does not support every query shape (e.g. reranking, * hybrid search, `orderBy`). On a MemWAL table those shapes error unless - * `useLsm(false)` is set, because a base-only read would silently exclude + * `useMemWal(false)` is set, because a base-only read would silently exclude * un-compacted MemWAL data. */ - useLsm(enable: boolean): this { - this.doCall((inner: NativeQueryType) => inner.useLsm(enable)); + useMemWal(enable: boolean): this { + this.doCall((inner: NativeQueryType) => inner.useMemWal(enable)); return this; } } @@ -871,8 +871,8 @@ export class TakeQuery extends QueryBase { * * @param enable - `false` reads the base table only. */ - useLsm(enable: boolean): this { - this.doCall((inner: NativeTakeQuery) => inner.useLsm(enable)); + useMemWal(enable: boolean): this { + this.doCall((inner: NativeTakeQuery) => inner.useMemWal(enable)); return this; } } diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 630d06f29..4d6500a5f 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -35,7 +35,7 @@ import { FunctionErrorsOptions, IndexConfig, IndexStatistics, - LsmStats, + MemWalStats, Branches as NativeBranches, OptimizeStats, RefreshColumnResult, @@ -59,9 +59,9 @@ import { sanitizeType } from "./sanitize"; import { IntoSql, toSQL } from "./util"; export { IndexConfig } from "./native"; export { - BucketStats, - GenerationStats, - LsmStats, + TableShardStats, + SsTableStats, + MemWalStats, MemtableStats, } from "./native"; @@ -217,7 +217,7 @@ export type TokenizeTableOptions = * `column` value, or upserts of that key can land in different shards and a * stale version can win). */ -export interface LsmWriteSpec { +export interface MemWalWriteSpec { /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */ specType: "bucket" | "identity" | "unsharded"; /** Bucket and identity variants: the sharding column. */ @@ -758,10 +758,10 @@ export abstract class Table { */ abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise; /** - * Install an {@link LsmWriteSpec} on this table, selecting Lance's MemWAL + * Install an {@link MemWalWriteSpec} on this table, selecting Lance's MemWAL * LSM-style write path for future `mergeInsert` calls. * - * `LsmWriteSpec` chooses one of three sharding strategies via `specType`: + * `MemWalWriteSpec` chooses one of three sharding strategies via `specType`: * * - `"bucket"` — hash-bucket writes by the single-column unenforced primary * key (`column` and `numBuckets` required). @@ -776,12 +776,12 @@ export abstract class Table { * including ones created later, and skips a kind the MemWAL cannot maintain. * Naming them pins an exact set, and a still-building index is rejected * rather than quietly omitted. - * @param {LsmWriteSpec} spec The sharding spec to install. + * @param {MemWalWriteSpec} spec The sharding spec to install. * @returns {Promise} * @example * ```ts * await table.setUnenforcedPrimaryKey("id"); - * await table.setLsmWriteSpec({ + * await table.enableMemWal({ * specType: "bucket", * column: "id", * numBuckets: 16, @@ -789,59 +789,59 @@ export abstract class Table { * }); * ``` */ - abstract setLsmWriteSpec(spec: LsmWriteSpec): Promise; + abstract enableMemWal(spec: MemWalWriteSpec): Promise; /** - * Remove the {@link LsmWriteSpec} from this table, reverting to the standard + * Remove the {@link MemWalWriteSpec} from this table, reverting to the standard * `mergeInsert` write path. * * Errors if no spec is currently set. * @returns {Promise} */ - abstract unsetLsmWriteSpec(): Promise; + abstract disableMemWal(): Promise; /** - * Read the {@link LsmWriteSpec} currently installed on this table. + * Read the {@link MemWalWriteSpec} currently installed on this table. * - * Resolves to `undefined` when the MemWAL LSM write path is not enabled (no - * spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}). + * Resolves to `undefined` when the MemWAL write path is not enabled (no + * spec has been set, or it was removed with {@link Table#disableMemWal}). * The spec is the one installed, including its maintained-index selection: * an absent `maintainedIndexes` for every index the table has, an empty * array for none. - * @returns {Promise} + * @returns {Promise} */ - abstract getLsmWriteSpec(): Promise; + abstract getMemWalSpec(): Promise; /** * Drain and close any cached MemWAL shard writers held for this table. * - * When an {@link LsmWriteSpec} is installed, `mergeInsert` opens MemWAL + * When an {@link MemWalWriteSpec} is installed, `mergeInsert` opens MemWAL * shard writers and caches them for reuse across calls. This closes them, * flushing pending data; writers reopen lazily on the next `mergeInsert`. * It is a no-op when no writers are cached. * @returns {Promise} */ - abstract closeLsmWriters(): Promise; + abstract closeMemWalWriters(): Promise; /** - * Seal every bucket's active memtable into a new L0 generation. + * Freeze every table shard's active memtable into a new SSTable. * - * Returns once the seal is committed. Sealing an empty memtable is a no-op, + * Returns once the freeze is committed. Freezing an empty memtable is a no-op, * so this is safe to call repeatedly. * @returns {Promise} */ - abstract flushLsm(): Promise; + abstract flushMemWal(): Promise; /** - * Trigger a background L0 → base compaction pass per bucket. + * Trigger a background SSTable compaction pass per table shard. * * Returns once the passes are *dispatched*, not once they finish — watch - * {@link Table#getLsmStats} for progress, or use - * {@link Table#checkpointLsm} to wait for convergence. + * {@link Table#getMemWalStats} for progress, or use + * {@link Table#checkpointMemWal} to wait for convergence. * @returns {Promise} */ - abstract compactLsm(): Promise; + abstract compactMemWal(): Promise; /** - * Converge this table's LSM write path into its base table. + * Converge this table's MemWAL write path into its base table. * - * Seals once, then triggers compaction and polls until the L0 that existed + * Freezes once, then triggers compaction and polls until the SSTables that existed * at the start is gone. The target set is fixed at the start, so - * generations created *during* the checkpoint are ignored — that is what + * SSTables created *during* the checkpoint are ignored — that is what * lets it terminate under write load, and what makes it best-effort: it * converges the fresh tier as of some instant. Idempotent, abandonable at * any point, and safe to run on a cadence. @@ -852,26 +852,26 @@ export abstract class Table { * @returns {Promise} * @example * ```ts - * const before = await table.getLsmStats(); - * await table.checkpointLsm(); - * const after = await table.getLsmStats(); + * const before = await table.getMemWalStats(); + * await table.checkpointMemWal(); + * const after = await table.getMemWalStats(); * ``` */ - abstract checkpointLsm(): Promise; + abstract checkpointMemWal(): Promise; /** - * Read live per-bucket LSM state. + * Read live per-table-shard MemWAL state. * - * Answers "how far behind is my fresh tier", "which bucket is hot", and + * Answers "how far behind is my fresh tier", "which table shard is hot", and * "why is my fresh-tier vector search brute-force". Mutates no table state. * - * Resolves to `undefined` only when the LSM write path is not enabled. - * @param {boolean} includeGenerationRows Also count rows per L0 generation. + * Resolves to `undefined` only when the MemWAL write path is not enabled. + * @param {boolean} includeSstableRows Also count rows per SSTable. * Off by default because each count opens an uncached Lance dataset. - * @returns {Promise} + * @returns {Promise} */ - abstract getLsmStats( - includeGenerationRows?: boolean, - ): Promise; + abstract getMemWalStats( + includeSstableRows?: boolean, + ): Promise; /** Retrieve the version of the table */ abstract version(): Promise; @@ -1474,43 +1474,43 @@ export class LocalTable extends Table { return await this.inner.setUnenforcedPrimaryKey(cols); } - async setLsmWriteSpec(spec: LsmWriteSpec): Promise { - return await this.inner.setLsmWriteSpec(spec); + async enableMemWal(spec: MemWalWriteSpec): Promise { + return await this.inner.enableMemWal(spec); } - async unsetLsmWriteSpec(): Promise { - return await this.inner.unsetLsmWriteSpec(); + async disableMemWal(): Promise { + return await this.inner.disableMemWal(); } - async getLsmWriteSpec(): Promise { + async getMemWalSpec(): Promise { // The native binding types `specType` as a plain `string`; narrow it back // to the public union. The Rust `From` impl only ever emits one of the // three valid values, so the cast is safe. - return ((await this.inner.getLsmWriteSpec()) ?? undefined) as - | LsmWriteSpec + return ((await this.inner.getMemWalSpec()) ?? undefined) as + | MemWalWriteSpec | undefined; } - async closeLsmWriters(): Promise { - return await this.inner.closeLsmWriters(); + async closeMemWalWriters(): Promise { + return await this.inner.closeMemWalWriters(); } - async flushLsm(): Promise { - return await this.inner.flushLsm(); + async flushMemWal(): Promise { + return await this.inner.flushMemWal(); } - async compactLsm(): Promise { - return await this.inner.compactLsm(); + async compactMemWal(): Promise { + return await this.inner.compactMemWal(); } - async checkpointLsm(): Promise { - return await this.inner.checkpointLsm(); + async checkpointMemWal(): Promise { + return await this.inner.checkpointMemWal(); } - async getLsmStats( - includeGenerationRows: boolean = false, - ): Promise { - return (await this.inner.getLsmStats(includeGenerationRows)) ?? undefined; + async getMemWalStats( + includeSstableRows: boolean = false, + ): Promise { + return (await this.inner.getMemWalStats(includeSstableRows)) ?? undefined; } async version(): Promise { diff --git a/nodejs/src/merge.rs b/nodejs/src/merge.rs index ac8649419..3d17a037e 100644 --- a/nodejs/src/merge.rs +++ b/nodejs/src/merge.rs @@ -50,10 +50,10 @@ impl NativeMergeInsertBuilder { this } - #[napi] - pub fn use_lsm(&self, enable: bool) -> Self { + #[napi(js_name = "useMemWal")] + pub fn use_memwal(&self, enable: bool) -> Self { let mut this = self.clone(); - this.inner.use_lsm(enable); + this.inner.use_memwal(enable); this } diff --git a/nodejs/src/query.rs b/nodejs/src/query.rs index 480426411..68db6f810 100644 --- a/nodejs/src/query.rs +++ b/nodejs/src/query.rs @@ -168,9 +168,9 @@ impl Query { self.inner = self.inner.clone().with_row_id(); } - #[napi] - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + #[napi(js_name = "useMemWal")] + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } #[napi] @@ -394,9 +394,9 @@ impl VectorQuery { self.inner = self.inner.clone().with_row_id(); } - #[napi] - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + #[napi(js_name = "useMemWal")] + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } #[napi] @@ -504,9 +504,9 @@ impl TakeQuery { self.inner = self.inner.clone().with_row_id(); } - #[napi] - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + #[napi(js_name = "useMemWal")] + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } #[napi(catch_unwind)] diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 32cdcce3c..5e29e23b6 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -580,64 +580,60 @@ impl Table { .default_error() } - #[napi(catch_unwind)] - pub async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> napi::Result<()> { - let native_spec = lancedb::table::LsmWriteSpec::try_from(spec)?; + #[napi(catch_unwind, js_name = "enableMemWal")] + pub async fn enable_memwal(&self, spec: MemWalWriteSpec) -> napi::Result<()> { + let native_spec = lancedb::table::MemWalWriteSpec::try_from(spec)?; self.inner_ref()? - .set_lsm_write_spec(native_spec) + .enable_memwal(native_spec) .await .default_error() } - #[napi(catch_unwind)] - pub async fn unset_lsm_write_spec(&self) -> napi::Result<()> { + #[napi(catch_unwind, js_name = "disableMemWal")] + pub async fn disable_memwal(&self) -> napi::Result<()> { + self.inner_ref()?.disable_memwal().await.default_error() + } + + #[napi(catch_unwind, js_name = "getMemWalSpec")] + pub async fn get_memwal_spec(&self) -> napi::Result> { + let spec = self.inner_ref()?.get_memwal_spec().await.default_error()?; + Ok(spec.map(MemWalWriteSpec::from)) + } + + #[napi(catch_unwind, js_name = "closeMemWalWriters")] + pub async fn close_memwal_writers(&self) -> napi::Result<()> { self.inner_ref()? - .unset_lsm_write_spec() + .close_memwal_writers() .await .default_error() } - #[napi(catch_unwind)] - pub async fn get_lsm_write_spec(&self) -> napi::Result> { - let spec = self - .inner_ref()? - .get_lsm_write_spec() - .await - .default_error()?; - Ok(spec.map(LsmWriteSpec::from)) + #[napi(catch_unwind, js_name = "flushMemWal")] + pub async fn flush_memwal(&self) -> napi::Result<()> { + self.inner_ref()?.flush_memwal().await.default_error() } - #[napi(catch_unwind)] - pub async fn close_lsm_writers(&self) -> napi::Result<()> { - self.inner_ref()?.close_lsm_writers().await.default_error() + #[napi(catch_unwind, js_name = "compactMemWal")] + pub async fn compact_memwal(&self) -> napi::Result<()> { + self.inner_ref()?.compact_memwal().await.default_error() } - #[napi(catch_unwind)] - pub async fn flush_lsm(&self) -> napi::Result<()> { - self.inner_ref()?.flush_lsm().await.default_error() + #[napi(catch_unwind, js_name = "checkpointMemWal")] + pub async fn checkpoint_memwal(&self) -> napi::Result<()> { + self.inner_ref()?.checkpoint_memwal().await.default_error() } - #[napi(catch_unwind)] - pub async fn compact_lsm(&self) -> napi::Result<()> { - self.inner_ref()?.compact_lsm().await.default_error() - } - - #[napi(catch_unwind)] - pub async fn checkpoint_lsm(&self) -> napi::Result<()> { - self.inner_ref()?.checkpoint_lsm().await.default_error() - } - - #[napi(catch_unwind)] - pub async fn get_lsm_stats( + #[napi(catch_unwind, js_name = "getMemWalStats")] + pub async fn get_memwal_stats( &self, - include_generation_rows: bool, - ) -> napi::Result> { + include_sstable_rows: bool, + ) -> napi::Result> { let stats = self .inner_ref()? - .get_lsm_stats(include_generation_rows) + .get_memwal_stats(include_sstable_rows) .await .default_error()?; - Ok(stats.map(LsmStats::from)) + Ok(stats.map(MemWalStats::from)) } #[napi(catch_unwind)] @@ -958,7 +954,7 @@ impl From for FtsToken { /// `column` is required. #[napi(object)] #[derive(Clone, Debug)] -pub struct LsmWriteSpec { +pub struct MemWalWriteSpec { /// One of `"bucket"`, `"identity"`, or `"unsharded"`. pub spec_type: String, /// Bucket and identity variants: the sharding column. @@ -972,31 +968,31 @@ pub struct LsmWriteSpec { pub writer_config_defaults: Option>, } -impl TryFrom for lancedb::table::LsmWriteSpec { +impl TryFrom for lancedb::table::MemWalWriteSpec { type Error = napi::Error; - fn try_from(value: LsmWriteSpec) -> napi::Result { + fn try_from(value: MemWalWriteSpec) -> napi::Result { let writer_config_defaults = value.writer_config_defaults.unwrap_or_default(); let spec = match value.spec_type.as_str() { "bucket" => { let column = value.column.ok_or_else(|| { - napi::Error::from_reason("LsmWriteSpec bucket requires `column`") + napi::Error::from_reason("MemWalWriteSpec bucket requires `column`") })?; let num_buckets = value.num_buckets.ok_or_else(|| { - napi::Error::from_reason("LsmWriteSpec bucket requires `numBuckets`") + napi::Error::from_reason("MemWalWriteSpec bucket requires `numBuckets`") })?; Self::bucket(column, num_buckets) } "identity" => { let column = value.column.ok_or_else(|| { - napi::Error::from_reason("LsmWriteSpec identity requires `column`") + napi::Error::from_reason("MemWalWriteSpec identity requires `column`") })?; Self::identity(column) } "unsharded" => Self::unsharded(), other => { return Err(napi::Error::from_reason(format!( - "LsmWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'", + "MemWalWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'", other ))); } @@ -1007,9 +1003,9 @@ impl TryFrom for lancedb::table::LsmWriteSpec { } } -impl From for LsmWriteSpec { - fn from(spec: lancedb::table::LsmWriteSpec) -> Self { - use lancedb::table::LsmWriteSpec as Native; +impl From for MemWalWriteSpec { + fn from(spec: lancedb::table::MemWalWriteSpec) -> Self { + use lancedb::table::MemWalWriteSpec as Native; match spec { Native::Bucket { column, @@ -1048,21 +1044,21 @@ impl From for LsmWriteSpec { } } -/// One flushed L0 generation. +/// One SSTable. #[napi(object)] #[derive(Clone, Debug)] -pub struct GenerationStats { - /// The generation number. Increases as memtables are sealed into L0. +pub struct SsTableStats { + /// The generation number. Increases as memtables are frozen into SSTables. pub generation: i64, - /// On-disk size of the generation. + /// On-disk size of the SSTable. pub bytes: i64, - /// Present only when `includeGenerationRows` was requested. Off by default + /// Present only when `includeSstableRows` was requested. Off by default /// because each count opens an uncached Lance dataset. pub rows: Option, } -impl From for GenerationStats { - fn from(g: lancedb::table::GenerationStats) -> Self { +impl From for SsTableStats { + fn from(g: lancedb::table::SsTableStats) -> Self { Self { generation: g.generation as i64, bytes: g.bytes as i64, @@ -1075,7 +1071,7 @@ impl From for GenerationStats { #[napi(object)] #[derive(Clone, Debug)] pub struct MemtableStats { - /// The generation this memtable will become once sealed. + /// The generation this memtable will become once frozen. pub generation: i64, /// Rows currently buffered. pub rows: i64, @@ -1100,13 +1096,13 @@ impl From for MemtableStats { } } -/// Live state of one bucket. A table is N buckets on one node; flattening to a -/// single number hides the one hot bucket that is usually why someone opened +/// Live state of one table shard. A table is N table shards on one node; flattening to a +/// single number hides the one hot table shard that is usually why someone opened /// this endpoint. #[napi(object)] #[derive(Clone, Debug)] -pub struct BucketStats { - /// The shard this bucket writes. +pub struct TableShardStats { + /// The id of this table shard. pub shard_id: String, /// `"Active"` or `"Sealed"` (drop-table 2PC in flight). pub status: String, @@ -1121,20 +1117,20 @@ pub struct BucketStats { /// Highest WAL position the writer has seen. The difference against /// `replayAfterWalEntryPosition` is the WAL lag. pub wal_entry_position_last_seen: i64, - /// Flushed L0 generations not yet merged into the base table. - pub generations: Vec, - /// Whether a pass owns this bucket's compaction latch right now. Says *a* + /// SSTables not yet merged into the base table. + pub sstables: Vec, + /// Whether a pass owns this table shard's compaction latch right now. Says *a* /// driver is running, not *whose*, and the latch is held from dispatch — /// including while the pass queues for a pod-wide compactor permit. Read it /// as "do not pile on", never as "mine is progressing". pub compacting: bool, - /// Oldest first, active last. Absent for a `"Sealed"` bucket, whose + /// Oldest first, active last. Absent for a `"Sealed"` table shard, whose /// in-memory state is torn down. pub memtables: Option>, } -impl From for BucketStats { - fn from(b: lancedb::table::BucketStats) -> Self { +impl From for TableShardStats { + fn from(b: lancedb::table::TableShardStats) -> Self { Self { shard_id: b.shard_id, status: b.status, @@ -1143,7 +1139,7 @@ impl From for BucketStats { current_generation: b.current_generation as i64, replay_after_wal_entry_position: b.replay_after_wal_entry_position as i64, wal_entry_position_last_seen: b.wal_entry_position_last_seen as i64, - generations: b.generations.into_iter().map(Into::into).collect(), + sstables: b.sstables.into_iter().map(Into::into).collect(), compacting: b.compacting, memtables: b .memtables @@ -1152,21 +1148,21 @@ impl From for BucketStats { } } -/// Live per-bucket LSM state, as returned by `Table#getLsmStats`. +/// Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`. /// -/// Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are +/// Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are /// the caller's to compute. #[napi(object)] #[derive(Clone, Debug)] -pub struct LsmStats { - /// One entry per bucket backing this table. - pub buckets: Vec, +pub struct MemWalStats { + /// One entry per table shard backing this table. + pub table_shards: Vec, } -impl From for LsmStats { - fn from(stats: lancedb::table::LsmStats) -> Self { +impl From for MemWalStats { + fn from(stats: lancedb::table::MemWalStats) -> Self { Self { - buckets: stats.buckets.into_iter().map(Into::into).collect(), + table_shards: stats.table_shards.into_iter().map(Into::into).collect(), } } } diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 39353f270..4ed64ee31 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -15,7 +15,7 @@ from ._lancedb import FtsToken from ._lancedb import FunctionErrorFragment as FunctionErrorFragment from ._lancedb import FunctionErrorRecord as FunctionErrorRecord from ._lancedb import FunctionErrors as FunctionErrors -from ._lancedb import LsmWriteSpec +from ._lancedb import MemWalWriteSpec from ._lancedb import tokenize as _tokenize from .common import URI, sanitize_uri from urllib.parse import urlparse @@ -610,7 +610,7 @@ __all__ = [ "SqlQuery", "LanceDBConnection", "LanceNamespaceDBConnection", - "LsmWriteSpec", + "MemWalWriteSpec", "RemoteDBConnection", "Session", "Table", diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 80554baf0..b9c033d7b 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -538,17 +538,17 @@ class Table: async def initial_storage_options(self) -> Optional[Dict[str, str]]: ... async def latest_storage_options(self) -> Optional[Dict[str, str]]: ... async def set_unenforced_primary_key(self, columns: List[str]) -> None: ... - async def set_lsm_write_spec(self, spec: LsmWriteSpec) -> None: ... - async def unset_lsm_write_spec(self) -> None: ... - async def get_lsm_write_spec(self) -> Optional[LsmWriteSpec]: ... + async def enable_memwal(self, spec: MemWalWriteSpec) -> None: ... + async def disable_memwal(self) -> None: ... + async def get_memwal_spec(self) -> Optional[MemWalWriteSpec]: ... # WAL-PK-FUSION: delete both. def hybrid_pk_fusion_learned(self) -> bool: ... def note_hybrid_pk_fusion(self) -> None: ... - async def checkpoint_lsm(self) -> None: ... - async def flush_lsm(self) -> None: ... - async def compact_lsm(self) -> None: ... - async def get_lsm_stats(self, include_generation_rows: bool) -> Optional[dict]: ... - async def close_lsm_writers(self) -> None: ... + async def checkpoint_memwal(self) -> None: ... + async def flush_memwal(self) -> None: ... + async def compact_memwal(self) -> None: ... + async def get_memwal_stats(self, include_sstable_rows: bool) -> Optional[dict]: ... + async def close_memwal_writers(self) -> None: ... @property def tags(self) -> Tags: ... @property @@ -661,7 +661,7 @@ class Query: def fast_search(self): ... def with_row_id(self): ... def postfilter(self): ... - def use_lsm(self, enable: bool): ... + def use_memwal(self, enable: bool): ... def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ... def nearest_to_text(self, query: dict) -> FTSQuery: ... def order_by(self, ordering: Optional[List[ColumnOrdering]]): ... @@ -678,7 +678,7 @@ class Query: class TakeQuery: def select(self, columns: List[str]): ... def with_row_id(self): ... - def use_lsm(self, enable: bool): ... + def use_memwal(self, enable: bool): ... async def output_schema(self) -> pa.Schema: ... async def execute(self) -> RecordBatchStream: ... async def explain_plan(self, verbose: Optional[bool]) -> str: ... @@ -697,7 +697,7 @@ class FTSQuery: def fast_search(self): ... def with_row_id(self): ... def postfilter(self): ... - def use_lsm(self, enable: bool): ... + def use_memwal(self, enable: bool): ... def get_query(self) -> str: ... def add_query_vector(self, query_vec: pa.Array) -> None: ... def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ... @@ -725,7 +725,7 @@ class VectorQuery: def column(self, column: str): ... def distance_type(self, distance_type: str): ... def postfilter(self): ... - def use_lsm(self, enable: bool): ... + def use_memwal(self, enable: bool): ... def refine_factor(self, refine_factor: int): ... def nprobes(self, nprobes: int): ... def search_effort(self, search_effort: Optional[float]): ... @@ -750,7 +750,7 @@ class HybridQuery: def fast_search(self): ... def with_row_id(self): ... def postfilter(self): ... - def use_lsm(self, enable: bool): ... + def use_memwal(self, enable: bool): ... def distance_type(self, distance_type: str): ... def refine_factor(self, refine_factor: int): ... def nprobes(self, nprobes: int): ... @@ -778,7 +778,7 @@ class PyQueryRequest: select_source_columns: Optional[Dict[str, str]] fast_search: Optional[bool] with_row_id: Optional[bool] - use_lsm: Optional[bool] + use_memwal: Optional[bool] column: Optional[str] query_vector: Optional[List[pa.Array]] nprobes: Optional[int] @@ -839,22 +839,26 @@ class FtsToken: text: str position: int -class LsmWriteSpec: +class MemWalWriteSpec: """Specification selecting Lance's MemWAL LSM-style write path for `merge_insert`.""" @staticmethod - def bucket(column: str, num_buckets: int) -> "LsmWriteSpec": ... + def bucket(column: str, num_buckets: int) -> "MemWalWriteSpec": ... @staticmethod - def identity(column: str) -> "LsmWriteSpec": ... + def identity(column: str) -> "MemWalWriteSpec": ... @staticmethod - def unsharded() -> "LsmWriteSpec": ... - def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "LsmWriteSpec": + def unsharded() -> "MemWalWriteSpec": ... + def with_maintained_indexes( + self, indexes: Optional[List[str]] + ) -> "MemWalWriteSpec": """Set which indexes the MemWAL keeps up to date. None resolves every index on the table at install, failing if one cannot be maintained; a list is verbatim, empty means none.""" ... - def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "LsmWriteSpec": + def with_writer_config_defaults( + self, defaults: Dict[str, str] + ) -> "MemWalWriteSpec": """Return a copy of this spec recording the given default `ShardWriter` configuration in the MemWAL index.""" ... diff --git a/python/python/lancedb/_wal_hybrid.py b/python/python/lancedb/_wal_hybrid.py index 67597950d..8e5d6ae17 100644 --- a/python/python/lancedb/_wal_hybrid.py +++ b/python/python/lancedb/_wal_hybrid.py @@ -76,7 +76,7 @@ _WAL_ROW_ID_REFUSAL = "does not support with_row_id" WAL_ROW_ID_UNSUPPORTED = ( "hybrid search on a MemWAL table cannot return _rowid: the fresh tier has " "no stable row id, and the ids the fusion joins on are synthesized from " - "the primary key. Set use_lsm(False) to read the base table only (results " + "the primary key. Set use_memwal(False) to read the base table only (results " "will exclude un-compacted MemWAL data)" ) @@ -151,13 +151,13 @@ def _stamp_surrogate_row_ids( return stamped -def pk_fusion_learned(table, use_lsm: Optional[bool]) -> bool: +def pk_fusion_learned(table, use_memwal: Optional[bool]) -> bool: """Whether a previous hybrid query on `table` was refused `_rowid`, so this one should go straight to the primary key. Learned from a refusal, never probed, so asking is free. """ - if table is None or use_lsm is False: + if table is None or use_memwal is False: return False # Only an unambiguous True forks to the key: a table type that answers the # question loosely keeps the `_rowid` path. @@ -192,7 +192,7 @@ class PkFusion: raise NotImplementedError( "hybrid search cannot project a blob column on a MemWAL table: " "fetching blobs needs a real _rowid, and the fresh tier has no " - "stable one. Set use_lsm(False) to read the base table only " + "stable one. Set use_memwal(False) to read the base table only " "(results will exclude un-compacted MemWAL data)" ) return cls(_pk_columns(schema)) diff --git a/python/python/lancedb/merge.py b/python/python/lancedb/merge.py index 115a47e2d..d7c25349e 100644 --- a/python/python/lancedb/merge.py +++ b/python/python/lancedb/merge.py @@ -37,7 +37,7 @@ class LanceMergeInsertBuilder(object): self._when_not_matched_by_source_condition_expr = None self._timeout = None self._use_index = True - self._use_lsm = None + self._use_memwal = None self._validate_single_shard = None def when_matched_update_all( @@ -115,22 +115,22 @@ class LanceMergeInsertBuilder(object): self._use_index = use_index return self - def use_lsm(self, enable: bool) -> LanceMergeInsertBuilder: + def use_memwal(self, enable: bool) -> LanceMergeInsertBuilder: """ Control MemWAL routing for this merge. - By default (unset), a `merge_insert` on a table with an LSM write spec is + By default (unset), a `merge_insert` on a table with a MemWAL write spec is routed through Lance's MemWAL shard writer, and a table without one uses the standard path. Parameters ---------- enable: bool - ``True`` forces MemWAL routing and errors if the table has no LSM + ``True`` forces MemWAL routing and errors if the table has no MemWAL write spec. ``False`` forces the standard write path even when a spec is set. """ - self._use_lsm = enable + self._use_memwal = enable return self def validate_single_shard( @@ -139,13 +139,13 @@ class LanceMergeInsertBuilder(object): """ Controls how an LSM merge checks that its input targets a single shard. - When a table has an LSM write spec, every row in a `merge_insert` call + When a table has a MemWAL write spec, every row in a `merge_insert` call must route to the same shard. When `True` (the default), every row is inspected to verify this. When `False`, only the first row is inspected and the shard it routes to is used for the whole input — a faster path for callers that have already pre-sharded their input. - Has no effect on tables without an LSM write spec. + Has no effect on tables without a MemWAL write spec. Parameters ---------- diff --git a/python/python/lancedb/permutation.py b/python/python/lancedb/permutation.py index 8287ef2fe..d97eccc07 100644 --- a/python/python/lancedb/permutation.py +++ b/python/python/lancedb/permutation.py @@ -48,7 +48,7 @@ class PermutationBuilder: By default, the permutation builder will create a single split that contains all rows in the same order as the base table. - Tables with an LSM write spec are rejected: unflushed rows have no row id. + Tables with a MemWAL write spec are rejected: unflushed rows have no row id. """ self._async = async_permutation_builder(table) diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index 9734c9b43..ebb4fdf16 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -890,7 +890,7 @@ class Query(pydantic.BaseModel): # MemWAL LSM read routing: None auto-routes when the table carries a write # spec, True forces the LSM scanner (errors without a spec), False reads the # base table only - use_lsm: Optional[bool] = None + use_memwal: Optional[bool] = None # size of the nearest neighbor list maintained during HNSW search ef: Optional[int] = None @@ -910,9 +910,9 @@ class Query(pydantic.BaseModel): query.full_text_query = req.full_text_search query.columns = req.select query.with_row_id = req.with_row_id - # use_lsm is a genuine tri-state (None / True / False); preserve it as-is + # use_memwal is a genuine tri-state (None / True / False); preserve it as-is # so a round-tripped query keeps an explicit False. - query.use_lsm = req.use_lsm + query.use_memwal = req.use_memwal query.vector_column = req.column query.vector = req.query_vector query.distance_type = req.distance_type @@ -1085,7 +1085,7 @@ class LanceQueryBuilder(ABC): self._with_row_address = None self._fragments = None self._fragment_ids = None - self._use_lsm = None + self._use_memwal = None self._vector = None self._text = None self._ef = None @@ -1520,19 +1520,19 @@ class LanceQueryBuilder(ABC): self._fragment_ids = fragment_ids return self - def use_lsm(self, enable: bool) -> Self: + def use_memwal(self, enable: bool) -> Self: """Control MemWAL LSM read routing for this query. - By default (unset), a query against a table with an LSM write spec is + By default (unset), a query against a table with a MemWAL write spec is routed through the LSM scanner so it also returns data written via the ``merge_insert`` LSM path that has not yet been compacted into the base - table (active/frozen memtables + flushed generations); a table without a + table (active/frozen memtables + SSTables); a table without a spec reads the base table. Parameters ---------- enable : bool - ``True`` forces the LSM scanner and errors if the table has no LSM + ``True`` forces the LSM scanner and errors if the table has no MemWAL write spec. ``False`` bypasses the MemWAL and reads the base table only, even when a spec is present. @@ -1541,7 +1541,7 @@ class LanceQueryBuilder(ABC): LanceQueryBuilder The LanceQueryBuilder object. """ - self._use_lsm = enable + self._use_memwal = enable return self def explain_plan(self, verbose: Optional[bool] = False) -> str: @@ -2055,7 +2055,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): with_row_address=self._with_row_address, fragments=self._fragments, fragment_ids=self._fragment_ids, - use_lsm=self._use_lsm, + use_memwal=self._use_memwal, offset=self._offset, fast_search=self._fast_search, ef=self._ef, @@ -2280,7 +2280,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder): with_row_address=self._with_row_address, fragments=self._fragments, fragment_ids=self._fragment_ids, - use_lsm=self._use_lsm, + use_memwal=self._use_memwal, full_text_query=FullTextSearchQuery( query=self._query_with_phrase_semantics(), columns=self._fts_columns ), @@ -2362,7 +2362,7 @@ class LanceEmptyQueryBuilder(LanceQueryBuilder): with_row_address=self._with_row_address, fragments=self._fragments, fragment_ids=self._fragment_ids, - use_lsm=self._use_lsm, + use_memwal=self._use_memwal, offset=self._offset, order_by=self._order_by, ) @@ -3040,7 +3040,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): # WAL-PK-FUSION: without the fallback, select `self._columns` as is. self._pk_fusion = None columns = self._columns - if _wal_hybrid.pk_fusion_learned(self._table, self._use_lsm): + if _wal_hybrid.pk_fusion_learned(self._table, self._use_memwal): self._pk_fusion = _wal_hybrid.PkFusion.for_query( self._table.schema, self._columns, @@ -3057,9 +3057,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): if self._with_row_id and self._pk_fusion is None: self._vector_query.with_row_id(True) self._fts_query.with_row_id(True) - if self._use_lsm is not None: - self._vector_query.use_lsm(self._use_lsm) - self._fts_query.use_lsm(self._use_lsm) + if self._use_memwal is not None: + self._vector_query.use_memwal(self._use_memwal) + self._fts_query.use_memwal(self._use_memwal) if self._phrase_query: self._fts_query.phrase_query(True) if self._distance_type: @@ -3723,25 +3723,25 @@ class AsyncStandardQuery(AsyncQueryBase): self._inner.fast_search() return self - def use_lsm(self, enable: bool) -> Self: + def use_memwal(self, enable: bool) -> Self: """ Control MemWAL LSM read routing for this query. - By default (unset), a query against a table with an LSM write spec (see - [AsyncTable.set_lsm_write_spec][lancedb.table.AsyncTable.set_lsm_write_spec]) + By default (unset), a query against a table with a MemWAL write spec (see + [AsyncTable.enable_memwal][lancedb.table.AsyncTable.enable_memwal]) is routed through the LSM scanner so it also returns data written via the ``merge_insert`` LSM path that has not yet been compacted into the base - table (the active/frozen in-memory memtables and the flushed generations), + table (the active/frozen in-memory memtables and the SSTables), deduplicated by primary key; a table without a spec reads the base table. Parameters ---------- enable : bool - ``True`` forces the LSM scanner and errors if the table has no LSM + ``True`` forces the LSM scanner and errors if the table has no MemWAL write spec. ``False`` bypasses the MemWAL and reads the base table only, even when a spec is present. """ - self._inner.use_lsm(enable) + self._inner.use_memwal(enable) return self def postfilter(self) -> Self: @@ -4388,7 +4388,7 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase): # WAL-PK-FUSION: without the fallback, keep only the two `with_row_id` # calls. self._pk_fusion = None - if _wal_hybrid.pk_fusion_learned(self._table, fts_req.use_lsm): + if _wal_hybrid.pk_fusion_learned(self._table, fts_req.use_memwal): self._pk_fusion = _wal_hybrid.PkFusion.for_query( await self._table.schema(), _query_request_projection(fts_req), @@ -4577,13 +4577,13 @@ class AsyncTakeQuery(AsyncQueryBase): def __init__(self, inner: LanceTakeQuery, table: Optional["AsyncTable"] = None): super().__init__(inner, table) - def use_lsm(self, enable: bool) -> "AsyncTakeQuery": + def use_memwal(self, enable: bool) -> "AsyncTakeQuery": """Control MemWAL LSM read routing for this take query. ``False`` bypasses the MemWAL and reads the base table only — the escape hatch, since take-by-row-id/offset is not supported on the LSM scanner. """ - self._inner.use_lsm(enable) + self._inner.use_memwal(enable) return self async def _plain_scan_to_pandas( @@ -4644,14 +4644,14 @@ class BaseQueryBuilder(object): self._inner.with_row_id() return self - def use_lsm(self, enable: bool) -> Self: + def use_memwal(self, enable: bool) -> Self: """ Control MemWAL LSM read routing for this query. ``False`` bypasses the MemWAL and reads the base table only, the escape hatch for shapes the LSM scanner cannot honor (e.g. take-by-row-id). """ - self._inner.use_lsm(enable) + self._inner.use_memwal(enable) return self def with_row_address(self, with_row_address: bool = True) -> Self: diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 5996fdeb3..2bec79d1e 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -31,7 +31,7 @@ from lancedb._lancedb import ( DropColumnsResult, FtsToken, IndexConfig, - LsmWriteSpec, + MemWalWriteSpec, MergeResult, UpdateResult, ) @@ -1037,17 +1037,17 @@ class RemoteTable(Table): """Set the unenforced primary key for this table to a single column.""" return LOOP.run(self._table.set_unenforced_primary_key(columns)) - def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None: - """Install an LsmWriteSpec.""" - return LOOP.run(self._table.set_lsm_write_spec(spec)) + def enable_memwal(self, spec: "MemWalWriteSpec") -> None: + """Install an MemWalWriteSpec.""" + return LOOP.run(self._table.enable_memwal(spec)) - def unset_lsm_write_spec(self) -> None: - """Remove the LsmWriteSpec.""" - return LOOP.run(self._table.unset_lsm_write_spec()) + def disable_memwal(self) -> None: + """Remove the MemWalWriteSpec.""" + return LOOP.run(self._table.disable_memwal()) - def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]: - """Read the installed LsmWriteSpec, or ``None``.""" - return LOOP.run(self._table.get_lsm_write_spec()) + def get_memwal_spec(self) -> Optional["MemWalWriteSpec"]: + """Read the installed MemWalWriteSpec, or ``None``.""" + return LOOP.run(self._table.get_memwal_spec()) # WAL-PK-FUSION: delete both hooks. def _hybrid_pk_fusion_learned(self) -> bool: @@ -1057,31 +1057,31 @@ class RemoteTable(Table): def _note_hybrid_pk_fusion(self) -> None: self._table._note_hybrid_pk_fusion() - def checkpoint_lsm(self) -> None: + def checkpoint_memwal(self) -> None: """Synchronous version of - [`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm].""" - return LOOP.run(self._table.checkpoint_lsm()) + [`AsyncTable.checkpoint_memwal`][lancedb.AsyncTable.checkpoint_memwal].""" + return LOOP.run(self._table.checkpoint_memwal()) - def flush_lsm(self) -> None: + def flush_memwal(self) -> None: """Synchronous version of - [`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm].""" - return LOOP.run(self._table.flush_lsm()) + [`AsyncTable.flush_memwal`][lancedb.AsyncTable.flush_memwal].""" + return LOOP.run(self._table.flush_memwal()) - def compact_lsm(self) -> None: + def compact_memwal(self) -> None: """Synchronous version of - [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm].""" - return LOOP.run(self._table.compact_lsm()) + [`AsyncTable.compact_memwal`][lancedb.AsyncTable.compact_memwal].""" + return LOOP.run(self._table.compact_memwal()) - def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]: + def get_memwal_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]: """Synchronous version of - [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats].""" + [`AsyncTable.get_memwal_stats`][lancedb.AsyncTable.get_memwal_stats].""" return LOOP.run( - self._table.get_lsm_stats(include_generation_rows=include_generation_rows) + self._table.get_memwal_stats(include_sstable_rows=include_sstable_rows) ) - def close_lsm_writers(self) -> None: + def close_memwal_writers(self) -> None: """No-op for remote tables (no local shard writers).""" - return LOOP.run(self._table.close_lsm_writers()) + return LOOP.run(self._table.close_memwal_writers()) def drop_index(self, index_name: str): return LOOP.run(self._table.drop_index(index_name)) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index d39053775..f0cb05427 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -223,7 +223,7 @@ if TYPE_CHECKING: DeleteResult, DropColumnsResult, FtsToken, - LsmWriteSpec, + MemWalWriteSpec, MergeResult, UpdateResult, ) @@ -4827,47 +4827,47 @@ class LanceTable(Table): [`AsyncTable.set_unenforced_primary_key`][lancedb.AsyncTable.set_unenforced_primary_key].""" return LOOP.run(self._table.set_unenforced_primary_key(columns)) - def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None: - """Install an LsmWriteSpec. See - [`AsyncTable.set_lsm_write_spec`][lancedb.AsyncTable.set_lsm_write_spec].""" - return LOOP.run(self._table.set_lsm_write_spec(spec)) + def enable_memwal(self, spec: "MemWalWriteSpec") -> None: + """Install an MemWalWriteSpec. See + [`AsyncTable.enable_memwal`][lancedb.AsyncTable.enable_memwal].""" + return LOOP.run(self._table.enable_memwal(spec)) - def unset_lsm_write_spec(self) -> None: - """Remove the LsmWriteSpec. See - [`AsyncTable.unset_lsm_write_spec`][lancedb.AsyncTable.unset_lsm_write_spec].""" - return LOOP.run(self._table.unset_lsm_write_spec()) + def disable_memwal(self) -> None: + """Remove the MemWalWriteSpec. See + [`AsyncTable.disable_memwal`][lancedb.AsyncTable.disable_memwal].""" + return LOOP.run(self._table.disable_memwal()) - def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]: - """Read the installed LsmWriteSpec, or ``None``. See - [`AsyncTable.get_lsm_write_spec`][lancedb.AsyncTable.get_lsm_write_spec].""" - return LOOP.run(self._table.get_lsm_write_spec()) + def get_memwal_spec(self) -> Optional["MemWalWriteSpec"]: + """Read the installed MemWalWriteSpec, or ``None``. See + [`AsyncTable.get_memwal_spec`][lancedb.AsyncTable.get_memwal_spec].""" + return LOOP.run(self._table.get_memwal_spec()) - def checkpoint_lsm(self) -> None: + def checkpoint_memwal(self) -> None: """Synchronous version of - [`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm].""" - return LOOP.run(self._table.checkpoint_lsm()) + [`AsyncTable.checkpoint_memwal`][lancedb.AsyncTable.checkpoint_memwal].""" + return LOOP.run(self._table.checkpoint_memwal()) - def flush_lsm(self) -> None: + def flush_memwal(self) -> None: """Synchronous version of - [`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm].""" - return LOOP.run(self._table.flush_lsm()) + [`AsyncTable.flush_memwal`][lancedb.AsyncTable.flush_memwal].""" + return LOOP.run(self._table.flush_memwal()) - def compact_lsm(self) -> None: + def compact_memwal(self) -> None: """Synchronous version of - [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm].""" - return LOOP.run(self._table.compact_lsm()) + [`AsyncTable.compact_memwal`][lancedb.AsyncTable.compact_memwal].""" + return LOOP.run(self._table.compact_memwal()) - def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]: + def get_memwal_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]: """Synchronous version of - [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats].""" + [`AsyncTable.get_memwal_stats`][lancedb.AsyncTable.get_memwal_stats].""" return LOOP.run( - self._table.get_lsm_stats(include_generation_rows=include_generation_rows) + self._table.get_memwal_stats(include_sstable_rows=include_sstable_rows) ) - def close_lsm_writers(self) -> None: + def close_memwal_writers(self) -> None: """Close cached MemWAL shard writers. See - [`AsyncTable.close_lsm_writers`][lancedb.AsyncTable.close_lsm_writers].""" - return LOOP.run(self._table.close_lsm_writers()) + [`AsyncTable.close_memwal_writers`][lancedb.AsyncTable.close_memwal_writers].""" + return LOOP.run(self._table.close_memwal_writers()) def uses_v2_manifest_paths(self) -> bool: """ @@ -5525,18 +5525,18 @@ class AsyncTable: columns = list(columns) await self._inner.set_unenforced_primary_key(columns) - async def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None: - """Install an LsmWriteSpec on this table. + async def enable_memwal(self, spec: "MemWalWriteSpec") -> None: + """Install an MemWalWriteSpec on this table. The spec selects Lance's MemWAL LSM-style write path for future - `merge_insert` calls. ``LsmWriteSpec`` chooses one of three sharding + `merge_insert` calls. ``MemWalWriteSpec`` chooses one of three sharding strategies: - - ``LsmWriteSpec.bucket(column, num_buckets)`` — hash-bucket writes by + - ``MemWalWriteSpec.bucket(column, num_buckets)`` — hash-bucket writes by the single-column unenforced primary key. - - ``LsmWriteSpec.identity(column)`` — shard by the raw value of a + - ``MemWalWriteSpec.identity(column)`` — shard by the raw value of a scalar column. - - ``LsmWriteSpec.unsharded()`` — route every write to a single shard. + - ``MemWalWriteSpec.unsharded()`` — route every write to a single shard. All variants require the table to have an unenforced primary key set via [`set_unenforced_primary_key`]; bucket sharding additionally @@ -5551,30 +5551,30 @@ class AsyncTable: Parameters ---------- - spec : LsmWriteSpec + spec : MemWalWriteSpec The sharding spec to install. Examples -------- - >>> from lancedb import LsmWriteSpec + >>> from lancedb import MemWalWriteSpec >>> # table.set_unenforced_primary_key("id") - >>> # table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 16)) + >>> # table.enable_memwal(MemWalWriteSpec.bucket("id", 16)) """ - await self._inner.set_lsm_write_spec(spec) + await self._inner.enable_memwal(spec) - async def unset_lsm_write_spec(self) -> None: - """Remove the LsmWriteSpec from this table. + async def disable_memwal(self) -> None: + """Remove the MemWalWriteSpec from this table. Reverts to the standard `merge_insert` write path. Errors if no spec is currently set. """ - await self._inner.unset_lsm_write_spec() + await self._inner.disable_memwal() - async def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]: - """Read the LsmWriteSpec currently installed on this table. + async def get_memwal_spec(self) -> Optional["MemWalWriteSpec"]: + """Read the MemWalWriteSpec currently installed on this table. - Returns ``None`` when the LSM write path is not enabled at all — no - spec has been set, or one was removed with `unset_lsm_write_spec`. + Returns ``None`` when the MemWAL write path is not enabled at all — no + spec has been set, or one was removed with `disable_memwal`. That is a different answer from a spec whose ``maintained_indexes`` is ``None``, which is an installed spec selecting indexes automatically. @@ -5584,7 +5584,7 @@ class AsyncTable: or gains later, ``[]`` maintains none, and a non-empty list maintains exactly those. All three round-trip. """ - return await self._inner.get_lsm_write_spec() + return await self._inner.get_memwal_spec() # WAL-PK-FUSION: delete both hooks. def _hybrid_pk_fusion_learned(self) -> bool: @@ -5595,19 +5595,19 @@ class AsyncTable: """See [`Table._note_hybrid_pk_fusion`][lancedb.table.Table].""" self._inner.note_hybrid_pk_fusion() - async def checkpoint_lsm(self) -> None: - """Converge this table's LSM write path into its base table. + async def checkpoint_memwal(self) -> None: + """Converge this table's MemWAL write path into its base table. - One flush, sealing every memtable into L0, then compaction triggers - until every generation that existed at that moment has reached base. - The loop runs client-side, reading progress from ``get_lsm_stats``. + One flush, freezing every memtable into an SSTable, then compaction triggers + until every SSTable that existed at that moment has reached base. + The loop runs client-side, reading progress from ``get_memwal_stats``. - Best-effort: generations created *while* it runs are deliberately not + Best-effort: SSTables created *while* it runs are deliberately not waited on, which is what lets it terminate on a table taking writes. Idempotent and safe on a cadence. There is no deadline, and the caller owns that. It returns when the - target generations are gone, raises on a terminal server fault, and + target SSTables are gone, raises on a terminal server fault, and otherwise waits however long the server takes. A slow table and a stuck one are the same picture from the client: the compactor pool is shared across every table on the node, so a checkpoint queued behind @@ -5615,56 +5615,56 @@ class AsyncTable: ``asyncio.wait_for`` for a wall-clock bound; abandoning it partway costs nothing. """ - await self._inner.checkpoint_lsm() + await self._inner.checkpoint_memwal() - async def flush_lsm(self) -> None: - """Seal every bucket's active memtable into L0. + async def flush_memwal(self) -> None: + """Freeze every table shard's active memtable into an SSTable. - Does not touch the base table — moving L0 into base is - `compact_lsm`. On a node that has not claimed this table, this claims + Does not touch the base table — compacting SSTables into base is + `compact_memwal`. On a node that has not claimed this table, this claims it and replays its WAL log first. """ - await self._inner.flush_lsm() + await self._inner.flush_memwal() - async def compact_lsm(self) -> None: - """Trigger a background L0 to base compaction pass per bucket. + async def compact_memwal(self) -> None: + """Trigger a background SSTable compaction pass per table shard. Returns once the passes are dispatched, not once they finish: watch - ``get_lsm_stats`` for progress, or use ``checkpoint_lsm`` to loop - until the current L0 has reached base. + ``get_memwal_stats`` for progress, or use ``checkpoint_memwal`` to loop + until the current SSTables have reached base. """ - await self._inner.compact_lsm() + await self._inner.compact_memwal() - async def get_lsm_stats( - self, *, include_generation_rows: bool = False + async def get_memwal_stats( + self, *, include_sstable_rows: bool = False ) -> Optional[dict]: - """Read live per-bucket LSM state. + """Read live per-table-shard MemWAL state. - Answers "how far behind is my fresh tier", "which bucket is hot", and + Answers "how far behind is my fresh tier", "which table shard is hot", and "why is my fresh-tier vector search brute-force". Mutates no table state, though on a node that has not claimed this table it claims it, exactly as a read would. - Returns ``None`` only when the LSM write path is not enabled. + Returns ``None`` only when the MemWAL write path is not enabled. Parameters ---------- - include_generation_rows - Report a row count per L0 generation. Off by default: each count - opens an uncached Lance dataset, and ``checkpoint_lsm`` polls this + include_sstable_rows + Report a row count per SSTable. Off by default: each count + opens an uncached Lance dataset, and ``checkpoint_memwal`` polls this needing only generation numbers. """ - return await self._inner.get_lsm_stats(include_generation_rows) + return await self._inner.get_memwal_stats(include_sstable_rows) - async def close_lsm_writers(self) -> None: + async def close_memwal_writers(self) -> None: """Drain and close any cached MemWAL shard writers for this table. - When an LSM write spec is installed, `merge_insert` opens MemWAL shard + When a MemWAL write spec is installed, `merge_insert` opens MemWAL shard writers and caches them for reuse across calls. This closes them, flushing pending data; writers reopen lazily on the next `merge_insert`. It is a no-op when no writers are cached. """ - await self._inner.close_lsm_writers() + await self._inner.close_memwal_writers() @property def name(self) -> str: @@ -6553,8 +6553,8 @@ class AsyncTable: take_query = self.take_offsets(query.take_offsets) if query.columns: take_query = take_query.select(query.columns) - if query.use_lsm is not None: - take_query = take_query.use_lsm(query.use_lsm) + if query.use_memwal is not None: + take_query = take_query.use_memwal(query.use_memwal) if query.with_row_id: take_query = take_query.with_row_id() return take_query @@ -6570,8 +6570,8 @@ class AsyncTable: async_query = async_query.where(query.filter) if query.fast_search: async_query = async_query.fast_search() - if query.use_lsm is not None: - async_query = async_query.use_lsm(query.use_lsm) + if query.use_memwal is not None: + async_query = async_query.use_memwal(query.use_memwal) if query.with_row_id: async_query = async_query.with_row_id() if query.order_by: @@ -6697,7 +6697,7 @@ class AsyncTable: when_not_matched_by_source_condition_expr=merge._when_not_matched_by_source_condition_expr, timeout=merge._timeout, use_index=merge._use_index, - use_lsm=merge._use_lsm, + use_memwal=merge._use_memwal, validate_single_shard=merge._validate_single_shard, ), ) diff --git a/python/python/tests/test_lsm_write_spec.py b/python/python/tests/test_memwal_spec.py similarity index 65% rename from python/python/tests/test_lsm_write_spec.py rename to python/python/tests/test_memwal_spec.py index ac767ea31..a1ec6a90d 100644 --- a/python/python/tests/test_lsm_write_spec.py +++ b/python/python/tests/test_memwal_spec.py @@ -1,8 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors -"""Tests for installing and clearing an LsmWriteSpec via -`Table.set_lsm_write_spec` / `Table.unset_lsm_write_spec`. +"""Tests for installing and clearing an MemWalWriteSpec via +`Table.enable_memwal` / `Table.disable_memwal`. """ from datetime import timedelta @@ -10,7 +10,7 @@ from datetime import timedelta import lancedb import pyarrow as pa import pytest -from lancedb._lancedb import LsmWriteSpec +from lancedb._lancedb import MemWalWriteSpec from lancedb.index import BTree SCHEMA = pa.schema( @@ -21,9 +21,9 @@ SCHEMA = pa.schema( ) -def test_lsm_write_spec_module_metadata(): - assert lancedb.LsmWriteSpec is LsmWriteSpec - assert LsmWriteSpec.__module__ == "lancedb._lancedb" +def test_memwal_spec_module_metadata(): + assert lancedb.MemWalWriteSpec is MemWalWriteSpec + assert MemWalWriteSpec.__module__ == "lancedb._lancedb" def _batch(ids, vs): @@ -43,35 +43,35 @@ def _make_table(tmp_path): return db, table -def test_set_lsm_write_spec_validates(tmp_path): +def test_enable_memwal_validates(tmp_path): _db, table = _make_table(tmp_path) # Out-of-range num_buckets. with pytest.raises(Exception, match="num_buckets"): - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 0)) + table.enable_memwal(MemWalWriteSpec.bucket("id", 0)) with pytest.raises(Exception, match="num_buckets"): - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1025)) + table.enable_memwal(MemWalWriteSpec.bucket("id", 1025)) # Happy path, then a set over the installed spec is refused. - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4)) + table.enable_memwal(MemWalWriteSpec.bucket("id", 4)) with pytest.raises(Exception, match="already set"): - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8)) + table.enable_memwal(MemWalWriteSpec.bucket("id", 8)) -def test_unset_lsm_write_spec(tmp_path): +def test_disable_memwal(tmp_path): _db, table = _make_table(tmp_path) # unset errors when no spec is set. - with pytest.raises(Exception, match="no LSM write spec"): - table.unset_lsm_write_spec() + with pytest.raises(Exception, match="no MemWAL write spec"): + table.disable_memwal() # Install a spec, then remove it; afterwards a fresh spec can be set. - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4)) - table.unset_lsm_write_spec() + table.enable_memwal(MemWalWriteSpec.bucket("id", 4)) + table.disable_memwal() # A second unset errors — there is no spec left to remove. - with pytest.raises(Exception, match="no LSM write spec"): - table.unset_lsm_write_spec() - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8)) + with pytest.raises(Exception, match="no MemWAL write spec"): + table.disable_memwal() + table.enable_memwal(MemWalWriteSpec.bucket("id", 8)) def test_set_unsharded_spec(tmp_path): @@ -79,12 +79,12 @@ def test_set_unsharded_spec(tmp_path): # Lance MemWAL still requires a primary key on the dataset; Unsharded # just skips per-row hashing. table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) - table.unset_lsm_write_spec() + table.enable_memwal(MemWalWriteSpec.unsharded()) + table.disable_memwal() -def test_lsm_write_spec_repr(): - s = LsmWriteSpec.bucket("id", 4) +def test_memwal_spec_repr(): + s = MemWalWriteSpec.bucket("id", 4) assert s.spec_type == "bucket" assert s.column == "id" assert s.num_buckets == 4 @@ -95,20 +95,20 @@ def test_lsm_write_spec_repr(): assert "id" in repr(s) assert "4" in repr(s) - u = LsmWriteSpec.unsharded() + u = MemWalWriteSpec.unsharded() assert u.spec_type == "unsharded" assert u.column is None assert u.num_buckets is None assert "unsharded" in repr(u) -def test_lsm_write_spec_with_maintained_indexes(): - s = LsmWriteSpec.bucket("id", 4).with_maintained_indexes(["idx_a", "idx_b"]) +def test_memwal_spec_with_maintained_indexes(): + s = MemWalWriteSpec.bucket("id", 4).with_maintained_indexes(["idx_a", "idx_b"]) assert s.maintained_indexes == ["idx_a", "idx_b"] @pytest.mark.asyncio -async def test_async_set_unset_lsm_write_spec(tmp_path): +async def test_async_set_disable_memwal(tmp_path): db = await lancedb.connect_async( tmp_path, read_consistency_interval=timedelta(seconds=0) ) @@ -118,11 +118,11 @@ async def test_async_set_unset_lsm_write_spec(tmp_path): ) await table.set_unenforced_primary_key("id") - await table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4)) - await table.unset_lsm_write_spec() + await table.enable_memwal(MemWalWriteSpec.bucket("id", 4)) + await table.disable_memwal() # A second unset errors. - with pytest.raises(Exception, match="no LSM write spec"): - await table.unset_lsm_write_spec() + with pytest.raises(Exception, match="no MemWAL write spec"): + await table.disable_memwal() def test_set_identity_spec(tmp_path): @@ -130,12 +130,12 @@ def test_set_identity_spec(tmp_path): # Identity sharding still requires an unenforced primary key on the # table; it shards by the raw value of the given column. table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.identity("v")) - table.unset_lsm_write_spec() + table.enable_memwal(MemWalWriteSpec.identity("v")) + table.disable_memwal() -def test_lsm_write_spec_identity_and_writer_config_defaults(): - s = LsmWriteSpec.identity("v") +def test_memwal_spec_identity_and_writer_config_defaults(): + s = MemWalWriteSpec.identity("v") assert s.spec_type == "identity" assert s.column == "v" assert s.num_buckets is None @@ -146,12 +146,12 @@ def test_lsm_write_spec_identity_and_writer_config_defaults(): assert "durable_write" in repr(s) -def test_get_lsm_write_spec(tmp_path): +def test_get_memwal_spec(tmp_path): _db, table = _make_table(tmp_path) table.set_unenforced_primary_key("id") # None when nothing is installed. - assert table.get_lsm_write_spec() is None + assert table.get_memwal_spec() is None # A real scalar index is needed to name it as a maintained index. table.create_index("id", config=BTree()) @@ -159,12 +159,12 @@ def test_get_lsm_write_spec(tmp_path): # Bucket spec round-trips, including maintained indexes and writer config # defaults. - table.set_lsm_write_spec( - LsmWriteSpec.bucket("id", 4) + table.enable_memwal( + MemWalWriteSpec.bucket("id", 4) .with_maintained_indexes([idx_name]) .with_writer_config_defaults({"durable_write": "false"}) ) - spec = table.get_lsm_write_spec() + spec = table.get_memwal_spec() assert spec is not None assert spec.spec_type == "bucket" assert spec.column == "id" @@ -173,30 +173,30 @@ def test_get_lsm_write_spec(tmp_path): assert spec.writer_config_defaults == {"durable_write": "false"} # After unset, None again. - table.unset_lsm_write_spec() - assert table.get_lsm_write_spec() is None + table.disable_memwal() + assert table.get_memwal_spec() is None # Identity round-trips (column recovered from the schema). An unnamed # maintained set reads back as None: what is stored is the intent to # maintain every index, not the set it resolves to now. - table.set_lsm_write_spec(LsmWriteSpec.identity("id")) - spec = table.get_lsm_write_spec() + table.enable_memwal(MemWalWriteSpec.identity("id")) + spec = table.get_memwal_spec() assert spec.spec_type == "identity" assert spec.column == "id" assert spec.maintained_indexes is None - table.unset_lsm_write_spec() + table.disable_memwal() # Unsharded round-trips (no routing column). Opting out is distinct from # the unnamed default. - table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) - spec = table.get_lsm_write_spec() + table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([])) + spec = table.get_memwal_spec() assert spec.spec_type == "unsharded" assert spec.column is None assert spec.maintained_indexes == [] @pytest.mark.asyncio -async def test_async_get_lsm_write_spec(tmp_path): +async def test_async_get_memwal_spec(tmp_path): db = await lancedb.connect_async( tmp_path, read_consistency_interval=timedelta(seconds=0) ) @@ -205,20 +205,20 @@ async def test_async_get_lsm_write_spec(tmp_path): pa.RecordBatchReader.from_batches(SCHEMA, [_batch(["seed"], [0])]), ) - assert await table.get_lsm_write_spec() is None + assert await table.get_memwal_spec() is None # A real scalar index is needed to name it as a maintained index. await table.create_index("id", config=BTree()) idx_name = (await table.list_indices())[0].name - await table.set_lsm_write_spec( - LsmWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name]) + await table.enable_memwal( + MemWalWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name]) ) - spec = await table.get_lsm_write_spec() + spec = await table.get_memwal_spec() assert spec is not None assert spec.spec_type == "bucket" assert spec.column == "id" assert spec.num_buckets == 8 assert spec.maintained_indexes == [idx_name] - await table.unset_lsm_write_spec() - assert await table.get_lsm_write_spec() is None + await table.disable_memwal() + assert await table.get_memwal_spec() is None diff --git a/python/python/tests/test_merge_insert_lsm.py b/python/python/tests/test_merge_insert_lsm.py index e74c21589..aec25993a 100644 --- a/python/python/tests/test_merge_insert_lsm.py +++ b/python/python/tests/test_merge_insert_lsm.py @@ -8,7 +8,7 @@ from datetime import timedelta import lancedb import pyarrow as pa import pytest -from lancedb._lancedb import LsmWriteSpec +from lancedb._lancedb import MemWalWriteSpec from lancedb.index import FTS, IvfPq SCHEMA = pa.schema( @@ -49,12 +49,12 @@ def _region_reader(rows): def _bucket_table(tmp_path): - """A table with ``id`` as the primary key and a single-bucket LSM spec.""" + """A table with ``id`` as the primary key and a single-bucket MemWAL spec.""" db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") # num_buckets = 1: every row routes to the single bucket. - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1)) + table.enable_memwal(MemWalWriteSpec.bucket("id", 1)) return table @@ -78,7 +78,7 @@ def test_lsm_merge_insert_unsharded(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.enable_memwal(MemWalWriteSpec.unsharded()) result = ( table.merge_insert("id") .when_matched_update_all() @@ -92,7 +92,7 @@ def test_lsm_merge_insert_identity(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _region_reader([(1, "us"), (2, "us")])) table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.identity("region")) + table.enable_memwal(MemWalWriteSpec.identity("region")) # All rows share one identity value, so they route to one shard. result = ( table.merge_insert([]) @@ -103,31 +103,31 @@ def test_lsm_merge_insert_identity(tmp_path): assert result.num_rows == 2 -def test_lsm_merge_insert_use_lsm_false(tmp_path): +def test_lsm_merge_insert_use_memwal_false(tmp_path): table = _bucket_table(tmp_path) # rows id = 1, 2, 3 - # use_lsm(False) opts out: the standard path runs and commits even with a spec. + # use_memwal(False) opts out: the standard path runs and commits even with a spec. result = ( table.merge_insert("id") .when_not_matched_insert_all() - .use_lsm(False) + .use_memwal(False) .execute(_reader([3, 4, 5])) ) assert result.num_inserted_rows == 2 assert table.count_rows() == 5 -def test_lsm_merge_insert_use_lsm_true_without_spec_errors(tmp_path): - # A table with a primary key but no LSM write spec installed. +def test_lsm_merge_insert_use_memwal_true_without_spec_errors(tmp_path): + # A table with a primary key but no MemWAL write spec installed. db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - # use_lsm(True) demands MemWAL routing; without a spec it errors. - with pytest.raises(Exception, match="use_lsm"): + # use_memwal(True) demands MemWAL routing; without a spec it errors. + with pytest.raises(Exception, match="use_memwal"): ( table.merge_insert("id") .when_matched_update_all() .when_not_matched_insert_all() - .use_lsm(True) + .use_memwal(True) .execute(_reader([3, 4, 5])) ) @@ -145,7 +145,7 @@ def test_lsm_merge_insert_validate_single_shard_off(tmp_path): def test_lsm_merge_insert_no_spec_uses_standard_path(tmp_path): - # A table with a primary key but no LSM write spec installed. + # A table with a primary key but no MemWAL write spec installed. db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") @@ -186,7 +186,7 @@ def test_lsm_close_writers(tmp_path): .when_not_matched_insert_all() .execute(_reader([7, 8])) ) - table.close_lsm_writers() + table.close_memwal_writers() # The writer reopens lazily on the next merge_insert. result = ( table.merge_insert([]) @@ -204,14 +204,14 @@ async def test_async_lsm_merge_insert(tmp_path): ) table = await db.create_table("t", _reader([1, 2, 3])) await table.set_unenforced_primary_key("id") - await table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1)) + await table.enable_memwal(MemWalWriteSpec.bucket("id", 1)) builder = ( table.merge_insert([]).when_matched_update_all().when_not_matched_insert_all() ) result = await builder.execute(_reader([3, 4, 5])) assert result.num_rows == 3 - await table.close_lsm_writers() + await table.close_memwal_writers() def _lsm_upsert(table, ids): @@ -228,7 +228,7 @@ def test_lsm_read_sees_active_memtable(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) # base ids 1,2,3 table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.enable_memwal(MemWalWriteSpec.unsharded()) _lsm_upsert(table, [4, 5]) # active memtable only, not committed to base @@ -236,8 +236,8 @@ def test_lsm_read_sees_active_memtable(tmp_path): lsm = table.search().to_arrow() assert sorted(lsm["id"].to_pylist()) == [1, 2, 3, 4, 5] - # use_lsm(False) bypasses the MemWAL and reads the base table only. - base_only = table.search().use_lsm(False).to_arrow() + # use_memwal(False) bypasses the MemWAL and reads the base table only. + base_only = table.search().use_memwal(False).to_arrow() assert sorted(base_only["id"].to_pylist()) == [1, 2, 3] @@ -245,7 +245,7 @@ def test_lsm_read_dedup_newest_wins(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) # id 2 -> value 1 table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.enable_memwal(MemWalWriteSpec.unsharded()) _lsm_upsert(table, [2, 3, 4]) # ids 2,3,4 -> values 0,1,2 @@ -258,22 +258,22 @@ def test_lsm_read_dedup_newest_wins(tmp_path): def test_lsm_read_without_spec_reads_base(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) - table.set_unenforced_primary_key("id") # no LSM write spec + table.set_unenforced_primary_key("id") # no MemWAL write spec - # No spec: default read and use_lsm(False) both read the base table, no error. + # No spec: default read and use_memwal(False) both read the base table, no error. assert sorted(table.search().to_arrow()["id"].to_pylist()) == [1, 2, 3] - assert sorted(table.search().use_lsm(False).to_arrow()["id"].to_pylist()) == [ + assert sorted(table.search().use_memwal(False).to_arrow()["id"].to_pylist()) == [ 1, 2, 3, ] -def test_lsm_read_unsupported_shape_errors_without_use_lsm_false(tmp_path): +def test_lsm_read_unsupported_shape_errors_without_use_memwal_false(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.enable_memwal(MemWalWriteSpec.unsharded()) _lsm_upsert(table, [4]) # with_row_id is unsupported by the LSM scanner; on a MemWAL table the default @@ -281,8 +281,8 @@ def test_lsm_read_unsupported_shape_errors_without_use_lsm_false(tmp_path): with pytest.raises(Exception): table.search().with_row_id(True).to_arrow() - # use_lsm(False) is the escape hatch: it reads the base table only. - base = table.search().with_row_id(True).use_lsm(False).to_arrow() + # use_memwal(False) is the escape hatch: it reads the base table only. + base = table.search().with_row_id(True).use_memwal(False).to_arrow() assert sorted(base["id"].to_pylist()) == [1, 2, 3] @@ -293,7 +293,7 @@ async def test_async_lsm_read(tmp_path): ) table = await db.create_table("t", _reader([1, 2, 3])) await table.set_unenforced_primary_key("id") - await table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + await table.enable_memwal(MemWalWriteSpec.unsharded()) builder = ( table.merge_insert([]).when_matched_update_all().when_not_matched_insert_all() @@ -331,7 +331,7 @@ def _vector_reader(rows): def _vector_table(tmp_path): """Base table whose vector column is indexed so its rows are visible to the LSM vector scanner (the base arm uses ``fast_search`` — indexed data only), plus an - unsharded LSM spec that maintains that index for the memtable. + unsharded MemWAL spec that maintains that index for the memtable. Rows 1,2 are category ``a``, row 3 is ``b``, and 4..60 are filler ``c`` that give the tiny IVF index enough data to train. @@ -353,8 +353,8 @@ def _vector_table(tmp_path): "vector", config=IvfPq(num_partitions=1, num_sub_vectors=2, num_bits=4) ) index_name = table.list_indices()[0].name - table.set_lsm_write_spec( - LsmWriteSpec.unsharded().with_maintained_indexes([index_name]) + table.enable_memwal( + MemWalWriteSpec.unsharded().with_maintained_indexes([index_name]) ) return table @@ -379,9 +379,9 @@ def test_lsm_read_vector_sees_memtable(tmp_path): assert {1, 2, 3} <= ids # indexed base rows assert 1000 in ids # in-flight memtable row - # use_lsm(False) bypasses the MemWAL, so the in-flight row is not visible. + # use_memwal(False) bypasses the MemWAL, so the in-flight row is not visible. base_ids = set( - table.search(query).use_lsm(False).limit(100).to_arrow()["id"].to_pylist() + table.search(query).use_memwal(False).limit(100).to_arrow()["id"].to_pylist() ) assert {1, 2, 3} <= base_ids assert 1000 not in base_ids @@ -453,8 +453,8 @@ def test_lsm_read_fts_sees_memtable(tmp_path): # Native FTS index (tantivy is not compatible with the LSM memtable index). table.create_index("text", config=FTS()) index_name = table.list_indices()[0].name - table.set_lsm_write_spec( - LsmWriteSpec.unsharded().with_maintained_indexes([index_name]) + table.enable_memwal( + MemWalWriteSpec.unsharded().with_maintained_indexes([index_name]) ) # in-flight doc 4 lands in the memtable's maintained FTS index. @@ -498,8 +498,8 @@ def test_lsm_read_vector_unsupported_knobs_error(tmp_path): with pytest.raises(Exception, match="use_index"): table.search(query).bypass_vector_index().to_arrow() - # use_lsm(False) is the escape hatch: the base-only standard path honors them. - base = table.search(query).distance_range(0.0, 100.0).use_lsm(False).to_arrow() + # use_memwal(False) is the escape hatch: the base-only standard path honors them. + base = table.search(query).distance_range(0.0, 100.0).use_memwal(False).to_arrow() assert 1000 not in set(base["id"].to_pylist()) @@ -544,7 +544,7 @@ def test_lsm_read_fts_unmaintained_index_errors(tmp_path): table.create_index("text", config=FTS()) # No maintained indexes: the active memtable FTS arm cannot serve un-compacted # docs, so the search would silently omit them — reject instead. - table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) + table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([])) with pytest.raises(Exception, match="maintained"): table.search("fox", query_type="fts", fts_columns="text").to_arrow() @@ -553,7 +553,7 @@ def test_lsm_read_time_travel_errors(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.enable_memwal(MemWalWriteSpec.unsharded()) pinned = table.version table.add(_reader([4, 5])) # standard add commits a newer version table.checkout(pinned) # detached head at the historical version @@ -562,8 +562,8 @@ def test_lsm_read_time_travel_errors(tmp_path): # historical version is rejected. with pytest.raises(Exception, match="time-travel"): table.search().to_arrow() - # use_lsm(False) reads the base table at the pinned version. - base = table.search().use_lsm(False).to_arrow() + # use_memwal(False) reads the base table at the pinned version. + base = table.search().use_memwal(False).to_arrow() assert sorted(base["id"].to_pylist()) == [1, 2, 3] @@ -571,14 +571,14 @@ def test_lsm_read_take_row_ids_errors(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.enable_memwal(MemWalWriteSpec.unsharded()) _lsm_upsert(table, [4]) # take-by-row-id auto-routes through the LSM scanner, which has no stable _rowid, # so it hard-errors instead of failing with an opaque column-not-found error. with pytest.raises(Exception, match="row id"): table.take_row_ids([0, 1]).to_arrow() - # use_lsm(False) is the escape hatch: it reads the base table. - base = table.take_row_ids([0, 1]).use_lsm(False).to_arrow() + # use_memwal(False) is the escape hatch: it reads the base table. + base = table.take_row_ids([0, 1]).use_memwal(False).to_arrow() assert base.num_rows == 2 @@ -588,8 +588,8 @@ def test_lsm_read_fts_postfilter_errors(tmp_path): table.set_unenforced_primary_key("id") table.create_index("text", config=FTS()) index_name = table.list_indices()[0].name - table.set_lsm_write_spec( - LsmWriteSpec.unsharded().with_maintained_indexes([index_name]) + table.enable_memwal( + MemWalWriteSpec.unsharded().with_maintained_indexes([index_name]) ) # The LSM scanner always prefilters; postfilter on FTS changes result semantics, # so it hard-errors (previously only the vector arm rejected it). @@ -608,9 +608,7 @@ def test_lsm_read_fts_multiple_same_type_indexes_errors(tmp_path): table.set_unenforced_primary_key("id") table.create_index("text", config=FTS(), name="fts_a") table.create_index("text", config=FTS(), name="fts_b", replace=False) - table.set_lsm_write_spec( - LsmWriteSpec.unsharded().with_maintained_indexes(["fts_a"]) - ) + table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes(["fts_a"])) # Two FTS indexes on the column: the base planner's chosen index is ambiguous, so # the scanner can't pick a catch-up watermark and rejects rather than risk # dropping rows the actually-used index has not caught up to. @@ -631,7 +629,7 @@ def test_lsm_read_vector_unmaintained_index_errors(tmp_path): ) # Spec with NO maintained indexes: the base vector index's catch-up is untracked, # so the scanner rejects rather than risk dropping compacted-but-unindexed rows. - table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) + table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([])) with pytest.raises(Exception, match="maintained"): table.search([1.0] * VECTOR_DIM).to_arrow() @@ -644,7 +642,7 @@ def test_lsm_read_fts_optimized_index_not_rejected(tmp_path): table.add(_fts_reader([(i, "lazy fox") for i in range(6, 11)])) table.optimize() # may split the FTS index into multiple physical segments name = table.list_indices()[0].name - table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([name])) + table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([name])) # Multiple physical segments of one logical index must not be miscounted as # multiple indexes and rejected. ids = set( diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index c22aa78cd..6ee41aa0c 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1845,10 +1845,10 @@ def lsm_test_table(lsm_handler): """A remote table whose LSM routes are served by ``lsm_handler``. ``lsm_handler(request, route)`` is called for ``/v1/table/test//`` - where route is one of flush_lsm, compact_lsm, get_lsm_stats, and is + where route is one of flush_memwal, compact_memwal, get_memwal_stats, and is responsible for writing the response. """ - routes = ("flush_lsm", "compact_lsm", "get_lsm_stats") + routes = ("flush_memwal", "compact_memwal", "get_memwal_stats") def handler(request): match = re.fullmatch(r"/v1/table/test/(\w+)/", request.path) @@ -1880,9 +1880,9 @@ def send_json(request, payload, status=200): request.wfile.write(json.dumps(payload).encode()) -def test_get_lsm_stats_sync(): +def test_get_memwal_stats_sync(): """The sync wrapper round-trips the server payload into a dict.""" - bucket = { + table_shard = { "shard_id": "b0", "status": "Active", "writer_epoch": 3, @@ -1890,7 +1890,7 @@ def test_get_lsm_stats_sync(): "current_generation": 6, "replay_after_wal_entry_position": 40, "wal_entry_position_last_seen": 42, - "generations": [{"generation": 5, "bytes": 1024, "rows": 7}], + "sstables": [{"generation": 5, "bytes": 1024, "rows": 7}], "compacting": False, "memtables": [ { @@ -1905,29 +1905,29 @@ def test_get_lsm_stats_sync(): seen_bodies = [] def lsm_handler(request, route): - assert route == "get_lsm_stats" + assert route == "get_memwal_stats" seen_bodies.append(read_json_body(request)) - send_json(request, {"lsm_stats": {"buckets": [bucket]}}) + send_json(request, {"memwal_stats": {"table_shards": [table_shard]}}) with lsm_test_table(lsm_handler) as table: - assert table.get_lsm_stats() == {"buckets": [bucket]} + assert table.get_memwal_stats() == {"table_shards": [table_shard]} # Off by default, and forwarded when asked for. - assert seen_bodies == [{"include_generation_rows": False}] - table.get_lsm_stats(include_generation_rows=True) - assert seen_bodies[-1] == {"include_generation_rows": True} + assert seen_bodies == [{"include_sstable_rows": False}] + table.get_memwal_stats(include_sstable_rows=True) + assert seen_bodies[-1] == {"include_sstable_rows": True} -def test_get_lsm_stats_sync_returns_none_when_lsm_disabled(): - """A null envelope means the LSM write path is not enabled, not an error.""" +def test_get_memwal_stats_sync_returns_none_when_memwal_disabled(): + """A null envelope means the MemWAL write path is not enabled, not an error.""" def lsm_handler(request, route): - send_json(request, {"lsm_stats": None}) + send_json(request, {"memwal_stats": None}) with lsm_test_table(lsm_handler) as table: - assert table.get_lsm_stats() is None + assert table.get_memwal_stats() is None -def test_flush_and_compact_lsm_sync(): +def test_flush_and_compact_memwal_sync(): """Both are one-shot POSTs answered 202 with no body.""" called = [] @@ -1937,13 +1937,13 @@ def test_flush_and_compact_lsm_sync(): request.end_headers() with lsm_test_table(lsm_handler) as table: - assert table.flush_lsm() is None - assert table.compact_lsm() is None - assert called == ["flush_lsm", "compact_lsm"] + assert table.flush_memwal() is None + assert table.compact_memwal() is None + assert called == ["flush_memwal", "compact_memwal"] -def test_checkpoint_lsm_sync(): - """Seal, read the watermark, and return once L0 holds nothing. +def test_checkpoint_memwal_sync(): + """Freeze, read the watermark, and return once no SSTables remain. The convergence loop itself is covered in Rust; this pins the sync binding to the endpoints it drives. @@ -1952,17 +1952,17 @@ def test_checkpoint_lsm_sync(): def lsm_handler(request, route): called.append(route) - if route == "get_lsm_stats": - # An empty L0 yields no target watermark, so the loop is done - # after the seal without ever polling compaction. - send_json(request, {"lsm_stats": {"buckets": []}}) + if route == "get_memwal_stats": + # An empty SSTable tier yields no target watermark, so the loop is done + # after the freeze without ever polling compaction. + send_json(request, {"memwal_stats": {"table_shards": []}}) else: request.send_response(202) request.end_headers() with lsm_test_table(lsm_handler) as table: - assert table.checkpoint_lsm() is None - assert called == ["flush_lsm", "get_lsm_stats"] + assert table.checkpoint_memwal() is None + assert called == ["flush_memwal", "get_memwal_stats"] @contextlib.contextmanager diff --git a/python/python/tests/test_wal_hybrid.py b/python/python/tests/test_wal_hybrid.py index 9c7efe151..675fc927a 100644 --- a/python/python/tests/test_wal_hybrid.py +++ b/python/python/tests/test_wal_hybrid.py @@ -81,15 +81,15 @@ def test_base_table_hybrid_still_joins_on_row_ids(): assert builder._vector_query._columns is None -def test_wal_hybrid_with_use_lsm_false_joins_on_row_ids(): - builder = _wal_hybrid().use_lsm(False) +def test_wal_hybrid_with_use_memwal_false_joins_on_row_ids(): + builder = _wal_hybrid().use_memwal(False) builder._create_query_builders() assert builder._pk_fusion is None def test_wal_hybrid_refuses_an_explicit_with_row_id(): builder = _wal_hybrid().with_row_id(True) - with pytest.raises(NotImplementedError, match="use_lsm"): + with pytest.raises(NotImplementedError, match="use_memwal"): builder._create_query_builders() @@ -141,7 +141,7 @@ def test_wal_hybrid_refuses_row_id_even_with_return_score_all(row_id_first: bool else: builder = builder.rerank(reranker).with_row_id(True) - with pytest.raises(NotImplementedError, match="use_lsm"): + with pytest.raises(NotImplementedError, match="use_memwal"): builder._create_query_builders() @@ -199,7 +199,7 @@ def test_fallback_explains_the_refusal_to_a_caller_who_asked_for_row_ids(): def run(): raise _REFUSAL - with pytest.raises(NotImplementedError, match="use_lsm") as info: + with pytest.raises(NotImplementedError, match="use_memwal") as info: with_pk_fallback( run, mock.Mock(), fused=lambda: False, caller_requested_row_id=True ) diff --git a/python/python/tests/utils.py b/python/python/tests/utils.py index 4882f0c28..b3548b678 100644 --- a/python/python/tests/utils.py +++ b/python/python/tests/utils.py @@ -73,10 +73,10 @@ class MockPermutationServer: }, }, ) - if path == f"/v1/table/{self.name}/get_lsm_write_spec/": + if path == f"/v1/table/{self.name}/get_memwal_spec/": self._read_body(request) - # Null spec: this table has no LSM write path. - return self._json(request, {"lsm_write_spec": None}) + # Null spec: this table has no MemWAL write path. + return self._json(request, {"memwal_spec": None}) if path == f"/v1/table/{self.name}/count_rows/": self._read_body(request) return self._json(request, self.num_rows) diff --git a/python/src/lib.rs b/python/src/lib.rs index 066e85927..8dd98d019 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -16,7 +16,7 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery}; use session::Session; use table::{ AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken, - FunctionErrorFragment, FunctionErrorRecord, FunctionErrors, LsmWriteSpec, MergeResult, + FunctionErrorFragment, FunctionErrorRecord, FunctionErrors, MemWalWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult, Table, UpdateFieldMetadataResult, UpdateResult, }; @@ -104,7 +104,7 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/src/query.rs b/python/src/query.rs index e5547881a..4f23fa1b4 100644 --- a/python/src/query.rs +++ b/python/src/query.rs @@ -333,7 +333,7 @@ pub struct PyQueryRequest { pub select_source_columns: Option>, pub fast_search: Option, pub with_row_id: Option, - pub use_lsm: Option, + pub use_memwal: Option, pub column: Option, pub query_vector: Option, pub nprobes: Option, @@ -366,7 +366,7 @@ impl From for PyQueryRequest { select: PySelect(query_request.select), fast_search: Some(query_request.fast_search), with_row_id: Some(query_request.with_row_id), - use_lsm: query_request.use_lsm, + use_memwal: query_request.use_memwal, column: None, query_vector: None, nprobes: None, @@ -395,7 +395,7 @@ impl From for PyQueryRequest { select: PySelect(vector_query.base.select), fast_search: Some(vector_query.base.fast_search), with_row_id: Some(vector_query.base.with_row_id), - use_lsm: vector_query.base.use_lsm, + use_memwal: vector_query.base.use_memwal, column: vector_query.column, query_vector: Some(PyQueryVectors(vector_query.query_vector)), nprobes: vector_query.nprobes, @@ -540,8 +540,8 @@ impl Query { self.inner = self.inner.clone().fast_search(); } - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } pub fn with_row_id(&mut self) { @@ -706,8 +706,8 @@ impl TakeQuery { self.inner = self.inner.clone().with_row_id(); } - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } #[pyo3(signature = ())] @@ -819,8 +819,8 @@ impl FTSQuery { self.inner = self.inner.clone().fast_search(); } - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } pub fn with_row_id(&mut self) { @@ -970,8 +970,8 @@ impl VectorQuery { self.inner = self.inner.clone().fast_search(); } - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } pub fn with_row_id(&mut self) { @@ -1178,9 +1178,9 @@ impl HybridQuery { self.inner_fts.postfilter(); } - pub fn use_lsm(&mut self, enable: bool) { - self.inner_vec.use_lsm(enable); - self.inner_fts.use_lsm(enable); + pub fn use_memwal(&mut self, enable: bool) { + self.inner_vec.use_memwal(enable); + self.inner_fts.use_memwal(enable); } pub fn add_query_vector(&mut self, vector: Bound<'_, PyAny>) -> PyResult<()> { diff --git a/python/src/table.rs b/python/src/table.rs index 23b48e3fd..c4053cc53 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -39,16 +39,16 @@ use pyo3::{ mod scannable; -/// Convert `LsmStats` to a Python dict, preserving the per-bucket list. +/// Convert `MemWalStats` to a Python dict, preserving the per-table-shard list. /// /// Deliberately not flattened to a table-level summary: a table is N -/// buckets on one node, and the per-bucket detail is the reason the -/// endpoint exists — flattening hides the single hot bucket someone opened +/// table shards on one node, and the per-shard detail is the reason the +/// endpoint exists — flattening hides the single hot table shard someone opened /// it to find. -fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult> { +fn memwal_stats_to_py(py: Python<'_>, stats: &lancedb::table::MemWalStats) -> PyResult> { let out = PyDict::new(py); - let buckets = PyList::empty(py); - for b in &stats.buckets { + let table_shards = PyList::empty(py); + for b in &stats.table_shards { let e = PyDict::new(py); e.set_item("shard_id", &b.shard_id)?; e.set_item("status", &b.status)?; @@ -64,15 +64,15 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult b.wal_entry_position_last_seen, )?; - let generations = PyList::empty(py); - for g in &b.generations { + let sstables = PyList::empty(py); + for g in &b.sstables { let ge = PyDict::new(py); ge.set_item("generation", g.generation)?; ge.set_item("bytes", g.bytes)?; ge.set_item("rows", g.rows)?; - generations.append(ge)?; + sstables.append(ge)?; } - e.set_item("generations", generations)?; + e.set_item("sstables", sstables)?; e.set_item("compacting", b.compacting)?; e.set_item( @@ -94,9 +94,9 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult }) .transpose()?, )?; - buckets.append(e)?; + table_shards.append(e)?; } - out.set_item("buckets", buckets)?; + out.set_item("table_shards", table_shards)?; Ok(out.unbind()) } @@ -270,17 +270,17 @@ fn fmt_maintained(maintained: &Option>) -> String { /// table has, including ones created later. #[pyclass(module = "lancedb._lancedb", from_py_object)] #[derive(Clone, Debug)] -pub struct LsmWriteSpec { - inner: lancedb::table::LsmWriteSpec, +pub struct MemWalWriteSpec { + inner: lancedb::table::MemWalWriteSpec, } #[pymethods] -impl LsmWriteSpec { +impl MemWalWriteSpec { /// Hash-bucket sharding by the unenforced primary key column. #[staticmethod] pub fn bucket(column: String, num_buckets: u32) -> Self { Self { - inner: lancedb::table::LsmWriteSpec::bucket(column, num_buckets), + inner: lancedb::table::MemWalWriteSpec::bucket(column, num_buckets), } } @@ -294,7 +294,7 @@ impl LsmWriteSpec { #[staticmethod] pub fn identity(column: String) -> Self { Self { - inner: lancedb::table::LsmWriteSpec::identity(column), + inner: lancedb::table::MemWalWriteSpec::identity(column), } } @@ -303,7 +303,7 @@ impl LsmWriteSpec { #[staticmethod] pub fn unsharded() -> Self { Self { - inner: lancedb::table::LsmWriteSpec::unsharded(), + inner: lancedb::table::MemWalWriteSpec::unsharded(), } } @@ -327,33 +327,33 @@ impl LsmWriteSpec { pub fn __repr__(&self) -> String { match &self.inner { - lancedb::table::LsmWriteSpec::Bucket { + lancedb::table::MemWalWriteSpec::Bucket { column, num_buckets, maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})", + "MemWalWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})", column, num_buckets, fmt_maintained(maintained_indexes), writer_config_defaults, ), - lancedb::table::LsmWriteSpec::Identity { + lancedb::table::MemWalWriteSpec::Identity { column, maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})", + "MemWalWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})", column, fmt_maintained(maintained_indexes), writer_config_defaults, ), - lancedb::table::LsmWriteSpec::Unsharded { + lancedb::table::MemWalWriteSpec::Unsharded { maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})", + "MemWalWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})", fmt_maintained(maintained_indexes), writer_config_defaults, ), @@ -365,9 +365,9 @@ impl LsmWriteSpec { #[getter] pub fn spec_type(&self) -> &'static str { match &self.inner { - lancedb::table::LsmWriteSpec::Bucket { .. } => "bucket", - lancedb::table::LsmWriteSpec::Identity { .. } => "identity", - lancedb::table::LsmWriteSpec::Unsharded { .. } => "unsharded", + lancedb::table::MemWalWriteSpec::Bucket { .. } => "bucket", + lancedb::table::MemWalWriteSpec::Identity { .. } => "identity", + lancedb::table::MemWalWriteSpec::Unsharded { .. } => "unsharded", } } @@ -375,9 +375,9 @@ impl LsmWriteSpec { #[getter] pub fn column(&self) -> Option { match &self.inner { - lancedb::table::LsmWriteSpec::Bucket { column, .. } - | lancedb::table::LsmWriteSpec::Identity { column, .. } => Some(column.clone()), - lancedb::table::LsmWriteSpec::Unsharded { .. } => None, + lancedb::table::MemWalWriteSpec::Bucket { column, .. } + | lancedb::table::MemWalWriteSpec::Identity { column, .. } => Some(column.clone()), + lancedb::table::MemWalWriteSpec::Unsharded { .. } => None, } } @@ -385,7 +385,7 @@ impl LsmWriteSpec { #[getter] pub fn num_buckets(&self) -> Option { match &self.inner { - lancedb::table::LsmWriteSpec::Bucket { num_buckets, .. } => Some(*num_buckets), + lancedb::table::MemWalWriteSpec::Bucket { num_buckets, .. } => Some(*num_buckets), _ => None, } } @@ -403,14 +403,14 @@ impl LsmWriteSpec { } } -impl From for lancedb::table::LsmWriteSpec { - fn from(spec: LsmWriteSpec) -> Self { +impl From for lancedb::table::MemWalWriteSpec { + fn from(spec: MemWalWriteSpec) -> Self { spec.inner } } -impl From for LsmWriteSpec { - fn from(inner: lancedb::table::LsmWriteSpec) -> Self { +impl From for MemWalWriteSpec { + fn from(inner: lancedb::table::MemWalWriteSpec) -> Self { Self { inner } } } @@ -1636,8 +1636,8 @@ impl Table { if let Some(use_index) = parameters.use_index { builder.use_index(use_index); } - if let Some(use_lsm) = parameters.use_lsm { - builder.use_lsm(use_lsm); + if let Some(use_memwal) = parameters.use_memwal { + builder.use_memwal(use_memwal); } if let Some(validate_single_shard) = parameters.validate_single_shard { builder.validate_single_shard(validate_single_shard); @@ -1662,29 +1662,29 @@ impl Table { }) } - pub fn set_lsm_write_spec<'a>( + pub fn enable_memwal<'a>( self_: PyRef<'a, Self>, - spec: LsmWriteSpec, + spec: MemWalWriteSpec, ) -> PyResult> { let inner = self_.inner_ref()?.clone(); - let native_spec = lancedb::table::LsmWriteSpec::from(spec); + let native_spec = lancedb::table::MemWalWriteSpec::from(spec); future_into_py(self_.py(), async move { - inner.set_lsm_write_spec(native_spec).await.infer_error() + inner.enable_memwal(native_spec).await.infer_error() }) } - pub fn unset_lsm_write_spec(self_: PyRef<'_, Self>) -> PyResult> { + pub fn disable_memwal(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - inner.unset_lsm_write_spec().await.infer_error() + inner.disable_memwal().await.infer_error() }) } - pub fn get_lsm_write_spec(self_: PyRef<'_, Self>) -> PyResult> { + pub fn get_memwal_spec(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - let spec = inner.get_lsm_write_spec().await.infer_error()?; - Ok(spec.map(LsmWriteSpec::from)) + let spec = inner.get_memwal_spec().await.infer_error()?; + Ok(spec.map(MemWalWriteSpec::from)) }) } @@ -1703,55 +1703,54 @@ impl Table { Ok(()) } - /// Converge the table's LSM write path into its base table. + /// Converge the table's MemWAL write path into its base table. /// /// Best-effort: with writes flowing, new rows may land after the last /// pass. Errors if the table stops making progress. - pub fn checkpoint_lsm(self_: PyRef<'_, Self>) -> PyResult> { + pub fn checkpoint_memwal(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - inner.checkpoint_lsm().await.infer_error() + inner.checkpoint_memwal().await.infer_error() }) } - /// Seal every bucket's active memtable into L0. - pub fn flush_lsm(self_: PyRef<'_, Self>) -> PyResult> { - let inner = self_.inner_ref()?.clone(); - future_into_py( - self_.py(), - async move { inner.flush_lsm().await.infer_error() }, - ) - } - - /// Trigger a background L0 → base pass per bucket. Returns once the - /// passes are dispatched, not once they finish — watch `get_lsm_stats`. - pub fn compact_lsm(self_: PyRef<'_, Self>) -> PyResult> { + /// Freeze every table shard's active memtable into an SSTable. + pub fn flush_memwal(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - inner.compact_lsm().await.infer_error() + inner.flush_memwal().await.infer_error() }) } - /// Live LSM state, or `None` when the LSM write path is not enabled. - #[pyo3(signature = (include_generation_rows=false))] - pub fn get_lsm_stats( + /// Trigger a background SSTable compaction pass per table shard. Returns once the + /// passes are dispatched, not once they finish — watch `get_memwal_stats`. + pub fn compact_memwal(self_: PyRef<'_, Self>) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + future_into_py(self_.py(), async move { + inner.compact_memwal().await.infer_error() + }) + } + + /// Live MemWAL state, or `None` when the MemWAL write path is not enabled. + #[pyo3(signature = (include_sstable_rows=false))] + pub fn get_memwal_stats( self_: PyRef<'_, Self>, - include_generation_rows: bool, + include_sstable_rows: bool, ) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { let stats = inner - .get_lsm_stats(include_generation_rows) + .get_memwal_stats(include_sstable_rows) .await .infer_error()?; - Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose()) + Python::attach(|py| stats.map(|s| memwal_stats_to_py(py, &s)).transpose()) }) } - pub fn close_lsm_writers(self_: PyRef<'_, Self>) -> PyResult> { + pub fn close_memwal_writers(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - inner.close_lsm_writers().await.infer_error() + inner.close_memwal_writers().await.infer_error() }) } @@ -2083,7 +2082,7 @@ pub struct MergeInsertParams { when_not_matched_by_source_condition_expr: Option, timeout: Option, use_index: Option, - use_lsm: Option, + use_memwal: Option, validate_single_shard: Option, } diff --git a/rust/lancedb/src/dataloader/permutation/builder.rs b/rust/lancedb/src/dataloader/permutation/builder.rs index ba0d0e180..e08f3eddc 100644 --- a/rust/lancedb/src/dataloader/permutation/builder.rs +++ b/rust/lancedb/src/dataloader/permutation/builder.rs @@ -250,17 +250,17 @@ impl PermutationBuilder { } // Unflushed rows have no row id, so a permutation cannot address them. - match self.base_table.base_table().get_lsm_write_spec().await { + match self.base_table.base_table().get_memwal_spec().await { Ok(Some(_)) => { return Err(Error::NotSupported { - message: "the data loader does not support tables with an LSM write \ + message: "the data loader does not support tables with a MemWAL write \ spec: rows that have not been flushed to the base table \ have no row id, so a permutation cannot reference them" .to_string(), }); } Ok(None) => {} - // No LSM write path means no spec. + // No MemWAL write path means no spec. Err(Error::NotSupported { .. }) => {} Err(err) => return Err(err), } @@ -489,9 +489,9 @@ mod tests { ) .unwrap() } - "/v1/table/remote_base/get_lsm_write_spec/" => http::Response::builder() + "/v1/table/remote_base/get_memwal_spec/" => http::Response::builder() .status(200) - .body(br#"{"lsm_write_spec":null}"#.to_vec()) + .body(br#"{"memwal_spec":null}"#.to_vec()) .unwrap(), "/v1/table/remote_base/count_rows/" => { let body = body.unwrap(); @@ -737,8 +737,8 @@ mod tests { /// permutation cannot reference them. Reading the base table alone would drop /// them from training without saying so, so the table is refused instead. #[tokio::test] - async fn test_permutation_rejects_lsm_write_spec() { - use crate::table::LsmWriteSpec; + async fn test_permutation_rejects_memwal_spec() { + use crate::table::MemWalWriteSpec; use arrow_array::{Int32Array, RecordBatchIterator}; use arrow_schema::{DataType, Field, Schema}; @@ -766,13 +766,13 @@ mod tests { table.set_unenforced_primary_key(["idx"]).await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); let err = PermutationBuilder::new(table).build().await.unwrap_err(); assert!( - err.to_string().contains("LSM write spec"), + err.to_string().contains("MemWAL write spec"), "expected the pre-check to refuse the table, got: {err}" ); } diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index 7750f66b5..68b05c531 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -167,7 +167,7 @@ //! # } //! ``` -// The MemWAL LSM read path (`table::query::lsm`) deepens the `create_plan` future's +// The MemWAL read path (`table::query::lsm`) deepens the `create_plan` future's // type graph enough to overflow the default trait-recursion limit while evaluating // auto-traits (`Send`) through the Linux io_uring build's moka cache. Raise it. #![recursion_limit = "256"] diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index f9634837a..b4ff22397 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -288,13 +288,15 @@ fn refresh_conflict_filter(dataset: &Dataset) -> Result { Ok(filter.build()) } -/// Reject MemWAL/LSM state because an ordinary dataset scan cannot see its +/// Reject MemWAL/MemWAL state because an ordinary dataset scan cannot see its /// un-compacted tiers. pub(crate) async fn ensure_no_mem_wal(dataset: &Dataset, role: &str, name: &str) -> Result<()> { let retained = !dataset.list_mem_wal_latest_shard_ids().await?.is_empty(); if retained || dataset.mem_wal_index_details().await?.is_some() { return Err(Error::NotSupported { - message: format!("{role} '{name}' has an LSM write spec or retained un-compacted rows"), + message: format!( + "{role} '{name}' has a MemWAL write spec or retained un-compacted rows" + ), }); } Ok(()) diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index d66932847..5f072cad6 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -547,22 +547,22 @@ pub trait QueryBase { /// Control MemWAL read routing for this query. /// /// By default (unset), when the table carries a MemWAL write spec (see - /// [`crate::Table::set_lsm_write_spec`]), reads are routed through the LSM + /// [`crate::Table::enable_memwal`]), reads are routed through the LSM /// scanner so they also return data written via the `merge_insert` LSM path /// that has not yet been compacted into the base table (active/frozen - /// memtables and flushed generations); a table without a spec reads the base + /// memtables and SSTables); a table without a spec reads the base /// table. /// - /// - `use_lsm(true)` forces LSM routing and errors if the table has no + /// - `use_memwal(true)` forces MemWAL routing and errors if the table has no /// MemWAL write spec. - /// - `use_lsm(false)` bypasses the MemWAL and reads the base table only, + /// - `use_memwal(false)` bypasses the MemWAL and reads the base table only, /// even when a spec is present. /// /// Note: the LSM scanner does not support every query shape (e.g. reranking, /// hybrid search, `order_by`). On a MemWAL table those shapes error unless - /// `use_lsm(false)` is set, because a base-only read would silently + /// `use_memwal(false)` is set, because a base-only read would silently /// exclude un-compacted MemWAL data. - fn use_lsm(self, enable: bool) -> Self; + fn use_memwal(self, enable: bool) -> Self; } pub trait HasQuery { @@ -634,8 +634,8 @@ impl QueryBase for T { self } - fn use_lsm(mut self, enable: bool) -> Self { - self.mut_query().use_lsm = Some(enable); + fn use_memwal(mut self, enable: bool) -> Self { + self.mut_query().use_memwal = Some(enable); self } } @@ -900,17 +900,17 @@ pub struct QueryRequest { /// Controls MemWAL read routing. When unset (the default), a query against a /// table that carries a MemWAL write spec (see - /// [`crate::Table::set_lsm_write_spec`]) is routed through the LSM scanner so + /// [`crate::Table::enable_memwal`]) is routed through the LSM scanner so /// it also sees data written via the `merge_insert` LSM path that has not yet /// been compacted into the base table — the active and frozen in-memory - /// memtables and the flushed (L0) generations, deduplicated by primary key + /// memtables and the SSTables, deduplicated by primary key /// against the base table (newest generation wins); a table without a spec /// reads the base table. /// - /// - `Some(true)` forces LSM routing and errors if the table has no MemWAL + /// - `Some(true)` forces MemWAL routing and errors if the table has no MemWAL /// write spec. /// - `Some(false)` reads only the base table, bypassing the MemWAL. - pub use_lsm: Option, + pub use_memwal: Option, } impl Default for QueryRequest { @@ -930,7 +930,7 @@ impl Default for QueryRequest { norm: None, disable_scoring_autoprojection: false, order_by: None, - use_lsm: None, + use_memwal: None, } } } diff --git a/rust/lancedb/src/query/wal_fusion.rs b/rust/lancedb/src/query/wal_fusion.rs index b438f6150..e8b6b3269 100644 --- a/rust/lancedb/src/query/wal_fusion.rs +++ b/rust/lancedb/src/query/wal_fusion.rs @@ -54,7 +54,7 @@ fn wal_row_id_unsupported() -> Error { Error::NotSupported { message: "hybrid search on a MemWAL table cannot return _rowid: the fresh tier has no \ stable row id, and the ids the fusion joins on are synthesized from the \ - primary key. Set use_lsm(false) to read the base table only (results will \ + primary key. Set use_memwal(false) to read the base table only (results will \ exclude un-compacted MemWAL data)" .to_string(), } @@ -80,7 +80,7 @@ where { // A previous query on this table was already refused, so skip straight to // the key. - if query.request.base.use_lsm != Some(false) && query.parent.hybrid_pk_fusion_learned() { + if query.request.base.use_memwal != Some(false) && query.parent.hybrid_pk_fusion_learned() { return run(Some(PkFusion::resolve(query).await?)).await; } match run(None).await { diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index d9da2c1a1..57843ca55 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -30,12 +30,12 @@ use crate::table::BranchDiff; use crate::table::CherryPickResult; use crate::table::DeleteResult; use crate::table::DropColumnsResult; -use crate::table::LsmStats; -use crate::table::LsmWriteSpec; +use crate::table::MemWalStats; +use crate::table::MemWalWriteSpec; use crate::table::MergeResult; use crate::table::Tags; use crate::table::UpdateResult; -use crate::table::lsm_stats::GetLsmStatsResponse; +use crate::table::memwal_stats::GetMemWalStatsResponse; use crate::table::merge::MergeFilter; use crate::table::primary_key; use crate::table::query::create_multi_vector_plan; @@ -1020,10 +1020,10 @@ impl RemoteTable { ) -> Result<()> { params.check_filter()?; body["prefilter"] = params.prefilter.into(); - // Only forward use_lsm when explicitly set; a server that predates it + // Only forward use_memwal when explicitly set; a server that predates it // ignores the field and routes as it would by default. - if let Some(use_lsm) = params.use_lsm { - body["use_lsm"] = serde_json::Value::Bool(use_lsm); + if let Some(use_memwal) = params.use_memwal { + body["use_memwal"] = serde_json::Value::Bool(use_memwal); } if let Some(offset) = params.offset { body["offset"] = serde_json::Value::Number(serde_json::Number::from(offset)); @@ -3227,42 +3227,43 @@ impl BaseTable for RemoteTable { Ok(()) } - async fn flush_lsm(&self) -> Result<()> { + async fn flush_memwal(&self) -> Result<()> { let request = self .client - .post(&format!("/v1/table/{}/flush_lsm/", self.identifier)); + .post(&format!("/v1/table/{}/flush_memwal/", self.identifier)); self.send_lsm_route(request).await?; Ok(()) } - async fn compact_lsm(&self) -> Result<()> { + async fn compact_memwal(&self) -> Result<()> { let request = self .client - .post(&format!("/v1/table/{}/compact_lsm/", self.identifier)); + .post(&format!("/v1/table/{}/compact_memwal/", self.identifier)); self.send_lsm_route(request).await?; Ok(()) } - async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result> { - // Read-semantics POST, like `get_lsm_write_spec`. + async fn get_memwal_stats(&self, include_sstable_rows: bool) -> Result> { + // Read-semantics POST, like `get_memwal_spec`. let request = self .client - .post(&format!("/v1/table/{}/get_lsm_stats/", self.identifier)) + .post(&format!("/v1/table/{}/get_memwal_stats/", self.identifier)) .json(&serde_json::json!({ - "include_generation_rows": include_generation_rows, + "include_sstable_rows": include_sstable_rows, })); let (request_id, response) = self.send_lsm_route(request).await?; let body = response.text().await.err_to_http(request_id.clone())?; - let parsed: GetLsmStatsResponse = serde_json::from_str(&body).map_err(|e| Error::Http { - source: format!("Failed to parse get_lsm_stats response: {e}").into(), - request_id, - status_code: None, - })?; - // `null` — and only — when the table has no LSM write path. - Ok(parsed.lsm_stats) + let parsed: GetMemWalStatsResponse = + serde_json::from_str(&body).map_err(|e| Error::Http { + source: format!("Failed to parse get_memwal_stats response: {e}").into(), + request_id, + status_code: None, + })?; + // `null` — and only — when the table has no MemWAL write path. + Ok(parsed.memwal_stats) } - async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> Result<()> { + async fn enable_memwal(&self, spec: MemWalWriteSpec) -> Result<()> { self.check_mutable().await?; // Map the spec onto the server's request DTO. `sharding` is internally @@ -3270,7 +3271,7 @@ impl BaseTable for RemoteTable { // `maintained_indexes` asks the server to resolve every maintainable // index at HEAD; a list is verbatim, an empty one meaning none. let sharding = match &spec { - LsmWriteSpec::Bucket { + MemWalWriteSpec::Bucket { column, num_buckets, .. @@ -3279,11 +3280,11 @@ impl BaseTable for RemoteTable { "column": column, "num_buckets": num_buckets, }), - LsmWriteSpec::Identity { column, .. } => serde_json::json!({ + MemWalWriteSpec::Identity { column, .. } => serde_json::json!({ "mode": "identity", "column": column, }), - LsmWriteSpec::Unsharded { .. } => serde_json::json!({ "mode": "unsharded" }), + MemWalWriteSpec::Unsharded { .. } => serde_json::json!({ "mode": "unsharded" }), }; let body = serde_json::json!({ "sharding": sharding, @@ -3293,45 +3294,40 @@ impl BaseTable for RemoteTable { let request = self .client - .post(&format!( - "/v1/table/{}/set_lsm_write_spec/", - self.identifier - )) + .post(&format!("/v1/table/{}/enable_memwal/", self.identifier)) .json(&body); let (request_id, response) = self.send(request, true).await?; self.check_table_response(&request_id, response).await?; Ok(()) } - async fn unset_lsm_write_spec(&self) -> Result<()> { + async fn disable_memwal(&self) -> Result<()> { self.check_mutable().await?; - let request = self.client.post(&format!( - "/v1/table/{}/unset_lsm_write_spec/", - self.identifier - )); + let request = self + .client + .post(&format!("/v1/table/{}/disable_memwal/", self.identifier)); let (request_id, response) = self.send(request, true).await?; self.check_table_response(&request_id, response).await?; self.wal_pk_fusion.forget(); // WAL-PK-FUSION: delete. Ok(()) } - async fn get_lsm_write_spec(&self) -> Result> { + async fn get_memwal_spec(&self) -> Result> { // Read counterpart to set/unset, resolved server-side against HEAD. The // server reads the spec from the `__lance_mem_wal` system index (shard // column mapped from its Lance field id against the current schema) and // re-encodes it into the same sophon-owned shape the set endpoint - // accepts — no lance/lancedb types cross the wire. `lsm_write_spec` is - // null when the LSM write path is not enabled for the table. - let request = self.client.post(&format!( - "/v1/table/{}/get_lsm_write_spec/", - self.identifier - )); + // accepts — no lance/lancedb types cross the wire. `memwal_spec` is + // null when the MemWAL write path is not enabled for the table. + let request = self + .client + .post(&format!("/v1/table/{}/get_memwal_spec/", self.identifier)); let (request_id, response) = self.send(request, true).await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; // Mirror of sophon's `Sharding` (internally tagged on `mode`) and - // `LsmWriteSpecBody` / `GetLsmWriteSpecResponse`. + // `MemWalWriteSpecBody` / `GetMemWalSpecResponse`. #[derive(Deserialize)] #[serde(tag = "mode", rename_all = "snake_case")] enum Sharding { @@ -3340,7 +3336,7 @@ impl BaseTable for RemoteTable { Identity { column: String }, } #[derive(Deserialize)] - struct LsmWriteSpecBody { + struct MemWalWriteSpecBody { sharding: Sharding, /// `null` selects every index the table has; `[]` selects none. #[serde(default)] @@ -3349,19 +3345,19 @@ impl BaseTable for RemoteTable { writer_config_defaults: std::collections::HashMap, } #[derive(Deserialize)] - struct GetLsmWriteSpecResponse { - lsm_write_spec: Option, + struct GetMemWalSpecResponse { + memwal_spec: Option, } - let parsed: GetLsmWriteSpecResponse = + let parsed: GetMemWalSpecResponse = serde_json::from_str(&body).map_err(|e| Error::Http { - source: format!("Failed to parse get_lsm_write_spec response: {}", e).into(), + source: format!("Failed to parse get_memwal_spec response: {}", e).into(), request_id, status_code: None, })?; - let Some(body) = parsed.lsm_write_spec else { - // The LSM write path is not enabled for this table. + let Some(body) = parsed.memwal_spec else { + // The MemWAL write path is not enabled for this table. return Ok(None); }; @@ -3369,9 +3365,9 @@ impl BaseTable for RemoteTable { Sharding::Bucket { column, num_buckets, - } => LsmWriteSpec::bucket(column, num_buckets), - Sharding::Identity { column } => LsmWriteSpec::identity(column), - Sharding::Unsharded => LsmWriteSpec::unsharded(), + } => MemWalWriteSpec::bucket(column, num_buckets), + Sharding::Identity { column } => MemWalWriteSpec::identity(column), + Sharding::Unsharded => MemWalWriteSpec::unsharded(), } .with_maintained_indexes(body.maintained_indexes) .with_writer_config_defaults(body.writer_config_defaults); @@ -3974,10 +3970,10 @@ pub struct MergeInsertRequest { // (the default is true) #[serde(skip_serializing_if = "is_true")] use_index: bool, - // Only serialize use_lsm when explicitly set (Some); a server that predates + // Only serialize use_memwal when explicitly set (Some); a server that predates // it ignores the field and routes as it would by default. #[serde(skip_serializing_if = "Option::is_none")] - use_lsm: Option, + use_memwal: Option, } impl MergeInsertRequest { @@ -4035,7 +4031,7 @@ impl TryFrom for MergeInsertRequest { when_not_matched_by_source_delete_filt, // Only serialize use_index when it's false for backwards compatibility use_index: value.use_index, - use_lsm: value.use_lsm, + use_memwal: value.use_memwal, }) } } @@ -10010,13 +10006,10 @@ mod tests { } #[tokio::test] - async fn test_set_lsm_write_spec_unsharded() { + async fn test_enable_memwal_unsharded() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!( - request.url().path(), - "/v1/table/my_table/set_lsm_write_spec/" - ); + assert_eq!(request.url().path(), "/v1/table/my_table/enable_memwal/"); let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!(body["sharding"], serde_json::json!({ "mode": "unsharded" })); @@ -10030,20 +10023,17 @@ mod tests { .body(r#"{"maintained_indexes":["id_idx"]}"#) .unwrap() }); - let spec = crate::table::LsmWriteSpec::unsharded() + let spec = crate::table::MemWalWriteSpec::unsharded() .with_maintained_indexes(vec!["id_idx".to_string()]) .with_writer_config_defaults([("max_memtable_rows", "1000")]); - table.set_lsm_write_spec(spec).await.unwrap(); + table.enable_memwal(spec).await.unwrap(); } #[tokio::test] - async fn test_set_lsm_write_spec_bucket() { + async fn test_enable_memwal_bucket() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!( - request.url().path(), - "/v1/table/my_table/set_lsm_write_spec/" - ); + assert_eq!(request.url().path(), "/v1/table/my_table/enable_memwal/"); let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!( @@ -10055,14 +10045,14 @@ mod tests { http::Response::builder().status(200).body("{}").unwrap() }); table - .set_lsm_write_spec(crate::table::LsmWriteSpec::bucket("id", 16)) + .enable_memwal(crate::table::MemWalWriteSpec::bucket("id", 16)) .await .unwrap(); } /// `[]` (none) must stay distinguishable on the wire from null (all). #[tokio::test] - async fn test_set_lsm_write_spec_no_maintained_indexes() { + async fn test_enable_memwal_no_maintained_indexes() { let table = Table::new_with_handler("my_table", |request| { let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); @@ -10070,21 +10060,18 @@ mod tests { http::Response::builder().status(200).body("{}").unwrap() }); table - .set_lsm_write_spec( - crate::table::LsmWriteSpec::bucket("id", 16).with_maintained_indexes(Vec::new()), + .enable_memwal( + crate::table::MemWalWriteSpec::bucket("id", 16).with_maintained_indexes(Vec::new()), ) .await .unwrap(); } #[tokio::test] - async fn test_set_lsm_write_spec_identity() { + async fn test_enable_memwal_identity() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!( - request.url().path(), - "/v1/table/my_table/set_lsm_write_spec/" - ); + assert_eq!(request.url().path(), "/v1/table/my_table/enable_memwal/"); let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!( @@ -10094,38 +10081,32 @@ mod tests { http::Response::builder().status(200).body("{}").unwrap() }); table - .set_lsm_write_spec(crate::table::LsmWriteSpec::identity("tenant")) + .enable_memwal(crate::table::MemWalWriteSpec::identity("tenant")) .await .unwrap(); } #[tokio::test] - async fn test_unset_lsm_write_spec() { + async fn test_disable_memwal() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!( - request.url().path(), - "/v1/table/my_table/unset_lsm_write_spec/" - ); + assert_eq!(request.url().path(), "/v1/table/my_table/disable_memwal/"); http::Response::builder().status(200).body("{}").unwrap() }); - table.unset_lsm_write_spec().await.unwrap(); + table.disable_memwal().await.unwrap(); } #[tokio::test] - async fn test_get_lsm_write_spec() { + async fn test_get_memwal_spec() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!( - request.url().path(), - "/v1/table/my_table/get_lsm_write_spec/" - ); + assert_eq!(request.url().path(), "/v1/table/my_table/get_memwal_spec/"); // The server resolves the spec and re-encodes it into the same // sophon-owned shape the set endpoint accepts (`Sharding` internally - // tagged on `mode`, wrapped in `lsm_write_spec`). + // tagged on `mode`, wrapped in `memwal_spec`). let response = serde_json::json!({ - "lsm_write_spec": { + "memwal_spec": { "sharding": { "mode": "bucket", "column": "id", "num_buckets": 4 }, "maintained_indexes": ["id_idx"], "writer_config_defaults": { "durable_write": "false" }, @@ -10138,12 +10119,12 @@ mod tests { }); let spec = table - .get_lsm_write_spec() + .get_memwal_spec() .await .unwrap() .expect("a spec should be reported"); match spec { - crate::table::LsmWriteSpec::Bucket { + crate::table::MemWalWriteSpec::Bucket { column, num_buckets, maintained_indexes, @@ -10170,13 +10151,13 @@ mod tests { #[case::no_index(serde_json::json!([]), Some(vec![]))] #[case::named(serde_json::json!(["id_idx"]), Some(vec!["id_idx".to_string()]))] #[tokio::test] - async fn test_get_lsm_write_spec_round_trips_the_selection( + async fn test_get_memwal_spec_round_trips_the_selection( #[case] reported: serde_json::Value, #[case] expected: Option>, ) { let table = Table::new_with_handler("my_table", move |_| { let response = serde_json::json!({ - "lsm_write_spec": { + "memwal_spec": { "sharding": { "mode": "unsharded" }, "maintained_indexes": reported, "writer_config_defaults": {}, @@ -10189,7 +10170,7 @@ mod tests { }); let spec = table - .get_lsm_write_spec() + .get_memwal_spec() .await .unwrap() .expect("a spec should be reported"); @@ -10201,35 +10182,32 @@ mod tests { } #[tokio::test] - async fn test_get_lsm_write_spec_absent() { + async fn test_get_memwal_spec_absent() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!( - request.url().path(), - "/v1/table/my_table/get_lsm_write_spec/" - ); - // Null spec → the LSM write path is not enabled. - let response = serde_json::json!({ "lsm_write_spec": null }); + assert_eq!(request.url().path(), "/v1/table/my_table/get_memwal_spec/"); + // Null spec → the MemWAL write path is not enabled. + let response = serde_json::json!({ "memwal_spec": null }); http::Response::builder() .status(200) .body(response.to_string()) .unwrap() }); - assert!(table.get_lsm_write_spec().await.unwrap().is_none()); + assert!(table.get_memwal_spec().await.unwrap().is_none()); } - /// Build a `get_lsm_stats` body for one bucket holding `generations`. - fn stats_body(generations: &[u64], compacting: bool) -> String { + /// Build a `get_memwal_stats` body for one table shard holding `sstables`. + fn stats_body(sstables: &[u64], compacting: bool) -> String { serde_json::json!({ - "lsm_stats": { - "buckets": [{ + "memwal_stats": { + "table_shards": [{ "shard_id": "b0", "status": "Active", "writer_epoch": 1, "manifest_version": 1, - "current_generation": generations.iter().max().copied().unwrap_or(0) + 1, + "current_generation": sstables.iter().max().copied().unwrap_or(0) + 1, "replay_after_wal_entry_position": 0, "wal_entry_position_last_seen": 0, - "generations": generations.iter() + "sstables": sstables.iter() .map(|g| serde_json::json!({ "generation": g, "bytes": 1 })) .collect::>(), "compacting": compacting, @@ -10240,7 +10218,7 @@ mod tests { .to_string() } - /// `flush_lsm` / `compact_lsm` answer 202 with no body at all. + /// `flush_memwal` / `compact_memwal` answer 202 with no body at all. fn accepted() -> http::Response { http::Response::builder() .status(202) @@ -10252,7 +10230,7 @@ mod tests { http::Response::builder().status(200).body(body).unwrap() } - /// A flush landing in an empty L0 finishes on the opening stats read + /// A flush landing in an empty SSTable tier finishes on the opening stats read /// alone. Asserting zero compacts is the point: "it returned Ok" is also /// true of a loop that ran a pointless pass. #[tokio::test(start_paused = true)] @@ -10261,18 +10239,18 @@ mod tests { let seen = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("compact_lsm") { + if path.contains("compact_memwal") { seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); panic!("an already-converged table must issue no compact calls"); } - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { return accepted(); } - assert_eq!(path, "/v1/table/my_table/get_lsm_stats/"); + assert_eq!(path, "/v1/table/my_table/get_memwal_stats/"); ok_json(stats_body(&[], false)) }); - table.checkpoint_lsm().await.unwrap(); + table.checkpoint_memwal().await.unwrap(); assert_eq!(compacts.load(std::sync::atomic::Ordering::SeqCst), 0); } @@ -10284,10 +10262,10 @@ mod tests { let seen = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { return accepted(); } - if path.contains("compact_lsm") { + if path.contains("compact_memwal") { seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); return accepted(); } @@ -10297,7 +10275,7 @@ mod tests { ok_json(stats_body(&left, false)) }); - table.checkpoint_lsm().await.unwrap(); + table.checkpoint_memwal().await.unwrap(); assert_eq!( compacts.load(std::sync::atomic::Ordering::SeqCst), 3, @@ -10306,7 +10284,7 @@ mod tests { } /// Generations created *during* the checkpoint are not waited on, which - /// is what lets the loop terminate on a table taking writes where "L0 is + /// is what lets the loop terminate on a table taking writes where "the SSTable tier is /// empty" never becomes true. #[tokio::test(start_paused = true)] async fn test_checkpoint_ignores_generations_created_while_it_runs() { @@ -10314,10 +10292,10 @@ mod tests { let seen = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { return accepted(); } - if path.contains("compact_lsm") { + if path.contains("compact_memwal") { seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); return accepted(); } @@ -10331,11 +10309,11 @@ mod tests { ok_json(body) }); - table.checkpoint_lsm().await.unwrap(); + table.checkpoint_memwal().await.unwrap(); assert_eq!( compacts.load(std::sync::atomic::Ordering::SeqCst), 1, - "the loop must not chase generations written after it started" + "the loop must not chase SSTables written after it started" ); } @@ -10348,11 +10326,11 @@ mod tests { let seen = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { return accepted(); } - if path.contains("compact_lsm") { - // First two triggers: every bucket already latched. + if path.contains("compact_memwal") { + // First two triggers: every table shard already latched. if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2 { return http::Response::builder() .status(429) @@ -10373,7 +10351,7 @@ mod tests { }); table - .checkpoint_lsm() + .checkpoint_memwal() .await .expect("contention must not abort the checkpoint"); assert_eq!( @@ -10393,7 +10371,7 @@ mod tests { let seen = polls.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") || path.contains("compact_lsm") { + if path.contains("flush_memwal") || path.contains("compact_memwal") { return accepted(); } // The opening read lands; the next two polls are latched out. @@ -10408,7 +10386,7 @@ mod tests { }); table - .checkpoint_lsm() + .checkpoint_memwal() .await .expect("a contended poll must be retried, not surfaced"); assert_eq!( @@ -10428,7 +10406,7 @@ mod tests { let seen = flushes.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 5 { return http::Response::builder() .status(429) @@ -10437,20 +10415,20 @@ mod tests { } return accepted(); } - if path.contains("compact_lsm") { + if path.contains("compact_memwal") { return accepted(); } ok_json(stats_body(&[], false)) }); table - .checkpoint_lsm() + .checkpoint_memwal() .await .expect("contention must not be reported as a lost claim"); assert_eq!( flushes.load(std::sync::atomic::Ordering::SeqCst), 6, - "five retries against one seal, then it lands" + "five retries against one freeze, then it lands" ); } @@ -10469,7 +10447,7 @@ mod tests { .unwrap() }); - let err = table.checkpoint_lsm().await.unwrap_err(); + let err = table.checkpoint_memwal().await.unwrap_err(); assert!( matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 429), "the fault that spent the budget must be the one reported: {err:?}" @@ -10499,7 +10477,7 @@ mod tests { .unwrap() }); - let err = table.checkpoint_lsm().await.unwrap_err(); + let err = table.checkpoint_memwal().await.unwrap_err(); let message = err.to_string(); assert!( matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 503), @@ -10519,7 +10497,7 @@ mod tests { /// A long stall with nothing compacting must keep waiting, not fail. The /// client cannot judge this: a checkpoint queued behind unrelated tables /// on the pod-wide compactor pool reports exactly these numbers — flat - /// generations, an idle latch — as one whose merges are failing. The + /// SSTable counts, an idle latch — as one whose merges are failing. The /// deadline is the caller's. #[tokio::test(start_paused = true)] async fn test_checkpoint_waits_out_a_long_stall_rather_than_failing() { @@ -10527,7 +10505,7 @@ mod tests { let seen = polls.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") || path.contains("compact_lsm") { + if path.contains("flush_memwal") || path.contains("compact_memwal") { return accepted(); } // Flat for far longer than any bound this loop ever had, with @@ -10537,7 +10515,7 @@ mod tests { }); table - .checkpoint_lsm() + .checkpoint_memwal() .await .expect("a stall is the server being slow, not the client's call to make"); assert!( @@ -10546,7 +10524,7 @@ mod tests { ); } - /// A pass already owns the latch on every outstanding bucket, so the loop + /// A pass already owns the latch on every outstanding table shard, so the loop /// waits rather than piling on triggers it would only refuse. This is the /// sole thing `compacting` is read for. #[tokio::test(start_paused = true)] @@ -10557,10 +10535,10 @@ mod tests { let seen_compacts = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { return accepted(); } - if path.contains("compact_lsm") { + if path.contains("compact_memwal") { seen_compacts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); return accepted(); } @@ -10574,32 +10552,32 @@ mod tests { }); table - .checkpoint_lsm() + .checkpoint_memwal() .await .expect("a running pass is progress, not a stall"); assert_eq!( compacts.load(std::sync::atomic::Ordering::SeqCst), 0, - "never trigger against a bucket already compacting" + "never trigger against a table shard already compacting" ); } /// WAL off ⇒ `None`; WAL on ⇒ a fully populated `Some` with no field - /// defaulting to a zero it did not measure. `include_generation_rows` + /// defaulting to a zero it did not measure. `include_sstable_rows` /// rides in the body and is off unless asked for. #[tokio::test] - async fn test_get_lsm_stats_round_trip() { + async fn test_get_memwal_stats_round_trip() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!(request.url().path(), "/v1/table/my_table/get_lsm_stats/"); + assert_eq!(request.url().path(), "/v1/table/my_table/get_memwal_stats/"); let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!( - body["include_generation_rows"], true, + body["include_sstable_rows"], true, "the flag must reach the server, not be silently dropped" ); let response = serde_json::json!({ - "lsm_stats": { - "buckets": [{ + "memwal_stats": { + "table_shards": [{ "shard_id": "b0", "status": "Active", "writer_epoch": 3, @@ -10607,7 +10585,7 @@ mod tests { "current_generation": 9, "replay_after_wal_entry_position": 100, "wal_entry_position_last_seen": 140, - "generations": [{ "generation": 8, "bytes": 4096, "rows": 30 }], + "sstables": [{ "generation": 8, "bytes": 4096, "rows": 30 }], "compacting": false, "memtables": [ { "generation": 9, "rows": 12, "bytes": 900, "batches": 2, @@ -10623,19 +10601,19 @@ mod tests { }); let stats = table - .get_lsm_stats(true) + .get_memwal_stats(true) .await .unwrap() .expect("a WAL-backed table reports Some"); - let bucket = &stats.buckets[0]; - assert_eq!(bucket.replay_after_wal_entry_position, 100); - assert_eq!(bucket.wal_entry_position_last_seen, 140); - assert!(!bucket.compacting); - assert_eq!(bucket.generations[0].generation, 8); - assert_eq!(bucket.generations[0].rows, Some(30)); + let table_shard = &stats.table_shards[0]; + assert_eq!(table_shard.replay_after_wal_entry_position, 100); + assert_eq!(table_shard.wal_entry_position_last_seen, 140); + assert!(!table_shard.compacting); + assert_eq!(table_shard.sstables[0].generation, 8); + assert_eq!(table_shard.sstables[0].rows, Some(30)); // The line that answers "why is my fresh-tier vector search // brute-force" — an absent index name is the whole explanation. - let memtables = bucket.memtables.as_ref().unwrap(); + let memtables = table_shard.memtables.as_ref().unwrap(); assert_eq!(memtables[0].indexes, vec!["vec_idx".to_string()]); } @@ -10655,7 +10633,7 @@ mod tests { .unwrap() }); - let err = table.checkpoint_lsm().await.unwrap_err(); + let err = table.checkpoint_memwal().await.unwrap_err(); assert!( matches!(err, Error::TableNotFound { .. }), "a missing table must say so: {err:?}" @@ -10676,12 +10654,12 @@ mod tests { let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { // First flush lands; the claim is then lost, and the // re-issued flush succeeds. return accepted(); } - if path.contains("compact_lsm") { + if path.contains("compact_memwal") { if n < 4 { return http::Response::builder() .status(421) @@ -10694,20 +10672,20 @@ mod tests { }); table - .checkpoint_lsm() + .checkpoint_memwal() .await .expect("a lost claim must be recovered by re-flushing, not surfaced"); } #[tokio::test] - async fn test_get_lsm_stats_absent_when_wal_off() { + async fn test_get_memwal_stats_absent_when_wal_off() { let table = Table::new_with_handler("my_table", |_request| { http::Response::builder() .status(200) - .body(serde_json::json!({ "lsm_stats": null }).to_string()) + .body(serde_json::json!({ "memwal_stats": null }).to_string()) .unwrap() }); - assert!(table.get_lsm_stats(false).await.unwrap().is_none()); + assert!(table.get_memwal_stats(false).await.unwrap().is_none()); } #[tokio::test] @@ -14726,7 +14704,7 @@ mod tests { /// Verbatim from sophon's `LSM_WITH_ROW_ID_UNSUPPORTED`; the fallback keys /// off this sentence, so a test that paraphrased it would prove nothing. - const WAL_ROW_ID_REFUSAL_BODY: &str = r#"{"code":13,"error":"Bad request: the MemWAL LSM scanner does not support with_row_id (the LSM scanner exposes _rowaddr, not a stable _rowid); set use_lsm(false) to read the base table only (results will exclude un-compacted MemWAL data)"}"#; + const WAL_ROW_ID_REFUSAL_BODY: &str = r#"{"code":13,"error":"Bad request: the MemWAL LSM scanner does not support with_row_id (the LSM scanner exposes _rowaddr, not a stable _rowid); set use_memwal(false) to read the base table only (results will exclude un-compacted MemWAL data)"}"#; use lance::dataset::ROW_ID; @@ -14985,7 +14963,7 @@ mod tests { // Translated, not passed through: the caller gets the reason and the // way out, which a bare 400 does not carry. assert!(matches!(err, Error::NotSupported { .. }), "{err:?}"); - assert!(err.to_string().contains("use_lsm(false)"), "{err}"); + assert!(err.to_string().contains("use_memwal(false)"), "{err}"); assert!( bodies .lock() diff --git a/rust/lancedb/src/remote/table/blobs.rs b/rust/lancedb/src/remote/table/blobs.rs index 901616fb2..fdcb8f440 100644 --- a/rust/lancedb/src/remote/table/blobs.rs +++ b/rust/lancedb/src/remote/table/blobs.rs @@ -685,7 +685,7 @@ impl RemoteTable { select: Select::columns(&[column]), with_row_id: true, // Row ids name base-table rows, and the MemWAL scanner has no stable `_rowid`. - use_lsm: Some(false), + use_memwal: Some(false), ..Default::default() }); let body = self @@ -1121,7 +1121,7 @@ mod tests { .unwrap(); assert_eq!(body["columns"], serde_json::json!(["image"])); assert_eq!(body["with_row_id"], serde_json::json!(true)); - assert_eq!(body["use_lsm"], serde_json::json!(false)); + assert_eq!(body["use_memwal"], serde_json::json!(false)); let requested = filter_row_ids(body["filter"].as_str().unwrap()); captured.lock().unwrap().push(TakeMockRequest::Query { row_ids: requested.clone(), diff --git a/rust/lancedb/src/remote/table/insert.rs b/rust/lancedb/src/remote/table/insert.rs index eef1d8e42..3a089eb95 100644 --- a/rust/lancedb/src/remote/table/insert.rs +++ b/rust/lancedb/src/remote/table/insert.rs @@ -1497,7 +1497,7 @@ mod tests { when_not_matched_by_source_delete: false, when_not_matched_by_source_delete_filt: None, use_index: true, - use_lsm: None, + use_memwal: None, }; let input: Arc = Arc::new(ErroringExec::new()); diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 74b99255a..c0a4a719e 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -80,7 +80,7 @@ pub mod datafusion; pub(crate) mod dataset; pub mod delete; pub mod freshness; -pub mod lsm_stats; +pub mod memwal_stats; pub mod merge; pub mod optimize; pub(crate) mod primary_key; @@ -107,7 +107,7 @@ use futures::future::join_all; pub use lance::dataset::refs::{BranchContents, Ref, TagContents, Tags as LanceTags}; pub use lance::dataset::scanner::DatasetRecordBatchStream; pub use lance_index::optimize::OptimizeOptions; -pub use lsm_stats::{BucketStats, GenerationStats, LsmStats, MemtableStats}; +pub use memwal_stats::{MemWalStats, MemtableStats, SsTableStats, TableShardStats}; pub use optimize::{CompactionOptions, OptimizeAction, OptimizeStats}; pub use refresh::RefreshColumnResult; pub use schema_evolution::{ @@ -327,20 +327,20 @@ pub use self::merge::MergeResult; /// Specification selecting Lance's MemWAL LSM-style write path for /// `merge_insert`. /// -/// Construct via [`LsmWriteSpec::bucket`], [`LsmWriteSpec::identity`], or -/// [`LsmWriteSpec::unsharded`], then optionally chain -/// [`LsmWriteSpec::with_maintained_indexes`] (indexes the MemWAL keeps up to -/// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default +/// Construct via [`MemWalWriteSpec::bucket`], [`MemWalWriteSpec::identity`], or +/// [`MemWalWriteSpec::unsharded`], then optionally chain +/// [`MemWalWriteSpec::with_maintained_indexes`] (indexes the MemWAL keeps up to +/// date) and [`MemWalWriteSpec::with_writer_config_defaults`] (default /// `ShardWriter` configuration recorded in the MemWAL index). /// /// A fresh spec maintains every index on the table, including ones created /// after it is installed. /// -/// Install a spec with [`Table::set_lsm_write_spec`] and remove it with -/// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch +/// Install a spec with [`Table::enable_memwal`] and remove it with +/// [`Table::disable_memwal`]. The actual `merge_insert` dispatch /// onto the MemWAL writer is a follow-up. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum LsmWriteSpec { +pub enum MemWalWriteSpec { /// Hash-bucket sharding by a scalar column. /// /// `column` must be a non-nested column with a supported scalar type. @@ -384,7 +384,7 @@ pub enum LsmWriteSpec { }, } -impl LsmWriteSpec { +impl MemWalWriteSpec { /// Construct a hash-bucket sharding spec maintaining every index on the table. pub fn bucket(column: impl Into, num_buckets: u32) -> Self { Self::Bucket { @@ -401,7 +401,7 @@ impl LsmWriteSpec { /// `column` must be a deterministic function of the unenforced primary /// key: every row with a given primary key must always produce the same /// `column` value. MemWAL dedups upserts by primary key but tracks - /// generations per shard, so if the same key is written with two + /// SSTables per shard, so if the same key is written with two /// different `column` values its versions land in different shards and a /// stale value can win. Typically `column` is the primary key itself, or /// a stable attribute of it (e.g. a tenant id). @@ -431,13 +431,13 @@ impl LsmWriteSpec { /// any added with [`NativeTable::set_mem_index_registry`]. /// /// ``` - /// # use lancedb::table::LsmWriteSpec; + /// # use lancedb::table::MemWalWriteSpec; /// // Every index the table has, now and later: - /// LsmWriteSpec::unsharded().with_maintained_indexes(None); + /// MemWalWriteSpec::unsharded().with_maintained_indexes(None); /// // Exactly these: - /// LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]); + /// MemWalWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]); /// // None at all: - /// LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new()); + /// MemWalWriteSpec::unsharded().with_maintained_indexes(Vec::new()); /// ``` pub fn with_maintained_indexes(mut self, indexes: impl Into>>) -> Self { let indexes = indexes.into(); @@ -674,37 +674,37 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { message: "set_unenforced_primary_key is not supported on this table type".into(), }) } - /// Install an [`LsmWriteSpec`] on this table. + /// Install an [`MemWalWriteSpec`] on this table. /// /// The spec selects Lance's MemWAL LSM-style write path for future /// `merge_insert` calls. /// /// The default implementation returns `NotSupported`. Implementations - /// that support the MemWAL LSM write path must override this. - async fn set_lsm_write_spec(&self, _spec: LsmWriteSpec) -> Result<()> { + /// that support the MemWAL write path must override this. + async fn enable_memwal(&self, _spec: MemWalWriteSpec) -> Result<()> { Err(Error::NotSupported { - message: "set_lsm_write_spec is not supported on this table type".into(), + message: "enable_memwal is not supported on this table type".into(), }) } - /// Remove the [`LsmWriteSpec`] from this table. + /// Remove the [`MemWalWriteSpec`] from this table. /// /// This is a no-op if no spec is currently set. /// /// The default implementation returns `NotSupported`. Implementations - /// that support the MemWAL LSM write path must override this. - async fn unset_lsm_write_spec(&self) -> Result<()> { + /// that support the MemWAL write path must override this. + async fn disable_memwal(&self) -> Result<()> { Err(Error::NotSupported { - message: "unset_lsm_write_spec is not supported on this table type".into(), + message: "disable_memwal is not supported on this table type".into(), }) } - /// Read the [`LsmWriteSpec`] currently installed on this table, returning - /// `None` when the MemWAL LSM write path is not enabled. + /// Read the [`MemWalWriteSpec`] currently installed on this table, returning + /// `None` when the MemWAL write path is not enabled. /// /// The default implementation returns `NotSupported`. Implementations that - /// support the MemWAL LSM write path must override this. - async fn get_lsm_write_spec(&self) -> Result> { + /// support the MemWAL write path must override this. + async fn get_memwal_spec(&self) -> Result> { Err(Error::NotSupported { - message: "get_lsm_write_spec is not supported on this table type".into(), + message: "get_memwal_spec is not supported on this table type".into(), }) } /// Whether a hybrid query on this table has already been told it cannot @@ -732,36 +732,36 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { /// the only cost of being late to notice is a base-only read that is still /// correct. fn note_hybrid_pk_fusion(&self) {} - /// Seal every bucket's active memtable into L0. + /// Freeze every table shard's active memtable into an SSTable. /// /// The default implementation returns `NotSupported`. - async fn flush_lsm(&self) -> Result<()> { + async fn flush_memwal(&self) -> Result<()> { Err(Error::NotSupported { - message: "flush_lsm is not supported on this table type".into(), + message: "flush_memwal is not supported on this table type".into(), }) } - /// Trigger a background L0 → base compaction pass per bucket. + /// Trigger a background SSTable compaction pass per table shard. /// /// The default implementation returns `NotSupported`. - async fn compact_lsm(&self) -> Result<()> { + async fn compact_memwal(&self) -> Result<()> { Err(Error::NotSupported { - message: "compact_lsm is not supported on this table type".into(), + message: "compact_memwal is not supported on this table type".into(), }) } - /// Read live LSM state, or `None` when the LSM write path is not + /// Read live MemWAL state, or `None` when the MemWAL write path is not /// enabled for this table. /// /// The default implementation returns `NotSupported`. - async fn get_lsm_stats(&self, _include_generation_rows: bool) -> Result> { + async fn get_memwal_stats(&self, _include_sstable_rows: bool) -> Result> { Err(Error::NotSupported { - message: "get_lsm_stats is not supported on this table type".into(), + message: "get_memwal_stats is not supported on this table type".into(), }) } /// Drain and close any cached MemWAL shard writers for this table. /// /// The default implementation is a no-op; table types that maintain /// MemWAL shard writers override it. - async fn close_lsm_writers(&self) -> Result<()> { + async fn close_memwal_writers(&self) -> Result<()> { Ok(()) } /// Names of the blob v2 columns in this table, in declaration order. @@ -2006,50 +2006,50 @@ impl Table { self.inner.set_unenforced_primary_key(&borrowed).await } - /// Install an [`LsmWriteSpec`] on this table, selecting Lance's MemWAL + /// Install an [`MemWalWriteSpec`] on this table, selecting Lance's MemWAL /// LSM-style write path for future `merge_insert` calls. /// - /// [`LsmWriteSpec`] chooses one of three sharding strategies: + /// [`MemWalWriteSpec`] chooses one of three sharding strategies: /// - /// - [`LsmWriteSpec::bucket`] — hash-bucket writes by a scalar column. - /// - [`LsmWriteSpec::identity`] — shard by the raw value of a scalar column. - /// - [`LsmWriteSpec::unsharded`] — route every write to a single shard. + /// - [`MemWalWriteSpec::bucket`] — hash-bucket writes by a scalar column. + /// - [`MemWalWriteSpec::identity`] — shard by the raw value of a scalar column. + /// - [`MemWalWriteSpec::unsharded`] — route every write to a single shard. /// /// A table carries one spec: this fails while one is installed, since the - /// generations already written were homed under it. - /// [`Table::unset_lsm_write_spec`] removes one. + /// SSTables already written were homed under it. + /// [`Table::disable_memwal`] removes one. /// /// # Example /// /// ``` - /// # use lancedb::table::{LsmWriteSpec, Table}; + /// # use lancedb::table::{MemWalWriteSpec, Table}; /// # async fn example(table: &Table) -> Result<(), Box> { /// table - /// .set_lsm_write_spec( - /// LsmWriteSpec::bucket("id", 16).with_maintained_indexes(vec!["id_idx".to_string()]), + /// .enable_memwal( + /// MemWalWriteSpec::bucket("id", 16).with_maintained_indexes(vec!["id_idx".to_string()]), /// ) /// .await?; /// # Ok(()) /// # } /// ``` - pub async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> Result<()> { - self.inner.set_lsm_write_spec(spec).await + pub async fn enable_memwal(&self, spec: MemWalWriteSpec) -> Result<()> { + self.inner.enable_memwal(spec).await } - /// Remove the [`LsmWriteSpec`] from this table, reverting to the standard + /// Remove the [`MemWalWriteSpec`] from this table, reverting to the standard /// `merge_insert` write path. /// /// Errors if no spec is currently set. - pub async fn unset_lsm_write_spec(&self) -> Result<()> { - self.inner.unset_lsm_write_spec().await + pub async fn disable_memwal(&self) -> Result<()> { + self.inner.disable_memwal().await } - /// Read the [`LsmWriteSpec`] currently installed on this table. + /// Read the [`MemWalWriteSpec`] currently installed on this table. /// - /// `Ok(None)` means the LSM write path is not enabled at all — no spec has - /// been set, or one was removed with [`Table::unset_lsm_write_spec`]. That + /// `Ok(None)` means the MemWAL write path is not enabled at all — no spec has + /// been set, or one was removed with [`Table::disable_memwal`]. That /// is a different answer from a spec whose - /// [`LsmWriteSpec::maintained_indexes`] is `None`, which is an installed + /// [`MemWalWriteSpec::maintained_indexes`] is `None`, which is an installed /// spec selecting indexes automatically. /// /// The spec read back is the one that was installed, selection included: @@ -2062,21 +2062,21 @@ impl Table { /// ``` /// # use lancedb::table::Table; /// # async fn example(table: &Table) -> Result<(), Box> { - /// if let Some(spec) = table.get_lsm_write_spec().await? { - /// println!("LSM write path enabled: {:?}", spec); + /// if let Some(spec) = table.get_memwal_spec().await? { + /// println!("MemWAL write path enabled: {:?}", spec); /// } /// # Ok(()) /// # } /// ``` - pub async fn get_lsm_write_spec(&self) -> Result> { - self.inner.get_lsm_write_spec().await + pub async fn get_memwal_spec(&self) -> Result> { + self.inner.get_memwal_spec().await } - /// Converge this table's LSM write path into its base table. + /// Converge this table's MemWAL write path into its base table. /// - /// One `flush` to seal every memtable into L0, then compaction triggers + /// One `flush` to freeze every memtable into an SSTable, then compaction triggers /// until every generation that existed at that moment has reached base. - /// The loop runs client-side, reading progress from `get_lsm_stats`, so + /// The loop runs client-side, reading progress from `get_memwal_stats`, so /// there is no held socket and nothing to reconcile if you drop this /// future partway through. /// @@ -2086,7 +2086,7 @@ impl Table { /// already-converged table costs two round trips and triggers nothing. /// /// **No deadline, and the caller owns that.** It returns when the target - /// generations are gone, propagates a terminal server fault, and + /// SSTables are gone, propagates a terminal server fault, and /// otherwise waits however long the server takes. A slow table and a /// stuck one are the same picture from here: the compactor pool is shared /// across every table on the node, so a checkpoint queued behind @@ -2099,66 +2099,69 @@ impl Table { /// ```no_run /// # use lancedb::Table; /// # async fn example(table: &Table) -> Result<(), Box> { - /// let before = table.get_lsm_stats(false).await?; - /// table.checkpoint_lsm().await?; - /// let after = table.get_lsm_stats(false).await?; + /// let before = table.get_memwal_stats(false).await?; + /// table.checkpoint_memwal().await?; + /// let after = table.get_memwal_stats(false).await?; /// # Ok(()) /// # } /// ``` - pub async fn checkpoint_lsm(&self) -> Result<()> { - checkpoint::checkpoint_lsm(self).await + pub async fn checkpoint_memwal(&self) -> Result<()> { + checkpoint::checkpoint_memwal(self).await } - /// Seal every bucket's active memtable into L0 without touching the + /// Freeze every table shard's active memtable into an SSTable without touching the /// base table. /// - /// Independently useful: flushing makes memtable rows readable from L0 at + /// Independently useful: flushing makes memtable rows readable from an SSTable at /// a lower per-query cost. On a node that has not claimed this table it /// claims it and replays the WAL log first — reporting "nothing to flush" /// without replaying would lie about durable data. - pub async fn flush_lsm(&self) -> Result<()> { - self.inner.flush_lsm().await + pub async fn flush_memwal(&self) -> Result<()> { + self.inner.flush_memwal().await } - /// Run one bounded L0 → base compaction pass per bucket, reporting what + /// Run one bounded SSTable compaction pass per table shard, reporting what /// it merged and what is left. /// /// One pass, not convergence: that bounds each request's cost and gives a /// caller driving its own cadence a progress signal per round trip. - pub async fn compact_lsm(&self) -> Result<()> { - self.inner.compact_lsm().await + pub async fn compact_memwal(&self) -> Result<()> { + self.inner.compact_memwal().await } - /// Read live per-bucket LSM state. + /// Read live per-table-shard MemWAL state. /// - /// Answers "how far behind is my fresh tier", "which bucket is hot", and + /// Answers "how far behind is my fresh tier", "which table shard is hot", and /// "why is my fresh-tier vector search brute-force". Mutates no table /// state, though on a node that has not claimed this table it claims it, /// exactly as a read would. /// - /// `include_generation_rows` reports a row count per L0 generation. Off by + /// `include_sstable_rows` reports a row count per SSTable. Off by /// default: each count opens an uncached Lance dataset, and - /// `checkpoint_lsm` polls this needing only generation numbers. + /// `checkpoint_memwal` polls this needing only generation numbers. /// - /// `Ok(None)` only when the LSM write path is not enabled, matching - /// [`Table::get_lsm_write_spec`]. Stats is fresh-tier only, so with the + /// `Ok(None)` only when the MemWAL write path is not enabled, matching + /// [`Table::get_memwal_spec`]. Stats is fresh-tier only, so with the /// WAL off there is no manifest to report and a struct of zeros would /// read as measurements. /// /// Do not build a checkpoint's termination on this: the completion /// predicate lives in the `flush` and `compact` responses. - pub async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result> { - self.inner.get_lsm_stats(include_generation_rows).await + pub async fn get_memwal_stats( + &self, + include_sstable_rows: bool, + ) -> Result> { + self.inner.get_memwal_stats(include_sstable_rows).await } /// Drain and close any cached MemWAL shard writers held for this table. /// - /// When an [`LsmWriteSpec`] is installed, `merge_insert` opens MemWAL shard + /// When an [`MemWalWriteSpec`] is installed, `merge_insert` opens MemWAL shard /// writers and caches them for reuse across calls. This closes them, /// flushing pending data; writers reopen lazily on the next `merge_insert`. /// It is a no-op when no writers are cached. - pub async fn close_lsm_writers(&self) -> Result<()> { - self.inner.close_lsm_writers().await + pub async fn close_memwal_writers(&self) -> Result<()> { + self.inner.close_memwal_writers().await } /// Retrieve the version of the table @@ -2776,8 +2779,8 @@ impl NativeTable { } /// Maintain the memtable index kinds in `registry` when this table writes - /// through its [`LsmWriteSpec`], and accept them when - /// [`LsmWriteSpec::with_maintained_indexes`] names one. + /// through its [`MemWalWriteSpec`], and accept them when + /// [`MemWalWriteSpec::with_maintained_indexes`] names one. /// /// The registry belongs to this handle, its clones and its branch handles, /// and is not stored with the table: a handle opened separately starts from @@ -2788,7 +2791,7 @@ impl NativeTable { /// ``` /// # use std::sync::Arc; /// # use lancedb::Table; - /// # use lancedb::table::{LsmWriteSpec, MemIndexRegistry, mem_index::MemIndexPlugin}; + /// # use lancedb::table::{MemWalWriteSpec, MemIndexRegistry, mem_index::MemIndexPlugin}; /// # async fn example( /// # table: &Table, /// # plugin: Arc, @@ -2797,8 +2800,8 @@ impl NativeTable { /// if let Some(native) = table.as_native() { /// native.set_mem_index_registry(registry); /// } - /// let spec = LsmWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]); - /// table.set_lsm_write_spec(spec).await?; + /// let spec = MemWalWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]); + /// table.enable_memwal(spec).await?; /// # Ok(()) /// # } /// ``` @@ -3691,20 +3694,20 @@ impl BaseTable for NativeTable { primary_key::set_unenforced_primary_key(self, columns).await } - async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> Result<()> { - merge::lsm::set_lsm_write_spec(self, spec).await + async fn enable_memwal(&self, spec: MemWalWriteSpec) -> Result<()> { + merge::lsm::enable_memwal(self, spec).await } - async fn unset_lsm_write_spec(&self) -> Result<()> { - merge::lsm::unset_lsm_write_spec(self).await + async fn disable_memwal(&self) -> Result<()> { + merge::lsm::disable_memwal(self).await } - async fn get_lsm_write_spec(&self) -> Result> { - merge::lsm::get_lsm_write_spec(self).await + async fn get_memwal_spec(&self) -> Result> { + merge::lsm::get_memwal_spec(self).await } - async fn close_lsm_writers(&self) -> Result<()> { - merge::lsm::close_lsm_writers(self).await + async fn close_memwal_writers(&self) -> Result<()> { + merge::lsm::close_memwal_writers(self).await } async fn blob_columns(&self) -> Result> { @@ -5510,7 +5513,7 @@ mod tests { } #[tokio::test] - async fn test_set_lsm_write_spec() { + async fn test_enable_memwal() { use arrow_array::StringArray; use lance::dataset::mem_wal::DatasetMemWalExt; @@ -5542,7 +5545,7 @@ mod tests { // Reject num_buckets out of range. for bad in [0u32, 1025] { let err = table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", bad)) + .enable_memwal(MemWalWriteSpec::bucket("id", bad)) .await .expect_err("should reject"); assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); @@ -5550,7 +5553,7 @@ mod tests { // Happy path: install spec; verify MemWAL details record it. table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 4)) + .enable_memwal(MemWalWriteSpec::bucket("id", 4)) .await .unwrap(); @@ -5576,14 +5579,14 @@ mod tests { // Mutation rejected. let err = table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 8)) + .enable_memwal(MemWalWriteSpec::bucket("id", 8)) .await .expect_err("mutation should be rejected"); assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); } #[tokio::test] - async fn test_set_lsm_write_spec_unsharded() { + async fn test_enable_memwal_unsharded() { use lance::dataset::mem_wal::DatasetMemWalExt; let tmp_dir = tempdir().unwrap(); @@ -5605,7 +5608,7 @@ mod tests { let table = conn.create_table("t", reader).execute().await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -5623,7 +5626,7 @@ mod tests { } #[tokio::test] - async fn test_set_lsm_write_spec_identity() { + async fn test_enable_memwal_identity() { use lance::dataset::mem_wal::DatasetMemWalExt; let tmp_dir = tempdir().unwrap(); @@ -5651,8 +5654,8 @@ mod tests { let table = conn.create_table("t", reader).execute().await.unwrap(); table - .set_lsm_write_spec( - LsmWriteSpec::identity("region") + .enable_memwal( + MemWalWriteSpec::identity("region") .with_writer_config_defaults([("durable_write", "false")]), ) .await @@ -5680,7 +5683,7 @@ mod tests { } #[tokio::test] - async fn test_unset_lsm_write_spec() { + async fn test_disable_memwal() { use lance::dataset::mem_wal::DatasetMemWalExt; let tmp_dir = tempdir().unwrap(); @@ -5702,11 +5705,11 @@ mod tests { let table = conn.create_table("t", reader).execute().await.unwrap(); // unset errors when no spec is set. - table.unset_lsm_write_spec().await.unwrap_err(); + table.disable_memwal().await.unwrap_err(); // Install a spec, then unset it. table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 4)) + .enable_memwal(MemWalWriteSpec::bucket("id", 4)) .await .unwrap(); { @@ -5714,16 +5717,16 @@ mod tests { assert!(dataset.mem_wal_index_details().await.unwrap().is_some()); } - table.unset_lsm_write_spec().await.unwrap(); + table.disable_memwal().await.unwrap(); { let dataset = table.as_native().unwrap().dataset.get().await.unwrap(); assert!(dataset.mem_wal_index_details().await.unwrap().is_none()); } // A second unset errors; a fresh spec can still be installed afterwards. - table.unset_lsm_write_spec().await.unwrap_err(); + table.disable_memwal().await.unwrap_err(); table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 8)) + .enable_memwal(MemWalWriteSpec::bucket("id", 8)) .await .unwrap(); { @@ -5733,7 +5736,7 @@ mod tests { } #[tokio::test] - async fn test_get_lsm_write_spec() { + async fn test_get_memwal_spec() { let tmp_dir = tempdir().unwrap(); let uri = tmp_dir.path().to_str().unwrap(); @@ -5759,7 +5762,7 @@ mod tests { let table = conn.create_table("t", reader).execute().await.unwrap(); // No spec installed yet. - assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); + assert_eq!(table.get_memwal_spec().await.unwrap(), None); // A real scalar index is needed to name it as a maintained index. table @@ -5771,35 +5774,35 @@ mod tests { // Bucket spec round-trips exactly, including the routing column (recovered // from its field id), maintained indexes, and writer config defaults. - let spec = LsmWriteSpec::bucket("id", 4) + let spec = MemWalWriteSpec::bucket("id", 4) .with_maintained_indexes(vec![idx_name.clone()]) .with_writer_config_defaults([("durable_write", "false")]); - table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); + table.enable_memwal(spec.clone()).await.unwrap(); + assert_eq!(table.get_memwal_spec().await.unwrap(), Some(spec)); // After unset, no spec is reported. - table.unset_lsm_write_spec().await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); + table.disable_memwal().await.unwrap(); + assert_eq!(table.get_memwal_spec().await.unwrap(), None); // Identity sharding round-trips (column recovered from the schema). // A spec left at its default round-trips as the default: what is stored // is the intent to maintain everything, not the set it resolves to now. - let spec = LsmWriteSpec::identity("region"); - table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); - table.unset_lsm_write_spec().await.unwrap(); + let spec = MemWalWriteSpec::identity("region"); + table.enable_memwal(spec.clone()).await.unwrap(); + assert_eq!(table.get_memwal_spec().await.unwrap(), Some(spec)); + table.disable_memwal().await.unwrap(); // Unsharded round-trips (no routing column). - let spec = LsmWriteSpec::unsharded(); - table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); + let spec = MemWalWriteSpec::unsharded(); + table.enable_memwal(spec.clone()).await.unwrap(); + assert_eq!(table.get_memwal_spec().await.unwrap(), Some(spec)); } /// The maintained set defaults to every index on the table. A named index /// the memtable cannot build fails the install, because the caller asked /// for it; an unnamed one is skipped, because it arrived by existing. #[tokio::test] - async fn test_set_lsm_write_spec_infers_maintained_indexes() { + async fn test_enable_memwal_infers_maintained_indexes() { let tmp_dir = tempdir().unwrap(); let uri = tmp_dir.path().to_str().unwrap(); @@ -5839,8 +5842,9 @@ mod tests { // Explicitly naming the bitmap index fails before anything commits. let err = table - .set_lsm_write_spec( - LsmWriteSpec::unsharded().with_maintained_indexes(vec!["tag_bitmap".to_string()]), + .enable_memwal( + MemWalWriteSpec::unsharded() + .with_maintained_indexes(vec!["tag_bitmap".to_string()]), ) .await .unwrap_err(); @@ -5848,18 +5852,18 @@ mod tests { matches!(err, Error::InvalidInput { ref message } if message.contains("tag_bitmap")), "expected the bitmap index to be rejected, got {err:?}" ); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); + assert_eq!(table.get_memwal_spec().await.unwrap(), None); // The default names nothing, so the bitmap is skipped rather than // refusing the table: it is maintained by existing, not by being asked // for. The intent is what is stored, not the set it resolves to today. table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); assert_eq!( table - .get_lsm_write_spec() + .get_memwal_spec() .await .unwrap() .unwrap() @@ -5870,16 +5874,16 @@ mod tests { // Narrowing to a named subset: unset first, since an installed spec // cannot be set over. - table.unset_lsm_write_spec().await.unwrap(); + table.disable_memwal().await.unwrap(); table - .set_lsm_write_spec( - LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]), + .enable_memwal( + MemWalWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]), ) .await .unwrap(); assert_eq!( table - .get_lsm_write_spec() + .get_memwal_spec() .await .unwrap() .unwrap() @@ -5888,14 +5892,14 @@ mod tests { ); // Opting out entirely is distinct from the default. - table.unset_lsm_write_spec().await.unwrap(); + table.disable_memwal().await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new())) + .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(Vec::new())) .await .unwrap(); assert_eq!( table - .get_lsm_write_spec() + .get_memwal_spec() .await .unwrap() .unwrap() @@ -5906,7 +5910,7 @@ mod tests { // An installed spec is never set over, whatever the new one asks for, // and the refused call leaves it alone. let err = table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 4)) + .enable_memwal(MemWalWriteSpec::bucket("id", 4)) .await .unwrap_err(); assert!( @@ -5916,7 +5920,7 @@ mod tests { ); assert_eq!( table - .get_lsm_write_spec() + .get_memwal_spec() .await .unwrap() .unwrap() diff --git a/rust/lancedb/src/table/checkpoint.rs b/rust/lancedb/src/table/checkpoint.rs index bb76604ed..c9a2f60d6 100644 --- a/rust/lancedb/src/table/checkpoint.rs +++ b/rust/lancedb/src/table/checkpoint.rs @@ -1,18 +1,18 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Converging a table's LSM write path into its base table. +//! Converging a table's MemWAL write path into its base table. //! -//! `checkpoint_lsm` seals once, then triggers compaction and watches -//! generation numbers until the L0 that existed at the start is gone. +//! `checkpoint_memwal` freezes once, then triggers compaction and watches +//! generation numbers until the SSTables that existed at the start are gone. //! -//! The loop runs in the client, not the server: `compact_lsm` dispatches a +//! The loop runs in the client, not the server: `compact_memwal` dispatches a //! pass and returns, so nothing holds a socket and a client can vanish //! mid-operation with nothing to reconcile. Completion is read from //! generation numbers in the shard manifest — durable state, unlike a count //! in a compact response, which a concurrent write invalidates. //! -//! The target set is fixed at the start, so generations created *during* the +//! The target set is fixed at the start, so SSTables created *during* the //! checkpoint are ignored. That is what lets it terminate under write load, //! and what makes it best-effort: it converges the fresh tier as of some //! instant. Idempotent, abandonable at any point, safe on a cadence. @@ -68,7 +68,7 @@ fn is_lost_claim(e: &Error) -> bool { status_of(e) == Some(421) } -/// Interval between `get_lsm_stats` polls. One interval is roughly one +/// Interval between `get_memwal_stats` polls. One interval is roughly one /// compaction pass, the granularity at which the answer can change. /// /// Fixed rather than configurable, matching `wait_for_index`. It costs @@ -149,15 +149,15 @@ where } } -/// Drive [`Table::checkpoint_lsm`]: seal once, fix the target watermark -/// from the resulting L0, then trigger and poll until it drains. -pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> { +/// Drive [`Table::checkpoint_memwal`]: freeze once, fix the target watermark +/// from the resulting SSTables, then trigger and poll until they drain. +pub(crate) async fn checkpoint_memwal(table: &Table) -> Result<()> { for reissue in 0..=MAX_REISSUES { - // The seal turns everything written before this call into a - // generation, so the watermark has to be read after it. Idempotent: - // sealing an empty memtable is a no-op, so a re-issue does not churn - // empty generations. - match issue(|| table.flush_lsm()).await? { + // The freeze turns everything written before this call into an + // SSTable, so the watermark has to be read after it. Idempotent: + // freezing an empty memtable is a no-op, so a re-issue does not churn + // empty SSTables. + match issue(|| table.flush_memwal()).await? { Attempt::Ok(()) => {} Attempt::ReissueFromFlush => { backoff(reissue).await; @@ -165,7 +165,7 @@ pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> { } } - let stats = match issue(|| table.get_lsm_stats(false)).await? { + let stats = match issue(|| table.get_memwal_stats(false)).await? { Attempt::Ok(stats) => stats, Attempt::ReissueFromFlush => { backoff(reissue).await; @@ -173,13 +173,13 @@ pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> { } }; let Some(stats) = stats else { - // Not WAL-backed; `flush_lsm` would have errored first but for a race. + // Not WAL-backed; `flush_memwal` would have errored first but for a race. return Ok(()); }; let targets: HashMap = stats - .buckets + .table_shards .iter() - .filter_map(|b| Some((b.shard_id.clone(), b.newest_generation()?))) + .filter_map(|b| Some((b.shard_id.clone(), b.newest_sstable_generation()?))) .collect(); if targets.is_empty() { return Ok(()); @@ -194,18 +194,18 @@ pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> { } } Err(Error::Runtime { - message: "checkpoint_lsm: the owning node kept losing its claim; \ + message: "checkpoint_memwal: the owning node kept losing its claim; \ re-issued from flush the maximum number of times" .into(), }) } -/// Trigger and poll until no bucket holds a generation at or below its +/// Trigger and poll until no table shard holds an SSTable generation at or below its /// target. /// /// No liveness bound, deliberately. The pod-wide compactor pool (a semaphore /// of 2 by default, shared across every table on the node) is taken *inside* -/// the pass, after the bucket latch, so a checkpoint queued behind unrelated +/// the pass, after the table shard latch, so a checkpoint queued behind unrelated /// tables is indistinguishable from one that is merging. An idle-poll counter /// here could only ever have fired on a table that would have finished. async fn drain_to_targets( @@ -213,24 +213,24 @@ async fn drain_to_targets( targets: &HashMap, ) -> Result { loop { - let stats = match issue(|| table.get_lsm_stats(false)).await? { + let stats = match issue(|| table.get_memwal_stats(false)).await? { Attempt::Ok(stats) => stats, Attempt::ReissueFromFlush => return Ok(CheckpointOutcome::ReissueFromFlush), }; let Some(stats) = stats else { return Ok(CheckpointOutcome::Done); }; - // `compacting` is the bucket's compaction latch, held from dispatch + // `compacting` is the table shard's compaction latch, held from dispatch // until the pass ends — including while it waits on the pod-wide - // permit. So it answers one question only: do not pile on. Buckets - // with nothing outstanding are skipped, not counted as idle. + // permit. So it answers one question only: do not pile on. Table + // shards with nothing outstanding are skipped, not counted as idle. let mut outstanding = 0; let mut all_compacting = true; - for b in &stats.buckets { + for b in &stats.table_shards { let Some(target) = targets.get(&b.shard_id) else { continue; }; - let n = b.outstanding_generations(*target); + let n = b.outstanding_sstables(*target); if n > 0 { outstanding += n; all_compacting &= b.compacting; @@ -241,11 +241,11 @@ async fn drain_to_targets( } if !all_compacting { - match table.compact_lsm().await { + match table.compact_memwal().await { Ok(()) => {} Err(e) if is_lost_claim(&e) => return Ok(CheckpointOutcome::ReissueFromFlush), Err(e) if !is_retryable(&e) => return Err(e), - // A 429 here means the server could latch no bucket at all, + // A 429 here means the server could latch no table shard at all, // which the poll above already handles. Not retried in place: // the latch it would contend for is the one doing the work, so // fall through and re-read — `POLL_INTERVAL` is the backoff. diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index 88c47992d..563f185b6 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -2197,7 +2197,7 @@ pub(crate) fn plan(schema: SchemaRef, columns: &[(String, String)]) -> Result, schema: Arc, fts_query: Option, - use_lsm: Option, + use_memwal: Option, } impl BaseTableAdapter { @@ -182,13 +182,13 @@ impl BaseTableAdapter { table, schema: Arc::new(schema), fts_query: None, - use_lsm: None, + use_memwal: None, }) } - /// Set the MemWAL read routing of every scan; see [`QueryRequest::use_lsm`]. - pub fn with_use_lsm(mut self, use_lsm: Option) -> Self { - self.use_lsm = use_lsm; + /// Set the MemWAL read routing of every scan; see [`QueryRequest::use_memwal`]. + pub fn with_use_memwal(mut self, use_memwal: Option) -> Self { + self.use_memwal = use_memwal; self } @@ -204,7 +204,7 @@ impl BaseTableAdapter { table: self.table.clone(), schema, fts_query: Some(fts_query), - use_lsm: self.use_lsm, + use_memwal: self.use_memwal, } } } @@ -232,7 +232,7 @@ impl TableProvider for BaseTableAdapter { let mut query = QueryRequest { full_text_search: self.fts_query.clone(), disable_scoring_autoprojection: disable_scoring, - use_lsm: self.use_lsm, + use_memwal: self.use_memwal, ..Default::default() }; diff --git a/rust/lancedb/src/table/lsm_stats.rs b/rust/lancedb/src/table/memwal_stats.rs similarity index 56% rename from rust/lancedb/src/table/lsm_stats.rs rename to rust/lancedb/src/table/memwal_stats.rs index 953aea90f..d9747d257 100644 --- a/rust/lancedb/src/table/lsm_stats.rs +++ b/rust/lancedb/src/table/memwal_stats.rs @@ -1,21 +1,21 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Live per-bucket LSM state — the shape [`crate::Table::get_lsm_stats`] +//! Live per-table-shard MemWAL state — the shape [`crate::Table::get_memwal_stats`] //! returns and [`super::checkpoint`] polls. //! -//! Nothing here is derived: sums and differences (total L0 bytes, WAL lag) +//! Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) //! are the caller's to compute. There is no "WAL is off" shape — that case is //! `None`, because a struct of zeros would read as measurements. use serde::Deserialize; -/// One flushed L0 generation. +/// One SSTable. #[derive(Debug, Clone, Deserialize)] -pub struct GenerationStats { +pub struct SsTableStats { pub generation: u64, pub bytes: u64, - /// Present only when `include_generation_rows` was requested. Off by + /// Present only when `include_sstable_rows` was requested. Off by /// default because each count opens an uncached Lance dataset, and the /// checkpoint loop polls this route needing only generation numbers. #[serde(default)] @@ -34,11 +34,11 @@ pub struct MemtableStats { pub indexes: Vec, } -/// Live state of one bucket. A table is N buckets on one node; flattening to -/// a single number hides the one hot bucket that is usually why someone +/// Live state of one table shard. A table is N table shards on one node; flattening to +/// a single number hides the one hot table shard that is usually why someone /// opened this endpoint. #[derive(Debug, Clone, Deserialize)] -pub struct BucketStats { +pub struct TableShardStats { pub shard_id: String, /// `Active` | `Sealed` (drop-table 2PC in flight). pub status: String, @@ -47,68 +47,68 @@ pub struct BucketStats { pub current_generation: u64, pub replay_after_wal_entry_position: u64, pub wal_entry_position_last_seen: u64, - pub generations: Vec, - /// Whether a pass owns this bucket's compaction latch right now. Says *a* + pub sstables: Vec, + /// Whether a pass owns this table shard's compaction latch right now. Says *a* /// driver is running, not *whose*, and the latch is held from dispatch — /// including while the pass queues for a pod-wide compactor permit. Read /// it as "do not pile on", never as "mine is progressing". pub compacting: bool, - /// Oldest first, active last. Absent for a `Sealed` bucket, whose + /// Oldest first, active last. Absent for a `Sealed` table shard, whose /// in-memory state is torn down. #[serde(default)] pub memtables: Option>, } -impl BucketStats { - /// The newest flushed generation, or `None` when L0 is empty. - pub(crate) fn newest_generation(&self) -> Option { - self.generations.iter().map(|g| g.generation).max() +impl TableShardStats { + /// The newest SSTable generation, or `None` when the tier is empty. + pub(crate) fn newest_sstable_generation(&self) -> Option { + self.sstables.iter().map(|g| g.generation).max() } - /// How many generations at or below `target` are still in L0. + /// How many SSTables at or below `target` are still uncompacted. /// /// A count, not a boolean: one pass drains a bounded prefix rather than /// the whole target set, so a boolean would read as "no progress" for /// every pass but the last. Compaction drains oldest-first, so this /// decreases monotonically. - pub(crate) fn outstanding_generations(&self, target: u64) -> usize { - self.generations + pub(crate) fn outstanding_sstables(&self, target: u64) -> usize { + self.sstables .iter() .filter(|g| g.generation <= target) .count() } } -/// Live LSM state, one entry per bucket. +/// Live MemWAL state, one entry per table shard. #[derive(Debug, Clone, Deserialize)] -pub struct LsmStats { - pub buckets: Vec, +pub struct MemWalStats { + pub table_shards: Vec, } -/// Server-side JSON envelope for `get_lsm_stats`. `lsm_stats` is null when -/// the table has no LSM write path. +/// Server-side JSON envelope for `get_memwal_stats`. `memwal_stats` is null when +/// the table has no MemWAL write path. #[derive(Debug, Deserialize)] -pub(crate) struct GetLsmStatsResponse { +pub(crate) struct GetMemWalStatsResponse { #[serde(default)] - pub lsm_stats: Option, + pub memwal_stats: Option, } #[cfg(test)] mod tests { use super::*; - fn bucket(shard: &str, generations: &[u64], compacting: bool) -> BucketStats { - BucketStats { + fn table_shard(shard: &str, sstables: &[u64], compacting: bool) -> TableShardStats { + TableShardStats { shard_id: shard.into(), status: "Active".into(), writer_epoch: 1, manifest_version: 1, - current_generation: generations.iter().max().copied().unwrap_or(0) + 1, + current_generation: sstables.iter().max().copied().unwrap_or(0) + 1, replay_after_wal_entry_position: 0, wal_entry_position_last_seen: 0, - generations: generations + sstables: sstables .iter() - .map(|g| GenerationStats { + .map(|g| SsTableStats { generation: *g, bytes: 1, rows: None, @@ -123,40 +123,46 @@ mod tests { /// generation created after it must not hold the loop open — that is why /// the predicate terminates under write load. #[test] - fn newer_generations_do_not_extend_the_target() { - let start = bucket("b0", &[7, 8], false); - let target = start.newest_generation().expect("L0 is non-empty"); + fn newer_sstables_do_not_extend_the_target() { + let start = table_shard("b0", &[7, 8], false); + let target = start + .newest_sstable_generation() + .expect("the SSTable tier is non-empty"); assert_eq!(target, 8); // Compaction drained 7 and 8; 9 and 10 arrived while it ran. - let later = bucket("b0", &[9, 10], false); + let later = table_shard("b0", &[9, 10], false); assert_eq!( - later.outstanding_generations(target), + later.outstanding_sstables(target), 0, - "generations above the target are somebody else's problem" + "sstables above the target are somebody else's problem" ); // Still holding 8 means still outstanding. assert_eq!( - bucket("b0", &[8, 9], false).outstanding_generations(target), + table_shard("b0", &[8, 9], false).outstanding_sstables(target), 1 ); } - /// The metric counts generations, not buckets: a pass drains a bounded - /// prefix, so one bucket going 3 → 2 → 1 → 0 is three steps. + /// The metric counts SSTables, not table shards: a pass drains a bounded + /// prefix, so one table shard going 3 → 2 → 1 → 0 is three steps. #[test] - fn progress_is_measured_in_generations() { + fn progress_is_measured_in_sstables() { let target = 3; let counts: Vec = [&[1u64, 2, 3][..], &[2, 3][..], &[3][..], &[][..]] .iter() - .map(|gens| bucket("b0", gens, false).outstanding_generations(target)) + .map(|gens| table_shard("b0", gens, false).outstanding_sstables(target)) .collect(); assert_eq!(counts, vec![3, 2, 1, 0]); } #[test] - fn empty_l0_has_no_target() { - assert!(bucket("b0", &[], false).newest_generation().is_none()); + fn an_empty_sstable_tier_has_no_target() { + assert!( + table_shard("b0", &[], false) + .newest_sstable_generation() + .is_none() + ); } } diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index c74980c5e..66ec48baf 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -47,7 +47,7 @@ pub struct MergeResult { /// Total number of rows written. /// /// On the standard `merge_insert` path this equals - /// `num_inserted_rows + num_updated_rows`. On the MemWAL LSM write path the + /// `num_inserted_rows + num_updated_rows`. On the MemWAL write path the /// insert/update breakdown is not known until compaction; in that mode /// `num_inserted_rows`, `num_updated_rows`, `num_deleted_rows`, `version` /// and `num_attempts` are all `0` and this field holds the total number of @@ -76,7 +76,7 @@ pub struct MergeInsertBuilder { pub(crate) when_not_matched_by_source_delete_filt: Option, pub(crate) timeout: Option, pub(crate) use_index: bool, - pub(crate) use_lsm: Option, + pub(crate) use_memwal: Option, pub(crate) validate_single_shard: bool, } @@ -92,7 +92,7 @@ impl MergeInsertBuilder { when_not_matched_by_source_delete_filt: None, timeout: None, use_index: true, - use_lsm: None, + use_memwal: None, validate_single_shard: true, } } @@ -193,27 +193,27 @@ impl MergeInsertBuilder { /// Control MemWAL routing for this `merge_insert`. /// /// By default (unset), a `merge_insert` on a table with an - /// [`LsmWriteSpec`](super::LsmWriteSpec) installed is routed through Lance's + /// [`MemWalWriteSpec`](super::MemWalWriteSpec) installed is routed through Lance's /// MemWAL shard writer; a table without one uses the standard path. /// - /// - `use_lsm(true)` forces MemWAL routing and errors if the table has no - /// LSM write spec. - /// - `use_lsm(false)` forces the standard write path even when a spec is set. - pub fn use_lsm(&mut self, enable: bool) -> &mut Self { - self.use_lsm = Some(enable); + /// - `use_memwal(true)` forces MemWAL routing and errors if the table has no + /// MemWAL write spec. + /// - `use_memwal(false)` forces the standard write path even when a spec is set. + pub fn use_memwal(&mut self, enable: bool) -> &mut Self { + self.use_memwal = Some(enable); self } /// Controls how an LSM `merge_insert` checks that its input targets a /// single shard. /// - /// When a table has an LSM write spec, every row in a `merge_insert` call + /// When a table has a MemWAL write spec, every row in a `merge_insert` call /// must route to the same shard. When `true` (the default), every row is /// inspected to verify this. When `false`, only the first row is inspected /// and the shard it routes to is used for the whole input — a faster path /// for callers that have already pre-sharded their input. /// - /// Has no effect on tables without an LSM write spec. + /// Has no effect on tables without a MemWAL write spec. pub fn validate_single_shard(&mut self, validate_single_shard: bool) -> &mut Self { self.validate_single_shard = validate_single_shard; self @@ -890,7 +890,7 @@ mod lsm_tests { use crate::connect; use crate::error::Error; - use crate::table::{LsmWriteSpec, Table}; + use crate::table::{MemWalWriteSpec, Table}; /// A reader of `[id: Int64, value: Int64]` rows; `value` is `0..n`. fn id_value_reader(ids: Vec) -> Box { @@ -974,7 +974,7 @@ mod lsm_tests { let table = id_value_table(&dir).await; // num_buckets = 1: every row routes to the single bucket. table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 1)) + .enable_memwal(MemWalWriteSpec::bucket("id", 1)) .await .unwrap(); @@ -1001,7 +1001,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -1030,7 +1030,7 @@ mod lsm_tests { .unwrap(); table.set_unenforced_primary_key(["id"]).await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::identity("region")) + .enable_memwal(MemWalWriteSpec::identity("region")) .await .unwrap(); @@ -1047,18 +1047,18 @@ mod lsm_tests { } #[tokio::test] - async fn lsm_merge_insert_use_lsm_false_falls_back() { + async fn lsm_merge_insert_use_memwal_false_falls_back() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 1)) + .enable_memwal(MemWalWriteSpec::bucket("id", 1)) .await .unwrap(); - // use_lsm(false) opts out: the standard path runs and commits even though + // use_memwal(false) opts out: the standard path runs and commits even though // a spec is installed. let mut builder = table.merge_insert(&["id"]); - builder.when_not_matched_insert_all().use_lsm(false); + builder.when_not_matched_insert_all().use_memwal(false); let result = builder .execute(id_value_reader(vec![3, 4, 5])) .await @@ -1069,17 +1069,17 @@ mod lsm_tests { } #[tokio::test] - async fn lsm_merge_insert_use_lsm_true_without_spec_errors() { + async fn lsm_merge_insert_use_memwal_true_without_spec_errors() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; - // use_lsm(true) demands MemWAL routing; without a write spec it errors + // use_memwal(true) demands MemWAL routing; without a write spec it errors // rather than silently falling back to the standard path. let mut builder = table.merge_insert(&["id"]); builder .when_matched_update_all(None) .when_not_matched_insert_all() - .use_lsm(true); + .use_memwal(true); let err = builder .execute(id_value_reader(vec![3, 4, 5])) .await @@ -1092,7 +1092,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 1)) + .enable_memwal(MemWalWriteSpec::bucket("id", 1)) .await .unwrap(); @@ -1109,7 +1109,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 1)) + .enable_memwal(MemWalWriteSpec::bucket("id", 1)) .await .unwrap(); @@ -1125,7 +1125,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 1)) + .enable_memwal(MemWalWriteSpec::bucket("id", 1)) .await .unwrap(); @@ -1135,7 +1135,7 @@ mod lsm_tests { .when_not_matched_insert_all(); builder.execute(id_value_reader(vec![7, 8])).await.unwrap(); - table.close_lsm_writers().await.unwrap(); + table.close_memwal_writers().await.unwrap(); // The writer reopens lazily on the next merge_insert. let mut builder = table.merge_insert(&[]); @@ -1160,7 +1160,7 @@ mod lsm_tests { .unwrap(); table.set_unenforced_primary_key(["id"]).await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::identity("region")) + .enable_memwal(MemWalWriteSpec::identity("region")) .await .unwrap(); @@ -1197,10 +1197,10 @@ mod lsm_tests { #[tokio::test] async fn lsm_merge_insert_no_spec_uses_standard_path() { let dir = tempdir().unwrap(); - // id_value_table sets a primary key but no LSM write spec. + // id_value_table sets a primary key but no MemWAL write spec. let table = id_value_table(&dir).await; - // Without a spec, a default merge_insert (use_lsm unset) simply uses + // Without a spec, a default merge_insert (use_memwal unset) simply uses // the standard path and commits — no opt-out required, no error. let mut builder = table.merge_insert(&["id"]); builder @@ -1228,7 +1228,7 @@ mod lsm_tests { .unwrap(); table.set_unenforced_primary_key(["id"]).await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::identity("region")) + .enable_memwal(MemWalWriteSpec::identity("region")) .await .unwrap(); @@ -1254,7 +1254,7 @@ mod lsm_tests { assert!(matches!(err, Error::InvalidInput { .. }), "got {err:?}"); // After closing the writer, a different shard can be written. - table.close_lsm_writers().await.unwrap(); + table.close_memwal_writers().await.unwrap(); let mut builder = table.merge_insert(&[]); builder .when_matched_update_all(None) @@ -1266,7 +1266,7 @@ mod lsm_tests { } // --------------------------------------------------------------------- - // LSM read path + // MemWAL read path // --------------------------------------------------------------------- use crate::arrow::SendableRecordBatchStream; @@ -1325,7 +1325,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; // base: ids 1,2,3 (value 0,1,2) table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -1340,8 +1340,8 @@ mod lsm_tests { vec![1, 2, 3, 4, 5] ); - // use_lsm(false) bypasses the MemWAL and reads the base table only. - let base_only = table.query().use_lsm(false).execute().await.unwrap(); + // use_memwal(false) bypasses the MemWAL and reads the base table only. + let base_only = table.query().use_memwal(false).execute().await.unwrap(); let rows = collect_id_value(base_only).await; assert_eq!( rows.iter().map(|(id, _)| *id).collect::>(), @@ -1350,14 +1350,14 @@ mod lsm_tests { } #[tokio::test] - async fn base_table_adapter_use_lsm_false_scans_the_base_only() { + async fn base_table_adapter_use_memwal_false_scans_the_base_only() { use crate::table::datafusion::BaseTableAdapter; use datafusion::prelude::SessionContext; let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; // base: ids 1,2,3 table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); // Ids 4,5 land in the active memtable, not the base table. @@ -1366,7 +1366,7 @@ mod lsm_tests { let adapter = BaseTableAdapter::try_new(table.base_table().clone()) .await .unwrap() - .with_use_lsm(Some(false)); + .with_use_memwal(Some(false)); let ctx = SessionContext::new(); ctx.register_table("base_only", Arc::new(adapter)).unwrap(); let batches = ctx @@ -1389,7 +1389,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); lsm_upsert(&table, vec![4, 5]).await; @@ -1407,7 +1407,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); lsm_upsert(&table, vec![4]).await; @@ -1427,7 +1427,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; // base: id 2 -> value 1 table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -1445,7 +1445,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); lsm_upsert(&table, vec![2, 3, 4]).await; // id 2 -> value 0 (shadows base) @@ -1460,14 +1460,14 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 8)) + .enable_memwal(MemWalWriteSpec::bucket("id", 8)) .await .unwrap(); - // Two single-row upserts that route to (likely) different buckets; each + // Two single-row upserts that route to (likely) different shards; each // closes the writer so the next opens a fresh shard. lsm_upsert(&table, vec![10]).await; - table.close_lsm_writers().await.unwrap(); + table.close_memwal_writers().await.unwrap(); lsm_upsert(&table, vec![11]).await; let lsm = table.query().execute().await.unwrap(); @@ -1482,14 +1482,14 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); lsm_upsert(&table, vec![4, 5]).await; // close flushes the active memtable to an on-disk generation and drops // the cached writer; the read must still see those rows via the shard // manifest snapshot. - table.close_lsm_writers().await.unwrap(); + table.close_memwal_writers().await.unwrap(); let lsm = table.query().execute().await.unwrap(); let ids: Vec = collect_id_value(lsm) @@ -1503,11 +1503,11 @@ mod lsm_tests { #[tokio::test] async fn lsm_read_without_spec_reads_base() { let dir = tempdir().unwrap(); - let table = id_value_table(&dir).await; // no LSM write spec + let table = id_value_table(&dir).await; // no MemWAL write spec // With no spec installed there is nothing to route: the default read and - // an explicit use_lsm(false) both read the base table without error. - for query in [table.query(), table.query().use_lsm(false)] { + // an explicit use_memwal(false) both read the base table without error. + for query in [table.query(), table.query().use_memwal(false)] { let rows = collect_id_value(query.execute().await.unwrap()).await; assert_eq!( rows.iter().map(|(id, _)| *id).collect::>(), @@ -1517,11 +1517,11 @@ mod lsm_tests { } #[tokio::test] - async fn lsm_read_unsupported_shape_errors_without_use_lsm_false() { + async fn lsm_read_unsupported_shape_errors_without_use_memwal_false() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); lsm_upsert(&table, vec![4]).await; @@ -1538,12 +1538,12 @@ mod lsm_tests { .expect("unsupported shape on a MemWAL table must error"); assert!(matches!(err, Error::NotSupported { .. }), "got {err:?}"); - // use_lsm(false) is the escape hatch: it reads the base table only. + // use_memwal(false) is the escape hatch: it reads the base table only. let rows = collect_id_value( table .query() .with_row_id() - .use_lsm(false) + .use_memwal(false) .execute() .await .unwrap(), @@ -1555,7 +1555,7 @@ mod lsm_tests { ); } - /// A table with `id`/`text` and a primary key, ready for an LSM write spec. + /// A table with `id`/`text` and a primary key, ready for a MemWAL write spec. async fn lsm_text_table(dir: &tempfile::TempDir) -> crate::Table { let conn = connect(dir.path().to_str().unwrap()) .execute() @@ -1570,7 +1570,7 @@ mod lsm_tests { table } - /// Upsert `rows` through the LSM write path. + /// Upsert `rows` through the MemWAL write path. async fn upsert_text(table: &crate::Table, rows: Vec<(i64, &str)>) { let mut builder = table.merge_insert(&[]); builder @@ -1614,7 +1614,7 @@ mod lsm_tests { let table = lsm_text_table(&dir).await; // The default set, on a table with no FTS index yet. table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -1648,7 +1648,7 @@ mod lsm_tests { /// A spec is replaced by unsetting it and setting the new one. /// - /// `unset_lsm_write_spec` drains the open writer, so the rows it held + /// `disable_memwal` drains the open writer, so the rows it held /// survive the replacement, and the writer the next write opens builds its /// MemTables from the new set. #[tokio::test] @@ -1668,15 +1668,15 @@ mod lsm_tests { // Installed maintaining nothing, then written to: the writer opens with // no FTS index. table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new())) + .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(Vec::new())) .await .unwrap(); upsert_text(&table, vec![(99, "zebra")]).await; // Setting over an installed spec is refused; unset, then set. let err = table - .set_lsm_write_spec( - LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index.clone()]), + .enable_memwal( + MemWalWriteSpec::unsharded().with_maintained_indexes(vec![fts_index.clone()]), ) .await .expect_err("an installed spec cannot be set over"); @@ -1684,9 +1684,9 @@ mod lsm_tests { err.to_string().contains("already set"), "unexpected error: {err}" ); - table.unset_lsm_write_spec().await.unwrap(); + table.disable_memwal().await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) + .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) .await .unwrap(); @@ -1716,7 +1716,7 @@ mod lsm_tests { /// A row written before the index exists is searchable once it does. /// /// The MemTable holding it was built without the index. Creating the index - /// replaces the writer's configs, which seals that MemTable and waits for + /// replaces the writer's configs, which freezes that MemTable and waits for /// its flush, so no resident MemTable is left that cannot answer the read. #[tokio::test] async fn lsm_a_row_written_before_the_index_is_searchable_after_it() { @@ -1727,7 +1727,7 @@ mod lsm_tests { let table = lsm_text_table(&dir).await; // No index yet, and the default set: maintain whatever the table has. table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -1783,7 +1783,7 @@ mod lsm_tests { .unwrap(); let fts_index = table.list_indices().await.unwrap()[0].name.clone(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) + .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) .await .unwrap(); @@ -1878,7 +1878,7 @@ mod lsm_tests { } let vec_index = table.list_indices().await.unwrap()[0].name.clone(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![vec_index])) + .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(vec![vec_index])) .await .unwrap(); @@ -2129,8 +2129,8 @@ mod lsm_tests { } /// A kind Lance has no memtable index for is refused when named until the - /// table's registry maintains it; then it is maintained, flushed with each - /// generation, and answers searches on the flushed generations. + /// table's registry maintains it; then it is maintained, flushed into each + /// SSTable, and answers searches on the SSTables. #[tokio::test] async fn lsm_maintains_an_index_kind_from_the_tables_registry() { let dir = tempdir().unwrap(); @@ -2150,10 +2150,11 @@ mod lsm_tests { .execute() .await .unwrap(); - let spec = - || LsmWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]); + let spec = || { + MemWalWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]) + }; - let refused = table.set_lsm_write_spec(spec()).await.unwrap_err(); + let refused = table.enable_memwal(spec()).await.unwrap_err(); assert!( refused.to_string().contains("region_bitmap") && refused @@ -2166,7 +2167,7 @@ mod lsm_tests { .with_plugin(Arc::new(BitmapAsBTree)) .unwrap(); table.as_native().unwrap().set_mem_index_registry(registry); - table.set_lsm_write_spec(spec()).await.unwrap(); + table.enable_memwal(spec()).await.unwrap(); // The registry is not stored with the table. let other = conn.open_table("t").execute().await.unwrap(); @@ -2224,13 +2225,13 @@ mod lsm_tests { active.index_store.index_names() ); - table.close_lsm_writers().await.unwrap(); + table.close_memwal_writers().await.unwrap(); assert_eq!(fresh().await, vec![10, 11]); - assert_generations_answer_through_region_bitmap(&table, vec![10, 11]).await; + assert_sstables_answer_through_region_bitmap(&table, vec![10, 11]).await; } - /// The generations each shard's manifest publishes. - async fn published_generations(table: &Table) -> Vec { + /// The SSTables each shard's manifest publishes. + async fn published_sstables(table: &Table) -> Vec { use lance::dataset::mem_wal::ShardManifestStore; let uri = table.uri().await.unwrap(); @@ -2238,7 +2239,7 @@ mod lsm_tests { .await .unwrap(); let mem_wal = std::path::Path::new(&uri).join("_mem_wal"); - let mut generations = Vec::new(); + let mut sstables = Vec::new(); for shard in std::fs::read_dir(&mem_wal).unwrap() { let Ok(shard_id) = shard.unwrap().file_name().to_string_lossy().parse() else { continue; @@ -2250,30 +2251,30 @@ mod lsm_tests { .expect("a shard directory has a manifest"); for sstable in manifest.sstables { let path = mem_wal.join(shard_id.to_string()).join(&sstable.path); - generations.push(lance::Dataset::open(path.to_str().unwrap()).await.unwrap()); + sstables.push(lance::Dataset::open(path.to_str().unwrap()).await.unwrap()); } } - generations + sstables } - /// Every published generation carries `region_bitmap` as a bitmap index on + /// Every published SSTable carries `region_bitmap` as a bitmap index on /// `region`, and a search for `region = 'fresh'` on it uses that index. - async fn assert_generations_answer_through_region_bitmap(table: &Table, fresh: Vec) { + async fn assert_sstables_answer_through_region_bitmap(table: &Table, fresh: Vec) { use lance::index::DatasetIndexExt; - let generations = published_generations(table).await; + let sstables = published_sstables(table).await; assert!( - !generations.is_empty(), - "closing the writer flushed a generation" + !sstables.is_empty(), + "closing the writer flushed an SSTable" ); let mut found = Vec::new(); - for generation in generations { - let region = generation.schema().field("region").unwrap().id; - let indices = generation.load_indices().await.unwrap(); + for sstable in sstables { + let region = sstable.schema().field("region").unwrap().id; + let indices = sstable.load_indices().await.unwrap(); let index = indices .iter() .find(|index| index.name == "region_bitmap") - .expect("the generation carries the index the plugin trained"); + .expect("the SSTable carries the index the plugin trained"); assert_eq!(index.fields, vec![region]); assert!( index @@ -2284,7 +2285,7 @@ mod lsm_tests { index.index_details ); - let mut scan = generation.scan(); + let mut scan = sstable.scan(); scan.filter("region = 'fresh'").unwrap(); let plan = scan.explain_plan(false).await.unwrap(); assert!(plan.contains("ScalarIndexQuery"), "{plan}"); @@ -2332,7 +2333,7 @@ mod lsm_tests { "a clone shares the handle's registry" ); table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -2375,13 +2376,13 @@ mod lsm_tests { active.index_store.index_names() ); - table.close_lsm_writers().await.unwrap(); - // Only the generation written after the index exists carries it. - let generations = published_generations(&table).await; + table.close_memwal_writers().await.unwrap(); + // Only the SSTable written after the index exists carries it. + let sstables = published_sstables(&table).await; let mut carrying = 0; - for generation in &generations { + for sstable in &sstables { use lance::index::DatasetIndexExt; - if generation + if sstable .load_indices() .await .unwrap() @@ -2393,8 +2394,8 @@ mod lsm_tests { } assert!( carrying >= 1, - "{} generations, none carry the index", - generations.len() + "{} SSTables, none carry the index", + sstables.len() ); } } diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index 33bf54535..d26be28cc 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -1,10 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! MemWAL LSM write path for `merge_insert`. +//! MemWAL write path for `merge_insert`. //! -//! [`set_lsm_write_spec`] installs an [`LsmWriteSpec`] on a table by creating -//! Lance's MemWAL index; [`unset_lsm_write_spec`] removes it. Once a spec is +//! [`enable_memwal`] installs an [`MemWalWriteSpec`] on a table by creating +//! Lance's MemWAL index; [`disable_memwal`] removes it. Once a spec is //! installed, `merge_insert` upsert calls are dispatched through Lance's MemWAL //! `ShardWriter` (LSM-style append) instead of the standard merge path — see //! [`lsm_dispatch_decision`] and [`execute_lsm_merge_insert`]. @@ -14,7 +14,7 @@ //! unsharded). [`MergeInsertBuilder::validate_single_shard`] controls whether //! every row is checked or only the first. A dataset writes to one shard at a //! time; its writer is cached in the [`ShardWriterCache`] held alongside the -//! dataset, and [`close_lsm_writers`] closes it. +//! dataset, and [`close_memwal_writers`] closes it. use std::collections::HashMap; use std::sync::Arc; @@ -40,9 +40,9 @@ use uuid::Uuid; use crate::error::{Error, Result}; use crate::index::IndexConfig; use crate::table::merge::{MergeInsertBuilder, MergeResult}; -use crate::table::{BaseTable, LsmWriteSpec, NativeTable}; +use crate::table::{BaseTable, MemWalWriteSpec, NativeTable}; -/// Spec id of the sole sharding spec installed by [`set_lsm_write_spec`]. +/// Spec id of the sole sharding spec installed by [`enable_memwal`]. /// Must match Lance's `InitializeMemWalBuilder` (`SHARDING_SPEC_ID`). const SHARDING_SPEC_ID: u32 = 1; @@ -61,23 +61,23 @@ const NUM_BUCKETS_PARAM: &str = "num_buckets"; const SHARD_NAMESPACE: Uuid = Uuid::from_u128(0x4c53_4d57_5249_5445_5f53_4841_5244_3031); // ============================================================================= -// set_lsm_write_spec +// enable_memwal // ============================================================================= -/// Install an [`LsmWriteSpec`] on the table. +/// Install an [`MemWalWriteSpec`] on the table. /// /// The bucket / identity / unsharded sharding spec is constructed and validated /// by Lance's /// [`InitializeMemWalBuilder`](lance::dataset::mem_wal::InitializeMemWalBuilder). #[allow(clippy::redundant_pub_crate)] -pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) -> Result<()> { +pub(crate) async fn enable_memwal(table: &NativeTable, spec: MemWalWriteSpec) -> Result<()> { table.dataset.ensure_mutable()?; { let dataset = table.dataset.get().await?; if dataset.mem_wal_index_details().await?.is_some() { return Err(Error::InvalidInput { - message: "set_lsm_write_spec: an LSM write spec is already set on this \ + message: "enable_memwal: a MemWAL write spec is already set on this \ table and cannot be changed" .into(), }); @@ -115,14 +115,14 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) let schema = arrow_schema::Schema::from(dataset.schema()); if !crate::table::computed_columns::computed_columns(&schema).is_empty() { return Err(Error::NotSupported { - message: "an LSM write spec cannot be installed on a table with computed \ + message: "a MemWAL write spec cannot be installed on a table with computed \ columns: rows in un-compacted tiers are invisible to refresh" .into(), }); } if crate::materialized_view::read_definition(&dataset.schema().metadata)?.is_some() { return Err(Error::NotSupported { - message: "an LSM write spec cannot be installed on a materialized view: \ + message: "a MemWAL write spec cannot be installed on a materialized view: \ rows in un-compacted tiers are invisible to refresh" .into(), }); @@ -131,7 +131,7 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) .initialize_mem_wal() .mem_index_registry(table.mem_index_registry()); let writer_config_defaults = match spec { - LsmWriteSpec::Bucket { + MemWalWriteSpec::Bucket { column, num_buckets, writer_config_defaults, @@ -140,7 +140,7 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) builder = builder.bucket_sharding(column, num_buckets); writer_config_defaults } - LsmWriteSpec::Identity { + MemWalWriteSpec::Identity { column, writer_config_defaults, .. @@ -148,7 +148,7 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) builder = builder.identity_sharding(column); writer_config_defaults } - LsmWriteSpec::Unsharded { + MemWalWriteSpec::Unsharded { writer_config_defaults, .. } => { @@ -178,22 +178,22 @@ fn index_name_list(indices: &[IndexConfig]) -> String { } // ============================================================================= -// unset_lsm_write_spec +// disable_memwal // ============================================================================= -/// Remove the [`LsmWriteSpec`] from the table by dropping the MemWAL index. +/// Remove the [`MemWalWriteSpec`] from the table by dropping the MemWAL index. /// /// Any cached shard writers are drained and closed first. Errors if no spec is /// currently set. #[allow(clippy::redundant_pub_crate)] -pub(crate) async fn unset_lsm_write_spec(table: &NativeTable) -> Result<()> { +pub(crate) async fn disable_memwal(table: &NativeTable) -> Result<()> { table.dataset.ensure_mutable()?; { let dataset = table.dataset.get().await?; if dataset.mem_wal_index_details().await?.is_none() { return Err(Error::InvalidInput { - message: "unset_lsm_write_spec: no LSM write spec is set on this table".into(), + message: "disable_memwal: no MemWAL write spec is set on this table".into(), }); } } @@ -209,46 +209,46 @@ pub(crate) async fn unset_lsm_write_spec(table: &NativeTable) -> Result<()> { } // ============================================================================= -// get_lsm_write_spec +// get_memwal_spec // ============================================================================= -/// Read the [`LsmWriteSpec`] currently installed on the table, if any. +/// Read the [`MemWalWriteSpec`] currently installed on the table, if any. /// /// Reconstructs the spec from the MemWAL index's stored details — the same -/// metadata [`set_lsm_write_spec`] writes — read directly via +/// metadata [`enable_memwal`] writes — read directly via /// [`mem_wal_index_details`](DatasetMemWalExt::mem_wal_index_details) (the raw /// manifest record, not the `describe_indices` enrichment path, so it is /// unaffected by system-index filtering there). Returns `Ok(None)` when no /// MemWAL index is installed. #[allow(clippy::redundant_pub_crate)] -pub(crate) async fn get_lsm_write_spec(table: &NativeTable) -> Result> { +pub(crate) async fn get_memwal_spec(table: &NativeTable) -> Result> { let dataset = table.dataset.get().await?; let Some(details) = dataset.mem_wal_index_details().await? else { return Ok(None); }; - let spec = lsm_write_spec_from_details(&details, dataset.schema())?; + let spec = memwal_spec_from_details(&details, dataset.schema())?; Ok(Some(spec)) } -/// Reconstruct the public [`LsmWriteSpec`] from stored MemWAL index details. +/// Reconstruct the public [`MemWalWriteSpec`] from stored MemWAL index details. /// /// The sharding transform and its parameters recover the mode; the routing /// column is resolved from the sharding field's source id back to its schema /// name (only bucket / identity carry a column). `maintained_indexes` and /// `writer_config_defaults` are copied verbatim, so the result round-trips -/// what `set_lsm_write_spec` installed. -fn lsm_write_spec_from_details( +/// what `enable_memwal` installed. +fn memwal_spec_from_details( details: &MemWalIndexDetails, schema: &LanceSchema, -) -> Result { +) -> Result { let spec = details .sharding_specs .first() .ok_or_else(|| Error::Runtime { - message: "get_lsm_write_spec: MemWAL index has no sharding spec".to_string(), + message: "get_memwal_spec: MemWAL index has no sharding spec".to_string(), })?; let field = spec.fields.first().ok_or_else(|| Error::Runtime { - message: "get_lsm_write_spec: MemWAL index has an empty sharding spec".to_string(), + message: "get_memwal_spec: MemWAL index has an empty sharding spec".to_string(), })?; let base = match field.transform.as_deref() { @@ -259,16 +259,16 @@ fn lsm_write_spec_from_details( .and_then(|s| s.parse::().ok()) .filter(|n| *n > 0) .ok_or_else(|| Error::Runtime { - message: "get_lsm_write_spec: MemWAL bucket spec has a missing or invalid num_buckets parameter".to_string(), + message: "get_memwal_spec: MemWAL bucket spec has a missing or invalid num_buckets parameter".to_string(), })?; - LsmWriteSpec::bucket(sharding_column(field, schema)?, num_buckets) + MemWalWriteSpec::bucket(sharding_column(field, schema)?, num_buckets) } - Some(IDENTITY_TRANSFORM) => LsmWriteSpec::identity(sharding_column(field, schema)?), - Some(UNSHARDED_TRANSFORM) => LsmWriteSpec::unsharded(), + Some(IDENTITY_TRANSFORM) => MemWalWriteSpec::identity(sharding_column(field, schema)?), + Some(UNSHARDED_TRANSFORM) => MemWalWriteSpec::unsharded(), other => { return Err(Error::Runtime { message: format!( - "get_lsm_write_spec: MemWAL index has an unsupported sharding transform {:?}", + "get_memwal_spec: MemWAL index has an unsupported sharding transform {:?}", other ), }); @@ -284,30 +284,30 @@ fn lsm_write_spec_from_details( /// Resolve the single routing column name from a sharding field's source id. /// -/// `set_lsm_write_spec` records the shard column by its Lance field id, so the +/// `enable_memwal` records the shard column by its Lance field id, so the /// name is recovered by looking that id up in the current schema. fn sharding_column(field: &ShardingField, schema: &LanceSchema) -> Result { let source_id = *field.source_ids.first().ok_or_else(|| Error::Runtime { - message: "get_lsm_write_spec: sharding field has no source column".to_string(), + message: "get_memwal_spec: sharding field has no source column".to_string(), })?; schema .field_by_id(source_id) .map(|f| f.name.clone()) .ok_or_else(|| Error::Runtime { message: format!( - "get_lsm_write_spec: sharding source field id {} not found in schema", + "get_memwal_spec: sharding source field id {} not found in schema", source_id ), }) } // ============================================================================= -// close_lsm_writers +// close_memwal_writers // ============================================================================= /// Drain and close every cached MemWAL shard writer for the table. #[allow(clippy::redundant_pub_crate)] -pub(crate) async fn close_lsm_writers(table: &NativeTable) -> Result<()> { +pub(crate) async fn close_memwal_writers(table: &NativeTable) -> Result<()> { table.dataset.shard_writer().drain_and_close().await } @@ -321,7 +321,7 @@ pub(crate) async fn close_lsm_writers(table: &NativeTable) -> Result<()> { /// so the writer lives where the dataset lives — cached for the session and /// reused across `merge_insert` calls. A dataset writes to one shard at a /// time; routing a `merge_insert` to a different shard requires closing the -/// current writer first via [`close_lsm_writers`]. `ShardWriter::put` takes +/// current writer first via [`close_memwal_writers`]. `ShardWriter::put` takes /// `&self`, so concurrent puts on the cached writer are safe; `close` consumes /// the writer, so the entry wraps it in `RwLock>`. #[derive(Default)] @@ -370,8 +370,8 @@ impl ShardWriterEntry { } /// The cached writer's latest in-memory manifest (current generation + - /// flushed generations). `Ok(None)` if the writer was already closed. - /// Used by the LSM read path to snapshot this shard authoritatively + /// SSTables). `Ok(None)` if the writer was already closed. + /// Used by the MemWAL read path to snapshot this shard authoritatively /// without re-reading the on-disk manifest. async fn manifest(&self) -> Result> { let guard = self.inner.read().await; @@ -449,7 +449,7 @@ impl ShardWriterCache { /// writer opened afterwards reads the committed set. /// /// Holds the entry's read lock across the refresh, so a concurrent close - /// waits rather than retiring the writer mid-seal. Writes take the same + /// waits rather than retiring the writer mid-freeze. Writes take the same /// lock first, so the order matches theirs. pub(crate) async fn refresh_maintained_indexes(&self, dataset: &Dataset) { let entry = { @@ -463,13 +463,13 @@ impl ShardWriterCache { let Some(writer) = guard.as_ref() else { return; }; - // Replacing the configs seals the active memtable so the next one carries + // Replacing the configs freezes the active memtable so the next one carries // the new indexes. The outgoing one keeps the set it was built with, so // its flush has to land before a read: while it is still frozen, an // indexed read sees a resident memtable that cannot answer and is // refused. - let sealed = match dataset.refresh_mem_wal_index_configs(writer).await { - Ok(sealed) => sealed, + let frozen = match dataset.refresh_mem_wal_index_configs(writer).await { + Ok(frozen) => frozen, Err(error) => { log::warn!( "the maintained index set changed but the open shard writer could \ @@ -478,18 +478,18 @@ impl ShardWriterCache { return; } }; - if let Some(fence) = sealed + if let Some(fence) = frozen && let Err(error) = fence.wait().await { log::warn!( "the shard writer took the new maintained index set, but the memtable \ - it sealed was not flushed, so an indexed read stays refused until \ + it froze was not flushed, so an indexed read stays refused until \ it is: {error}" ); } } - /// Snapshot the cached writer's shard for the LSM read path: its shard id, + /// Snapshot the cached writer's shard for the MemWAL read path: its shard id, /// authoritative in-memory manifest, and active + frozen memtable refs. /// Returns `None` when no writer is currently cached (e.g. nothing has been /// written this session, or the writer was closed). @@ -537,7 +537,7 @@ fn check_shard_match(cached: Uuid, wanted: Uuid) -> Result<()> { } Err(Error::InvalidInput { message: format!( - "merge_insert: a shard writer is already open for shard {} but this input routes to shard {}; call close_lsm_writers before writing to a different shard", + "merge_insert: a shard writer is already open for shard {} but this input routes to shard {}; call close_memwal_writers before writing to a different shard", cached, wanted ), }) @@ -569,7 +569,7 @@ pub(crate) struct LsmPlan { /// Outcome of [`lsm_dispatch_decision`]. #[allow(clippy::redundant_pub_crate)] pub(crate) enum LsmDispatch { - /// No LSM write spec applies; use the standard `merge_insert` path. + /// No MemWAL write spec applies; use the standard `merge_insert` path. Standard, /// Route the `merge_insert` through the MemWAL shard writer. Lsm(LsmPlan), @@ -583,18 +583,18 @@ pub(crate) async fn lsm_dispatch_decision( params: &MergeInsertBuilder, ) -> Result { // Explicit opt-out: use the standard path regardless of any installed spec. - if params.use_lsm == Some(false) { + if params.use_memwal == Some(false) { return Ok(LsmDispatch::Standard); } let dataset = table.dataset.get().await?; let Some(details) = dataset.mem_wal_index_details().await? else { - // No write spec installed. `use_lsm(true)` demanded MemWAL routing, so + // No write spec installed. `use_memwal(true)` demanded MemWAL routing, so // that is an error; otherwise fall back to the standard path. - if params.use_lsm == Some(true) { + if params.use_memwal == Some(true) { return Err(Error::InvalidInput { - message: "use_lsm(true) was set but the table has no MemWAL write spec; \ - install one with set_lsm_write_spec or leave use_lsm unset" + message: "use_memwal(true) was set but the table has no MemWAL write spec; \ + install one with enable_memwal or leave use_memwal unset" .to_string(), }); } @@ -616,7 +616,7 @@ pub(crate) async fn lsm_dispatch_decision( if !params.on.is_empty() && params.on != pk_cols { return Err(Error::InvalidInput { message: format!( - "merge_insert: `on` columns {:?} must match the table's unenforced primary key {:?} when an LSM write spec is set; pass an empty `on` to default to the primary key", + "merge_insert: `on` columns {:?} must match the table's unenforced primary key {:?} when a MemWAL write spec is set; pass an empty `on` to default to the primary key", params.on, pk_cols ), }); @@ -624,7 +624,7 @@ pub(crate) async fn lsm_dispatch_decision( if !is_upsert_only(params) { return Err(Error::InvalidInput { - message: "merge_insert: when an LSM write spec is set, only the upsert form (when_matched_update_all without a filter + when_not_matched_insert_all, no by-source delete) is supported; call use_lsm(false) to use the standard merge_insert path".to_string(), + message: "merge_insert: when a MemWAL write spec is set, only the upsert form (when_matched_update_all without a filter + when_not_matched_insert_all, no by-source delete) is supported; call use_memwal(false) to use the standard merge_insert path".to_string(), }); } diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index 3ab54ae07..90a1d3217 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -95,8 +95,8 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R return Ok(false); } // A MemWAL write spec means reads auto-route through the LSM scanner in - // `create_plan` even when `use_lsm` is unset. The namespace request has no - // use_lsm field, so pushing the default query down would silently omit + // `create_plan` even when `use_memwal` is unset. The namespace request has no + // use_memwal field, so pushing the default query down would silently omit // un-compacted rows — force local execution whenever a spec is installed. let dataset = table.dataset.get().await?; if dataset.mem_wal_index_details().await?.is_some() { @@ -106,10 +106,10 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R } fn requires_local_namespace_execution(query: &AnyQuery) -> bool { - // The namespace QueryTable request cannot represent approx_mode, search_effort or use_lsm. - // For use_lsm that is worse than a tuning miss: MemWAL read routing lives only + // The namespace QueryTable request cannot represent approx_mode, search_effort or use_memwal. + // For use_memwal that is worse than a tuning miss: MemWAL read routing lives only // in `create_plan`, so a pushed-down query would return stale base-only data. - if query.base().use_lsm.is_some() || query.base().take_offsets.is_some() { + if query.base().use_memwal.is_some() || query.base().take_offsets.is_some() { return true; } matches!( @@ -167,25 +167,25 @@ pub async fn create_plan( let ds_ref = table.dataset.get().await?; - // MemWAL read routing driven by `use_lsm`: + // MemWAL read routing driven by `use_memwal`: // * unset — route through the LSM scanner iff the table carries a write spec - // * Some(true) — force LSM routing; error if the table has no write spec + // * Some(true) — force MemWAL routing; error if the table has no write spec // * Some(false) — read the base table only, bypassing the MemWAL // The LSM scanner surfaces in-flight `merge_insert` data (active/frozen - // memtables + flushed generations); validation and dispatch live in `lsm`. + // memtables + SSTables); validation and dispatch live in `lsm`. let has_spec = ds_ref.mem_wal_index_details().await?.is_some(); - let use_lsm = match query.base.use_lsm { + let use_memwal = match query.base.use_memwal { Some(true) if !has_spec => { return Err(Error::InvalidInput { - message: "use_lsm(true) was set but the table has no MemWAL write spec; \ - install one with set_lsm_write_spec or leave use_lsm unset" + message: "use_memwal(true) was set but the table has no MemWAL write spec; \ + install one with enable_memwal or leave use_memwal unset" .to_string(), }); } Some(enable) => enable, None => has_spec, }; - if use_lsm { + if use_memwal { return lsm::create_lsm_plan(table, ds_ref, query).await; } @@ -1195,7 +1195,7 @@ mod tests { table.set_unenforced_primary_key(["id"]).await.unwrap(); table - .set_lsm_write_spec(crate::table::LsmWriteSpec::unsharded()) + .enable_memwal(crate::table::MemWalWriteSpec::unsharded()) .await .unwrap(); let lsm_error = table @@ -1395,7 +1395,7 @@ mod tests { } #[tokio::test] - async fn test_execute_query_use_lsm_with_namespace_pushdown_runs_locally() { + async fn test_execute_query_use_memwal_with_namespace_pushdown_runs_locally() { use crate::connect; use crate::table::query::execute_query; use arrow_array::{Int32Array, RecordBatch}; @@ -1418,7 +1418,7 @@ mod tests { .unwrap(); let table = conn - .create_table("test_use_lsm_namespace_fallback", batch) + .create_table("test_use_memwal_namespace_fallback", batch) .execute() .await .unwrap(); @@ -1429,13 +1429,13 @@ mod tests { .pushdown_operations .insert(NamespaceClientPushdownOperation::QueryTable); - // `use_lsm` set (even to false) must force local execution — the namespace - // request has no use_lsm field, so a pushdown would silently ignore it. + // `use_memwal` set (even to false) must force local execution — the namespace + // request has no use_memwal field, so a pushdown would silently ignore it. let query_vector = Arc::new(Float32Array::from(vec![0.0, 0.0])); let query = AnyQuery::VectorQuery(VectorQueryRequest { base: QueryRequest { limit: Some(1), - use_lsm: Some(false), + use_memwal: Some(false), ..Default::default() }, column: Some("vector".to_string()), diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 4798da431..32966e4a6 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -1,14 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! MemWAL LSM read path. +//! MemWAL read path. //! -//! When a table has an LSM write spec installed (see [`set_lsm_write_spec`]), +//! When a table has a MemWAL write spec installed (see [`enable_memwal`]), //! reads are routed through Lance's [`LsmScanner`] instead of the plain //! base-table scan unless the query sets -//! [`use_lsm(false)`](crate::query::QueryBase::use_lsm). This makes data +//! [`use_memwal(false)`](crate::query::QueryBase::use_memwal). This makes data //! written via the LSM `merge_insert` path — which lives in the active/frozen -//! in-memory memtables and the flushed SSTable generations until an external +//! in-memory memtables and the SSTables until an external //! compaction merges it into the base table — visible to queries, deduplicated by //! primary key (newest generation wins). //! @@ -17,10 +17,10 @@ //! three run through a single [`LsmScanner`], so a `where` filter is honored as a //! prefilter uniformly — including for vector search, where `LsmScanner` threads //! it into the vector planner's prefilter. Shapes the LSM path cannot honor are -//! rejected with [`Error::NotSupported`]; the caller must set `use_lsm(false)` to +//! rejected with [`Error::NotSupported`]; the caller must set `use_memwal(false)` to //! run those against the base table. //! -//! [`set_lsm_write_spec`]: crate::Table::set_lsm_write_spec +//! [`enable_memwal`]: crate::Table::enable_memwal use std::collections::HashMap; use std::sync::Arc; @@ -57,7 +57,7 @@ const LSM_OVERFETCH_FACTOR: f64 = 2.0; /// The caller guarantees `ds_ref` carries a MemWAL write spec (routing is decided /// in [`create_plan`](super::create_plan)). Errors with [`Error::NotSupported`] /// for query shapes the LSM scanner cannot honor — the caller must set -/// `use_lsm(false)` to run those against the base table. +/// `use_memwal(false)` to run those against the base table. pub(super) async fn create_lsm_plan( table: &NativeTable, ds_ref: Arc, @@ -67,11 +67,11 @@ pub(super) async fn create_lsm_plan( // A time-traveled (checked-out) handle pins an older dataset version, but the // WAL manifests and cached writer expose current live state — mixing them would - // surface WAL rows written after the requested version. `use_lsm(false)` reads + // surface WAL rows written after the requested version. `use_memwal(false)` reads // the base table at the pinned version. if table.dataset.time_travel_version().is_some() { return Err(Error::NotSupported { - message: "the MemWAL LSM scanner cannot read from a time-traveled dataset version; set use_lsm(false) to read the base table at this version".to_string(), + message: "the MemWAL LSM scanner cannot read from a time-traveled dataset version; set use_memwal(false) to read the base table at this version".to_string(), }); } @@ -80,7 +80,7 @@ pub(super) async fn create_lsm_plan( .mem_wal_index_details() .await? .ok_or_else(|| Error::Runtime { - message: "the MemWAL LSM write spec disappeared during read planning".to_string(), + message: "the MemWAL write spec disappeared during read planning".to_string(), })?; let pk_columns = pk_columns(&ds_ref)?; @@ -135,10 +135,10 @@ pub(super) async fn create_lsm_plan( restore_projection(plan, &query, &pk_columns) } -/// Reject query shapes the LSM read path does not implement. On a MemWAL table +/// Reject query shapes the MemWAL read path does not implement. On a MemWAL table /// reads route through the LSM scanner by default, so an unsupported shape is a /// hard error rather than a silent fallback to the base-only scan — which would -/// exclude un-compacted MemWAL data. The caller must set `use_lsm(false)` to +/// exclude un-compacted MemWAL data. The caller must set `use_memwal(false)` to /// run these against the base table, accepting that the results omit un-compacted /// MemWAL data. /// @@ -148,7 +148,7 @@ fn reject_unsupported(query: &VectorQueryRequest) -> Result<()> { let unsupported = |what: &str| { Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner does not support {what}; set use_lsm(false) to read the base table only (results will exclude un-compacted MemWAL data)" + "the MemWAL LSM scanner does not support {what}; set use_memwal(false) to read the base table only (results will exclude un-compacted MemWAL data)" ), }) }; @@ -275,7 +275,7 @@ fn exclusion_watermarks( exclude } -/// Assemble the per-shard snapshots (flushed SSTable generations) and the +/// Assemble the per-shard snapshots (SSTables) and the /// in-memory memtables (active + frozen) for the table. /// /// Snapshots for all shards come from their on-disk manifests; for the shard @@ -413,7 +413,7 @@ fn base_scanner( QueryFilter::Datafusion(expr) => scanner.filter_expr(expr.clone()), QueryFilter::Substrait(_) => { return Err(Error::NotSupported { - message: "the MemWAL LSM scanner does not support Substrait filters; set use_lsm(false) to read the base table only".to_string(), + message: "the MemWAL LSM scanner does not support Substrait filters; set use_memwal(false) to read the base table only".to_string(), }); } }; @@ -458,11 +458,11 @@ async fn fts_plan( offset: Option, ) -> Result> { // Pre-check for a lancedb-flavored error; `LsmScanner` also validates the - // single-column requirement, but without the `use_lsm(false)` guidance. + // single-column requirement, but without the `use_memwal(false)` guidance. let columns: Vec = fts.columns().into_iter().collect(); if columns.len() > 1 { return Err(Error::NotSupported { - message: "the MemWAL LSM scanner full-text search supports a single column; set use_lsm(false) to read the base table only".to_string(), + message: "the MemWAL LSM scanner full-text search supports a single column; set use_memwal(false) to read the base table only".to_string(), }); } let column = columns.first().ok_or_else(|| Error::NotSupported { @@ -482,7 +482,7 @@ async fn fts_plan( { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise un-compacted documents are omitted. set use_lsm(false) to read the base table only" + "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (MemWalWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise un-compacted documents are omitted. set use_memwal(false) to read the base table only" ), }); } @@ -633,7 +633,7 @@ fn resolve_single_index( if names.len() > 1 { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner cannot resolve the {arm} index catch-up watermark for '{column}': it has multiple {arm} indexes; set use_lsm(false) to read the base table only" + "the MemWAL LSM scanner cannot resolve the {arm} index catch-up watermark for '{column}': it has multiple {arm} indexes; set use_memwal(false) to read the base table only" ), }); } @@ -682,7 +682,7 @@ fn restore_projection( /// /// Note: the base and SSTable arms use `fast_search` (indexed data only), so a /// base-table row not covered by a vector index is invisible here — it surfaces -/// only via the memtable or `use_lsm(false)`. +/// only via the memtable or `use_memwal(false)`. #[allow(clippy::too_many_arguments)] async fn vector_plan( dataset: &Dataset, @@ -717,7 +717,7 @@ async fn vector_plan( if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only" + "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (MemWalWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_memwal(false) to read the base table only" ), }); } @@ -726,7 +726,7 @@ async fn vector_plan( // clear error rather than failing deep in the planner. if is_binary_vector_column(&arrow_schema, &column) { return Err(Error::NotSupported { - message: "the MemWAL LSM scanner does not support binary (uint8) vector search; set use_lsm(false) to read the base table only".to_string(), + message: "the MemWAL LSM scanner does not support binary (uint8) vector search; set use_memwal(false) to read the base table only".to_string(), }); } @@ -855,7 +855,7 @@ mod tests { fn exclusion_watermark_gates_on_lagging_index_catchup() { let shard = Uuid::from_u128(1); let details = MemWalIndexDetails { - // Compaction has drained generations through 5 into the base table... + // Compaction has drained SSTables through generation 5 into the base table... compacted_sstables: vec![CompactedSsTable::new(shard, 5)], // ...but the FTS index has only caught up through generation 2. index_catchup: vec![IndexCatchupProgress::new( diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 0119ba36d..4274aaa6f 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -87,7 +87,7 @@ async fn execute_refresh_column_with_source( column: &str, ) -> Result { table.dataset.ensure_mutable()?; - ensure_no_lsm_write_spec(table).await?; + ensure_no_memwal_spec(table).await?; let dataset = table.dataset.get().await?; let expression = declared_expression(&dataset, column)?; @@ -303,7 +303,7 @@ pub(crate) async fn execute_refresh_column_async( // Validate before spawning so bad input is reported by this call rather // than only by the job. table.dataset.ensure_mutable()?; - ensure_no_lsm_write_spec(table).await?; + ensure_no_memwal_spec(table).await?; let dataset = table.dataset.get().await?; declared_expression(&dataset, column)?; drop(dataset); @@ -323,12 +323,12 @@ pub(crate) async fn execute_refresh_column_async( }))) } -/// Refuse to refresh under an LSM write spec. +/// Refuse to refresh under a MemWAL write spec. /// /// Refresh enumerates base fragments, and a write spec keeps visible rows in /// un-compacted MemWAL tiers it cannot reach -- success would silently omit /// readable rows. -async fn ensure_no_lsm_write_spec(table: &NativeTable) -> Result<()> { +async fn ensure_no_memwal_spec(table: &NativeTable) -> Result<()> { use lance::dataset::mem_wal::DatasetMemWalExt; // Unset drops the MemWAL index, so the spec alone stops describing a table @@ -341,7 +341,7 @@ async fn ensure_no_lsm_write_spec(table: &NativeTable) -> Result<()> { .list_mem_wal_latest_shard_ids() .await? .is_empty(); - if retained_sstables || table.get_lsm_write_spec().await?.is_some() { + if retained_sstables || table.get_memwal_spec().await?.is_some() { return Err(Error::NotSupported { message: "refresh_column is not supported on a table with an LSM write \ spec: rows in un-compacted tiers are invisible to refresh" @@ -1637,7 +1637,7 @@ mod tests { /// writer left in that state. #[tokio::test] async fn test_refresh_refuses_a_foreign_lsm_state() { - use crate::table::LsmWriteSpec; + use crate::table::MemWalWriteSpec; let tmp_dir = tempfile::tempdir().unwrap(); let conn = connect(tmp_dir.path().to_str().unwrap()) @@ -1655,7 +1655,7 @@ mod tests { let table = conn.create_table("lsm", batch).execute().await.unwrap(); table.set_unenforced_primary_key(["x"]).await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); super::super::computed_columns::add_foreign_kind(&table, "doubled", "sql").await; @@ -1673,7 +1673,7 @@ mod tests { /// flag still marks retained SSTable rows; refresh refuses on the flag. #[tokio::test] async fn test_refresh_refuses_retained_catchup_state() { - use crate::table::LsmWriteSpec; + use crate::table::MemWalWriteSpec; let tmp_dir = tempfile::tempdir().unwrap(); let conn = connect(tmp_dir.path().to_str().unwrap()) @@ -1697,14 +1697,14 @@ mod tests { .unwrap(); table.set_unenforced_primary_key(["x"]).await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); let mut merge = table.merge_insert(&["x"]); merge .when_matched_update_all(None) .when_not_matched_insert_all() - .use_lsm(true); + .use_memwal(true); merge .execute(Box::new(arrow_array::RecordBatchIterator::new( vec![Ok(batch)], @@ -1712,7 +1712,7 @@ mod tests { ))) .await .unwrap(); - table.unset_lsm_write_spec().await.unwrap(); + table.disable_memwal().await.unwrap(); super::super::computed_columns::add_foreign_kind(&table, "doubled", "sql").await; let err = table.refresh_column("doubled").await.unwrap_err(); diff --git a/rust/lancedb/src/table/schema_evolution.rs b/rust/lancedb/src/table/schema_evolution.rs index 4f229835b..1bb57d6c0 100644 --- a/rust/lancedb/src/table/schema_evolution.rs +++ b/rust/lancedb/src/table/schema_evolution.rs @@ -131,7 +131,7 @@ pub(crate) async fn execute_declare( ) -> Result { use lance::dataset::mem_wal::DatasetMemWalExt; - // An LSM write spec keeps visible rows in tiers refresh cannot reach; + // A MemWAL write spec keeps visible rows in tiers refresh cannot reach; // checked against latest committed state, not this handle's snapshot. table.checkout_latest().await?; computed_columns::ensure_not_function_bound( @@ -149,7 +149,7 @@ pub(crate) async fn execute_declare( .list_mem_wal_latest_shard_ids() .await? .is_empty(); - if retained_sstables || table.get_lsm_write_spec().await?.is_some() { + if retained_sstables || table.get_memwal_spec().await?.is_some() { return Err(Error::NotSupported { message: "computed columns are not supported on a table with an LSM write \ spec: rows in un-compacted tiers are invisible to refresh"