From 0968c21f4cee62443a4e9096ed301b4da9cb36f0 Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Fri, 16 Oct 2026 11:41:27 -0500 Subject: [PATCH 1/7] feat(remote): support set_unenforced_primary_key (#4158) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary `RemoteTable::set_unenforced_primary_key` returned `NotSupported`, so the call failed against LanceDB Cloud and enterprise from every SDK — Python and TypeScript both forward to it. That also blocked sharded LSM writes, since the server rejects bucket/identity sharding on a table that declares no unenforced primary key, and the SDK had no way to declare one. No server-side support was missing. The unenforced primary key is Lance schema field metadata, and the existing `update_field_metadata` endpoint writes exactly that, so the remote table now installs the key through it. The commit layer behind that endpoint installs the position, enforces immutability and runs `verify_primary_key()` — the same code a native table reaches, so both paths agree on semantics and not just on error messages. ## Changes - `rust/lancedb/src/table/primary_key.rs`: the request validation (single column, column exists, supported dtype, key not already set) and the metadata edit (set the 1-indexed position, clear the legacy boolean key) move into `validate()` and `install_edit()`, shared by both table types so they cannot drift. Native behaviour is unchanged. - `rust/lancedb/src/remote/table.rs`: real implementation — `check_mutable()`, validate against the described schema, then one `update_field_metadata` request. - `python/python/lancedb/remote/table.py`: drop the "Not supported on LanceDB Cloud." docstring. No binding changes were needed; Python and TypeScript already forward to the Rust `BaseTable` method. ## Tests Three mocked-server tests in `rust/lancedb/src/remote/table.rs`: - the happy path, asserting the request body carries `lance-schema:unenforced-primary-key:position: "1"` and a null legacy key on the right field path; - the invalid requests a native table also rejects (empty, compound, unknown column, unsupported dtype), each failing before any write reaches the server; - the already-set case, rejected from the schema the server already reports. `cargo test --features remote -p lancedb --lib`: 1186 passed, 0 failed. `cargo clippy --features remote -p lancedb --lib --tests` is clean. ## Note for enterprise Sophon pins the lancedb submodule at `v0.39.0-beta.6`, so this reaches enterprise on the next bump. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_011xV8EebEZDmDDV8hVBm6MD --------- Co-authored-by: Claude Opus 5 (1M context) --- nodejs/__test__/remote.test.ts | 4 - python/python/lancedb/remote/table.py | 2 +- python/python/tests/test_remote_db.py | 1 - rust/lancedb/src/remote/table.rs | 145 ++++++++++++++++++++++++-- rust/lancedb/src/table.rs | 2 +- rust/lancedb/src/table/primary_key.rs | 120 +++++++++++++-------- 6 files changed, 214 insertions(+), 60 deletions(-) diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 4f70ac1aa..c06cb737e 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -125,10 +125,6 @@ describe("remote connection", () => { const cases: [() => Promise, string][] = [ [() => table.usesV2ManifestPaths(), "uses_v2_manifest_paths"], [() => table.migrateManifestPathsV2(), "migrate_manifest_paths_v2"], - [ - () => table.setUnenforcedPrimaryKey("id"), - "set_unenforced_primary_key", - ], [() => table.optimize(), "optimize"], [ () => table.createIndex("vector", { config: Index.hnswPq() }), diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 53b6c402a..5996fdeb3 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -1034,7 +1034,7 @@ class RemoteTable(Table): return LOOP.run(self._table.drop_columns(columns)) def set_unenforced_primary_key(self, columns: Union[str, Iterable[str]]) -> None: - """Not supported for remote tables.""" + """Set the unenforced primary key for this table to a single column.""" return LOOP.run(self._table.set_unenforced_primary_key(columns)) def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None: diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 717490dcd..c22aa78cd 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1623,7 +1623,6 @@ def remote_unsupported_operations(): return [ ("uses_v2_manifest_paths", (), {}, "uses_v2_manifest_paths"), ("migrate_v2_manifest_paths", (), {}, "migrate_manifest_paths_v2"), - ("set_unenforced_primary_key", (["id"],), {}, "set_unenforced_primary_key"), ("fetch_blob_ranges", ("image", [(1, 0, 1)]), {}, "fetch_blob_ranges"), ("optimize", (), {}, "optimize"), ("create_index", ("vector",), {"config": HnswPq()}, "IVF_HNSW_PQ"), diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 7d7a6301a..d9da2c1a1 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -37,6 +37,7 @@ use crate::table::Tags; use crate::table::UpdateResult; use crate::table::lsm_stats::GetLsmStatsResponse; use crate::table::merge::MergeFilter; +use crate::table::primary_key; use crate::table::query::create_multi_vector_plan; use crate::table::write_progress::FinishOnDrop; use crate::table::{ @@ -74,6 +75,7 @@ use lance::arrow::json::{JsonDataType, JsonSchema}; use lance::dataset::refs::TagContents; use lance::dataset::scanner::DatasetRecordBatchStream; use lance::dataset::{ColumnAlteration, NewColumnTransform, Version}; +use lance_core::datatypes::Schema as LanceSchema; use lance_datafusion::exec::{OneShotExec, execute_plan}; use reqwest::{RequestBuilder, Response}; use serde::{Deserialize, Serialize}; @@ -3202,10 +3204,27 @@ impl BaseTable for RemoteTable { } } - async fn set_unenforced_primary_key(&self, _columns: &[&str]) -> Result<()> { - Err(Error::NotSupported { - message: "set_unenforced_primary_key is not supported for remote tables.".into(), - }) + /// The unenforced primary key is Lance schema field metadata, so this + /// installs it through the `update_field_metadata` endpoint. The commit + /// layer behind that endpoint is what actually installs and validates the + /// key, exactly as on a native table; the checks here only fail fast with + /// the same messages a native table gives. + async fn set_unenforced_primary_key(&self, columns: &[&str]) -> Result<()> { + self.check_mutable().await?; + + let arrow_schema = self.schema().await?; + let schema = LanceSchema::try_from(arrow_schema.as_ref()).map_err(|e| Error::Schema { + message: format!("Invalid schema: {}", e), + })?; + primary_key::validate(&schema, columns)?; + + self.update_field_metadata(&[FieldMetadataUpdate { + path: columns[0].to_string(), + metadata: primary_key::install_edit(), + replace: false, + }]) + .await?; + Ok(()) } async fn flush_lsm(&self) -> Result<()> { @@ -5936,10 +5955,6 @@ mod tests { let mut fts = FullTextSearchQuery::new("test".into()); fts.wand_factor = Some(1.0); let results = [ - ( - "set_unenforced_primary_key", - table.set_unenforced_primary_key(["id"]).await, - ), ( "optimize", table.optimize(OptimizeAction::All).await.map(|_| ()), @@ -13365,6 +13380,120 @@ mod tests { assert_eq!(result.version, 7); } + /// The unenforced primary key is field metadata, so the remote table + /// installs it through the `update_field_metadata` endpoint. + #[tokio::test] + async fn test_set_unenforced_primary_key() { + let table = Table::new_with_handler("my_table", |request| { + assert_eq!(request.method(), "POST"); + match request.url().path() { + "/v1/table/my_table/describe/" => { + let schema = Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, true), + ]); + http::Response::builder() + .status(200) + .body(describe_response(&schema)) + .unwrap() + } + "/v1/table/my_table/update_field_metadata/" => { + let body = request_body_json(&request); + assert_eq!(body["updates"].as_array().unwrap().len(), 1); + let update = &body["updates"][0]; + assert_eq!(update["path"], "id"); + assert_eq!(update["replace"], json!(false)); + assert_eq!( + update["metadata"]["lance-schema:unenforced-primary-key:position"], + "1" + ); + assert_eq!( + update["metadata"]["lance-schema:unenforced-primary-key"], + json!(null) + ); + http::Response::builder() + .status(200) + .body(r#"{"version": 3, "fields": {}}"#.to_string()) + .unwrap() + } + path => panic!("Unexpected path: {}", path), + } + }); + + table.set_unenforced_primary_key(["id"]).await.unwrap(); + } + + /// Requests the native table rejects are rejected here too, before any + /// write reaches the server. + #[tokio::test] + async fn test_set_unenforced_primary_key_rejects_invalid_requests() { + let table = Table::new_with_handler("my_table", |request| match request.url().path() { + "/v1/table/my_table/describe/" => { + let schema = Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("score", DataType::Float32, true), + ]); + http::Response::builder() + .status(200) + .body(describe_response(&schema)) + .unwrap() + } + path => panic!("Unexpected path: {}", path), + }); + + for columns in [ + vec![], + vec!["id", "score"], + vec!["nonexistent"], + vec!["score"], + ] { + let err = table + .set_unenforced_primary_key(columns.clone()) + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { .. }), + "unexpected error for {:?}: {:?}", + columns, + err + ); + } + } + + /// The key is immutable once set, and the schema the server already + /// reports is enough to say so. + #[tokio::test] + async fn test_set_unenforced_primary_key_already_set() { + let table = Table::new_with_handler("my_table", |request| match request.url().path() { + "/v1/table/my_table/describe/" => { + let schema = Schema::new(vec![ + Field::new("id", DataType::Int64, false).with_metadata(HashMap::from([( + "lance-schema:unenforced-primary-key:position".to_string(), + "1".to_string(), + )])), + Field::new("name", DataType::Utf8, false), + ]); + http::Response::builder() + .status(200) + .body(describe_response(&schema)) + .unwrap() + } + path => panic!("Unexpected path: {}", path), + }); + + for column in ["name", "id"] { + let err = table + .set_unenforced_primary_key([column]) + .await + .unwrap_err(); + assert!( + err.to_string().contains("already set"), + "unexpected error: {:?}", + err + ); + } + } + // ----- Branch support ----- /// Parse a request's in-memory JSON body. Only valid for JSON-body ops diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index ca58b54de..74b99255a 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -83,7 +83,7 @@ pub mod freshness; pub mod lsm_stats; pub mod merge; pub mod optimize; -mod primary_key; +pub(crate) mod primary_key; pub mod query; pub mod refresh; pub mod schema_evolution; diff --git a/rust/lancedb/src/table/primary_key.rs b/rust/lancedb/src/table/primary_key.rs index 8a7b48efa..d399155fd 100644 --- a/rust/lancedb/src/table/primary_key.rs +++ b/rust/lancedb/src/table/primary_key.rs @@ -11,24 +11,26 @@ //! Only a single-column primary key is supported, and the key cannot be //! changed once set. +use std::collections::HashMap; + use arrow_schema::DataType; -use lance_core::datatypes::{LANCE_UNENFORCED_PRIMARY_KEY, LANCE_UNENFORCED_PRIMARY_KEY_POSITION}; +use lance_core::datatypes::{ + Field as LanceField, LANCE_UNENFORCED_PRIMARY_KEY, LANCE_UNENFORCED_PRIMARY_KEY_POSITION, + Schema as LanceSchema, +}; use crate::error::{Error, Result}; use crate::table::NativeTable; -/// Set the unenforced primary key on `table` to the single column in `columns`. +/// Validate a `set_unenforced_primary_key` request against `schema`, returning +/// the field the key would be installed on. /// -/// Fails if `columns` is not exactly one column (compound primary keys are not -/// supported), if the column does not exist or has an unsupported dtype, or if -/// the table already has an unenforced primary key (changing the primary key -/// is not supported). -pub(super) async fn set_unenforced_primary_key( - table: &NativeTable, - columns: &[&str], -) -> Result<()> { - table.dataset.ensure_mutable()?; - +/// Shared by [`NativeTable`] and the remote table so both reject the same +/// requests with the same messages. Fails if `columns` is not exactly one +/// column (compound primary keys are not supported), if the column does not +/// exist or has an unsupported dtype, or if the table already has an +/// unenforced primary key (changing the primary key is not supported). +pub fn validate<'a>(schema: &'a LanceSchema, columns: &[&str]) -> Result<&'a LanceField> { if columns.is_empty() { return Err(Error::InvalidInput { message: "set_unenforced_primary_key: a column is required".into(), @@ -44,43 +46,71 @@ pub(super) async fn set_unenforced_primary_key( } let column = columns[0]; + // The primary key is immutable once set. The Lance commit layer is the + // source of truth for this (it also covers the concurrent-writer race); + // this check just fails fast with a clear message. + if !schema.unenforced_primary_key().is_empty() { + return Err(Error::InvalidInput { + message: "set_unenforced_primary_key: an unenforced primary key is already set on this table; changing it is not supported".into(), + }); + } + + let field = schema.field(column).ok_or_else(|| Error::InvalidInput { + message: format!( + "set_unenforced_primary_key: column '{}' not found on table", + column + ), + })?; + if !is_supported_pk_dtype(&field.data_type()) { + return Err(Error::InvalidInput { + message: format!( + "set_unenforced_primary_key: column '{}' has dtype {:?} which is not supported as a primary key. Supported: Int32, Int64, Utf8, LargeUtf8, Binary, LargeBinary, FixedSizeBinary", + column, + field.data_type() + ), + }); + } + Ok(field) +} + +/// The field metadata edit that installs the primary key on a field: keys to +/// set (`Some`) or delete (`None`). +/// +/// Position metadata is 1-indexed; `Schema::unenforced_primary_key` treats +/// position 0 as a legacy "no specific position" fallback, so the legacy +/// boolean key is cleared and only the position governs. +pub fn install_edit() -> HashMap> { + HashMap::from([ + (LANCE_UNENFORCED_PRIMARY_KEY.to_string(), None), + ( + LANCE_UNENFORCED_PRIMARY_KEY_POSITION.to_string(), + Some("1".to_string()), + ), + ]) +} + +/// Set the unenforced primary key on `table` to the single column in `columns`. +pub(super) async fn set_unenforced_primary_key( + table: &NativeTable, + columns: &[&str], +) -> Result<()> { + table.dataset.ensure_mutable()?; + let updates = { let dataset = table.dataset.get().await?; - let schema = dataset.schema(); + let field = validate(dataset.schema(), columns)?; - // The primary key is immutable once set. The Lance commit layer is the - // source of truth for this (it also covers the concurrent-writer race); - // this check just fails fast with a clear message. - if !schema.unenforced_primary_key().is_empty() { - return Err(Error::InvalidInput { - message: "set_unenforced_primary_key: an unenforced primary key is already set on this table; changing it is not supported".into(), - }); - } - - let field = schema.field(column).ok_or_else(|| Error::InvalidInput { - message: format!( - "set_unenforced_primary_key: column '{}' not found on table", - column - ), - })?; - if !is_supported_pk_dtype(&field.data_type()) { - return Err(Error::InvalidInput { - message: format!( - "set_unenforced_primary_key: column '{}' has dtype {:?} which is not supported as a primary key. Supported: Int32, Int64, Utf8, LargeUtf8, Binary, LargeBinary, FixedSizeBinary", - column, - field.data_type() - ), - }); - } - - // Position metadata is 1-indexed; `Schema::unenforced_primary_key` - // treats position 0 as a legacy "no specific position" fallback. let mut metadata = field.metadata.clone(); - metadata.remove(LANCE_UNENFORCED_PRIMARY_KEY); - metadata.insert( - LANCE_UNENFORCED_PRIMARY_KEY_POSITION.to_string(), - "1".to_string(), - ); + for (key, value) in install_edit() { + match value { + Some(value) => { + metadata.insert(key, value); + } + None => { + metadata.remove(&key); + } + } + } vec![(field_id_to_u32(field.id, &field.name)?, metadata)] }; From fe34eb354977f53e01855ddec4fba6b3a4df1333 Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Fri, 16 Oct 2026 13:33:57 -0500 Subject: [PATCH 2/7] refactor!: rename LSM APIs to MemWAL and align SSTable/table shard naming (#4075) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Renames the MemWAL surface in two passes, across Rust, Python, TypeScript, and Java, including the wire format. ### 1. MemWAL APIs drop "LSM" The write path these APIs drive is Lance's MemWAL; "LSM" described its shape rather than naming it. Turning it on and off now reads as enabling a feature rather than installing a spec. | Before | After | |---|---| | `set_lsm_write_spec` | `enable_memwal` | | `unset_lsm_write_spec` | `disable_memwal` | | `get_lsm_write_spec` | `get_memwal_spec` | | `flush_lsm` / `compact_lsm` / `checkpoint_lsm` | `flush_memwal` / `compact_memwal` / `checkpoint_memwal` | | `get_lsm_stats` | `get_memwal_stats` | | `close_lsm_writers` | `close_memwal_writers` | | `use_lsm` (query, merge insert) | `use_memwal` | | `LsmWriteSpec` / `LsmStats` | `MemWalWriteSpec` / `MemWalStats` | | Java `LanceDbTableLsm` | `LanceDbTableMemWal` | TypeScript and Java use camelCase (`enableMemWal`, `getMemWalStats`, ...); the napi methods set `js_name` so the generated names keep the `MemWal` casing. flush/compact/checkpoint keep a `_memwal` suffix so `compact_memwal` is not confused with `compact_files` / `optimize`, which compact the base table. Internal names (`LsmMode`, `LsmScanner`, the `merge::lsm` / `query::lsm` modules) are unchanged. ### 2. Stats use SSTable and table shard Aligns the stats surface with the MemWAL naming settled upstream in lance-format/lance#7943 and #7957, where the persisted unit became an **SSTable**. **Generation → SSTable.** A `GenerationStats` describes one flushed MemTable, which is an SSTable. The generation *number* is kept — an SSTable is identified by its generation — so only the noun moved. **Bucket → table shard.** Each entry is one MemWAL shard. "Bucket" names only the hash sharding *transform*, so it was already wrong for `identity` and `year` sharding, which produce shards and no buckets at all. | Before | After | |---|---| | `GenerationStats` | `SsTableStats` | | `BucketStats` | `TableShardStats` | | `stats.buckets` | `stats.table_shards` | | `BucketStats.generations` | `TableShardStats.sstables` | | `include_generation_rows` | `include_sstable_rows` | | `newest_generation` | `newest_sstable_generation` | | `outstanding_generations` | `outstanding_sstables` | Docs and comments follow suit: "seal" → "freeze", "L0" / "flushed generations" → SSTables, "bucket" (meaning a shard) → table shard. `MemWalWriteSpec::bucket`, `num_buckets`, and the `"Sealed"` shard status value are **unchanged** — those name the sharding transform or a server value, not the shard. ## Compatibility Breaking, for an experimental API. The wire format changes with the SDKs, so this must roll out together with the matching server change: - Routes: `/set_lsm_write_spec/` → `/enable_memwal/`, `/unset_lsm_write_spec/` → `/disable_memwal/`, `/get_lsm_write_spec/` → `/get_memwal_spec/`, `/flush_lsm/` → `/flush_memwal/`, `/compact_lsm/` → `/compact_memwal/`, `/get_lsm_stats/` → `/get_memwal_stats/` - Response keys: `lsm_write_spec` → `memwal_spec`, `lsm_stats` → `memwal_stats`, `buckets` → `table_shards`, `generations` → `sstables` - Request fields: `include_generation_rows` → `include_sstable_rows`; `use_lsm` → `use_memwal` on query and merge-insert bodies ## Validation - Rust: `cargo test --features remote -p lancedb --lib --tests` — 1467 lib tests and all integration test targets pass; `cargo clippy` clean - Python: rebuilt the extension; `test_remote_db`, `test_memwal_spec`, `test_merge_insert_lsm`, `test_query`, `test_permutation`, `test_table` — 655 passed; `ruff format` / `ruff check` clean - TypeScript: `pnpm build`; `table`, `merge`, `query` tests — 593 passed; `biome check` clean; `docs/src/js` regenerated with `npm run docs` - Java: `mvn test` — 37 passed (Spotless applied with JDK 11; the pinned google-java-format does not run on JDK 21) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 (1M context) --- docs/src/js/classes/AutoQuery.md | 12 +- docs/src/js/classes/MergeInsertBuilder.md | 12 +- docs/src/js/classes/Query.md | 12 +- docs/src/js/classes/Table.md | 232 ++++++------- docs/src/js/classes/TakeQuery.md | 4 +- docs/src/js/classes/VectorQuery.md | 12 +- docs/src/js/globals.md | 8 +- docs/src/js/interfaces/GenerationStats.md | 40 --- docs/src/js/interfaces/LsmStats.md | 22 -- docs/src/js/interfaces/MemWalStats.md | 22 ++ .../{LsmWriteSpec.md => MemWalWriteSpec.md} | 4 +- docs/src/js/interfaces/MemtableStats.md | 2 +- docs/src/js/interfaces/SsTableStats.md | 40 +++ .../{BucketStats.md => TableShardStats.md} | 34 +- docs/src/python/python.md | 2 +- .../materialized-views.mdx | 2 +- docs/web/maintenance/background-jobs.mdx | 2 +- docs/web/sql/ddl/table.mdx | 8 +- java/README.md | 18 +- .../src/main/java/com/lancedb/JsonFields.java | 6 +- .../LanceDbNamespaceClientBuilder.java | 4 +- .../java/com/lancedb/LanceDbRestClient.java | 13 +- ...bTableLsm.java => LanceDbTableMemWal.java} | 180 +++++----- .../src/main/java/com/lancedb/LsmStats.java | 56 ---- .../main/java/com/lancedb/MemWalStats.java | 56 ++++ ...LsmWriteSpec.java => MemWalWriteSpec.java} | 41 +-- .../main/java/com/lancedb/MemtableStats.java | 2 +- ...GenerationStats.java => SsTableStats.java} | 16 +- ...{BucketStats.java => TableShardStats.java} | 60 ++-- ...mTest.java => LanceDbTableMemWalTest.java} | 299 ++++++++--------- nodejs/__test__/table.test.ts | 98 +++--- nodejs/lancedb/index.ts | 8 +- nodejs/lancedb/merge.ts | 12 +- nodejs/lancedb/query.ts | 14 +- nodejs/lancedb/table.ts | 120 +++---- nodejs/src/merge.rs | 6 +- nodejs/src/query.rs | 18 +- nodejs/src/table.rs | 144 ++++---- python/python/lancedb/__init__.py | 4 +- python/python/lancedb/_lancedb.pyi | 44 +-- python/python/lancedb/_wal_hybrid.py | 8 +- python/python/lancedb/merge.py | 14 +- python/python/lancedb/permutation.py | 2 +- python/python/lancedb/query.py | 54 +-- python/python/lancedb/remote/table.py | 48 +-- python/python/lancedb/table.py | 162 ++++----- ..._lsm_write_spec.py => test_memwal_spec.py} | 108 +++--- python/python/tests/test_merge_insert_lsm.py | 102 +++--- python/python/tests/test_remote_db.py | 54 +-- python/python/tests/test_wal_hybrid.py | 10 +- python/python/tests/utils.py | 6 +- python/src/lib.rs | 4 +- python/src/query.rs | 28 +- python/src/table.rs | 141 ++++---- .../src/dataloader/permutation/builder.rs | 18 +- rust/lancedb/src/lib.rs | 2 +- rust/lancedb/src/materialized_view/refresh.rs | 6 +- rust/lancedb/src/query.rs | 26 +- rust/lancedb/src/query/wal_fusion.rs | 4 +- rust/lancedb/src/remote/table.rs | 312 ++++++++---------- rust/lancedb/src/remote/table/blobs.rs | 4 +- rust/lancedb/src/remote/table/insert.rs | 2 +- rust/lancedb/src/table.rs | 294 +++++++++-------- rust/lancedb/src/table/checkpoint.rs | 58 ++-- rust/lancedb/src/table/computed_columns.rs | 24 +- rust/lancedb/src/table/datafusion.rs | 14 +- .../table/{lsm_stats.rs => memwal_stats.rs} | 92 +++--- rust/lancedb/src/table/merge.rs | 197 +++++------ rust/lancedb/src/table/merge/lsm.rs | 118 +++---- rust/lancedb/src/table/query.rs | 36 +- rust/lancedb/src/table/query/lsm.rs | 46 +-- rust/lancedb/src/table/refresh.rs | 22 +- rust/lancedb/src/table/schema_evolution.rs | 4 +- 73 files changed, 1853 insertions(+), 1856 deletions(-) delete mode 100644 docs/src/js/interfaces/GenerationStats.md delete mode 100644 docs/src/js/interfaces/LsmStats.md create mode 100644 docs/src/js/interfaces/MemWalStats.md rename docs/src/js/interfaces/{LsmWriteSpec.md => MemWalWriteSpec.md} (93%) create mode 100644 docs/src/js/interfaces/SsTableStats.md rename docs/src/js/interfaces/{BucketStats.md => TableShardStats.md} (71%) rename java/lancedb-core/src/main/java/com/lancedb/{LanceDbTableLsm.java => LanceDbTableMemWal.java} (57%) delete mode 100644 java/lancedb-core/src/main/java/com/lancedb/LsmStats.java create mode 100644 java/lancedb-core/src/main/java/com/lancedb/MemWalStats.java rename java/lancedb-core/src/main/java/com/lancedb/{LsmWriteSpec.java => MemWalWriteSpec.java} (87%) rename java/lancedb-core/src/main/java/com/lancedb/{GenerationStats.java => SsTableStats.java} (75%) rename java/lancedb-core/src/main/java/com/lancedb/{BucketStats.java => TableShardStats.java} (76%) rename java/lancedb-core/src/test/java/com/lancedb/{LanceDbTableLsmTest.java => LanceDbTableMemWalTest.java} (63%) rename python/python/tests/{test_lsm_write_spec.py => test_memwal_spec.py} (65%) rename rust/lancedb/src/table/{lsm_stats.rs => memwal_stats.rs} (56%) diff --git a/docs/src/js/classes/AutoQuery.md b/docs/src/js/classes/AutoQuery.md index 12f517dbd..1e9859215 100644 --- a/docs/src/js/classes/AutoQuery.md +++ b/docs/src/js/classes/AutoQuery.md @@ -425,19 +425,19 @@ Empty results retain the query's output schema. *** -### useLsm() +### useMemWal() ```ts -useLsm(enable): this +useMemWal(enable): this ``` Control MemWAL read routing for this query. By default (unset), when the table carries a MemWAL write spec (see -[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so +[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so they also return data written via the `mergeInsert` LSM path that has not yet been compacted into the base table (the active/frozen in-memory memtables and -the flushed generations), deduplicated by primary key; a table without a spec +the SSTables), deduplicated by primary key; a table without a spec reads the base table. #### Parameters @@ -448,7 +448,7 @@ reads the base table. even when a spec is present. Note: the LSM scanner does not support every query shape (e.g. reranking, hybrid search, `orderBy`). On a MemWAL table those shapes error unless - `useLsm(false)` is set, because a base-only read would silently exclude + `useMemWal(false)` is set, because a base-only read would silently exclude un-compacted MemWAL data. #### Returns @@ -457,7 +457,7 @@ reads the base table. #### Inherited from -`StandardQueryBase.useLsm` +`StandardQueryBase.useMemWal` *** diff --git a/docs/src/js/classes/MergeInsertBuilder.md b/docs/src/js/classes/MergeInsertBuilder.md index 81349a0bc..cf40837d3 100644 --- a/docs/src/js/classes/MergeInsertBuilder.md +++ b/docs/src/js/classes/MergeInsertBuilder.md @@ -76,15 +76,15 @@ the query optimizer chooses a suboptimal path. *** -### useLsm() +### useMemWal() ```ts -useLsm(enable): MergeInsertBuilder +useMemWal(enable): MergeInsertBuilder ``` Control MemWAL routing for this merge. -By default (unset), a `mergeInsert` on a table with an LSM write spec is +By default (unset), a `mergeInsert` on a table with a MemWAL write spec is routed through Lance's MemWAL shard writer, and a table without one uses the standard path. @@ -92,7 +92,7 @@ standard path. * **enable**: `boolean` `true` forces MemWAL routing and errors if the table has no - LSM write spec. `false` forces the standard write path even when a spec is set. + MemWAL write spec. `false` forces the standard write path even when a spec is set. #### Returns @@ -108,12 +108,12 @@ validateSingleShard(validateSingleShard): MergeInsertBuilder Controls how an LSM merge checks that its input targets a single shard. -When a table has an LSM write spec, every row in a `mergeInsert` call must +When a table has a MemWAL write spec, every row in a `mergeInsert` call must route to the same shard. When `true` (the default), every row is inspected to verify this. When `false`, only the first row is inspected and the shard it routes to is used for the whole input — a faster path for callers that have already pre-sharded their input. Has no effect on tables without -an LSM write spec. +a MemWAL write spec. #### Parameters diff --git a/docs/src/js/classes/Query.md b/docs/src/js/classes/Query.md index 3038bf2b3..9d83404f4 100644 --- a/docs/src/js/classes/Query.md +++ b/docs/src/js/classes/Query.md @@ -499,19 +499,19 @@ Empty results retain the query's output schema. *** -### useLsm() +### useMemWal() ```ts -useLsm(enable): this +useMemWal(enable): this ``` Control MemWAL read routing for this query. By default (unset), when the table carries a MemWAL write spec (see -[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so +[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so they also return data written via the `mergeInsert` LSM path that has not yet been compacted into the base table (the active/frozen in-memory memtables and -the flushed generations), deduplicated by primary key; a table without a spec +the SSTables), deduplicated by primary key; a table without a spec reads the base table. #### Parameters @@ -522,7 +522,7 @@ reads the base table. even when a spec is present. Note: the LSM scanner does not support every query shape (e.g. reranking, hybrid search, `orderBy`). On a MemWAL table those shapes error unless - `useLsm(false)` is set, because a base-only read would silently exclude + `useMemWal(false)` is set, because a base-only read would silently exclude un-compacted MemWAL data. #### Returns @@ -531,7 +531,7 @@ reads the base table. #### Inherited from -`StandardQueryBase.useLsm` +`StandardQueryBase.useMemWal` *** diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 59c3209c8..3eaf9c9ca 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -228,17 +228,17 @@ version of the table. *** -### checkpointLsm() +### checkpointMemWal() ```ts -abstract checkpointLsm(): Promise +abstract checkpointMemWal(): Promise ``` -Converge this table's LSM write path into its base table. +Converge this table's MemWAL write path into its base table. -Seals once, then triggers compaction and polls until the L0 that existed +Freezes once, then triggers compaction and polls until the SSTables that existed at the start is gone. The target set is fixed at the start, so -generations created *during* the checkpoint are ignored — that is what +SSTables created *during* the checkpoint are ignored — that is what lets it terminate under write load, and what makes it best-effort: it converges the fresh tier as of some instant. Idempotent, abandonable at any point, and safe to run on a cadence. @@ -254,9 +254,9 @@ is merging. The caller owns the deadline. #### Example ```ts -const before = await table.getLsmStats(); -await table.checkpointLsm(); -const after = await table.getLsmStats(); +const before = await table.getMemWalStats(); +await table.checkpointMemWal(); +const after = await table.getMemWalStats(); ``` *** @@ -279,15 +279,15 @@ Any attempt to use the table after it is closed will result in an error. *** -### closeLsmWriters() +### closeMemWalWriters() ```ts -abstract closeLsmWriters(): Promise +abstract closeMemWalWriters(): Promise ``` Drain and close any cached MemWAL shard writers held for this table. -When an [LsmWriteSpec](../interfaces/LsmWriteSpec.md) is installed, `mergeInsert` opens MemWAL +When an [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) is installed, `mergeInsert` opens MemWAL shard writers and caches them for reuse across calls. This closes them, flushing pending data; writers reopen lazily on the next `mergeInsert`. It is a no-op when no writers are cached. @@ -298,17 +298,17 @@ It is a no-op when no writers are cached. *** -### compactLsm() +### compactMemWal() ```ts -abstract compactLsm(): Promise +abstract compactMemWal(): Promise ``` -Trigger a background L0 → base compaction pass per bucket. +Trigger a background SSTable compaction pass per table shard. Returns once the passes are *dispatched*, not once they finish — watch -[Table#getLsmStats](Table.md#getlsmstats) for progress, or use -[Table#checkpointLsm](Table.md#checkpointlsm) to wait for convergence. +[Table#getMemWalStats](Table.md#getmemwalstats) for progress, or use +[Table#checkpointMemWal](Table.md#checkpointmemwal) to wait for convergence. #### Returns @@ -449,6 +449,23 @@ containing the new version number of the table *** +### disableMemWal() + +```ts +abstract disableMemWal(): Promise +``` + +Remove the [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) from this table, reverting to the standard +`mergeInsert` write path. + +Errors if no spec is currently set. + +#### Returns + +`Promise`<`void`> + +*** + ### display() ```ts @@ -514,6 +531,54 @@ Drop an index from the table. *** +### enableMemWal() + +```ts +abstract enableMemWal(spec): Promise +``` + +Install an [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) on this table, selecting Lance's MemWAL +LSM-style write path for future `mergeInsert` calls. + +`MemWalWriteSpec` chooses one of three sharding strategies via `specType`: + +- `"bucket"` — hash-bucket writes by the single-column unenforced primary + key (`column` and `numBuckets` required). +- `"identity"` — shard by the raw value of a scalar `column`. +- `"unsharded"` — route every write to a single shard. + +All variants require the table to have an unenforced primary key +([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally +requires it to be the single column being bucketed. + +Omitting `maintainedIndexes` maintains every index the table has, +including ones created later, and skips a kind the MemWAL cannot maintain. +Naming them pins an exact set, and a still-building index is rejected +rather than quietly omitted. + +#### Parameters + +* **spec**: [`MemWalWriteSpec`](../interfaces/MemWalWriteSpec.md) + The sharding spec to install. + +#### Returns + +`Promise`<`void`> + +#### Example + +```ts +await table.setUnenforcedPrimaryKey("id"); +await table.enableMemWal({ + specType: "bucket", + column: "id", + numBuckets: 16, + maintainedIndexes: ["id_idx"], +}); +``` + +*** + ### fetchBlobFiles() ```ts @@ -582,15 +647,15 @@ memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values. *** -### flushLsm() +### flushMemWal() ```ts -abstract flushLsm(): Promise +abstract flushMemWal(): Promise ``` -Seal every bucket's active memtable into a new L0 generation. +Freeze every table shard's active memtable into a new SSTable. -Returns once the seal is committed. Sealing an empty memtable is a no-op, +Returns once the freeze is committed. Freezing an empty memtable is a no-op, so this is safe to call repeatedly. #### Returns @@ -634,48 +699,48 @@ const { records, truncated } = await table.functionErrors({ column: "embedding" *** -### getLsmStats() +### getMemWalSpec() ```ts -abstract getLsmStats(includeGenerationRows?): Promise +abstract getMemWalSpec(): Promise ``` -Read live per-bucket LSM state. +Read the [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) currently installed on this table. -Answers "how far behind is my fresh tier", "which bucket is hot", and -"why is my fresh-tier vector search brute-force". Mutates no table state. - -Resolves to `undefined` only when the LSM write path is not enabled. - -#### Parameters - -* **includeGenerationRows?**: `boolean` - Also count rows per L0 generation. - Off by default because each count opens an uncached Lance dataset. - -#### Returns - -`Promise`<`undefined` \| [`LsmStats`](../interfaces/LsmStats.md)> - -*** - -### getLsmWriteSpec() - -```ts -abstract getLsmWriteSpec(): Promise -``` - -Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on this table. - -Resolves to `undefined` when the MemWAL LSM write path is not enabled (no -spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)). +Resolves to `undefined` when the MemWAL write path is not enabled (no +spec has been set, or it was removed with [Table#disableMemWal](Table.md#disablememwal)). The spec is the one installed, including its maintained-index selection: an absent `maintainedIndexes` for every index the table has, an empty array for none. #### Returns -`Promise`<`undefined` \| [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)> +`Promise`<`undefined` \| [`MemWalWriteSpec`](../interfaces/MemWalWriteSpec.md)> + +*** + +### getMemWalStats() + +```ts +abstract getMemWalStats(includeSstableRows?): Promise +``` + +Read live per-table-shard MemWAL state. + +Answers "how far behind is my fresh tier", "which table shard is hot", and +"why is my fresh-tier vector search brute-force". Mutates no table state. + +Resolves to `undefined` only when the MemWAL write path is not enabled. + +#### Parameters + +* **includeSstableRows?**: `boolean` + Also count rows per SSTable. + Off by default because each count opens an uncached Lance dataset. + +#### Returns + +`Promise`<`undefined` \| [`MemWalStats`](../interfaces/MemWalStats.md)> *** @@ -1093,54 +1158,6 @@ of the given query *** -### setLsmWriteSpec() - -```ts -abstract setLsmWriteSpec(spec): Promise -``` - -Install an [LsmWriteSpec](../interfaces/LsmWriteSpec.md) on this table, selecting Lance's MemWAL -LSM-style write path for future `mergeInsert` calls. - -`LsmWriteSpec` chooses one of three sharding strategies via `specType`: - -- `"bucket"` — hash-bucket writes by the single-column unenforced primary - key (`column` and `numBuckets` required). -- `"identity"` — shard by the raw value of a scalar `column`. -- `"unsharded"` — route every write to a single shard. - -All variants require the table to have an unenforced primary key -([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally -requires it to be the single column being bucketed. - -Omitting `maintainedIndexes` maintains every index the table has, -including ones created later, and skips a kind the MemWAL cannot maintain. -Naming them pins an exact set, and a still-building index is rejected -rather than quietly omitted. - -#### Parameters - -* **spec**: [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md) - The sharding spec to install. - -#### Returns - -`Promise`<`void`> - -#### Example - -```ts -await table.setUnenforcedPrimaryKey("id"); -await table.setLsmWriteSpec({ - specType: "bucket", - column: "id", - numBuckets: 16, - maintainedIndexes: ["id_idx"], -}); -``` - -*** - ### setUnenforcedPrimaryKey() ```ts @@ -1295,23 +1312,6 @@ same tokenizer model files must also exist locally. *** -### unsetLsmWriteSpec() - -```ts -abstract unsetLsmWriteSpec(): Promise -``` - -Remove the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) from this table, reverting to the standard -`mergeInsert` write path. - -Errors if no spec is currently set. - -#### Returns - -`Promise`<`void`> - -*** - ### update() #### update(opts) diff --git a/docs/src/js/classes/TakeQuery.md b/docs/src/js/classes/TakeQuery.md index f3d59a2a3..9e8d03149 100644 --- a/docs/src/js/classes/TakeQuery.md +++ b/docs/src/js/classes/TakeQuery.md @@ -275,10 +275,10 @@ Empty results retain the query's output schema. *** -### useLsm() +### useMemWal() ```ts -useLsm(enable): this +useMemWal(enable): this ``` Control MemWAL read routing for this take query. diff --git a/docs/src/js/classes/VectorQuery.md b/docs/src/js/classes/VectorQuery.md index 4809ebefd..e329cb254 100644 --- a/docs/src/js/classes/VectorQuery.md +++ b/docs/src/js/classes/VectorQuery.md @@ -781,19 +781,19 @@ Empty results retain the query's output schema. *** -### useLsm() +### useMemWal() ```ts -useLsm(enable): this +useMemWal(enable): this ``` Control MemWAL read routing for this query. By default (unset), when the table carries a MemWAL write spec (see -[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so +[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so they also return data written via the `mergeInsert` LSM path that has not yet been compacted into the base table (the active/frozen in-memory memtables and -the flushed generations), deduplicated by primary key; a table without a spec +the SSTables), deduplicated by primary key; a table without a spec reads the base table. #### Parameters @@ -804,7 +804,7 @@ reads the base table. even when a spec is present. Note: the LSM scanner does not support every query shape (e.g. reranking, hybrid search, `orderBy`). On a MemWAL table those shapes error unless - `useLsm(false)` is set, because a base-only read would silently exclude + `useMemWal(false)` is set, because a base-only read would silently exclude un-compacted MemWAL data. #### Returns @@ -813,7 +813,7 @@ reads the base table. #### Inherited from -`StandardQueryBase.useLsm` +`StandardQueryBase.useMemWal` *** diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 9cf3136af..739b9033a 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -66,7 +66,6 @@ - [BranchDiff](interfaces/BranchDiff.md) - [BranchIndexSummary](interfaces/BranchIndexSummary.md) - [BranchRowCountSummary](interfaces/BranchRowCountSummary.md) -- [BucketStats](interfaces/BucketStats.md) - [CatalogOptions](interfaces/CatalogOptions.md) - [CherryPickError](interfaces/CherryPickError.md) - [CherryPickPreview](interfaces/CherryPickPreview.md) @@ -98,7 +97,6 @@ - [FunctionErrorRecord](interfaces/FunctionErrorRecord.md) - [FunctionErrors](interfaces/FunctionErrors.md) - [FunctionErrorsOptions](interfaces/FunctionErrorsOptions.md) -- [GenerationStats](interfaces/GenerationStats.md) - [HnswPqOptions](interfaces/HnswPqOptions.md) - [HnswSqOptions](interfaces/HnswSqOptions.md) - [IndexConfig](interfaces/IndexConfig.md) @@ -115,9 +113,9 @@ - [ListTablesOptions](interfaces/ListTablesOptions.md) - [ListTablesResponse](interfaces/ListTablesResponse.md) - [ListingOptions](interfaces/ListingOptions.md) -- [LsmStats](interfaces/LsmStats.md) -- [LsmWriteSpec](interfaces/LsmWriteSpec.md) - [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md) +- [MemWalStats](interfaces/MemWalStats.md) +- [MemWalWriteSpec](interfaces/MemWalWriteSpec.md) - [MemtableStats](interfaces/MemtableStats.md) - [MergeResult](interfaces/MergeResult.md) - [NativeOAuthConfig](interfaces/NativeOAuthConfig.md) @@ -140,7 +138,9 @@ - [SplitHashOptions](interfaces/SplitHashOptions.md) - [SplitRandomOptions](interfaces/SplitRandomOptions.md) - [SplitSequentialOptions](interfaces/SplitSequentialOptions.md) +- [SsTableStats](interfaces/SsTableStats.md) - [TableNamesOptions](interfaces/TableNamesOptions.md) +- [TableShardStats](interfaces/TableShardStats.md) - [TableStatistics](interfaces/TableStatistics.md) - [TimeoutConfig](interfaces/TimeoutConfig.md) - [TlsConfig](interfaces/TlsConfig.md) diff --git a/docs/src/js/interfaces/GenerationStats.md b/docs/src/js/interfaces/GenerationStats.md deleted file mode 100644 index 19dd2afda..000000000 --- a/docs/src/js/interfaces/GenerationStats.md +++ /dev/null @@ -1,40 +0,0 @@ -[**@lancedb/lancedb**](../README.md) • **Docs** - -*** - -[@lancedb/lancedb](../globals.md) / GenerationStats - -# Interface: GenerationStats - -One flushed L0 generation. - -## Properties - -### bytes - -```ts -bytes: number; -``` - -On-disk size of the generation. - -*** - -### generation - -```ts -generation: number; -``` - -The generation number. Increases as memtables are sealed into L0. - -*** - -### rows? - -```ts -optional rows: number; -``` - -Present only when `includeGenerationRows` was requested. Off by default -because each count opens an uncached Lance dataset. diff --git a/docs/src/js/interfaces/LsmStats.md b/docs/src/js/interfaces/LsmStats.md deleted file mode 100644 index 76a2f50db..000000000 --- a/docs/src/js/interfaces/LsmStats.md +++ /dev/null @@ -1,22 +0,0 @@ -[**@lancedb/lancedb**](../README.md) • **Docs** - -*** - -[@lancedb/lancedb](../globals.md) / LsmStats - -# Interface: LsmStats - -Live per-bucket LSM state, as returned by `Table#getLsmStats`. - -Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are -the caller's to compute. - -## Properties - -### buckets - -```ts -buckets: BucketStats[]; -``` - -One entry per bucket backing this table. diff --git a/docs/src/js/interfaces/MemWalStats.md b/docs/src/js/interfaces/MemWalStats.md new file mode 100644 index 000000000..4221867fc --- /dev/null +++ b/docs/src/js/interfaces/MemWalStats.md @@ -0,0 +1,22 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MemWalStats + +# Interface: MemWalStats + +Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`. + +Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are +the caller's to compute. + +## Properties + +### tableShards + +```ts +tableShards: TableShardStats[]; +``` + +One entry per table shard backing this table. diff --git a/docs/src/js/interfaces/LsmWriteSpec.md b/docs/src/js/interfaces/MemWalWriteSpec.md similarity index 93% rename from docs/src/js/interfaces/LsmWriteSpec.md rename to docs/src/js/interfaces/MemWalWriteSpec.md index 693542dbd..50b74da41 100644 --- a/docs/src/js/interfaces/LsmWriteSpec.md +++ b/docs/src/js/interfaces/MemWalWriteSpec.md @@ -2,9 +2,9 @@ *** -[@lancedb/lancedb](../globals.md) / LsmWriteSpec +[@lancedb/lancedb](../globals.md) / MemWalWriteSpec -# Interface: LsmWriteSpec +# Interface: MemWalWriteSpec Specification selecting Lance's MemWAL LSM-style write path for `mergeInsert`. diff --git a/docs/src/js/interfaces/MemtableStats.md b/docs/src/js/interfaces/MemtableStats.md index fdc1e4467..63b8eace8 100644 --- a/docs/src/js/interfaces/MemtableStats.md +++ b/docs/src/js/interfaces/MemtableStats.md @@ -36,7 +36,7 @@ Estimated in-memory size. generation: number; ``` -The generation this memtable will become once sealed. +The generation this memtable will become once frozen. *** diff --git a/docs/src/js/interfaces/SsTableStats.md b/docs/src/js/interfaces/SsTableStats.md new file mode 100644 index 000000000..7b01a46f2 --- /dev/null +++ b/docs/src/js/interfaces/SsTableStats.md @@ -0,0 +1,40 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / SsTableStats + +# Interface: SsTableStats + +One SSTable. + +## Properties + +### bytes + +```ts +bytes: number; +``` + +On-disk size of the SSTable. + +*** + +### generation + +```ts +generation: number; +``` + +The generation number. Increases as memtables are frozen into SSTables. + +*** + +### rows? + +```ts +optional rows: number; +``` + +Present only when `includeSstableRows` was requested. Off by default +because each count opens an uncached Lance dataset. diff --git a/docs/src/js/interfaces/BucketStats.md b/docs/src/js/interfaces/TableShardStats.md similarity index 71% rename from docs/src/js/interfaces/BucketStats.md rename to docs/src/js/interfaces/TableShardStats.md index 3f5095672..64be5e863 100644 --- a/docs/src/js/interfaces/BucketStats.md +++ b/docs/src/js/interfaces/TableShardStats.md @@ -2,12 +2,12 @@ *** -[@lancedb/lancedb](../globals.md) / BucketStats +[@lancedb/lancedb](../globals.md) / TableShardStats -# Interface: BucketStats +# Interface: TableShardStats -Live state of one bucket. A table is N buckets on one node; flattening to a -single number hides the one hot bucket that is usually why someone opened +Live state of one table shard. A table is N table shards on one node; flattening to a +single number hides the one hot table shard that is usually why someone opened this endpoint. ## Properties @@ -18,7 +18,7 @@ this endpoint. compacting: boolean; ``` -Whether a pass owns this bucket's compaction latch right now. Says *a* +Whether a pass owns this table shard's compaction latch right now. Says *a* driver is running, not *whose*, and the latch is held from dispatch — including while the pass queues for a pod-wide compactor permit. Read it as "do not pile on", never as "mine is progressing". @@ -35,16 +35,6 @@ The generation the active memtable will become. *** -### generations - -```ts -generations: GenerationStats[]; -``` - -Flushed L0 generations not yet merged into the base table. - -*** - ### manifestVersion ```ts @@ -61,7 +51,7 @@ Version of the shard manifest these numbers were read from. optional memtables: MemtableStats[]; ``` -Oldest first, active last. Absent for a `"Sealed"` bucket, whose +Oldest first, active last. Absent for a `"Sealed"` table shard, whose in-memory state is torn down. *** @@ -82,7 +72,17 @@ WAL position replay resumes from. shardId: string; ``` -The shard this bucket writes. +The id of this table shard. + +*** + +### sstables + +```ts +sstables: SsTableStats[]; +``` + +SSTables not yet merged into the base table. *** diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 24372a83d..8405852f2 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -297,7 +297,7 @@ listing a storage directory. ::: lancedb.table.Branches -::: lancedb.LsmWriteSpec +::: lancedb.MemWalWriteSpec ## Functions and Jobs diff --git a/docs/web/feature-engineering/materialized-views.mdx b/docs/web/feature-engineering/materialized-views.mdx index ab949adb1..ffe4fc959 100644 --- a/docs/web/feature-engineering/materialized-views.mdx +++ b/docs/web/feature-engineering/materialized-views.mdx @@ -52,7 +52,7 @@ In Python SDK 0.40.0, create the source with above does. Stable row IDs let the view track source rows across updates and compaction. Set this option when creating the table; it cannot be added later. -A source with an LSM write specification or retained, uncompacted write-ahead-log +A source with a MemWAL write spec or retained, uncompacted write-ahead-log rows is not accepted by the local materialized-view implementation. ## Refresh behavior {#incremental} diff --git a/docs/web/maintenance/background-jobs.mdx b/docs/web/maintenance/background-jobs.mdx index 151828e5d..2fe5d549c 100644 --- a/docs/web/maintenance/background-jobs.mdx +++ b/docs/web/maintenance/background-jobs.mdx @@ -32,7 +32,7 @@ log from growing without bound: | Job type | What it does | | --- | --- | -| `sstable_compaction` | Folds a WAL bucket's SSTables into the base table by primary key. Distinct from `compact`, which merges base-table fragments -- the two are unrelated. | +| `sstable_compaction` | Folds a table shard's SSTables into the base table by primary key. Distinct from `compact`, which merges base-table fragments -- the two are unrelated. | | `wal_index_catchup` | Extends a base-table index over the rows `sstable_compaction` just folded in, and records how far the index has caught up. Nothing else advances that record. | ## Index maintenance {#index} diff --git a/docs/web/sql/ddl/table.mdx b/docs/web/sql/ddl/table.mdx index 390c24b65..05406e677 100644 --- a/docs/web/sql/ddl/table.mdx +++ b/docs/web/sql/ddl/table.mdx @@ -102,10 +102,10 @@ duplicates impossible. It tells the planner what to assume. The column must be declared `NOT NULL`. ```sql -ALTER TABLE t SET WAL -ALTER TABLE t SET WAL WITH (sharding = 'bucket', column = 'id', buckets = 4) -ALTER TABLE t SET WAL WITH (maintained_indexes = '') -ALTER TABLE t UNSET WAL +ALTER TABLE t SET MEMWAL +ALTER TABLE t SET MEMWAL WITH (sharding = 'bucket', column = 'id', buckets = 4) +ALTER TABLE t SET MEMWAL WITH (maintained_indexes = '') +ALTER TABLE t UNSET MEMWAL ``` A write-ahead log makes small frequent writes cheap by batching them out of the diff --git a/java/README.md b/java/README.md index 4abe5f50e..0e54d20e4 100644 --- a/java/README.md +++ b/java/README.md @@ -29,7 +29,7 @@ LanceNamespace namespaceClient = LanceDbNamespaceClientBuilder.newBuilder() .build(); ``` -## MemWAL LSM write path +## MemWAL write path Most table operations reach LanceDB through the `LanceNamespace` above, which is generated from the Lance Namespace specification. The MemWAL LSM routes are not part @@ -37,27 +37,27 @@ of that specification, so they are issued through a separate client: ```java import com.lancedb.LanceDbRestClient; -import com.lancedb.LanceDbTableLsm; -import com.lancedb.LsmWriteSpec; +import com.lancedb.LanceDbTableMemWal; +import com.lancedb.MemWalWriteSpec; LanceDbRestClient client = LanceDbNamespaceClientBuilder.newBuilder() .apiKey("your_lancedb_cloud_api_key") .database("your_database_name") .buildRestClient(); -LanceDbTableLsm lsm = new LanceDbTableLsm(client, "my_table"); +LanceDbTableMemWal memWal = new LanceDbTableMemWal(client, "my_table"); // Route future merge_insert upserts through the MemWAL, hash-bucketed by `id`. -lsm.setLsmWriteSpec(LsmWriteSpec.bucket("id", 16)); +memWal.enableMemWal(MemWalWriteSpec.bucket("id", 16)); // ... merge_insert traffic ... // Converge the fresh tier into the base table. -lsm.checkpointLsm(); +memWal.checkpointMemWal(); -// Inspect live per-bucket state. -lsm.getLsmStats().ifPresent(stats -> stats.buckets().forEach(bucket -> - System.out.println(bucket.shardId() + ": " + bucket.generations().size() + " L0 generations"))); +// Inspect live per-table-shard state. +memWal.getMemWalStats().ifPresent(stats -> stats.tableShards().forEach(tableShard -> + System.out.println(tableShard.shardId() + ": " + tableShard.sstables().size() + " SSTables"))); client.close(); ``` diff --git a/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java b/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java index b78e2411a..e0a288eaf 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java +++ b/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java @@ -20,12 +20,12 @@ import com.fasterxml.jackson.databind.JsonNode; * *

Every reader fails closed: a missing, null, or wrong-typed field throws rather than * defaulting. That mirrors the serde decoding the Rust client applies to the same payloads in - * {@code rust/lancedb/src/table/lsm_stats.rs}, where a required field has no default and a + * {@code rust/lancedb/src/table/memwal_stats.rs}, where a required field has no default and a * malformed response is an error rather than a zero. * *

The alternative — Jackson's {@code path()}, which yields a missing node that reads as an empty - * array or a zero — is unsafe here because {@link LanceDbTableLsm#checkpointLsm()} decides - * convergence from these numbers. A defaulted {@code generations} array is indistinguishable from a + * array or a zero — is unsafe here because {@link LanceDbTableMemWal#checkpointMemWal()} decides + * convergence from these numbers. A defaulted {@code sstables} array is indistinguishable from a * drained one, so a malformed response would report a checkpoint that never happened. */ final class JsonFields { diff --git a/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java b/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java index da241dfd5..05aed8c74 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java +++ b/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java @@ -151,8 +151,8 @@ public class LanceDbNamespaceClientBuilder { * Build a {@link LanceDbRestClient} for the same endpoint. * *

Needed only for LanceDB routes that the Lance Namespace specification does not cover — the - * MemWAL LSM write path, reached through {@link LanceDbTableLsm}. Every other table operation - * belongs on the {@link LanceNamespace} from {@link #build()}. + * MemWAL write path, reached through {@link LanceDbTableMemWal}. Every other table + * operation belongs on the {@link LanceNamespace} from {@link #build()}. * *

The returned client owns an HTTP connection pool; close it when you are done with it. * diff --git a/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java b/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java index baafbb9df..5275c8346 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java +++ b/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java @@ -31,9 +31,9 @@ import java.io.UncheckedIOException; * specification does not cover. * *

Most table operations reach LanceDB through {@link org.lance.namespace.LanceNamespace}, which - * is generated from the namespace spec. A handful of routes — the MemWAL LSM write path in + * is generated from the namespace spec. A handful of routes — the MemWAL write path in * particular — are served by the same endpoint but are not part of that spec, so they are issued - * directly here. See {@link LanceDbTableLsm}. + * directly here. See {@link LanceDbTableMemWal}. * *

Obtain one from {@link LanceDbNamespaceClientBuilder#buildRestClient()}. */ @@ -50,8 +50,8 @@ public class LanceDbRestClient implements Closeable { this.apiKey = apiKey; this.database = database; // Automatic retries off, deliberately. The default strategy retries 429 and 503 — - // exactly the two statuses LanceDbTableLsm.checkpointLsm() acts on — which would - // silently double its explicit retry budget and would also retry compact_lsm in + // exactly the two statuses LanceDbTableMemWal.checkpointMemWal() acts on — which would + // silently double its explicit retry budget and would also retry compact_memwal in // place, where the loop is designed to fall through to a fresh stats poll instead. // The checkpoint loop owns the 421/429/503 transitions; the transport must not. this.http = HttpClients.custom().disableAutomaticRetries().build(); @@ -98,8 +98,9 @@ public class LanceDbRestClient implements Closeable { /** * A non-2xx response. * - *

The status is exposed because callers act on it: {@link LanceDbTableLsm#checkpointLsm()} - * treats 429 and 503 as retryable and 421 as a lost node claim. + *

The status is exposed because callers act on it: {@link + * LanceDbTableMemWal#checkpointMemWal()} treats 429 and 503 as retryable and 421 as a lost node + * claim. */ public static class HttpException extends RuntimeException { private static final long serialVersionUID = 1L; diff --git a/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java b/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableMemWal.java similarity index 57% rename from java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java rename to java/lancedb-core/src/main/java/com/lancedb/LanceDbTableMemWal.java index bb1467ecd..75c6c33b1 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java +++ b/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableMemWal.java @@ -22,11 +22,11 @@ import java.util.Optional; import java.util.OptionalLong; /** - * The MemWAL LSM write path for one LanceDB Cloud or Enterprise table. + * The MemWAL write path for one LanceDB Cloud or Enterprise table. * - *

Installing an {@link LsmWriteSpec} routes {@code mergeInsert} upserts through Lance's MemWAL — - * an LSM-style append — instead of the standard merge path. Rows land in an in-memory memtable, - * seal into L0 generations, and are merged into the base table by compaction. + *

Installing an {@link MemWalWriteSpec} routes {@code mergeInsert} upserts through Lance's + * MemWAL — an LSM-style append — instead of the standard merge path. Rows land in an in-memory + * memtable, freeze into SSTables, and are merged into the base table by compaction. * *

These routes are not part of the Lance Namespace specification, so they are issued directly * rather than through {@link org.lance.namespace.LanceNamespace}. @@ -37,23 +37,23 @@ import java.util.OptionalLong; * .database("your_database_name") * .buildRestClient(); * - * LanceDbTableLsm lsm = new LanceDbTableLsm(client, "my_table"); - * lsm.setLsmWriteSpec(LsmWriteSpec.bucket("id", 16)); + * LanceDbTableMemWal memWal = new LanceDbTableMemWal(client, "my_table"); + * memWal.enableMemWal(MemWalWriteSpec.bucket("id", 16)); * // ... merge_insert traffic ... - * lsm.checkpointLsm(); + * memWal.checkpointMemWal(); * } */ -public class LanceDbTableLsm { +public class LanceDbTableMemWal { /** - * Interval between {@code get_lsm_stats} polls during a checkpoint. One interval is roughly one - * compaction pass, the granularity at which the answer can change. + * Interval between {@code get_memwal_stats} polls during a checkpoint. One interval is roughly + * one compaction pass, the granularity at which the answer can change. */ private static final long POLL_INTERVAL_MS = 5_000L; /** - * Cap on re-issues from {@code flushLsm} after a 421, so a crash-looping node cannot turn flush → - * compact → 421 → flush into a spin. + * Cap on re-issues from {@code flushMemWal} after a 421, so a crash-looping node cannot turn + * flush → compact → 421 → flush into a spin. * *

Deliberately not shared with {@link #MAX_RETRIES}: a claim that keeps evaporating is a * broken node, while contention is routine and wants a real budget. @@ -79,7 +79,7 @@ public class LanceDbTableLsm { * @param tableIdentifier The table's full identifier, {@code $}-delimited when it sits inside a * namespace, such as {@code analytics$events}. */ - public LanceDbTableLsm(LanceDbRestClient client, String tableIdentifier) { + public LanceDbTableMemWal(LanceDbRestClient client, String tableIdentifier) { if (client == null) { throw new IllegalArgumentException("Client cannot be null"); } @@ -91,140 +91,140 @@ public class LanceDbTableLsm { } /** - * Install an {@link LsmWriteSpec} on this table, selecting the MemWAL LSM write path for future - * {@code mergeInsert} calls. + * Install an {@link MemWalWriteSpec} on this table, selecting the MemWAL write path for + * future {@code mergeInsert} calls. * *

All variants require the table to have an unenforced primary key; bucket sharding * additionally requires it to be the single column being bucketed. */ - public void setLsmWriteSpec(LsmWriteSpec spec) { + public void enableMemWal(MemWalWriteSpec spec) { if (spec == null) { throw new IllegalArgumentException("Spec cannot be null"); } - client.post(route("set_lsm_write_spec"), spec.toRequestBody()); + client.post(route("enable_memwal"), spec.toRequestBody()); } /** - * Remove the {@link LsmWriteSpec} from this table, reverting to the standard {@code mergeInsert} - * write path. + * Remove the {@link MemWalWriteSpec} from this table, reverting to the standard {@code + * mergeInsert} write path. * *

Errors if no spec is currently set. */ - public void unsetLsmWriteSpec() { - client.post(route("unset_lsm_write_spec"), null); + public void disableMemWal() { + client.post(route("disable_memwal"), null); } /** - * Read the {@link LsmWriteSpec} currently installed on this table. + * Read the {@link MemWalWriteSpec} currently installed on this table. * - *

Empty when the LSM write path is not enabled at all — no spec has been set, or one was - * removed with {@link #unsetLsmWriteSpec()}. That is a different answer from a present spec - * whose {@link LsmWriteSpec#maintainedIndexes()} is null, which selects indexes automatically. + *

Empty when the MemWAL write path is not enabled at all — no spec has been set, or one was + * removed with {@link #disableMemWal()}. That is a different answer from a present spec whose + * {@link MemWalWriteSpec#maintainedIndexes()} is null, which selects indexes automatically. * *

The spec read back is the one that was installed, selection included: null maintains every - * supported index the table has now or gains later, an empty list maintains none, and a - * non-empty list maintains exactly those. All three round-trip. + * supported index the table has now or gains later, an empty list maintains none, and a non-empty + * list maintains exactly those. All three round-trip. */ - public Optional getLsmWriteSpec() { - JsonNode response = client.post(route("get_lsm_write_spec"), null); - if (response == null || !response.hasNonNull("lsm_write_spec")) { + public Optional getMemWalSpec() { + JsonNode response = client.post(route("get_memwal_spec"), null); + if (response == null || !response.hasNonNull("memwal_spec")) { return Optional.empty(); } - return Optional.of(LsmWriteSpec.fromJson(response.get("lsm_write_spec"))); + return Optional.of(MemWalWriteSpec.fromJson(response.get("memwal_spec"))); } /** - * Seal every bucket's active memtable into a new L0 generation. + * Freeze every table shard's active memtable into a new SSTable. * - *

Returns once the seal is committed. Sealing an empty memtable is a no-op, so this is safe to - * call repeatedly. + *

Returns once the freeze is committed. Freezing an empty memtable is a no-op, so this is safe + * to call repeatedly. */ - public void flushLsm() { - client.post(route("flush_lsm"), null); + public void flushMemWal() { + client.post(route("flush_memwal"), null); } /** - * Trigger a background L0 → base compaction pass per bucket. + * Trigger a background SSTable compaction pass per table shard. * *

Returns once the passes are dispatched, not once they finish — watch {@link - * #getLsmStats}, or use {@link #checkpointLsm} to wait for convergence. + * #getMemWalStats}, or use {@link #checkpointMemWal} to wait for convergence. */ - public void compactLsm() { - client.post(route("compact_lsm"), null); + public void compactMemWal() { + client.post(route("compact_memwal"), null); } /** - * Read live per-bucket LSM state. + * Read live per-table-shard MemWAL state. * - *

Answers "how far behind is my fresh tier", "which bucket is hot", and "why is my fresh-tier - * vector search brute-force". Mutates no table state. + *

Answers "how far behind is my fresh tier", "which table shard is hot", and "why is my + * fresh-tier vector search brute-force". Mutates no table state. * - *

Empty only when the LSM write path is not enabled — that is, when the server sends an absent - * or null {@code lsm_stats}. A stats object that is present is decoded strictly, and a malformed - * one throws rather than decoding to something empty, because {@link #checkpointLsm} reads - * convergence out of these numbers and cannot tell a defaulted array from a drained one. + *

Empty only when the MemWAL write path is not enabled — that is, when the server sends an + * absent or null {@code memwal_stats}. A stats object that is present is decoded strictly, and a + * malformed one throws rather than decoding to something empty, because {@link #checkpointMemWal} + * reads convergence out of these numbers and cannot tell a defaulted array from a drained one. * - * @param includeGenerationRows Also count rows per L0 generation. Off by default because each - * count opens an uncached Lance dataset. + * @param includeSstableRows Also count rows per SSTable. Off by default because each count opens + * an uncached Lance dataset. * @throws IllegalStateException if the response is absent or does not decode. */ - public Optional getLsmStats(boolean includeGenerationRows) { + public Optional getMemWalStats(boolean includeSstableRows) { Map body = new LinkedHashMap(); - body.put("include_generation_rows", includeGenerationRows); - JsonNode response = client.post(route("get_lsm_stats"), body); + body.put("include_sstable_rows", includeSstableRows); + JsonNode response = client.post(route("get_memwal_stats"), body); if (response == null) { - throw new IllegalStateException("get_lsm_stats returned an empty response body"); + throw new IllegalStateException("get_memwal_stats returned an empty response body"); } - JsonNode stats = response.get("lsm_stats"); + JsonNode stats = response.get("memwal_stats"); if (stats == null || stats.isNull()) { return Optional.empty(); } - return Optional.of(LsmStats.fromJson(stats)); + return Optional.of(MemWalStats.fromJson(stats)); } - /** Equivalent to {@code getLsmStats(false)}. */ - public Optional getLsmStats() { - return getLsmStats(false); + /** Equivalent to {@code getMemWalStats(false)}. */ + public Optional getMemWalStats() { + return getMemWalStats(false); } /** - * Converge this table's LSM write path into its base table. + * Converge this table's MemWAL write path into its base table. * - *

Seals once, fixes a target watermark from the resulting L0, then triggers compaction and - * polls until that L0 is gone. The target set is fixed at the start, so generations created - * during the checkpoint are ignored — that is what lets it terminate under write load, - * and what makes it best-effort: it converges the fresh tier as of some instant. Idempotent, - * abandonable at any point, safe on a cadence. + *

Freezes once, fixes a target watermark from the resulting SSTables, then triggers compaction + * and polls until those SSTables are gone. The target set is fixed at the start, so sstables + * created during the checkpoint are ignored — that is what lets it terminate under write + * load, and what makes it best-effort: it converges the fresh tier as of some instant. + * Idempotent, abandonable at any point, safe on a cadence. * - *

The loop runs here, not on the server: {@link #compactLsm} dispatches a pass and returns, so - * nothing holds a socket and a client can vanish mid-operation with nothing to reconcile. + *

The loop runs here, not on the server: {@link #compactMemWal} dispatches a pass and returns, + * so nothing holds a socket and a client can vanish mid-operation with nothing to reconcile. * Completion is read from generation numbers in the shard manifest — durable state, unlike a * count in a compact response, which a concurrent write invalidates. * *

No liveness bound — the caller owns the deadline. The compactor pool is shared across * tables, so a checkpoint queued behind unrelated work looks exactly like one that is merging. */ - public void checkpointLsm() { + public void checkpointMemWal() { for (int reissue = 0; reissue <= MAX_REISSUES; reissue++) { - // The seal turns everything written before this call into a generation, so the - // watermark has to be read after it. Idempotent: sealing an empty memtable is a - // no-op, so a re-issue does not churn empty generations. - if (issueVoid(this::flushLsm)) { + // The freeze turns everything written before this call into an SSTable, so the + // watermark has to be read after it. Idempotent: freezing an empty memtable is a + // no-op, so a re-issue does not churn empty sstables. + if (issueVoid(this::flushMemWal)) { backoff(reissue); continue; } - Attempt> stats = issue(() -> getLsmStats(false)); + Attempt> stats = issue(() -> getMemWalStats(false)); if (stats.lostClaim) { backoff(reissue); continue; } if (!stats.value.isPresent()) { - // Not WAL-backed; flushLsm would have errored first but for a race. + // Not WAL-backed; flushMemWal would have errored first but for a race. return; } - Map targets = newestGenerations(stats.value.get()); + Map targets = newestSstableGenerations(stats.value.get()); if (targets.isEmpty()) { return; } @@ -235,18 +235,18 @@ public class LanceDbTableLsm { backoff(reissue); } throw new IllegalStateException( - "checkpointLsm: the owning node kept losing its claim; re-issued from flush the maximum " + "checkpointMemWal: the owning node kept losing its claim; re-issued from flush the maximum " + "number of times"); } /** - * Trigger and poll until no bucket holds a generation at or below its target. + * Trigger and poll until no table shard holds an SSTable generation at or below its target. * * @return true when the drain finished, false when the table needs re-claiming from flush. */ private boolean drainToTargets(Map targets) { while (true) { - Attempt> stats = issue(() -> getLsmStats(false)); + Attempt> stats = issue(() -> getMemWalStats(false)); if (stats.lostClaim) { return false; } @@ -254,21 +254,21 @@ public class LanceDbTableLsm { return true; } - // `compacting` is the bucket's compaction latch, held from dispatch until the pass + // `compacting` is the table shard's compaction latch, held from dispatch until the pass // ends — including while it waits on a pod-wide permit. So it answers one question - // only: do not pile on. Buckets with nothing outstanding are skipped, not counted + // only: do not pile on. Table shards with nothing outstanding are skipped, not counted // as idle. long outstanding = 0; boolean allCompacting = true; - for (BucketStats bucket : stats.value.get().buckets()) { - Long target = targets.get(bucket.shardId()); + for (TableShardStats tableShard : stats.value.get().tableShards()) { + Long target = targets.get(tableShard.shardId()); if (target == null) { continue; } - long remaining = bucket.outstandingGenerations(target); + long remaining = tableShard.outstandingSstables(target); if (remaining > 0) { outstanding += remaining; - allCompacting &= bucket.compacting(); + allCompacting &= tableShard.compacting(); } } if (outstanding == 0) { @@ -277,7 +277,7 @@ public class LanceDbTableLsm { if (!allCompacting) { try { - compactLsm(); + compactMemWal(); } catch (LanceDbRestClient.HttpException e) { if (isLostClaim(e)) { return false; @@ -285,7 +285,7 @@ public class LanceDbTableLsm { if (!isRetryable(e)) { throw e; } - // A 429 here means the server could latch no bucket at all, which the poll + // A 429 here means the server could latch no table shard at all, which the poll // above already handles. Not retried in place: the latch it would contend for // is the one doing the work, so fall through and re-read — POLL_INTERVAL_MS is // the backoff. @@ -295,13 +295,13 @@ public class LanceDbTableLsm { } } - /** The newest generation held by each bucket, skipping buckets holding none. */ - private static Map newestGenerations(LsmStats stats) { + /** The newest SSTable generation held by each table shard, skipping table shards holding none. */ + private static Map newestSstableGenerations(MemWalStats stats) { Map targets = new HashMap(); - for (BucketStats bucket : stats.buckets()) { - OptionalLong newest = bucket.newestGeneration(); + for (TableShardStats tableShard : stats.tableShards()) { + OptionalLong newest = tableShard.newestSstableGeneration(); if (newest.isPresent()) { - targets.put(bucket.shardId(), newest.getAsLong()); + targets.put(tableShard.shardId(), newest.getAsLong()); } } return targets; diff --git a/java/lancedb-core/src/main/java/com/lancedb/LsmStats.java b/java/lancedb-core/src/main/java/com/lancedb/LsmStats.java deleted file mode 100644 index 3496ebc96..000000000 --- a/java/lancedb-core/src/main/java/com/lancedb/LsmStats.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.lancedb; - -import com.fasterxml.jackson.databind.JsonNode; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Live per-bucket LSM state, as returned by {@link LanceDbTableLsm#getLsmStats()}. - * - *

Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are the caller's to - * compute. There is no "LSM is off" shape — that case is an empty {@link java.util.Optional}, - * because a stats object of zeros would read as measurements. - */ -public class LsmStats { - private static final String CONTEXT = "lsm stats"; - - private final List buckets; - - LsmStats(List buckets) { - this.buckets = Collections.unmodifiableList(buckets); - } - - /** One entry per bucket. */ - public List buckets() { - return buckets; - } - - static LsmStats fromJson(JsonNode node) { - JsonFields.requiredObject(node, CONTEXT); - List buckets = new ArrayList(); - for (JsonNode bucket : JsonFields.requiredArray(node, "buckets", CONTEXT)) { - buckets.add(BucketStats.fromJson(bucket)); - } - return new LsmStats(buckets); - } - - @Override - public String toString() { - return "LsmStats{buckets=" + buckets + "}"; - } -} diff --git a/java/lancedb-core/src/main/java/com/lancedb/MemWalStats.java b/java/lancedb-core/src/main/java/com/lancedb/MemWalStats.java new file mode 100644 index 000000000..c84220508 --- /dev/null +++ b/java/lancedb-core/src/main/java/com/lancedb/MemWalStats.java @@ -0,0 +1,56 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb; + +import com.fasterxml.jackson.databind.JsonNode; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Live per-table-shard MemWAL state, as returned by {@link LanceDbTableMemWal#getMemWalStats()}. + * + *

Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are the caller's + * to compute. There is no "LSM is off" shape — that case is an empty {@link java.util.Optional}, + * because a stats object of zeros would read as measurements. + */ +public class MemWalStats { + private static final String CONTEXT = "memwal stats"; + + private final List tableShards; + + MemWalStats(List tableShards) { + this.tableShards = Collections.unmodifiableList(tableShards); + } + + /** One entry per table shard. */ + public List tableShards() { + return tableShards; + } + + static MemWalStats fromJson(JsonNode node) { + JsonFields.requiredObject(node, CONTEXT); + List tableShards = new ArrayList(); + for (JsonNode tableShard : JsonFields.requiredArray(node, "table_shards", CONTEXT)) { + tableShards.add(TableShardStats.fromJson(tableShard)); + } + return new MemWalStats(tableShards); + } + + @Override + public String toString() { + return "MemWalStats{tableShards=" + tableShards + "}"; + } +} diff --git a/java/lancedb-core/src/main/java/com/lancedb/LsmWriteSpec.java b/java/lancedb-core/src/main/java/com/lancedb/MemWalWriteSpec.java similarity index 87% rename from java/lancedb-core/src/main/java/com/lancedb/LsmWriteSpec.java rename to java/lancedb-core/src/main/java/com/lancedb/MemWalWriteSpec.java index c02298c45..95f541214 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/LsmWriteSpec.java +++ b/java/lancedb-core/src/main/java/com/lancedb/MemWalWriteSpec.java @@ -27,16 +27,16 @@ import java.util.Map; * *

Construct via {@link #bucket}, {@link #identity}, or {@link #unsharded}, then optionally chain * {@link #withMaintainedIndexes} and {@link #withWriterConfigDefaults}. Install it with {@link - * LanceDbTableLsm#setLsmWriteSpec} and remove it with {@link LanceDbTableLsm#unsetLsmWriteSpec}. + * LanceDbTableMemWal#enableMemWal} and remove it with {@link LanceDbTableMemWal#disableMemWal}. * *

This is deliberately not {@code org.lance.memwal.InitializeMemWalParams}. That type is Lance's * own, and its maintained-index default is the opposite of this one: it defaults to maintaining * nothing, while a fresh spec here maintains every index. It also cannot express * the null that asks the server to resolve the set. */ -public class LsmWriteSpec { +public class MemWalWriteSpec { - private static final String CONTEXT = "get_lsm_write_spec response"; + private static final String CONTEXT = "get_memwal_spec response"; /** How writes are routed to MemWAL shards. */ public enum Sharding { @@ -73,7 +73,7 @@ public class LsmWriteSpec { private final List maintainedIndexes; private final Map writerConfigDefaults; - private LsmWriteSpec( + private MemWalWriteSpec( Sharding sharding, String column, Integer numBuckets, @@ -95,11 +95,11 @@ public class LsmWriteSpec { * @param column A non-nested column with a supported scalar type. * @param numBuckets The number of buckets, in {@code [1, 1024]}. */ - public static LsmWriteSpec bucket(String column, int numBuckets) { + public static MemWalWriteSpec bucket(String column, int numBuckets) { if (column == null || column.trim().isEmpty()) { throw new IllegalArgumentException("Column cannot be null or empty"); } - return new LsmWriteSpec( + return new MemWalWriteSpec( Sharding.BUCKET, column, numBuckets, null, new HashMap()); } @@ -111,16 +111,17 @@ public class LsmWriteSpec { * with a given primary key must always produce the same {@code column} value, or upserts of that * key can land in different shards and a stale version can win. */ - public static LsmWriteSpec identity(String column) { + public static MemWalWriteSpec identity(String column) { if (column == null || column.trim().isEmpty()) { throw new IllegalArgumentException("Column cannot be null or empty"); } - return new LsmWriteSpec(Sharding.IDENTITY, column, null, null, new HashMap()); + return new MemWalWriteSpec( + Sharding.IDENTITY, column, null, null, new HashMap()); } /** No sharding — every write goes to a single MemWAL shard — maintaining every index. */ - public static LsmWriteSpec unsharded() { - return new LsmWriteSpec(Sharding.UNSHARDED, null, null, null, new HashMap()); + public static MemWalWriteSpec unsharded() { + return new MemWalWriteSpec(Sharding.UNSHARDED, null, null, null, new HashMap()); } /** @@ -131,8 +132,8 @@ public class LsmWriteSpec { * *

Note that {@code null} and the empty list mean opposite things here. */ - public LsmWriteSpec withMaintainedIndexes(List maintainedIndexes) { - return new LsmWriteSpec( + public MemWalWriteSpec withMaintainedIndexes(List maintainedIndexes) { + return new MemWalWriteSpec( sharding, column, numBuckets, @@ -149,11 +150,11 @@ public class LsmWriteSpec { * max_unflushed_memtable_bytes}, and {@code enable_memtable}. Duration knobs carry an {@code _ms} * suffix, such as {@code max_wal_flush_interval_ms}. */ - public LsmWriteSpec withWriterConfigDefaults(Map writerConfigDefaults) { + public MemWalWriteSpec withWriterConfigDefaults(Map writerConfigDefaults) { if (writerConfigDefaults == null) { throw new IllegalArgumentException("writerConfigDefaults cannot be null"); } - return new LsmWriteSpec( + return new MemWalWriteSpec( sharding, column, numBuckets, @@ -191,7 +192,7 @@ public class LsmWriteSpec { return Collections.unmodifiableMap(writerConfigDefaults); } - /** Render this spec as the {@code set_lsm_write_spec} request body. */ + /** Render this spec as the {@code enable_memwal} request body. */ Map toRequestBody() { Map shardingBody = new LinkedHashMap(); shardingBody.put("mode", sharding.wireName()); @@ -211,15 +212,15 @@ public class LsmWriteSpec { } /** - * Rebuild a spec from a {@code get_lsm_write_spec} response body. + * Rebuild a spec from a {@code get_memwal_spec} response body. * *

A null {@code maintained_indexes} selects every index the table has; an empty list selects * none. */ - static LsmWriteSpec fromJson(JsonNode node) { + static MemWalWriteSpec fromJson(JsonNode node) { JsonNode shardingNode = node.get("sharding"); if (shardingNode == null || shardingNode.get("mode") == null) { - throw new IllegalStateException("get_lsm_write_spec response has no sharding mode"); + throw new IllegalStateException("get_memwal_spec response has no sharding mode"); } Sharding sharding = Sharding.fromWireName(shardingNode.get("mode").asText()); @@ -252,12 +253,12 @@ public class LsmWriteSpec { .forEachRemaining(name -> defaults.put(name, defaultsNode.get(name).asText())); } - return new LsmWriteSpec(sharding, column, numBuckets, maintainedIndexes, defaults); + return new MemWalWriteSpec(sharding, column, numBuckets, maintainedIndexes, defaults); } @Override public String toString() { - return "LsmWriteSpec{sharding=" + return "MemWalWriteSpec{sharding=" + sharding + ", column=" + column diff --git a/java/lancedb-core/src/main/java/com/lancedb/MemtableStats.java b/java/lancedb-core/src/main/java/com/lancedb/MemtableStats.java index 777e915aa..b9ef5a89d 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/MemtableStats.java +++ b/java/lancedb-core/src/main/java/com/lancedb/MemtableStats.java @@ -37,7 +37,7 @@ public class MemtableStats { this.indexes = Collections.unmodifiableList(indexes); } - /** The generation this memtable will become once sealed. */ + /** The generation this memtable will become once frozen. */ public long generation() { return generation; } diff --git a/java/lancedb-core/src/main/java/com/lancedb/GenerationStats.java b/java/lancedb-core/src/main/java/com/lancedb/SsTableStats.java similarity index 75% rename from java/lancedb-core/src/main/java/com/lancedb/GenerationStats.java rename to java/lancedb-core/src/main/java/com/lancedb/SsTableStats.java index 12222407c..6e57c1ddf 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/GenerationStats.java +++ b/java/lancedb-core/src/main/java/com/lancedb/SsTableStats.java @@ -17,21 +17,21 @@ import com.fasterxml.jackson.databind.JsonNode; import java.util.OptionalLong; -/** One flushed L0 generation. */ -public class GenerationStats { +/** One SSTable. */ +public class SsTableStats { private static final String CONTEXT = "generation stats"; private final long generation; private final long bytes; private final Long rows; - GenerationStats(long generation, long bytes, Long rows) { + SsTableStats(long generation, long bytes, Long rows) { this.generation = generation; this.bytes = bytes; this.rows = rows; } - /** The generation number. Increases as memtables are sealed into L0. */ + /** The generation number. Increases as memtables are frozen into SSTables. */ public long generation() { return generation; } @@ -42,16 +42,16 @@ public class GenerationStats { } /** - * Rows in this generation, present only when {@code includeGenerationRows} was requested. Off by + * Rows in this generation, present only when {@code includeSstableRows} was requested. Off by * default because each count opens an uncached Lance dataset. */ public OptionalLong rows() { return rows == null ? OptionalLong.empty() : OptionalLong.of(rows); } - static GenerationStats fromJson(JsonNode node) { + static SsTableStats fromJson(JsonNode node) { JsonFields.requiredObject(node, CONTEXT); - return new GenerationStats( + return new SsTableStats( JsonFields.requiredLong(node, "generation", CONTEXT), JsonFields.requiredLong(node, "bytes", CONTEXT), JsonFields.optionalLong(node, "rows", CONTEXT)); @@ -59,6 +59,6 @@ public class GenerationStats { @Override public String toString() { - return "GenerationStats{generation=" + generation + ", bytes=" + bytes + ", rows=" + rows + "}"; + return "SsTableStats{generation=" + generation + ", bytes=" + bytes + ", rows=" + rows + "}"; } } diff --git a/java/lancedb-core/src/main/java/com/lancedb/BucketStats.java b/java/lancedb-core/src/main/java/com/lancedb/TableShardStats.java similarity index 76% rename from java/lancedb-core/src/main/java/com/lancedb/BucketStats.java rename to java/lancedb-core/src/main/java/com/lancedb/TableShardStats.java index 2a8060c5d..dcbc4fa93 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/BucketStats.java +++ b/java/lancedb-core/src/main/java/com/lancedb/TableShardStats.java @@ -22,11 +22,11 @@ import java.util.Optional; import java.util.OptionalLong; /** - * Live state of one bucket. A table is N buckets on one node; flattening to a single number hides - * the one hot bucket that is usually why someone opened this endpoint. + * Live state of one table shard. A table is N table shards on one node; flattening to a single + * number hides the one hot table shard that is usually why someone opened this endpoint. */ -public class BucketStats { - private static final String CONTEXT = "bucket stats"; +public class TableShardStats { + private static final String CONTEXT = "tableShard stats"; private final String shardId; private final String status; @@ -35,11 +35,11 @@ public class BucketStats { private final long currentGeneration; private final long replayAfterWalEntryPosition; private final long walEntryPositionLastSeen; - private final List generations; + private final List sstables; private final boolean compacting; private final List memtables; - BucketStats( + TableShardStats( String shardId, String status, long writerEpoch, @@ -47,7 +47,7 @@ public class BucketStats { long currentGeneration, long replayAfterWalEntryPosition, long walEntryPositionLastSeen, - List generations, + List sstables, boolean compacting, List memtables) { this.shardId = shardId; @@ -57,12 +57,12 @@ public class BucketStats { this.currentGeneration = currentGeneration; this.replayAfterWalEntryPosition = replayAfterWalEntryPosition; this.walEntryPositionLastSeen = walEntryPositionLastSeen; - this.generations = Collections.unmodifiableList(generations); + this.sstables = Collections.unmodifiableList(sstables); this.compacting = compacting; this.memtables = memtables == null ? null : Collections.unmodifiableList(memtables); } - /** The shard this bucket writes. */ + /** The id of this table shard. */ public String shardId() { return shardId; } @@ -100,13 +100,13 @@ public class BucketStats { return walEntryPositionLastSeen; } - /** Flushed L0 generations not yet merged into the base table. */ - public List generations() { - return generations; + /** SSTables not yet merged into the base table. */ + public List sstables() { + return sstables; } /** - * Whether a pass owns this bucket's compaction latch right now. Says a driver is + * Whether a pass owns this table shard's compaction latch right now. Says a driver is * running, not whose, and the latch is held from dispatch — including while the pass * queues for a pod-wide compactor permit. Read it as "do not pile on", never as "mine is * progressing". @@ -115,15 +115,17 @@ public class BucketStats { return compacting; } - /** Oldest first, active last. Empty for a {@code "Sealed"} bucket, whose state is torn down. */ + /** + * Oldest first, active last. Empty for a {@code "Sealed"} table shard, whose state is torn down. + */ public Optional> memtables() { return Optional.ofNullable(memtables); } - /** The newest flushed generation, or empty when L0 is empty. */ - OptionalLong newestGeneration() { + /** The newest SSTable generation, or empty when the tier is empty. */ + OptionalLong newestSstableGeneration() { OptionalLong newest = OptionalLong.empty(); - for (GenerationStats generation : generations) { + for (SsTableStats generation : sstables) { if (!newest.isPresent() || generation.generation() > newest.getAsLong()) { newest = OptionalLong.of(generation.generation()); } @@ -132,15 +134,15 @@ public class BucketStats { } /** - * How many generations at or below {@code target} are still in L0. + * How many SSTables at or below {@code target} are still uncompacted. * *

A count, not a boolean: one pass drains a bounded prefix rather than the whole target set, * so a boolean would read as "no progress" for every pass but the last. Compaction drains * oldest-first, so this decreases monotonically. */ - long outstandingGenerations(long target) { + long outstandingSstables(long target) { long count = 0; - for (GenerationStats generation : generations) { + for (SsTableStats generation : sstables) { if (generation.generation() <= target) { count++; } @@ -148,11 +150,11 @@ public class BucketStats { return count; } - static BucketStats fromJson(JsonNode node) { + static TableShardStats fromJson(JsonNode node) { JsonFields.requiredObject(node, CONTEXT); - List generations = new ArrayList(); - for (JsonNode generation : JsonFields.requiredArray(node, "generations", CONTEXT)) { - generations.add(GenerationStats.fromJson(generation)); + List sstables = new ArrayList(); + for (JsonNode generation : JsonFields.requiredArray(node, "sstables", CONTEXT)) { + sstables.add(SsTableStats.fromJson(generation)); } JsonNode memtablesNode = JsonFields.optionalArray(node, "memtables", CONTEXT); @@ -164,7 +166,7 @@ public class BucketStats { } } - return new BucketStats( + return new TableShardStats( JsonFields.requiredText(node, "shard_id", CONTEXT), JsonFields.requiredText(node, "status", CONTEXT), JsonFields.requiredLong(node, "writer_epoch", CONTEXT), @@ -172,21 +174,21 @@ public class BucketStats { JsonFields.requiredLong(node, "current_generation", CONTEXT), JsonFields.requiredLong(node, "replay_after_wal_entry_position", CONTEXT), JsonFields.requiredLong(node, "wal_entry_position_last_seen", CONTEXT), - generations, + sstables, JsonFields.requiredBoolean(node, "compacting", CONTEXT), memtables); } @Override public String toString() { - return "BucketStats{shardId=" + return "TableShardStats{shardId=" + shardId + ", status=" + status + ", currentGeneration=" + currentGeneration - + ", generations=" - + generations + + ", sstables=" + + sstables + ", compacting=" + compacting + "}"; diff --git a/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java b/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableMemWalTest.java similarity index 63% rename from java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java rename to java/lancedb-core/src/test/java/com/lancedb/LanceDbTableMemWalTest.java index 66ce140b6..349479988 100644 --- a/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java +++ b/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableMemWalTest.java @@ -46,12 +46,12 @@ import static org.junit.jupiter.api.Assertions.*; *

The wire assertions mirror the Rust mocked-endpoint tests in {@code * rust/lancedb/src/remote/table.rs}, which are the contract these routes have to match. */ -public class LanceDbTableLsmTest { +public class LanceDbTableMemWalTest { private static final ObjectMapper MAPPER = new ObjectMapper(); private HttpServer server; private LanceDbRestClient client; - private LanceDbTableLsm lsm; + private LanceDbTableMemWal memWal; private final List requestPaths = Collections.synchronizedList(new ArrayList()); private final List requestBodies = Collections.synchronizedList(new ArrayList()); @@ -101,7 +101,7 @@ public class LanceDbTableLsmTest { .database("test-db") .endpoint("http://127.0.0.1:" + server.getAddress().getPort()) .buildRestClient(); - lsm = new LanceDbTableLsm(client, "my_table"); + memWal = new LanceDbTableMemWal(client, "my_table"); } @AfterEach @@ -115,12 +115,12 @@ public class LanceDbTableLsmTest { // =========================================================================== @Test - public void testSetLsmWriteSpecUnsharded() throws Exception { - enqueue("set_lsm_write_spec", 200, ""); + public void testEnableMemWalUnsharded() throws Exception { + enqueue("enable_memwal", 200, ""); - lsm.setLsmWriteSpec(LsmWriteSpec.unsharded()); + memWal.enableMemWal(MemWalWriteSpec.unsharded()); - assertEquals("/v1/table/my_table/set_lsm_write_spec/", requestPaths.get(0)); + assertEquals("/v1/table/my_table/enable_memwal/", requestPaths.get(0)); JsonNode body = MAPPER.readTree(requestBodies.get(0)); assertEquals("unsharded", body.get("sharding").get("mode").asText()); assertFalse(body.get("sharding").has("column")); @@ -128,11 +128,11 @@ public class LanceDbTableLsmTest { } @Test - public void testSetLsmWriteSpecBucket() throws Exception { - enqueue("set_lsm_write_spec", 200, ""); + public void testEnableMemWalBucket() throws Exception { + enqueue("enable_memwal", 200, ""); - lsm.setLsmWriteSpec( - LsmWriteSpec.bucket("id", 16).withMaintainedIndexes(Arrays.asList("id_idx"))); + memWal.enableMemWal( + MemWalWriteSpec.bucket("id", 16).withMaintainedIndexes(Arrays.asList("id_idx"))); JsonNode body = MAPPER.readTree(requestBodies.get(0)); assertEquals("bucket", body.get("sharding").get("mode").asText()); @@ -143,10 +143,10 @@ public class LanceDbTableLsmTest { } @Test - public void testSetLsmWriteSpecIdentity() throws Exception { - enqueue("set_lsm_write_spec", 200, ""); + public void testEnableMemWalIdentity() throws Exception { + enqueue("enable_memwal", 200, ""); - lsm.setLsmWriteSpec(LsmWriteSpec.identity("tenant")); + memWal.enableMemWal(MemWalWriteSpec.identity("tenant")); JsonNode body = MAPPER.readTree(requestBodies.get(0)); assertEquals("identity", body.get("sharding").get("mode").asText()); @@ -160,55 +160,55 @@ public class LanceDbTableLsmTest { */ @Test public void testMaintainedIndexesNullAndEmptyAreDistinctOnTheWire() throws Exception { - enqueue("set_lsm_write_spec", 200, ""); + enqueue("enable_memwal", 200, ""); - lsm.setLsmWriteSpec(LsmWriteSpec.unsharded()); + memWal.enableMemWal(MemWalWriteSpec.unsharded()); JsonNode fresh = MAPPER.readTree(requestBodies.get(0)); assertTrue(fresh.has("maintained_indexes"), "the key must be present"); assertTrue(fresh.get("maintained_indexes").isNull(), "a fresh spec sends null, not []"); - lsm.setLsmWriteSpec( - LsmWriteSpec.unsharded().withMaintainedIndexes(Collections.emptyList())); + memWal.enableMemWal( + MemWalWriteSpec.unsharded().withMaintainedIndexes(Collections.emptyList())); JsonNode none = MAPPER.readTree(requestBodies.get(1)); assertTrue(none.get("maintained_indexes").isArray()); assertEquals(0, none.get("maintained_indexes").size()); } @Test - public void testSetLsmWriteSpecWriterConfigDefaults() throws Exception { - enqueue("set_lsm_write_spec", 200, ""); + public void testEnableMemWalWriterConfigDefaults() throws Exception { + enqueue("enable_memwal", 200, ""); Map defaults = new HashMap(); defaults.put("max_memtable_rows", "50000"); - lsm.setLsmWriteSpec(LsmWriteSpec.unsharded().withWriterConfigDefaults(defaults)); + memWal.enableMemWal(MemWalWriteSpec.unsharded().withWriterConfigDefaults(defaults)); JsonNode body = MAPPER.readTree(requestBodies.get(0)); assertEquals("50000", body.get("writer_config_defaults").get("max_memtable_rows").asText()); } @Test - public void testUnsetLsmWriteSpec() { - enqueue("unset_lsm_write_spec", 200, ""); + public void testDisableMemWal() { + enqueue("disable_memwal", 200, ""); - lsm.unsetLsmWriteSpec(); + memWal.disableMemWal(); - assertEquals("/v1/table/my_table/unset_lsm_write_spec/", requestPaths.get(0)); + assertEquals("/v1/table/my_table/disable_memwal/", requestPaths.get(0)); assertEquals("", requestBodies.get(0)); } @Test - public void testGetLsmWriteSpec() { + public void testGetMemWalSpec() { enqueue( - "get_lsm_write_spec", + "get_memwal_spec", 200, - "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"bucket\",\"column\":\"id\"," + "{\"memwal_spec\":{\"sharding\":{\"mode\":\"bucket\",\"column\":\"id\"," + "\"num_buckets\":16},\"maintained_indexes\":[\"id_idx\"]," + "\"writer_config_defaults\":{\"durable_write\":\"true\"}}}"); - Optional spec = lsm.getLsmWriteSpec(); + Optional spec = memWal.getMemWalSpec(); assertTrue(spec.isPresent()); - assertEquals(LsmWriteSpec.Sharding.BUCKET, spec.get().sharding()); + assertEquals(MemWalWriteSpec.Sharding.BUCKET, spec.get().sharding()); assertEquals("id", spec.get().column()); assertEquals(Integer.valueOf(16), spec.get().numBuckets()); assertEquals(Arrays.asList("id_idx"), spec.get().maintainedIndexes()); @@ -217,28 +217,28 @@ public class LanceDbTableLsmTest { /** A null selection, every index the table has, reads back as null rather than an empty list. */ @Test - public void testGetLsmWriteSpecRoundTripsEveryIndexSelection() { + public void testGetMemWalSpecRoundTripsEveryIndexSelection() { enqueue( - "get_lsm_write_spec", + "get_memwal_spec", 200, - "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "\"maintained_indexes\":null,\"writer_config_defaults\":{}}}"); - Optional spec = lsm.getLsmWriteSpec(); + Optional spec = memWal.getMemWalSpec(); assertTrue(spec.isPresent()); assertNull(spec.get().maintainedIndexes()); } @Test - public void testGetLsmWriteSpecRoundTripsNoIndexSelection() { + public void testGetMemWalSpecRoundTripsNoIndexSelection() { enqueue( - "get_lsm_write_spec", + "get_memwal_spec", 200, - "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "\"maintained_indexes\":[],\"writer_config_defaults\":{}}}"); - Optional spec = lsm.getLsmWriteSpec(); + Optional spec = memWal.getMemWalSpec(); assertTrue(spec.isPresent()); assertEquals(Collections.emptyList(), spec.get().maintainedIndexes()); @@ -249,32 +249,32 @@ public class LanceDbTableLsmTest { * rejected: reading a string or an object as null would turn a malformed response into a policy. */ @Test - public void testGetLsmWriteSpecRejectsANonArrayMaintainedIndexes() { + public void testGetMemWalSpecRejectsANonArrayMaintainedIndexes() { enqueue( - "get_lsm_write_spec", + "get_memwal_spec", 200, - "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "\"maintained_indexes\":\"id_idx\",\"writer_config_defaults\":{}}}"); - assertThrows(IllegalStateException.class, () -> lsm.getLsmWriteSpec()); + assertThrows(IllegalStateException.class, () -> memWal.getMemWalSpec()); } @Test - public void testGetLsmWriteSpecRejectsNonStringIndexNames() { + public void testGetMemWalSpecRejectsNonStringIndexNames() { enqueue( - "get_lsm_write_spec", + "get_memwal_spec", 200, - "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "\"maintained_indexes\":[7],\"writer_config_defaults\":{}}}"); - assertThrows(IllegalStateException.class, () -> lsm.getLsmWriteSpec()); + assertThrows(IllegalStateException.class, () -> memWal.getMemWalSpec()); } @Test - public void testGetLsmWriteSpecAbsent() { - enqueue("get_lsm_write_spec", 200, "{\"lsm_write_spec\":null}"); + public void testGetMemWalSpecAbsent() { + enqueue("get_memwal_spec", 200, "{\"memwal_spec\":null}"); - assertFalse(lsm.getLsmWriteSpec().isPresent()); + assertFalse(memWal.getMemWalSpec().isPresent()); } // =========================================================================== @@ -282,15 +282,15 @@ public class LanceDbTableLsmTest { // =========================================================================== @Test - public void testGetLsmStats() throws Exception { - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L))); + public void testGetMemWalStats() throws Exception { + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); - Optional got = lsm.getLsmStats(true); + Optional got = memWal.getMemWalStats(true); - assertEquals("/v1/table/my_table/get_lsm_stats/", requestPaths.get(0)); - assertTrue(MAPPER.readTree(requestBodies.get(0)).get("include_generation_rows").asBoolean()); + assertEquals("/v1/table/my_table/get_memwal_stats/", requestPaths.get(0)); + assertTrue(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean()); assertTrue(got.isPresent()); - BucketStats decoded = got.get().buckets().get(0); + TableShardStats decoded = got.get().tableShards().get(0); assertEquals("shard-0", decoded.shardId()); assertEquals("Active", decoded.status()); assertEquals(1, decoded.writerEpoch()); @@ -298,30 +298,30 @@ public class LanceDbTableLsmTest { assertEquals(9, decoded.currentGeneration()); assertFalse(decoded.compacting()); assertEquals(Arrays.asList(7L, 8L), generationNumbers(decoded)); - assertEquals(1024, decoded.generations().get(0).bytes()); - assertFalse(decoded.generations().get(0).rows().isPresent(), "rows absent unless requested"); + assertEquals(1024, decoded.sstables().get(0).bytes()); + assertFalse(decoded.sstables().get(0).rows().isPresent(), "rows absent unless requested"); assertFalse(decoded.memtables().isPresent(), "absent memtables stay absent"); } /** The optional fields decode when the server does send them. */ @Test - public void testGetLsmStatsDecodesOptionalFields() { + public void testGetMemWalStatsDecodesOptionalFields() { enqueue( - "get_lsm_stats", + "get_memwal_stats", 200, - "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "{\"memwal_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9," + "\"replay_after_wal_entry_position\":3,\"wal_entry_position_last_seen\":11," - + "\"generations\":[{\"generation\":7,\"bytes\":1024,\"rows\":42}]," + + "\"sstables\":[{\"generation\":7,\"bytes\":1024,\"rows\":42}]," + "\"compacting\":true,\"memtables\":[{\"generation\":8,\"rows\":5," + "\"bytes\":64,\"batches\":2,\"indexes\":[\"id_idx\"]}]}]}}"); - BucketStats decoded = lsm.getLsmStats(true).get().buckets().get(0); + TableShardStats decoded = memWal.getMemWalStats(true).get().tableShards().get(0); assertEquals(3, decoded.replayAfterWalEntryPosition()); assertEquals(11, decoded.walEntryPositionLastSeen()); assertTrue(decoded.compacting()); - assertEquals(42, decoded.generations().get(0).rows().getAsLong()); + assertEquals(42, decoded.sstables().get(0).rows().getAsLong()); assertTrue(decoded.memtables().isPresent()); MemtableStats memtable = decoded.memtables().get().get(0); assertEquals(8, memtable.generation()); @@ -332,19 +332,19 @@ public class LanceDbTableLsmTest { } @Test - public void testGetLsmStatsAbsentWhenLsmDisabled() { - enqueue("get_lsm_stats", 200, "{\"lsm_stats\":null}"); + public void testGetMemWalStatsAbsentWhenMemWalDisabled() { + enqueue("get_memwal_stats", 200, "{\"memwal_stats\":null}"); - assertFalse(lsm.getLsmStats().isPresent()); + assertFalse(memWal.getMemWalStats().isPresent()); } @Test - public void testGetLsmStatsDefaultsToExcludingGenerationRows() throws Exception { - enqueue("get_lsm_stats", 200, stats()); + public void testGetMemWalStatsDefaultsToExcludingGenerationRows() throws Exception { + enqueue("get_memwal_stats", 200, stats()); - lsm.getLsmStats(); + memWal.getMemWalStats(); - assertFalse(MAPPER.readTree(requestBodies.get(0)).get("include_generation_rows").asBoolean()); + assertFalse(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean()); } // =========================================================================== @@ -353,22 +353,22 @@ public class LanceDbTableLsmTest { @Test public void testFlushAndCompactRoutes() { - enqueue("flush_lsm", 200, ""); - enqueue("compact_lsm", 200, ""); + enqueue("flush_memwal", 200, ""); + enqueue("compact_memwal", 200, ""); - lsm.flushLsm(); - lsm.compactLsm(); + memWal.flushMemWal(); + memWal.compactMemWal(); - assertEquals("/v1/table/my_table/flush_lsm/", requestPaths.get(0)); - assertEquals("/v1/table/my_table/compact_lsm/", requestPaths.get(1)); + assertEquals("/v1/table/my_table/flush_memwal/", requestPaths.get(0)); + assertEquals("/v1/table/my_table/compact_memwal/", requestPaths.get(1)); } @Test public void testHttpErrorCarriesStatus() { - enqueue("flush_lsm", 404, "no such table"); + enqueue("flush_memwal", 404, "no such table"); LanceDbRestClient.HttpException e = - assertThrows(LanceDbRestClient.HttpException.class, () -> lsm.flushLsm()); + assertThrows(LanceDbRestClient.HttpException.class, () -> memWal.flushMemWal()); assertEquals(404, e.statusCode()); } @@ -377,97 +377,98 @@ public class LanceDbTableLsmTest { // =========================================================================== @Test - public void testCheckpointReturnsWhenLsmDisabled() { - enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 200, "{\"lsm_stats\":null}"); + public void testCheckpointReturnsWhenMemWalDisabled() { + enqueue("flush_memwal", 200, ""); + enqueue("get_memwal_stats", 200, "{\"memwal_stats\":null}"); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(0, countCalls("compact_lsm"), "nothing to compact when the LSM path is off"); + assertEquals(0, countCalls("compact_memwal"), "nothing to compact when the LSM path is off"); } @Test public void testCheckpointReturnsWhenNoGenerationsOutstanding() { - enqueue("flush_lsm", 200, ""); - // A bucket with no L0 generations yields no target, so the drain never starts. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false))); + enqueue("flush_memwal", 200, ""); + // A table shard with no SSTables yields no target, so the drain never starts. + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false))); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(0, countCalls("compact_lsm")); + assertEquals(0, countCalls("compact_memwal")); } @Test public void testCheckpointConvergesOnceTargetGenerationsAreGone() { - enqueue("flush_lsm", 200, ""); - // Watermark read: shard-0 holds generations 7 and 8, so target = 8. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L))); + enqueue("flush_memwal", 200, ""); + // Watermark read: shard-0 holds sstables 7 and 8, so target = 8. + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); // First drain poll: both still outstanding, nothing compacting -> dispatch a pass. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L))); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); // Second drain poll: drained past the target -> done. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 9L))); - enqueue("compact_lsm", 200, ""); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 9L))); + enqueue("compact_memwal", 200, ""); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(1, countCalls("compact_lsm"), "one pass dispatched"); - assertEquals(3, countCalls("get_lsm_stats"), "watermark read plus two drain polls"); + assertEquals(1, countCalls("compact_memwal"), "one pass dispatched"); + assertEquals(3, countCalls("get_memwal_stats"), "watermark read plus two drain polls"); } @Test public void testCheckpointDoesNotPileOnWhileEveryTargetBucketIsCompacting() { - enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", true, 4L))); + enqueue("flush_memwal", 200, ""); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", true, 4L))); // Still compacting on the first poll, so no pass is dispatched; then it drains. - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", true, 4L))); - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 5L))); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", true, 4L))); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 5L))); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(0, countCalls("compact_lsm"), "a latched bucket is left alone"); + assertEquals(0, countCalls("compact_memwal"), "a latched table shard is left alone"); } @Test public void testCheckpointRetriesFromFlushAfterLostClaim() { // 421 on the watermark read: the node lost its claim, so the whole thing restarts // from flush rather than retrying the read in place. - enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 421, "no claim"); - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false))); + enqueue("flush_memwal", 200, ""); + enqueue("get_memwal_stats", 421, "no claim"); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false))); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(2, countCalls("flush_lsm"), "re-issued from flush"); + assertEquals(2, countCalls("flush_memwal"), "re-issued from flush"); } @Test public void testCheckpointRetriesRetryableStatusInPlace() { - enqueue("flush_lsm", 429, "latch held"); - enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false))); + enqueue("flush_memwal", 429, "latch held"); + enqueue("flush_memwal", 200, ""); + enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false))); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(2, countCalls("flush_lsm"), "429 retried in place, not re-issued"); + assertEquals(2, countCalls("flush_memwal"), "429 retried in place, not re-issued"); } @Test public void testCheckpointPropagatesTerminalStatus() { - enqueue("flush_lsm", 400, "bad request"); + enqueue("flush_memwal", 400, "bad request"); LanceDbRestClient.HttpException e = - assertThrows(LanceDbRestClient.HttpException.class, () -> lsm.checkpointLsm()); + assertThrows(LanceDbRestClient.HttpException.class, () -> memWal.checkpointMemWal()); assertEquals(400, e.statusCode()); - assertEquals(1, countCalls("flush_lsm"), "a terminal status is not retried"); + assertEquals(1, countCalls("flush_memwal"), "a terminal status is not retried"); } @Test public void testCheckpointGivesUpAfterRepeatedLostClaims() { - enqueue("flush_lsm", 421, "no claim"); + enqueue("flush_memwal", 421, "no claim"); - IllegalStateException e = assertThrows(IllegalStateException.class, () -> lsm.checkpointLsm()); + IllegalStateException e = + assertThrows(IllegalStateException.class, () -> memWal.checkpointMemWal()); assertTrue(e.getMessage().contains("kept losing its claim"), e.getMessage()); - assertEquals(4, countCalls("flush_lsm"), "the initial attempt plus MAX_REISSUES"); + assertEquals(4, countCalls("flush_memwal"), "the initial attempt plus MAX_REISSUES"); } // =========================================================================== @@ -476,50 +477,51 @@ public class LanceDbTableLsmTest { /** * A stats payload that does not decode must fail closed. Every one of these bodies used to be - * read as "no buckets", which is indistinguishable from a drained table, so {@code checkpointLsm} - * reported convergence for a checkpoint that never ran. + * read as "no table shards", which is indistinguishable from a drained table, so {@code + * checkpointMemWal} reported convergence for a checkpoint that never ran. */ @Test public void testCheckpointRejectsMalformedStats() { Map malformed = new LinkedHashMap(); malformed.put("no response body at all", ""); - malformed.put("stats object with no buckets", "{\"lsm_stats\":{}}"); - malformed.put("bucket missing its required fields", "{\"lsm_stats\":{\"buckets\":[{}]}}"); + malformed.put("stats object with no table shards", "{\"memwal_stats\":{}}"); malformed.put( - "bucket missing generations", - "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "table shard missing its required fields", "{\"memwal_stats\":{\"table_shards\":[{}]}}"); + malformed.put( + "table shard missing sstables", + "{\"memwal_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9," + "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0," + "\"compacting\":false}]}}"); malformed.put( "generation with a non-numeric generation number", - "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "{\"memwal_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9," + "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0," - + "\"generations\":[{\"generation\":\"7\",\"bytes\":1024}]," + + "\"sstables\":[{\"generation\":\"7\",\"bytes\":1024}]," + "\"compacting\":false}]}}"); for (Map.Entry each : malformed.entrySet()) { setUpFresh(); - enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 200, each.getValue()); + enqueue("flush_memwal", 200, ""); + enqueue("get_memwal_stats", 200, each.getValue()); assertThrows( IllegalStateException.class, - () -> lsm.checkpointLsm(), + () -> memWal.checkpointMemWal(), each.getKey() + " must not report convergence"); } } - /** The one shape that legitimately means "this table has no LSM write path". */ + /** The one shape that legitimately means "this table has no MemWAL write path". */ @Test public void testCheckpointTreatsNullStatsAsNotWalBacked() { - enqueue("flush_lsm", 200, ""); - enqueue("get_lsm_stats", 200, "{\"lsm_stats\":null}"); + enqueue("flush_memwal", 200, ""); + enqueue("get_memwal_stats", 200, "{\"memwal_stats\":null}"); - lsm.checkpointLsm(); + memWal.checkpointMemWal(); - assertEquals(1, countCalls("get_lsm_stats")); + assertEquals(1, countCalls("get_memwal_stats")); } // =========================================================================== @@ -529,40 +531,41 @@ public class LanceDbTableLsmTest { /** * The transport must not retry on the checkpoint loop's behalf. Apache HttpClient's default * strategy retries exactly 429 and 503 — the two statuses {@code isRetryable} owns — which - * doubled every budget here and also retried {@code compact_lsm} in place, where the loop is + * doubled every budget here and also retried {@code compact_memwal} in place, where the loop is * built to fall through to a fresh stats poll instead. */ @Test public void testCheckpointRetryBudgetIsNotDoubledByTheTransport() { - enqueue("flush_lsm", 429, "latch held"); + enqueue("flush_memwal", 429, "latch held"); LanceDbRestClient.HttpException e = - assertThrows(LanceDbRestClient.HttpException.class, () -> lsm.checkpointLsm()); + assertThrows(LanceDbRestClient.HttpException.class, () -> memWal.checkpointMemWal()); assertEquals(429, e.statusCode(), "the exhausted budget propagates the last error as itself"); - assertEquals(9, countCalls("flush_lsm"), "the initial request plus MAX_RETRIES, and no more"); + assertEquals( + 9, countCalls("flush_memwal"), "the initial request plus MAX_RETRIES, and no more"); } // =========================================================================== // harness // =========================================================================== - private static List generationNumbers(BucketStats bucket) { + private static List generationNumbers(TableShardStats tableShard) { List numbers = new ArrayList(); - for (GenerationStats generation : bucket.generations()) { + for (SsTableStats generation : tableShard.sstables()) { numbers.add(generation.generation()); } return numbers; } - /** Build an {@code lsm_stats} response body from bucket fragments. */ - private static String stats(String... buckets) { - return "{\"lsm_stats\":{\"buckets\":[" + String.join(",", buckets) + "]}}"; + /** Build a {@code memwal_stats} response body from table shard fragments. */ + private static String stats(String... tableShards) { + return "{\"memwal_stats\":{\"table_shards\":[" + String.join(",", tableShards) + "]}}"; } - private static String bucket(String shardId, boolean compacting, Long... generations) { + private static String tableShard(String shardId, boolean compacting, Long... sstables) { StringBuilder gens = new StringBuilder(); - for (Long generation : generations) { + for (Long generation : sstables) { if (gens.length() > 0) { gens.append(","); } @@ -572,7 +575,7 @@ public class LanceDbTableLsmTest { + shardId + "\",\"status\":\"Active\",\"writer_epoch\":1,\"manifest_version\":2," + "\"current_generation\":9,\"replay_after_wal_entry_position\":0," - + "\"wal_entry_position_last_seen\":0,\"generations\":[" + + "\"wal_entry_position_last_seen\":0,\"sstables\":[" + gens + "],\"compacting\":" + compacting @@ -597,7 +600,7 @@ public class LanceDbTableLsmTest { return requestPaths.stream().filter(path -> operationOf(path).equals(operation)).count(); } - /** {@code /v1/table/my_table/flush_lsm/} -> {@code flush_lsm}. */ + /** {@code /v1/table/my_table/flush_memwal/} -> {@code flush_memwal}. */ private static String operationOf(String path) { String[] segments = path.split("/"); return segments.length == 0 ? "" : segments[segments.length - 1]; diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 6c770494b..f53f859b8 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -640,11 +640,11 @@ describe.each(arrowVersions)( ); }); - it("should expose useLsm on takeRowIds as the base-only escape hatch", async () => { + it("should expose useMemWal on takeRowIds as the base-only escape hatch", async () => { await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]); - // useLsm(false) is reachable on TakeQuery (the escape hatch for MemWAL tables, + // useMemWal(false) is reachable on TakeQuery (the escape hatch for MemWAL tables, // where take-by-row-id auto-routes to the LSM scanner and is rejected). - const res = await table.takeRowIds([0, 2]).useLsm(false).toArray(); + const res = await table.takeRowIds([0, 2]).useMemWal(false).toArray(); expect(res.map((r) => r.id)).toEqual([1, 3]); }); @@ -4628,7 +4628,7 @@ describe("setUnenforcedPrimaryKey", () => { }); }); -describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { +describe("enableMemWal / disableMemWal", () => { let tmpDir: tmp.DirResult; beforeEach(() => { @@ -4648,16 +4648,16 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { const table = await makeTable(conn); await table.setUnenforcedPrimaryKey("id"); - await table.setLsmWriteSpec({ + await table.enableMemWal({ specType: "bucket", column: "id", numBuckets: 4, }); - await table.unsetLsmWriteSpec(); + await table.disableMemWal(); // A second unset errors — there is no spec left to remove. - await expect(table.unsetLsmWriteSpec()).rejects.toThrow(); + await expect(table.disableMemWal()).rejects.toThrow(); // A fresh spec can be installed after unset. - await table.setLsmWriteSpec({ + await table.enableMemWal({ specType: "bucket", column: "id", numBuckets: 8, @@ -4669,8 +4669,8 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { const table = await makeTable(conn); await table.setUnenforcedPrimaryKey("id"); - await table.setLsmWriteSpec({ specType: "unsharded" }); - await table.unsetLsmWriteSpec(); + await table.enableMemWal({ specType: "unsharded" }); + await table.disableMemWal(); }); it("installs an identity spec", async () => { @@ -4678,8 +4678,8 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { const table = await makeTable(conn); await table.setUnenforcedPrimaryKey("id"); - await table.setLsmWriteSpec({ specType: "identity", column: "id" }); - await table.unsetLsmWriteSpec(); + await table.enableMemWal({ specType: "identity", column: "id" }); + await table.disableMemWal(); }); it("rejects an invalid spec", async () => { @@ -4689,7 +4689,7 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { await table.setUnenforcedPrimaryKey("id"); // num_buckets out of range. await expect( - table.setLsmWriteSpec({ + table.enableMemWal({ specType: "bucket", column: "id", numBuckets: 0, @@ -4697,7 +4697,7 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { ).rejects.toThrow(); // Column mismatch. await expect( - table.setLsmWriteSpec({ + table.enableMemWal({ specType: "bucket", column: "missing", numBuckets: 4, @@ -4705,13 +4705,13 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { ).rejects.toThrow(); }); - it("reads back the installed spec via getLsmWriteSpec", async () => { + it("reads back the installed spec via getMemWalSpec", async () => { const conn = await connect(tmpDir.name); const table = await makeTable(conn); await table.setUnenforcedPrimaryKey("id"); // Nothing installed yet. - expect(await table.getLsmWriteSpec()).toBeUndefined(); + expect(await table.getMemWalSpec()).toBeUndefined(); // A real scalar index is needed to name it as a maintained index. await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]); @@ -4722,14 +4722,14 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { // defaults. Lance writer-config keys are canonically snake_case. // biome-ignore lint/style/useNamingConvention: Lance writer-config keys are snake_case const writerConfigDefaults = { durable_write: "false" }; - await table.setLsmWriteSpec({ + await table.enableMemWal({ specType: "bucket", column: "id", numBuckets: 4, maintainedIndexes: [indexName], writerConfigDefaults, }); - const spec = await table.getLsmWriteSpec(); + const spec = await table.getMemWalSpec(); expect(spec).toBeDefined(); expect(spec?.specType).toBe("bucket"); expect(spec?.column).toBe("id"); @@ -4738,19 +4738,19 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => { expect(spec?.writerConfigDefaults).toEqual(writerConfigDefaults); // After unset, undefined again. - await table.unsetLsmWriteSpec(); - expect(await table.getLsmWriteSpec()).toBeUndefined(); + await table.disableMemWal(); + expect(await table.getMemWalSpec()).toBeUndefined(); // Identity round-trips (column recovered from the schema). - await table.setLsmWriteSpec({ specType: "identity", column: "id" }); - const identity = await table.getLsmWriteSpec(); + await table.enableMemWal({ specType: "identity", column: "id" }); + const identity = await table.getMemWalSpec(); expect(identity?.specType).toBe("identity"); expect(identity?.column).toBe("id"); - await table.unsetLsmWriteSpec(); + await table.disableMemWal(); // Unsharded round-trips (no routing column). - await table.setLsmWriteSpec({ specType: "unsharded" }); - const unsharded = await table.getLsmWriteSpec(); + await table.enableMemWal({ specType: "unsharded" }); + const unsharded = await table.getMemWalSpec(); expect(unsharded?.specType).toBe("unsharded"); expect(unsharded?.column).toBeFalsy(); }); @@ -4779,7 +4779,7 @@ describe("LSM merge insert", () => { ]); await table.setUnenforcedPrimaryKey("id"); // numBuckets = 1: every row routes to the single bucket. - await table.setLsmWriteSpec({ + await table.enableMemWal({ specType: "bucket", column: "id", numBuckets: 1, @@ -4804,17 +4804,17 @@ describe("LSM merge insert", () => { expect(res.version).toBe(0); expect(res.numInsertedRows).toBe(0); - await table.closeLsmWriters(); + await table.closeMemWalWriters(); }); - it("falls back to the standard path with useLsm(false)", async () => { + it("falls back to the standard path with useMemWal(false)", async () => { const conn = await connect(tmpDir.name); const table = await bucketTable(conn); const res = await table .mergeInsert("id") .whenNotMatchedInsertAll() - .useLsm(false) + .useMemWal(false) .execute([ { id: "b", value: 9 }, { id: "e", value: 5 }, @@ -4837,7 +4837,7 @@ describe("LSM merge insert", () => { expect(res.numRows).toBe(1); }); - it("rejects a non-upsert merge under an LSM spec", async () => { + it("rejects a non-upsert merge under a MemWAL spec", async () => { const conn = await connect(tmpDir.name); const table = await bucketTable(conn); @@ -4863,22 +4863,24 @@ describe("LSM merge insert", () => { const lsm = await table.query().toArray(); expect(lsm.map((r) => r.id).sort()).toEqual(["a", "b", "c"]); - // useLsm(false) bypasses the MemWAL and reads the base table only. - const baseOnly = await table.query().useLsm(false).toArray(); + // useMemWal(false) bypasses the MemWAL and reads the base table only. + const baseOnly = await table.query().useMemWal(false).toArray(); expect(baseOnly.map((r) => r.id).sort()).toEqual(["a", "b"]); }); - it("reads the base table when no LSM spec is installed", async () => { + it("reads the base table when no MemWAL spec is installed", async () => { const conn = await connect(tmpDir.name); const table = await conn.createEmptyTable( "plain", new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]), ); - // No spec: default read and useLsm(false) both succeed against the base table. + // No spec: default read and useMemWal(false) both succeed against the base table. await expect(table.query().toArray()).resolves.toBeDefined(); - await expect(table.query().useLsm(false).toArray()).resolves.toBeDefined(); - // useLsm(true) demands MemWAL routing; without a spec it errors. - await expect(table.query().useLsm(true).toArray()).rejects.toThrow(); + await expect( + table.query().useMemWal(false).toArray(), + ).resolves.toBeDefined(); + // useMemWal(true) demands MemWAL routing; without a spec it errors. + await expect(table.query().useMemWal(true).toArray()).rejects.toThrow(); }); }); @@ -4896,7 +4898,7 @@ describe("LSM convergence and stats", () => { new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]), ); await table.setUnenforcedPrimaryKey("id"); - await table.setLsmWriteSpec({ specType: "unsharded" }); + await table.enableMemWal({ specType: "unsharded" }); return table; } @@ -4904,34 +4906,34 @@ describe("LSM convergence and stats", () => { // rejects them rather than answering. What is asserted here is that the // bindings reach the core at all; the behavior against a real endpoint is // covered by the mocked endpoint tests in rust/lancedb/src/remote/table.rs. - it("rejects flushLsm on a local table", async () => { + it("rejects flushMemWal on a local table", async () => { const conn = await connect(tmpDir.name); const table = await lsmTable(conn); - await expect(table.flushLsm()).rejects.toThrow(/not supported/i); + await expect(table.flushMemWal()).rejects.toThrow(/not supported/i); }); - it("rejects compactLsm on a local table", async () => { + it("rejects compactMemWal on a local table", async () => { const conn = await connect(tmpDir.name); const table = await lsmTable(conn); - await expect(table.compactLsm()).rejects.toThrow(/not supported/i); + await expect(table.compactMemWal()).rejects.toThrow(/not supported/i); }); - it("rejects getLsmStats on a local table", async () => { + it("rejects getMemWalStats on a local table", async () => { const conn = await connect(tmpDir.name); const table = await lsmTable(conn); - await expect(table.getLsmStats()).rejects.toThrow(/not supported/i); - await expect(table.getLsmStats(true)).rejects.toThrow(/not supported/i); + await expect(table.getMemWalStats()).rejects.toThrow(/not supported/i); + await expect(table.getMemWalStats(true)).rejects.toThrow(/not supported/i); }); - it("rejects checkpointLsm on a local table", async () => { + it("rejects checkpointMemWal on a local table", async () => { const conn = await connect(tmpDir.name); const table = await lsmTable(conn); - // checkpointLsm seals first, so it surfaces flushLsm's rejection. - await expect(table.checkpointLsm()).rejects.toThrow(/not supported/i); + // checkpointMemWal freezes first, so it surfaces flushMemWal's rejection. + await expect(table.checkpointMemWal()).rejects.toThrow(/not supported/i); }); }); diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index be46e23dc..3254a0742 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -167,10 +167,10 @@ export { WriteProgress, FtsToken, TokenizeTableOptions, - LsmWriteSpec, - LsmStats, - BucketStats, - GenerationStats, + MemWalWriteSpec, + MemWalStats, + TableShardStats, + SsTableStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, diff --git a/nodejs/lancedb/merge.ts b/nodejs/lancedb/merge.ts index 5f1d8704a..0c6cdaa29 100644 --- a/nodejs/lancedb/merge.ts +++ b/nodejs/lancedb/merge.ts @@ -90,25 +90,25 @@ export class MergeInsertBuilder { /** * Control MemWAL routing for this merge. * - * By default (unset), a `mergeInsert` on a table with an LSM write spec is + * By default (unset), a `mergeInsert` on a table with a MemWAL write spec is * routed through Lance's MemWAL shard writer, and a table without one uses the * standard path. * * @param enable - `true` forces MemWAL routing and errors if the table has no - * LSM write spec. `false` forces the standard write path even when a spec is set. + * MemWAL write spec. `false` forces the standard write path even when a spec is set. */ - useLsm(enable: boolean): MergeInsertBuilder { - return new MergeInsertBuilder(this.#native.useLsm(enable), this.#schema); + useMemWal(enable: boolean): MergeInsertBuilder { + return new MergeInsertBuilder(this.#native.useMemWal(enable), this.#schema); } /** * Controls how an LSM merge checks that its input targets a single shard. * - * When a table has an LSM write spec, every row in a `mergeInsert` call must + * When a table has a MemWAL write spec, every row in a `mergeInsert` call must * route to the same shard. When `true` (the default), every row is inspected * to verify this. When `false`, only the first row is inspected and the * shard it routes to is used for the whole input — a faster path for callers * that have already pre-sharded their input. Has no effect on tables without - * an LSM write spec. + * a MemWAL write spec. * * @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`. */ diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index cce505c97..3583e5992 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -464,10 +464,10 @@ export class StandardQueryBase< * Control MemWAL read routing for this query. * * By default (unset), when the table carries a MemWAL write spec (see - * {@link Table#setLsmWriteSpec}), reads are routed through the LSM scanner so + * {@link Table#enableMemWal}), reads are routed through the LSM scanner so * they also return data written via the `mergeInsert` LSM path that has not yet * been compacted into the base table (the active/frozen in-memory memtables and - * the flushed generations), deduplicated by primary key; a table without a spec + * the SSTables), deduplicated by primary key; a table without a spec * reads the base table. * * @param enable - `true` forces the LSM scanner and errors if the table has no @@ -476,11 +476,11 @@ export class StandardQueryBase< * * Note: the LSM scanner does not support every query shape (e.g. reranking, * hybrid search, `orderBy`). On a MemWAL table those shapes error unless - * `useLsm(false)` is set, because a base-only read would silently exclude + * `useMemWal(false)` is set, because a base-only read would silently exclude * un-compacted MemWAL data. */ - useLsm(enable: boolean): this { - this.doCall((inner: NativeQueryType) => inner.useLsm(enable)); + useMemWal(enable: boolean): this { + this.doCall((inner: NativeQueryType) => inner.useMemWal(enable)); return this; } } @@ -871,8 +871,8 @@ export class TakeQuery extends QueryBase { * * @param enable - `false` reads the base table only. */ - useLsm(enable: boolean): this { - this.doCall((inner: NativeTakeQuery) => inner.useLsm(enable)); + useMemWal(enable: boolean): this { + this.doCall((inner: NativeTakeQuery) => inner.useMemWal(enable)); return this; } } diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 630d06f29..4d6500a5f 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -35,7 +35,7 @@ import { FunctionErrorsOptions, IndexConfig, IndexStatistics, - LsmStats, + MemWalStats, Branches as NativeBranches, OptimizeStats, RefreshColumnResult, @@ -59,9 +59,9 @@ import { sanitizeType } from "./sanitize"; import { IntoSql, toSQL } from "./util"; export { IndexConfig } from "./native"; export { - BucketStats, - GenerationStats, - LsmStats, + TableShardStats, + SsTableStats, + MemWalStats, MemtableStats, } from "./native"; @@ -217,7 +217,7 @@ export type TokenizeTableOptions = * `column` value, or upserts of that key can land in different shards and a * stale version can win). */ -export interface LsmWriteSpec { +export interface MemWalWriteSpec { /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */ specType: "bucket" | "identity" | "unsharded"; /** Bucket and identity variants: the sharding column. */ @@ -758,10 +758,10 @@ export abstract class Table { */ abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise; /** - * Install an {@link LsmWriteSpec} on this table, selecting Lance's MemWAL + * Install an {@link MemWalWriteSpec} on this table, selecting Lance's MemWAL * LSM-style write path for future `mergeInsert` calls. * - * `LsmWriteSpec` chooses one of three sharding strategies via `specType`: + * `MemWalWriteSpec` chooses one of three sharding strategies via `specType`: * * - `"bucket"` — hash-bucket writes by the single-column unenforced primary * key (`column` and `numBuckets` required). @@ -776,12 +776,12 @@ export abstract class Table { * including ones created later, and skips a kind the MemWAL cannot maintain. * Naming them pins an exact set, and a still-building index is rejected * rather than quietly omitted. - * @param {LsmWriteSpec} spec The sharding spec to install. + * @param {MemWalWriteSpec} spec The sharding spec to install. * @returns {Promise} * @example * ```ts * await table.setUnenforcedPrimaryKey("id"); - * await table.setLsmWriteSpec({ + * await table.enableMemWal({ * specType: "bucket", * column: "id", * numBuckets: 16, @@ -789,59 +789,59 @@ export abstract class Table { * }); * ``` */ - abstract setLsmWriteSpec(spec: LsmWriteSpec): Promise; + abstract enableMemWal(spec: MemWalWriteSpec): Promise; /** - * Remove the {@link LsmWriteSpec} from this table, reverting to the standard + * Remove the {@link MemWalWriteSpec} from this table, reverting to the standard * `mergeInsert` write path. * * Errors if no spec is currently set. * @returns {Promise} */ - abstract unsetLsmWriteSpec(): Promise; + abstract disableMemWal(): Promise; /** - * Read the {@link LsmWriteSpec} currently installed on this table. + * Read the {@link MemWalWriteSpec} currently installed on this table. * - * Resolves to `undefined` when the MemWAL LSM write path is not enabled (no - * spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}). + * Resolves to `undefined` when the MemWAL write path is not enabled (no + * spec has been set, or it was removed with {@link Table#disableMemWal}). * The spec is the one installed, including its maintained-index selection: * an absent `maintainedIndexes` for every index the table has, an empty * array for none. - * @returns {Promise} + * @returns {Promise} */ - abstract getLsmWriteSpec(): Promise; + abstract getMemWalSpec(): Promise; /** * Drain and close any cached MemWAL shard writers held for this table. * - * When an {@link LsmWriteSpec} is installed, `mergeInsert` opens MemWAL + * When an {@link MemWalWriteSpec} is installed, `mergeInsert` opens MemWAL * shard writers and caches them for reuse across calls. This closes them, * flushing pending data; writers reopen lazily on the next `mergeInsert`. * It is a no-op when no writers are cached. * @returns {Promise} */ - abstract closeLsmWriters(): Promise; + abstract closeMemWalWriters(): Promise; /** - * Seal every bucket's active memtable into a new L0 generation. + * Freeze every table shard's active memtable into a new SSTable. * - * Returns once the seal is committed. Sealing an empty memtable is a no-op, + * Returns once the freeze is committed. Freezing an empty memtable is a no-op, * so this is safe to call repeatedly. * @returns {Promise} */ - abstract flushLsm(): Promise; + abstract flushMemWal(): Promise; /** - * Trigger a background L0 → base compaction pass per bucket. + * Trigger a background SSTable compaction pass per table shard. * * Returns once the passes are *dispatched*, not once they finish — watch - * {@link Table#getLsmStats} for progress, or use - * {@link Table#checkpointLsm} to wait for convergence. + * {@link Table#getMemWalStats} for progress, or use + * {@link Table#checkpointMemWal} to wait for convergence. * @returns {Promise} */ - abstract compactLsm(): Promise; + abstract compactMemWal(): Promise; /** - * Converge this table's LSM write path into its base table. + * Converge this table's MemWAL write path into its base table. * - * Seals once, then triggers compaction and polls until the L0 that existed + * Freezes once, then triggers compaction and polls until the SSTables that existed * at the start is gone. The target set is fixed at the start, so - * generations created *during* the checkpoint are ignored — that is what + * SSTables created *during* the checkpoint are ignored — that is what * lets it terminate under write load, and what makes it best-effort: it * converges the fresh tier as of some instant. Idempotent, abandonable at * any point, and safe to run on a cadence. @@ -852,26 +852,26 @@ export abstract class Table { * @returns {Promise} * @example * ```ts - * const before = await table.getLsmStats(); - * await table.checkpointLsm(); - * const after = await table.getLsmStats(); + * const before = await table.getMemWalStats(); + * await table.checkpointMemWal(); + * const after = await table.getMemWalStats(); * ``` */ - abstract checkpointLsm(): Promise; + abstract checkpointMemWal(): Promise; /** - * Read live per-bucket LSM state. + * Read live per-table-shard MemWAL state. * - * Answers "how far behind is my fresh tier", "which bucket is hot", and + * Answers "how far behind is my fresh tier", "which table shard is hot", and * "why is my fresh-tier vector search brute-force". Mutates no table state. * - * Resolves to `undefined` only when the LSM write path is not enabled. - * @param {boolean} includeGenerationRows Also count rows per L0 generation. + * Resolves to `undefined` only when the MemWAL write path is not enabled. + * @param {boolean} includeSstableRows Also count rows per SSTable. * Off by default because each count opens an uncached Lance dataset. - * @returns {Promise} + * @returns {Promise} */ - abstract getLsmStats( - includeGenerationRows?: boolean, - ): Promise; + abstract getMemWalStats( + includeSstableRows?: boolean, + ): Promise; /** Retrieve the version of the table */ abstract version(): Promise; @@ -1474,43 +1474,43 @@ export class LocalTable extends Table { return await this.inner.setUnenforcedPrimaryKey(cols); } - async setLsmWriteSpec(spec: LsmWriteSpec): Promise { - return await this.inner.setLsmWriteSpec(spec); + async enableMemWal(spec: MemWalWriteSpec): Promise { + return await this.inner.enableMemWal(spec); } - async unsetLsmWriteSpec(): Promise { - return await this.inner.unsetLsmWriteSpec(); + async disableMemWal(): Promise { + return await this.inner.disableMemWal(); } - async getLsmWriteSpec(): Promise { + async getMemWalSpec(): Promise { // The native binding types `specType` as a plain `string`; narrow it back // to the public union. The Rust `From` impl only ever emits one of the // three valid values, so the cast is safe. - return ((await this.inner.getLsmWriteSpec()) ?? undefined) as - | LsmWriteSpec + return ((await this.inner.getMemWalSpec()) ?? undefined) as + | MemWalWriteSpec | undefined; } - async closeLsmWriters(): Promise { - return await this.inner.closeLsmWriters(); + async closeMemWalWriters(): Promise { + return await this.inner.closeMemWalWriters(); } - async flushLsm(): Promise { - return await this.inner.flushLsm(); + async flushMemWal(): Promise { + return await this.inner.flushMemWal(); } - async compactLsm(): Promise { - return await this.inner.compactLsm(); + async compactMemWal(): Promise { + return await this.inner.compactMemWal(); } - async checkpointLsm(): Promise { - return await this.inner.checkpointLsm(); + async checkpointMemWal(): Promise { + return await this.inner.checkpointMemWal(); } - async getLsmStats( - includeGenerationRows: boolean = false, - ): Promise { - return (await this.inner.getLsmStats(includeGenerationRows)) ?? undefined; + async getMemWalStats( + includeSstableRows: boolean = false, + ): Promise { + return (await this.inner.getMemWalStats(includeSstableRows)) ?? undefined; } async version(): Promise { diff --git a/nodejs/src/merge.rs b/nodejs/src/merge.rs index ac8649419..3d17a037e 100644 --- a/nodejs/src/merge.rs +++ b/nodejs/src/merge.rs @@ -50,10 +50,10 @@ impl NativeMergeInsertBuilder { this } - #[napi] - pub fn use_lsm(&self, enable: bool) -> Self { + #[napi(js_name = "useMemWal")] + pub fn use_memwal(&self, enable: bool) -> Self { let mut this = self.clone(); - this.inner.use_lsm(enable); + this.inner.use_memwal(enable); this } diff --git a/nodejs/src/query.rs b/nodejs/src/query.rs index 480426411..68db6f810 100644 --- a/nodejs/src/query.rs +++ b/nodejs/src/query.rs @@ -168,9 +168,9 @@ impl Query { self.inner = self.inner.clone().with_row_id(); } - #[napi] - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + #[napi(js_name = "useMemWal")] + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } #[napi] @@ -394,9 +394,9 @@ impl VectorQuery { self.inner = self.inner.clone().with_row_id(); } - #[napi] - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + #[napi(js_name = "useMemWal")] + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } #[napi] @@ -504,9 +504,9 @@ impl TakeQuery { self.inner = self.inner.clone().with_row_id(); } - #[napi] - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + #[napi(js_name = "useMemWal")] + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } #[napi(catch_unwind)] diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 32cdcce3c..5e29e23b6 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -580,64 +580,60 @@ impl Table { .default_error() } - #[napi(catch_unwind)] - pub async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> napi::Result<()> { - let native_spec = lancedb::table::LsmWriteSpec::try_from(spec)?; + #[napi(catch_unwind, js_name = "enableMemWal")] + pub async fn enable_memwal(&self, spec: MemWalWriteSpec) -> napi::Result<()> { + let native_spec = lancedb::table::MemWalWriteSpec::try_from(spec)?; self.inner_ref()? - .set_lsm_write_spec(native_spec) + .enable_memwal(native_spec) .await .default_error() } - #[napi(catch_unwind)] - pub async fn unset_lsm_write_spec(&self) -> napi::Result<()> { + #[napi(catch_unwind, js_name = "disableMemWal")] + pub async fn disable_memwal(&self) -> napi::Result<()> { + self.inner_ref()?.disable_memwal().await.default_error() + } + + #[napi(catch_unwind, js_name = "getMemWalSpec")] + pub async fn get_memwal_spec(&self) -> napi::Result> { + let spec = self.inner_ref()?.get_memwal_spec().await.default_error()?; + Ok(spec.map(MemWalWriteSpec::from)) + } + + #[napi(catch_unwind, js_name = "closeMemWalWriters")] + pub async fn close_memwal_writers(&self) -> napi::Result<()> { self.inner_ref()? - .unset_lsm_write_spec() + .close_memwal_writers() .await .default_error() } - #[napi(catch_unwind)] - pub async fn get_lsm_write_spec(&self) -> napi::Result> { - let spec = self - .inner_ref()? - .get_lsm_write_spec() - .await - .default_error()?; - Ok(spec.map(LsmWriteSpec::from)) + #[napi(catch_unwind, js_name = "flushMemWal")] + pub async fn flush_memwal(&self) -> napi::Result<()> { + self.inner_ref()?.flush_memwal().await.default_error() } - #[napi(catch_unwind)] - pub async fn close_lsm_writers(&self) -> napi::Result<()> { - self.inner_ref()?.close_lsm_writers().await.default_error() + #[napi(catch_unwind, js_name = "compactMemWal")] + pub async fn compact_memwal(&self) -> napi::Result<()> { + self.inner_ref()?.compact_memwal().await.default_error() } - #[napi(catch_unwind)] - pub async fn flush_lsm(&self) -> napi::Result<()> { - self.inner_ref()?.flush_lsm().await.default_error() + #[napi(catch_unwind, js_name = "checkpointMemWal")] + pub async fn checkpoint_memwal(&self) -> napi::Result<()> { + self.inner_ref()?.checkpoint_memwal().await.default_error() } - #[napi(catch_unwind)] - pub async fn compact_lsm(&self) -> napi::Result<()> { - self.inner_ref()?.compact_lsm().await.default_error() - } - - #[napi(catch_unwind)] - pub async fn checkpoint_lsm(&self) -> napi::Result<()> { - self.inner_ref()?.checkpoint_lsm().await.default_error() - } - - #[napi(catch_unwind)] - pub async fn get_lsm_stats( + #[napi(catch_unwind, js_name = "getMemWalStats")] + pub async fn get_memwal_stats( &self, - include_generation_rows: bool, - ) -> napi::Result> { + include_sstable_rows: bool, + ) -> napi::Result> { let stats = self .inner_ref()? - .get_lsm_stats(include_generation_rows) + .get_memwal_stats(include_sstable_rows) .await .default_error()?; - Ok(stats.map(LsmStats::from)) + Ok(stats.map(MemWalStats::from)) } #[napi(catch_unwind)] @@ -958,7 +954,7 @@ impl From for FtsToken { /// `column` is required. #[napi(object)] #[derive(Clone, Debug)] -pub struct LsmWriteSpec { +pub struct MemWalWriteSpec { /// One of `"bucket"`, `"identity"`, or `"unsharded"`. pub spec_type: String, /// Bucket and identity variants: the sharding column. @@ -972,31 +968,31 @@ pub struct LsmWriteSpec { pub writer_config_defaults: Option>, } -impl TryFrom for lancedb::table::LsmWriteSpec { +impl TryFrom for lancedb::table::MemWalWriteSpec { type Error = napi::Error; - fn try_from(value: LsmWriteSpec) -> napi::Result { + fn try_from(value: MemWalWriteSpec) -> napi::Result { let writer_config_defaults = value.writer_config_defaults.unwrap_or_default(); let spec = match value.spec_type.as_str() { "bucket" => { let column = value.column.ok_or_else(|| { - napi::Error::from_reason("LsmWriteSpec bucket requires `column`") + napi::Error::from_reason("MemWalWriteSpec bucket requires `column`") })?; let num_buckets = value.num_buckets.ok_or_else(|| { - napi::Error::from_reason("LsmWriteSpec bucket requires `numBuckets`") + napi::Error::from_reason("MemWalWriteSpec bucket requires `numBuckets`") })?; Self::bucket(column, num_buckets) } "identity" => { let column = value.column.ok_or_else(|| { - napi::Error::from_reason("LsmWriteSpec identity requires `column`") + napi::Error::from_reason("MemWalWriteSpec identity requires `column`") })?; Self::identity(column) } "unsharded" => Self::unsharded(), other => { return Err(napi::Error::from_reason(format!( - "LsmWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'", + "MemWalWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'", other ))); } @@ -1007,9 +1003,9 @@ impl TryFrom for lancedb::table::LsmWriteSpec { } } -impl From for LsmWriteSpec { - fn from(spec: lancedb::table::LsmWriteSpec) -> Self { - use lancedb::table::LsmWriteSpec as Native; +impl From for MemWalWriteSpec { + fn from(spec: lancedb::table::MemWalWriteSpec) -> Self { + use lancedb::table::MemWalWriteSpec as Native; match spec { Native::Bucket { column, @@ -1048,21 +1044,21 @@ impl From for LsmWriteSpec { } } -/// One flushed L0 generation. +/// One SSTable. #[napi(object)] #[derive(Clone, Debug)] -pub struct GenerationStats { - /// The generation number. Increases as memtables are sealed into L0. +pub struct SsTableStats { + /// The generation number. Increases as memtables are frozen into SSTables. pub generation: i64, - /// On-disk size of the generation. + /// On-disk size of the SSTable. pub bytes: i64, - /// Present only when `includeGenerationRows` was requested. Off by default + /// Present only when `includeSstableRows` was requested. Off by default /// because each count opens an uncached Lance dataset. pub rows: Option, } -impl From for GenerationStats { - fn from(g: lancedb::table::GenerationStats) -> Self { +impl From for SsTableStats { + fn from(g: lancedb::table::SsTableStats) -> Self { Self { generation: g.generation as i64, bytes: g.bytes as i64, @@ -1075,7 +1071,7 @@ impl From for GenerationStats { #[napi(object)] #[derive(Clone, Debug)] pub struct MemtableStats { - /// The generation this memtable will become once sealed. + /// The generation this memtable will become once frozen. pub generation: i64, /// Rows currently buffered. pub rows: i64, @@ -1100,13 +1096,13 @@ impl From for MemtableStats { } } -/// Live state of one bucket. A table is N buckets on one node; flattening to a -/// single number hides the one hot bucket that is usually why someone opened +/// Live state of one table shard. A table is N table shards on one node; flattening to a +/// single number hides the one hot table shard that is usually why someone opened /// this endpoint. #[napi(object)] #[derive(Clone, Debug)] -pub struct BucketStats { - /// The shard this bucket writes. +pub struct TableShardStats { + /// The id of this table shard. pub shard_id: String, /// `"Active"` or `"Sealed"` (drop-table 2PC in flight). pub status: String, @@ -1121,20 +1117,20 @@ pub struct BucketStats { /// Highest WAL position the writer has seen. The difference against /// `replayAfterWalEntryPosition` is the WAL lag. pub wal_entry_position_last_seen: i64, - /// Flushed L0 generations not yet merged into the base table. - pub generations: Vec, - /// Whether a pass owns this bucket's compaction latch right now. Says *a* + /// SSTables not yet merged into the base table. + pub sstables: Vec, + /// Whether a pass owns this table shard's compaction latch right now. Says *a* /// driver is running, not *whose*, and the latch is held from dispatch — /// including while the pass queues for a pod-wide compactor permit. Read it /// as "do not pile on", never as "mine is progressing". pub compacting: bool, - /// Oldest first, active last. Absent for a `"Sealed"` bucket, whose + /// Oldest first, active last. Absent for a `"Sealed"` table shard, whose /// in-memory state is torn down. pub memtables: Option>, } -impl From for BucketStats { - fn from(b: lancedb::table::BucketStats) -> Self { +impl From for TableShardStats { + fn from(b: lancedb::table::TableShardStats) -> Self { Self { shard_id: b.shard_id, status: b.status, @@ -1143,7 +1139,7 @@ impl From for BucketStats { current_generation: b.current_generation as i64, replay_after_wal_entry_position: b.replay_after_wal_entry_position as i64, wal_entry_position_last_seen: b.wal_entry_position_last_seen as i64, - generations: b.generations.into_iter().map(Into::into).collect(), + sstables: b.sstables.into_iter().map(Into::into).collect(), compacting: b.compacting, memtables: b .memtables @@ -1152,21 +1148,21 @@ impl From for BucketStats { } } -/// Live per-bucket LSM state, as returned by `Table#getLsmStats`. +/// Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`. /// -/// Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are +/// Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are /// the caller's to compute. #[napi(object)] #[derive(Clone, Debug)] -pub struct LsmStats { - /// One entry per bucket backing this table. - pub buckets: Vec, +pub struct MemWalStats { + /// One entry per table shard backing this table. + pub table_shards: Vec, } -impl From for LsmStats { - fn from(stats: lancedb::table::LsmStats) -> Self { +impl From for MemWalStats { + fn from(stats: lancedb::table::MemWalStats) -> Self { Self { - buckets: stats.buckets.into_iter().map(Into::into).collect(), + table_shards: stats.table_shards.into_iter().map(Into::into).collect(), } } } diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 39353f270..4ed64ee31 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -15,7 +15,7 @@ from ._lancedb import FtsToken from ._lancedb import FunctionErrorFragment as FunctionErrorFragment from ._lancedb import FunctionErrorRecord as FunctionErrorRecord from ._lancedb import FunctionErrors as FunctionErrors -from ._lancedb import LsmWriteSpec +from ._lancedb import MemWalWriteSpec from ._lancedb import tokenize as _tokenize from .common import URI, sanitize_uri from urllib.parse import urlparse @@ -610,7 +610,7 @@ __all__ = [ "SqlQuery", "LanceDBConnection", "LanceNamespaceDBConnection", - "LsmWriteSpec", + "MemWalWriteSpec", "RemoteDBConnection", "Session", "Table", diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 80554baf0..b9c033d7b 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -538,17 +538,17 @@ class Table: async def initial_storage_options(self) -> Optional[Dict[str, str]]: ... async def latest_storage_options(self) -> Optional[Dict[str, str]]: ... async def set_unenforced_primary_key(self, columns: List[str]) -> None: ... - async def set_lsm_write_spec(self, spec: LsmWriteSpec) -> None: ... - async def unset_lsm_write_spec(self) -> None: ... - async def get_lsm_write_spec(self) -> Optional[LsmWriteSpec]: ... + async def enable_memwal(self, spec: MemWalWriteSpec) -> None: ... + async def disable_memwal(self) -> None: ... + async def get_memwal_spec(self) -> Optional[MemWalWriteSpec]: ... # WAL-PK-FUSION: delete both. def hybrid_pk_fusion_learned(self) -> bool: ... def note_hybrid_pk_fusion(self) -> None: ... - async def checkpoint_lsm(self) -> None: ... - async def flush_lsm(self) -> None: ... - async def compact_lsm(self) -> None: ... - async def get_lsm_stats(self, include_generation_rows: bool) -> Optional[dict]: ... - async def close_lsm_writers(self) -> None: ... + async def checkpoint_memwal(self) -> None: ... + async def flush_memwal(self) -> None: ... + async def compact_memwal(self) -> None: ... + async def get_memwal_stats(self, include_sstable_rows: bool) -> Optional[dict]: ... + async def close_memwal_writers(self) -> None: ... @property def tags(self) -> Tags: ... @property @@ -661,7 +661,7 @@ class Query: def fast_search(self): ... def with_row_id(self): ... def postfilter(self): ... - def use_lsm(self, enable: bool): ... + def use_memwal(self, enable: bool): ... def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ... def nearest_to_text(self, query: dict) -> FTSQuery: ... def order_by(self, ordering: Optional[List[ColumnOrdering]]): ... @@ -678,7 +678,7 @@ class Query: class TakeQuery: def select(self, columns: List[str]): ... def with_row_id(self): ... - def use_lsm(self, enable: bool): ... + def use_memwal(self, enable: bool): ... async def output_schema(self) -> pa.Schema: ... async def execute(self) -> RecordBatchStream: ... async def explain_plan(self, verbose: Optional[bool]) -> str: ... @@ -697,7 +697,7 @@ class FTSQuery: def fast_search(self): ... def with_row_id(self): ... def postfilter(self): ... - def use_lsm(self, enable: bool): ... + def use_memwal(self, enable: bool): ... def get_query(self) -> str: ... def add_query_vector(self, query_vec: pa.Array) -> None: ... def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ... @@ -725,7 +725,7 @@ class VectorQuery: def column(self, column: str): ... def distance_type(self, distance_type: str): ... def postfilter(self): ... - def use_lsm(self, enable: bool): ... + def use_memwal(self, enable: bool): ... def refine_factor(self, refine_factor: int): ... def nprobes(self, nprobes: int): ... def search_effort(self, search_effort: Optional[float]): ... @@ -750,7 +750,7 @@ class HybridQuery: def fast_search(self): ... def with_row_id(self): ... def postfilter(self): ... - def use_lsm(self, enable: bool): ... + def use_memwal(self, enable: bool): ... def distance_type(self, distance_type: str): ... def refine_factor(self, refine_factor: int): ... def nprobes(self, nprobes: int): ... @@ -778,7 +778,7 @@ class PyQueryRequest: select_source_columns: Optional[Dict[str, str]] fast_search: Optional[bool] with_row_id: Optional[bool] - use_lsm: Optional[bool] + use_memwal: Optional[bool] column: Optional[str] query_vector: Optional[List[pa.Array]] nprobes: Optional[int] @@ -839,22 +839,26 @@ class FtsToken: text: str position: int -class LsmWriteSpec: +class MemWalWriteSpec: """Specification selecting Lance's MemWAL LSM-style write path for `merge_insert`.""" @staticmethod - def bucket(column: str, num_buckets: int) -> "LsmWriteSpec": ... + def bucket(column: str, num_buckets: int) -> "MemWalWriteSpec": ... @staticmethod - def identity(column: str) -> "LsmWriteSpec": ... + def identity(column: str) -> "MemWalWriteSpec": ... @staticmethod - def unsharded() -> "LsmWriteSpec": ... - def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "LsmWriteSpec": + def unsharded() -> "MemWalWriteSpec": ... + def with_maintained_indexes( + self, indexes: Optional[List[str]] + ) -> "MemWalWriteSpec": """Set which indexes the MemWAL keeps up to date. None resolves every index on the table at install, failing if one cannot be maintained; a list is verbatim, empty means none.""" ... - def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "LsmWriteSpec": + def with_writer_config_defaults( + self, defaults: Dict[str, str] + ) -> "MemWalWriteSpec": """Return a copy of this spec recording the given default `ShardWriter` configuration in the MemWAL index.""" ... diff --git a/python/python/lancedb/_wal_hybrid.py b/python/python/lancedb/_wal_hybrid.py index 67597950d..8e5d6ae17 100644 --- a/python/python/lancedb/_wal_hybrid.py +++ b/python/python/lancedb/_wal_hybrid.py @@ -76,7 +76,7 @@ _WAL_ROW_ID_REFUSAL = "does not support with_row_id" WAL_ROW_ID_UNSUPPORTED = ( "hybrid search on a MemWAL table cannot return _rowid: the fresh tier has " "no stable row id, and the ids the fusion joins on are synthesized from " - "the primary key. Set use_lsm(False) to read the base table only (results " + "the primary key. Set use_memwal(False) to read the base table only (results " "will exclude un-compacted MemWAL data)" ) @@ -151,13 +151,13 @@ def _stamp_surrogate_row_ids( return stamped -def pk_fusion_learned(table, use_lsm: Optional[bool]) -> bool: +def pk_fusion_learned(table, use_memwal: Optional[bool]) -> bool: """Whether a previous hybrid query on `table` was refused `_rowid`, so this one should go straight to the primary key. Learned from a refusal, never probed, so asking is free. """ - if table is None or use_lsm is False: + if table is None or use_memwal is False: return False # Only an unambiguous True forks to the key: a table type that answers the # question loosely keeps the `_rowid` path. @@ -192,7 +192,7 @@ class PkFusion: raise NotImplementedError( "hybrid search cannot project a blob column on a MemWAL table: " "fetching blobs needs a real _rowid, and the fresh tier has no " - "stable one. Set use_lsm(False) to read the base table only " + "stable one. Set use_memwal(False) to read the base table only " "(results will exclude un-compacted MemWAL data)" ) return cls(_pk_columns(schema)) diff --git a/python/python/lancedb/merge.py b/python/python/lancedb/merge.py index 115a47e2d..d7c25349e 100644 --- a/python/python/lancedb/merge.py +++ b/python/python/lancedb/merge.py @@ -37,7 +37,7 @@ class LanceMergeInsertBuilder(object): self._when_not_matched_by_source_condition_expr = None self._timeout = None self._use_index = True - self._use_lsm = None + self._use_memwal = None self._validate_single_shard = None def when_matched_update_all( @@ -115,22 +115,22 @@ class LanceMergeInsertBuilder(object): self._use_index = use_index return self - def use_lsm(self, enable: bool) -> LanceMergeInsertBuilder: + def use_memwal(self, enable: bool) -> LanceMergeInsertBuilder: """ Control MemWAL routing for this merge. - By default (unset), a `merge_insert` on a table with an LSM write spec is + By default (unset), a `merge_insert` on a table with a MemWAL write spec is routed through Lance's MemWAL shard writer, and a table without one uses the standard path. Parameters ---------- enable: bool - ``True`` forces MemWAL routing and errors if the table has no LSM + ``True`` forces MemWAL routing and errors if the table has no MemWAL write spec. ``False`` forces the standard write path even when a spec is set. """ - self._use_lsm = enable + self._use_memwal = enable return self def validate_single_shard( @@ -139,13 +139,13 @@ class LanceMergeInsertBuilder(object): """ Controls how an LSM merge checks that its input targets a single shard. - When a table has an LSM write spec, every row in a `merge_insert` call + When a table has a MemWAL write spec, every row in a `merge_insert` call must route to the same shard. When `True` (the default), every row is inspected to verify this. When `False`, only the first row is inspected and the shard it routes to is used for the whole input — a faster path for callers that have already pre-sharded their input. - Has no effect on tables without an LSM write spec. + Has no effect on tables without a MemWAL write spec. Parameters ---------- diff --git a/python/python/lancedb/permutation.py b/python/python/lancedb/permutation.py index 8287ef2fe..d97eccc07 100644 --- a/python/python/lancedb/permutation.py +++ b/python/python/lancedb/permutation.py @@ -48,7 +48,7 @@ class PermutationBuilder: By default, the permutation builder will create a single split that contains all rows in the same order as the base table. - Tables with an LSM write spec are rejected: unflushed rows have no row id. + Tables with a MemWAL write spec are rejected: unflushed rows have no row id. """ self._async = async_permutation_builder(table) diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index 9734c9b43..ebb4fdf16 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -890,7 +890,7 @@ class Query(pydantic.BaseModel): # MemWAL LSM read routing: None auto-routes when the table carries a write # spec, True forces the LSM scanner (errors without a spec), False reads the # base table only - use_lsm: Optional[bool] = None + use_memwal: Optional[bool] = None # size of the nearest neighbor list maintained during HNSW search ef: Optional[int] = None @@ -910,9 +910,9 @@ class Query(pydantic.BaseModel): query.full_text_query = req.full_text_search query.columns = req.select query.with_row_id = req.with_row_id - # use_lsm is a genuine tri-state (None / True / False); preserve it as-is + # use_memwal is a genuine tri-state (None / True / False); preserve it as-is # so a round-tripped query keeps an explicit False. - query.use_lsm = req.use_lsm + query.use_memwal = req.use_memwal query.vector_column = req.column query.vector = req.query_vector query.distance_type = req.distance_type @@ -1085,7 +1085,7 @@ class LanceQueryBuilder(ABC): self._with_row_address = None self._fragments = None self._fragment_ids = None - self._use_lsm = None + self._use_memwal = None self._vector = None self._text = None self._ef = None @@ -1520,19 +1520,19 @@ class LanceQueryBuilder(ABC): self._fragment_ids = fragment_ids return self - def use_lsm(self, enable: bool) -> Self: + def use_memwal(self, enable: bool) -> Self: """Control MemWAL LSM read routing for this query. - By default (unset), a query against a table with an LSM write spec is + By default (unset), a query against a table with a MemWAL write spec is routed through the LSM scanner so it also returns data written via the ``merge_insert`` LSM path that has not yet been compacted into the base - table (active/frozen memtables + flushed generations); a table without a + table (active/frozen memtables + SSTables); a table without a spec reads the base table. Parameters ---------- enable : bool - ``True`` forces the LSM scanner and errors if the table has no LSM + ``True`` forces the LSM scanner and errors if the table has no MemWAL write spec. ``False`` bypasses the MemWAL and reads the base table only, even when a spec is present. @@ -1541,7 +1541,7 @@ class LanceQueryBuilder(ABC): LanceQueryBuilder The LanceQueryBuilder object. """ - self._use_lsm = enable + self._use_memwal = enable return self def explain_plan(self, verbose: Optional[bool] = False) -> str: @@ -2055,7 +2055,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): with_row_address=self._with_row_address, fragments=self._fragments, fragment_ids=self._fragment_ids, - use_lsm=self._use_lsm, + use_memwal=self._use_memwal, offset=self._offset, fast_search=self._fast_search, ef=self._ef, @@ -2280,7 +2280,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder): with_row_address=self._with_row_address, fragments=self._fragments, fragment_ids=self._fragment_ids, - use_lsm=self._use_lsm, + use_memwal=self._use_memwal, full_text_query=FullTextSearchQuery( query=self._query_with_phrase_semantics(), columns=self._fts_columns ), @@ -2362,7 +2362,7 @@ class LanceEmptyQueryBuilder(LanceQueryBuilder): with_row_address=self._with_row_address, fragments=self._fragments, fragment_ids=self._fragment_ids, - use_lsm=self._use_lsm, + use_memwal=self._use_memwal, offset=self._offset, order_by=self._order_by, ) @@ -3040,7 +3040,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): # WAL-PK-FUSION: without the fallback, select `self._columns` as is. self._pk_fusion = None columns = self._columns - if _wal_hybrid.pk_fusion_learned(self._table, self._use_lsm): + if _wal_hybrid.pk_fusion_learned(self._table, self._use_memwal): self._pk_fusion = _wal_hybrid.PkFusion.for_query( self._table.schema, self._columns, @@ -3057,9 +3057,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): if self._with_row_id and self._pk_fusion is None: self._vector_query.with_row_id(True) self._fts_query.with_row_id(True) - if self._use_lsm is not None: - self._vector_query.use_lsm(self._use_lsm) - self._fts_query.use_lsm(self._use_lsm) + if self._use_memwal is not None: + self._vector_query.use_memwal(self._use_memwal) + self._fts_query.use_memwal(self._use_memwal) if self._phrase_query: self._fts_query.phrase_query(True) if self._distance_type: @@ -3723,25 +3723,25 @@ class AsyncStandardQuery(AsyncQueryBase): self._inner.fast_search() return self - def use_lsm(self, enable: bool) -> Self: + def use_memwal(self, enable: bool) -> Self: """ Control MemWAL LSM read routing for this query. - By default (unset), a query against a table with an LSM write spec (see - [AsyncTable.set_lsm_write_spec][lancedb.table.AsyncTable.set_lsm_write_spec]) + By default (unset), a query against a table with a MemWAL write spec (see + [AsyncTable.enable_memwal][lancedb.table.AsyncTable.enable_memwal]) is routed through the LSM scanner so it also returns data written via the ``merge_insert`` LSM path that has not yet been compacted into the base - table (the active/frozen in-memory memtables and the flushed generations), + table (the active/frozen in-memory memtables and the SSTables), deduplicated by primary key; a table without a spec reads the base table. Parameters ---------- enable : bool - ``True`` forces the LSM scanner and errors if the table has no LSM + ``True`` forces the LSM scanner and errors if the table has no MemWAL write spec. ``False`` bypasses the MemWAL and reads the base table only, even when a spec is present. """ - self._inner.use_lsm(enable) + self._inner.use_memwal(enable) return self def postfilter(self) -> Self: @@ -4388,7 +4388,7 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase): # WAL-PK-FUSION: without the fallback, keep only the two `with_row_id` # calls. self._pk_fusion = None - if _wal_hybrid.pk_fusion_learned(self._table, fts_req.use_lsm): + if _wal_hybrid.pk_fusion_learned(self._table, fts_req.use_memwal): self._pk_fusion = _wal_hybrid.PkFusion.for_query( await self._table.schema(), _query_request_projection(fts_req), @@ -4577,13 +4577,13 @@ class AsyncTakeQuery(AsyncQueryBase): def __init__(self, inner: LanceTakeQuery, table: Optional["AsyncTable"] = None): super().__init__(inner, table) - def use_lsm(self, enable: bool) -> "AsyncTakeQuery": + def use_memwal(self, enable: bool) -> "AsyncTakeQuery": """Control MemWAL LSM read routing for this take query. ``False`` bypasses the MemWAL and reads the base table only — the escape hatch, since take-by-row-id/offset is not supported on the LSM scanner. """ - self._inner.use_lsm(enable) + self._inner.use_memwal(enable) return self async def _plain_scan_to_pandas( @@ -4644,14 +4644,14 @@ class BaseQueryBuilder(object): self._inner.with_row_id() return self - def use_lsm(self, enable: bool) -> Self: + def use_memwal(self, enable: bool) -> Self: """ Control MemWAL LSM read routing for this query. ``False`` bypasses the MemWAL and reads the base table only, the escape hatch for shapes the LSM scanner cannot honor (e.g. take-by-row-id). """ - self._inner.use_lsm(enable) + self._inner.use_memwal(enable) return self def with_row_address(self, with_row_address: bool = True) -> Self: diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 5996fdeb3..2bec79d1e 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -31,7 +31,7 @@ from lancedb._lancedb import ( DropColumnsResult, FtsToken, IndexConfig, - LsmWriteSpec, + MemWalWriteSpec, MergeResult, UpdateResult, ) @@ -1037,17 +1037,17 @@ class RemoteTable(Table): """Set the unenforced primary key for this table to a single column.""" return LOOP.run(self._table.set_unenforced_primary_key(columns)) - def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None: - """Install an LsmWriteSpec.""" - return LOOP.run(self._table.set_lsm_write_spec(spec)) + def enable_memwal(self, spec: "MemWalWriteSpec") -> None: + """Install an MemWalWriteSpec.""" + return LOOP.run(self._table.enable_memwal(spec)) - def unset_lsm_write_spec(self) -> None: - """Remove the LsmWriteSpec.""" - return LOOP.run(self._table.unset_lsm_write_spec()) + def disable_memwal(self) -> None: + """Remove the MemWalWriteSpec.""" + return LOOP.run(self._table.disable_memwal()) - def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]: - """Read the installed LsmWriteSpec, or ``None``.""" - return LOOP.run(self._table.get_lsm_write_spec()) + def get_memwal_spec(self) -> Optional["MemWalWriteSpec"]: + """Read the installed MemWalWriteSpec, or ``None``.""" + return LOOP.run(self._table.get_memwal_spec()) # WAL-PK-FUSION: delete both hooks. def _hybrid_pk_fusion_learned(self) -> bool: @@ -1057,31 +1057,31 @@ class RemoteTable(Table): def _note_hybrid_pk_fusion(self) -> None: self._table._note_hybrid_pk_fusion() - def checkpoint_lsm(self) -> None: + def checkpoint_memwal(self) -> None: """Synchronous version of - [`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm].""" - return LOOP.run(self._table.checkpoint_lsm()) + [`AsyncTable.checkpoint_memwal`][lancedb.AsyncTable.checkpoint_memwal].""" + return LOOP.run(self._table.checkpoint_memwal()) - def flush_lsm(self) -> None: + def flush_memwal(self) -> None: """Synchronous version of - [`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm].""" - return LOOP.run(self._table.flush_lsm()) + [`AsyncTable.flush_memwal`][lancedb.AsyncTable.flush_memwal].""" + return LOOP.run(self._table.flush_memwal()) - def compact_lsm(self) -> None: + def compact_memwal(self) -> None: """Synchronous version of - [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm].""" - return LOOP.run(self._table.compact_lsm()) + [`AsyncTable.compact_memwal`][lancedb.AsyncTable.compact_memwal].""" + return LOOP.run(self._table.compact_memwal()) - def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]: + def get_memwal_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]: """Synchronous version of - [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats].""" + [`AsyncTable.get_memwal_stats`][lancedb.AsyncTable.get_memwal_stats].""" return LOOP.run( - self._table.get_lsm_stats(include_generation_rows=include_generation_rows) + self._table.get_memwal_stats(include_sstable_rows=include_sstable_rows) ) - def close_lsm_writers(self) -> None: + def close_memwal_writers(self) -> None: """No-op for remote tables (no local shard writers).""" - return LOOP.run(self._table.close_lsm_writers()) + return LOOP.run(self._table.close_memwal_writers()) def drop_index(self, index_name: str): return LOOP.run(self._table.drop_index(index_name)) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index d39053775..f0cb05427 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -223,7 +223,7 @@ if TYPE_CHECKING: DeleteResult, DropColumnsResult, FtsToken, - LsmWriteSpec, + MemWalWriteSpec, MergeResult, UpdateResult, ) @@ -4827,47 +4827,47 @@ class LanceTable(Table): [`AsyncTable.set_unenforced_primary_key`][lancedb.AsyncTable.set_unenforced_primary_key].""" return LOOP.run(self._table.set_unenforced_primary_key(columns)) - def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None: - """Install an LsmWriteSpec. See - [`AsyncTable.set_lsm_write_spec`][lancedb.AsyncTable.set_lsm_write_spec].""" - return LOOP.run(self._table.set_lsm_write_spec(spec)) + def enable_memwal(self, spec: "MemWalWriteSpec") -> None: + """Install an MemWalWriteSpec. See + [`AsyncTable.enable_memwal`][lancedb.AsyncTable.enable_memwal].""" + return LOOP.run(self._table.enable_memwal(spec)) - def unset_lsm_write_spec(self) -> None: - """Remove the LsmWriteSpec. See - [`AsyncTable.unset_lsm_write_spec`][lancedb.AsyncTable.unset_lsm_write_spec].""" - return LOOP.run(self._table.unset_lsm_write_spec()) + def disable_memwal(self) -> None: + """Remove the MemWalWriteSpec. See + [`AsyncTable.disable_memwal`][lancedb.AsyncTable.disable_memwal].""" + return LOOP.run(self._table.disable_memwal()) - def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]: - """Read the installed LsmWriteSpec, or ``None``. See - [`AsyncTable.get_lsm_write_spec`][lancedb.AsyncTable.get_lsm_write_spec].""" - return LOOP.run(self._table.get_lsm_write_spec()) + def get_memwal_spec(self) -> Optional["MemWalWriteSpec"]: + """Read the installed MemWalWriteSpec, or ``None``. See + [`AsyncTable.get_memwal_spec`][lancedb.AsyncTable.get_memwal_spec].""" + return LOOP.run(self._table.get_memwal_spec()) - def checkpoint_lsm(self) -> None: + def checkpoint_memwal(self) -> None: """Synchronous version of - [`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm].""" - return LOOP.run(self._table.checkpoint_lsm()) + [`AsyncTable.checkpoint_memwal`][lancedb.AsyncTable.checkpoint_memwal].""" + return LOOP.run(self._table.checkpoint_memwal()) - def flush_lsm(self) -> None: + def flush_memwal(self) -> None: """Synchronous version of - [`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm].""" - return LOOP.run(self._table.flush_lsm()) + [`AsyncTable.flush_memwal`][lancedb.AsyncTable.flush_memwal].""" + return LOOP.run(self._table.flush_memwal()) - def compact_lsm(self) -> None: + def compact_memwal(self) -> None: """Synchronous version of - [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm].""" - return LOOP.run(self._table.compact_lsm()) + [`AsyncTable.compact_memwal`][lancedb.AsyncTable.compact_memwal].""" + return LOOP.run(self._table.compact_memwal()) - def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]: + def get_memwal_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]: """Synchronous version of - [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats].""" + [`AsyncTable.get_memwal_stats`][lancedb.AsyncTable.get_memwal_stats].""" return LOOP.run( - self._table.get_lsm_stats(include_generation_rows=include_generation_rows) + self._table.get_memwal_stats(include_sstable_rows=include_sstable_rows) ) - def close_lsm_writers(self) -> None: + def close_memwal_writers(self) -> None: """Close cached MemWAL shard writers. See - [`AsyncTable.close_lsm_writers`][lancedb.AsyncTable.close_lsm_writers].""" - return LOOP.run(self._table.close_lsm_writers()) + [`AsyncTable.close_memwal_writers`][lancedb.AsyncTable.close_memwal_writers].""" + return LOOP.run(self._table.close_memwal_writers()) def uses_v2_manifest_paths(self) -> bool: """ @@ -5525,18 +5525,18 @@ class AsyncTable: columns = list(columns) await self._inner.set_unenforced_primary_key(columns) - async def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None: - """Install an LsmWriteSpec on this table. + async def enable_memwal(self, spec: "MemWalWriteSpec") -> None: + """Install an MemWalWriteSpec on this table. The spec selects Lance's MemWAL LSM-style write path for future - `merge_insert` calls. ``LsmWriteSpec`` chooses one of three sharding + `merge_insert` calls. ``MemWalWriteSpec`` chooses one of three sharding strategies: - - ``LsmWriteSpec.bucket(column, num_buckets)`` — hash-bucket writes by + - ``MemWalWriteSpec.bucket(column, num_buckets)`` — hash-bucket writes by the single-column unenforced primary key. - - ``LsmWriteSpec.identity(column)`` — shard by the raw value of a + - ``MemWalWriteSpec.identity(column)`` — shard by the raw value of a scalar column. - - ``LsmWriteSpec.unsharded()`` — route every write to a single shard. + - ``MemWalWriteSpec.unsharded()`` — route every write to a single shard. All variants require the table to have an unenforced primary key set via [`set_unenforced_primary_key`]; bucket sharding additionally @@ -5551,30 +5551,30 @@ class AsyncTable: Parameters ---------- - spec : LsmWriteSpec + spec : MemWalWriteSpec The sharding spec to install. Examples -------- - >>> from lancedb import LsmWriteSpec + >>> from lancedb import MemWalWriteSpec >>> # table.set_unenforced_primary_key("id") - >>> # table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 16)) + >>> # table.enable_memwal(MemWalWriteSpec.bucket("id", 16)) """ - await self._inner.set_lsm_write_spec(spec) + await self._inner.enable_memwal(spec) - async def unset_lsm_write_spec(self) -> None: - """Remove the LsmWriteSpec from this table. + async def disable_memwal(self) -> None: + """Remove the MemWalWriteSpec from this table. Reverts to the standard `merge_insert` write path. Errors if no spec is currently set. """ - await self._inner.unset_lsm_write_spec() + await self._inner.disable_memwal() - async def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]: - """Read the LsmWriteSpec currently installed on this table. + async def get_memwal_spec(self) -> Optional["MemWalWriteSpec"]: + """Read the MemWalWriteSpec currently installed on this table. - Returns ``None`` when the LSM write path is not enabled at all — no - spec has been set, or one was removed with `unset_lsm_write_spec`. + Returns ``None`` when the MemWAL write path is not enabled at all — no + spec has been set, or one was removed with `disable_memwal`. That is a different answer from a spec whose ``maintained_indexes`` is ``None``, which is an installed spec selecting indexes automatically. @@ -5584,7 +5584,7 @@ class AsyncTable: or gains later, ``[]`` maintains none, and a non-empty list maintains exactly those. All three round-trip. """ - return await self._inner.get_lsm_write_spec() + return await self._inner.get_memwal_spec() # WAL-PK-FUSION: delete both hooks. def _hybrid_pk_fusion_learned(self) -> bool: @@ -5595,19 +5595,19 @@ class AsyncTable: """See [`Table._note_hybrid_pk_fusion`][lancedb.table.Table].""" self._inner.note_hybrid_pk_fusion() - async def checkpoint_lsm(self) -> None: - """Converge this table's LSM write path into its base table. + async def checkpoint_memwal(self) -> None: + """Converge this table's MemWAL write path into its base table. - One flush, sealing every memtable into L0, then compaction triggers - until every generation that existed at that moment has reached base. - The loop runs client-side, reading progress from ``get_lsm_stats``. + One flush, freezing every memtable into an SSTable, then compaction triggers + until every SSTable that existed at that moment has reached base. + The loop runs client-side, reading progress from ``get_memwal_stats``. - Best-effort: generations created *while* it runs are deliberately not + Best-effort: SSTables created *while* it runs are deliberately not waited on, which is what lets it terminate on a table taking writes. Idempotent and safe on a cadence. There is no deadline, and the caller owns that. It returns when the - target generations are gone, raises on a terminal server fault, and + target SSTables are gone, raises on a terminal server fault, and otherwise waits however long the server takes. A slow table and a stuck one are the same picture from the client: the compactor pool is shared across every table on the node, so a checkpoint queued behind @@ -5615,56 +5615,56 @@ class AsyncTable: ``asyncio.wait_for`` for a wall-clock bound; abandoning it partway costs nothing. """ - await self._inner.checkpoint_lsm() + await self._inner.checkpoint_memwal() - async def flush_lsm(self) -> None: - """Seal every bucket's active memtable into L0. + async def flush_memwal(self) -> None: + """Freeze every table shard's active memtable into an SSTable. - Does not touch the base table — moving L0 into base is - `compact_lsm`. On a node that has not claimed this table, this claims + Does not touch the base table — compacting SSTables into base is + `compact_memwal`. On a node that has not claimed this table, this claims it and replays its WAL log first. """ - await self._inner.flush_lsm() + await self._inner.flush_memwal() - async def compact_lsm(self) -> None: - """Trigger a background L0 to base compaction pass per bucket. + async def compact_memwal(self) -> None: + """Trigger a background SSTable compaction pass per table shard. Returns once the passes are dispatched, not once they finish: watch - ``get_lsm_stats`` for progress, or use ``checkpoint_lsm`` to loop - until the current L0 has reached base. + ``get_memwal_stats`` for progress, or use ``checkpoint_memwal`` to loop + until the current SSTables have reached base. """ - await self._inner.compact_lsm() + await self._inner.compact_memwal() - async def get_lsm_stats( - self, *, include_generation_rows: bool = False + async def get_memwal_stats( + self, *, include_sstable_rows: bool = False ) -> Optional[dict]: - """Read live per-bucket LSM state. + """Read live per-table-shard MemWAL state. - Answers "how far behind is my fresh tier", "which bucket is hot", and + Answers "how far behind is my fresh tier", "which table shard is hot", and "why is my fresh-tier vector search brute-force". Mutates no table state, though on a node that has not claimed this table it claims it, exactly as a read would. - Returns ``None`` only when the LSM write path is not enabled. + Returns ``None`` only when the MemWAL write path is not enabled. Parameters ---------- - include_generation_rows - Report a row count per L0 generation. Off by default: each count - opens an uncached Lance dataset, and ``checkpoint_lsm`` polls this + include_sstable_rows + Report a row count per SSTable. Off by default: each count + opens an uncached Lance dataset, and ``checkpoint_memwal`` polls this needing only generation numbers. """ - return await self._inner.get_lsm_stats(include_generation_rows) + return await self._inner.get_memwal_stats(include_sstable_rows) - async def close_lsm_writers(self) -> None: + async def close_memwal_writers(self) -> None: """Drain and close any cached MemWAL shard writers for this table. - When an LSM write spec is installed, `merge_insert` opens MemWAL shard + When a MemWAL write spec is installed, `merge_insert` opens MemWAL shard writers and caches them for reuse across calls. This closes them, flushing pending data; writers reopen lazily on the next `merge_insert`. It is a no-op when no writers are cached. """ - await self._inner.close_lsm_writers() + await self._inner.close_memwal_writers() @property def name(self) -> str: @@ -6553,8 +6553,8 @@ class AsyncTable: take_query = self.take_offsets(query.take_offsets) if query.columns: take_query = take_query.select(query.columns) - if query.use_lsm is not None: - take_query = take_query.use_lsm(query.use_lsm) + if query.use_memwal is not None: + take_query = take_query.use_memwal(query.use_memwal) if query.with_row_id: take_query = take_query.with_row_id() return take_query @@ -6570,8 +6570,8 @@ class AsyncTable: async_query = async_query.where(query.filter) if query.fast_search: async_query = async_query.fast_search() - if query.use_lsm is not None: - async_query = async_query.use_lsm(query.use_lsm) + if query.use_memwal is not None: + async_query = async_query.use_memwal(query.use_memwal) if query.with_row_id: async_query = async_query.with_row_id() if query.order_by: @@ -6697,7 +6697,7 @@ class AsyncTable: when_not_matched_by_source_condition_expr=merge._when_not_matched_by_source_condition_expr, timeout=merge._timeout, use_index=merge._use_index, - use_lsm=merge._use_lsm, + use_memwal=merge._use_memwal, validate_single_shard=merge._validate_single_shard, ), ) diff --git a/python/python/tests/test_lsm_write_spec.py b/python/python/tests/test_memwal_spec.py similarity index 65% rename from python/python/tests/test_lsm_write_spec.py rename to python/python/tests/test_memwal_spec.py index ac767ea31..a1ec6a90d 100644 --- a/python/python/tests/test_lsm_write_spec.py +++ b/python/python/tests/test_memwal_spec.py @@ -1,8 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors -"""Tests for installing and clearing an LsmWriteSpec via -`Table.set_lsm_write_spec` / `Table.unset_lsm_write_spec`. +"""Tests for installing and clearing an MemWalWriteSpec via +`Table.enable_memwal` / `Table.disable_memwal`. """ from datetime import timedelta @@ -10,7 +10,7 @@ from datetime import timedelta import lancedb import pyarrow as pa import pytest -from lancedb._lancedb import LsmWriteSpec +from lancedb._lancedb import MemWalWriteSpec from lancedb.index import BTree SCHEMA = pa.schema( @@ -21,9 +21,9 @@ SCHEMA = pa.schema( ) -def test_lsm_write_spec_module_metadata(): - assert lancedb.LsmWriteSpec is LsmWriteSpec - assert LsmWriteSpec.__module__ == "lancedb._lancedb" +def test_memwal_spec_module_metadata(): + assert lancedb.MemWalWriteSpec is MemWalWriteSpec + assert MemWalWriteSpec.__module__ == "lancedb._lancedb" def _batch(ids, vs): @@ -43,35 +43,35 @@ def _make_table(tmp_path): return db, table -def test_set_lsm_write_spec_validates(tmp_path): +def test_enable_memwal_validates(tmp_path): _db, table = _make_table(tmp_path) # Out-of-range num_buckets. with pytest.raises(Exception, match="num_buckets"): - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 0)) + table.enable_memwal(MemWalWriteSpec.bucket("id", 0)) with pytest.raises(Exception, match="num_buckets"): - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1025)) + table.enable_memwal(MemWalWriteSpec.bucket("id", 1025)) # Happy path, then a set over the installed spec is refused. - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4)) + table.enable_memwal(MemWalWriteSpec.bucket("id", 4)) with pytest.raises(Exception, match="already set"): - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8)) + table.enable_memwal(MemWalWriteSpec.bucket("id", 8)) -def test_unset_lsm_write_spec(tmp_path): +def test_disable_memwal(tmp_path): _db, table = _make_table(tmp_path) # unset errors when no spec is set. - with pytest.raises(Exception, match="no LSM write spec"): - table.unset_lsm_write_spec() + with pytest.raises(Exception, match="no MemWAL write spec"): + table.disable_memwal() # Install a spec, then remove it; afterwards a fresh spec can be set. - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4)) - table.unset_lsm_write_spec() + table.enable_memwal(MemWalWriteSpec.bucket("id", 4)) + table.disable_memwal() # A second unset errors — there is no spec left to remove. - with pytest.raises(Exception, match="no LSM write spec"): - table.unset_lsm_write_spec() - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8)) + with pytest.raises(Exception, match="no MemWAL write spec"): + table.disable_memwal() + table.enable_memwal(MemWalWriteSpec.bucket("id", 8)) def test_set_unsharded_spec(tmp_path): @@ -79,12 +79,12 @@ def test_set_unsharded_spec(tmp_path): # Lance MemWAL still requires a primary key on the dataset; Unsharded # just skips per-row hashing. table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) - table.unset_lsm_write_spec() + table.enable_memwal(MemWalWriteSpec.unsharded()) + table.disable_memwal() -def test_lsm_write_spec_repr(): - s = LsmWriteSpec.bucket("id", 4) +def test_memwal_spec_repr(): + s = MemWalWriteSpec.bucket("id", 4) assert s.spec_type == "bucket" assert s.column == "id" assert s.num_buckets == 4 @@ -95,20 +95,20 @@ def test_lsm_write_spec_repr(): assert "id" in repr(s) assert "4" in repr(s) - u = LsmWriteSpec.unsharded() + u = MemWalWriteSpec.unsharded() assert u.spec_type == "unsharded" assert u.column is None assert u.num_buckets is None assert "unsharded" in repr(u) -def test_lsm_write_spec_with_maintained_indexes(): - s = LsmWriteSpec.bucket("id", 4).with_maintained_indexes(["idx_a", "idx_b"]) +def test_memwal_spec_with_maintained_indexes(): + s = MemWalWriteSpec.bucket("id", 4).with_maintained_indexes(["idx_a", "idx_b"]) assert s.maintained_indexes == ["idx_a", "idx_b"] @pytest.mark.asyncio -async def test_async_set_unset_lsm_write_spec(tmp_path): +async def test_async_set_disable_memwal(tmp_path): db = await lancedb.connect_async( tmp_path, read_consistency_interval=timedelta(seconds=0) ) @@ -118,11 +118,11 @@ async def test_async_set_unset_lsm_write_spec(tmp_path): ) await table.set_unenforced_primary_key("id") - await table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4)) - await table.unset_lsm_write_spec() + await table.enable_memwal(MemWalWriteSpec.bucket("id", 4)) + await table.disable_memwal() # A second unset errors. - with pytest.raises(Exception, match="no LSM write spec"): - await table.unset_lsm_write_spec() + with pytest.raises(Exception, match="no MemWAL write spec"): + await table.disable_memwal() def test_set_identity_spec(tmp_path): @@ -130,12 +130,12 @@ def test_set_identity_spec(tmp_path): # Identity sharding still requires an unenforced primary key on the # table; it shards by the raw value of the given column. table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.identity("v")) - table.unset_lsm_write_spec() + table.enable_memwal(MemWalWriteSpec.identity("v")) + table.disable_memwal() -def test_lsm_write_spec_identity_and_writer_config_defaults(): - s = LsmWriteSpec.identity("v") +def test_memwal_spec_identity_and_writer_config_defaults(): + s = MemWalWriteSpec.identity("v") assert s.spec_type == "identity" assert s.column == "v" assert s.num_buckets is None @@ -146,12 +146,12 @@ def test_lsm_write_spec_identity_and_writer_config_defaults(): assert "durable_write" in repr(s) -def test_get_lsm_write_spec(tmp_path): +def test_get_memwal_spec(tmp_path): _db, table = _make_table(tmp_path) table.set_unenforced_primary_key("id") # None when nothing is installed. - assert table.get_lsm_write_spec() is None + assert table.get_memwal_spec() is None # A real scalar index is needed to name it as a maintained index. table.create_index("id", config=BTree()) @@ -159,12 +159,12 @@ def test_get_lsm_write_spec(tmp_path): # Bucket spec round-trips, including maintained indexes and writer config # defaults. - table.set_lsm_write_spec( - LsmWriteSpec.bucket("id", 4) + table.enable_memwal( + MemWalWriteSpec.bucket("id", 4) .with_maintained_indexes([idx_name]) .with_writer_config_defaults({"durable_write": "false"}) ) - spec = table.get_lsm_write_spec() + spec = table.get_memwal_spec() assert spec is not None assert spec.spec_type == "bucket" assert spec.column == "id" @@ -173,30 +173,30 @@ def test_get_lsm_write_spec(tmp_path): assert spec.writer_config_defaults == {"durable_write": "false"} # After unset, None again. - table.unset_lsm_write_spec() - assert table.get_lsm_write_spec() is None + table.disable_memwal() + assert table.get_memwal_spec() is None # Identity round-trips (column recovered from the schema). An unnamed # maintained set reads back as None: what is stored is the intent to # maintain every index, not the set it resolves to now. - table.set_lsm_write_spec(LsmWriteSpec.identity("id")) - spec = table.get_lsm_write_spec() + table.enable_memwal(MemWalWriteSpec.identity("id")) + spec = table.get_memwal_spec() assert spec.spec_type == "identity" assert spec.column == "id" assert spec.maintained_indexes is None - table.unset_lsm_write_spec() + table.disable_memwal() # Unsharded round-trips (no routing column). Opting out is distinct from # the unnamed default. - table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) - spec = table.get_lsm_write_spec() + table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([])) + spec = table.get_memwal_spec() assert spec.spec_type == "unsharded" assert spec.column is None assert spec.maintained_indexes == [] @pytest.mark.asyncio -async def test_async_get_lsm_write_spec(tmp_path): +async def test_async_get_memwal_spec(tmp_path): db = await lancedb.connect_async( tmp_path, read_consistency_interval=timedelta(seconds=0) ) @@ -205,20 +205,20 @@ async def test_async_get_lsm_write_spec(tmp_path): pa.RecordBatchReader.from_batches(SCHEMA, [_batch(["seed"], [0])]), ) - assert await table.get_lsm_write_spec() is None + assert await table.get_memwal_spec() is None # A real scalar index is needed to name it as a maintained index. await table.create_index("id", config=BTree()) idx_name = (await table.list_indices())[0].name - await table.set_lsm_write_spec( - LsmWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name]) + await table.enable_memwal( + MemWalWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name]) ) - spec = await table.get_lsm_write_spec() + spec = await table.get_memwal_spec() assert spec is not None assert spec.spec_type == "bucket" assert spec.column == "id" assert spec.num_buckets == 8 assert spec.maintained_indexes == [idx_name] - await table.unset_lsm_write_spec() - assert await table.get_lsm_write_spec() is None + await table.disable_memwal() + assert await table.get_memwal_spec() is None diff --git a/python/python/tests/test_merge_insert_lsm.py b/python/python/tests/test_merge_insert_lsm.py index e74c21589..aec25993a 100644 --- a/python/python/tests/test_merge_insert_lsm.py +++ b/python/python/tests/test_merge_insert_lsm.py @@ -8,7 +8,7 @@ from datetime import timedelta import lancedb import pyarrow as pa import pytest -from lancedb._lancedb import LsmWriteSpec +from lancedb._lancedb import MemWalWriteSpec from lancedb.index import FTS, IvfPq SCHEMA = pa.schema( @@ -49,12 +49,12 @@ def _region_reader(rows): def _bucket_table(tmp_path): - """A table with ``id`` as the primary key and a single-bucket LSM spec.""" + """A table with ``id`` as the primary key and a single-bucket MemWAL spec.""" db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") # num_buckets = 1: every row routes to the single bucket. - table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1)) + table.enable_memwal(MemWalWriteSpec.bucket("id", 1)) return table @@ -78,7 +78,7 @@ def test_lsm_merge_insert_unsharded(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.enable_memwal(MemWalWriteSpec.unsharded()) result = ( table.merge_insert("id") .when_matched_update_all() @@ -92,7 +92,7 @@ def test_lsm_merge_insert_identity(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _region_reader([(1, "us"), (2, "us")])) table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.identity("region")) + table.enable_memwal(MemWalWriteSpec.identity("region")) # All rows share one identity value, so they route to one shard. result = ( table.merge_insert([]) @@ -103,31 +103,31 @@ def test_lsm_merge_insert_identity(tmp_path): assert result.num_rows == 2 -def test_lsm_merge_insert_use_lsm_false(tmp_path): +def test_lsm_merge_insert_use_memwal_false(tmp_path): table = _bucket_table(tmp_path) # rows id = 1, 2, 3 - # use_lsm(False) opts out: the standard path runs and commits even with a spec. + # use_memwal(False) opts out: the standard path runs and commits even with a spec. result = ( table.merge_insert("id") .when_not_matched_insert_all() - .use_lsm(False) + .use_memwal(False) .execute(_reader([3, 4, 5])) ) assert result.num_inserted_rows == 2 assert table.count_rows() == 5 -def test_lsm_merge_insert_use_lsm_true_without_spec_errors(tmp_path): - # A table with a primary key but no LSM write spec installed. +def test_lsm_merge_insert_use_memwal_true_without_spec_errors(tmp_path): + # A table with a primary key but no MemWAL write spec installed. db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - # use_lsm(True) demands MemWAL routing; without a spec it errors. - with pytest.raises(Exception, match="use_lsm"): + # use_memwal(True) demands MemWAL routing; without a spec it errors. + with pytest.raises(Exception, match="use_memwal"): ( table.merge_insert("id") .when_matched_update_all() .when_not_matched_insert_all() - .use_lsm(True) + .use_memwal(True) .execute(_reader([3, 4, 5])) ) @@ -145,7 +145,7 @@ def test_lsm_merge_insert_validate_single_shard_off(tmp_path): def test_lsm_merge_insert_no_spec_uses_standard_path(tmp_path): - # A table with a primary key but no LSM write spec installed. + # A table with a primary key but no MemWAL write spec installed. db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") @@ -186,7 +186,7 @@ def test_lsm_close_writers(tmp_path): .when_not_matched_insert_all() .execute(_reader([7, 8])) ) - table.close_lsm_writers() + table.close_memwal_writers() # The writer reopens lazily on the next merge_insert. result = ( table.merge_insert([]) @@ -204,14 +204,14 @@ async def test_async_lsm_merge_insert(tmp_path): ) table = await db.create_table("t", _reader([1, 2, 3])) await table.set_unenforced_primary_key("id") - await table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1)) + await table.enable_memwal(MemWalWriteSpec.bucket("id", 1)) builder = ( table.merge_insert([]).when_matched_update_all().when_not_matched_insert_all() ) result = await builder.execute(_reader([3, 4, 5])) assert result.num_rows == 3 - await table.close_lsm_writers() + await table.close_memwal_writers() def _lsm_upsert(table, ids): @@ -228,7 +228,7 @@ def test_lsm_read_sees_active_memtable(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) # base ids 1,2,3 table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.enable_memwal(MemWalWriteSpec.unsharded()) _lsm_upsert(table, [4, 5]) # active memtable only, not committed to base @@ -236,8 +236,8 @@ def test_lsm_read_sees_active_memtable(tmp_path): lsm = table.search().to_arrow() assert sorted(lsm["id"].to_pylist()) == [1, 2, 3, 4, 5] - # use_lsm(False) bypasses the MemWAL and reads the base table only. - base_only = table.search().use_lsm(False).to_arrow() + # use_memwal(False) bypasses the MemWAL and reads the base table only. + base_only = table.search().use_memwal(False).to_arrow() assert sorted(base_only["id"].to_pylist()) == [1, 2, 3] @@ -245,7 +245,7 @@ def test_lsm_read_dedup_newest_wins(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) # id 2 -> value 1 table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.enable_memwal(MemWalWriteSpec.unsharded()) _lsm_upsert(table, [2, 3, 4]) # ids 2,3,4 -> values 0,1,2 @@ -258,22 +258,22 @@ def test_lsm_read_dedup_newest_wins(tmp_path): def test_lsm_read_without_spec_reads_base(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) - table.set_unenforced_primary_key("id") # no LSM write spec + table.set_unenforced_primary_key("id") # no MemWAL write spec - # No spec: default read and use_lsm(False) both read the base table, no error. + # No spec: default read and use_memwal(False) both read the base table, no error. assert sorted(table.search().to_arrow()["id"].to_pylist()) == [1, 2, 3] - assert sorted(table.search().use_lsm(False).to_arrow()["id"].to_pylist()) == [ + assert sorted(table.search().use_memwal(False).to_arrow()["id"].to_pylist()) == [ 1, 2, 3, ] -def test_lsm_read_unsupported_shape_errors_without_use_lsm_false(tmp_path): +def test_lsm_read_unsupported_shape_errors_without_use_memwal_false(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.enable_memwal(MemWalWriteSpec.unsharded()) _lsm_upsert(table, [4]) # with_row_id is unsupported by the LSM scanner; on a MemWAL table the default @@ -281,8 +281,8 @@ def test_lsm_read_unsupported_shape_errors_without_use_lsm_false(tmp_path): with pytest.raises(Exception): table.search().with_row_id(True).to_arrow() - # use_lsm(False) is the escape hatch: it reads the base table only. - base = table.search().with_row_id(True).use_lsm(False).to_arrow() + # use_memwal(False) is the escape hatch: it reads the base table only. + base = table.search().with_row_id(True).use_memwal(False).to_arrow() assert sorted(base["id"].to_pylist()) == [1, 2, 3] @@ -293,7 +293,7 @@ async def test_async_lsm_read(tmp_path): ) table = await db.create_table("t", _reader([1, 2, 3])) await table.set_unenforced_primary_key("id") - await table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + await table.enable_memwal(MemWalWriteSpec.unsharded()) builder = ( table.merge_insert([]).when_matched_update_all().when_not_matched_insert_all() @@ -331,7 +331,7 @@ def _vector_reader(rows): def _vector_table(tmp_path): """Base table whose vector column is indexed so its rows are visible to the LSM vector scanner (the base arm uses ``fast_search`` — indexed data only), plus an - unsharded LSM spec that maintains that index for the memtable. + unsharded MemWAL spec that maintains that index for the memtable. Rows 1,2 are category ``a``, row 3 is ``b``, and 4..60 are filler ``c`` that give the tiny IVF index enough data to train. @@ -353,8 +353,8 @@ def _vector_table(tmp_path): "vector", config=IvfPq(num_partitions=1, num_sub_vectors=2, num_bits=4) ) index_name = table.list_indices()[0].name - table.set_lsm_write_spec( - LsmWriteSpec.unsharded().with_maintained_indexes([index_name]) + table.enable_memwal( + MemWalWriteSpec.unsharded().with_maintained_indexes([index_name]) ) return table @@ -379,9 +379,9 @@ def test_lsm_read_vector_sees_memtable(tmp_path): assert {1, 2, 3} <= ids # indexed base rows assert 1000 in ids # in-flight memtable row - # use_lsm(False) bypasses the MemWAL, so the in-flight row is not visible. + # use_memwal(False) bypasses the MemWAL, so the in-flight row is not visible. base_ids = set( - table.search(query).use_lsm(False).limit(100).to_arrow()["id"].to_pylist() + table.search(query).use_memwal(False).limit(100).to_arrow()["id"].to_pylist() ) assert {1, 2, 3} <= base_ids assert 1000 not in base_ids @@ -453,8 +453,8 @@ def test_lsm_read_fts_sees_memtable(tmp_path): # Native FTS index (tantivy is not compatible with the LSM memtable index). table.create_index("text", config=FTS()) index_name = table.list_indices()[0].name - table.set_lsm_write_spec( - LsmWriteSpec.unsharded().with_maintained_indexes([index_name]) + table.enable_memwal( + MemWalWriteSpec.unsharded().with_maintained_indexes([index_name]) ) # in-flight doc 4 lands in the memtable's maintained FTS index. @@ -498,8 +498,8 @@ def test_lsm_read_vector_unsupported_knobs_error(tmp_path): with pytest.raises(Exception, match="use_index"): table.search(query).bypass_vector_index().to_arrow() - # use_lsm(False) is the escape hatch: the base-only standard path honors them. - base = table.search(query).distance_range(0.0, 100.0).use_lsm(False).to_arrow() + # use_memwal(False) is the escape hatch: the base-only standard path honors them. + base = table.search(query).distance_range(0.0, 100.0).use_memwal(False).to_arrow() assert 1000 not in set(base["id"].to_pylist()) @@ -544,7 +544,7 @@ def test_lsm_read_fts_unmaintained_index_errors(tmp_path): table.create_index("text", config=FTS()) # No maintained indexes: the active memtable FTS arm cannot serve un-compacted # docs, so the search would silently omit them — reject instead. - table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) + table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([])) with pytest.raises(Exception, match="maintained"): table.search("fox", query_type="fts", fts_columns="text").to_arrow() @@ -553,7 +553,7 @@ def test_lsm_read_time_travel_errors(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.enable_memwal(MemWalWriteSpec.unsharded()) pinned = table.version table.add(_reader([4, 5])) # standard add commits a newer version table.checkout(pinned) # detached head at the historical version @@ -562,8 +562,8 @@ def test_lsm_read_time_travel_errors(tmp_path): # historical version is rejected. with pytest.raises(Exception, match="time-travel"): table.search().to_arrow() - # use_lsm(False) reads the base table at the pinned version. - base = table.search().use_lsm(False).to_arrow() + # use_memwal(False) reads the base table at the pinned version. + base = table.search().use_memwal(False).to_arrow() assert sorted(base["id"].to_pylist()) == [1, 2, 3] @@ -571,14 +571,14 @@ def test_lsm_read_take_row_ids_errors(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.enable_memwal(MemWalWriteSpec.unsharded()) _lsm_upsert(table, [4]) # take-by-row-id auto-routes through the LSM scanner, which has no stable _rowid, # so it hard-errors instead of failing with an opaque column-not-found error. with pytest.raises(Exception, match="row id"): table.take_row_ids([0, 1]).to_arrow() - # use_lsm(False) is the escape hatch: it reads the base table. - base = table.take_row_ids([0, 1]).use_lsm(False).to_arrow() + # use_memwal(False) is the escape hatch: it reads the base table. + base = table.take_row_ids([0, 1]).use_memwal(False).to_arrow() assert base.num_rows == 2 @@ -588,8 +588,8 @@ def test_lsm_read_fts_postfilter_errors(tmp_path): table.set_unenforced_primary_key("id") table.create_index("text", config=FTS()) index_name = table.list_indices()[0].name - table.set_lsm_write_spec( - LsmWriteSpec.unsharded().with_maintained_indexes([index_name]) + table.enable_memwal( + MemWalWriteSpec.unsharded().with_maintained_indexes([index_name]) ) # The LSM scanner always prefilters; postfilter on FTS changes result semantics, # so it hard-errors (previously only the vector arm rejected it). @@ -608,9 +608,7 @@ def test_lsm_read_fts_multiple_same_type_indexes_errors(tmp_path): table.set_unenforced_primary_key("id") table.create_index("text", config=FTS(), name="fts_a") table.create_index("text", config=FTS(), name="fts_b", replace=False) - table.set_lsm_write_spec( - LsmWriteSpec.unsharded().with_maintained_indexes(["fts_a"]) - ) + table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes(["fts_a"])) # Two FTS indexes on the column: the base planner's chosen index is ambiguous, so # the scanner can't pick a catch-up watermark and rejects rather than risk # dropping rows the actually-used index has not caught up to. @@ -631,7 +629,7 @@ def test_lsm_read_vector_unmaintained_index_errors(tmp_path): ) # Spec with NO maintained indexes: the base vector index's catch-up is untracked, # so the scanner rejects rather than risk dropping compacted-but-unindexed rows. - table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) + table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([])) with pytest.raises(Exception, match="maintained"): table.search([1.0] * VECTOR_DIM).to_arrow() @@ -644,7 +642,7 @@ def test_lsm_read_fts_optimized_index_not_rejected(tmp_path): table.add(_fts_reader([(i, "lazy fox") for i in range(6, 11)])) table.optimize() # may split the FTS index into multiple physical segments name = table.list_indices()[0].name - table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([name])) + table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([name])) # Multiple physical segments of one logical index must not be miscounted as # multiple indexes and rejected. ids = set( diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index c22aa78cd..6ee41aa0c 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1845,10 +1845,10 @@ def lsm_test_table(lsm_handler): """A remote table whose LSM routes are served by ``lsm_handler``. ``lsm_handler(request, route)`` is called for ``/v1/table/test//`` - where route is one of flush_lsm, compact_lsm, get_lsm_stats, and is + where route is one of flush_memwal, compact_memwal, get_memwal_stats, and is responsible for writing the response. """ - routes = ("flush_lsm", "compact_lsm", "get_lsm_stats") + routes = ("flush_memwal", "compact_memwal", "get_memwal_stats") def handler(request): match = re.fullmatch(r"/v1/table/test/(\w+)/", request.path) @@ -1880,9 +1880,9 @@ def send_json(request, payload, status=200): request.wfile.write(json.dumps(payload).encode()) -def test_get_lsm_stats_sync(): +def test_get_memwal_stats_sync(): """The sync wrapper round-trips the server payload into a dict.""" - bucket = { + table_shard = { "shard_id": "b0", "status": "Active", "writer_epoch": 3, @@ -1890,7 +1890,7 @@ def test_get_lsm_stats_sync(): "current_generation": 6, "replay_after_wal_entry_position": 40, "wal_entry_position_last_seen": 42, - "generations": [{"generation": 5, "bytes": 1024, "rows": 7}], + "sstables": [{"generation": 5, "bytes": 1024, "rows": 7}], "compacting": False, "memtables": [ { @@ -1905,29 +1905,29 @@ def test_get_lsm_stats_sync(): seen_bodies = [] def lsm_handler(request, route): - assert route == "get_lsm_stats" + assert route == "get_memwal_stats" seen_bodies.append(read_json_body(request)) - send_json(request, {"lsm_stats": {"buckets": [bucket]}}) + send_json(request, {"memwal_stats": {"table_shards": [table_shard]}}) with lsm_test_table(lsm_handler) as table: - assert table.get_lsm_stats() == {"buckets": [bucket]} + assert table.get_memwal_stats() == {"table_shards": [table_shard]} # Off by default, and forwarded when asked for. - assert seen_bodies == [{"include_generation_rows": False}] - table.get_lsm_stats(include_generation_rows=True) - assert seen_bodies[-1] == {"include_generation_rows": True} + assert seen_bodies == [{"include_sstable_rows": False}] + table.get_memwal_stats(include_sstable_rows=True) + assert seen_bodies[-1] == {"include_sstable_rows": True} -def test_get_lsm_stats_sync_returns_none_when_lsm_disabled(): - """A null envelope means the LSM write path is not enabled, not an error.""" +def test_get_memwal_stats_sync_returns_none_when_memwal_disabled(): + """A null envelope means the MemWAL write path is not enabled, not an error.""" def lsm_handler(request, route): - send_json(request, {"lsm_stats": None}) + send_json(request, {"memwal_stats": None}) with lsm_test_table(lsm_handler) as table: - assert table.get_lsm_stats() is None + assert table.get_memwal_stats() is None -def test_flush_and_compact_lsm_sync(): +def test_flush_and_compact_memwal_sync(): """Both are one-shot POSTs answered 202 with no body.""" called = [] @@ -1937,13 +1937,13 @@ def test_flush_and_compact_lsm_sync(): request.end_headers() with lsm_test_table(lsm_handler) as table: - assert table.flush_lsm() is None - assert table.compact_lsm() is None - assert called == ["flush_lsm", "compact_lsm"] + assert table.flush_memwal() is None + assert table.compact_memwal() is None + assert called == ["flush_memwal", "compact_memwal"] -def test_checkpoint_lsm_sync(): - """Seal, read the watermark, and return once L0 holds nothing. +def test_checkpoint_memwal_sync(): + """Freeze, read the watermark, and return once no SSTables remain. The convergence loop itself is covered in Rust; this pins the sync binding to the endpoints it drives. @@ -1952,17 +1952,17 @@ def test_checkpoint_lsm_sync(): def lsm_handler(request, route): called.append(route) - if route == "get_lsm_stats": - # An empty L0 yields no target watermark, so the loop is done - # after the seal without ever polling compaction. - send_json(request, {"lsm_stats": {"buckets": []}}) + if route == "get_memwal_stats": + # An empty SSTable tier yields no target watermark, so the loop is done + # after the freeze without ever polling compaction. + send_json(request, {"memwal_stats": {"table_shards": []}}) else: request.send_response(202) request.end_headers() with lsm_test_table(lsm_handler) as table: - assert table.checkpoint_lsm() is None - assert called == ["flush_lsm", "get_lsm_stats"] + assert table.checkpoint_memwal() is None + assert called == ["flush_memwal", "get_memwal_stats"] @contextlib.contextmanager diff --git a/python/python/tests/test_wal_hybrid.py b/python/python/tests/test_wal_hybrid.py index 9c7efe151..675fc927a 100644 --- a/python/python/tests/test_wal_hybrid.py +++ b/python/python/tests/test_wal_hybrid.py @@ -81,15 +81,15 @@ def test_base_table_hybrid_still_joins_on_row_ids(): assert builder._vector_query._columns is None -def test_wal_hybrid_with_use_lsm_false_joins_on_row_ids(): - builder = _wal_hybrid().use_lsm(False) +def test_wal_hybrid_with_use_memwal_false_joins_on_row_ids(): + builder = _wal_hybrid().use_memwal(False) builder._create_query_builders() assert builder._pk_fusion is None def test_wal_hybrid_refuses_an_explicit_with_row_id(): builder = _wal_hybrid().with_row_id(True) - with pytest.raises(NotImplementedError, match="use_lsm"): + with pytest.raises(NotImplementedError, match="use_memwal"): builder._create_query_builders() @@ -141,7 +141,7 @@ def test_wal_hybrid_refuses_row_id_even_with_return_score_all(row_id_first: bool else: builder = builder.rerank(reranker).with_row_id(True) - with pytest.raises(NotImplementedError, match="use_lsm"): + with pytest.raises(NotImplementedError, match="use_memwal"): builder._create_query_builders() @@ -199,7 +199,7 @@ def test_fallback_explains_the_refusal_to_a_caller_who_asked_for_row_ids(): def run(): raise _REFUSAL - with pytest.raises(NotImplementedError, match="use_lsm") as info: + with pytest.raises(NotImplementedError, match="use_memwal") as info: with_pk_fallback( run, mock.Mock(), fused=lambda: False, caller_requested_row_id=True ) diff --git a/python/python/tests/utils.py b/python/python/tests/utils.py index 4882f0c28..b3548b678 100644 --- a/python/python/tests/utils.py +++ b/python/python/tests/utils.py @@ -73,10 +73,10 @@ class MockPermutationServer: }, }, ) - if path == f"/v1/table/{self.name}/get_lsm_write_spec/": + if path == f"/v1/table/{self.name}/get_memwal_spec/": self._read_body(request) - # Null spec: this table has no LSM write path. - return self._json(request, {"lsm_write_spec": None}) + # Null spec: this table has no MemWAL write path. + return self._json(request, {"memwal_spec": None}) if path == f"/v1/table/{self.name}/count_rows/": self._read_body(request) return self._json(request, self.num_rows) diff --git a/python/src/lib.rs b/python/src/lib.rs index 066e85927..8dd98d019 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -16,7 +16,7 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery}; use session::Session; use table::{ AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken, - FunctionErrorFragment, FunctionErrorRecord, FunctionErrors, LsmWriteSpec, MergeResult, + FunctionErrorFragment, FunctionErrorRecord, FunctionErrors, MemWalWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult, Table, UpdateFieldMetadataResult, UpdateResult, }; @@ -104,7 +104,7 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/src/query.rs b/python/src/query.rs index e5547881a..4f23fa1b4 100644 --- a/python/src/query.rs +++ b/python/src/query.rs @@ -333,7 +333,7 @@ pub struct PyQueryRequest { pub select_source_columns: Option>, pub fast_search: Option, pub with_row_id: Option, - pub use_lsm: Option, + pub use_memwal: Option, pub column: Option, pub query_vector: Option, pub nprobes: Option, @@ -366,7 +366,7 @@ impl From for PyQueryRequest { select: PySelect(query_request.select), fast_search: Some(query_request.fast_search), with_row_id: Some(query_request.with_row_id), - use_lsm: query_request.use_lsm, + use_memwal: query_request.use_memwal, column: None, query_vector: None, nprobes: None, @@ -395,7 +395,7 @@ impl From for PyQueryRequest { select: PySelect(vector_query.base.select), fast_search: Some(vector_query.base.fast_search), with_row_id: Some(vector_query.base.with_row_id), - use_lsm: vector_query.base.use_lsm, + use_memwal: vector_query.base.use_memwal, column: vector_query.column, query_vector: Some(PyQueryVectors(vector_query.query_vector)), nprobes: vector_query.nprobes, @@ -540,8 +540,8 @@ impl Query { self.inner = self.inner.clone().fast_search(); } - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } pub fn with_row_id(&mut self) { @@ -706,8 +706,8 @@ impl TakeQuery { self.inner = self.inner.clone().with_row_id(); } - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } #[pyo3(signature = ())] @@ -819,8 +819,8 @@ impl FTSQuery { self.inner = self.inner.clone().fast_search(); } - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } pub fn with_row_id(&mut self) { @@ -970,8 +970,8 @@ impl VectorQuery { self.inner = self.inner.clone().fast_search(); } - pub fn use_lsm(&mut self, enable: bool) { - self.inner = self.inner.clone().use_lsm(enable); + pub fn use_memwal(&mut self, enable: bool) { + self.inner = self.inner.clone().use_memwal(enable); } pub fn with_row_id(&mut self) { @@ -1178,9 +1178,9 @@ impl HybridQuery { self.inner_fts.postfilter(); } - pub fn use_lsm(&mut self, enable: bool) { - self.inner_vec.use_lsm(enable); - self.inner_fts.use_lsm(enable); + pub fn use_memwal(&mut self, enable: bool) { + self.inner_vec.use_memwal(enable); + self.inner_fts.use_memwal(enable); } pub fn add_query_vector(&mut self, vector: Bound<'_, PyAny>) -> PyResult<()> { diff --git a/python/src/table.rs b/python/src/table.rs index 23b48e3fd..c4053cc53 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -39,16 +39,16 @@ use pyo3::{ mod scannable; -/// Convert `LsmStats` to a Python dict, preserving the per-bucket list. +/// Convert `MemWalStats` to a Python dict, preserving the per-table-shard list. /// /// Deliberately not flattened to a table-level summary: a table is N -/// buckets on one node, and the per-bucket detail is the reason the -/// endpoint exists — flattening hides the single hot bucket someone opened +/// table shards on one node, and the per-shard detail is the reason the +/// endpoint exists — flattening hides the single hot table shard someone opened /// it to find. -fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult> { +fn memwal_stats_to_py(py: Python<'_>, stats: &lancedb::table::MemWalStats) -> PyResult> { let out = PyDict::new(py); - let buckets = PyList::empty(py); - for b in &stats.buckets { + let table_shards = PyList::empty(py); + for b in &stats.table_shards { let e = PyDict::new(py); e.set_item("shard_id", &b.shard_id)?; e.set_item("status", &b.status)?; @@ -64,15 +64,15 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult b.wal_entry_position_last_seen, )?; - let generations = PyList::empty(py); - for g in &b.generations { + let sstables = PyList::empty(py); + for g in &b.sstables { let ge = PyDict::new(py); ge.set_item("generation", g.generation)?; ge.set_item("bytes", g.bytes)?; ge.set_item("rows", g.rows)?; - generations.append(ge)?; + sstables.append(ge)?; } - e.set_item("generations", generations)?; + e.set_item("sstables", sstables)?; e.set_item("compacting", b.compacting)?; e.set_item( @@ -94,9 +94,9 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult }) .transpose()?, )?; - buckets.append(e)?; + table_shards.append(e)?; } - out.set_item("buckets", buckets)?; + out.set_item("table_shards", table_shards)?; Ok(out.unbind()) } @@ -270,17 +270,17 @@ fn fmt_maintained(maintained: &Option>) -> String { /// table has, including ones created later. #[pyclass(module = "lancedb._lancedb", from_py_object)] #[derive(Clone, Debug)] -pub struct LsmWriteSpec { - inner: lancedb::table::LsmWriteSpec, +pub struct MemWalWriteSpec { + inner: lancedb::table::MemWalWriteSpec, } #[pymethods] -impl LsmWriteSpec { +impl MemWalWriteSpec { /// Hash-bucket sharding by the unenforced primary key column. #[staticmethod] pub fn bucket(column: String, num_buckets: u32) -> Self { Self { - inner: lancedb::table::LsmWriteSpec::bucket(column, num_buckets), + inner: lancedb::table::MemWalWriteSpec::bucket(column, num_buckets), } } @@ -294,7 +294,7 @@ impl LsmWriteSpec { #[staticmethod] pub fn identity(column: String) -> Self { Self { - inner: lancedb::table::LsmWriteSpec::identity(column), + inner: lancedb::table::MemWalWriteSpec::identity(column), } } @@ -303,7 +303,7 @@ impl LsmWriteSpec { #[staticmethod] pub fn unsharded() -> Self { Self { - inner: lancedb::table::LsmWriteSpec::unsharded(), + inner: lancedb::table::MemWalWriteSpec::unsharded(), } } @@ -327,33 +327,33 @@ impl LsmWriteSpec { pub fn __repr__(&self) -> String { match &self.inner { - lancedb::table::LsmWriteSpec::Bucket { + lancedb::table::MemWalWriteSpec::Bucket { column, num_buckets, maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})", + "MemWalWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})", column, num_buckets, fmt_maintained(maintained_indexes), writer_config_defaults, ), - lancedb::table::LsmWriteSpec::Identity { + lancedb::table::MemWalWriteSpec::Identity { column, maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})", + "MemWalWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})", column, fmt_maintained(maintained_indexes), writer_config_defaults, ), - lancedb::table::LsmWriteSpec::Unsharded { + lancedb::table::MemWalWriteSpec::Unsharded { maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})", + "MemWalWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})", fmt_maintained(maintained_indexes), writer_config_defaults, ), @@ -365,9 +365,9 @@ impl LsmWriteSpec { #[getter] pub fn spec_type(&self) -> &'static str { match &self.inner { - lancedb::table::LsmWriteSpec::Bucket { .. } => "bucket", - lancedb::table::LsmWriteSpec::Identity { .. } => "identity", - lancedb::table::LsmWriteSpec::Unsharded { .. } => "unsharded", + lancedb::table::MemWalWriteSpec::Bucket { .. } => "bucket", + lancedb::table::MemWalWriteSpec::Identity { .. } => "identity", + lancedb::table::MemWalWriteSpec::Unsharded { .. } => "unsharded", } } @@ -375,9 +375,9 @@ impl LsmWriteSpec { #[getter] pub fn column(&self) -> Option { match &self.inner { - lancedb::table::LsmWriteSpec::Bucket { column, .. } - | lancedb::table::LsmWriteSpec::Identity { column, .. } => Some(column.clone()), - lancedb::table::LsmWriteSpec::Unsharded { .. } => None, + lancedb::table::MemWalWriteSpec::Bucket { column, .. } + | lancedb::table::MemWalWriteSpec::Identity { column, .. } => Some(column.clone()), + lancedb::table::MemWalWriteSpec::Unsharded { .. } => None, } } @@ -385,7 +385,7 @@ impl LsmWriteSpec { #[getter] pub fn num_buckets(&self) -> Option { match &self.inner { - lancedb::table::LsmWriteSpec::Bucket { num_buckets, .. } => Some(*num_buckets), + lancedb::table::MemWalWriteSpec::Bucket { num_buckets, .. } => Some(*num_buckets), _ => None, } } @@ -403,14 +403,14 @@ impl LsmWriteSpec { } } -impl From for lancedb::table::LsmWriteSpec { - fn from(spec: LsmWriteSpec) -> Self { +impl From for lancedb::table::MemWalWriteSpec { + fn from(spec: MemWalWriteSpec) -> Self { spec.inner } } -impl From for LsmWriteSpec { - fn from(inner: lancedb::table::LsmWriteSpec) -> Self { +impl From for MemWalWriteSpec { + fn from(inner: lancedb::table::MemWalWriteSpec) -> Self { Self { inner } } } @@ -1636,8 +1636,8 @@ impl Table { if let Some(use_index) = parameters.use_index { builder.use_index(use_index); } - if let Some(use_lsm) = parameters.use_lsm { - builder.use_lsm(use_lsm); + if let Some(use_memwal) = parameters.use_memwal { + builder.use_memwal(use_memwal); } if let Some(validate_single_shard) = parameters.validate_single_shard { builder.validate_single_shard(validate_single_shard); @@ -1662,29 +1662,29 @@ impl Table { }) } - pub fn set_lsm_write_spec<'a>( + pub fn enable_memwal<'a>( self_: PyRef<'a, Self>, - spec: LsmWriteSpec, + spec: MemWalWriteSpec, ) -> PyResult> { let inner = self_.inner_ref()?.clone(); - let native_spec = lancedb::table::LsmWriteSpec::from(spec); + let native_spec = lancedb::table::MemWalWriteSpec::from(spec); future_into_py(self_.py(), async move { - inner.set_lsm_write_spec(native_spec).await.infer_error() + inner.enable_memwal(native_spec).await.infer_error() }) } - pub fn unset_lsm_write_spec(self_: PyRef<'_, Self>) -> PyResult> { + pub fn disable_memwal(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - inner.unset_lsm_write_spec().await.infer_error() + inner.disable_memwal().await.infer_error() }) } - pub fn get_lsm_write_spec(self_: PyRef<'_, Self>) -> PyResult> { + pub fn get_memwal_spec(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - let spec = inner.get_lsm_write_spec().await.infer_error()?; - Ok(spec.map(LsmWriteSpec::from)) + let spec = inner.get_memwal_spec().await.infer_error()?; + Ok(spec.map(MemWalWriteSpec::from)) }) } @@ -1703,55 +1703,54 @@ impl Table { Ok(()) } - /// Converge the table's LSM write path into its base table. + /// Converge the table's MemWAL write path into its base table. /// /// Best-effort: with writes flowing, new rows may land after the last /// pass. Errors if the table stops making progress. - pub fn checkpoint_lsm(self_: PyRef<'_, Self>) -> PyResult> { + pub fn checkpoint_memwal(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - inner.checkpoint_lsm().await.infer_error() + inner.checkpoint_memwal().await.infer_error() }) } - /// Seal every bucket's active memtable into L0. - pub fn flush_lsm(self_: PyRef<'_, Self>) -> PyResult> { - let inner = self_.inner_ref()?.clone(); - future_into_py( - self_.py(), - async move { inner.flush_lsm().await.infer_error() }, - ) - } - - /// Trigger a background L0 → base pass per bucket. Returns once the - /// passes are dispatched, not once they finish — watch `get_lsm_stats`. - pub fn compact_lsm(self_: PyRef<'_, Self>) -> PyResult> { + /// Freeze every table shard's active memtable into an SSTable. + pub fn flush_memwal(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - inner.compact_lsm().await.infer_error() + inner.flush_memwal().await.infer_error() }) } - /// Live LSM state, or `None` when the LSM write path is not enabled. - #[pyo3(signature = (include_generation_rows=false))] - pub fn get_lsm_stats( + /// Trigger a background SSTable compaction pass per table shard. Returns once the + /// passes are dispatched, not once they finish — watch `get_memwal_stats`. + pub fn compact_memwal(self_: PyRef<'_, Self>) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + future_into_py(self_.py(), async move { + inner.compact_memwal().await.infer_error() + }) + } + + /// Live MemWAL state, or `None` when the MemWAL write path is not enabled. + #[pyo3(signature = (include_sstable_rows=false))] + pub fn get_memwal_stats( self_: PyRef<'_, Self>, - include_generation_rows: bool, + include_sstable_rows: bool, ) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { let stats = inner - .get_lsm_stats(include_generation_rows) + .get_memwal_stats(include_sstable_rows) .await .infer_error()?; - Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose()) + Python::attach(|py| stats.map(|s| memwal_stats_to_py(py, &s)).transpose()) }) } - pub fn close_lsm_writers(self_: PyRef<'_, Self>) -> PyResult> { + pub fn close_memwal_writers(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - inner.close_lsm_writers().await.infer_error() + inner.close_memwal_writers().await.infer_error() }) } @@ -2083,7 +2082,7 @@ pub struct MergeInsertParams { when_not_matched_by_source_condition_expr: Option, timeout: Option, use_index: Option, - use_lsm: Option, + use_memwal: Option, validate_single_shard: Option, } diff --git a/rust/lancedb/src/dataloader/permutation/builder.rs b/rust/lancedb/src/dataloader/permutation/builder.rs index ba0d0e180..e08f3eddc 100644 --- a/rust/lancedb/src/dataloader/permutation/builder.rs +++ b/rust/lancedb/src/dataloader/permutation/builder.rs @@ -250,17 +250,17 @@ impl PermutationBuilder { } // Unflushed rows have no row id, so a permutation cannot address them. - match self.base_table.base_table().get_lsm_write_spec().await { + match self.base_table.base_table().get_memwal_spec().await { Ok(Some(_)) => { return Err(Error::NotSupported { - message: "the data loader does not support tables with an LSM write \ + message: "the data loader does not support tables with a MemWAL write \ spec: rows that have not been flushed to the base table \ have no row id, so a permutation cannot reference them" .to_string(), }); } Ok(None) => {} - // No LSM write path means no spec. + // No MemWAL write path means no spec. Err(Error::NotSupported { .. }) => {} Err(err) => return Err(err), } @@ -489,9 +489,9 @@ mod tests { ) .unwrap() } - "/v1/table/remote_base/get_lsm_write_spec/" => http::Response::builder() + "/v1/table/remote_base/get_memwal_spec/" => http::Response::builder() .status(200) - .body(br#"{"lsm_write_spec":null}"#.to_vec()) + .body(br#"{"memwal_spec":null}"#.to_vec()) .unwrap(), "/v1/table/remote_base/count_rows/" => { let body = body.unwrap(); @@ -737,8 +737,8 @@ mod tests { /// permutation cannot reference them. Reading the base table alone would drop /// them from training without saying so, so the table is refused instead. #[tokio::test] - async fn test_permutation_rejects_lsm_write_spec() { - use crate::table::LsmWriteSpec; + async fn test_permutation_rejects_memwal_spec() { + use crate::table::MemWalWriteSpec; use arrow_array::{Int32Array, RecordBatchIterator}; use arrow_schema::{DataType, Field, Schema}; @@ -766,13 +766,13 @@ mod tests { table.set_unenforced_primary_key(["idx"]).await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); let err = PermutationBuilder::new(table).build().await.unwrap_err(); assert!( - err.to_string().contains("LSM write spec"), + err.to_string().contains("MemWAL write spec"), "expected the pre-check to refuse the table, got: {err}" ); } diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index 7750f66b5..68b05c531 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -167,7 +167,7 @@ //! # } //! ``` -// The MemWAL LSM read path (`table::query::lsm`) deepens the `create_plan` future's +// The MemWAL read path (`table::query::lsm`) deepens the `create_plan` future's // type graph enough to overflow the default trait-recursion limit while evaluating // auto-traits (`Send`) through the Linux io_uring build's moka cache. Raise it. #![recursion_limit = "256"] diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index f9634837a..b4ff22397 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -288,13 +288,15 @@ fn refresh_conflict_filter(dataset: &Dataset) -> Result { Ok(filter.build()) } -/// Reject MemWAL/LSM state because an ordinary dataset scan cannot see its +/// Reject MemWAL/MemWAL state because an ordinary dataset scan cannot see its /// un-compacted tiers. pub(crate) async fn ensure_no_mem_wal(dataset: &Dataset, role: &str, name: &str) -> Result<()> { let retained = !dataset.list_mem_wal_latest_shard_ids().await?.is_empty(); if retained || dataset.mem_wal_index_details().await?.is_some() { return Err(Error::NotSupported { - message: format!("{role} '{name}' has an LSM write spec or retained un-compacted rows"), + message: format!( + "{role} '{name}' has a MemWAL write spec or retained un-compacted rows" + ), }); } Ok(()) diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index d66932847..5f072cad6 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -547,22 +547,22 @@ pub trait QueryBase { /// Control MemWAL read routing for this query. /// /// By default (unset), when the table carries a MemWAL write spec (see - /// [`crate::Table::set_lsm_write_spec`]), reads are routed through the LSM + /// [`crate::Table::enable_memwal`]), reads are routed through the LSM /// scanner so they also return data written via the `merge_insert` LSM path /// that has not yet been compacted into the base table (active/frozen - /// memtables and flushed generations); a table without a spec reads the base + /// memtables and SSTables); a table without a spec reads the base /// table. /// - /// - `use_lsm(true)` forces LSM routing and errors if the table has no + /// - `use_memwal(true)` forces MemWAL routing and errors if the table has no /// MemWAL write spec. - /// - `use_lsm(false)` bypasses the MemWAL and reads the base table only, + /// - `use_memwal(false)` bypasses the MemWAL and reads the base table only, /// even when a spec is present. /// /// Note: the LSM scanner does not support every query shape (e.g. reranking, /// hybrid search, `order_by`). On a MemWAL table those shapes error unless - /// `use_lsm(false)` is set, because a base-only read would silently + /// `use_memwal(false)` is set, because a base-only read would silently /// exclude un-compacted MemWAL data. - fn use_lsm(self, enable: bool) -> Self; + fn use_memwal(self, enable: bool) -> Self; } pub trait HasQuery { @@ -634,8 +634,8 @@ impl QueryBase for T { self } - fn use_lsm(mut self, enable: bool) -> Self { - self.mut_query().use_lsm = Some(enable); + fn use_memwal(mut self, enable: bool) -> Self { + self.mut_query().use_memwal = Some(enable); self } } @@ -900,17 +900,17 @@ pub struct QueryRequest { /// Controls MemWAL read routing. When unset (the default), a query against a /// table that carries a MemWAL write spec (see - /// [`crate::Table::set_lsm_write_spec`]) is routed through the LSM scanner so + /// [`crate::Table::enable_memwal`]) is routed through the LSM scanner so /// it also sees data written via the `merge_insert` LSM path that has not yet /// been compacted into the base table — the active and frozen in-memory - /// memtables and the flushed (L0) generations, deduplicated by primary key + /// memtables and the SSTables, deduplicated by primary key /// against the base table (newest generation wins); a table without a spec /// reads the base table. /// - /// - `Some(true)` forces LSM routing and errors if the table has no MemWAL + /// - `Some(true)` forces MemWAL routing and errors if the table has no MemWAL /// write spec. /// - `Some(false)` reads only the base table, bypassing the MemWAL. - pub use_lsm: Option, + pub use_memwal: Option, } impl Default for QueryRequest { @@ -930,7 +930,7 @@ impl Default for QueryRequest { norm: None, disable_scoring_autoprojection: false, order_by: None, - use_lsm: None, + use_memwal: None, } } } diff --git a/rust/lancedb/src/query/wal_fusion.rs b/rust/lancedb/src/query/wal_fusion.rs index b438f6150..e8b6b3269 100644 --- a/rust/lancedb/src/query/wal_fusion.rs +++ b/rust/lancedb/src/query/wal_fusion.rs @@ -54,7 +54,7 @@ fn wal_row_id_unsupported() -> Error { Error::NotSupported { message: "hybrid search on a MemWAL table cannot return _rowid: the fresh tier has no \ stable row id, and the ids the fusion joins on are synthesized from the \ - primary key. Set use_lsm(false) to read the base table only (results will \ + primary key. Set use_memwal(false) to read the base table only (results will \ exclude un-compacted MemWAL data)" .to_string(), } @@ -80,7 +80,7 @@ where { // A previous query on this table was already refused, so skip straight to // the key. - if query.request.base.use_lsm != Some(false) && query.parent.hybrid_pk_fusion_learned() { + if query.request.base.use_memwal != Some(false) && query.parent.hybrid_pk_fusion_learned() { return run(Some(PkFusion::resolve(query).await?)).await; } match run(None).await { diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index d9da2c1a1..57843ca55 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -30,12 +30,12 @@ use crate::table::BranchDiff; use crate::table::CherryPickResult; use crate::table::DeleteResult; use crate::table::DropColumnsResult; -use crate::table::LsmStats; -use crate::table::LsmWriteSpec; +use crate::table::MemWalStats; +use crate::table::MemWalWriteSpec; use crate::table::MergeResult; use crate::table::Tags; use crate::table::UpdateResult; -use crate::table::lsm_stats::GetLsmStatsResponse; +use crate::table::memwal_stats::GetMemWalStatsResponse; use crate::table::merge::MergeFilter; use crate::table::primary_key; use crate::table::query::create_multi_vector_plan; @@ -1020,10 +1020,10 @@ impl RemoteTable { ) -> Result<()> { params.check_filter()?; body["prefilter"] = params.prefilter.into(); - // Only forward use_lsm when explicitly set; a server that predates it + // Only forward use_memwal when explicitly set; a server that predates it // ignores the field and routes as it would by default. - if let Some(use_lsm) = params.use_lsm { - body["use_lsm"] = serde_json::Value::Bool(use_lsm); + if let Some(use_memwal) = params.use_memwal { + body["use_memwal"] = serde_json::Value::Bool(use_memwal); } if let Some(offset) = params.offset { body["offset"] = serde_json::Value::Number(serde_json::Number::from(offset)); @@ -3227,42 +3227,43 @@ impl BaseTable for RemoteTable { Ok(()) } - async fn flush_lsm(&self) -> Result<()> { + async fn flush_memwal(&self) -> Result<()> { let request = self .client - .post(&format!("/v1/table/{}/flush_lsm/", self.identifier)); + .post(&format!("/v1/table/{}/flush_memwal/", self.identifier)); self.send_lsm_route(request).await?; Ok(()) } - async fn compact_lsm(&self) -> Result<()> { + async fn compact_memwal(&self) -> Result<()> { let request = self .client - .post(&format!("/v1/table/{}/compact_lsm/", self.identifier)); + .post(&format!("/v1/table/{}/compact_memwal/", self.identifier)); self.send_lsm_route(request).await?; Ok(()) } - async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result> { - // Read-semantics POST, like `get_lsm_write_spec`. + async fn get_memwal_stats(&self, include_sstable_rows: bool) -> Result> { + // Read-semantics POST, like `get_memwal_spec`. let request = self .client - .post(&format!("/v1/table/{}/get_lsm_stats/", self.identifier)) + .post(&format!("/v1/table/{}/get_memwal_stats/", self.identifier)) .json(&serde_json::json!({ - "include_generation_rows": include_generation_rows, + "include_sstable_rows": include_sstable_rows, })); let (request_id, response) = self.send_lsm_route(request).await?; let body = response.text().await.err_to_http(request_id.clone())?; - let parsed: GetLsmStatsResponse = serde_json::from_str(&body).map_err(|e| Error::Http { - source: format!("Failed to parse get_lsm_stats response: {e}").into(), - request_id, - status_code: None, - })?; - // `null` — and only — when the table has no LSM write path. - Ok(parsed.lsm_stats) + let parsed: GetMemWalStatsResponse = + serde_json::from_str(&body).map_err(|e| Error::Http { + source: format!("Failed to parse get_memwal_stats response: {e}").into(), + request_id, + status_code: None, + })?; + // `null` — and only — when the table has no MemWAL write path. + Ok(parsed.memwal_stats) } - async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> Result<()> { + async fn enable_memwal(&self, spec: MemWalWriteSpec) -> Result<()> { self.check_mutable().await?; // Map the spec onto the server's request DTO. `sharding` is internally @@ -3270,7 +3271,7 @@ impl BaseTable for RemoteTable { // `maintained_indexes` asks the server to resolve every maintainable // index at HEAD; a list is verbatim, an empty one meaning none. let sharding = match &spec { - LsmWriteSpec::Bucket { + MemWalWriteSpec::Bucket { column, num_buckets, .. @@ -3279,11 +3280,11 @@ impl BaseTable for RemoteTable { "column": column, "num_buckets": num_buckets, }), - LsmWriteSpec::Identity { column, .. } => serde_json::json!({ + MemWalWriteSpec::Identity { column, .. } => serde_json::json!({ "mode": "identity", "column": column, }), - LsmWriteSpec::Unsharded { .. } => serde_json::json!({ "mode": "unsharded" }), + MemWalWriteSpec::Unsharded { .. } => serde_json::json!({ "mode": "unsharded" }), }; let body = serde_json::json!({ "sharding": sharding, @@ -3293,45 +3294,40 @@ impl BaseTable for RemoteTable { let request = self .client - .post(&format!( - "/v1/table/{}/set_lsm_write_spec/", - self.identifier - )) + .post(&format!("/v1/table/{}/enable_memwal/", self.identifier)) .json(&body); let (request_id, response) = self.send(request, true).await?; self.check_table_response(&request_id, response).await?; Ok(()) } - async fn unset_lsm_write_spec(&self) -> Result<()> { + async fn disable_memwal(&self) -> Result<()> { self.check_mutable().await?; - let request = self.client.post(&format!( - "/v1/table/{}/unset_lsm_write_spec/", - self.identifier - )); + let request = self + .client + .post(&format!("/v1/table/{}/disable_memwal/", self.identifier)); let (request_id, response) = self.send(request, true).await?; self.check_table_response(&request_id, response).await?; self.wal_pk_fusion.forget(); // WAL-PK-FUSION: delete. Ok(()) } - async fn get_lsm_write_spec(&self) -> Result> { + async fn get_memwal_spec(&self) -> Result> { // Read counterpart to set/unset, resolved server-side against HEAD. The // server reads the spec from the `__lance_mem_wal` system index (shard // column mapped from its Lance field id against the current schema) and // re-encodes it into the same sophon-owned shape the set endpoint - // accepts — no lance/lancedb types cross the wire. `lsm_write_spec` is - // null when the LSM write path is not enabled for the table. - let request = self.client.post(&format!( - "/v1/table/{}/get_lsm_write_spec/", - self.identifier - )); + // accepts — no lance/lancedb types cross the wire. `memwal_spec` is + // null when the MemWAL write path is not enabled for the table. + let request = self + .client + .post(&format!("/v1/table/{}/get_memwal_spec/", self.identifier)); let (request_id, response) = self.send(request, true).await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; // Mirror of sophon's `Sharding` (internally tagged on `mode`) and - // `LsmWriteSpecBody` / `GetLsmWriteSpecResponse`. + // `MemWalWriteSpecBody` / `GetMemWalSpecResponse`. #[derive(Deserialize)] #[serde(tag = "mode", rename_all = "snake_case")] enum Sharding { @@ -3340,7 +3336,7 @@ impl BaseTable for RemoteTable { Identity { column: String }, } #[derive(Deserialize)] - struct LsmWriteSpecBody { + struct MemWalWriteSpecBody { sharding: Sharding, /// `null` selects every index the table has; `[]` selects none. #[serde(default)] @@ -3349,19 +3345,19 @@ impl BaseTable for RemoteTable { writer_config_defaults: std::collections::HashMap, } #[derive(Deserialize)] - struct GetLsmWriteSpecResponse { - lsm_write_spec: Option, + struct GetMemWalSpecResponse { + memwal_spec: Option, } - let parsed: GetLsmWriteSpecResponse = + let parsed: GetMemWalSpecResponse = serde_json::from_str(&body).map_err(|e| Error::Http { - source: format!("Failed to parse get_lsm_write_spec response: {}", e).into(), + source: format!("Failed to parse get_memwal_spec response: {}", e).into(), request_id, status_code: None, })?; - let Some(body) = parsed.lsm_write_spec else { - // The LSM write path is not enabled for this table. + let Some(body) = parsed.memwal_spec else { + // The MemWAL write path is not enabled for this table. return Ok(None); }; @@ -3369,9 +3365,9 @@ impl BaseTable for RemoteTable { Sharding::Bucket { column, num_buckets, - } => LsmWriteSpec::bucket(column, num_buckets), - Sharding::Identity { column } => LsmWriteSpec::identity(column), - Sharding::Unsharded => LsmWriteSpec::unsharded(), + } => MemWalWriteSpec::bucket(column, num_buckets), + Sharding::Identity { column } => MemWalWriteSpec::identity(column), + Sharding::Unsharded => MemWalWriteSpec::unsharded(), } .with_maintained_indexes(body.maintained_indexes) .with_writer_config_defaults(body.writer_config_defaults); @@ -3974,10 +3970,10 @@ pub struct MergeInsertRequest { // (the default is true) #[serde(skip_serializing_if = "is_true")] use_index: bool, - // Only serialize use_lsm when explicitly set (Some); a server that predates + // Only serialize use_memwal when explicitly set (Some); a server that predates // it ignores the field and routes as it would by default. #[serde(skip_serializing_if = "Option::is_none")] - use_lsm: Option, + use_memwal: Option, } impl MergeInsertRequest { @@ -4035,7 +4031,7 @@ impl TryFrom for MergeInsertRequest { when_not_matched_by_source_delete_filt, // Only serialize use_index when it's false for backwards compatibility use_index: value.use_index, - use_lsm: value.use_lsm, + use_memwal: value.use_memwal, }) } } @@ -10010,13 +10006,10 @@ mod tests { } #[tokio::test] - async fn test_set_lsm_write_spec_unsharded() { + async fn test_enable_memwal_unsharded() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!( - request.url().path(), - "/v1/table/my_table/set_lsm_write_spec/" - ); + assert_eq!(request.url().path(), "/v1/table/my_table/enable_memwal/"); let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!(body["sharding"], serde_json::json!({ "mode": "unsharded" })); @@ -10030,20 +10023,17 @@ mod tests { .body(r#"{"maintained_indexes":["id_idx"]}"#) .unwrap() }); - let spec = crate::table::LsmWriteSpec::unsharded() + let spec = crate::table::MemWalWriteSpec::unsharded() .with_maintained_indexes(vec!["id_idx".to_string()]) .with_writer_config_defaults([("max_memtable_rows", "1000")]); - table.set_lsm_write_spec(spec).await.unwrap(); + table.enable_memwal(spec).await.unwrap(); } #[tokio::test] - async fn test_set_lsm_write_spec_bucket() { + async fn test_enable_memwal_bucket() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!( - request.url().path(), - "/v1/table/my_table/set_lsm_write_spec/" - ); + assert_eq!(request.url().path(), "/v1/table/my_table/enable_memwal/"); let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!( @@ -10055,14 +10045,14 @@ mod tests { http::Response::builder().status(200).body("{}").unwrap() }); table - .set_lsm_write_spec(crate::table::LsmWriteSpec::bucket("id", 16)) + .enable_memwal(crate::table::MemWalWriteSpec::bucket("id", 16)) .await .unwrap(); } /// `[]` (none) must stay distinguishable on the wire from null (all). #[tokio::test] - async fn test_set_lsm_write_spec_no_maintained_indexes() { + async fn test_enable_memwal_no_maintained_indexes() { let table = Table::new_with_handler("my_table", |request| { let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); @@ -10070,21 +10060,18 @@ mod tests { http::Response::builder().status(200).body("{}").unwrap() }); table - .set_lsm_write_spec( - crate::table::LsmWriteSpec::bucket("id", 16).with_maintained_indexes(Vec::new()), + .enable_memwal( + crate::table::MemWalWriteSpec::bucket("id", 16).with_maintained_indexes(Vec::new()), ) .await .unwrap(); } #[tokio::test] - async fn test_set_lsm_write_spec_identity() { + async fn test_enable_memwal_identity() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!( - request.url().path(), - "/v1/table/my_table/set_lsm_write_spec/" - ); + assert_eq!(request.url().path(), "/v1/table/my_table/enable_memwal/"); let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!( @@ -10094,38 +10081,32 @@ mod tests { http::Response::builder().status(200).body("{}").unwrap() }); table - .set_lsm_write_spec(crate::table::LsmWriteSpec::identity("tenant")) + .enable_memwal(crate::table::MemWalWriteSpec::identity("tenant")) .await .unwrap(); } #[tokio::test] - async fn test_unset_lsm_write_spec() { + async fn test_disable_memwal() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!( - request.url().path(), - "/v1/table/my_table/unset_lsm_write_spec/" - ); + assert_eq!(request.url().path(), "/v1/table/my_table/disable_memwal/"); http::Response::builder().status(200).body("{}").unwrap() }); - table.unset_lsm_write_spec().await.unwrap(); + table.disable_memwal().await.unwrap(); } #[tokio::test] - async fn test_get_lsm_write_spec() { + async fn test_get_memwal_spec() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!( - request.url().path(), - "/v1/table/my_table/get_lsm_write_spec/" - ); + assert_eq!(request.url().path(), "/v1/table/my_table/get_memwal_spec/"); // The server resolves the spec and re-encodes it into the same // sophon-owned shape the set endpoint accepts (`Sharding` internally - // tagged on `mode`, wrapped in `lsm_write_spec`). + // tagged on `mode`, wrapped in `memwal_spec`). let response = serde_json::json!({ - "lsm_write_spec": { + "memwal_spec": { "sharding": { "mode": "bucket", "column": "id", "num_buckets": 4 }, "maintained_indexes": ["id_idx"], "writer_config_defaults": { "durable_write": "false" }, @@ -10138,12 +10119,12 @@ mod tests { }); let spec = table - .get_lsm_write_spec() + .get_memwal_spec() .await .unwrap() .expect("a spec should be reported"); match spec { - crate::table::LsmWriteSpec::Bucket { + crate::table::MemWalWriteSpec::Bucket { column, num_buckets, maintained_indexes, @@ -10170,13 +10151,13 @@ mod tests { #[case::no_index(serde_json::json!([]), Some(vec![]))] #[case::named(serde_json::json!(["id_idx"]), Some(vec!["id_idx".to_string()]))] #[tokio::test] - async fn test_get_lsm_write_spec_round_trips_the_selection( + async fn test_get_memwal_spec_round_trips_the_selection( #[case] reported: serde_json::Value, #[case] expected: Option>, ) { let table = Table::new_with_handler("my_table", move |_| { let response = serde_json::json!({ - "lsm_write_spec": { + "memwal_spec": { "sharding": { "mode": "unsharded" }, "maintained_indexes": reported, "writer_config_defaults": {}, @@ -10189,7 +10170,7 @@ mod tests { }); let spec = table - .get_lsm_write_spec() + .get_memwal_spec() .await .unwrap() .expect("a spec should be reported"); @@ -10201,35 +10182,32 @@ mod tests { } #[tokio::test] - async fn test_get_lsm_write_spec_absent() { + async fn test_get_memwal_spec_absent() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!( - request.url().path(), - "/v1/table/my_table/get_lsm_write_spec/" - ); - // Null spec → the LSM write path is not enabled. - let response = serde_json::json!({ "lsm_write_spec": null }); + assert_eq!(request.url().path(), "/v1/table/my_table/get_memwal_spec/"); + // Null spec → the MemWAL write path is not enabled. + let response = serde_json::json!({ "memwal_spec": null }); http::Response::builder() .status(200) .body(response.to_string()) .unwrap() }); - assert!(table.get_lsm_write_spec().await.unwrap().is_none()); + assert!(table.get_memwal_spec().await.unwrap().is_none()); } - /// Build a `get_lsm_stats` body for one bucket holding `generations`. - fn stats_body(generations: &[u64], compacting: bool) -> String { + /// Build a `get_memwal_stats` body for one table shard holding `sstables`. + fn stats_body(sstables: &[u64], compacting: bool) -> String { serde_json::json!({ - "lsm_stats": { - "buckets": [{ + "memwal_stats": { + "table_shards": [{ "shard_id": "b0", "status": "Active", "writer_epoch": 1, "manifest_version": 1, - "current_generation": generations.iter().max().copied().unwrap_or(0) + 1, + "current_generation": sstables.iter().max().copied().unwrap_or(0) + 1, "replay_after_wal_entry_position": 0, "wal_entry_position_last_seen": 0, - "generations": generations.iter() + "sstables": sstables.iter() .map(|g| serde_json::json!({ "generation": g, "bytes": 1 })) .collect::>(), "compacting": compacting, @@ -10240,7 +10218,7 @@ mod tests { .to_string() } - /// `flush_lsm` / `compact_lsm` answer 202 with no body at all. + /// `flush_memwal` / `compact_memwal` answer 202 with no body at all. fn accepted() -> http::Response { http::Response::builder() .status(202) @@ -10252,7 +10230,7 @@ mod tests { http::Response::builder().status(200).body(body).unwrap() } - /// A flush landing in an empty L0 finishes on the opening stats read + /// A flush landing in an empty SSTable tier finishes on the opening stats read /// alone. Asserting zero compacts is the point: "it returned Ok" is also /// true of a loop that ran a pointless pass. #[tokio::test(start_paused = true)] @@ -10261,18 +10239,18 @@ mod tests { let seen = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("compact_lsm") { + if path.contains("compact_memwal") { seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); panic!("an already-converged table must issue no compact calls"); } - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { return accepted(); } - assert_eq!(path, "/v1/table/my_table/get_lsm_stats/"); + assert_eq!(path, "/v1/table/my_table/get_memwal_stats/"); ok_json(stats_body(&[], false)) }); - table.checkpoint_lsm().await.unwrap(); + table.checkpoint_memwal().await.unwrap(); assert_eq!(compacts.load(std::sync::atomic::Ordering::SeqCst), 0); } @@ -10284,10 +10262,10 @@ mod tests { let seen = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { return accepted(); } - if path.contains("compact_lsm") { + if path.contains("compact_memwal") { seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); return accepted(); } @@ -10297,7 +10275,7 @@ mod tests { ok_json(stats_body(&left, false)) }); - table.checkpoint_lsm().await.unwrap(); + table.checkpoint_memwal().await.unwrap(); assert_eq!( compacts.load(std::sync::atomic::Ordering::SeqCst), 3, @@ -10306,7 +10284,7 @@ mod tests { } /// Generations created *during* the checkpoint are not waited on, which - /// is what lets the loop terminate on a table taking writes where "L0 is + /// is what lets the loop terminate on a table taking writes where "the SSTable tier is /// empty" never becomes true. #[tokio::test(start_paused = true)] async fn test_checkpoint_ignores_generations_created_while_it_runs() { @@ -10314,10 +10292,10 @@ mod tests { let seen = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { return accepted(); } - if path.contains("compact_lsm") { + if path.contains("compact_memwal") { seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); return accepted(); } @@ -10331,11 +10309,11 @@ mod tests { ok_json(body) }); - table.checkpoint_lsm().await.unwrap(); + table.checkpoint_memwal().await.unwrap(); assert_eq!( compacts.load(std::sync::atomic::Ordering::SeqCst), 1, - "the loop must not chase generations written after it started" + "the loop must not chase SSTables written after it started" ); } @@ -10348,11 +10326,11 @@ mod tests { let seen = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { return accepted(); } - if path.contains("compact_lsm") { - // First two triggers: every bucket already latched. + if path.contains("compact_memwal") { + // First two triggers: every table shard already latched. if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2 { return http::Response::builder() .status(429) @@ -10373,7 +10351,7 @@ mod tests { }); table - .checkpoint_lsm() + .checkpoint_memwal() .await .expect("contention must not abort the checkpoint"); assert_eq!( @@ -10393,7 +10371,7 @@ mod tests { let seen = polls.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") || path.contains("compact_lsm") { + if path.contains("flush_memwal") || path.contains("compact_memwal") { return accepted(); } // The opening read lands; the next two polls are latched out. @@ -10408,7 +10386,7 @@ mod tests { }); table - .checkpoint_lsm() + .checkpoint_memwal() .await .expect("a contended poll must be retried, not surfaced"); assert_eq!( @@ -10428,7 +10406,7 @@ mod tests { let seen = flushes.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 5 { return http::Response::builder() .status(429) @@ -10437,20 +10415,20 @@ mod tests { } return accepted(); } - if path.contains("compact_lsm") { + if path.contains("compact_memwal") { return accepted(); } ok_json(stats_body(&[], false)) }); table - .checkpoint_lsm() + .checkpoint_memwal() .await .expect("contention must not be reported as a lost claim"); assert_eq!( flushes.load(std::sync::atomic::Ordering::SeqCst), 6, - "five retries against one seal, then it lands" + "five retries against one freeze, then it lands" ); } @@ -10469,7 +10447,7 @@ mod tests { .unwrap() }); - let err = table.checkpoint_lsm().await.unwrap_err(); + let err = table.checkpoint_memwal().await.unwrap_err(); assert!( matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 429), "the fault that spent the budget must be the one reported: {err:?}" @@ -10499,7 +10477,7 @@ mod tests { .unwrap() }); - let err = table.checkpoint_lsm().await.unwrap_err(); + let err = table.checkpoint_memwal().await.unwrap_err(); let message = err.to_string(); assert!( matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 503), @@ -10519,7 +10497,7 @@ mod tests { /// A long stall with nothing compacting must keep waiting, not fail. The /// client cannot judge this: a checkpoint queued behind unrelated tables /// on the pod-wide compactor pool reports exactly these numbers — flat - /// generations, an idle latch — as one whose merges are failing. The + /// SSTable counts, an idle latch — as one whose merges are failing. The /// deadline is the caller's. #[tokio::test(start_paused = true)] async fn test_checkpoint_waits_out_a_long_stall_rather_than_failing() { @@ -10527,7 +10505,7 @@ mod tests { let seen = polls.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") || path.contains("compact_lsm") { + if path.contains("flush_memwal") || path.contains("compact_memwal") { return accepted(); } // Flat for far longer than any bound this loop ever had, with @@ -10537,7 +10515,7 @@ mod tests { }); table - .checkpoint_lsm() + .checkpoint_memwal() .await .expect("a stall is the server being slow, not the client's call to make"); assert!( @@ -10546,7 +10524,7 @@ mod tests { ); } - /// A pass already owns the latch on every outstanding bucket, so the loop + /// A pass already owns the latch on every outstanding table shard, so the loop /// waits rather than piling on triggers it would only refuse. This is the /// sole thing `compacting` is read for. #[tokio::test(start_paused = true)] @@ -10557,10 +10535,10 @@ mod tests { let seen_compacts = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { return accepted(); } - if path.contains("compact_lsm") { + if path.contains("compact_memwal") { seen_compacts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); return accepted(); } @@ -10574,32 +10552,32 @@ mod tests { }); table - .checkpoint_lsm() + .checkpoint_memwal() .await .expect("a running pass is progress, not a stall"); assert_eq!( compacts.load(std::sync::atomic::Ordering::SeqCst), 0, - "never trigger against a bucket already compacting" + "never trigger against a table shard already compacting" ); } /// WAL off ⇒ `None`; WAL on ⇒ a fully populated `Some` with no field - /// defaulting to a zero it did not measure. `include_generation_rows` + /// defaulting to a zero it did not measure. `include_sstable_rows` /// rides in the body and is off unless asked for. #[tokio::test] - async fn test_get_lsm_stats_round_trip() { + async fn test_get_memwal_stats_round_trip() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!(request.url().path(), "/v1/table/my_table/get_lsm_stats/"); + assert_eq!(request.url().path(), "/v1/table/my_table/get_memwal_stats/"); let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!( - body["include_generation_rows"], true, + body["include_sstable_rows"], true, "the flag must reach the server, not be silently dropped" ); let response = serde_json::json!({ - "lsm_stats": { - "buckets": [{ + "memwal_stats": { + "table_shards": [{ "shard_id": "b0", "status": "Active", "writer_epoch": 3, @@ -10607,7 +10585,7 @@ mod tests { "current_generation": 9, "replay_after_wal_entry_position": 100, "wal_entry_position_last_seen": 140, - "generations": [{ "generation": 8, "bytes": 4096, "rows": 30 }], + "sstables": [{ "generation": 8, "bytes": 4096, "rows": 30 }], "compacting": false, "memtables": [ { "generation": 9, "rows": 12, "bytes": 900, "batches": 2, @@ -10623,19 +10601,19 @@ mod tests { }); let stats = table - .get_lsm_stats(true) + .get_memwal_stats(true) .await .unwrap() .expect("a WAL-backed table reports Some"); - let bucket = &stats.buckets[0]; - assert_eq!(bucket.replay_after_wal_entry_position, 100); - assert_eq!(bucket.wal_entry_position_last_seen, 140); - assert!(!bucket.compacting); - assert_eq!(bucket.generations[0].generation, 8); - assert_eq!(bucket.generations[0].rows, Some(30)); + let table_shard = &stats.table_shards[0]; + assert_eq!(table_shard.replay_after_wal_entry_position, 100); + assert_eq!(table_shard.wal_entry_position_last_seen, 140); + assert!(!table_shard.compacting); + assert_eq!(table_shard.sstables[0].generation, 8); + assert_eq!(table_shard.sstables[0].rows, Some(30)); // The line that answers "why is my fresh-tier vector search // brute-force" — an absent index name is the whole explanation. - let memtables = bucket.memtables.as_ref().unwrap(); + let memtables = table_shard.memtables.as_ref().unwrap(); assert_eq!(memtables[0].indexes, vec!["vec_idx".to_string()]); } @@ -10655,7 +10633,7 @@ mod tests { .unwrap() }); - let err = table.checkpoint_lsm().await.unwrap_err(); + let err = table.checkpoint_memwal().await.unwrap_err(); assert!( matches!(err, Error::TableNotFound { .. }), "a missing table must say so: {err:?}" @@ -10676,12 +10654,12 @@ mod tests { let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - if path.contains("flush_lsm") { + if path.contains("flush_memwal") { // First flush lands; the claim is then lost, and the // re-issued flush succeeds. return accepted(); } - if path.contains("compact_lsm") { + if path.contains("compact_memwal") { if n < 4 { return http::Response::builder() .status(421) @@ -10694,20 +10672,20 @@ mod tests { }); table - .checkpoint_lsm() + .checkpoint_memwal() .await .expect("a lost claim must be recovered by re-flushing, not surfaced"); } #[tokio::test] - async fn test_get_lsm_stats_absent_when_wal_off() { + async fn test_get_memwal_stats_absent_when_wal_off() { let table = Table::new_with_handler("my_table", |_request| { http::Response::builder() .status(200) - .body(serde_json::json!({ "lsm_stats": null }).to_string()) + .body(serde_json::json!({ "memwal_stats": null }).to_string()) .unwrap() }); - assert!(table.get_lsm_stats(false).await.unwrap().is_none()); + assert!(table.get_memwal_stats(false).await.unwrap().is_none()); } #[tokio::test] @@ -14726,7 +14704,7 @@ mod tests { /// Verbatim from sophon's `LSM_WITH_ROW_ID_UNSUPPORTED`; the fallback keys /// off this sentence, so a test that paraphrased it would prove nothing. - const WAL_ROW_ID_REFUSAL_BODY: &str = r#"{"code":13,"error":"Bad request: the MemWAL LSM scanner does not support with_row_id (the LSM scanner exposes _rowaddr, not a stable _rowid); set use_lsm(false) to read the base table only (results will exclude un-compacted MemWAL data)"}"#; + const WAL_ROW_ID_REFUSAL_BODY: &str = r#"{"code":13,"error":"Bad request: the MemWAL LSM scanner does not support with_row_id (the LSM scanner exposes _rowaddr, not a stable _rowid); set use_memwal(false) to read the base table only (results will exclude un-compacted MemWAL data)"}"#; use lance::dataset::ROW_ID; @@ -14985,7 +14963,7 @@ mod tests { // Translated, not passed through: the caller gets the reason and the // way out, which a bare 400 does not carry. assert!(matches!(err, Error::NotSupported { .. }), "{err:?}"); - assert!(err.to_string().contains("use_lsm(false)"), "{err}"); + assert!(err.to_string().contains("use_memwal(false)"), "{err}"); assert!( bodies .lock() diff --git a/rust/lancedb/src/remote/table/blobs.rs b/rust/lancedb/src/remote/table/blobs.rs index 901616fb2..fdcb8f440 100644 --- a/rust/lancedb/src/remote/table/blobs.rs +++ b/rust/lancedb/src/remote/table/blobs.rs @@ -685,7 +685,7 @@ impl RemoteTable { select: Select::columns(&[column]), with_row_id: true, // Row ids name base-table rows, and the MemWAL scanner has no stable `_rowid`. - use_lsm: Some(false), + use_memwal: Some(false), ..Default::default() }); let body = self @@ -1121,7 +1121,7 @@ mod tests { .unwrap(); assert_eq!(body["columns"], serde_json::json!(["image"])); assert_eq!(body["with_row_id"], serde_json::json!(true)); - assert_eq!(body["use_lsm"], serde_json::json!(false)); + assert_eq!(body["use_memwal"], serde_json::json!(false)); let requested = filter_row_ids(body["filter"].as_str().unwrap()); captured.lock().unwrap().push(TakeMockRequest::Query { row_ids: requested.clone(), diff --git a/rust/lancedb/src/remote/table/insert.rs b/rust/lancedb/src/remote/table/insert.rs index eef1d8e42..3a089eb95 100644 --- a/rust/lancedb/src/remote/table/insert.rs +++ b/rust/lancedb/src/remote/table/insert.rs @@ -1497,7 +1497,7 @@ mod tests { when_not_matched_by_source_delete: false, when_not_matched_by_source_delete_filt: None, use_index: true, - use_lsm: None, + use_memwal: None, }; let input: Arc = Arc::new(ErroringExec::new()); diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 74b99255a..c0a4a719e 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -80,7 +80,7 @@ pub mod datafusion; pub(crate) mod dataset; pub mod delete; pub mod freshness; -pub mod lsm_stats; +pub mod memwal_stats; pub mod merge; pub mod optimize; pub(crate) mod primary_key; @@ -107,7 +107,7 @@ use futures::future::join_all; pub use lance::dataset::refs::{BranchContents, Ref, TagContents, Tags as LanceTags}; pub use lance::dataset::scanner::DatasetRecordBatchStream; pub use lance_index::optimize::OptimizeOptions; -pub use lsm_stats::{BucketStats, GenerationStats, LsmStats, MemtableStats}; +pub use memwal_stats::{MemWalStats, MemtableStats, SsTableStats, TableShardStats}; pub use optimize::{CompactionOptions, OptimizeAction, OptimizeStats}; pub use refresh::RefreshColumnResult; pub use schema_evolution::{ @@ -327,20 +327,20 @@ pub use self::merge::MergeResult; /// Specification selecting Lance's MemWAL LSM-style write path for /// `merge_insert`. /// -/// Construct via [`LsmWriteSpec::bucket`], [`LsmWriteSpec::identity`], or -/// [`LsmWriteSpec::unsharded`], then optionally chain -/// [`LsmWriteSpec::with_maintained_indexes`] (indexes the MemWAL keeps up to -/// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default +/// Construct via [`MemWalWriteSpec::bucket`], [`MemWalWriteSpec::identity`], or +/// [`MemWalWriteSpec::unsharded`], then optionally chain +/// [`MemWalWriteSpec::with_maintained_indexes`] (indexes the MemWAL keeps up to +/// date) and [`MemWalWriteSpec::with_writer_config_defaults`] (default /// `ShardWriter` configuration recorded in the MemWAL index). /// /// A fresh spec maintains every index on the table, including ones created /// after it is installed. /// -/// Install a spec with [`Table::set_lsm_write_spec`] and remove it with -/// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch +/// Install a spec with [`Table::enable_memwal`] and remove it with +/// [`Table::disable_memwal`]. The actual `merge_insert` dispatch /// onto the MemWAL writer is a follow-up. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum LsmWriteSpec { +pub enum MemWalWriteSpec { /// Hash-bucket sharding by a scalar column. /// /// `column` must be a non-nested column with a supported scalar type. @@ -384,7 +384,7 @@ pub enum LsmWriteSpec { }, } -impl LsmWriteSpec { +impl MemWalWriteSpec { /// Construct a hash-bucket sharding spec maintaining every index on the table. pub fn bucket(column: impl Into, num_buckets: u32) -> Self { Self::Bucket { @@ -401,7 +401,7 @@ impl LsmWriteSpec { /// `column` must be a deterministic function of the unenforced primary /// key: every row with a given primary key must always produce the same /// `column` value. MemWAL dedups upserts by primary key but tracks - /// generations per shard, so if the same key is written with two + /// SSTables per shard, so if the same key is written with two /// different `column` values its versions land in different shards and a /// stale value can win. Typically `column` is the primary key itself, or /// a stable attribute of it (e.g. a tenant id). @@ -431,13 +431,13 @@ impl LsmWriteSpec { /// any added with [`NativeTable::set_mem_index_registry`]. /// /// ``` - /// # use lancedb::table::LsmWriteSpec; + /// # use lancedb::table::MemWalWriteSpec; /// // Every index the table has, now and later: - /// LsmWriteSpec::unsharded().with_maintained_indexes(None); + /// MemWalWriteSpec::unsharded().with_maintained_indexes(None); /// // Exactly these: - /// LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]); + /// MemWalWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]); /// // None at all: - /// LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new()); + /// MemWalWriteSpec::unsharded().with_maintained_indexes(Vec::new()); /// ``` pub fn with_maintained_indexes(mut self, indexes: impl Into>>) -> Self { let indexes = indexes.into(); @@ -674,37 +674,37 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { message: "set_unenforced_primary_key is not supported on this table type".into(), }) } - /// Install an [`LsmWriteSpec`] on this table. + /// Install an [`MemWalWriteSpec`] on this table. /// /// The spec selects Lance's MemWAL LSM-style write path for future /// `merge_insert` calls. /// /// The default implementation returns `NotSupported`. Implementations - /// that support the MemWAL LSM write path must override this. - async fn set_lsm_write_spec(&self, _spec: LsmWriteSpec) -> Result<()> { + /// that support the MemWAL write path must override this. + async fn enable_memwal(&self, _spec: MemWalWriteSpec) -> Result<()> { Err(Error::NotSupported { - message: "set_lsm_write_spec is not supported on this table type".into(), + message: "enable_memwal is not supported on this table type".into(), }) } - /// Remove the [`LsmWriteSpec`] from this table. + /// Remove the [`MemWalWriteSpec`] from this table. /// /// This is a no-op if no spec is currently set. /// /// The default implementation returns `NotSupported`. Implementations - /// that support the MemWAL LSM write path must override this. - async fn unset_lsm_write_spec(&self) -> Result<()> { + /// that support the MemWAL write path must override this. + async fn disable_memwal(&self) -> Result<()> { Err(Error::NotSupported { - message: "unset_lsm_write_spec is not supported on this table type".into(), + message: "disable_memwal is not supported on this table type".into(), }) } - /// Read the [`LsmWriteSpec`] currently installed on this table, returning - /// `None` when the MemWAL LSM write path is not enabled. + /// Read the [`MemWalWriteSpec`] currently installed on this table, returning + /// `None` when the MemWAL write path is not enabled. /// /// The default implementation returns `NotSupported`. Implementations that - /// support the MemWAL LSM write path must override this. - async fn get_lsm_write_spec(&self) -> Result> { + /// support the MemWAL write path must override this. + async fn get_memwal_spec(&self) -> Result> { Err(Error::NotSupported { - message: "get_lsm_write_spec is not supported on this table type".into(), + message: "get_memwal_spec is not supported on this table type".into(), }) } /// Whether a hybrid query on this table has already been told it cannot @@ -732,36 +732,36 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { /// the only cost of being late to notice is a base-only read that is still /// correct. fn note_hybrid_pk_fusion(&self) {} - /// Seal every bucket's active memtable into L0. + /// Freeze every table shard's active memtable into an SSTable. /// /// The default implementation returns `NotSupported`. - async fn flush_lsm(&self) -> Result<()> { + async fn flush_memwal(&self) -> Result<()> { Err(Error::NotSupported { - message: "flush_lsm is not supported on this table type".into(), + message: "flush_memwal is not supported on this table type".into(), }) } - /// Trigger a background L0 → base compaction pass per bucket. + /// Trigger a background SSTable compaction pass per table shard. /// /// The default implementation returns `NotSupported`. - async fn compact_lsm(&self) -> Result<()> { + async fn compact_memwal(&self) -> Result<()> { Err(Error::NotSupported { - message: "compact_lsm is not supported on this table type".into(), + message: "compact_memwal is not supported on this table type".into(), }) } - /// Read live LSM state, or `None` when the LSM write path is not + /// Read live MemWAL state, or `None` when the MemWAL write path is not /// enabled for this table. /// /// The default implementation returns `NotSupported`. - async fn get_lsm_stats(&self, _include_generation_rows: bool) -> Result> { + async fn get_memwal_stats(&self, _include_sstable_rows: bool) -> Result> { Err(Error::NotSupported { - message: "get_lsm_stats is not supported on this table type".into(), + message: "get_memwal_stats is not supported on this table type".into(), }) } /// Drain and close any cached MemWAL shard writers for this table. /// /// The default implementation is a no-op; table types that maintain /// MemWAL shard writers override it. - async fn close_lsm_writers(&self) -> Result<()> { + async fn close_memwal_writers(&self) -> Result<()> { Ok(()) } /// Names of the blob v2 columns in this table, in declaration order. @@ -2006,50 +2006,50 @@ impl Table { self.inner.set_unenforced_primary_key(&borrowed).await } - /// Install an [`LsmWriteSpec`] on this table, selecting Lance's MemWAL + /// Install an [`MemWalWriteSpec`] on this table, selecting Lance's MemWAL /// LSM-style write path for future `merge_insert` calls. /// - /// [`LsmWriteSpec`] chooses one of three sharding strategies: + /// [`MemWalWriteSpec`] chooses one of three sharding strategies: /// - /// - [`LsmWriteSpec::bucket`] — hash-bucket writes by a scalar column. - /// - [`LsmWriteSpec::identity`] — shard by the raw value of a scalar column. - /// - [`LsmWriteSpec::unsharded`] — route every write to a single shard. + /// - [`MemWalWriteSpec::bucket`] — hash-bucket writes by a scalar column. + /// - [`MemWalWriteSpec::identity`] — shard by the raw value of a scalar column. + /// - [`MemWalWriteSpec::unsharded`] — route every write to a single shard. /// /// A table carries one spec: this fails while one is installed, since the - /// generations already written were homed under it. - /// [`Table::unset_lsm_write_spec`] removes one. + /// SSTables already written were homed under it. + /// [`Table::disable_memwal`] removes one. /// /// # Example /// /// ``` - /// # use lancedb::table::{LsmWriteSpec, Table}; + /// # use lancedb::table::{MemWalWriteSpec, Table}; /// # async fn example(table: &Table) -> Result<(), Box> { /// table - /// .set_lsm_write_spec( - /// LsmWriteSpec::bucket("id", 16).with_maintained_indexes(vec!["id_idx".to_string()]), + /// .enable_memwal( + /// MemWalWriteSpec::bucket("id", 16).with_maintained_indexes(vec!["id_idx".to_string()]), /// ) /// .await?; /// # Ok(()) /// # } /// ``` - pub async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> Result<()> { - self.inner.set_lsm_write_spec(spec).await + pub async fn enable_memwal(&self, spec: MemWalWriteSpec) -> Result<()> { + self.inner.enable_memwal(spec).await } - /// Remove the [`LsmWriteSpec`] from this table, reverting to the standard + /// Remove the [`MemWalWriteSpec`] from this table, reverting to the standard /// `merge_insert` write path. /// /// Errors if no spec is currently set. - pub async fn unset_lsm_write_spec(&self) -> Result<()> { - self.inner.unset_lsm_write_spec().await + pub async fn disable_memwal(&self) -> Result<()> { + self.inner.disable_memwal().await } - /// Read the [`LsmWriteSpec`] currently installed on this table. + /// Read the [`MemWalWriteSpec`] currently installed on this table. /// - /// `Ok(None)` means the LSM write path is not enabled at all — no spec has - /// been set, or one was removed with [`Table::unset_lsm_write_spec`]. That + /// `Ok(None)` means the MemWAL write path is not enabled at all — no spec has + /// been set, or one was removed with [`Table::disable_memwal`]. That /// is a different answer from a spec whose - /// [`LsmWriteSpec::maintained_indexes`] is `None`, which is an installed + /// [`MemWalWriteSpec::maintained_indexes`] is `None`, which is an installed /// spec selecting indexes automatically. /// /// The spec read back is the one that was installed, selection included: @@ -2062,21 +2062,21 @@ impl Table { /// ``` /// # use lancedb::table::Table; /// # async fn example(table: &Table) -> Result<(), Box> { - /// if let Some(spec) = table.get_lsm_write_spec().await? { - /// println!("LSM write path enabled: {:?}", spec); + /// if let Some(spec) = table.get_memwal_spec().await? { + /// println!("MemWAL write path enabled: {:?}", spec); /// } /// # Ok(()) /// # } /// ``` - pub async fn get_lsm_write_spec(&self) -> Result> { - self.inner.get_lsm_write_spec().await + pub async fn get_memwal_spec(&self) -> Result> { + self.inner.get_memwal_spec().await } - /// Converge this table's LSM write path into its base table. + /// Converge this table's MemWAL write path into its base table. /// - /// One `flush` to seal every memtable into L0, then compaction triggers + /// One `flush` to freeze every memtable into an SSTable, then compaction triggers /// until every generation that existed at that moment has reached base. - /// The loop runs client-side, reading progress from `get_lsm_stats`, so + /// The loop runs client-side, reading progress from `get_memwal_stats`, so /// there is no held socket and nothing to reconcile if you drop this /// future partway through. /// @@ -2086,7 +2086,7 @@ impl Table { /// already-converged table costs two round trips and triggers nothing. /// /// **No deadline, and the caller owns that.** It returns when the target - /// generations are gone, propagates a terminal server fault, and + /// SSTables are gone, propagates a terminal server fault, and /// otherwise waits however long the server takes. A slow table and a /// stuck one are the same picture from here: the compactor pool is shared /// across every table on the node, so a checkpoint queued behind @@ -2099,66 +2099,69 @@ impl Table { /// ```no_run /// # use lancedb::Table; /// # async fn example(table: &Table) -> Result<(), Box> { - /// let before = table.get_lsm_stats(false).await?; - /// table.checkpoint_lsm().await?; - /// let after = table.get_lsm_stats(false).await?; + /// let before = table.get_memwal_stats(false).await?; + /// table.checkpoint_memwal().await?; + /// let after = table.get_memwal_stats(false).await?; /// # Ok(()) /// # } /// ``` - pub async fn checkpoint_lsm(&self) -> Result<()> { - checkpoint::checkpoint_lsm(self).await + pub async fn checkpoint_memwal(&self) -> Result<()> { + checkpoint::checkpoint_memwal(self).await } - /// Seal every bucket's active memtable into L0 without touching the + /// Freeze every table shard's active memtable into an SSTable without touching the /// base table. /// - /// Independently useful: flushing makes memtable rows readable from L0 at + /// Independently useful: flushing makes memtable rows readable from an SSTable at /// a lower per-query cost. On a node that has not claimed this table it /// claims it and replays the WAL log first — reporting "nothing to flush" /// without replaying would lie about durable data. - pub async fn flush_lsm(&self) -> Result<()> { - self.inner.flush_lsm().await + pub async fn flush_memwal(&self) -> Result<()> { + self.inner.flush_memwal().await } - /// Run one bounded L0 → base compaction pass per bucket, reporting what + /// Run one bounded SSTable compaction pass per table shard, reporting what /// it merged and what is left. /// /// One pass, not convergence: that bounds each request's cost and gives a /// caller driving its own cadence a progress signal per round trip. - pub async fn compact_lsm(&self) -> Result<()> { - self.inner.compact_lsm().await + pub async fn compact_memwal(&self) -> Result<()> { + self.inner.compact_memwal().await } - /// Read live per-bucket LSM state. + /// Read live per-table-shard MemWAL state. /// - /// Answers "how far behind is my fresh tier", "which bucket is hot", and + /// Answers "how far behind is my fresh tier", "which table shard is hot", and /// "why is my fresh-tier vector search brute-force". Mutates no table /// state, though on a node that has not claimed this table it claims it, /// exactly as a read would. /// - /// `include_generation_rows` reports a row count per L0 generation. Off by + /// `include_sstable_rows` reports a row count per SSTable. Off by /// default: each count opens an uncached Lance dataset, and - /// `checkpoint_lsm` polls this needing only generation numbers. + /// `checkpoint_memwal` polls this needing only generation numbers. /// - /// `Ok(None)` only when the LSM write path is not enabled, matching - /// [`Table::get_lsm_write_spec`]. Stats is fresh-tier only, so with the + /// `Ok(None)` only when the MemWAL write path is not enabled, matching + /// [`Table::get_memwal_spec`]. Stats is fresh-tier only, so with the /// WAL off there is no manifest to report and a struct of zeros would /// read as measurements. /// /// Do not build a checkpoint's termination on this: the completion /// predicate lives in the `flush` and `compact` responses. - pub async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result> { - self.inner.get_lsm_stats(include_generation_rows).await + pub async fn get_memwal_stats( + &self, + include_sstable_rows: bool, + ) -> Result> { + self.inner.get_memwal_stats(include_sstable_rows).await } /// Drain and close any cached MemWAL shard writers held for this table. /// - /// When an [`LsmWriteSpec`] is installed, `merge_insert` opens MemWAL shard + /// When an [`MemWalWriteSpec`] is installed, `merge_insert` opens MemWAL shard /// writers and caches them for reuse across calls. This closes them, /// flushing pending data; writers reopen lazily on the next `merge_insert`. /// It is a no-op when no writers are cached. - pub async fn close_lsm_writers(&self) -> Result<()> { - self.inner.close_lsm_writers().await + pub async fn close_memwal_writers(&self) -> Result<()> { + self.inner.close_memwal_writers().await } /// Retrieve the version of the table @@ -2776,8 +2779,8 @@ impl NativeTable { } /// Maintain the memtable index kinds in `registry` when this table writes - /// through its [`LsmWriteSpec`], and accept them when - /// [`LsmWriteSpec::with_maintained_indexes`] names one. + /// through its [`MemWalWriteSpec`], and accept them when + /// [`MemWalWriteSpec::with_maintained_indexes`] names one. /// /// The registry belongs to this handle, its clones and its branch handles, /// and is not stored with the table: a handle opened separately starts from @@ -2788,7 +2791,7 @@ impl NativeTable { /// ``` /// # use std::sync::Arc; /// # use lancedb::Table; - /// # use lancedb::table::{LsmWriteSpec, MemIndexRegistry, mem_index::MemIndexPlugin}; + /// # use lancedb::table::{MemWalWriteSpec, MemIndexRegistry, mem_index::MemIndexPlugin}; /// # async fn example( /// # table: &Table, /// # plugin: Arc, @@ -2797,8 +2800,8 @@ impl NativeTable { /// if let Some(native) = table.as_native() { /// native.set_mem_index_registry(registry); /// } - /// let spec = LsmWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]); - /// table.set_lsm_write_spec(spec).await?; + /// let spec = MemWalWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]); + /// table.enable_memwal(spec).await?; /// # Ok(()) /// # } /// ``` @@ -3691,20 +3694,20 @@ impl BaseTable for NativeTable { primary_key::set_unenforced_primary_key(self, columns).await } - async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> Result<()> { - merge::lsm::set_lsm_write_spec(self, spec).await + async fn enable_memwal(&self, spec: MemWalWriteSpec) -> Result<()> { + merge::lsm::enable_memwal(self, spec).await } - async fn unset_lsm_write_spec(&self) -> Result<()> { - merge::lsm::unset_lsm_write_spec(self).await + async fn disable_memwal(&self) -> Result<()> { + merge::lsm::disable_memwal(self).await } - async fn get_lsm_write_spec(&self) -> Result> { - merge::lsm::get_lsm_write_spec(self).await + async fn get_memwal_spec(&self) -> Result> { + merge::lsm::get_memwal_spec(self).await } - async fn close_lsm_writers(&self) -> Result<()> { - merge::lsm::close_lsm_writers(self).await + async fn close_memwal_writers(&self) -> Result<()> { + merge::lsm::close_memwal_writers(self).await } async fn blob_columns(&self) -> Result> { @@ -5510,7 +5513,7 @@ mod tests { } #[tokio::test] - async fn test_set_lsm_write_spec() { + async fn test_enable_memwal() { use arrow_array::StringArray; use lance::dataset::mem_wal::DatasetMemWalExt; @@ -5542,7 +5545,7 @@ mod tests { // Reject num_buckets out of range. for bad in [0u32, 1025] { let err = table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", bad)) + .enable_memwal(MemWalWriteSpec::bucket("id", bad)) .await .expect_err("should reject"); assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); @@ -5550,7 +5553,7 @@ mod tests { // Happy path: install spec; verify MemWAL details record it. table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 4)) + .enable_memwal(MemWalWriteSpec::bucket("id", 4)) .await .unwrap(); @@ -5576,14 +5579,14 @@ mod tests { // Mutation rejected. let err = table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 8)) + .enable_memwal(MemWalWriteSpec::bucket("id", 8)) .await .expect_err("mutation should be rejected"); assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); } #[tokio::test] - async fn test_set_lsm_write_spec_unsharded() { + async fn test_enable_memwal_unsharded() { use lance::dataset::mem_wal::DatasetMemWalExt; let tmp_dir = tempdir().unwrap(); @@ -5605,7 +5608,7 @@ mod tests { let table = conn.create_table("t", reader).execute().await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -5623,7 +5626,7 @@ mod tests { } #[tokio::test] - async fn test_set_lsm_write_spec_identity() { + async fn test_enable_memwal_identity() { use lance::dataset::mem_wal::DatasetMemWalExt; let tmp_dir = tempdir().unwrap(); @@ -5651,8 +5654,8 @@ mod tests { let table = conn.create_table("t", reader).execute().await.unwrap(); table - .set_lsm_write_spec( - LsmWriteSpec::identity("region") + .enable_memwal( + MemWalWriteSpec::identity("region") .with_writer_config_defaults([("durable_write", "false")]), ) .await @@ -5680,7 +5683,7 @@ mod tests { } #[tokio::test] - async fn test_unset_lsm_write_spec() { + async fn test_disable_memwal() { use lance::dataset::mem_wal::DatasetMemWalExt; let tmp_dir = tempdir().unwrap(); @@ -5702,11 +5705,11 @@ mod tests { let table = conn.create_table("t", reader).execute().await.unwrap(); // unset errors when no spec is set. - table.unset_lsm_write_spec().await.unwrap_err(); + table.disable_memwal().await.unwrap_err(); // Install a spec, then unset it. table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 4)) + .enable_memwal(MemWalWriteSpec::bucket("id", 4)) .await .unwrap(); { @@ -5714,16 +5717,16 @@ mod tests { assert!(dataset.mem_wal_index_details().await.unwrap().is_some()); } - table.unset_lsm_write_spec().await.unwrap(); + table.disable_memwal().await.unwrap(); { let dataset = table.as_native().unwrap().dataset.get().await.unwrap(); assert!(dataset.mem_wal_index_details().await.unwrap().is_none()); } // A second unset errors; a fresh spec can still be installed afterwards. - table.unset_lsm_write_spec().await.unwrap_err(); + table.disable_memwal().await.unwrap_err(); table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 8)) + .enable_memwal(MemWalWriteSpec::bucket("id", 8)) .await .unwrap(); { @@ -5733,7 +5736,7 @@ mod tests { } #[tokio::test] - async fn test_get_lsm_write_spec() { + async fn test_get_memwal_spec() { let tmp_dir = tempdir().unwrap(); let uri = tmp_dir.path().to_str().unwrap(); @@ -5759,7 +5762,7 @@ mod tests { let table = conn.create_table("t", reader).execute().await.unwrap(); // No spec installed yet. - assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); + assert_eq!(table.get_memwal_spec().await.unwrap(), None); // A real scalar index is needed to name it as a maintained index. table @@ -5771,35 +5774,35 @@ mod tests { // Bucket spec round-trips exactly, including the routing column (recovered // from its field id), maintained indexes, and writer config defaults. - let spec = LsmWriteSpec::bucket("id", 4) + let spec = MemWalWriteSpec::bucket("id", 4) .with_maintained_indexes(vec![idx_name.clone()]) .with_writer_config_defaults([("durable_write", "false")]); - table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); + table.enable_memwal(spec.clone()).await.unwrap(); + assert_eq!(table.get_memwal_spec().await.unwrap(), Some(spec)); // After unset, no spec is reported. - table.unset_lsm_write_spec().await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); + table.disable_memwal().await.unwrap(); + assert_eq!(table.get_memwal_spec().await.unwrap(), None); // Identity sharding round-trips (column recovered from the schema). // A spec left at its default round-trips as the default: what is stored // is the intent to maintain everything, not the set it resolves to now. - let spec = LsmWriteSpec::identity("region"); - table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); - table.unset_lsm_write_spec().await.unwrap(); + let spec = MemWalWriteSpec::identity("region"); + table.enable_memwal(spec.clone()).await.unwrap(); + assert_eq!(table.get_memwal_spec().await.unwrap(), Some(spec)); + table.disable_memwal().await.unwrap(); // Unsharded round-trips (no routing column). - let spec = LsmWriteSpec::unsharded(); - table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); + let spec = MemWalWriteSpec::unsharded(); + table.enable_memwal(spec.clone()).await.unwrap(); + assert_eq!(table.get_memwal_spec().await.unwrap(), Some(spec)); } /// The maintained set defaults to every index on the table. A named index /// the memtable cannot build fails the install, because the caller asked /// for it; an unnamed one is skipped, because it arrived by existing. #[tokio::test] - async fn test_set_lsm_write_spec_infers_maintained_indexes() { + async fn test_enable_memwal_infers_maintained_indexes() { let tmp_dir = tempdir().unwrap(); let uri = tmp_dir.path().to_str().unwrap(); @@ -5839,8 +5842,9 @@ mod tests { // Explicitly naming the bitmap index fails before anything commits. let err = table - .set_lsm_write_spec( - LsmWriteSpec::unsharded().with_maintained_indexes(vec!["tag_bitmap".to_string()]), + .enable_memwal( + MemWalWriteSpec::unsharded() + .with_maintained_indexes(vec!["tag_bitmap".to_string()]), ) .await .unwrap_err(); @@ -5848,18 +5852,18 @@ mod tests { matches!(err, Error::InvalidInput { ref message } if message.contains("tag_bitmap")), "expected the bitmap index to be rejected, got {err:?}" ); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); + assert_eq!(table.get_memwal_spec().await.unwrap(), None); // The default names nothing, so the bitmap is skipped rather than // refusing the table: it is maintained by existing, not by being asked // for. The intent is what is stored, not the set it resolves to today. table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); assert_eq!( table - .get_lsm_write_spec() + .get_memwal_spec() .await .unwrap() .unwrap() @@ -5870,16 +5874,16 @@ mod tests { // Narrowing to a named subset: unset first, since an installed spec // cannot be set over. - table.unset_lsm_write_spec().await.unwrap(); + table.disable_memwal().await.unwrap(); table - .set_lsm_write_spec( - LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]), + .enable_memwal( + MemWalWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]), ) .await .unwrap(); assert_eq!( table - .get_lsm_write_spec() + .get_memwal_spec() .await .unwrap() .unwrap() @@ -5888,14 +5892,14 @@ mod tests { ); // Opting out entirely is distinct from the default. - table.unset_lsm_write_spec().await.unwrap(); + table.disable_memwal().await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new())) + .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(Vec::new())) .await .unwrap(); assert_eq!( table - .get_lsm_write_spec() + .get_memwal_spec() .await .unwrap() .unwrap() @@ -5906,7 +5910,7 @@ mod tests { // An installed spec is never set over, whatever the new one asks for, // and the refused call leaves it alone. let err = table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 4)) + .enable_memwal(MemWalWriteSpec::bucket("id", 4)) .await .unwrap_err(); assert!( @@ -5916,7 +5920,7 @@ mod tests { ); assert_eq!( table - .get_lsm_write_spec() + .get_memwal_spec() .await .unwrap() .unwrap() diff --git a/rust/lancedb/src/table/checkpoint.rs b/rust/lancedb/src/table/checkpoint.rs index bb76604ed..c9a2f60d6 100644 --- a/rust/lancedb/src/table/checkpoint.rs +++ b/rust/lancedb/src/table/checkpoint.rs @@ -1,18 +1,18 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Converging a table's LSM write path into its base table. +//! Converging a table's MemWAL write path into its base table. //! -//! `checkpoint_lsm` seals once, then triggers compaction and watches -//! generation numbers until the L0 that existed at the start is gone. +//! `checkpoint_memwal` freezes once, then triggers compaction and watches +//! generation numbers until the SSTables that existed at the start are gone. //! -//! The loop runs in the client, not the server: `compact_lsm` dispatches a +//! The loop runs in the client, not the server: `compact_memwal` dispatches a //! pass and returns, so nothing holds a socket and a client can vanish //! mid-operation with nothing to reconcile. Completion is read from //! generation numbers in the shard manifest — durable state, unlike a count //! in a compact response, which a concurrent write invalidates. //! -//! The target set is fixed at the start, so generations created *during* the +//! The target set is fixed at the start, so SSTables created *during* the //! checkpoint are ignored. That is what lets it terminate under write load, //! and what makes it best-effort: it converges the fresh tier as of some //! instant. Idempotent, abandonable at any point, safe on a cadence. @@ -68,7 +68,7 @@ fn is_lost_claim(e: &Error) -> bool { status_of(e) == Some(421) } -/// Interval between `get_lsm_stats` polls. One interval is roughly one +/// Interval between `get_memwal_stats` polls. One interval is roughly one /// compaction pass, the granularity at which the answer can change. /// /// Fixed rather than configurable, matching `wait_for_index`. It costs @@ -149,15 +149,15 @@ where } } -/// Drive [`Table::checkpoint_lsm`]: seal once, fix the target watermark -/// from the resulting L0, then trigger and poll until it drains. -pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> { +/// Drive [`Table::checkpoint_memwal`]: freeze once, fix the target watermark +/// from the resulting SSTables, then trigger and poll until they drain. +pub(crate) async fn checkpoint_memwal(table: &Table) -> Result<()> { for reissue in 0..=MAX_REISSUES { - // The seal turns everything written before this call into a - // generation, so the watermark has to be read after it. Idempotent: - // sealing an empty memtable is a no-op, so a re-issue does not churn - // empty generations. - match issue(|| table.flush_lsm()).await? { + // The freeze turns everything written before this call into an + // SSTable, so the watermark has to be read after it. Idempotent: + // freezing an empty memtable is a no-op, so a re-issue does not churn + // empty SSTables. + match issue(|| table.flush_memwal()).await? { Attempt::Ok(()) => {} Attempt::ReissueFromFlush => { backoff(reissue).await; @@ -165,7 +165,7 @@ pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> { } } - let stats = match issue(|| table.get_lsm_stats(false)).await? { + let stats = match issue(|| table.get_memwal_stats(false)).await? { Attempt::Ok(stats) => stats, Attempt::ReissueFromFlush => { backoff(reissue).await; @@ -173,13 +173,13 @@ pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> { } }; let Some(stats) = stats else { - // Not WAL-backed; `flush_lsm` would have errored first but for a race. + // Not WAL-backed; `flush_memwal` would have errored first but for a race. return Ok(()); }; let targets: HashMap = stats - .buckets + .table_shards .iter() - .filter_map(|b| Some((b.shard_id.clone(), b.newest_generation()?))) + .filter_map(|b| Some((b.shard_id.clone(), b.newest_sstable_generation()?))) .collect(); if targets.is_empty() { return Ok(()); @@ -194,18 +194,18 @@ pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> { } } Err(Error::Runtime { - message: "checkpoint_lsm: the owning node kept losing its claim; \ + message: "checkpoint_memwal: the owning node kept losing its claim; \ re-issued from flush the maximum number of times" .into(), }) } -/// Trigger and poll until no bucket holds a generation at or below its +/// Trigger and poll until no table shard holds an SSTable generation at or below its /// target. /// /// No liveness bound, deliberately. The pod-wide compactor pool (a semaphore /// of 2 by default, shared across every table on the node) is taken *inside* -/// the pass, after the bucket latch, so a checkpoint queued behind unrelated +/// the pass, after the table shard latch, so a checkpoint queued behind unrelated /// tables is indistinguishable from one that is merging. An idle-poll counter /// here could only ever have fired on a table that would have finished. async fn drain_to_targets( @@ -213,24 +213,24 @@ async fn drain_to_targets( targets: &HashMap, ) -> Result { loop { - let stats = match issue(|| table.get_lsm_stats(false)).await? { + let stats = match issue(|| table.get_memwal_stats(false)).await? { Attempt::Ok(stats) => stats, Attempt::ReissueFromFlush => return Ok(CheckpointOutcome::ReissueFromFlush), }; let Some(stats) = stats else { return Ok(CheckpointOutcome::Done); }; - // `compacting` is the bucket's compaction latch, held from dispatch + // `compacting` is the table shard's compaction latch, held from dispatch // until the pass ends — including while it waits on the pod-wide - // permit. So it answers one question only: do not pile on. Buckets - // with nothing outstanding are skipped, not counted as idle. + // permit. So it answers one question only: do not pile on. Table + // shards with nothing outstanding are skipped, not counted as idle. let mut outstanding = 0; let mut all_compacting = true; - for b in &stats.buckets { + for b in &stats.table_shards { let Some(target) = targets.get(&b.shard_id) else { continue; }; - let n = b.outstanding_generations(*target); + let n = b.outstanding_sstables(*target); if n > 0 { outstanding += n; all_compacting &= b.compacting; @@ -241,11 +241,11 @@ async fn drain_to_targets( } if !all_compacting { - match table.compact_lsm().await { + match table.compact_memwal().await { Ok(()) => {} Err(e) if is_lost_claim(&e) => return Ok(CheckpointOutcome::ReissueFromFlush), Err(e) if !is_retryable(&e) => return Err(e), - // A 429 here means the server could latch no bucket at all, + // A 429 here means the server could latch no table shard at all, // which the poll above already handles. Not retried in place: // the latch it would contend for is the one doing the work, so // fall through and re-read — `POLL_INTERVAL` is the backoff. diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index 88c47992d..563f185b6 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -2197,7 +2197,7 @@ pub(crate) fn plan(schema: SchemaRef, columns: &[(String, String)]) -> Result, schema: Arc, fts_query: Option, - use_lsm: Option, + use_memwal: Option, } impl BaseTableAdapter { @@ -182,13 +182,13 @@ impl BaseTableAdapter { table, schema: Arc::new(schema), fts_query: None, - use_lsm: None, + use_memwal: None, }) } - /// Set the MemWAL read routing of every scan; see [`QueryRequest::use_lsm`]. - pub fn with_use_lsm(mut self, use_lsm: Option) -> Self { - self.use_lsm = use_lsm; + /// Set the MemWAL read routing of every scan; see [`QueryRequest::use_memwal`]. + pub fn with_use_memwal(mut self, use_memwal: Option) -> Self { + self.use_memwal = use_memwal; self } @@ -204,7 +204,7 @@ impl BaseTableAdapter { table: self.table.clone(), schema, fts_query: Some(fts_query), - use_lsm: self.use_lsm, + use_memwal: self.use_memwal, } } } @@ -232,7 +232,7 @@ impl TableProvider for BaseTableAdapter { let mut query = QueryRequest { full_text_search: self.fts_query.clone(), disable_scoring_autoprojection: disable_scoring, - use_lsm: self.use_lsm, + use_memwal: self.use_memwal, ..Default::default() }; diff --git a/rust/lancedb/src/table/lsm_stats.rs b/rust/lancedb/src/table/memwal_stats.rs similarity index 56% rename from rust/lancedb/src/table/lsm_stats.rs rename to rust/lancedb/src/table/memwal_stats.rs index 953aea90f..d9747d257 100644 --- a/rust/lancedb/src/table/lsm_stats.rs +++ b/rust/lancedb/src/table/memwal_stats.rs @@ -1,21 +1,21 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Live per-bucket LSM state — the shape [`crate::Table::get_lsm_stats`] +//! Live per-table-shard MemWAL state — the shape [`crate::Table::get_memwal_stats`] //! returns and [`super::checkpoint`] polls. //! -//! Nothing here is derived: sums and differences (total L0 bytes, WAL lag) +//! Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) //! are the caller's to compute. There is no "WAL is off" shape — that case is //! `None`, because a struct of zeros would read as measurements. use serde::Deserialize; -/// One flushed L0 generation. +/// One SSTable. #[derive(Debug, Clone, Deserialize)] -pub struct GenerationStats { +pub struct SsTableStats { pub generation: u64, pub bytes: u64, - /// Present only when `include_generation_rows` was requested. Off by + /// Present only when `include_sstable_rows` was requested. Off by /// default because each count opens an uncached Lance dataset, and the /// checkpoint loop polls this route needing only generation numbers. #[serde(default)] @@ -34,11 +34,11 @@ pub struct MemtableStats { pub indexes: Vec, } -/// Live state of one bucket. A table is N buckets on one node; flattening to -/// a single number hides the one hot bucket that is usually why someone +/// Live state of one table shard. A table is N table shards on one node; flattening to +/// a single number hides the one hot table shard that is usually why someone /// opened this endpoint. #[derive(Debug, Clone, Deserialize)] -pub struct BucketStats { +pub struct TableShardStats { pub shard_id: String, /// `Active` | `Sealed` (drop-table 2PC in flight). pub status: String, @@ -47,68 +47,68 @@ pub struct BucketStats { pub current_generation: u64, pub replay_after_wal_entry_position: u64, pub wal_entry_position_last_seen: u64, - pub generations: Vec, - /// Whether a pass owns this bucket's compaction latch right now. Says *a* + pub sstables: Vec, + /// Whether a pass owns this table shard's compaction latch right now. Says *a* /// driver is running, not *whose*, and the latch is held from dispatch — /// including while the pass queues for a pod-wide compactor permit. Read /// it as "do not pile on", never as "mine is progressing". pub compacting: bool, - /// Oldest first, active last. Absent for a `Sealed` bucket, whose + /// Oldest first, active last. Absent for a `Sealed` table shard, whose /// in-memory state is torn down. #[serde(default)] pub memtables: Option>, } -impl BucketStats { - /// The newest flushed generation, or `None` when L0 is empty. - pub(crate) fn newest_generation(&self) -> Option { - self.generations.iter().map(|g| g.generation).max() +impl TableShardStats { + /// The newest SSTable generation, or `None` when the tier is empty. + pub(crate) fn newest_sstable_generation(&self) -> Option { + self.sstables.iter().map(|g| g.generation).max() } - /// How many generations at or below `target` are still in L0. + /// How many SSTables at or below `target` are still uncompacted. /// /// A count, not a boolean: one pass drains a bounded prefix rather than /// the whole target set, so a boolean would read as "no progress" for /// every pass but the last. Compaction drains oldest-first, so this /// decreases monotonically. - pub(crate) fn outstanding_generations(&self, target: u64) -> usize { - self.generations + pub(crate) fn outstanding_sstables(&self, target: u64) -> usize { + self.sstables .iter() .filter(|g| g.generation <= target) .count() } } -/// Live LSM state, one entry per bucket. +/// Live MemWAL state, one entry per table shard. #[derive(Debug, Clone, Deserialize)] -pub struct LsmStats { - pub buckets: Vec, +pub struct MemWalStats { + pub table_shards: Vec, } -/// Server-side JSON envelope for `get_lsm_stats`. `lsm_stats` is null when -/// the table has no LSM write path. +/// Server-side JSON envelope for `get_memwal_stats`. `memwal_stats` is null when +/// the table has no MemWAL write path. #[derive(Debug, Deserialize)] -pub(crate) struct GetLsmStatsResponse { +pub(crate) struct GetMemWalStatsResponse { #[serde(default)] - pub lsm_stats: Option, + pub memwal_stats: Option, } #[cfg(test)] mod tests { use super::*; - fn bucket(shard: &str, generations: &[u64], compacting: bool) -> BucketStats { - BucketStats { + fn table_shard(shard: &str, sstables: &[u64], compacting: bool) -> TableShardStats { + TableShardStats { shard_id: shard.into(), status: "Active".into(), writer_epoch: 1, manifest_version: 1, - current_generation: generations.iter().max().copied().unwrap_or(0) + 1, + current_generation: sstables.iter().max().copied().unwrap_or(0) + 1, replay_after_wal_entry_position: 0, wal_entry_position_last_seen: 0, - generations: generations + sstables: sstables .iter() - .map(|g| GenerationStats { + .map(|g| SsTableStats { generation: *g, bytes: 1, rows: None, @@ -123,40 +123,46 @@ mod tests { /// generation created after it must not hold the loop open — that is why /// the predicate terminates under write load. #[test] - fn newer_generations_do_not_extend_the_target() { - let start = bucket("b0", &[7, 8], false); - let target = start.newest_generation().expect("L0 is non-empty"); + fn newer_sstables_do_not_extend_the_target() { + let start = table_shard("b0", &[7, 8], false); + let target = start + .newest_sstable_generation() + .expect("the SSTable tier is non-empty"); assert_eq!(target, 8); // Compaction drained 7 and 8; 9 and 10 arrived while it ran. - let later = bucket("b0", &[9, 10], false); + let later = table_shard("b0", &[9, 10], false); assert_eq!( - later.outstanding_generations(target), + later.outstanding_sstables(target), 0, - "generations above the target are somebody else's problem" + "sstables above the target are somebody else's problem" ); // Still holding 8 means still outstanding. assert_eq!( - bucket("b0", &[8, 9], false).outstanding_generations(target), + table_shard("b0", &[8, 9], false).outstanding_sstables(target), 1 ); } - /// The metric counts generations, not buckets: a pass drains a bounded - /// prefix, so one bucket going 3 → 2 → 1 → 0 is three steps. + /// The metric counts SSTables, not table shards: a pass drains a bounded + /// prefix, so one table shard going 3 → 2 → 1 → 0 is three steps. #[test] - fn progress_is_measured_in_generations() { + fn progress_is_measured_in_sstables() { let target = 3; let counts: Vec = [&[1u64, 2, 3][..], &[2, 3][..], &[3][..], &[][..]] .iter() - .map(|gens| bucket("b0", gens, false).outstanding_generations(target)) + .map(|gens| table_shard("b0", gens, false).outstanding_sstables(target)) .collect(); assert_eq!(counts, vec![3, 2, 1, 0]); } #[test] - fn empty_l0_has_no_target() { - assert!(bucket("b0", &[], false).newest_generation().is_none()); + fn an_empty_sstable_tier_has_no_target() { + assert!( + table_shard("b0", &[], false) + .newest_sstable_generation() + .is_none() + ); } } diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index c74980c5e..66ec48baf 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -47,7 +47,7 @@ pub struct MergeResult { /// Total number of rows written. /// /// On the standard `merge_insert` path this equals - /// `num_inserted_rows + num_updated_rows`. On the MemWAL LSM write path the + /// `num_inserted_rows + num_updated_rows`. On the MemWAL write path the /// insert/update breakdown is not known until compaction; in that mode /// `num_inserted_rows`, `num_updated_rows`, `num_deleted_rows`, `version` /// and `num_attempts` are all `0` and this field holds the total number of @@ -76,7 +76,7 @@ pub struct MergeInsertBuilder { pub(crate) when_not_matched_by_source_delete_filt: Option, pub(crate) timeout: Option, pub(crate) use_index: bool, - pub(crate) use_lsm: Option, + pub(crate) use_memwal: Option, pub(crate) validate_single_shard: bool, } @@ -92,7 +92,7 @@ impl MergeInsertBuilder { when_not_matched_by_source_delete_filt: None, timeout: None, use_index: true, - use_lsm: None, + use_memwal: None, validate_single_shard: true, } } @@ -193,27 +193,27 @@ impl MergeInsertBuilder { /// Control MemWAL routing for this `merge_insert`. /// /// By default (unset), a `merge_insert` on a table with an - /// [`LsmWriteSpec`](super::LsmWriteSpec) installed is routed through Lance's + /// [`MemWalWriteSpec`](super::MemWalWriteSpec) installed is routed through Lance's /// MemWAL shard writer; a table without one uses the standard path. /// - /// - `use_lsm(true)` forces MemWAL routing and errors if the table has no - /// LSM write spec. - /// - `use_lsm(false)` forces the standard write path even when a spec is set. - pub fn use_lsm(&mut self, enable: bool) -> &mut Self { - self.use_lsm = Some(enable); + /// - `use_memwal(true)` forces MemWAL routing and errors if the table has no + /// MemWAL write spec. + /// - `use_memwal(false)` forces the standard write path even when a spec is set. + pub fn use_memwal(&mut self, enable: bool) -> &mut Self { + self.use_memwal = Some(enable); self } /// Controls how an LSM `merge_insert` checks that its input targets a /// single shard. /// - /// When a table has an LSM write spec, every row in a `merge_insert` call + /// When a table has a MemWAL write spec, every row in a `merge_insert` call /// must route to the same shard. When `true` (the default), every row is /// inspected to verify this. When `false`, only the first row is inspected /// and the shard it routes to is used for the whole input — a faster path /// for callers that have already pre-sharded their input. /// - /// Has no effect on tables without an LSM write spec. + /// Has no effect on tables without a MemWAL write spec. pub fn validate_single_shard(&mut self, validate_single_shard: bool) -> &mut Self { self.validate_single_shard = validate_single_shard; self @@ -890,7 +890,7 @@ mod lsm_tests { use crate::connect; use crate::error::Error; - use crate::table::{LsmWriteSpec, Table}; + use crate::table::{MemWalWriteSpec, Table}; /// A reader of `[id: Int64, value: Int64]` rows; `value` is `0..n`. fn id_value_reader(ids: Vec) -> Box { @@ -974,7 +974,7 @@ mod lsm_tests { let table = id_value_table(&dir).await; // num_buckets = 1: every row routes to the single bucket. table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 1)) + .enable_memwal(MemWalWriteSpec::bucket("id", 1)) .await .unwrap(); @@ -1001,7 +1001,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -1030,7 +1030,7 @@ mod lsm_tests { .unwrap(); table.set_unenforced_primary_key(["id"]).await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::identity("region")) + .enable_memwal(MemWalWriteSpec::identity("region")) .await .unwrap(); @@ -1047,18 +1047,18 @@ mod lsm_tests { } #[tokio::test] - async fn lsm_merge_insert_use_lsm_false_falls_back() { + async fn lsm_merge_insert_use_memwal_false_falls_back() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 1)) + .enable_memwal(MemWalWriteSpec::bucket("id", 1)) .await .unwrap(); - // use_lsm(false) opts out: the standard path runs and commits even though + // use_memwal(false) opts out: the standard path runs and commits even though // a spec is installed. let mut builder = table.merge_insert(&["id"]); - builder.when_not_matched_insert_all().use_lsm(false); + builder.when_not_matched_insert_all().use_memwal(false); let result = builder .execute(id_value_reader(vec![3, 4, 5])) .await @@ -1069,17 +1069,17 @@ mod lsm_tests { } #[tokio::test] - async fn lsm_merge_insert_use_lsm_true_without_spec_errors() { + async fn lsm_merge_insert_use_memwal_true_without_spec_errors() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; - // use_lsm(true) demands MemWAL routing; without a write spec it errors + // use_memwal(true) demands MemWAL routing; without a write spec it errors // rather than silently falling back to the standard path. let mut builder = table.merge_insert(&["id"]); builder .when_matched_update_all(None) .when_not_matched_insert_all() - .use_lsm(true); + .use_memwal(true); let err = builder .execute(id_value_reader(vec![3, 4, 5])) .await @@ -1092,7 +1092,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 1)) + .enable_memwal(MemWalWriteSpec::bucket("id", 1)) .await .unwrap(); @@ -1109,7 +1109,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 1)) + .enable_memwal(MemWalWriteSpec::bucket("id", 1)) .await .unwrap(); @@ -1125,7 +1125,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 1)) + .enable_memwal(MemWalWriteSpec::bucket("id", 1)) .await .unwrap(); @@ -1135,7 +1135,7 @@ mod lsm_tests { .when_not_matched_insert_all(); builder.execute(id_value_reader(vec![7, 8])).await.unwrap(); - table.close_lsm_writers().await.unwrap(); + table.close_memwal_writers().await.unwrap(); // The writer reopens lazily on the next merge_insert. let mut builder = table.merge_insert(&[]); @@ -1160,7 +1160,7 @@ mod lsm_tests { .unwrap(); table.set_unenforced_primary_key(["id"]).await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::identity("region")) + .enable_memwal(MemWalWriteSpec::identity("region")) .await .unwrap(); @@ -1197,10 +1197,10 @@ mod lsm_tests { #[tokio::test] async fn lsm_merge_insert_no_spec_uses_standard_path() { let dir = tempdir().unwrap(); - // id_value_table sets a primary key but no LSM write spec. + // id_value_table sets a primary key but no MemWAL write spec. let table = id_value_table(&dir).await; - // Without a spec, a default merge_insert (use_lsm unset) simply uses + // Without a spec, a default merge_insert (use_memwal unset) simply uses // the standard path and commits — no opt-out required, no error. let mut builder = table.merge_insert(&["id"]); builder @@ -1228,7 +1228,7 @@ mod lsm_tests { .unwrap(); table.set_unenforced_primary_key(["id"]).await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::identity("region")) + .enable_memwal(MemWalWriteSpec::identity("region")) .await .unwrap(); @@ -1254,7 +1254,7 @@ mod lsm_tests { assert!(matches!(err, Error::InvalidInput { .. }), "got {err:?}"); // After closing the writer, a different shard can be written. - table.close_lsm_writers().await.unwrap(); + table.close_memwal_writers().await.unwrap(); let mut builder = table.merge_insert(&[]); builder .when_matched_update_all(None) @@ -1266,7 +1266,7 @@ mod lsm_tests { } // --------------------------------------------------------------------- - // LSM read path + // MemWAL read path // --------------------------------------------------------------------- use crate::arrow::SendableRecordBatchStream; @@ -1325,7 +1325,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; // base: ids 1,2,3 (value 0,1,2) table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -1340,8 +1340,8 @@ mod lsm_tests { vec![1, 2, 3, 4, 5] ); - // use_lsm(false) bypasses the MemWAL and reads the base table only. - let base_only = table.query().use_lsm(false).execute().await.unwrap(); + // use_memwal(false) bypasses the MemWAL and reads the base table only. + let base_only = table.query().use_memwal(false).execute().await.unwrap(); let rows = collect_id_value(base_only).await; assert_eq!( rows.iter().map(|(id, _)| *id).collect::>(), @@ -1350,14 +1350,14 @@ mod lsm_tests { } #[tokio::test] - async fn base_table_adapter_use_lsm_false_scans_the_base_only() { + async fn base_table_adapter_use_memwal_false_scans_the_base_only() { use crate::table::datafusion::BaseTableAdapter; use datafusion::prelude::SessionContext; let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; // base: ids 1,2,3 table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); // Ids 4,5 land in the active memtable, not the base table. @@ -1366,7 +1366,7 @@ mod lsm_tests { let adapter = BaseTableAdapter::try_new(table.base_table().clone()) .await .unwrap() - .with_use_lsm(Some(false)); + .with_use_memwal(Some(false)); let ctx = SessionContext::new(); ctx.register_table("base_only", Arc::new(adapter)).unwrap(); let batches = ctx @@ -1389,7 +1389,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); lsm_upsert(&table, vec![4, 5]).await; @@ -1407,7 +1407,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); lsm_upsert(&table, vec![4]).await; @@ -1427,7 +1427,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; // base: id 2 -> value 1 table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -1445,7 +1445,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); lsm_upsert(&table, vec![2, 3, 4]).await; // id 2 -> value 0 (shadows base) @@ -1460,14 +1460,14 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::bucket("id", 8)) + .enable_memwal(MemWalWriteSpec::bucket("id", 8)) .await .unwrap(); - // Two single-row upserts that route to (likely) different buckets; each + // Two single-row upserts that route to (likely) different shards; each // closes the writer so the next opens a fresh shard. lsm_upsert(&table, vec![10]).await; - table.close_lsm_writers().await.unwrap(); + table.close_memwal_writers().await.unwrap(); lsm_upsert(&table, vec![11]).await; let lsm = table.query().execute().await.unwrap(); @@ -1482,14 +1482,14 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); lsm_upsert(&table, vec![4, 5]).await; // close flushes the active memtable to an on-disk generation and drops // the cached writer; the read must still see those rows via the shard // manifest snapshot. - table.close_lsm_writers().await.unwrap(); + table.close_memwal_writers().await.unwrap(); let lsm = table.query().execute().await.unwrap(); let ids: Vec = collect_id_value(lsm) @@ -1503,11 +1503,11 @@ mod lsm_tests { #[tokio::test] async fn lsm_read_without_spec_reads_base() { let dir = tempdir().unwrap(); - let table = id_value_table(&dir).await; // no LSM write spec + let table = id_value_table(&dir).await; // no MemWAL write spec // With no spec installed there is nothing to route: the default read and - // an explicit use_lsm(false) both read the base table without error. - for query in [table.query(), table.query().use_lsm(false)] { + // an explicit use_memwal(false) both read the base table without error. + for query in [table.query(), table.query().use_memwal(false)] { let rows = collect_id_value(query.execute().await.unwrap()).await; assert_eq!( rows.iter().map(|(id, _)| *id).collect::>(), @@ -1517,11 +1517,11 @@ mod lsm_tests { } #[tokio::test] - async fn lsm_read_unsupported_shape_errors_without_use_lsm_false() { + async fn lsm_read_unsupported_shape_errors_without_use_memwal_false() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); lsm_upsert(&table, vec![4]).await; @@ -1538,12 +1538,12 @@ mod lsm_tests { .expect("unsupported shape on a MemWAL table must error"); assert!(matches!(err, Error::NotSupported { .. }), "got {err:?}"); - // use_lsm(false) is the escape hatch: it reads the base table only. + // use_memwal(false) is the escape hatch: it reads the base table only. let rows = collect_id_value( table .query() .with_row_id() - .use_lsm(false) + .use_memwal(false) .execute() .await .unwrap(), @@ -1555,7 +1555,7 @@ mod lsm_tests { ); } - /// A table with `id`/`text` and a primary key, ready for an LSM write spec. + /// A table with `id`/`text` and a primary key, ready for a MemWAL write spec. async fn lsm_text_table(dir: &tempfile::TempDir) -> crate::Table { let conn = connect(dir.path().to_str().unwrap()) .execute() @@ -1570,7 +1570,7 @@ mod lsm_tests { table } - /// Upsert `rows` through the LSM write path. + /// Upsert `rows` through the MemWAL write path. async fn upsert_text(table: &crate::Table, rows: Vec<(i64, &str)>) { let mut builder = table.merge_insert(&[]); builder @@ -1614,7 +1614,7 @@ mod lsm_tests { let table = lsm_text_table(&dir).await; // The default set, on a table with no FTS index yet. table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -1648,7 +1648,7 @@ mod lsm_tests { /// A spec is replaced by unsetting it and setting the new one. /// - /// `unset_lsm_write_spec` drains the open writer, so the rows it held + /// `disable_memwal` drains the open writer, so the rows it held /// survive the replacement, and the writer the next write opens builds its /// MemTables from the new set. #[tokio::test] @@ -1668,15 +1668,15 @@ mod lsm_tests { // Installed maintaining nothing, then written to: the writer opens with // no FTS index. table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new())) + .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(Vec::new())) .await .unwrap(); upsert_text(&table, vec![(99, "zebra")]).await; // Setting over an installed spec is refused; unset, then set. let err = table - .set_lsm_write_spec( - LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index.clone()]), + .enable_memwal( + MemWalWriteSpec::unsharded().with_maintained_indexes(vec![fts_index.clone()]), ) .await .expect_err("an installed spec cannot be set over"); @@ -1684,9 +1684,9 @@ mod lsm_tests { err.to_string().contains("already set"), "unexpected error: {err}" ); - table.unset_lsm_write_spec().await.unwrap(); + table.disable_memwal().await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) + .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) .await .unwrap(); @@ -1716,7 +1716,7 @@ mod lsm_tests { /// A row written before the index exists is searchable once it does. /// /// The MemTable holding it was built without the index. Creating the index - /// replaces the writer's configs, which seals that MemTable and waits for + /// replaces the writer's configs, which freezes that MemTable and waits for /// its flush, so no resident MemTable is left that cannot answer the read. #[tokio::test] async fn lsm_a_row_written_before_the_index_is_searchable_after_it() { @@ -1727,7 +1727,7 @@ mod lsm_tests { let table = lsm_text_table(&dir).await; // No index yet, and the default set: maintain whatever the table has. table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -1783,7 +1783,7 @@ mod lsm_tests { .unwrap(); let fts_index = table.list_indices().await.unwrap()[0].name.clone(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) + .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) .await .unwrap(); @@ -1878,7 +1878,7 @@ mod lsm_tests { } let vec_index = table.list_indices().await.unwrap()[0].name.clone(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![vec_index])) + .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(vec![vec_index])) .await .unwrap(); @@ -2129,8 +2129,8 @@ mod lsm_tests { } /// A kind Lance has no memtable index for is refused when named until the - /// table's registry maintains it; then it is maintained, flushed with each - /// generation, and answers searches on the flushed generations. + /// table's registry maintains it; then it is maintained, flushed into each + /// SSTable, and answers searches on the SSTables. #[tokio::test] async fn lsm_maintains_an_index_kind_from_the_tables_registry() { let dir = tempdir().unwrap(); @@ -2150,10 +2150,11 @@ mod lsm_tests { .execute() .await .unwrap(); - let spec = - || LsmWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]); + let spec = || { + MemWalWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]) + }; - let refused = table.set_lsm_write_spec(spec()).await.unwrap_err(); + let refused = table.enable_memwal(spec()).await.unwrap_err(); assert!( refused.to_string().contains("region_bitmap") && refused @@ -2166,7 +2167,7 @@ mod lsm_tests { .with_plugin(Arc::new(BitmapAsBTree)) .unwrap(); table.as_native().unwrap().set_mem_index_registry(registry); - table.set_lsm_write_spec(spec()).await.unwrap(); + table.enable_memwal(spec()).await.unwrap(); // The registry is not stored with the table. let other = conn.open_table("t").execute().await.unwrap(); @@ -2224,13 +2225,13 @@ mod lsm_tests { active.index_store.index_names() ); - table.close_lsm_writers().await.unwrap(); + table.close_memwal_writers().await.unwrap(); assert_eq!(fresh().await, vec![10, 11]); - assert_generations_answer_through_region_bitmap(&table, vec![10, 11]).await; + assert_sstables_answer_through_region_bitmap(&table, vec![10, 11]).await; } - /// The generations each shard's manifest publishes. - async fn published_generations(table: &Table) -> Vec { + /// The SSTables each shard's manifest publishes. + async fn published_sstables(table: &Table) -> Vec { use lance::dataset::mem_wal::ShardManifestStore; let uri = table.uri().await.unwrap(); @@ -2238,7 +2239,7 @@ mod lsm_tests { .await .unwrap(); let mem_wal = std::path::Path::new(&uri).join("_mem_wal"); - let mut generations = Vec::new(); + let mut sstables = Vec::new(); for shard in std::fs::read_dir(&mem_wal).unwrap() { let Ok(shard_id) = shard.unwrap().file_name().to_string_lossy().parse() else { continue; @@ -2250,30 +2251,30 @@ mod lsm_tests { .expect("a shard directory has a manifest"); for sstable in manifest.sstables { let path = mem_wal.join(shard_id.to_string()).join(&sstable.path); - generations.push(lance::Dataset::open(path.to_str().unwrap()).await.unwrap()); + sstables.push(lance::Dataset::open(path.to_str().unwrap()).await.unwrap()); } } - generations + sstables } - /// Every published generation carries `region_bitmap` as a bitmap index on + /// Every published SSTable carries `region_bitmap` as a bitmap index on /// `region`, and a search for `region = 'fresh'` on it uses that index. - async fn assert_generations_answer_through_region_bitmap(table: &Table, fresh: Vec) { + async fn assert_sstables_answer_through_region_bitmap(table: &Table, fresh: Vec) { use lance::index::DatasetIndexExt; - let generations = published_generations(table).await; + let sstables = published_sstables(table).await; assert!( - !generations.is_empty(), - "closing the writer flushed a generation" + !sstables.is_empty(), + "closing the writer flushed an SSTable" ); let mut found = Vec::new(); - for generation in generations { - let region = generation.schema().field("region").unwrap().id; - let indices = generation.load_indices().await.unwrap(); + for sstable in sstables { + let region = sstable.schema().field("region").unwrap().id; + let indices = sstable.load_indices().await.unwrap(); let index = indices .iter() .find(|index| index.name == "region_bitmap") - .expect("the generation carries the index the plugin trained"); + .expect("the SSTable carries the index the plugin trained"); assert_eq!(index.fields, vec![region]); assert!( index @@ -2284,7 +2285,7 @@ mod lsm_tests { index.index_details ); - let mut scan = generation.scan(); + let mut scan = sstable.scan(); scan.filter("region = 'fresh'").unwrap(); let plan = scan.explain_plan(false).await.unwrap(); assert!(plan.contains("ScalarIndexQuery"), "{plan}"); @@ -2332,7 +2333,7 @@ mod lsm_tests { "a clone shares the handle's registry" ); table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); @@ -2375,13 +2376,13 @@ mod lsm_tests { active.index_store.index_names() ); - table.close_lsm_writers().await.unwrap(); - // Only the generation written after the index exists carries it. - let generations = published_generations(&table).await; + table.close_memwal_writers().await.unwrap(); + // Only the SSTable written after the index exists carries it. + let sstables = published_sstables(&table).await; let mut carrying = 0; - for generation in &generations { + for sstable in &sstables { use lance::index::DatasetIndexExt; - if generation + if sstable .load_indices() .await .unwrap() @@ -2393,8 +2394,8 @@ mod lsm_tests { } assert!( carrying >= 1, - "{} generations, none carry the index", - generations.len() + "{} SSTables, none carry the index", + sstables.len() ); } } diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index 33bf54535..d26be28cc 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -1,10 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! MemWAL LSM write path for `merge_insert`. +//! MemWAL write path for `merge_insert`. //! -//! [`set_lsm_write_spec`] installs an [`LsmWriteSpec`] on a table by creating -//! Lance's MemWAL index; [`unset_lsm_write_spec`] removes it. Once a spec is +//! [`enable_memwal`] installs an [`MemWalWriteSpec`] on a table by creating +//! Lance's MemWAL index; [`disable_memwal`] removes it. Once a spec is //! installed, `merge_insert` upsert calls are dispatched through Lance's MemWAL //! `ShardWriter` (LSM-style append) instead of the standard merge path — see //! [`lsm_dispatch_decision`] and [`execute_lsm_merge_insert`]. @@ -14,7 +14,7 @@ //! unsharded). [`MergeInsertBuilder::validate_single_shard`] controls whether //! every row is checked or only the first. A dataset writes to one shard at a //! time; its writer is cached in the [`ShardWriterCache`] held alongside the -//! dataset, and [`close_lsm_writers`] closes it. +//! dataset, and [`close_memwal_writers`] closes it. use std::collections::HashMap; use std::sync::Arc; @@ -40,9 +40,9 @@ use uuid::Uuid; use crate::error::{Error, Result}; use crate::index::IndexConfig; use crate::table::merge::{MergeInsertBuilder, MergeResult}; -use crate::table::{BaseTable, LsmWriteSpec, NativeTable}; +use crate::table::{BaseTable, MemWalWriteSpec, NativeTable}; -/// Spec id of the sole sharding spec installed by [`set_lsm_write_spec`]. +/// Spec id of the sole sharding spec installed by [`enable_memwal`]. /// Must match Lance's `InitializeMemWalBuilder` (`SHARDING_SPEC_ID`). const SHARDING_SPEC_ID: u32 = 1; @@ -61,23 +61,23 @@ const NUM_BUCKETS_PARAM: &str = "num_buckets"; const SHARD_NAMESPACE: Uuid = Uuid::from_u128(0x4c53_4d57_5249_5445_5f53_4841_5244_3031); // ============================================================================= -// set_lsm_write_spec +// enable_memwal // ============================================================================= -/// Install an [`LsmWriteSpec`] on the table. +/// Install an [`MemWalWriteSpec`] on the table. /// /// The bucket / identity / unsharded sharding spec is constructed and validated /// by Lance's /// [`InitializeMemWalBuilder`](lance::dataset::mem_wal::InitializeMemWalBuilder). #[allow(clippy::redundant_pub_crate)] -pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) -> Result<()> { +pub(crate) async fn enable_memwal(table: &NativeTable, spec: MemWalWriteSpec) -> Result<()> { table.dataset.ensure_mutable()?; { let dataset = table.dataset.get().await?; if dataset.mem_wal_index_details().await?.is_some() { return Err(Error::InvalidInput { - message: "set_lsm_write_spec: an LSM write spec is already set on this \ + message: "enable_memwal: a MemWAL write spec is already set on this \ table and cannot be changed" .into(), }); @@ -115,14 +115,14 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) let schema = arrow_schema::Schema::from(dataset.schema()); if !crate::table::computed_columns::computed_columns(&schema).is_empty() { return Err(Error::NotSupported { - message: "an LSM write spec cannot be installed on a table with computed \ + message: "a MemWAL write spec cannot be installed on a table with computed \ columns: rows in un-compacted tiers are invisible to refresh" .into(), }); } if crate::materialized_view::read_definition(&dataset.schema().metadata)?.is_some() { return Err(Error::NotSupported { - message: "an LSM write spec cannot be installed on a materialized view: \ + message: "a MemWAL write spec cannot be installed on a materialized view: \ rows in un-compacted tiers are invisible to refresh" .into(), }); @@ -131,7 +131,7 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) .initialize_mem_wal() .mem_index_registry(table.mem_index_registry()); let writer_config_defaults = match spec { - LsmWriteSpec::Bucket { + MemWalWriteSpec::Bucket { column, num_buckets, writer_config_defaults, @@ -140,7 +140,7 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) builder = builder.bucket_sharding(column, num_buckets); writer_config_defaults } - LsmWriteSpec::Identity { + MemWalWriteSpec::Identity { column, writer_config_defaults, .. @@ -148,7 +148,7 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) builder = builder.identity_sharding(column); writer_config_defaults } - LsmWriteSpec::Unsharded { + MemWalWriteSpec::Unsharded { writer_config_defaults, .. } => { @@ -178,22 +178,22 @@ fn index_name_list(indices: &[IndexConfig]) -> String { } // ============================================================================= -// unset_lsm_write_spec +// disable_memwal // ============================================================================= -/// Remove the [`LsmWriteSpec`] from the table by dropping the MemWAL index. +/// Remove the [`MemWalWriteSpec`] from the table by dropping the MemWAL index. /// /// Any cached shard writers are drained and closed first. Errors if no spec is /// currently set. #[allow(clippy::redundant_pub_crate)] -pub(crate) async fn unset_lsm_write_spec(table: &NativeTable) -> Result<()> { +pub(crate) async fn disable_memwal(table: &NativeTable) -> Result<()> { table.dataset.ensure_mutable()?; { let dataset = table.dataset.get().await?; if dataset.mem_wal_index_details().await?.is_none() { return Err(Error::InvalidInput { - message: "unset_lsm_write_spec: no LSM write spec is set on this table".into(), + message: "disable_memwal: no MemWAL write spec is set on this table".into(), }); } } @@ -209,46 +209,46 @@ pub(crate) async fn unset_lsm_write_spec(table: &NativeTable) -> Result<()> { } // ============================================================================= -// get_lsm_write_spec +// get_memwal_spec // ============================================================================= -/// Read the [`LsmWriteSpec`] currently installed on the table, if any. +/// Read the [`MemWalWriteSpec`] currently installed on the table, if any. /// /// Reconstructs the spec from the MemWAL index's stored details — the same -/// metadata [`set_lsm_write_spec`] writes — read directly via +/// metadata [`enable_memwal`] writes — read directly via /// [`mem_wal_index_details`](DatasetMemWalExt::mem_wal_index_details) (the raw /// manifest record, not the `describe_indices` enrichment path, so it is /// unaffected by system-index filtering there). Returns `Ok(None)` when no /// MemWAL index is installed. #[allow(clippy::redundant_pub_crate)] -pub(crate) async fn get_lsm_write_spec(table: &NativeTable) -> Result> { +pub(crate) async fn get_memwal_spec(table: &NativeTable) -> Result> { let dataset = table.dataset.get().await?; let Some(details) = dataset.mem_wal_index_details().await? else { return Ok(None); }; - let spec = lsm_write_spec_from_details(&details, dataset.schema())?; + let spec = memwal_spec_from_details(&details, dataset.schema())?; Ok(Some(spec)) } -/// Reconstruct the public [`LsmWriteSpec`] from stored MemWAL index details. +/// Reconstruct the public [`MemWalWriteSpec`] from stored MemWAL index details. /// /// The sharding transform and its parameters recover the mode; the routing /// column is resolved from the sharding field's source id back to its schema /// name (only bucket / identity carry a column). `maintained_indexes` and /// `writer_config_defaults` are copied verbatim, so the result round-trips -/// what `set_lsm_write_spec` installed. -fn lsm_write_spec_from_details( +/// what `enable_memwal` installed. +fn memwal_spec_from_details( details: &MemWalIndexDetails, schema: &LanceSchema, -) -> Result { +) -> Result { let spec = details .sharding_specs .first() .ok_or_else(|| Error::Runtime { - message: "get_lsm_write_spec: MemWAL index has no sharding spec".to_string(), + message: "get_memwal_spec: MemWAL index has no sharding spec".to_string(), })?; let field = spec.fields.first().ok_or_else(|| Error::Runtime { - message: "get_lsm_write_spec: MemWAL index has an empty sharding spec".to_string(), + message: "get_memwal_spec: MemWAL index has an empty sharding spec".to_string(), })?; let base = match field.transform.as_deref() { @@ -259,16 +259,16 @@ fn lsm_write_spec_from_details( .and_then(|s| s.parse::().ok()) .filter(|n| *n > 0) .ok_or_else(|| Error::Runtime { - message: "get_lsm_write_spec: MemWAL bucket spec has a missing or invalid num_buckets parameter".to_string(), + message: "get_memwal_spec: MemWAL bucket spec has a missing or invalid num_buckets parameter".to_string(), })?; - LsmWriteSpec::bucket(sharding_column(field, schema)?, num_buckets) + MemWalWriteSpec::bucket(sharding_column(field, schema)?, num_buckets) } - Some(IDENTITY_TRANSFORM) => LsmWriteSpec::identity(sharding_column(field, schema)?), - Some(UNSHARDED_TRANSFORM) => LsmWriteSpec::unsharded(), + Some(IDENTITY_TRANSFORM) => MemWalWriteSpec::identity(sharding_column(field, schema)?), + Some(UNSHARDED_TRANSFORM) => MemWalWriteSpec::unsharded(), other => { return Err(Error::Runtime { message: format!( - "get_lsm_write_spec: MemWAL index has an unsupported sharding transform {:?}", + "get_memwal_spec: MemWAL index has an unsupported sharding transform {:?}", other ), }); @@ -284,30 +284,30 @@ fn lsm_write_spec_from_details( /// Resolve the single routing column name from a sharding field's source id. /// -/// `set_lsm_write_spec` records the shard column by its Lance field id, so the +/// `enable_memwal` records the shard column by its Lance field id, so the /// name is recovered by looking that id up in the current schema. fn sharding_column(field: &ShardingField, schema: &LanceSchema) -> Result { let source_id = *field.source_ids.first().ok_or_else(|| Error::Runtime { - message: "get_lsm_write_spec: sharding field has no source column".to_string(), + message: "get_memwal_spec: sharding field has no source column".to_string(), })?; schema .field_by_id(source_id) .map(|f| f.name.clone()) .ok_or_else(|| Error::Runtime { message: format!( - "get_lsm_write_spec: sharding source field id {} not found in schema", + "get_memwal_spec: sharding source field id {} not found in schema", source_id ), }) } // ============================================================================= -// close_lsm_writers +// close_memwal_writers // ============================================================================= /// Drain and close every cached MemWAL shard writer for the table. #[allow(clippy::redundant_pub_crate)] -pub(crate) async fn close_lsm_writers(table: &NativeTable) -> Result<()> { +pub(crate) async fn close_memwal_writers(table: &NativeTable) -> Result<()> { table.dataset.shard_writer().drain_and_close().await } @@ -321,7 +321,7 @@ pub(crate) async fn close_lsm_writers(table: &NativeTable) -> Result<()> { /// so the writer lives where the dataset lives — cached for the session and /// reused across `merge_insert` calls. A dataset writes to one shard at a /// time; routing a `merge_insert` to a different shard requires closing the -/// current writer first via [`close_lsm_writers`]. `ShardWriter::put` takes +/// current writer first via [`close_memwal_writers`]. `ShardWriter::put` takes /// `&self`, so concurrent puts on the cached writer are safe; `close` consumes /// the writer, so the entry wraps it in `RwLock>`. #[derive(Default)] @@ -370,8 +370,8 @@ impl ShardWriterEntry { } /// The cached writer's latest in-memory manifest (current generation + - /// flushed generations). `Ok(None)` if the writer was already closed. - /// Used by the LSM read path to snapshot this shard authoritatively + /// SSTables). `Ok(None)` if the writer was already closed. + /// Used by the MemWAL read path to snapshot this shard authoritatively /// without re-reading the on-disk manifest. async fn manifest(&self) -> Result> { let guard = self.inner.read().await; @@ -449,7 +449,7 @@ impl ShardWriterCache { /// writer opened afterwards reads the committed set. /// /// Holds the entry's read lock across the refresh, so a concurrent close - /// waits rather than retiring the writer mid-seal. Writes take the same + /// waits rather than retiring the writer mid-freeze. Writes take the same /// lock first, so the order matches theirs. pub(crate) async fn refresh_maintained_indexes(&self, dataset: &Dataset) { let entry = { @@ -463,13 +463,13 @@ impl ShardWriterCache { let Some(writer) = guard.as_ref() else { return; }; - // Replacing the configs seals the active memtable so the next one carries + // Replacing the configs freezes the active memtable so the next one carries // the new indexes. The outgoing one keeps the set it was built with, so // its flush has to land before a read: while it is still frozen, an // indexed read sees a resident memtable that cannot answer and is // refused. - let sealed = match dataset.refresh_mem_wal_index_configs(writer).await { - Ok(sealed) => sealed, + let frozen = match dataset.refresh_mem_wal_index_configs(writer).await { + Ok(frozen) => frozen, Err(error) => { log::warn!( "the maintained index set changed but the open shard writer could \ @@ -478,18 +478,18 @@ impl ShardWriterCache { return; } }; - if let Some(fence) = sealed + if let Some(fence) = frozen && let Err(error) = fence.wait().await { log::warn!( "the shard writer took the new maintained index set, but the memtable \ - it sealed was not flushed, so an indexed read stays refused until \ + it froze was not flushed, so an indexed read stays refused until \ it is: {error}" ); } } - /// Snapshot the cached writer's shard for the LSM read path: its shard id, + /// Snapshot the cached writer's shard for the MemWAL read path: its shard id, /// authoritative in-memory manifest, and active + frozen memtable refs. /// Returns `None` when no writer is currently cached (e.g. nothing has been /// written this session, or the writer was closed). @@ -537,7 +537,7 @@ fn check_shard_match(cached: Uuid, wanted: Uuid) -> Result<()> { } Err(Error::InvalidInput { message: format!( - "merge_insert: a shard writer is already open for shard {} but this input routes to shard {}; call close_lsm_writers before writing to a different shard", + "merge_insert: a shard writer is already open for shard {} but this input routes to shard {}; call close_memwal_writers before writing to a different shard", cached, wanted ), }) @@ -569,7 +569,7 @@ pub(crate) struct LsmPlan { /// Outcome of [`lsm_dispatch_decision`]. #[allow(clippy::redundant_pub_crate)] pub(crate) enum LsmDispatch { - /// No LSM write spec applies; use the standard `merge_insert` path. + /// No MemWAL write spec applies; use the standard `merge_insert` path. Standard, /// Route the `merge_insert` through the MemWAL shard writer. Lsm(LsmPlan), @@ -583,18 +583,18 @@ pub(crate) async fn lsm_dispatch_decision( params: &MergeInsertBuilder, ) -> Result { // Explicit opt-out: use the standard path regardless of any installed spec. - if params.use_lsm == Some(false) { + if params.use_memwal == Some(false) { return Ok(LsmDispatch::Standard); } let dataset = table.dataset.get().await?; let Some(details) = dataset.mem_wal_index_details().await? else { - // No write spec installed. `use_lsm(true)` demanded MemWAL routing, so + // No write spec installed. `use_memwal(true)` demanded MemWAL routing, so // that is an error; otherwise fall back to the standard path. - if params.use_lsm == Some(true) { + if params.use_memwal == Some(true) { return Err(Error::InvalidInput { - message: "use_lsm(true) was set but the table has no MemWAL write spec; \ - install one with set_lsm_write_spec or leave use_lsm unset" + message: "use_memwal(true) was set but the table has no MemWAL write spec; \ + install one with enable_memwal or leave use_memwal unset" .to_string(), }); } @@ -616,7 +616,7 @@ pub(crate) async fn lsm_dispatch_decision( if !params.on.is_empty() && params.on != pk_cols { return Err(Error::InvalidInput { message: format!( - "merge_insert: `on` columns {:?} must match the table's unenforced primary key {:?} when an LSM write spec is set; pass an empty `on` to default to the primary key", + "merge_insert: `on` columns {:?} must match the table's unenforced primary key {:?} when a MemWAL write spec is set; pass an empty `on` to default to the primary key", params.on, pk_cols ), }); @@ -624,7 +624,7 @@ pub(crate) async fn lsm_dispatch_decision( if !is_upsert_only(params) { return Err(Error::InvalidInput { - message: "merge_insert: when an LSM write spec is set, only the upsert form (when_matched_update_all without a filter + when_not_matched_insert_all, no by-source delete) is supported; call use_lsm(false) to use the standard merge_insert path".to_string(), + message: "merge_insert: when a MemWAL write spec is set, only the upsert form (when_matched_update_all without a filter + when_not_matched_insert_all, no by-source delete) is supported; call use_memwal(false) to use the standard merge_insert path".to_string(), }); } diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index 3ab54ae07..90a1d3217 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -95,8 +95,8 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R return Ok(false); } // A MemWAL write spec means reads auto-route through the LSM scanner in - // `create_plan` even when `use_lsm` is unset. The namespace request has no - // use_lsm field, so pushing the default query down would silently omit + // `create_plan` even when `use_memwal` is unset. The namespace request has no + // use_memwal field, so pushing the default query down would silently omit // un-compacted rows — force local execution whenever a spec is installed. let dataset = table.dataset.get().await?; if dataset.mem_wal_index_details().await?.is_some() { @@ -106,10 +106,10 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R } fn requires_local_namespace_execution(query: &AnyQuery) -> bool { - // The namespace QueryTable request cannot represent approx_mode, search_effort or use_lsm. - // For use_lsm that is worse than a tuning miss: MemWAL read routing lives only + // The namespace QueryTable request cannot represent approx_mode, search_effort or use_memwal. + // For use_memwal that is worse than a tuning miss: MemWAL read routing lives only // in `create_plan`, so a pushed-down query would return stale base-only data. - if query.base().use_lsm.is_some() || query.base().take_offsets.is_some() { + if query.base().use_memwal.is_some() || query.base().take_offsets.is_some() { return true; } matches!( @@ -167,25 +167,25 @@ pub async fn create_plan( let ds_ref = table.dataset.get().await?; - // MemWAL read routing driven by `use_lsm`: + // MemWAL read routing driven by `use_memwal`: // * unset — route through the LSM scanner iff the table carries a write spec - // * Some(true) — force LSM routing; error if the table has no write spec + // * Some(true) — force MemWAL routing; error if the table has no write spec // * Some(false) — read the base table only, bypassing the MemWAL // The LSM scanner surfaces in-flight `merge_insert` data (active/frozen - // memtables + flushed generations); validation and dispatch live in `lsm`. + // memtables + SSTables); validation and dispatch live in `lsm`. let has_spec = ds_ref.mem_wal_index_details().await?.is_some(); - let use_lsm = match query.base.use_lsm { + let use_memwal = match query.base.use_memwal { Some(true) if !has_spec => { return Err(Error::InvalidInput { - message: "use_lsm(true) was set but the table has no MemWAL write spec; \ - install one with set_lsm_write_spec or leave use_lsm unset" + message: "use_memwal(true) was set but the table has no MemWAL write spec; \ + install one with enable_memwal or leave use_memwal unset" .to_string(), }); } Some(enable) => enable, None => has_spec, }; - if use_lsm { + if use_memwal { return lsm::create_lsm_plan(table, ds_ref, query).await; } @@ -1195,7 +1195,7 @@ mod tests { table.set_unenforced_primary_key(["id"]).await.unwrap(); table - .set_lsm_write_spec(crate::table::LsmWriteSpec::unsharded()) + .enable_memwal(crate::table::MemWalWriteSpec::unsharded()) .await .unwrap(); let lsm_error = table @@ -1395,7 +1395,7 @@ mod tests { } #[tokio::test] - async fn test_execute_query_use_lsm_with_namespace_pushdown_runs_locally() { + async fn test_execute_query_use_memwal_with_namespace_pushdown_runs_locally() { use crate::connect; use crate::table::query::execute_query; use arrow_array::{Int32Array, RecordBatch}; @@ -1418,7 +1418,7 @@ mod tests { .unwrap(); let table = conn - .create_table("test_use_lsm_namespace_fallback", batch) + .create_table("test_use_memwal_namespace_fallback", batch) .execute() .await .unwrap(); @@ -1429,13 +1429,13 @@ mod tests { .pushdown_operations .insert(NamespaceClientPushdownOperation::QueryTable); - // `use_lsm` set (even to false) must force local execution — the namespace - // request has no use_lsm field, so a pushdown would silently ignore it. + // `use_memwal` set (even to false) must force local execution — the namespace + // request has no use_memwal field, so a pushdown would silently ignore it. let query_vector = Arc::new(Float32Array::from(vec![0.0, 0.0])); let query = AnyQuery::VectorQuery(VectorQueryRequest { base: QueryRequest { limit: Some(1), - use_lsm: Some(false), + use_memwal: Some(false), ..Default::default() }, column: Some("vector".to_string()), diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 4798da431..32966e4a6 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -1,14 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! MemWAL LSM read path. +//! MemWAL read path. //! -//! When a table has an LSM write spec installed (see [`set_lsm_write_spec`]), +//! When a table has a MemWAL write spec installed (see [`enable_memwal`]), //! reads are routed through Lance's [`LsmScanner`] instead of the plain //! base-table scan unless the query sets -//! [`use_lsm(false)`](crate::query::QueryBase::use_lsm). This makes data +//! [`use_memwal(false)`](crate::query::QueryBase::use_memwal). This makes data //! written via the LSM `merge_insert` path — which lives in the active/frozen -//! in-memory memtables and the flushed SSTable generations until an external +//! in-memory memtables and the SSTables until an external //! compaction merges it into the base table — visible to queries, deduplicated by //! primary key (newest generation wins). //! @@ -17,10 +17,10 @@ //! three run through a single [`LsmScanner`], so a `where` filter is honored as a //! prefilter uniformly — including for vector search, where `LsmScanner` threads //! it into the vector planner's prefilter. Shapes the LSM path cannot honor are -//! rejected with [`Error::NotSupported`]; the caller must set `use_lsm(false)` to +//! rejected with [`Error::NotSupported`]; the caller must set `use_memwal(false)` to //! run those against the base table. //! -//! [`set_lsm_write_spec`]: crate::Table::set_lsm_write_spec +//! [`enable_memwal`]: crate::Table::enable_memwal use std::collections::HashMap; use std::sync::Arc; @@ -57,7 +57,7 @@ const LSM_OVERFETCH_FACTOR: f64 = 2.0; /// The caller guarantees `ds_ref` carries a MemWAL write spec (routing is decided /// in [`create_plan`](super::create_plan)). Errors with [`Error::NotSupported`] /// for query shapes the LSM scanner cannot honor — the caller must set -/// `use_lsm(false)` to run those against the base table. +/// `use_memwal(false)` to run those against the base table. pub(super) async fn create_lsm_plan( table: &NativeTable, ds_ref: Arc, @@ -67,11 +67,11 @@ pub(super) async fn create_lsm_plan( // A time-traveled (checked-out) handle pins an older dataset version, but the // WAL manifests and cached writer expose current live state — mixing them would - // surface WAL rows written after the requested version. `use_lsm(false)` reads + // surface WAL rows written after the requested version. `use_memwal(false)` reads // the base table at the pinned version. if table.dataset.time_travel_version().is_some() { return Err(Error::NotSupported { - message: "the MemWAL LSM scanner cannot read from a time-traveled dataset version; set use_lsm(false) to read the base table at this version".to_string(), + message: "the MemWAL LSM scanner cannot read from a time-traveled dataset version; set use_memwal(false) to read the base table at this version".to_string(), }); } @@ -80,7 +80,7 @@ pub(super) async fn create_lsm_plan( .mem_wal_index_details() .await? .ok_or_else(|| Error::Runtime { - message: "the MemWAL LSM write spec disappeared during read planning".to_string(), + message: "the MemWAL write spec disappeared during read planning".to_string(), })?; let pk_columns = pk_columns(&ds_ref)?; @@ -135,10 +135,10 @@ pub(super) async fn create_lsm_plan( restore_projection(plan, &query, &pk_columns) } -/// Reject query shapes the LSM read path does not implement. On a MemWAL table +/// Reject query shapes the MemWAL read path does not implement. On a MemWAL table /// reads route through the LSM scanner by default, so an unsupported shape is a /// hard error rather than a silent fallback to the base-only scan — which would -/// exclude un-compacted MemWAL data. The caller must set `use_lsm(false)` to +/// exclude un-compacted MemWAL data. The caller must set `use_memwal(false)` to /// run these against the base table, accepting that the results omit un-compacted /// MemWAL data. /// @@ -148,7 +148,7 @@ fn reject_unsupported(query: &VectorQueryRequest) -> Result<()> { let unsupported = |what: &str| { Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner does not support {what}; set use_lsm(false) to read the base table only (results will exclude un-compacted MemWAL data)" + "the MemWAL LSM scanner does not support {what}; set use_memwal(false) to read the base table only (results will exclude un-compacted MemWAL data)" ), }) }; @@ -275,7 +275,7 @@ fn exclusion_watermarks( exclude } -/// Assemble the per-shard snapshots (flushed SSTable generations) and the +/// Assemble the per-shard snapshots (SSTables) and the /// in-memory memtables (active + frozen) for the table. /// /// Snapshots for all shards come from their on-disk manifests; for the shard @@ -413,7 +413,7 @@ fn base_scanner( QueryFilter::Datafusion(expr) => scanner.filter_expr(expr.clone()), QueryFilter::Substrait(_) => { return Err(Error::NotSupported { - message: "the MemWAL LSM scanner does not support Substrait filters; set use_lsm(false) to read the base table only".to_string(), + message: "the MemWAL LSM scanner does not support Substrait filters; set use_memwal(false) to read the base table only".to_string(), }); } }; @@ -458,11 +458,11 @@ async fn fts_plan( offset: Option, ) -> Result> { // Pre-check for a lancedb-flavored error; `LsmScanner` also validates the - // single-column requirement, but without the `use_lsm(false)` guidance. + // single-column requirement, but without the `use_memwal(false)` guidance. let columns: Vec = fts.columns().into_iter().collect(); if columns.len() > 1 { return Err(Error::NotSupported { - message: "the MemWAL LSM scanner full-text search supports a single column; set use_lsm(false) to read the base table only".to_string(), + message: "the MemWAL LSM scanner full-text search supports a single column; set use_memwal(false) to read the base table only".to_string(), }); } let column = columns.first().ok_or_else(|| Error::NotSupported { @@ -482,7 +482,7 @@ async fn fts_plan( { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise un-compacted documents are omitted. set use_lsm(false) to read the base table only" + "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (MemWalWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise un-compacted documents are omitted. set use_memwal(false) to read the base table only" ), }); } @@ -633,7 +633,7 @@ fn resolve_single_index( if names.len() > 1 { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner cannot resolve the {arm} index catch-up watermark for '{column}': it has multiple {arm} indexes; set use_lsm(false) to read the base table only" + "the MemWAL LSM scanner cannot resolve the {arm} index catch-up watermark for '{column}': it has multiple {arm} indexes; set use_memwal(false) to read the base table only" ), }); } @@ -682,7 +682,7 @@ fn restore_projection( /// /// Note: the base and SSTable arms use `fast_search` (indexed data only), so a /// base-table row not covered by a vector index is invisible here — it surfaces -/// only via the memtable or `use_lsm(false)`. +/// only via the memtable or `use_memwal(false)`. #[allow(clippy::too_many_arguments)] async fn vector_plan( dataset: &Dataset, @@ -717,7 +717,7 @@ async fn vector_plan( if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only" + "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (MemWalWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_memwal(false) to read the base table only" ), }); } @@ -726,7 +726,7 @@ async fn vector_plan( // clear error rather than failing deep in the planner. if is_binary_vector_column(&arrow_schema, &column) { return Err(Error::NotSupported { - message: "the MemWAL LSM scanner does not support binary (uint8) vector search; set use_lsm(false) to read the base table only".to_string(), + message: "the MemWAL LSM scanner does not support binary (uint8) vector search; set use_memwal(false) to read the base table only".to_string(), }); } @@ -855,7 +855,7 @@ mod tests { fn exclusion_watermark_gates_on_lagging_index_catchup() { let shard = Uuid::from_u128(1); let details = MemWalIndexDetails { - // Compaction has drained generations through 5 into the base table... + // Compaction has drained SSTables through generation 5 into the base table... compacted_sstables: vec![CompactedSsTable::new(shard, 5)], // ...but the FTS index has only caught up through generation 2. index_catchup: vec![IndexCatchupProgress::new( diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 0119ba36d..4274aaa6f 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -87,7 +87,7 @@ async fn execute_refresh_column_with_source( column: &str, ) -> Result { table.dataset.ensure_mutable()?; - ensure_no_lsm_write_spec(table).await?; + ensure_no_memwal_spec(table).await?; let dataset = table.dataset.get().await?; let expression = declared_expression(&dataset, column)?; @@ -303,7 +303,7 @@ pub(crate) async fn execute_refresh_column_async( // Validate before spawning so bad input is reported by this call rather // than only by the job. table.dataset.ensure_mutable()?; - ensure_no_lsm_write_spec(table).await?; + ensure_no_memwal_spec(table).await?; let dataset = table.dataset.get().await?; declared_expression(&dataset, column)?; drop(dataset); @@ -323,12 +323,12 @@ pub(crate) async fn execute_refresh_column_async( }))) } -/// Refuse to refresh under an LSM write spec. +/// Refuse to refresh under a MemWAL write spec. /// /// Refresh enumerates base fragments, and a write spec keeps visible rows in /// un-compacted MemWAL tiers it cannot reach -- success would silently omit /// readable rows. -async fn ensure_no_lsm_write_spec(table: &NativeTable) -> Result<()> { +async fn ensure_no_memwal_spec(table: &NativeTable) -> Result<()> { use lance::dataset::mem_wal::DatasetMemWalExt; // Unset drops the MemWAL index, so the spec alone stops describing a table @@ -341,7 +341,7 @@ async fn ensure_no_lsm_write_spec(table: &NativeTable) -> Result<()> { .list_mem_wal_latest_shard_ids() .await? .is_empty(); - if retained_sstables || table.get_lsm_write_spec().await?.is_some() { + if retained_sstables || table.get_memwal_spec().await?.is_some() { return Err(Error::NotSupported { message: "refresh_column is not supported on a table with an LSM write \ spec: rows in un-compacted tiers are invisible to refresh" @@ -1637,7 +1637,7 @@ mod tests { /// writer left in that state. #[tokio::test] async fn test_refresh_refuses_a_foreign_lsm_state() { - use crate::table::LsmWriteSpec; + use crate::table::MemWalWriteSpec; let tmp_dir = tempfile::tempdir().unwrap(); let conn = connect(tmp_dir.path().to_str().unwrap()) @@ -1655,7 +1655,7 @@ mod tests { let table = conn.create_table("lsm", batch).execute().await.unwrap(); table.set_unenforced_primary_key(["x"]).await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); super::super::computed_columns::add_foreign_kind(&table, "doubled", "sql").await; @@ -1673,7 +1673,7 @@ mod tests { /// flag still marks retained SSTable rows; refresh refuses on the flag. #[tokio::test] async fn test_refresh_refuses_retained_catchup_state() { - use crate::table::LsmWriteSpec; + use crate::table::MemWalWriteSpec; let tmp_dir = tempfile::tempdir().unwrap(); let conn = connect(tmp_dir.path().to_str().unwrap()) @@ -1697,14 +1697,14 @@ mod tests { .unwrap(); table.set_unenforced_primary_key(["x"]).await.unwrap(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .enable_memwal(MemWalWriteSpec::unsharded()) .await .unwrap(); let mut merge = table.merge_insert(&["x"]); merge .when_matched_update_all(None) .when_not_matched_insert_all() - .use_lsm(true); + .use_memwal(true); merge .execute(Box::new(arrow_array::RecordBatchIterator::new( vec![Ok(batch)], @@ -1712,7 +1712,7 @@ mod tests { ))) .await .unwrap(); - table.unset_lsm_write_spec().await.unwrap(); + table.disable_memwal().await.unwrap(); super::super::computed_columns::add_foreign_kind(&table, "doubled", "sql").await; let err = table.refresh_column("doubled").await.unwrap_err(); diff --git a/rust/lancedb/src/table/schema_evolution.rs b/rust/lancedb/src/table/schema_evolution.rs index 4f229835b..1bb57d6c0 100644 --- a/rust/lancedb/src/table/schema_evolution.rs +++ b/rust/lancedb/src/table/schema_evolution.rs @@ -131,7 +131,7 @@ pub(crate) async fn execute_declare( ) -> Result { use lance::dataset::mem_wal::DatasetMemWalExt; - // An LSM write spec keeps visible rows in tiers refresh cannot reach; + // A MemWAL write spec keeps visible rows in tiers refresh cannot reach; // checked against latest committed state, not this handle's snapshot. table.checkout_latest().await?; computed_columns::ensure_not_function_bound( @@ -149,7 +149,7 @@ pub(crate) async fn execute_declare( .list_mem_wal_latest_shard_ids() .await? .is_empty(); - if retained_sstables || table.get_lsm_write_spec().await?.is_some() { + if retained_sstables || table.get_memwal_spec().await?.is_some() { return Err(Error::NotSupported { message: "computed columns are not supported on a table with an LSM write \ spec: rows in un-compacted tiers are invisible to refresh" From ea1d2b172ae240c22a89d05b1c66ca6d3a5f90a4 Mon Sep 17 00:00:00 2001 From: XY Zhan Date: Fri, 16 Oct 2026 15:27:53 -0400 Subject: [PATCH 3/7] fix: require a primary key to enable MemWAL (#4561) ## Why Every MemWAL write and read resolves rows by the table's unenforced primary key, but `enable_memwal` accepted a table without one. The spec installed, and every MemWAL `merge_insert` then failed with "table has a MemWAL index but no unenforced primary key" until MemWAL was disabled. Lance's MemWAL initialization doesn't catch it: its key checks pass when there is no key. ## What changes `enable_memwal` refuses a spec on a table with no unenforced primary key, for every sharding mode, before anything is committed. The check runs against the dataset the spec is committed to; a key cannot be removed once set, and an overwrite or restore of the table conflicts with that commit. Servers that install specs through this method get the check too. The Rust doc on `Table::enable_memwal` now states the requirement; the Python and TypeScript docs already did. ## Tests - New `test_enable_memwal_requires_primary_key` (Rust and Python): unsharded, bucket and identity specs are all refused without a key, a refused spec leaves nothing installed, and the same table accepts a spec once a key is set. The Rust test fails with the check disabled. - Tests that installed a spec on a table without a key now declare one first (6 Rust, 3 Python). - `cargo nextest run -p lancedb --features remote --lib`: 1529 passed. `cargo clippy -p lancedb --features remote --tests --examples -- -D warnings` and `cargo fmt --check` are clean; no integration test or example installs a spec. - Python: `test_memwal_spec.py` and `test_merge_insert_lsm.py`, 43 passed; `ruff format` and `ruff check` are clean. --- python/python/tests/test_memwal_spec.py | 15 +++++++ rust/lancedb/src/table.rs | 59 +++++++++++++++++++++++++ rust/lancedb/src/table/merge/lsm.rs | 11 +++++ 3 files changed, 85 insertions(+) diff --git a/python/python/tests/test_memwal_spec.py b/python/python/tests/test_memwal_spec.py index a1ec6a90d..d292c9185 100644 --- a/python/python/tests/test_memwal_spec.py +++ b/python/python/tests/test_memwal_spec.py @@ -45,6 +45,7 @@ def _make_table(tmp_path): def test_enable_memwal_validates(tmp_path): _db, table = _make_table(tmp_path) + table.set_unenforced_primary_key("id") # Out-of-range num_buckets. with pytest.raises(Exception, match="num_buckets"): @@ -60,6 +61,7 @@ def test_enable_memwal_validates(tmp_path): def test_disable_memwal(tmp_path): _db, table = _make_table(tmp_path) + table.set_unenforced_primary_key("id") # unset errors when no spec is set. with pytest.raises(Exception, match="no MemWAL write spec"): @@ -74,6 +76,18 @@ def test_disable_memwal(tmp_path): table.enable_memwal(MemWalWriteSpec.bucket("id", 8)) +def test_enable_memwal_requires_primary_key(tmp_path): + _db, table = _make_table(tmp_path) + for spec in ( + MemWalWriteSpec.unsharded(), + MemWalWriteSpec.bucket("id", 4), + MemWalWriteSpec.identity("id"), + ): + with pytest.raises(Exception, match="primary key"): + table.enable_memwal(spec) + assert table.get_memwal_spec() is None + + def test_set_unsharded_spec(tmp_path): _db, table = _make_table(tmp_path) # Lance MemWAL still requires a primary key on the dataset; Unsharded @@ -206,6 +220,7 @@ async def test_async_get_memwal_spec(tmp_path): ) assert await table.get_memwal_spec() is None + await table.set_unenforced_primary_key("id") # A real scalar index is needed to name it as a maintained index. await table.create_index("id", config=BTree()) diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index c0a4a719e..c6193d750 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -2015,6 +2015,9 @@ impl Table { /// - [`MemWalWriteSpec::identity`] — shard by the raw value of a scalar column. /// - [`MemWalWriteSpec::unsharded`] — route every write to a single shard. /// + /// Every spec requires the table to declare an unenforced primary key + /// ([`Table::set_unenforced_primary_key`]); otherwise this fails. + /// /// A table carries one spec: this fails while one is installed, since the /// SSTables already written were homed under it. /// [`Table::disable_memwal`] removes one. @@ -5541,6 +5544,7 @@ mod tests { .await .unwrap(); let table = conn.create_table("t", reader).execute().await.unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); // Reject num_buckets out of range. for bad in [0u32, 1025] { @@ -5585,6 +5589,56 @@ mod tests { assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); } + /// A spec is refused on a table without an unenforced primary key, for every + /// sharding mode, and a refused spec leaves nothing installed. + #[tokio::test] + async fn test_enable_memwal_requires_primary_key() { + use lance::dataset::mem_wal::DatasetMemWalExt; + + let tmp_dir = tempdir().unwrap(); + let conn = ConnectBuilder::new(tmp_dir.path().to_str().unwrap()) + .read_consistency_interval(Duration::from_secs(0)) + .execute() + .await + .unwrap(); + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(arrow_array::Int64Array::from(vec![1, 2]))], + ) + .unwrap(); + let reader: Box = + Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema)); + let table = conn.create_table("t", reader).execute().await.unwrap(); + let installed = |table: &Table| { + let table = table.clone(); + async move { + let dataset = table.as_native().unwrap().dataset.get().await.unwrap(); + dataset.mem_wal_index_details().await.unwrap().is_some() + } + }; + + for spec in [ + MemWalWriteSpec::unsharded(), + MemWalWriteSpec::bucket("id", 4), + MemWalWriteSpec::identity("id"), + ] { + let err = table.enable_memwal(spec).await.unwrap_err(); + assert!( + matches!(&err, Error::InvalidInput { message } if message.contains("primary key")), + "got {err:?}" + ); + assert!(!installed(&table).await); + } + + table.set_unenforced_primary_key(["id"]).await.unwrap(); + table + .enable_memwal(MemWalWriteSpec::unsharded()) + .await + .unwrap(); + assert!(installed(&table).await); + } + #[tokio::test] async fn test_enable_memwal_unsharded() { use lance::dataset::mem_wal::DatasetMemWalExt; @@ -5606,6 +5660,7 @@ mod tests { .await .unwrap(); let table = conn.create_table("t", reader).execute().await.unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); table .enable_memwal(MemWalWriteSpec::unsharded()) @@ -5652,6 +5707,7 @@ mod tests { .await .unwrap(); let table = conn.create_table("t", reader).execute().await.unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); table .enable_memwal( @@ -5703,6 +5759,7 @@ mod tests { .await .unwrap(); let table = conn.create_table("t", reader).execute().await.unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); // unset errors when no spec is set. table.disable_memwal().await.unwrap_err(); @@ -5760,6 +5817,7 @@ mod tests { .await .unwrap(); let table = conn.create_table("t", reader).execute().await.unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); // No spec installed yet. assert_eq!(table.get_memwal_spec().await.unwrap(), None); @@ -5826,6 +5884,7 @@ mod tests { .await .unwrap(); let table = conn.create_table("t", reader).execute().await.unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); table .create_index(&["id"], Index::BTree(Default::default())) diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index d26be28cc..39e4fbf28 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -127,6 +127,17 @@ pub(crate) async fn enable_memwal(table: &NativeTable, spec: MemWalWriteSpec) -> .into(), }); } + // Every MemWAL write and read resolves rows by the unenforced primary key, + // so a spec on a table without one leaves the table unwritable through the + // MemWAL path. A key cannot be removed once set, and an overwrite or restore + // of the table conflicts with the commit below. + if dataset.schema().unenforced_primary_key().is_empty() { + return Err(Error::InvalidInput { + message: "a MemWAL write spec requires the table to declare an unenforced \ + primary key; set one with set_unenforced_primary_key first" + .into(), + }); + } let mut builder = dataset .initialize_mem_wal() .mem_index_registry(table.mem_index_registry()); From b301a47aa8b4ed0e1bad6ae7706029b376f9a107 Mon Sep 17 00:00:00 2001 From: Bilal Atique <74999164+BilalAtique@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:50:38 +0500 Subject: [PATCH 4/7] feat(node): add blobMode to return blob bytes from queries (#4353) --- docs/src/js/globals.md | 1 + .../js/interfaces/QueryExecutionOptions.md | 32 +++ docs/src/js/type-aliases/BlobMode.md | 13 + nodejs/__test__/table.test.ts | 183 +++++++++++++ nodejs/lancedb/index.ts | 1 + nodejs/lancedb/query.ts | 255 ++++++++++++++++-- nodejs/lancedb/table.ts | 10 +- nodejs/src/query.rs | 27 +- 8 files changed, 497 insertions(+), 25 deletions(-) create mode 100644 docs/src/js/type-aliases/BlobMode.md diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 739b9033a..7310f3f90 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -161,6 +161,7 @@ - [BaseTokenizer](type-aliases/BaseTokenizer.md) - [BlobData](type-aliases/BlobData.md) - [BlobInput](type-aliases/BlobInput.md) +- [BlobMode](type-aliases/BlobMode.md) - [BlobOptions](type-aliases/BlobOptions.md) - [BlobRange](type-aliases/BlobRange.md) - [BlobReadOptions](type-aliases/BlobReadOptions.md) diff --git a/docs/src/js/interfaces/QueryExecutionOptions.md b/docs/src/js/interfaces/QueryExecutionOptions.md index 46b06ce7e..8d3749f38 100644 --- a/docs/src/js/interfaces/QueryExecutionOptions.md +++ b/docs/src/js/interfaces/QueryExecutionOptions.md @@ -10,6 +10,38 @@ Options that control the behavior of a particular query execution ## Properties +### blobMode? + +```ts +optional blobMode: BlobMode; +``` + +How to return blob v2 columns. Applies to `toArrow()` and `toArray()`. + +- `"descriptions"` (default): each blob is a descriptor. Read the bytes + with [Table.fetchBlobs](../classes/Table.md#fetchblobs) or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles). +- `"bytes"`: each top-level blob column holds the blob bytes as + `LargeBinary`. The query also reads `_rowid` to fetch the bytes, and + leaves it out of the result unless [QueryBase.withRowId](../classes/QueryBase.md#withrowid) was + called. All the bytes are held in memory. + +Blobs nested in a struct or a list keep their descriptors. + +The rows and the bytes come from the same table version. If the table +changes while the query runs, the query runs again, up to three times. + +#### Example + +```ts +const rows = await table + .query() + .select(["id", "image"]) + .toArray({ blobMode: "bytes" }); +const image: Uint8Array | null = rows[0].image; +``` + +*** + ### maxBatchLength? ```ts diff --git a/docs/src/js/type-aliases/BlobMode.md b/docs/src/js/type-aliases/BlobMode.md new file mode 100644 index 000000000..67d7c47ad --- /dev/null +++ b/docs/src/js/type-aliases/BlobMode.md @@ -0,0 +1,13 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / BlobMode + +# Type Alias: BlobMode + +```ts +type BlobMode: "descriptions" | "bytes"; +``` + +How a query returns blob v2 columns. See [QueryExecutionOptions.blobMode](../interfaces/QueryExecutionOptions.md#blobmode). diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index f53f859b8..b30121725 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -10,6 +10,7 @@ import * as tmp from "tmp"; import { AutoQuery, type BlobInput, + type BlobMode, Connection, MatchQuery, PhraseQuery, @@ -18,6 +19,7 @@ import { VectorQuery, blob, connect, + isBlobField, tokenize, } from "../lancedb"; import { @@ -3016,6 +3018,161 @@ describe("when dealing with blob columns", () => { expect(await handle!.readRange(0n, 4n)).toEqual(payload.subarray(0, 4)); }); + it("toArrow returns blob bytes with blobMode bytes", async () => { + const { table, alpha, beta } = await openBlobTable(); + const result = await table.query().toArrow({ blobMode: "bytes" }); + const field = result.schema.fields.find((f) => f.name === "image")!; + expect(arrow.DataType.isLargeBinary(field.type)).toBe(true); + expect(isBlobField(field)).toBe(false); + expect(result.schema.fields.map((f) => f.name)).toEqual(["id", "image"]); + const byId = new Map( + result.toArray().map((row) => [Number(row.id), row.image]), + ); + expect(Buffer.from(byId.get(1)!)).toEqual(alpha); + expect(Buffer.from(byId.get(2)!)).toEqual(beta); + expect(byId.get(3)).toBeNull(); + }); + + it("toArray returns blob bytes and keeps _rowid only when asked", async () => { + const { table, rowIds, alpha } = await openBlobTable(); + const [plain] = await table + .query() + .where("id = 1") + .select(["id", "image"]) + .toArray({ blobMode: "bytes" }); + expect(Object.keys(plain)).toEqual(["id", "image"]); + expect(Buffer.from(plain.image)).toEqual(alpha); + + const [withRowId] = await table + .query() + .where("id = 1") + .withRowId() + .toArray({ blobMode: "bytes" }); + expect(withRowId._rowid).toBe(rowIds[0]); + expect(Buffer.from(withRowId.image)).toEqual(alpha); + }); + + it("fills a renamed blob column with bytes", async () => { + const { table, beta } = await openBlobTable(); + const [row] = await table + .query() + .where("id = 2") + .select( + new Map([ + ["key", "id"], + ["picture", "image"], + ]), + ) + .toArray({ blobMode: "bytes" }); + expect(Number(row.key)).toBe(2); + expect(Buffer.from(row.picture)).toEqual(beta); + }); + + it("returns blob bytes from a take query", async () => { + const { table, rowIds, alpha, beta } = await openBlobTable(); + const rows = await table + .takeRowIds([rowIds[1], rowIds[0]]) + .toArray({ blobMode: "bytes" }); + const byId = new Map(rows.map((row) => [Number(row.id), row])); + expect(Buffer.from(byId.get(1)!.image)).toEqual(alpha); + expect(Buffer.from(byId.get(2)!.image)).toEqual(beta); + expect(Object.keys(rows[0])).not.toContain("_rowid"); + }); + + it("does not change later executions of the same query", async () => { + const { table } = await openBlobTable(); + const query = table.query().where("id = 1"); + await query.toArray({ blobMode: "bytes" }); + const [row] = await query.toArray(); + expect(Object.keys(row)).not.toContain("_rowid"); + expect(Number(row.image.size)).toBe(5); + }); + + it("skips row ids when no blob column is selected", async () => { + const { table } = await openBlobTable(); + const result = await table + .query() + .select(["id"]) + .toArrow({ blobMode: "bytes" }); + expect(result.schema.fields.map((f) => f.name)).toEqual(["id"]); + expect(result.numRows).toBe(3); + }); + + it("returns an empty bytes column for an empty result", async () => { + const { table } = await openBlobTable(); + const result = await table + .query() + .where("id > 100") + .toArrow({ blobMode: "bytes" }); + expect(result.numRows).toBe(0); + expect(result.schema.fields.map((f) => f.name)).toEqual(["id", "image"]); + const field = result.schema.fields.find((f) => f.name === "image")!; + expect(arrow.DataType.isLargeBinary(field.type)).toBe(true); + expect(isBlobField(field)).toBe(false); + }); + + it("keeps the descriptor schema for an empty result", async () => { + const { table } = await openBlobTable(); + const result = await table.query().where("id > 100").toArrow(); + expect(result.numRows).toBe(0); + expect(result.schema.fields.map((f) => f.name)).toEqual(["id", "image"]); + const field = result.schema.fields.find((f) => f.name === "image")!; + expect(arrow.DataType.isStruct(field.type)).toBe(true); + }); + + it("rejects an unknown blobMode", async () => { + const { table } = await openBlobTable(); + await expect( + table.query().toArray({ blobMode: "lazy" as BlobMode }), + ).rejects.toThrow(/blobMode must be/); + }); + + it("reruns the query when the table changes before the version check", async () => { + const { table, native, update } = await openVersionedBlobTable(); + const liveVersion = native.version.bind(native); + native.version = async () => { + native.version = liveVersion; + await update(2n, "new"); + return liveVersion(); + }; + const [row] = await table.query().toArray({ blobMode: "bytes" }); + expect(row.generation).toBe(2n); + expect(Buffer.from(row.image).toString()).toBe("new"); + }); + + it("reads bytes from the query's version when a write lands before the fetch", async () => { + const { table, native, update } = await openVersionedBlobTable(); + const liveVersion = native.version.bind(native); + native.version = async () => { + native.version = liveVersion; + const version = await liveVersion(); + await update(2n, "new"); + return version; + }; + const [row] = await table.query().toArray({ blobMode: "bytes" }); + expect(row.generation).toBe(1n); + expect(Buffer.from(row.image).toString()).toBe("old"); + }); + + it("fails when the table changes during every attempt", async () => { + const { table, native, update } = await openVersionedBlobTable(); + const liveVersion = native.version.bind(native); + let generation = 1n; + native.version = async () => { + generation += 1n; + await update(generation, `generation ${generation}`); + return liveVersion(); + }; + try { + await expect( + table.query().toArray({ blobMode: "bytes" }), + ).rejects.toThrow(/changed during each of 3 attempts/); + expect(generation).toBe(4n); + } finally { + native.version = liveVersion; + } + }); + it("rejects fetchBlobs on a non-blob column", async () => { const { table, rowIds } = await openBlobTable(); await expect(table.fetchBlobs("id", rowIds)).rejects.toThrow(/blob/i); @@ -3198,6 +3355,32 @@ describe("when dealing with blob columns", () => { ).map((value) => (value == null ? null : Number(value.size))); } + async function openVersionedBlobTable() { + const db = await connect(tmpDir.name, { + storageOptions: { newTableEnableStableRowIds: "true" }, + }); + const schema = new Schema([ + new Field("id", new Int64(), false), + new Field("generation", new Int64(), true), + blob("image"), + ]); + const table = await db.createTable( + "versions", + [{ id: 1n, generation: 1n, image: Buffer.from("old") }], + { schema }, + ); + const native = ( + table as unknown as { inner: { version: () => Promise } } + ).inner; + const update = async (generation: bigint, image: string) => { + await table + .mergeInsert("id") + .whenMatchedUpdateAll() + .execute([{ id: 1n, generation, image: Buffer.from(image) }]); + }; + return { table, native, update }; + } + async function openBlobTable() { const db = await connect(tmpDir.name); const schema = new Schema([ diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 3254a0742..3096cbe6b 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -122,6 +122,7 @@ export { VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, + BlobMode, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index 3583e5992..ef7443bbb 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -3,12 +3,18 @@ import { Table as ArrowTable, + Field, type IntoVector, + LargeBinary, RecordBatch, + Schema, + Struct, extractVectorBuffer, fromBufferToRecordBatch, fromRecordBatchToBuffer, + makeData, tableFromIPC, + vectorFromArray, } from "./arrow"; import { type IvfPqOptions } from "./indices"; import { @@ -41,6 +47,112 @@ export async function* RecordBatchIterator( } } +/** + * How a query returns blob v2 columns. See {@link QueryExecutionOptions.blobMode}. + */ +export type BlobMode = "descriptions" | "bytes"; + +/** + * The table a query reads, and the parts of the query that decide which blob + * columns `blobMode: "bytes"` fills in. + * + * @hidden + */ +export interface QueryBlobContext { + table: NativeTable; + withRowId: boolean; + projection?: [string, string][]; +} + +const BLOB_BYTES_ATTEMPTS = 3; + +function blobSources( + blobColumns: string[], + projection: [string, string][] | undefined, +): Map { + const columns = new Set(blobColumns); + const pairs = + projection ?? + blobColumns.map((column): [string, string] => [column, column]); + return new Map(pairs.filter(([, source]) => columns.has(source))); +} + +async function readBlobBytes( + table: ArrowTable, + sources: Map, + snapshot: NativeTable, + dropRowId: boolean, +): Promise { + const rowIdColumn = table.getChild("_rowid"); + if (table.numRows > 0 && rowIdColumn === null) { + throw new Error('blobMode "bytes" needs _rowid in the query result'); + } + const rowIds = + rowIdColumn === null + ? [] + : Array.from(rowIdColumn.toArray() as BigUint64Array); + const replacements = new Map(); + for (const [output, source] of sources) { + const index = table.schema.fields.findIndex( + (field) => field.name === output, + ); + if (index < 0) { + continue; + } + const values = + rowIds.length === 0 ? [] : await snapshot.fetchBlobs(source, rowIds); + replacements.set( + index, + values.map((value) => value ?? null), + ); + } + const result = withBinaryColumns(table, replacements); + if (!dropRowId) { + return result; + } + return result.select( + result.schema.fields + .map((field) => field.name) + .filter((name) => name !== "_rowid"), + ); +} + +// A fresh field, not a clone: the descriptor field's metadata must not follow +// the bytes. +function withBinaryColumns( + table: ArrowTable, + replacements: Map, +): ArrowTable { + if (replacements.size === 0) { + return table; + } + const fields = table.schema.fields.map((field, index) => + replacements.has(index) + ? new Field(field.name, new LargeBinary(), true) + : field, + ); + const schema = new Schema(fields, table.schema.metadata); + let offset = 0; + const batches = table.batches.map((batch) => { + const children = batch.data.children.map((child, index) => { + const values = replacements.get(index); + if (values === undefined) { + return child; + } + return vectorFromArray( + values.slice(offset, offset + batch.numRows), + new LargeBinary(), + ).data[0]; + }); + offset += batch.numRows; + return new RecordBatch( + schema, + makeData({ type: new Struct(fields), length: batch.numRows, children }), + ); + }); + return new ArrowTable(schema, batches); +} + /** * Options that control the behavior of a particular query execution */ @@ -57,6 +169,32 @@ export interface QueryExecutionOptions { * Timeout for query execution in milliseconds */ timeoutMs?: number; + + /** + * How to return blob v2 columns. Applies to `toArrow()` and `toArray()`. + * + * - `"descriptions"` (default): each blob is a descriptor. Read the bytes + * with {@link Table.fetchBlobs} or {@link Table.fetchBlobFiles}. + * - `"bytes"`: each top-level blob column holds the blob bytes as + * `LargeBinary`. The query also reads `_rowid` to fetch the bytes, and + * leaves it out of the result unless {@link QueryBase.withRowId} was + * called. All the bytes are held in memory. + * + * Blobs nested in a struct or a list keep their descriptors. + * + * The rows and the bytes come from the same table version. If the table + * changes while the query runs, the query runs again, up to three times. + * + * @example + * ```ts + * const rows = await table + * .query() + * .select(["id", "image"]) + * .toArray({ blobMode: "bytes" }); + * const image: Uint8Array | null = rows[0].image; + * ``` + */ + blobMode?: BlobMode; } export type AnalyzePlanDistributedMetrics = "aggregate" | "per_worker" | "full"; @@ -119,10 +257,19 @@ export class QueryBase< /** * @hidden */ - protected constructor(inner?: NativeQueryType | Promise) { + protected blobContext?: QueryBlobContext; + + /** + * @hidden + */ + protected constructor( + inner?: NativeQueryType | Promise, + blobContext?: QueryBlobContext, + ) { if (inner !== undefined) { this.inner = inner; } + this.blobContext = blobContext; } // call a function on the inner (either a promise or the actual object) @@ -193,14 +340,22 @@ export class QueryBase< }); }; + let projection: [string, string][]; if (typeof columns === "string") { selectColumns([columns]); + projection = [[columns, columns]]; } else if (Array.isArray(columns)) { selectColumns(columns); + projection = columns.map((column) => [column, column]); } else if (columns instanceof Map) { - selectMapping(Array.from(columns.entries())); + projection = Array.from(columns.entries()); + selectMapping(projection); } else { - selectMapping(Object.entries(columns)); + projection = Object.entries(columns); + selectMapping(projection); + } + if (this.blobContext !== undefined) { + this.blobContext = { ...this.blobContext, projection }; } return this; @@ -215,6 +370,9 @@ export class QueryBase< */ withRowId(): this { this.doCall((inner: NativeQueryType) => inner.withRowId()); + if (this.blobContext !== undefined) { + this.blobContext = { ...this.blobContext, withRowId: true }; + } return this; } @@ -223,9 +381,14 @@ export class QueryBase< */ protected async nativeExecute( options?: Partial, + withRowId = false, ): Promise { const inner = await this.getInner(); - return inner.execute(options?.maxBatchLength, options?.timeoutMs); + return inner.execute( + options?.maxBatchLength, + options?.timeoutMs, + withRowId, + ); } /** @@ -257,8 +420,41 @@ export class QueryBase< * Empty results retain the query's output schema. */ async toArrow(options?: Partial): Promise { + const context = this.blobBytesContext(options?.blobMode); + if (context === undefined) { + return this.collect(options, false); + } + for (let attempt = 1; ; attempt++) { + // The snapshot pins the bytes. The query runs on the live table, so an + // unchanged version afterwards proves it read the snapshot's version. + const snapshot = await context.table.querySnapshot(); + const version = await snapshot.version(); + const sources = blobSources( + await snapshot.blobColumns(), + context.projection, + ); + if (sources.size === 0) { + return this.collect(options, false); + } + const addRowId = !context.withRowId; + const table = await this.collect(options, addRowId); + if ((await context.table.version()) === version) { + return readBlobBytes(table, sources, snapshot, addRowId); + } + if (attempt === BLOB_BYTES_ATTEMPTS) { + throw new Error( + `blobMode "bytes": the table changed during each of ${BLOB_BYTES_ATTEMPTS} attempts to read it`, + ); + } + } + } + + private async collect( + options: Partial | undefined, + withRowId: boolean, + ): Promise { const batches = []; - const iterator = this.nativeExecute(options); + const iterator = this.nativeExecute(options, withRowId); for await (const batch of RecordBatchIterator(iterator)) { batches.push(batch); } @@ -268,6 +464,23 @@ export class QueryBase< return new ArrowTable(batches); } + private blobBytesContext( + mode: BlobMode | undefined, + ): QueryBlobContext | undefined { + if (mode === undefined || mode === "descriptions") { + return undefined; + } + if (mode !== "bytes") { + throw new Error( + `blobMode must be "descriptions" or "bytes", got "${String(mode)}"`, + ); + } + if (this.blobContext === undefined) { + throw new Error('blobMode "bytes" is not supported for this query'); + } + return this.blobContext; + } + /** Collect the results as an array of objects. */ // biome-ignore lint/suspicious/noExplicitAny: arrow.toArrow() returns any[] async toArray(options?: Partial): Promise { @@ -355,8 +568,11 @@ export class StandardQueryBase< extends QueryBase implements ExecutableQuery { - constructor(inner?: NativeQueryType | Promise) { - super(inner); + constructor( + inner?: NativeQueryType | Promise, + blobContext?: QueryBlobContext, + ) { + super(inner, blobContext); } /** @@ -505,8 +721,11 @@ export class VectorQuery extends StandardQueryBase { /** * @hidden */ - constructor(inner: NativeVectorQuery | Promise) { - super(inner); + constructor( + inner: NativeVectorQuery | Promise, + blobContext?: QueryBlobContext, + ) { + super(inner, blobContext); } /** @@ -760,7 +979,7 @@ export class VectorQuery extends StandardQueryBase { addQueryVectorToNative(inner, outcome.value); return inner; })(); - return new VectorQuery(res); + return new VectorQuery(res, this.blobContext); } else { this.doVectorCall((inner) => addQueryVectorToNative(inner, vector)); return this; @@ -849,7 +1068,7 @@ export function createAutoQuery( return nearestToNative(route.table.query(), vector); }; - return new AutoQuery(createInner); + return new AutoQuery(createInner, { table, withRowId: false }); } /** @@ -858,8 +1077,8 @@ export function createAutoQuery( * @hideconstructor */ export class TakeQuery extends QueryBase { - constructor(inner: NativeTakeQuery) { - super(inner); + constructor(inner: NativeTakeQuery, blobContext?: QueryBlobContext) { + super(inner, blobContext); } /** @@ -898,8 +1117,9 @@ export class AutoQuery extends StandardQueryBase< private readonly createInner: () => Promise< NativeQuery | NativeVectorQuery >, + blobContext?: QueryBlobContext, ) { - super(); + super(undefined, blobContext); } /** @hidden */ @@ -933,7 +1153,7 @@ export class Query extends StandardQueryBase { * @hidden */ constructor(tbl: NativeTable) { - super(tbl.query()); + super(tbl.query(), { table: tbl, withRowId: false }); } /** @@ -979,14 +1199,15 @@ export class Query extends StandardQueryBase { const nativeQuery = inner.then(async (resolvedInner) => nearestToNative(resolvedInner, await vector), ); - return new VectorQuery(nativeQuery); + return new VectorQuery(nativeQuery, this.blobContext); } if (vector instanceof Promise) { return new VectorQuery( vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)), + this.blobContext, ); } - return new VectorQuery(nearestToNative(inner, vector)); + return new VectorQuery(nearestToNative(inner, vector), this.blobContext); } nearestToText(query: string | FullTextQuery, columns?: string[]): Query { diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 4d6500a5f..2b5319bec 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -1240,11 +1240,17 @@ export class LocalTable extends Table { } takeOffsets(offsets: number[]): TakeQuery { - return new TakeQuery(this.inner.takeOffsets(offsets)); + return new TakeQuery(this.inner.takeOffsets(offsets), { + table: this.inner, + withRowId: false, + }); } takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery { - return new TakeQuery(this.inner.takeRowIds(rowIdsToBigInts(rowIds))); + return new TakeQuery(this.inner.takeRowIds(rowIdsToBigInts(rowIds)), { + table: this.inner, + withRowId: false, + }); } blobColumns(): Promise { diff --git a/nodejs/src/query.rs b/nodejs/src/query.rs index 68db6f810..dba8f665a 100644 --- a/nodejs/src/query.rs +++ b/nodejs/src/query.rs @@ -196,6 +196,7 @@ impl Query { &self, max_batch_length: Option, timeout_ms: Option, + with_row_id: Option, ) -> napi::Result { let mut execution_opts = QueryExecutionOptions::default(); if let Some(max_batch_length) = max_batch_length { @@ -204,8 +205,12 @@ impl Query { if let Some(timeout_ms) = timeout_ms { execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64)) } - let inner_stream = self - .inner + let query = if with_row_id == Some(true) { + self.inner.clone().with_row_id() + } else { + self.inner.clone() + }; + let inner_stream = query .execute_with_options(execution_opts) .await .map_err(|e| { @@ -432,6 +437,7 @@ impl VectorQuery { &self, max_batch_length: Option, timeout_ms: Option, + with_row_id: Option, ) -> napi::Result { let mut execution_opts = QueryExecutionOptions::default(); if let Some(max_batch_length) = max_batch_length { @@ -440,8 +446,12 @@ impl VectorQuery { if let Some(timeout_ms) = timeout_ms { execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64)) } - let inner_stream = self - .inner + let query = if with_row_id == Some(true) { + self.inner.clone().with_row_id() + } else { + self.inner.clone() + }; + let inner_stream = query .execute_with_options(execution_opts) .await .map_err(|e| { @@ -520,6 +530,7 @@ impl TakeQuery { &self, max_batch_length: Option, timeout_ms: Option, + with_row_id: Option, ) -> napi::Result { let mut execution_opts = QueryExecutionOptions::default(); if let Some(max_batch_length) = max_batch_length { @@ -528,8 +539,12 @@ impl TakeQuery { if let Some(timeout_ms) = timeout_ms { execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64)) } - let inner_stream = self - .inner + let query = if with_row_id == Some(true) { + self.inner.clone().with_row_id() + } else { + self.inner.clone() + }; + let inner_stream = query .execute_with_options(execution_opts) .await .map_err(|e| { From 5f746d1e0111ae9f59932af5e81a274bff64a9df Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 16 Oct 2026 12:58:37 -0700 Subject: [PATCH 5/7] fix: refuse a materialized view whose sources lead back to itself (#4494) Creating a view now follows its source through each view's stored coordinate, up to 32 levels, and refuses when the walk reaches the name being created. One table open per level; no catalog scan. Part of ENT-2922. --- rust/lancedb/src/materialized_view.rs | 98 +++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index c04a6cbbe..95e3baac7 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -42,6 +42,9 @@ pub use refresh::{RefreshMaterializedViewResult, RefreshMode}; /// Schema metadata key holding the view definition; see [`DEFINITION_FORMAT`]. pub const DEFINITION_META_KEY: &str = "mv.definition"; +/// How many views may stand between a view and a plain table. +const MAX_SOURCE_DEPTH: usize = 32; + /// Field metadata namespace for declarations about schema structure, such as /// an unenforced primary key. const SCHEMA_DECLARATION_META_PREFIX: &str = "lance-schema:"; @@ -1238,6 +1241,7 @@ impl PreparedDeclaration { namespace_path: &[String], name: &str, ) -> Result { + ensure_no_source_cycle(&self.database, namespace_path, name, &self.definition).await?; let empty: Vec> = vec![]; let reader: Box = @@ -1254,6 +1258,65 @@ impl PreparedDeclaration { } } +/// Refuse to create `name` when following its source through each view's +/// stored definition leads back to `name`: with no plain table beneath it, +/// such a view would only ever refresh from itself. The walk follows the +/// coordinates the views already record, so it opens one table per level. +async fn ensure_no_source_cycle( + database: &Arc, + namespace_path: &[String], + name: &str, + definition: &MaterializedViewDefinition, +) -> Result<()> { + let mut path = vec![name.to_string()]; + let mut next = ( + definition.source_namespace.clone(), + definition.source_table.clone(), + ); + for _ in 0..MAX_SOURCE_DEPTH { + let (namespace, source) = next; + path.push(source.clone()); + if namespace == namespace_path && source == name { + return Err(Error::InvalidInput { + message: format!( + "materialized view '{name}' would read itself: {}", + path.join(" -> ") + ), + }); + } + let table = match database + .open_table(OpenTableRequest { + name: source, + namespace_path: namespace, + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await + { + Ok(table) => table, + // A missing source ends the chain, so it cannot lead back here. + Err(Error::TableNotFound { .. }) => return Ok(()), + Err(e) => return Err(e), + }; + let schema = table.schema().await?; + // Only a definition this version reads can be followed. + match read_definition(schema.metadata()) { + Ok(Some(StoredDefinition::Query(upstream))) => { + next = (upstream.source_namespace, upstream.source_table); + } + _ => return Ok(()), + } + } + Err(Error::InvalidInput { + message: format!( + "materialized view '{name}' would nest deeper than {MAX_SOURCE_DEPTH} views" + ), + }) +} + /// Column definitions are positional over the view schema: carry each /// field's entry to its place in `fields`, physical for a field that had none. fn rewrite_column_definitions( @@ -2017,6 +2080,41 @@ mod tests { } } + #[tokio::test] + async fn a_view_cannot_lead_back_to_itself() { + let conn = connect("memory://").execute().await.unwrap(); + let rows = || record_batch!(("x", Int32, [1, 2])).unwrap(); + conn.create_table("src", rows()).execute().await.unwrap(); + conn.create_materialized_view("m", "src") + .execute() + .await + .unwrap(); + conn.create_materialized_view("m2", "m") + .execute() + .await + .unwrap(); + conn.drop_table("src", &[]).await.unwrap(); + + let err = conn + .create_materialized_view("src", "m2") + .execute() + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("materialized view 'src' would read itself: src -> m2 -> m -> src"), + "{err}" + ); + assert!(conn.open_table("src").execute().await.is_err()); + + // Back under a plain table, the same chain is fine to extend. + conn.create_table("src", rows()).execute().await.unwrap(); + conn.create_materialized_view("m3", "m2") + .execute() + .await + .unwrap(); + } + #[tokio::test] async fn direct_writes_to_a_view_are_refused() { use arrow_schema::Schema; From 44ba9c6dee530516cd00cb4e86477715c6a57ca1 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:43:55 -0700 Subject: [PATCH 6/7] fix: support remote blob range reads (#4553) --- python/python/lancedb/table.py | 5 +- python/python/tests/test_remote_db.py | 46 +++++++- rust/lancedb/src/remote/table.rs | 20 ++-- rust/lancedb/src/remote/table/blobs.rs | 151 ++++++++++++++++++++++++- rust/lancedb/src/table.rs | 2 + 5 files changed, 210 insertions(+), 14 deletions(-) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index f0cb05427..a1a54ee8e 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -2190,8 +2190,9 @@ class Table(ABC): One missing or deleted row ID rejects the entire batch; no partial results are returned. - Row IDs can be obtained from a query with ``with_row_id(True)``. This - API is currently supported only by local tables. + Row IDs can be obtained from a query with ``with_row_id(True)``. + Remote tables read the requested ranges through version-pinned blob + handles using HTTP byte-range requests. """ @abstractmethod diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 6ee41aa0c..e9768b0bf 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1623,7 +1623,6 @@ def remote_unsupported_operations(): return [ ("uses_v2_manifest_paths", (), {}, "uses_v2_manifest_paths"), ("migrate_v2_manifest_paths", (), {}, "migrate_manifest_paths_v2"), - ("fetch_blob_ranges", ("image", [(1, 0, 1)]), {}, "fetch_blob_ranges"), ("optimize", (), {}, "optimize"), ("create_index", ("vector",), {"config": HnswPq()}, "IVF_HNSW_PQ"), ] @@ -3362,13 +3361,20 @@ def blob_remote_table(*, server_version=Version("0.5.0")): else: assert body["with_row_id"] is True if columns == ["image"]: - # fetch_blob_files sizes its handles from a descriptor take. + # Blob handles are sized from a descriptor take. assert body["filter"].startswith("_rowid IN") response_table = blob_descriptor_take_table() + row_ids = [ + int(value) for value in re.findall(r"\d+", body["filter"]) + ] + response_table = response_table.filter( + pa.compute.is_in(response_table["_rowid"], pa.array(row_ids)) + ) else: response_table = blob_query_response_table() request.send_response(200) request.send_header("Content-Type", "application/vnd.apache.arrow.file") + request.send_header("x-lancedb-version", "1") request.end_headers() with pa.ipc.new_file(request.wfile, response_table.schema) as writer: writer.write_table(response_table) @@ -3400,6 +3406,42 @@ def test_remote_blob_columns_and_fetch(): assert blobs.to_pylist() == [b"alpha", None, b"gamma"] +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +async def test_remote_fetch_blob_ranges(use_async): + with blob_remote_table() as remote_table: + table = remote_table._table if use_async else remote_table + requests = [ + (30, 2, 3), + (10, 1, 3), + (20, 0, 1), + (10, 0, 2), + (40, 0, 0), + (10, 5, 0), + (30, 2, 3), + ] + result = table.fetch_blob_ranges("image", requests) + if use_async: + result = await result + assert isinstance(result, pa.LargeBinaryArray) + assert result.to_pylist() == [b"mma", b"lph", None, b"al", b"", b"", b"mma"] + + for invalid, message in [ + ([(10, 0, 1), (99, 0, 1)], "only 1 exist"), + ([(10, 4, 2)], "exceeds blob size"), + ([(20, 2**64 - 1, 1)], "overflows u64"), + ]: + with pytest.raises(ValueError, match=message): + result = table.fetch_blob_ranges("image", invalid) + if use_async: + await result + + result = table.fetch_blob_ranges("image", []) + if use_async: + result = await result + assert result.to_pylist() == [] + + @pytest.mark.asyncio async def test_async_remote_to_lance_rejects_storage_access(monkeypatch): with blob_remote_table() as remote_table: diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 57843ca55..d21583c91 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2884,12 +2884,10 @@ impl BaseTable for RemoteTable { async fn fetch_blob_ranges( &self, - _column: &str, - _requests: &[crate::blob::BlobRangeRequest], + column: &str, + requests: &[crate::blob::BlobRangeRequest], ) -> Result { - Err(Error::NotSupported { - message: "fetch_blob_ranges is not supported for remote tables.".into(), - }) + self.fetch_blob_ranges_impl(column, requests).await } async fn fetch_blob_files( @@ -5394,6 +5392,14 @@ mod tests { err.to_string() .contains("blobs inside lists cannot be fetched") ); + let err = table + .fetch_blob_ranges(column, [crate::blob::BlobRangeRequest::new(10, 0, 1)]) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("blobs inside lists cannot be fetched") + ); } } @@ -5955,10 +5961,6 @@ mod tests { "optimize", table.optimize(OptimizeAction::All).await.map(|_| ()), ), - ( - "fetch_blob_ranges", - table.fetch_blob_ranges("image", []).await.map(|_| ()), - ), ( "Wand factor", table diff --git a/rust/lancedb/src/remote/table/blobs.rs b/rust/lancedb/src/remote/table/blobs.rs index fdcb8f440..e7219280c 100644 --- a/rust/lancedb/src/remote/table/blobs.rs +++ b/rust/lancedb/src/remote/table/blobs.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; +use arrow_array::builder::LargeBinaryBuilder; use arrow_array::{Array, LargeBinaryArray, StructArray, UInt8Array, UInt64Array}; use arrow_schema::DataType; use bytes::Bytes; @@ -20,7 +21,7 @@ use reqwest::{Response, StatusCode, header}; use tokio::sync::{Mutex, OwnedSemaphorePermit}; use crate::Error; -use crate::blob::BlobFile; +use crate::blob::{BlobFile, BlobRangeRequest}; use crate::error::Result; use crate::query::{QueryFilter, QueryRequest, Select}; use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient}; @@ -538,6 +539,37 @@ impl RemoteTable { Ok(blobs) } + /// Materialize ranges through version-pinned handles, preserving nulls and order. + pub(super) async fn fetch_blob_ranges_impl( + &self, + column: &str, + requests: &[BlobRangeRequest], + ) -> Result { + let row_ids = requests + .iter() + .map(|request| request.row_id) + .collect::>(); + let files = self.fetch_blob_files_impl(column, &row_ids).await?; + let mut builder = LargeBinaryBuilder::with_capacity(requests.len(), 0); + for (file, request) in files.into_iter().zip(requests) { + let end = + request + .offset + .checked_add(request.length) + .ok_or_else(|| Error::InvalidInput { + message: "blob range offset + length overflows u64".into(), + })?; + match file { + Some(file) => { + let bytes = file.read_range(request.offset..end).await?; + builder.append_value(bytes); + } + None => builder.append_null(), + } + } + Ok(builder.finish()) + } + /// Open seekable handles for `row_ids`. /// /// Descriptors for the requested rows come from one `_rowid` take query per @@ -1191,6 +1223,123 @@ mod tests { (table, requests) } + #[tokio::test] + async fn remote_blob_ranges_preserve_order_nulls_and_empty_ranges() { + let (table, requests) = mock_take_blob_table( + vec![ + (10, Some((BlobKind::Inline, PAYLOAD.len() as u64))), + (20, None), + (30, Some((BlobKind::Packed, PAYLOAD.len() as u64))), + (40, Some((BlobKind::Inline, 0))), + ], + Some("7"), + ); + + let ranges = table + .fetch_blob_ranges( + "image", + &[ + BlobRangeRequest::new(30, 6, 5), + BlobRangeRequest::new(10, 0, 3), + BlobRangeRequest::new(20, 0, 1), + BlobRangeRequest::new(10, 2, 2), + BlobRangeRequest::new(40, 0, 0), + BlobRangeRequest::new(10, PAYLOAD.len() as u64, 0), + BlobRangeRequest::new(30, 6, 5), + ], + ) + .await + .unwrap(); + + assert_eq!( + ranges, + LargeBinaryArray::from(vec![ + Some(&PAYLOAD[6..11]), + Some(&PAYLOAD[0..3]), + None, + Some(&PAYLOAD[2..4]), + Some(b"".as_slice()), + Some(b"".as_slice()), + Some(&PAYLOAD[6..11]), + ]) + ); + let requests = requests.lock().unwrap(); + assert_eq!( + requests[0], + TakeMockRequest::Query { + row_ids: vec![10, 20, 30, 40], + version: None, + } + ); + // Only nonempty, non-null ranges are read, at the descriptor take's version. + let reads = requests[1..] + .iter() + .map(|request| match request { + TakeMockRequest::Range { + row_id, + range, + version, + } => { + assert_eq!(version.as_deref(), Some("7")); + (*row_id, range.as_str()) + } + other => panic!("unexpected request: {other:?}"), + }) + .collect::>(); + assert_eq!( + reads, + [ + (30, "bytes=6-10"), + (10, "bytes=0-2"), + (10, "bytes=2-3"), + (30, "bytes=6-10") + ] + ); + } + + #[tokio::test] + async fn remote_blob_ranges_reject_missing_rows_and_invalid_ranges() { + for (ranges, message) in [ + ( + vec![ + BlobRangeRequest::new(10, 0, 1), + BlobRangeRequest::new(99, 0, 1), + ], + "only 1 exist", + ), + (vec![BlobRangeRequest::new(10, 35, 2)], "exceeds blob size"), + (vec![BlobRangeRequest::new(10, 37, 0)], "exceeds blob size"), + ( + vec![BlobRangeRequest::new(10, u64::MAX, 1)], + "overflows u64", + ), + ( + vec![BlobRangeRequest::new(20, u64::MAX, 1)], + "overflows u64", + ), + ] { + let (table, requests) = mock_take_blob_table( + vec![ + (10, Some((BlobKind::Inline, PAYLOAD.len() as u64))), + (20, None), + ], + Some("7"), + ); + let error = table.fetch_blob_ranges("image", &ranges).await.unwrap_err(); + assert!(error.to_string().contains(message), "{error}"); + // Missing rows and invalid ranges fail without reading any bytes. + assert_eq!(requests.lock().unwrap().len(), 1); + } + } + + #[tokio::test] + async fn remote_blob_ranges_accept_empty_requests_without_http() { + let (table, requests) = mock_take_blob_table(vec![], Some("7")); + let ranges = table.fetch_blob_ranges("image", &[]).await.unwrap(); + assert!(ranges.is_empty()); + assert!(requests.lock().unwrap().is_empty()); + } + #[tokio::test] async fn remote_blob_files_are_sized_from_one_descriptor_take() { let payload_size = PAYLOAD.len() as u64; diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index c6193d750..802c2f517 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -1304,6 +1304,8 @@ impl Table { /// ranges for the same blob. The output has the same length and order as /// the requests. Null blobs produce null output slots; empty ranges on /// non-null blobs produce empty byte strings. + /// Remote tables read through version-pinned [`BlobFile`] handles using + /// HTTP byte-range requests. /// /// `_rowid` values stay valid after compaction when the table has stable /// row ids. One missing or deleted row id rejects the whole request; no From 91541408e75214885834bca5f8a7ddd43da51664 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Fri, 16 Oct 2026 13:48:19 -0700 Subject: [PATCH 7/7] chore: update lance dependency to v14.0.0-beta.19 (#4562) Updates the Rust workspace Lance dependencies and Java lance-core from 14.0.0-beta.18 to [v14.0.0-beta.19](https://github.com/lance-format/lance/releases/tag/v14.0.0-beta.19), refreshing Cargo.lock. No compatibility fixes were required. Validation: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and `git diff --check`. Co-authored-by: Daniel Rammer --- Cargo.lock | 92 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 28 ++++++++-------- java/pom.xml | 2 +- 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88aabd4f5..ad9eb6d41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arc-swap", "arrow", @@ -5133,8 +5133,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5156,7 +5156,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5170,7 +5170,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-schema", @@ -5179,8 +5179,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrayref", "crunchy", @@ -5190,8 +5190,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5228,8 +5228,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow", "arrow-array", @@ -5260,8 +5260,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow", "arrow-array", @@ -5278,8 +5278,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "proc-macro2", "quote", @@ -5288,8 +5288,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-arith", "arrow-array", @@ -5322,8 +5322,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-arith", "arrow-array", @@ -5355,8 +5355,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "datafusion", "geo-traits", @@ -5370,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arc-swap", "arrow", @@ -5442,8 +5442,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-schema", @@ -5466,8 +5466,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow", "arrow-array", @@ -5510,8 +5510,8 @@ dependencies = [ [[package]] name = "lance-jsonb" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "byteorder", "ethnum", @@ -5528,8 +5528,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-schema", @@ -5543,8 +5543,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow", "async-trait", @@ -5558,8 +5558,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arc-swap", "arrow", @@ -5613,8 +5613,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5628,8 +5628,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow", "arrow-array", @@ -5669,8 +5669,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-schema", @@ -5683,8 +5683,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index c6b9e84fe..a4aaeeaca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=14.0.0-beta.19", default-features = false, "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=14.0.0-beta.19", default-features = false, "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=14.0.0-beta.19", default-features = false, "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index d6d1fdf98..c85bf04ca 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 14.0.0-beta.18 + 14.0.0-beta.19 2.18.11 false 2.30.0