From 706a9c327fb324e7961620cf1e159fe8c15f7fab Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Fri, 7 Aug 2026 14:50:22 -0500 Subject: [PATCH] feat: infer maintained indexes when an LsmWriteSpec omits them (#3748) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What `LsmWriteSpec::maintained_indexes` becomes `Option>`: | value | meaning | |---|---| | `None` (new default) | every index the MemWAL supports, resolved when the spec is installed | | `Some([])` | maintain nothing — a scan/filter-only WAL table | | `Some([..])` | exactly these, taken verbatim | `with_maintained_indexes` keeps its signature; `with_no_maintained_indexes()` is new. Surfaced through the remote path (null on the wire), Python, and Node. ## Why Callers had to state the maintained set by hand every time, which is both tedious and easy to get wrong — the common case is "maintain what I already built." Resolution filters on `IndexConfig::is_memwal_maintainable`, delegating to lance's `is_maintainable_index_type`. This is load-bearing rather than cosmetic: lance does **not** skip an index type its memtable cannot build, it errors when the shard writer opens, so sweeping up a bitmap index would fail every memtable claim and leave the table unwritable. The inferred set excludes those, and an explicit list naming one is now rejected at spec time instead of at claim time. ## Behavior change A freshly constructed spec used to maintain **nothing**; it now maintains **everything supported**. This flipped because napi collapses `undefined` and `null` to `None`, so TypeScript cannot express "absent means nothing, null means all" — any other choice makes the bindings disagree with the wire. The error direction also favors it: an unwanted maintained index costs memory, while a silently unmaintained one degrades FTS to an unscored scan. Three existing tests encoded the old default and are updated rather than worked around. ## Caveat The resolved set is a snapshot, not a subscription. An index created after the spec is installed is not maintained until the spec is unset and set again. `get_lsm_write_spec` therefore always reports a concrete list — `None` never round-trips. ## Dependency Needs a lance release carrying `is_maintainable_index_type` (lance-format/lance#8095) before this builds against the pinned tag. Draft until then. ## Testing 38 Rust LSM tests and 10 Python tests pass against a local lance build, including new coverage that a bitmap index is excluded from inference and rejected when named, and that `[]` stays distinguishable from null on the wire. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 (1M context) --- docs/src/js/classes/Table.md | 12 +- docs/src/js/interfaces/LsmWriteSpec.md | 4 +- nodejs/lancedb/table.ts | 18 +- nodejs/src/table.rs | 12 +- python/python/lancedb/_lancedb.pyi | 11 +- python/python/lancedb/table.py | 13 +- python/python/tests/test_lsm_write_spec.py | 15 +- python/python/tests/test_merge_insert_lsm.py | 4 +- python/src/table.rs | 46 +++-- rust/lancedb/src/remote/table.rs | 30 ++- rust/lancedb/src/table.rs | 202 +++++++++++++++---- rust/lancedb/src/table/merge.rs | 4 +- rust/lancedb/src/table/merge/lsm.rs | 82 +++++++- 13 files changed, 360 insertions(+), 93 deletions(-) diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 11fca32d0..3fa3b08db 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -431,9 +431,10 @@ Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on th Resolves to `undefined` when the MemWAL LSM write path is not enabled (no spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)). -The returned spec — including its `maintainedIndexes` and -`writerConfigDefaults` — mirrors what was passed to -[Table#setLsmWriteSpec](Table.md#setlsmwritespec). +The returned spec mirrors what was passed to +[Table#setLsmWriteSpec](Table.md#setlsmwritespec), except that `maintainedIndexes` always +reports the concrete list resolved when the spec was set — `undefined` +never round-trips. #### Returns @@ -806,6 +807,11 @@ All variants require the table to have an unenforced primary key ([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally requires it to be the single column being bucketed. +Omitting `maintainedIndexes` maintains every index on the table, resolved +here, failing if one cannot be maintained — name them to install anyway. +Naming them pins an exact set, and a still-building index is rejected +rather than quietly omitted. + #### Parameters * **spec**: [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md) diff --git a/docs/src/js/interfaces/LsmWriteSpec.md b/docs/src/js/interfaces/LsmWriteSpec.md index 8a588df6a..f2ae91186 100644 --- a/docs/src/js/interfaces/LsmWriteSpec.md +++ b/docs/src/js/interfaces/LsmWriteSpec.md @@ -34,7 +34,9 @@ Bucket and identity variants: the sharding column. optional maintainedIndexes: string[]; ``` -Names of indexes the MemWAL should keep up to date during writes. +Indexes the MemWAL keeps up to date. Omit to maintain every supported +index, resolved on install — a snapshot, so indexes created later are not +maintained. Pass `[]` for none. *** diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 3359a2643..04705475b 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -197,7 +197,11 @@ export interface LsmWriteSpec { column?: string; /** Bucket variant: the number of buckets, in `[1, 1024]`. */ numBuckets?: number; - /** Names of indexes the MemWAL should keep up to date during writes. */ + /** + * Indexes the MemWAL keeps up to date. Omit to maintain every supported + * index, resolved on install — a snapshot, so indexes created later are not + * maintained. Pass `[]` for none. + */ maintainedIndexes?: string[]; /** Default `ShardWriter` configuration recorded in the MemWAL index. */ writerConfigDefaults?: Record; @@ -595,6 +599,11 @@ export abstract class Table { * All variants require the table to have an unenforced primary key * ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally * requires it to be the single column being bucketed. + * + * Omitting `maintainedIndexes` maintains every index on the table, resolved + * here, failing if one cannot be maintained — name them to install anyway. + * Naming them pins an exact set, and a still-building index is rejected + * rather than quietly omitted. * @param {LsmWriteSpec} spec The sharding spec to install. * @returns {Promise} * @example @@ -622,9 +631,10 @@ export abstract class Table { * * Resolves to `undefined` when the MemWAL LSM write path is not enabled (no * spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}). - * The returned spec — including its `maintainedIndexes` and - * `writerConfigDefaults` — mirrors what was passed to - * {@link Table#setLsmWriteSpec}. + * The returned spec mirrors what was passed to + * {@link Table#setLsmWriteSpec}, except that `maintainedIndexes` always + * reports the concrete list resolved when the spec was set — `undefined` + * never round-trips. * @returns {Promise} */ abstract getLsmWriteSpec(): Promise; diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 2ac2fecb2..d26a44845 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -772,7 +772,8 @@ pub struct LsmWriteSpec { pub column: Option, /// Bucket variant: the number of buckets, in `[1, 1024]`. pub num_buckets: Option, - /// Names of indexes the MemWAL should keep up to date during writes. + /// Indexes the MemWAL keeps up to date. Omitted resolves every + /// maintainable index on install; an empty array means none. pub maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. pub writer_config_defaults: Option>, @@ -782,7 +783,6 @@ impl TryFrom for lancedb::table::LsmWriteSpec { type Error = napi::Error; fn try_from(value: LsmWriteSpec) -> napi::Result { - let maintained = value.maintained_indexes.unwrap_or_default(); let writer_config_defaults = value.writer_config_defaults.unwrap_or_default(); let spec = match value.spec_type.as_str() { "bucket" => { @@ -809,7 +809,7 @@ impl TryFrom for lancedb::table::LsmWriteSpec { } }; Ok(spec - .with_maintained_indexes(maintained) + .with_maintained_indexes(value.maintained_indexes) .with_writer_config_defaults(writer_config_defaults)) } } @@ -827,7 +827,7 @@ impl From for LsmWriteSpec { spec_type: "bucket".to_string(), column: Some(column), num_buckets: Some(num_buckets), - maintained_indexes: Some(maintained_indexes), + maintained_indexes, writer_config_defaults: Some(writer_config_defaults), }, Native::Identity { @@ -838,7 +838,7 @@ impl From for LsmWriteSpec { spec_type: "identity".to_string(), column: Some(column), num_buckets: None, - maintained_indexes: Some(maintained_indexes), + maintained_indexes, writer_config_defaults: Some(writer_config_defaults), }, Native::Unsharded { @@ -848,7 +848,7 @@ impl From for LsmWriteSpec { spec_type: "unsharded".to_string(), column: None, num_buckets: None, - maintained_indexes: Some(maintained_indexes), + maintained_indexes, writer_config_defaults: Some(writer_config_defaults), }, } diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index fad2744d3..f87fd3d13 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -653,9 +653,10 @@ class LsmWriteSpec: def identity(column: str) -> "LsmWriteSpec": ... @staticmethod def unsharded() -> "LsmWriteSpec": ... - def with_maintained_indexes(self, indexes: List[str]) -> "LsmWriteSpec": - """Return a copy of this spec asking the MemWAL to keep the named - indexes up to date as rows are appended.""" + def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "LsmWriteSpec": + """Set which indexes the MemWAL keeps up to date. None resolves every + index on the table at install, failing if one cannot be maintained; + a list is verbatim, empty means none.""" ... def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "LsmWriteSpec": """Return a copy of this spec recording the given default @@ -670,7 +671,9 @@ class LsmWriteSpec: @property def num_buckets(self) -> Optional[int]: ... @property - def maintained_indexes(self) -> List[str]: ... + def maintained_indexes(self) -> Optional[List[str]]: + """Indexes the MemWAL keeps up to date, or None for every supported one.""" + ... @property def writer_config_defaults(self) -> Dict[str, str]: ... diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 0828f04dc..f0d7dc8c8 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -4676,6 +4676,13 @@ class AsyncTable: via [`set_unenforced_primary_key`]; bucket sharding additionally requires it to be the single column being bucketed. + By default the MemWAL maintains every index on the table, resolved + here — a snapshot, so an index created afterwards needs the spec unset + and set again. This fails if one cannot be maintained; name the set + with ``with_maintained_indexes`` to install anyway. That pins an exact + set (a still-building index is rejected, not omitted); ``[]`` maintains + none. + Parameters ---------- spec : LsmWriteSpec @@ -4702,9 +4709,9 @@ class AsyncTable: Returns ``None`` when the MemWAL LSM write path is not enabled (no spec has been set, or it was removed with `unset_lsm_write_spec`). - The returned spec — including its ``maintained_indexes`` and - ``writer_config_defaults`` — mirrors what was passed to - `set_lsm_write_spec`. + The returned spec mirrors what was passed to `set_lsm_write_spec`, + except that ``maintained_indexes`` always reports the concrete list + resolved when the spec was set — ``None`` never round-trips. """ return await self._inner.get_lsm_write_spec() diff --git a/python/python/tests/test_lsm_write_spec.py b/python/python/tests/test_lsm_write_spec.py index d38918f09..218793b89 100644 --- a/python/python/tests/test_lsm_write_spec.py +++ b/python/python/tests/test_lsm_write_spec.py @@ -83,7 +83,9 @@ def test_lsm_write_spec_repr(): assert s.spec_type == "bucket" assert s.column == "id" assert s.num_buckets == 4 - assert s.maintained_indexes == [] + # A fresh spec defers its maintained set to install time. + assert s.maintained_indexes is None + assert s.with_maintained_indexes([]).maintained_indexes == [] assert "bucket" in repr(s) assert "id" in repr(s) assert "4" in repr(s) @@ -169,18 +171,23 @@ def test_get_lsm_write_spec(tmp_path): table.unset_lsm_write_spec() assert table.get_lsm_write_spec() is None - # Identity round-trips (column recovered from the schema). + # Identity round-trips (column recovered from the schema). Leaving the + # maintained set to be inferred picks up the index on the table, so the + # spec reads back naming it rather than as "infer". table.set_lsm_write_spec(LsmWriteSpec.identity("id")) spec = table.get_lsm_write_spec() assert spec.spec_type == "identity" assert spec.column == "id" + assert spec.maintained_indexes == [idx_name] table.unset_lsm_write_spec() - # Unsharded round-trips (no routing column). - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + # Unsharded round-trips (no routing column). Opting out is distinct from + # the inferred default. + table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) spec = table.get_lsm_write_spec() assert spec.spec_type == "unsharded" assert spec.column is None + assert spec.maintained_indexes == [] @pytest.mark.asyncio diff --git a/python/python/tests/test_merge_insert_lsm.py b/python/python/tests/test_merge_insert_lsm.py index 5674a05ab..e74c21589 100644 --- a/python/python/tests/test_merge_insert_lsm.py +++ b/python/python/tests/test_merge_insert_lsm.py @@ -544,7 +544,7 @@ def test_lsm_read_fts_unmaintained_index_errors(tmp_path): table.create_index("text", config=FTS()) # No maintained indexes: the active memtable FTS arm cannot serve un-compacted # docs, so the search would silently omit them — reject instead. - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) with pytest.raises(Exception, match="maintained"): table.search("fox", query_type="fts", fts_columns="text").to_arrow() @@ -631,7 +631,7 @@ def test_lsm_read_vector_unmaintained_index_errors(tmp_path): ) # Spec with NO maintained indexes: the base vector index's catch-up is untracked, # so the scanner rejects rather than risk dropping compacted-but-unindexed rows. - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) with pytest.raises(Exception, match="maintained"): table.search([1.0] * VECTOR_DIM).to_arrow() diff --git a/python/src/table.rs b/python/src/table.rs index 119388708..20a93556f 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -246,12 +246,22 @@ impl From for MergeResult { } } +/// Render for `__repr__`, so the default reads as Python's `None` rather than +/// Rust's `Some([..])`. +fn fmt_maintained(maintained: &Option>) -> String { + match maintained { + Some(names) => format!("{:?}", names), + None => "None".to_string(), + } +} + /// Specification selecting Lance's MemWAL LSM-style write path for /// `merge_insert`. /// /// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()` /// classmethods, then optionally chain `with_maintained_indexes(...)` and -/// `with_writer_config_defaults(...)`. +/// `with_writer_config_defaults(...)`. A fresh spec maintains every index the +/// MemWAL supports, resolved on install. #[pyclass(from_py_object)] #[derive(Clone, Debug)] pub struct LsmWriteSpec { @@ -291,11 +301,11 @@ impl LsmWriteSpec { } } - /// Replace the list of indexes the MemWAL should keep up to date as - /// rows are appended. Each name must reference an index that - /// already exists on the table at the time `set_lsm_write_spec` - /// is called. - pub fn with_maintained_indexes(&self, indexes: Vec) -> Self { + /// Set which indexes the MemWAL maintains. `None` (the default) + /// resolves every supported index on install; a list is verbatim, + /// and an empty list maintains nothing. + #[pyo3(signature = (indexes))] + pub fn with_maintained_indexes(&self, indexes: Option>) -> Self { Self { inner: self.inner.clone().with_maintained_indexes(indexes), } @@ -317,23 +327,29 @@ impl LsmWriteSpec { maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={:?}, writer_config_defaults={:?})", - column, num_buckets, maintained_indexes, writer_config_defaults, + "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})", + column, + num_buckets, + fmt_maintained(maintained_indexes), + writer_config_defaults, ), lancedb::table::LsmWriteSpec::Identity { column, maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.identity(column={:?}, maintained_indexes={:?}, writer_config_defaults={:?})", - column, maintained_indexes, writer_config_defaults, + "LsmWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})", + column, + fmt_maintained(maintained_indexes), + writer_config_defaults, ), lancedb::table::LsmWriteSpec::Unsharded { maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.unsharded(maintained_indexes={:?}, writer_config_defaults={:?})", - maintained_indexes, writer_config_defaults, + "LsmWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})", + fmt_maintained(maintained_indexes), + writer_config_defaults, ), } } @@ -368,10 +384,10 @@ impl LsmWriteSpec { } } - /// Names of indexes the MemWAL should keep up to date during writes. + /// Indexes the MemWAL keeps up to date, or `None` for every supported one. #[getter] - pub fn maintained_indexes(&self) -> Vec { - self.inner.maintained_indexes().to_vec() + pub fn maintained_indexes(&self) -> Option> { + self.inner.maintained_indexes().map(<[String]>::to_vec) } /// Default `ShardWriter` configuration recorded by this spec. diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index f3e872cbe..0d843dd54 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2520,9 +2520,9 @@ impl BaseTable for RemoteTable { self.check_mutable().await?; // Map the spec onto the server's request DTO. `sharding` is internally - // tagged on `mode` to mirror sophon's `Sharding` enum; `maintained_indexes` - // and `writer_config_defaults` are sent verbatim (an empty list means "no - // maintained indexes", not "default to all"). + // tagged on `mode` to mirror sophon's `Sharding` enum. A null + // `maintained_indexes` asks the server to resolve every maintainable + // index at HEAD; a list is verbatim, an empty one meaning none. let sharding = match &spec { LsmWriteSpec::Bucket { column, @@ -6599,7 +6599,7 @@ mod tests { .unwrap() }); let spec = crate::table::LsmWriteSpec::unsharded() - .with_maintained_indexes(["id_idx"]) + .with_maintained_indexes(vec!["id_idx".to_string()]) .with_writer_config_defaults([("max_memtable_rows", "1000")]); table.set_lsm_write_spec(spec).await.unwrap(); } @@ -6618,7 +6618,8 @@ mod tests { body["sharding"], serde_json::json!({ "mode": "bucket", "column": "id", "num_buckets": 16 }) ); - assert_eq!(body["maintained_indexes"], serde_json::json!([])); + // An unpinned maintained set sends null: resolve server-side. + assert_eq!(body["maintained_indexes"], serde_json::Value::Null); http::Response::builder().status(200).body("{}").unwrap() }); table @@ -6627,6 +6628,23 @@ mod tests { .unwrap(); } + /// `[]` (none) must stay distinguishable on the wire from null (all). + #[tokio::test] + async fn test_set_lsm_write_spec_no_maintained_indexes() { + let table = Table::new_with_handler("my_table", |request| { + let body = request.body().unwrap().as_bytes().unwrap(); + let body: serde_json::Value = serde_json::from_slice(body).unwrap(); + assert_eq!(body["maintained_indexes"], serde_json::json!([])); + http::Response::builder().status(200).body("{}").unwrap() + }); + table + .set_lsm_write_spec( + crate::table::LsmWriteSpec::bucket("id", 16).with_maintained_indexes(Vec::new()), + ) + .await + .unwrap(); + } + #[tokio::test] async fn test_set_lsm_write_spec_identity() { let table = Table::new_with_handler("my_table", |request| { @@ -6701,7 +6719,7 @@ mod tests { } => { assert_eq!(column, "id"); assert_eq!(num_buckets, 4); - assert_eq!(maintained_indexes, vec!["id_idx".to_string()]); + assert_eq!(maintained_indexes, Some(vec!["id_idx".to_string()])); assert_eq!( writer_config_defaults .get("durable_write") diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 74ab17921..e23bb7c47 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -371,6 +371,8 @@ pub use self::merge::MergeResult; /// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default /// `ShardWriter` configuration recorded in the MemWAL index). /// +/// A fresh spec maintains every index on the table, resolved on install. +/// /// Install a spec with [`Table::set_lsm_write_spec`] and remove it with /// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch /// onto the MemWAL writer is a follow-up. @@ -385,9 +387,12 @@ pub enum LsmWriteSpec { Bucket { column: String, num_buckets: u32, - /// Names of indexes (already created on the table) that the - /// MemWAL should maintain in-memory as rows are appended. - maintained_indexes: Vec, + /// Indexes the MemWAL maintains in-memory as rows are appended. + /// + /// `None` means every index it can maintain, resolved on install — a + /// snapshot, so indexes created later need the spec unset and re-set. + /// `Some([])` maintains nothing. + maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, }, @@ -397,35 +402,41 @@ pub enum LsmWriteSpec { /// distinct value of `column` becomes its own shard. Identity { column: String, - /// Names of indexes (already created on the table) that the - /// MemWAL should maintain in-memory as rows are appended. - maintained_indexes: Vec, + /// Indexes the MemWAL maintains in-memory as rows are appended. + /// + /// `None` means every index it can maintain, resolved on install — a + /// snapshot, so indexes created later need the spec unset and re-set. + /// `Some([])` maintains nothing. + maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, }, /// No sharding — every `merge_insert` call writes to a single MemWAL shard. Unsharded { - /// Names of indexes (already created on the table) that the - /// MemWAL should maintain in-memory as rows are appended. - maintained_indexes: Vec, + /// Indexes the MemWAL maintains in-memory as rows are appended. + /// + /// `None` means every index it can maintain, resolved on install — a + /// snapshot, so indexes created later need the spec unset and re-set. + /// `Some([])` maintains nothing. + maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, }, } impl LsmWriteSpec { - /// Construct a hash-bucket sharding spec with no maintained indexes. + /// Construct a hash-bucket sharding spec maintaining every index on the table. pub fn bucket(column: impl Into, num_buckets: u32) -> Self { Self::Bucket { column: column.into(), num_buckets, - maintained_indexes: Vec::new(), + maintained_indexes: None, writer_config_defaults: HashMap::new(), } } /// Construct an identity-sharding spec (shard by the raw value of - /// `column`) with no maintained indexes. + /// `column`) maintaining every index on the table. /// /// `column` must be a deterministic function of the unenforced primary /// key: every row with a given primary key must always produce the same @@ -437,28 +448,37 @@ impl LsmWriteSpec { pub fn identity(column: impl Into) -> Self { Self::Identity { column: column.into(), - maintained_indexes: Vec::new(), + maintained_indexes: None, writer_config_defaults: HashMap::new(), } } - /// Construct an unsharded spec with no maintained indexes. + /// Construct an unsharded spec maintaining every index on the table. pub fn unsharded() -> Self { Self::Unsharded { - maintained_indexes: Vec::new(), + maintained_indexes: None, writer_config_defaults: HashMap::new(), } } - /// Replace the list of indexes the MemWAL should keep up to date as - /// rows are appended. Each name must reference an index that already - /// exists on the table at the time `set_lsm_write_spec` is called. - pub fn with_maintained_indexes(mut self, indexes: I) -> Self - where - I: IntoIterator, - S: Into, - { - let v: Vec = indexes.into_iter().map(Into::into).collect(); + /// Set which indexes the MemWAL maintains. + /// + /// `None` (the default) resolves to every index on the table at install, + /// failing if one cannot be maintained — name the set to install anyway. A + /// list is verbatim: each name must already exist and be maintainable, and + /// an empty list maintains nothing. + /// + /// ``` + /// # use lancedb::table::LsmWriteSpec; + /// // Every index the table has when the spec is installed: + /// LsmWriteSpec::unsharded().with_maintained_indexes(None); + /// // Exactly these: + /// LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]); + /// // None at all: + /// LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new()); + /// ``` + pub fn with_maintained_indexes(mut self, indexes: impl Into>>) -> Self { + let indexes = indexes.into(); match &mut self { Self::Bucket { maintained_indexes, .. @@ -468,7 +488,7 @@ impl LsmWriteSpec { } | Self::Unsharded { maintained_indexes, .. - } => *maintained_indexes = v, + } => *maintained_indexes = indexes, } self } @@ -504,8 +524,9 @@ impl LsmWriteSpec { self } - /// Borrow the list of index names this spec asks MemWAL to maintain. - pub fn maintained_indexes(&self) -> &[String] { + /// Borrow the list of index names this spec asks MemWAL to maintain, or + /// `None` when it asks for every index on the table. + pub fn maintained_indexes(&self) -> Option<&[String]> { match self { Self::Bucket { maintained_indexes, .. @@ -515,7 +536,7 @@ impl LsmWriteSpec { } | Self::Unsharded { maintained_indexes, .. - } => maintained_indexes, + } => maintained_indexes.as_deref(), } } @@ -1713,7 +1734,7 @@ impl Table { /// # async fn example(table: &Table) -> Result<(), Box> { /// table /// .set_lsm_write_spec( - /// LsmWriteSpec::bucket("id", 16).with_maintained_indexes(["id_idx"]), + /// LsmWriteSpec::bucket("id", 16).with_maintained_indexes(vec!["id_idx".to_string()]), /// ) /// .await?; /// # Ok(()) @@ -1735,9 +1756,10 @@ impl Table { /// /// Returns `Ok(None)` when the MemWAL LSM write path is not enabled (no /// spec has been set, or it was removed with [`Table::unset_lsm_write_spec`]). - /// The returned spec — including its [`LsmWriteSpec::maintained_indexes`] and - /// [`LsmWriteSpec::writer_config_defaults`] — mirrors what was passed to - /// [`Table::set_lsm_write_spec`]. + /// The returned spec mirrors what was passed to + /// [`Table::set_lsm_write_spec`], except that + /// [`LsmWriteSpec::maintained_indexes`] always reports the concrete list + /// resolved when the spec was set — `None` never round-trips. /// /// # Example /// @@ -5065,7 +5087,7 @@ mod tests { // Bucket spec round-trips exactly, including the routing column (recovered // from its field id), maintained indexes, and writer config defaults. let spec = LsmWriteSpec::bucket("id", 4) - .with_maintained_indexes([idx_name]) + .with_maintained_indexes(vec![idx_name.clone()]) .with_writer_config_defaults([("durable_write", "false")]); table.set_lsm_write_spec(spec.clone()).await.unwrap(); assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); @@ -5075,15 +5097,125 @@ mod tests { assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); // Identity sharding round-trips (column recovered from the schema). + // A spec left at its default maintains every index on the table, so it + // reads back naming the one on the table rather than as "infer". let spec = LsmWriteSpec::identity("region"); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); + assert_eq!( + table.get_lsm_write_spec().await.unwrap(), + Some(spec.with_maintained_indexes(vec![idx_name.clone()])) + ); table.unset_lsm_write_spec().await.unwrap(); // Unsharded round-trips (no routing column). let spec = LsmWriteSpec::unsharded(); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); + assert_eq!( + table.get_lsm_write_spec().await.unwrap(), + Some(spec.with_maintained_indexes(vec![idx_name])) + ); + } + + /// The maintained set defaults to every index on the table, resolved at + /// install. An index the memtable cannot build fails the install rather + /// than being dropped: maintaining it would take the table offline for + /// writes, dropping it would hide that from the caller. + #[tokio::test] + async fn test_set_lsm_write_spec_infers_maintained_indexes() { + let tmp_dir = tempdir().unwrap(); + let uri = tmp_dir.path().to_str().unwrap(); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("tag", DataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(arrow_array::Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + ], + ) + .unwrap(); + let reader: Box = + Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema.clone())); + let conn = ConnectBuilder::new(uri) + .read_consistency_interval(Duration::from_secs(0)) + .execute() + .await + .unwrap(); + let table = conn.create_table("t", reader).execute().await.unwrap(); + + table + .create_index(&["id"], Index::BTree(Default::default())) + .name("id_btree".to_string()) + .execute() + .await + .unwrap(); + table + .create_index(&["tag"], Index::Bitmap(Default::default())) + .name("tag_bitmap".to_string()) + .execute() + .await + .unwrap(); + + // Explicitly naming the bitmap index fails before anything commits. + let err = table + .set_lsm_write_spec( + LsmWriteSpec::unsharded().with_maintained_indexes(vec!["tag_bitmap".to_string()]), + ) + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { ref message } if message.contains("tag_bitmap")), + "expected the bitmap index to be rejected, got {err:?}" + ); + assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); + + // The default covers every index, so the bitmap fails it too. + let err = table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { ref message } + if message.contains("tag_bitmap") && message.contains("maintained_indexes")), + "expected the inferred set to be rejected, got {err:?}" + ); + assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); + + // Naming the maintainable subset installs. + table + .set_lsm_write_spec( + LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]), + ) + .await + .unwrap(); + assert_eq!( + table + .get_lsm_write_spec() + .await + .unwrap() + .unwrap() + .maintained_indexes(), + Some(["id_btree".to_string()].as_slice()) + ); + + // Opting out entirely is distinct from the default. + table.unset_lsm_write_spec().await.unwrap(); + table + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new())) + .await + .unwrap(); + assert_eq!( + table + .get_lsm_write_spec() + .await + .unwrap() + .unwrap() + .maintained_indexes(), + Some([].as_slice()) + ); } #[tokio::test] diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 82a1d1473..3a5b6882d 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -1161,7 +1161,7 @@ mod lsm_tests { .unwrap(); let fts_index = table.list_indices().await.unwrap()[0].name.clone(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([fts_index])) + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) .await .unwrap(); @@ -1254,7 +1254,7 @@ mod lsm_tests { .unwrap(); let vec_index = table.list_indices().await.unwrap()[0].name.clone(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([vec_index])) + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![vec_index])) .await .unwrap(); diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index 0eb7c0231..87c427b3c 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -29,6 +29,7 @@ use arrow_schema::{DataType, Schema as ArrowSchema, SchemaRef}; use lance::Dataset; use lance::dataset::mem_wal::{ DatasetMemWalExt, ShardWriter, ShardWriterConfig, evaluate_sharding_spec, + validate_maintained_indexes, }; use lance::index::DatasetIndexExt; use lance_core::datatypes::Schema as LanceSchema; @@ -37,8 +38,9 @@ use tokio::sync::RwLock; use uuid::Uuid; use crate::error::{Error, Result}; +use crate::index::IndexConfig; use crate::table::merge::{MergeInsertBuilder, MergeResult}; -use crate::table::{LsmWriteSpec, NativeTable}; +use crate::table::{BaseTable, LsmWriteSpec, NativeTable}; /// Spec id of the sole sharding spec installed by [`set_lsm_write_spec`]. /// Must match Lance's `InitializeMemWalBuilder` (`SHARDING_SPEC_ID`). @@ -80,32 +82,44 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) } } + // Before the builder borrows the dataset clone. `list_indices` merges an + // index's segments into one entry, so the result needs no dedup. + let maintained_indexes = { + let dataset = table.dataset.get().await?; + resolve_maintained_indexes( + &dataset, + &table.list_indices().await?, + spec.maintained_indexes(), + ) + .await? + }; + let mut dataset = (*table.dataset.get().await?).clone(); let mut builder = dataset.initialize_mem_wal(); - let (maintained_indexes, writer_config_defaults) = match spec { + let writer_config_defaults = match spec { LsmWriteSpec::Bucket { column, num_buckets, - maintained_indexes, writer_config_defaults, + .. } => { builder = builder.bucket_sharding(column, num_buckets); - (maintained_indexes, writer_config_defaults) + writer_config_defaults } LsmWriteSpec::Identity { column, - maintained_indexes, writer_config_defaults, + .. } => { builder = builder.identity_sharding(column); - (maintained_indexes, writer_config_defaults) + writer_config_defaults } LsmWriteSpec::Unsharded { - maintained_indexes, writer_config_defaults, + .. } => { builder = builder.unsharded(); - (maintained_indexes, writer_config_defaults) + writer_config_defaults } }; builder = builder.maintained_indexes(maintained_indexes); @@ -117,6 +131,58 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) Ok(()) } +/// Resolve a spec's maintained-index selection against `indices`, as reported +/// by [`Table::list_indices`](crate::Table::list_indices). +/// +/// `None` means every index on the table, snapshotted now. Lance validates +/// either selection against its shard-writer rules, so a spec that installs is +/// one the MemWAL can open. +/// +/// An unmaintainable index fails an inferred set rather than being dropped from +/// it — dropping would leave the caller believing it is maintained. +async fn resolve_maintained_indexes( + dataset: &Dataset, + indices: &[IndexConfig], + requested: Option<&[String]>, +) -> Result> { + let Some(requested) = requested else { + let all: Vec = indices.iter().map(|index| index.name.clone()).collect(); + validate_maintained_indexes(dataset, &all) + .await + .map_err(|source| Error::InvalidInput { + message: format!( + "cannot maintain every index on this table: {source}. Set \ + maintained_indexes explicitly to choose from {}", + index_name_list(indices), + ), + })?; + return Ok(all); + }; + for name in requested { + if !indices.iter().any(|index| &index.name == name) { + return Err(Error::InvalidInput { + message: format!( + "maintained index '{}' does not exist on this table; it has {}", + name, + index_name_list(indices), + ), + }); + } + } + validate_maintained_indexes(dataset, requested).await?; + Ok(requested.to_vec()) +} + +/// Index names for an error message. +fn index_name_list(indices: &[IndexConfig]) -> String { + if indices.is_empty() { + return "no indexes".to_string(); + } + let mut names: Vec<&str> = indices.iter().map(|index| index.name.as_str()).collect(); + names.sort_unstable(); + format!("[{}]", names.join(", ")) +} + // ============================================================================= // unset_lsm_write_spec // =============================================================================