From 0654cbd2c0d515d545b674e881853b0d7bc82920 Mon Sep 17 00:00:00 2001 From: XYZhan Date: Fri, 25 Sep 2026 18:24:56 -0400 Subject: [PATCH] feat: let a table's maintained index set be changed in place Changing the set meant unsetting the spec and setting it again, which drops the MemWAL and everything in its fresh tier. `set_lsm_write_spec` now accepts a repeat that differs only there and updates the set on the live index; the sharding still cannot move, because the generations already written were homed under it. The default set also stops being a snapshot. It is stored as the intent to maintain everything, which lance re-reads as the table changes, so an index created later is maintained without another call. An index of a kind the MemWAL cannot mirror is skipped under that intent rather than refusing the table, since nobody asked for it by name; a named one is still refused. The LSM read path asks the spec whether an index is maintained rather than searching the stored list, which an unnamed set leaves empty -- reading the list alone would refuse full-text and vector search on every table using the default. --- rust/lancedb/src/table.rs | 101 +++++++++++++++--------- rust/lancedb/src/table/merge/lsm.rs | 118 ++++++++++++++-------------- rust/lancedb/src/table/query/lsm.rs | 72 +++++++++++------ 3 files changed, 168 insertions(+), 123 deletions(-) diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index dcb3e1578..f5372e675 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -330,7 +330,8 @@ pub use self::merge::MergeResult; /// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default /// `ShardWriter` configuration recorded in the MemWAL index). /// -/// A fresh spec maintains every index on the table, resolved on install. +/// A fresh spec maintains every index on the table, including ones created +/// after it is installed. /// /// Install a spec with [`Table::set_lsm_write_spec`] and remove it with /// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch @@ -348,9 +349,8 @@ pub enum LsmWriteSpec { num_buckets: u32, /// Indexes the MemWAL maintains in-memory as rows are appended. /// - /// `None` means every index it can maintain, resolved on install — a - /// snapshot, so indexes created later need the spec unset and re-set. - /// `Some([])` maintains nothing. + /// `None` means every index the table has, including ones created + /// later. `Some([])` maintains nothing. maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, @@ -363,9 +363,8 @@ pub enum LsmWriteSpec { column: String, /// Indexes the MemWAL maintains in-memory as rows are appended. /// - /// `None` means every index it can maintain, resolved on install — a - /// snapshot, so indexes created later need the spec unset and re-set. - /// `Some([])` maintains nothing. + /// `None` means every index the table has, including ones created + /// later. `Some([])` maintains nothing. maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, @@ -374,9 +373,8 @@ pub enum LsmWriteSpec { Unsharded { /// Indexes the MemWAL maintains in-memory as rows are appended. /// - /// `None` means every index it can maintain, resolved on install — a - /// snapshot, so indexes created later need the spec unset and re-set. - /// `Some([])` maintains nothing. + /// `None` means every index the table has, including ones created + /// later. `Some([])` maintains nothing. maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, @@ -422,14 +420,15 @@ impl LsmWriteSpec { /// Set which indexes the MemWAL maintains. /// - /// `None` (the default) resolves to every index on the table at install, - /// failing if one cannot be maintained — name the set to install anyway. A - /// list is verbatim: each name must already exist and be maintainable, and - /// an empty list maintains nothing. + /// `None` (the default) is every index on the table, re-read as the table + /// changes, so an index created later is maintained too; one of a kind the + /// MemWAL cannot mirror is skipped rather than failing the table. A list is + /// verbatim: each name must already exist and be maintainable, and an empty + /// list maintains nothing. /// /// ``` /// # use lancedb::table::LsmWriteSpec; - /// // Every index the table has when the spec is installed: + /// // Every index the table has, now and later: /// LsmWriteSpec::unsharded().with_maintained_indexes(None); /// // Exactly these: /// LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]); @@ -1985,6 +1984,10 @@ impl Table { /// - [`LsmWriteSpec::identity`] — shard by the raw value of a scalar column. /// - [`LsmWriteSpec::unsharded`] — route every write to a single shard. /// + /// Calling this again on a table that already has a spec is how the + /// maintained index set is changed; the sharding cannot move, because the + /// generations already written were homed under it. + /// /// # Example /// /// ``` @@ -5648,29 +5651,22 @@ mod tests { assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); // Identity sharding round-trips (column recovered from the schema). - // A spec left at its default maintains every index on the table, so it - // reads back naming the one on the table rather than as "infer". + // A spec left at its default round-trips as the default: what is stored + // is the intent to maintain everything, not the set it resolves to now. let spec = LsmWriteSpec::identity("region"); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!( - table.get_lsm_write_spec().await.unwrap(), - Some(spec.with_maintained_indexes(vec![idx_name.clone()])) - ); + assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); table.unset_lsm_write_spec().await.unwrap(); // Unsharded round-trips (no routing column). let spec = LsmWriteSpec::unsharded(); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!( - table.get_lsm_write_spec().await.unwrap(), - Some(spec.with_maintained_indexes(vec![idx_name])) - ); + assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); } - /// The maintained set defaults to every index on the table, resolved at - /// install. An index the memtable cannot build fails the install rather - /// than being dropped: maintaining it would take the table offline for - /// writes, dropping it would hide that from the caller. + /// The maintained set defaults to every index on the table. A named index + /// the memtable cannot build fails the install, because the caller asked + /// for it; an unnamed one is skipped, because it arrived by existing. #[tokio::test] async fn test_set_lsm_write_spec_infers_maintained_indexes() { let tmp_dir = tempdir().unwrap(); @@ -5723,19 +5719,25 @@ mod tests { ); assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); - // The default covers every index, so the bitmap fails it too. - let err = table + // The default names nothing, so the bitmap is skipped rather than + // refusing the table: it is maintained by existing, not by being asked + // for. The intent is what is stored, not the set it resolves to today. + table .set_lsm_write_spec(LsmWriteSpec::unsharded()) .await - .unwrap_err(); - assert!( - matches!(err, Error::InvalidInput { ref message } - if message.contains("tag_bitmap") && message.contains("maintained_indexes")), - "expected the inferred set to be rejected, got {err:?}" + .unwrap(); + assert_eq!( + table + .get_lsm_write_spec() + .await + .unwrap() + .unwrap() + .maintained_indexes(), + None, + "an unnamed set is stored as the intent to maintain everything" ); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); - // Naming the maintainable subset installs. + // Naming the maintainable subset narrows it, with no unset in between. table .set_lsm_write_spec( LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]), @@ -5753,7 +5755,6 @@ mod tests { ); // Opting out entirely is distinct from the default. - table.unset_lsm_write_spec().await.unwrap(); table .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new())) .await @@ -5767,6 +5768,28 @@ mod tests { .maintained_indexes(), Some([].as_slice()) ); + + // The sharding cannot move, so the repeat that changes it is refused + // and the installed spec is left alone. + let err = table + .set_lsm_write_spec(LsmWriteSpec::bucket("id", 4)) + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { ref message } + if message.contains("only its maintained index set can be changed")), + "expected the sharding change to be refused, got {err:?}" + ); + assert_eq!( + table + .get_lsm_write_spec() + .await + .unwrap() + .unwrap() + .maintained_indexes(), + Some([].as_slice()), + "a refused call changes nothing" + ); } #[tokio::test] diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index fc925c014..029dfa36c 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -73,27 +73,57 @@ const SHARD_NAMESPACE: Uuid = Uuid::from_u128(0x4c53_4d57_5249_5445_5f53_4841_52 pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) -> Result<()> { table.dataset.ensure_mutable()?; + // A named set is checked against the table here, where the caller can still + // be told which names exist. An unnamed set is not resolved at all: it is an + // intent lance re-reads whenever it builds a MemTable, so an index created + // later is maintained without another call. + let maintained_indexes = match spec.maintained_indexes() { + Some(requested) => { + let indices = table.list_indices().await?; + for name in requested { + if !indices.iter().any(|index| &index.name == name) { + return Err(Error::InvalidInput { + message: format!( + "maintained index '{}' does not exist on this table; it has {}", + name, + index_name_list(&indices), + ), + }); + } + } + // Before the builder borrows the dataset clone. + let dataset = table.dataset.get().await?; + validate_maintained_indexes(&dataset, requested).await?; + Some(requested.to_vec()) + } + None => None, + }; + + // A table that already has a spec accepts one kind of repeat: the same + // sharding with a different maintained set. The rest of a spec describes + // how the generations already written were homed, so it cannot move. { let dataset = table.dataset.get().await?; - if dataset.mem_wal_index_details().await?.is_some() { - return Err(Error::InvalidInput { - message: "set_lsm_write_spec: an LSM write spec is already set on this table; mutation is not supported".into(), - }); + if let Some(details) = dataset.mem_wal_index_details().await? { + let installed = lsm_write_spec_from_details(&details, dataset.schema())?; + if forget_maintained_indexes(&installed) != forget_maintained_indexes(&spec) { + return Err(Error::InvalidInput { + message: "set_lsm_write_spec: an LSM write spec is already set on this \ + table; only its maintained index set can be changed" + .into(), + }); + } + drop(dataset); + table.checkout_latest().await?; + let mut dataset = (*table.dataset.get().await?).clone(); + dataset + .update_mem_wal_maintained_indexes(maintained_indexes) + .await?; + table.dataset.update(dataset); + return Ok(()); } } - // Before the builder borrows the dataset clone. `list_indices` merges an - // index's segments into one entry, so the result needs no dedup. - let maintained_indexes = { - let dataset = table.dataset.get().await?; - resolve_maintained_indexes( - &dataset, - &table.list_indices().await?, - spec.maintained_indexes(), - ) - .await? - }; - table.checkout_latest().await?; let mut dataset = (*table.dataset.get().await?).clone(); let schema = arrow_schema::Schema::from(dataset.schema()); @@ -138,7 +168,9 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) writer_config_defaults } }; - builder = builder.maintained_indexes(maintained_indexes); + if let Some(maintained_indexes) = maintained_indexes { + builder = builder.maintained_indexes(maintained_indexes); + } for (key, value) in writer_config_defaults { builder = builder.add_writer_config_default(key, value); } @@ -147,48 +179,6 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) Ok(()) } -/// Resolve a spec's maintained-index selection against `indices`, as reported -/// by [`Table::list_indices`](crate::Table::list_indices). -/// -/// `None` means every index on the table, snapshotted now. Lance validates -/// either selection against its shard-writer rules, so a spec that installs is -/// one the MemWAL can open. -/// -/// An unmaintainable index fails an inferred set rather than being dropped from -/// it — dropping would leave the caller believing it is maintained. -async fn resolve_maintained_indexes( - dataset: &Dataset, - indices: &[IndexConfig], - requested: Option<&[String]>, -) -> Result> { - let Some(requested) = requested else { - let all: Vec = indices.iter().map(|index| index.name.clone()).collect(); - validate_maintained_indexes(dataset, &all) - .await - .map_err(|source| Error::InvalidInput { - message: format!( - "cannot maintain every index on this table: {source}. Set \ - maintained_indexes explicitly to choose from {}", - index_name_list(indices), - ), - })?; - return Ok(all); - }; - for name in requested { - if !indices.iter().any(|index| &index.name == name) { - return Err(Error::InvalidInput { - message: format!( - "maintained index '{}' does not exist on this table; it has {}", - name, - index_name_list(indices), - ), - }); - } - } - validate_maintained_indexes(dataset, requested).await?; - Ok(requested.to_vec()) -} - /// Index names for an error message. fn index_name_list(indices: &[IndexConfig]) -> String { if indices.is_empty() { @@ -298,10 +288,18 @@ fn lsm_write_spec_from_details( }; Ok(base - .with_maintained_indexes(details.maintained_indexes.clone()) + .with_maintained_indexes( + (!details.maintain_all_indexes).then(|| details.maintained_indexes.clone()), + ) .with_writer_config_defaults(details.writer_config_defaults.clone())) } +/// A spec with its maintained set cleared, so two specs compare on everything +/// that cannot be changed after install. +fn forget_maintained_indexes(spec: &LsmWriteSpec) -> LsmWriteSpec { + spec.clone().with_maintained_indexes(None) +} + /// Resolve the single routing column name from a sharding field's source id. /// /// `set_lsm_write_spec` records the shard column by its Lance field id, so the diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 86c1fe5f2..5f399fce7 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -471,14 +471,7 @@ async fn fts_plan( // Without a maintained in-memory FTS index for this column, the active memtable // arm produces an empty plan (`active_source_can_execute_fts` returns false), so // the search silently omits un-compacted documents. Reject rather than mislead. - if !index_maintained( - dataset, - column, - &details.maintained_indexes, - "InvertedIndexDetails", - ) - .await? - { + if !index_maintained(dataset, column, details, "InvertedIndexDetails").await? { return Err(Error::NotSupported { message: format!( "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise un-compacted documents are omitted. set use_lsm(false) to read the base table only" @@ -493,6 +486,14 @@ async fn fts_plan( Ok(scanner.create_plan().await?) } +/// Whether the spec maintains the index named `name`. +/// +/// An unnamed set names nothing and covers everything, so the stored list alone +/// cannot answer this. +fn spec_maintains(details: &MemWalIndexDetails, name: &str) -> bool { + details.maintain_all_indexes || details.maintained_indexes.iter().any(|n| n == name) +} + /// Whether an index of `type_url_suffix` covering `column` is in the MemWAL spec's /// maintained set. Only a maintained index has its catch-up tracked (so exclusion is /// gated correctly) and its in-memory arm kept current; an unmaintained base index @@ -502,7 +503,7 @@ async fn fts_plan( async fn index_maintained( dataset: &Dataset, column: &str, - maintained: &[String], + details: &MemWalIndexDetails, type_url_suffix: &str, ) -> Result { use lance::index::DatasetIndexExt; @@ -512,7 +513,7 @@ async fn index_maintained( let indices = dataset.load_indices().await?; Ok(indices.iter().any(|idx| { idx.fields.contains(&field.id) - && maintained.iter().any(|m| m == &idx.name) + && spec_maintains(details, &idx.name) && idx .index_details .as_ref() @@ -583,9 +584,7 @@ async fn arm_maintained_index_names( }) .map(|idx| idx.name.clone()) .collect(); - if let Some(name) = - resolve_single_index(segment_names, &details.maintained_indexes, arm, &column)? - { + if let Some(name) = resolve_single_index(segment_names, details, arm, &column)? { names.push(name); } } @@ -604,7 +603,7 @@ async fn arm_maintained_index_names( /// compaction watermark). fn resolve_single_index( mut names: Vec, - maintained: &[String], + details: &MemWalIndexDetails, arm: &str, column: &str, ) -> Result> { @@ -620,7 +619,7 @@ fn resolve_single_index( Ok(names .into_iter() .next() - .filter(|name| maintained.contains(name))) + .filter(|name| spec_maintains(details, name))) } /// Drop the primary-key columns Lance appends internally for dedup when the user's @@ -692,14 +691,7 @@ async fn vector_plan( // maintained, its catch-up is untracked and exclusion falls back to the // compaction watermark — dropping compacted SSTables the (lagging) base index has // not re-indexed. Reject rather than silently omit rows, mirroring the FTS arm. - if !index_maintained( - dataset, - &column, - &details.maintained_indexes, - "VectorIndexDetails", - ) - .await? - { + if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? { return Err(Error::NotSupported { message: format!( "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only" @@ -888,7 +880,10 @@ mod tests { #[test] fn resolve_single_index_dedupes_segments() { - let maintained = vec!["fts_idx".to_string()]; + let maintained = MemWalIndexDetails { + maintained_indexes: vec!["fts_idx".to_string()], + ..Default::default() + }; // Two physical segments of ONE logical index must not count as "multiple". assert_eq!( resolve_single_index( @@ -917,4 +912,33 @@ mod tests { None ); } + + /// A spec that maintains every index names none of them, so the stored list + /// is empty while every index on the table is in fact maintained. Reading + /// the list alone would refuse the read path on the default spec. + #[test] + fn an_unnamed_set_maintains_every_index() { + let all = MemWalIndexDetails { + maintained_indexes: Vec::new(), + maintain_all_indexes: true, + ..Default::default() + }; + assert!(spec_maintains(&all, "fts_idx")); + assert_eq!( + resolve_single_index(vec!["fts_idx".to_string()], &all, "full-text", "text").unwrap(), + Some("fts_idx".to_string()), + ); + + // The same empty list without the flag maintains nothing. + let none = MemWalIndexDetails { + maintained_indexes: Vec::new(), + maintain_all_indexes: false, + ..Default::default() + }; + assert!(!spec_maintains(&none, "fts_idx")); + assert_eq!( + resolve_single_index(vec!["fts_idx".to_string()], &none, "full-text", "text").unwrap(), + None, + ); + } }