diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index dcb3e1578..f5372e675 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -330,7 +330,8 @@ pub use self::merge::MergeResult; /// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default /// `ShardWriter` configuration recorded in the MemWAL index). /// -/// A fresh spec maintains every index on the table, resolved on install. +/// A fresh spec maintains every index on the table, including ones created +/// after it is installed. /// /// Install a spec with [`Table::set_lsm_write_spec`] and remove it with /// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch @@ -348,9 +349,8 @@ pub enum LsmWriteSpec { num_buckets: u32, /// Indexes the MemWAL maintains in-memory as rows are appended. /// - /// `None` means every index it can maintain, resolved on install — a - /// snapshot, so indexes created later need the spec unset and re-set. - /// `Some([])` maintains nothing. + /// `None` means every index the table has, including ones created + /// later. `Some([])` maintains nothing. maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, @@ -363,9 +363,8 @@ pub enum LsmWriteSpec { column: String, /// Indexes the MemWAL maintains in-memory as rows are appended. /// - /// `None` means every index it can maintain, resolved on install — a - /// snapshot, so indexes created later need the spec unset and re-set. - /// `Some([])` maintains nothing. + /// `None` means every index the table has, including ones created + /// later. `Some([])` maintains nothing. maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, @@ -374,9 +373,8 @@ pub enum LsmWriteSpec { Unsharded { /// Indexes the MemWAL maintains in-memory as rows are appended. /// - /// `None` means every index it can maintain, resolved on install — a - /// snapshot, so indexes created later need the spec unset and re-set. - /// `Some([])` maintains nothing. + /// `None` means every index the table has, including ones created + /// later. `Some([])` maintains nothing. maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, @@ -422,14 +420,15 @@ impl LsmWriteSpec { /// Set which indexes the MemWAL maintains. /// - /// `None` (the default) resolves to every index on the table at install, - /// failing if one cannot be maintained — name the set to install anyway. A - /// list is verbatim: each name must already exist and be maintainable, and - /// an empty list maintains nothing. + /// `None` (the default) is every index on the table, re-read as the table + /// changes, so an index created later is maintained too; one of a kind the + /// MemWAL cannot mirror is skipped rather than failing the table. A list is + /// verbatim: each name must already exist and be maintainable, and an empty + /// list maintains nothing. /// /// ``` /// # use lancedb::table::LsmWriteSpec; - /// // Every index the table has when the spec is installed: + /// // Every index the table has, now and later: /// LsmWriteSpec::unsharded().with_maintained_indexes(None); /// // Exactly these: /// LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]); @@ -1985,6 +1984,10 @@ impl Table { /// - [`LsmWriteSpec::identity`] — shard by the raw value of a scalar column. /// - [`LsmWriteSpec::unsharded`] — route every write to a single shard. /// + /// Calling this again on a table that already has a spec is how the + /// maintained index set is changed; the sharding cannot move, because the + /// generations already written were homed under it. + /// /// # Example /// /// ``` @@ -5648,29 +5651,22 @@ mod tests { assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); // Identity sharding round-trips (column recovered from the schema). - // A spec left at its default maintains every index on the table, so it - // reads back naming the one on the table rather than as "infer". + // A spec left at its default round-trips as the default: what is stored + // is the intent to maintain everything, not the set it resolves to now. let spec = LsmWriteSpec::identity("region"); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!( - table.get_lsm_write_spec().await.unwrap(), - Some(spec.with_maintained_indexes(vec![idx_name.clone()])) - ); + assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); table.unset_lsm_write_spec().await.unwrap(); // Unsharded round-trips (no routing column). let spec = LsmWriteSpec::unsharded(); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!( - table.get_lsm_write_spec().await.unwrap(), - Some(spec.with_maintained_indexes(vec![idx_name])) - ); + assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); } - /// The maintained set defaults to every index on the table, resolved at - /// install. An index the memtable cannot build fails the install rather - /// than being dropped: maintaining it would take the table offline for - /// writes, dropping it would hide that from the caller. + /// The maintained set defaults to every index on the table. A named index + /// the memtable cannot build fails the install, because the caller asked + /// for it; an unnamed one is skipped, because it arrived by existing. #[tokio::test] async fn test_set_lsm_write_spec_infers_maintained_indexes() { let tmp_dir = tempdir().unwrap(); @@ -5723,19 +5719,25 @@ mod tests { ); assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); - // The default covers every index, so the bitmap fails it too. - let err = table + // The default names nothing, so the bitmap is skipped rather than + // refusing the table: it is maintained by existing, not by being asked + // for. The intent is what is stored, not the set it resolves to today. + table .set_lsm_write_spec(LsmWriteSpec::unsharded()) .await - .unwrap_err(); - assert!( - matches!(err, Error::InvalidInput { ref message } - if message.contains("tag_bitmap") && message.contains("maintained_indexes")), - "expected the inferred set to be rejected, got {err:?}" + .unwrap(); + assert_eq!( + table + .get_lsm_write_spec() + .await + .unwrap() + .unwrap() + .maintained_indexes(), + None, + "an unnamed set is stored as the intent to maintain everything" ); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); - // Naming the maintainable subset installs. + // Naming the maintainable subset narrows it, with no unset in between. table .set_lsm_write_spec( LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]), @@ -5753,7 +5755,6 @@ mod tests { ); // Opting out entirely is distinct from the default. - table.unset_lsm_write_spec().await.unwrap(); table .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new())) .await @@ -5767,6 +5768,28 @@ mod tests { .maintained_indexes(), Some([].as_slice()) ); + + // The sharding cannot move, so the repeat that changes it is refused + // and the installed spec is left alone. + let err = table + .set_lsm_write_spec(LsmWriteSpec::bucket("id", 4)) + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { ref message } + if message.contains("only its maintained index set can be changed")), + "expected the sharding change to be refused, got {err:?}" + ); + assert_eq!( + table + .get_lsm_write_spec() + .await + .unwrap() + .unwrap() + .maintained_indexes(), + Some([].as_slice()), + "a refused call changes nothing" + ); } #[tokio::test] diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 0a0d00da1..4d7b0fff8 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -160,6 +160,10 @@ impl NativeTable { builder = builder.name(name); } builder.await?; + self.dataset + .shard_writer() + .refresh_maintained_indexes(&dataset) + .await; self.dataset.update(dataset); Ok(()) } diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 8f5585829..89d44baa1 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -1251,6 +1251,30 @@ mod lsm_tests { ); } + /// A table with `id`/`text` and a primary key, ready for an LSM write spec. + async fn lsm_text_table(dir: &tempfile::TempDir) -> crate::Table { + let conn = connect(dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let table = conn + .create_table("t", id_text_reader(vec![(1, "alpha")])) + .execute() + .await + .unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); + table + } + + /// Upsert `rows` through the LSM write path. + async fn upsert_text(table: &crate::Table, rows: Vec<(i64, &str)>) { + let mut builder = table.merge_insert(&[]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + builder.execute(id_text_reader(rows)).await.unwrap(); + } + /// A reader of `[id: Int64, text: Utf8]` rows. fn id_text_reader(rows: Vec<(i64, &str)>) -> Box { let schema = Arc::new(Schema::new(vec![ @@ -1270,6 +1294,148 @@ mod lsm_tests { Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema)) } + /// Under the default set, an index created while a writer is open reaches + /// that writer. + /// + /// The default maintains every index the table has, so creating one changes + /// what the table maintains without anyone touching the spec. A writer left + /// on its old configs would keep building MemTables the new index does not + /// cover. + #[tokio::test] + async fn lsm_default_set_picks_up_an_index_created_while_writing() { + use crate::index::Index; + use lance_index::scalar::FullTextSearchQuery; + + let dir = tempdir().unwrap(); + let table = lsm_text_table(&dir).await; + // The default set, on a table with no FTS index yet. + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + + // Opens the writer, whose MemTable carries no FTS index. + upsert_text(&table, vec![(99, "zebra")]).await; + + // Created with the writer still open and never reopened. + table + .create_index(&["text"], Index::FTS(Default::default())) + .execute() + .await + .unwrap(); + + upsert_text(&table, vec![(100, "zebra stripes")]).await; + + let query = FullTextSearchQuery::new("stripes".to_string()) + .with_column("text".to_string()) + .unwrap(); + let batches = table + .query() + .full_text_search(query) + .execute() + .await + .expect("the created index reached the open writer") + .try_collect::>() + .await + .unwrap(); + let found: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(found, 1, "the row written after create_index must be found"); + } + + /// Changing the set on a live table reaches the writer already open on it. + /// + /// Without that, the commit would land and the writer would keep building + /// MemTables from the old set, so a query needing the newly maintained + /// index would stay refused until something reopened the writer. + #[tokio::test] + async fn lsm_set_change_reaches_an_already_open_writer() { + use crate::index::Index; + use lance_index::scalar::FullTextSearchQuery; + + let dir = tempdir().unwrap(); + let table = lsm_text_table(&dir).await; + table + .create_index(&["text"], Index::FTS(Default::default())) + .execute() + .await + .unwrap(); + let fts_index = table.list_indices().await.unwrap()[0].name.clone(); + + // Installed maintaining nothing, then written to: the writer opens with + // no FTS index. + table + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new())) + .await + .unwrap(); + upsert_text(&table, vec![(99, "zebra")]).await; + + // Change the set in place on the live table. + table + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) + .await + .unwrap(); + + // A row written after the change is answered by the index the writer + // picked up. + upsert_text(&table, vec![(100, "zebra stripes")]).await; + + let query = FullTextSearchQuery::new("stripes".to_string()) + .with_column("text".to_string()) + .unwrap(); + let batches = table + .query() + .full_text_search(query) + .execute() + .await + .expect("the set change reached the open writer") + .try_collect::>() + .await + .unwrap(); + let found: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(found, 1, "the row written after the change must be found"); + } + + /// A writer opened before the index existed keeps a MemTable without it, so + /// an indexed read is refused rather than served without those rows. + /// + /// The default set maintains every index the table has, which the spec + /// records as intent. That intent alone would let the read through while + /// the live MemTable still cannot answer it. + #[tokio::test] + async fn lsm_read_refuses_when_a_resident_memtable_predates_the_index() { + use crate::index::Index; + use lance_index::scalar::FullTextSearchQuery; + + let dir = tempdir().unwrap(); + let table = lsm_text_table(&dir).await; + // No index yet, and the default set: maintain whatever the table has. + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + + // Opens the writer, whose MemTable therefore carries no FTS index. + upsert_text(&table, vec![(99, "zebra")]).await; + + // Built afterwards: the spec now resolves to it, the MemTable does not. + table + .create_index(&["text"], Index::FTS(Default::default())) + .execute() + .await + .unwrap(); + + let query = FullTextSearchQuery::new("zebra".to_string()) + .with_column("text".to_string()) + .unwrap(); + let Err(error) = table.query().full_text_search(query).execute().await else { + panic!("an indexed read the resident MemTable cannot answer must be refused"); + }; + assert!( + error.to_string().contains("resident MemTable"), + "unexpected error: {error}" + ); + } + #[tokio::test] async fn lsm_read_full_text_search() { use crate::index::Index; diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index fc925c014..bd8acd8a8 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -73,27 +73,62 @@ const SHARD_NAMESPACE: Uuid = Uuid::from_u128(0x4c53_4d57_5249_5445_5f53_4841_52 pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) -> Result<()> { table.dataset.ensure_mutable()?; + // A named set is checked against the table here, where the caller can still + // be told which names exist. An unnamed set is not resolved at all: it is an + // intent lance re-reads whenever it builds a MemTable, so an index created + // later is maintained without another call. + let maintained_indexes = match spec.maintained_indexes() { + Some(requested) => { + let indices = table.list_indices().await?; + for name in requested { + if !indices.iter().any(|index| &index.name == name) { + return Err(Error::InvalidInput { + message: format!( + "maintained index '{}' does not exist on this table; it has {}", + name, + index_name_list(&indices), + ), + }); + } + } + // Before the builder borrows the dataset clone. + let dataset = table.dataset.get().await?; + validate_maintained_indexes(&dataset, requested).await?; + Some(requested.to_vec()) + } + None => None, + }; + + // A table that already has a spec accepts one kind of repeat: the same + // sharding with a different maintained set. The rest of a spec describes + // how the generations already written were homed, so it cannot move. { let dataset = table.dataset.get().await?; - if dataset.mem_wal_index_details().await?.is_some() { - return Err(Error::InvalidInput { - message: "set_lsm_write_spec: an LSM write spec is already set on this table; mutation is not supported".into(), - }); + if let Some(details) = dataset.mem_wal_index_details().await? { + let installed = lsm_write_spec_from_details(&details, dataset.schema())?; + if !same_except_maintained_indexes(&installed, &spec) { + return Err(Error::InvalidInput { + message: "set_lsm_write_spec: an LSM write spec is already set on this \ + table; only its maintained index set can be changed" + .into(), + }); + } + drop(dataset); + table.checkout_latest().await?; + let mut dataset = (*table.dataset.get().await?).clone(); + dataset + .update_mem_wal_maintained_indexes(maintained_indexes) + .await?; + table + .dataset + .shard_writer() + .refresh_maintained_indexes(&dataset) + .await; + table.dataset.update(dataset); + return Ok(()); } } - // Before the builder borrows the dataset clone. `list_indices` merges an - // index's segments into one entry, so the result needs no dedup. - let maintained_indexes = { - let dataset = table.dataset.get().await?; - resolve_maintained_indexes( - &dataset, - &table.list_indices().await?, - spec.maintained_indexes(), - ) - .await? - }; - table.checkout_latest().await?; let mut dataset = (*table.dataset.get().await?).clone(); let schema = arrow_schema::Schema::from(dataset.schema()); @@ -138,7 +173,9 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) writer_config_defaults } }; - builder = builder.maintained_indexes(maintained_indexes); + if let Some(maintained_indexes) = maintained_indexes { + builder = builder.maintained_indexes(maintained_indexes); + } for (key, value) in writer_config_defaults { builder = builder.add_writer_config_default(key, value); } @@ -147,48 +184,6 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) Ok(()) } -/// Resolve a spec's maintained-index selection against `indices`, as reported -/// by [`Table::list_indices`](crate::Table::list_indices). -/// -/// `None` means every index on the table, snapshotted now. Lance validates -/// either selection against its shard-writer rules, so a spec that installs is -/// one the MemWAL can open. -/// -/// An unmaintainable index fails an inferred set rather than being dropped from -/// it — dropping would leave the caller believing it is maintained. -async fn resolve_maintained_indexes( - dataset: &Dataset, - indices: &[IndexConfig], - requested: Option<&[String]>, -) -> Result> { - let Some(requested) = requested else { - let all: Vec = indices.iter().map(|index| index.name.clone()).collect(); - validate_maintained_indexes(dataset, &all) - .await - .map_err(|source| Error::InvalidInput { - message: format!( - "cannot maintain every index on this table: {source}. Set \ - maintained_indexes explicitly to choose from {}", - index_name_list(indices), - ), - })?; - return Ok(all); - }; - for name in requested { - if !indices.iter().any(|index| &index.name == name) { - return Err(Error::InvalidInput { - message: format!( - "maintained index '{}' does not exist on this table; it has {}", - name, - index_name_list(indices), - ), - }); - } - } - validate_maintained_indexes(dataset, requested).await?; - Ok(requested.to_vec()) -} - /// Index names for an error message. fn index_name_list(indices: &[IndexConfig]) -> String { if indices.is_empty() { @@ -298,10 +293,17 @@ fn lsm_write_spec_from_details( }; Ok(base - .with_maintained_indexes(details.maintained_indexes.clone()) + .with_maintained_indexes( + (!details.maintain_all_indexes).then(|| details.maintained_indexes.clone()), + ) .with_writer_config_defaults(details.writer_config_defaults.clone())) } +/// Whether two specs agree on everything an installed spec cannot change. +fn same_except_maintained_indexes(a: &LsmWriteSpec, b: &LsmWriteSpec) -> bool { + a.clone().with_maintained_indexes(None) == b.clone().with_maintained_indexes(None) +} + /// Resolve the single routing column name from a sharding field's source id. /// /// `set_lsm_write_spec` records the shard column by its Lance field id, so the @@ -460,6 +462,37 @@ impl ShardWriterCache { Ok(entry) } + /// Install `dataset`'s maintained index set on the cached writer, if one is + /// open, so a change reaches a table that is already being written to. + /// + /// Reported rather than propagated: both callers have already committed the + /// change durably, and the writer this process happens to hold may be + /// fenced or poisoned -- which is not a reason to fail creating an index. + /// Any writer opened afterwards reads the committed set. + /// + /// Holds the entry's read lock across the refresh, so a concurrent close + /// waits rather than retiring the writer mid-seal. Writes take the same + /// lock first, so the order matches theirs. + pub(crate) async fn refresh_maintained_indexes(&self, dataset: &Dataset) { + let entry = { + let guard = self.slot.read().await; + guard.as_ref().map(|(_, entry)| entry.clone()) + }; + let Some(entry) = entry else { + return; + }; + let guard = entry.inner.read().await; + let Some(writer) = guard.as_ref() else { + return; + }; + if let Err(error) = dataset.refresh_mem_wal_index_configs(writer).await { + log::warn!( + "the maintained index set changed but the open shard writer could \ + not be refreshed, so it keeps the set it opened with: {error}" + ); + } + } + /// Snapshot the cached writer's shard for the LSM read path: its shard id, /// authoritative in-memory manifest, and active + frozen memtable refs. /// Returns `None` when no writer is currently cached (e.g. nothing has been diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 86c1fe5f2..bdeb2e1f4 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -33,7 +33,7 @@ use datafusion_physical_plan::expressions::Column; use datafusion_physical_plan::projection::ProjectionExec; use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr}; use lance::Dataset; -use lance::dataset::mem_wal::scanner::InMemoryMemTables; +use lance::dataset::mem_wal::scanner::{InMemoryMemTableRef, InMemoryMemTables}; use lance::dataset::mem_wal::{ DatasetMemWalExt, LsmScanner, ShardManifestStore, ShardSnapshot, ShardWriterConfig, }; @@ -471,17 +471,14 @@ async fn fts_plan( // Without a maintained in-memory FTS index for this column, the active memtable // arm produces an empty plan (`active_source_can_execute_fts` returns false), so // the search silently omits un-compacted documents. Reject rather than mislead. - if !index_maintained( - dataset, - column, - &details.maintained_indexes, - "InvertedIndexDetails", - ) - .await? + if !index_maintained(dataset, column, details, "InvertedIndexDetails").await? + || !resident_memtables_carry(&in_memory, |memtable| { + memtable.index_store.get_fts_by_column(column).is_some() + }) { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise un-compacted documents are omitted. set use_lsm(false) to read the base table only" + "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise un-compacted documents are omitted. set use_lsm(false) to read the base table only" ), }); } @@ -493,6 +490,27 @@ async fn fts_plan( Ok(scanner.create_plan().await?) } +/// Whether the spec maintains the index named `name`. +/// +/// An unnamed set names nothing and covers everything, so the stored list alone +/// cannot answer this. +fn spec_maintains(details: &MemWalIndexDetails, name: &str) -> bool { + details.maintain_all_indexes || details.maintained_indexes.iter().any(|n| n == name) +} + +/// Whether every resident MemTable carries the index an arm needs. +/// +/// The spec is intent; a MemTable built before the index joined the set +/// contributes nothing to an indexed read, so its rows would go missing. +fn resident_memtables_carry( + in_memory: &HashMap, + carries: impl Fn(&InMemoryMemTableRef) -> bool, +) -> bool { + in_memory + .values() + .all(|tables| carries(&tables.active) && tables.frozen.iter().all(&carries)) +} + /// Whether an index of `type_url_suffix` covering `column` is in the MemWAL spec's /// maintained set. Only a maintained index has its catch-up tracked (so exclusion is /// gated correctly) and its in-memory arm kept current; an unmaintained base index @@ -502,7 +520,7 @@ async fn fts_plan( async fn index_maintained( dataset: &Dataset, column: &str, - maintained: &[String], + details: &MemWalIndexDetails, type_url_suffix: &str, ) -> Result { use lance::index::DatasetIndexExt; @@ -512,7 +530,7 @@ async fn index_maintained( let indices = dataset.load_indices().await?; Ok(indices.iter().any(|idx| { idx.fields.contains(&field.id) - && maintained.iter().any(|m| m == &idx.name) + && spec_maintains(details, &idx.name) && idx .index_details .as_ref() @@ -583,9 +601,7 @@ async fn arm_maintained_index_names( }) .map(|idx| idx.name.clone()) .collect(); - if let Some(name) = - resolve_single_index(segment_names, &details.maintained_indexes, arm, &column)? - { + if let Some(name) = resolve_single_index(segment_names, details, arm, &column)? { names.push(name); } } @@ -604,7 +620,7 @@ async fn arm_maintained_index_names( /// compaction watermark). fn resolve_single_index( mut names: Vec, - maintained: &[String], + details: &MemWalIndexDetails, arm: &str, column: &str, ) -> Result> { @@ -620,7 +636,7 @@ fn resolve_single_index( Ok(names .into_iter() .next() - .filter(|name| maintained.contains(name))) + .filter(|name| spec_maintains(details, name))) } /// Drop the primary-key columns Lance appends internally for dedup when the user's @@ -692,17 +708,14 @@ async fn vector_plan( // maintained, its catch-up is untracked and exclusion falls back to the // compaction watermark — dropping compacted SSTables the (lagging) base index has // not re-indexed. Reject rather than silently omit rows, mirroring the FTS arm. - if !index_maintained( - dataset, - &column, - &details.maintained_indexes, - "VectorIndexDetails", - ) - .await? + if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? + || !resident_memtables_carry(&in_memory, |memtable| { + memtable.index_store.get_hnsw_by_column(&column).is_some() + }) { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only" + "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only" ), }); } @@ -888,7 +901,10 @@ mod tests { #[test] fn resolve_single_index_dedupes_segments() { - let maintained = vec!["fts_idx".to_string()]; + let maintained = MemWalIndexDetails { + maintained_indexes: vec!["fts_idx".to_string()], + ..Default::default() + }; // Two physical segments of ONE logical index must not count as "multiple". assert_eq!( resolve_single_index( @@ -917,4 +933,33 @@ mod tests { None ); } + + /// A spec that maintains every index names none of them, so the stored list + /// is empty while every index on the table is in fact maintained. Reading + /// the list alone would refuse the read path on the default spec. + #[test] + fn an_unnamed_set_maintains_every_index() { + let all = MemWalIndexDetails { + maintained_indexes: Vec::new(), + maintain_all_indexes: true, + ..Default::default() + }; + assert!(spec_maintains(&all, "fts_idx")); + assert_eq!( + resolve_single_index(vec!["fts_idx".to_string()], &all, "full-text", "text").unwrap(), + Some("fts_idx".to_string()), + ); + + // The same empty list without the flag maintains nothing. + let none = MemWalIndexDetails { + maintained_indexes: Vec::new(), + maintain_all_indexes: false, + ..Default::default() + }; + assert!(!spec_maintains(&none, "fts_idx")); + assert_eq!( + resolve_single_index(vec!["fts_idx".to_string()], &none, "full-text", "text").unwrap(), + None, + ); + } }