diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 9fa5a1314..354eef5a5 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -1663,14 +1663,13 @@ mod lsm_tests { ); } - /// A writer opened before the index existed keeps a MemTable without it, so - /// an indexed read is refused rather than served without those rows. + /// A row written before the index exists is searchable once it does. /// - /// The default set maintains every index the table has, which the spec - /// records as intent. That intent alone would let the read through while - /// the live MemTable still cannot answer it. + /// The MemTable holding it was built without the index. Creating the index + /// replaces the writer's configs, which seals that MemTable and waits for + /// its flush, so no resident MemTable is left that cannot answer the read. #[tokio::test] - async fn lsm_read_refuses_when_a_resident_memtable_predates_the_index() { + async fn lsm_a_row_written_before_the_index_is_searchable_after_it() { use crate::index::Index; use lance_index::scalar::FullTextSearchQuery; @@ -1695,13 +1694,17 @@ mod lsm_tests { let query = FullTextSearchQuery::new("zebra".to_string()) .with_column("text".to_string()) .unwrap(); - let Err(error) = table.query().full_text_search(query).execute().await else { - panic!("an indexed read the resident MemTable cannot answer must be refused"); - }; - assert!( - error.to_string().contains("resident MemTable"), - "unexpected error: {error}" - ); + let batches = table + .query() + .full_text_search(query) + .execute() + .await + .expect("no resident MemTable is left that cannot answer") + .try_collect::>() + .await + .unwrap(); + let found: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(found, 1, "the row written before the index must be found"); } #[tokio::test] diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index d2be0e5b1..0f86fdcac 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -460,10 +460,28 @@ impl ShardWriterCache { let Some(writer) = guard.as_ref() else { return; }; - if let Err(error) = dataset.refresh_mem_wal_index_configs(writer).await { + // Replacing the configs seals the active memtable so the next one carries + // the new indexes. The outgoing one keeps the set it was built with, so + // its flush has to land before a read: while it is still frozen, an + // indexed read sees a resident memtable that cannot answer and is + // refused. + let sealed = match dataset.refresh_mem_wal_index_configs(writer).await { + Ok(sealed) => sealed, + Err(error) => { + log::warn!( + "the maintained index set changed but the open shard writer could \ + not be refreshed, so it keeps the set it opened with: {error}" + ); + return; + } + }; + if let Some(fence) = sealed + && let Err(error) = fence.wait().await + { log::warn!( - "the maintained index set changed but the open shard writer could \ - not be refreshed, so it keeps the set it opened with: {error}" + "the shard writer took the new maintained index set, but the memtable \ + it sealed was not flushed, so an indexed read stays refused until \ + it is: {error}" ); } }