fix: wait for the seal a maintained-index refresh triggers

Replacing a writer's index configs seals the active memtable so the next one
carries the new indexes, and returns a fence covering that seal's flushes. The
outgoing memtable keeps the set it was built with and stays resident, frozen,
until it has been flushed — and an indexed read refuses while any resident
memtable cannot answer it.

Discarding the fence left that a race: the flush usually won, so the read
succeeded, and under load it did not. Awaiting the fence makes creating an index
on a live writer mean what it says by the time it returns.

The read a MemTable predating the index used to refuse now succeeds, because no
such MemTable survives the refresh; its test pins that instead.

rust/lancedb/src/table/merge.rs

@@ -1663,14 +1663,13 @@ mod lsm_tests {
         );
     }
 
-    /// A writer opened before the index existed keeps a MemTable without it, so
-    /// an indexed read is refused rather than served without those rows.
+    /// A row written before the index exists is searchable once it does.
     ///
-    /// The default set maintains every index the table has, which the spec
-    /// records as intent. That intent alone would let the read through while
-    /// the live MemTable still cannot answer it.
+    /// The MemTable holding it was built without the index. Creating the index
+    /// replaces the writer's configs, which seals that MemTable and waits for
+    /// its flush, so no resident MemTable is left that cannot answer the read.
     #[tokio::test]
-    async fn lsm_read_refuses_when_a_resident_memtable_predates_the_index() {
+    async fn lsm_a_row_written_before_the_index_is_searchable_after_it() {
         use crate::index::Index;
         use lance_index::scalar::FullTextSearchQuery;
 
@@ -1695,13 +1694,17 @@ mod lsm_tests {
         let query = FullTextSearchQuery::new("zebra".to_string())
             .with_column("text".to_string())
             .unwrap();
-        let Err(error) = table.query().full_text_search(query).execute().await else {
-            panic!("an indexed read the resident MemTable cannot answer must be refused");
-        };
-        assert!(
-            error.to_string().contains("resident MemTable"),
-            "unexpected error: {error}"
-        );
+        let batches = table
+            .query()
+            .full_text_search(query)
+            .execute()
+            .await
+            .expect("no resident MemTable is left that cannot answer")
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let found: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(found, 1, "the row written before the index must be found");
     }
 
     #[tokio::test]

rust/lancedb/src/table/merge/lsm.rs

@@ -460,10 +460,28 @@ impl ShardWriterCache {
         let Some(writer) = guard.as_ref() else {
             return;
         };
-        if let Err(error) = dataset.refresh_mem_wal_index_configs(writer).await {
+        // Replacing the configs seals the active memtable so the next one carries
+        // the new indexes. The outgoing one keeps the set it was built with, so
+        // its flush has to land before a read: while it is still frozen, an
+        // indexed read sees a resident memtable that cannot answer and is
+        // refused.
+        let sealed = match dataset.refresh_mem_wal_index_configs(writer).await {
+            Ok(sealed) => sealed,
+            Err(error) => {
+                log::warn!(
+                    "the maintained index set changed but the open shard writer could \
+                     not be refreshed, so it keeps the set it opened with: {error}"
+                );
+                return;
+            }
+        };
+        if let Some(fence) = sealed
+            && let Err(error) = fence.wait().await
+        {
             log::warn!(
-                "the maintained index set changed but the open shard writer could \
-                 not be refreshed, so it keeps the set it opened with: {error}"
+                "the shard writer took the new maintained index set, but the memtable \
+                 it sealed was not flushed, so an indexed read stays refused until \
+                 it is: {error}"
             );
         }
     }