feat: let a table's maintained index set be changed in place

Changing the set meant unsetting the spec and setting it again, which drops
the MemWAL and everything in its fresh tier. `set_lsm_write_spec` now accepts
a repeat that differs only there and updates the set on the live index; the
sharding still cannot move, because the generations already written were
homed under it.

The default set also stops being a snapshot. It is stored as the intent to
maintain everything, which lance re-reads as the table changes, so an index
created later is maintained without another call. An index of a kind the
MemWAL cannot mirror is skipped under that intent rather than refusing the
table, since nobody asked for it by name; a named one is still refused.

The LSM read path asks the spec whether an index is maintained rather than
searching the stored list, which an unnamed set leaves empty -- reading the
list alone would refuse full-text and vector search on every table using the
default.

rust/lancedb/src/table.rs

@@ -330,7 +330,8 @@ pub use self::merge::MergeResult;
 /// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default
 /// `ShardWriter` configuration recorded in the MemWAL index).
 ///
-/// A fresh spec maintains every index on the table, resolved on install.
+/// A fresh spec maintains every index on the table, including ones created
+/// after it is installed.
 ///
 /// Install a spec with [`Table::set_lsm_write_spec`] and remove it with
 /// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch
@@ -348,9 +349,8 @@ pub enum LsmWriteSpec {
         num_buckets: u32,
         /// Indexes the MemWAL maintains in-memory as rows are appended.
         ///
-        /// `None` means every index it can maintain, resolved on install — a
-        /// snapshot, so indexes created later need the spec unset and re-set.
-        /// `Some([])` maintains nothing.
+        /// `None` means every index the table has, including ones created
+        /// later. `Some([])` maintains nothing.
         maintained_indexes: Option<Vec<String>>,
         /// Default `ShardWriter` configuration recorded in the MemWAL index.
         writer_config_defaults: HashMap<String, String>,
@@ -363,9 +363,8 @@ pub enum LsmWriteSpec {
         column: String,
         /// Indexes the MemWAL maintains in-memory as rows are appended.
         ///
-        /// `None` means every index it can maintain, resolved on install — a
-        /// snapshot, so indexes created later need the spec unset and re-set.
-        /// `Some([])` maintains nothing.
+        /// `None` means every index the table has, including ones created
+        /// later. `Some([])` maintains nothing.
         maintained_indexes: Option<Vec<String>>,
         /// Default `ShardWriter` configuration recorded in the MemWAL index.
         writer_config_defaults: HashMap<String, String>,
@@ -374,9 +373,8 @@ pub enum LsmWriteSpec {
     Unsharded {
         /// Indexes the MemWAL maintains in-memory as rows are appended.
         ///
-        /// `None` means every index it can maintain, resolved on install — a
-        /// snapshot, so indexes created later need the spec unset and re-set.
-        /// `Some([])` maintains nothing.
+        /// `None` means every index the table has, including ones created
+        /// later. `Some([])` maintains nothing.
         maintained_indexes: Option<Vec<String>>,
         /// Default `ShardWriter` configuration recorded in the MemWAL index.
         writer_config_defaults: HashMap<String, String>,
@@ -422,14 +420,15 @@ impl LsmWriteSpec {
 
     /// Set which indexes the MemWAL maintains.
     ///
-    /// `None` (the default) resolves to every index on the table at install,
-    /// failing if one cannot be maintained — name the set to install anyway. A
-    /// list is verbatim: each name must already exist and be maintainable, and
-    /// an empty list maintains nothing.
+    /// `None` (the default) is every index on the table, re-read as the table
+    /// changes, so an index created later is maintained too; one of a kind the
+    /// MemWAL cannot mirror is skipped rather than failing the table. A list is
+    /// verbatim: each name must already exist and be maintainable, and an empty
+    /// list maintains nothing.
     ///
     /// ```
     /// # use lancedb::table::LsmWriteSpec;
-    /// // Every index the table has when the spec is installed:
+    /// // Every index the table has, now and later:
     /// LsmWriteSpec::unsharded().with_maintained_indexes(None);
     /// // Exactly these:
     /// LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]);
@@ -1985,6 +1984,10 @@ impl Table {
     /// - [`LsmWriteSpec::identity`] — shard by the raw value of a scalar column.
     /// - [`LsmWriteSpec::unsharded`] — route every write to a single shard.
     ///
+    /// Calling this again on a table that already has a spec is how the
+    /// maintained index set is changed; the sharding cannot move, because the
+    /// generations already written were homed under it.
+    ///
     /// # Example
     ///
     /// ```
@@ -5648,29 +5651,22 @@ mod tests {
         assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
 
         // Identity sharding round-trips (column recovered from the schema).
-        // A spec left at its default maintains every index on the table, so it
-        // reads back naming the one on the table rather than as "infer".
+        // A spec left at its default round-trips as the default: what is stored
+        // is the intent to maintain everything, not the set it resolves to now.
         let spec = LsmWriteSpec::identity("region");
         table.set_lsm_write_spec(spec.clone()).await.unwrap();
-        assert_eq!(
-            table.get_lsm_write_spec().await.unwrap(),
-            Some(spec.with_maintained_indexes(vec![idx_name.clone()]))
-        );
+        assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
         table.unset_lsm_write_spec().await.unwrap();
 
         // Unsharded round-trips (no routing column).
         let spec = LsmWriteSpec::unsharded();
         table.set_lsm_write_spec(spec.clone()).await.unwrap();
-        assert_eq!(
-            table.get_lsm_write_spec().await.unwrap(),
-            Some(spec.with_maintained_indexes(vec![idx_name]))
-        );
+        assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
     }
 
-    /// The maintained set defaults to every index on the table, resolved at
-    /// install. An index the memtable cannot build fails the install rather
-    /// than being dropped: maintaining it would take the table offline for
-    /// writes, dropping it would hide that from the caller.
+    /// The maintained set defaults to every index on the table. A named index
+    /// the memtable cannot build fails the install, because the caller asked
+    /// for it; an unnamed one is skipped, because it arrived by existing.
     #[tokio::test]
     async fn test_set_lsm_write_spec_infers_maintained_indexes() {
         let tmp_dir = tempdir().unwrap();
@@ -5723,19 +5719,25 @@ mod tests {
         );
         assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
 
-        // The default covers every index, so the bitmap fails it too.
-        let err = table
+        // The default names nothing, so the bitmap is skipped rather than
+        // refusing the table: it is maintained by existing, not by being asked
+        // for. The intent is what is stored, not the set it resolves to today.
+        table
             .set_lsm_write_spec(LsmWriteSpec::unsharded())
             .await
-            .unwrap_err();
-        assert!(
-            matches!(err, Error::InvalidInput { ref message }
-                if message.contains("tag_bitmap") && message.contains("maintained_indexes")),
-            "expected the inferred set to be rejected, got {err:?}"
+            .unwrap();
+        assert_eq!(
+            table
+                .get_lsm_write_spec()
+                .await
+                .unwrap()
+                .unwrap()
+                .maintained_indexes(),
+            None,
+            "an unnamed set is stored as the intent to maintain everything"
         );
-        assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
 
-        // Naming the maintainable subset installs.
+        // Naming the maintainable subset narrows it, with no unset in between.
         table
             .set_lsm_write_spec(
                 LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]),
@@ -5753,7 +5755,6 @@ mod tests {
         );
 
         // Opting out entirely is distinct from the default.
-        table.unset_lsm_write_spec().await.unwrap();
         table
             .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new()))
             .await
@@ -5767,6 +5768,28 @@ mod tests {
                 .maintained_indexes(),
             Some([].as_slice())
         );
+
+        // The sharding cannot move, so the repeat that changes it is refused
+        // and the installed spec is left alone.
+        let err = table
+            .set_lsm_write_spec(LsmWriteSpec::bucket("id", 4))
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(err, Error::InvalidInput { ref message }
+                if message.contains("only its maintained index set can be changed")),
+            "expected the sharding change to be refused, got {err:?}"
+        );
+        assert_eq!(
+            table
+                .get_lsm_write_spec()
+                .await
+                .unwrap()
+                .unwrap()
+                .maintained_indexes(),
+            Some([].as_slice()),
+            "a refused call changes nothing"
+        );
     }
 
     #[tokio::test]

rust/lancedb/src/table/create_index.rs

@@ -160,6 +160,10 @@ impl NativeTable {
             builder = builder.name(name);
         }
         builder.await?;
+        self.dataset
+            .shard_writer()
+            .refresh_maintained_indexes(&dataset)
+            .await;
         self.dataset.update(dataset);
         Ok(())
     }

rust/lancedb/src/table/merge.rs

@@ -1251,6 +1251,30 @@ mod lsm_tests {
         );
     }
 
+    /// A table with `id`/`text` and a primary key, ready for an LSM write spec.
+    async fn lsm_text_table(dir: &tempfile::TempDir) -> crate::Table {
+        let conn = connect(dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+        let table = conn
+            .create_table("t", id_text_reader(vec![(1, "alpha")]))
+            .execute()
+            .await
+            .unwrap();
+        table.set_unenforced_primary_key(["id"]).await.unwrap();
+        table
+    }
+
+    /// Upsert `rows` through the LSM write path.
+    async fn upsert_text(table: &crate::Table, rows: Vec<(i64, &str)>) {
+        let mut builder = table.merge_insert(&[]);
+        builder
+            .when_matched_update_all(None)
+            .when_not_matched_insert_all();
+        builder.execute(id_text_reader(rows)).await.unwrap();
+    }
+
     /// A reader of `[id: Int64, text: Utf8]` rows.
     fn id_text_reader(rows: Vec<(i64, &str)>) -> Box<dyn RecordBatchReader + Send> {
         let schema = Arc::new(Schema::new(vec![
@@ -1270,6 +1294,148 @@ mod lsm_tests {
         Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema))
     }
 
+    /// Under the default set, an index created while a writer is open reaches
+    /// that writer.
+    ///
+    /// The default maintains every index the table has, so creating one changes
+    /// what the table maintains without anyone touching the spec. A writer left
+    /// on its old configs would keep building MemTables the new index does not
+    /// cover.
+    #[tokio::test]
+    async fn lsm_default_set_picks_up_an_index_created_while_writing() {
+        use crate::index::Index;
+        use lance_index::scalar::FullTextSearchQuery;
+
+        let dir = tempdir().unwrap();
+        let table = lsm_text_table(&dir).await;
+        // The default set, on a table with no FTS index yet.
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+
+        // Opens the writer, whose MemTable carries no FTS index.
+        upsert_text(&table, vec![(99, "zebra")]).await;
+
+        // Created with the writer still open and never reopened.
+        table
+            .create_index(&["text"], Index::FTS(Default::default()))
+            .execute()
+            .await
+            .unwrap();
+
+        upsert_text(&table, vec![(100, "zebra stripes")]).await;
+
+        let query = FullTextSearchQuery::new("stripes".to_string())
+            .with_column("text".to_string())
+            .unwrap();
+        let batches = table
+            .query()
+            .full_text_search(query)
+            .execute()
+            .await
+            .expect("the created index reached the open writer")
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let found: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(found, 1, "the row written after create_index must be found");
+    }
+
+    /// Changing the set on a live table reaches the writer already open on it.
+    ///
+    /// Without that, the commit would land and the writer would keep building
+    /// MemTables from the old set, so a query needing the newly maintained
+    /// index would stay refused until something reopened the writer.
+    #[tokio::test]
+    async fn lsm_set_change_reaches_an_already_open_writer() {
+        use crate::index::Index;
+        use lance_index::scalar::FullTextSearchQuery;
+
+        let dir = tempdir().unwrap();
+        let table = lsm_text_table(&dir).await;
+        table
+            .create_index(&["text"], Index::FTS(Default::default()))
+            .execute()
+            .await
+            .unwrap();
+        let fts_index = table.list_indices().await.unwrap()[0].name.clone();
+
+        // Installed maintaining nothing, then written to: the writer opens with
+        // no FTS index.
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new()))
+            .await
+            .unwrap();
+        upsert_text(&table, vec![(99, "zebra")]).await;
+
+        // Change the set in place on the live table.
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index]))
+            .await
+            .unwrap();
+
+        // A row written after the change is answered by the index the writer
+        // picked up.
+        upsert_text(&table, vec![(100, "zebra stripes")]).await;
+
+        let query = FullTextSearchQuery::new("stripes".to_string())
+            .with_column("text".to_string())
+            .unwrap();
+        let batches = table
+            .query()
+            .full_text_search(query)
+            .execute()
+            .await
+            .expect("the set change reached the open writer")
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let found: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(found, 1, "the row written after the change must be found");
+    }
+
+    /// A writer opened before the index existed keeps a MemTable without it, so
+    /// an indexed read is refused rather than served without those rows.
+    ///
+    /// The default set maintains every index the table has, which the spec
+    /// records as intent. That intent alone would let the read through while
+    /// the live MemTable still cannot answer it.
+    #[tokio::test]
+    async fn lsm_read_refuses_when_a_resident_memtable_predates_the_index() {
+        use crate::index::Index;
+        use lance_index::scalar::FullTextSearchQuery;
+
+        let dir = tempdir().unwrap();
+        let table = lsm_text_table(&dir).await;
+        // No index yet, and the default set: maintain whatever the table has.
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+
+        // Opens the writer, whose MemTable therefore carries no FTS index.
+        upsert_text(&table, vec![(99, "zebra")]).await;
+
+        // Built afterwards: the spec now resolves to it, the MemTable does not.
+        table
+            .create_index(&["text"], Index::FTS(Default::default()))
+            .execute()
+            .await
+            .unwrap();
+
+        let query = FullTextSearchQuery::new("zebra".to_string())
+            .with_column("text".to_string())
+            .unwrap();
+        let Err(error) = table.query().full_text_search(query).execute().await else {
+            panic!("an indexed read the resident MemTable cannot answer must be refused");
+        };
+        assert!(
+            error.to_string().contains("resident MemTable"),
+            "unexpected error: {error}"
+        );
+    }
+
     #[tokio::test]
     async fn lsm_read_full_text_search() {
         use crate::index::Index;

rust/lancedb/src/table/merge/lsm.rs

@@ -73,27 +73,62 @@ const SHARD_NAMESPACE: Uuid = Uuid::from_u128(0x4c53_4d57_5249_5445_5f53_4841_52
 pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) -> Result<()> {
     table.dataset.ensure_mutable()?;
 
+    // A named set is checked against the table here, where the caller can still
+    // be told which names exist. An unnamed set is not resolved at all: it is an
+    // intent lance re-reads whenever it builds a MemTable, so an index created
+    // later is maintained without another call.
+    let maintained_indexes = match spec.maintained_indexes() {
+        Some(requested) => {
+            let indices = table.list_indices().await?;
+            for name in requested {
+                if !indices.iter().any(|index| &index.name == name) {
+                    return Err(Error::InvalidInput {
+                        message: format!(
+                            "maintained index '{}' does not exist on this table; it has {}",
+                            name,
+                            index_name_list(&indices),
+                        ),
+                    });
+                }
+            }
+            // Before the builder borrows the dataset clone.
+            let dataset = table.dataset.get().await?;
+            validate_maintained_indexes(&dataset, requested).await?;
+            Some(requested.to_vec())
+        }
+        None => None,
+    };
+
+    // A table that already has a spec accepts one kind of repeat: the same
+    // sharding with a different maintained set. The rest of a spec describes
+    // how the generations already written were homed, so it cannot move.
     {
         let dataset = table.dataset.get().await?;
-        if dataset.mem_wal_index_details().await?.is_some() {
-            return Err(Error::InvalidInput {
-                message: "set_lsm_write_spec: an LSM write spec is already set on this table; mutation is not supported".into(),
-            });
+        if let Some(details) = dataset.mem_wal_index_details().await? {
+            let installed = lsm_write_spec_from_details(&details, dataset.schema())?;
+            if !same_except_maintained_indexes(&installed, &spec) {
+                return Err(Error::InvalidInput {
+                    message: "set_lsm_write_spec: an LSM write spec is already set on this \
+                              table; only its maintained index set can be changed"
+                        .into(),
+                });
+            }
+            drop(dataset);
+            table.checkout_latest().await?;
+            let mut dataset = (*table.dataset.get().await?).clone();
+            dataset
+                .update_mem_wal_maintained_indexes(maintained_indexes)
+                .await?;
+            table
+                .dataset
+                .shard_writer()
+                .refresh_maintained_indexes(&dataset)
+                .await;
+            table.dataset.update(dataset);
+            return Ok(());
         }
     }
 
-    // Before the builder borrows the dataset clone. `list_indices` merges an
-    // index's segments into one entry, so the result needs no dedup.
-    let maintained_indexes = {
-        let dataset = table.dataset.get().await?;
-        resolve_maintained_indexes(
-            &dataset,
-            &table.list_indices().await?,
-            spec.maintained_indexes(),
-        )
-        .await?
-    };
-
     table.checkout_latest().await?;
     let mut dataset = (*table.dataset.get().await?).clone();
     let schema = arrow_schema::Schema::from(dataset.schema());
@@ -138,7 +173,9 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
             writer_config_defaults
         }
     };
-    builder = builder.maintained_indexes(maintained_indexes);
+    if let Some(maintained_indexes) = maintained_indexes {
+        builder = builder.maintained_indexes(maintained_indexes);
+    }
     for (key, value) in writer_config_defaults {
         builder = builder.add_writer_config_default(key, value);
     }
@@ -147,48 +184,6 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
     Ok(())
 }
 
-/// Resolve a spec's maintained-index selection against `indices`, as reported
-/// by [`Table::list_indices`](crate::Table::list_indices).
-///
-/// `None` means every index on the table, snapshotted now. Lance validates
-/// either selection against its shard-writer rules, so a spec that installs is
-/// one the MemWAL can open.
-///
-/// An unmaintainable index fails an inferred set rather than being dropped from
-/// it — dropping would leave the caller believing it is maintained.
-async fn resolve_maintained_indexes(
-    dataset: &Dataset,
-    indices: &[IndexConfig],
-    requested: Option<&[String]>,
-) -> Result<Vec<String>> {
-    let Some(requested) = requested else {
-        let all: Vec<String> = indices.iter().map(|index| index.name.clone()).collect();
-        validate_maintained_indexes(dataset, &all)
-            .await
-            .map_err(|source| Error::InvalidInput {
-                message: format!(
-                    "cannot maintain every index on this table: {source}. Set \
-                     maintained_indexes explicitly to choose from {}",
-                    index_name_list(indices),
-                ),
-            })?;
-        return Ok(all);
-    };
-    for name in requested {
-        if !indices.iter().any(|index| &index.name == name) {
-            return Err(Error::InvalidInput {
-                message: format!(
-                    "maintained index '{}' does not exist on this table; it has {}",
-                    name,
-                    index_name_list(indices),
-                ),
-            });
-        }
-    }
-    validate_maintained_indexes(dataset, requested).await?;
-    Ok(requested.to_vec())
-}
-
 /// Index names for an error message.
 fn index_name_list(indices: &[IndexConfig]) -> String {
     if indices.is_empty() {
@@ -298,10 +293,17 @@ fn lsm_write_spec_from_details(
     };
 
     Ok(base
-        .with_maintained_indexes(details.maintained_indexes.clone())
+        .with_maintained_indexes(
+            (!details.maintain_all_indexes).then(|| details.maintained_indexes.clone()),
+        )
         .with_writer_config_defaults(details.writer_config_defaults.clone()))
 }
 
+/// Whether two specs agree on everything an installed spec cannot change.
+fn same_except_maintained_indexes(a: &LsmWriteSpec, b: &LsmWriteSpec) -> bool {
+    a.clone().with_maintained_indexes(None) == b.clone().with_maintained_indexes(None)
+}
+
 /// Resolve the single routing column name from a sharding field's source id.
 ///
 /// `set_lsm_write_spec` records the shard column by its Lance field id, so the
@@ -460,6 +462,37 @@ impl ShardWriterCache {
         Ok(entry)
     }
 
+    /// Install `dataset`'s maintained index set on the cached writer, if one is
+    /// open, so a change reaches a table that is already being written to.
+    ///
+    /// Reported rather than propagated: both callers have already committed the
+    /// change durably, and the writer this process happens to hold may be
+    /// fenced or poisoned -- which is not a reason to fail creating an index.
+    /// Any writer opened afterwards reads the committed set.
+    ///
+    /// Holds the entry's read lock across the refresh, so a concurrent close
+    /// waits rather than retiring the writer mid-seal. Writes take the same
+    /// lock first, so the order matches theirs.
+    pub(crate) async fn refresh_maintained_indexes(&self, dataset: &Dataset) {
+        let entry = {
+            let guard = self.slot.read().await;
+            guard.as_ref().map(|(_, entry)| entry.clone())
+        };
+        let Some(entry) = entry else {
+            return;
+        };
+        let guard = entry.inner.read().await;
+        let Some(writer) = guard.as_ref() else {
+            return;
+        };
+        if let Err(error) = dataset.refresh_mem_wal_index_configs(writer).await {
+            log::warn!(
+                "the maintained index set changed but the open shard writer could \
+                 not be refreshed, so it keeps the set it opened with: {error}"
+            );
+        }
+    }
+
     /// Snapshot the cached writer's shard for the LSM read path: its shard id,
     /// authoritative in-memory manifest, and active + frozen memtable refs.
     /// Returns `None` when no writer is currently cached (e.g. nothing has been

rust/lancedb/src/table/query/lsm.rs

@@ -33,7 +33,7 @@ use datafusion_physical_plan::expressions::Column;
 use datafusion_physical_plan::projection::ProjectionExec;
 use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr};
 use lance::Dataset;
-use lance::dataset::mem_wal::scanner::InMemoryMemTables;
+use lance::dataset::mem_wal::scanner::{InMemoryMemTableRef, InMemoryMemTables};
 use lance::dataset::mem_wal::{
     DatasetMemWalExt, LsmScanner, ShardManifestStore, ShardSnapshot, ShardWriterConfig,
 };
@@ -471,17 +471,14 @@ async fn fts_plan(
     // Without a maintained in-memory FTS index for this column, the active memtable
     // arm produces an empty plan (`active_source_can_execute_fts` returns false), so
     // the search silently omits un-compacted documents. Reject rather than mislead.
-    if !index_maintained(
-        dataset,
-        column,
-        &details.maintained_indexes,
-        "InvertedIndexDetails",
-    )
-    .await?
+    if !index_maintained(dataset, column, details, "InvertedIndexDetails").await?
+        || !resident_memtables_carry(&in_memory, |memtable| {
+            memtable.index_store.get_fts_by_column(column).is_some()
+        })
     {
         return Err(Error::NotSupported {
             message: format!(
-                "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise un-compacted documents are omitted. set use_lsm(false) to read the base table only"
+                "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise un-compacted documents are omitted. set use_lsm(false) to read the base table only"
             ),
         });
     }
@@ -493,6 +490,27 @@ async fn fts_plan(
     Ok(scanner.create_plan().await?)
 }
 
+/// Whether the spec maintains the index named `name`.
+///
+/// An unnamed set names nothing and covers everything, so the stored list alone
+/// cannot answer this.
+fn spec_maintains(details: &MemWalIndexDetails, name: &str) -> bool {
+    details.maintain_all_indexes || details.maintained_indexes.iter().any(|n| n == name)
+}
+
+/// Whether every resident MemTable carries the index an arm needs.
+///
+/// The spec is intent; a MemTable built before the index joined the set
+/// contributes nothing to an indexed read, so its rows would go missing.
+fn resident_memtables_carry(
+    in_memory: &HashMap<Uuid, InMemoryMemTables>,
+    carries: impl Fn(&InMemoryMemTableRef) -> bool,
+) -> bool {
+    in_memory
+        .values()
+        .all(|tables| carries(&tables.active) && tables.frozen.iter().all(&carries))
+}
+
 /// Whether an index of `type_url_suffix` covering `column` is in the MemWAL spec's
 /// maintained set. Only a maintained index has its catch-up tracked (so exclusion is
 /// gated correctly) and its in-memory arm kept current; an unmaintained base index
@@ -502,7 +520,7 @@ async fn fts_plan(
 async fn index_maintained(
     dataset: &Dataset,
     column: &str,
-    maintained: &[String],
+    details: &MemWalIndexDetails,
     type_url_suffix: &str,
 ) -> Result<bool> {
     use lance::index::DatasetIndexExt;
@@ -512,7 +530,7 @@ async fn index_maintained(
     let indices = dataset.load_indices().await?;
     Ok(indices.iter().any(|idx| {
         idx.fields.contains(&field.id)
-            && maintained.iter().any(|m| m == &idx.name)
+            && spec_maintains(details, &idx.name)
             && idx
                 .index_details
                 .as_ref()
@@ -583,9 +601,7 @@ async fn arm_maintained_index_names(
             })
             .map(|idx| idx.name.clone())
             .collect();
-        if let Some(name) =
-            resolve_single_index(segment_names, &details.maintained_indexes, arm, &column)?
-        {
+        if let Some(name) = resolve_single_index(segment_names, details, arm, &column)? {
             names.push(name);
         }
     }
@@ -604,7 +620,7 @@ async fn arm_maintained_index_names(
 /// compaction watermark).
 fn resolve_single_index(
     mut names: Vec<String>,
-    maintained: &[String],
+    details: &MemWalIndexDetails,
     arm: &str,
     column: &str,
 ) -> Result<Option<String>> {
@@ -620,7 +636,7 @@ fn resolve_single_index(
     Ok(names
         .into_iter()
         .next()
-        .filter(|name| maintained.contains(name)))
+        .filter(|name| spec_maintains(details, name)))
 }
 
 /// Drop the primary-key columns Lance appends internally for dedup when the user's
@@ -692,17 +708,14 @@ async fn vector_plan(
     // maintained, its catch-up is untracked and exclusion falls back to the
     // compaction watermark — dropping compacted SSTables the (lagging) base index has
     // not re-indexed. Reject rather than silently omit rows, mirroring the FTS arm.
-    if !index_maintained(
-        dataset,
-        &column,
-        &details.maintained_indexes,
-        "VectorIndexDetails",
-    )
-    .await?
+    if !index_maintained(dataset, &column, details, "VectorIndexDetails").await?
+        || !resident_memtables_carry(&in_memory, |memtable| {
+            memtable.index_store.get_hnsw_by_column(&column).is_some()
+        })
     {
         return Err(Error::NotSupported {
             message: format!(
-                "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only"
+                "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only"
             ),
         });
     }
@@ -888,7 +901,10 @@ mod tests {
 
     #[test]
     fn resolve_single_index_dedupes_segments() {
-        let maintained = vec!["fts_idx".to_string()];
+        let maintained = MemWalIndexDetails {
+            maintained_indexes: vec!["fts_idx".to_string()],
+            ..Default::default()
+        };
         // Two physical segments of ONE logical index must not count as "multiple".
         assert_eq!(
             resolve_single_index(
@@ -917,4 +933,33 @@ mod tests {
             None
         );
     }
+
+    /// A spec that maintains every index names none of them, so the stored list
+    /// is empty while every index on the table is in fact maintained. Reading
+    /// the list alone would refuse the read path on the default spec.
+    #[test]
+    fn an_unnamed_set_maintains_every_index() {
+        let all = MemWalIndexDetails {
+            maintained_indexes: Vec::new(),
+            maintain_all_indexes: true,
+            ..Default::default()
+        };
+        assert!(spec_maintains(&all, "fts_idx"));
+        assert_eq!(
+            resolve_single_index(vec!["fts_idx".to_string()], &all, "full-text", "text").unwrap(),
+            Some("fts_idx".to_string()),
+        );
+
+        // The same empty list without the flag maintains nothing.
+        let none = MemWalIndexDetails {
+            maintained_indexes: Vec::new(),
+            maintain_all_indexes: false,
+            ..Default::default()
+        };
+        assert!(!spec_maintains(&none, "fts_idx"));
+        assert_eq!(
+            resolve_single_index(vec!["fts_idx".to_string()], &none, "full-text", "text").unwrap(),
+            None,
+        );
+    }
 }