feat: infer maintained indexes when an LsmWriteSpec omits them

`LsmWriteSpec::maintained_indexes` becomes `Option<Vec<String>>`. `None` —
now the default for a freshly constructed spec — maintains every index the
MemWAL supports, resolved when the spec is installed. `Some([])` maintains
none, and an explicit list is still taken verbatim.

Resolution filters on `IndexConfig::is_memwal_maintainable`, which delegates
to lance's `is_maintainable_index_type`. Sweeping up an index the memtable
cannot build (a bitmap, say) would fail every memtable claim and leave the
table unwritable, so the inferred set excludes them and an explicit list
naming one is now rejected up front rather than at claim time.

The resolved set is a snapshot, not a subscription: an index created after
the spec is installed is not maintained until the spec is unset and set
again. That is why the default flipped from "maintain nothing" — an
unwanted maintained index costs memory, while a silently unmaintained one
degrades FTS to an unscored scan.

Note this needs a lance release carrying `is_maintainable_index_type`
before it builds against the pinned tag.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

python/python/lancedb/_lancedb.pyi

@@ -608,8 +608,13 @@ class LsmWriteSpec:
     @staticmethod
     def unsharded() -> "LsmWriteSpec": ...
     def with_maintained_indexes(self, indexes: List[str]) -> "LsmWriteSpec":
-        """Return a copy of this spec asking the MemWAL to keep the named
-        indexes up to date as rows are appended."""
+        """Return a copy of this spec asking the MemWAL to keep exactly the
+        named indexes up to date as rows are appended, replacing the default
+        of every index the MemWAL supports."""
+        ...
+    def with_no_maintained_indexes(self) -> "LsmWriteSpec":
+        """Return a copy of this spec asking the MemWAL to maintain no
+        indexes at all."""
         ...
     def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "LsmWriteSpec":
         """Return a copy of this spec recording the given default
@@ -624,7 +629,10 @@ class LsmWriteSpec:
     @property
     def num_buckets(self) -> Optional[int]: ...
     @property
-    def maintained_indexes(self) -> List[str]: ...
+    def maintained_indexes(self) -> Optional[List[str]]:
+        """Names of indexes the MemWAL keeps up to date, or None when the spec
+        asks for every index the MemWAL supports."""
+        ...
     @property
     def writer_config_defaults(self) -> Dict[str, str]: ...
 

python/python/lancedb/table.py

@@ -4632,6 +4632,14 @@ class AsyncTable:
         via [`set_unenforced_primary_key`]; bucket sharding additionally
         requires it to be the single column being bucketed.
 
+        Unless the spec says otherwise, the MemWAL maintains every index on
+        the table that it supports, resolved here. That is a snapshot, not a
+        subscription: an index created afterwards is not maintained until the
+        spec is unset and set again. Use ``with_maintained_indexes`` to pin an
+        exact set — names must already be committed, so a still-building index
+        is rejected rather than quietly omitted — or
+        ``with_no_maintained_indexes`` to maintain none.
+
         Parameters
         ----------
         spec : LsmWriteSpec

python/python/tests/test_lsm_write_spec.py

@@ -83,7 +83,9 @@ def test_lsm_write_spec_repr():
     assert s.spec_type == "bucket"
     assert s.column == "id"
     assert s.num_buckets == 4
-    assert s.maintained_indexes == []
+    # A fresh spec defers its maintained set to install time.
+    assert s.maintained_indexes is None
+    assert s.with_no_maintained_indexes().maintained_indexes == []
     assert "bucket" in repr(s)
     assert "id" in repr(s)
     assert "4" in repr(s)
@@ -169,18 +171,23 @@ def test_get_lsm_write_spec(tmp_path):
     table.unset_lsm_write_spec()
     assert table.get_lsm_write_spec() is None
 
-    # Identity round-trips (column recovered from the schema).
+    # Identity round-trips (column recovered from the schema). Leaving the
+    # maintained set to be inferred picks up the index on the table, so the
+    # spec reads back naming it rather than as "infer".
     table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
     spec = table.get_lsm_write_spec()
     assert spec.spec_type == "identity"
     assert spec.column == "id"
+    assert spec.maintained_indexes == [idx_name]
     table.unset_lsm_write_spec()
 
-    # Unsharded round-trips (no routing column).
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    # Unsharded round-trips (no routing column). Opting out is distinct from
+    # the inferred default.
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_no_maintained_indexes())
     spec = table.get_lsm_write_spec()
     assert spec.spec_type == "unsharded"
     assert spec.column is None
+    assert spec.maintained_indexes == []
 
 
 @pytest.mark.asyncio

python/src/table.rs

@@ -185,12 +185,23 @@ impl From<lancedb::table::MergeResult> for MergeResult {
     }
 }
 
+/// Render a spec's maintained-index selection for `__repr__`, where the
+/// inferred default reads as Python's `None` rather than Rust's `Some([..])`.
+fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
+    match maintained {
+        Some(names) => format!("{:?}", names),
+        None => "None".to_string(),
+    }
+}
+
 /// Specification selecting Lance's MemWAL LSM-style write path for
 /// `merge_insert`.
 ///
 /// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()`
-/// classmethods, then optionally chain `with_maintained_indexes(...)` and
-/// `with_writer_config_defaults(...)`.
+/// classmethods, then optionally chain `with_maintained_indexes(...)` /
+/// `with_no_maintained_indexes()` and `with_writer_config_defaults(...)`.
+/// A freshly constructed spec maintains every index the MemWAL supports,
+/// resolved when the spec is installed.
 #[pyclass(from_py_object)]
 #[derive(Clone, Debug)]
 pub struct LsmWriteSpec {
@@ -230,16 +241,24 @@ impl LsmWriteSpec {
         }
     }
 
-    /// Replace the list of indexes the MemWAL should keep up to date as
-    /// rows are appended. Each name must reference an index that
-    /// already exists on the table at the time `set_lsm_write_spec`
-    /// is called.
+    /// Maintain exactly `indexes`, replacing the default of "every index
+    /// the MemWAL supports". Each name must reference an index that
+    /// already exists on the table, of a type the MemWAL can maintain,
+    /// at the time `set_lsm_write_spec` is called.
     pub fn with_maintained_indexes(&self, indexes: Vec<String>) -> Self {
         Self {
             inner: self.inner.clone().with_maintained_indexes(indexes),
         }
     }
 
+    /// Maintain no indexes at all — the memtable serves scans and filters
+    /// without any in-memory index.
+    pub fn with_no_maintained_indexes(&self) -> Self {
+        Self {
+            inner: self.inner.clone().with_no_maintained_indexes(),
+        }
+    }
+
     /// Replace the default `ShardWriter` configuration recorded in the
     /// MemWAL index, so every writer starts from the same defaults.
     pub fn with_writer_config_defaults(&self, defaults: HashMap<String, String>) -> Self {
@@ -256,23 +275,29 @@ impl LsmWriteSpec {
                 maintained_indexes,
                 writer_config_defaults,
             } => format!(
-                "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={:?}, writer_config_defaults={:?})",
-                column, num_buckets, maintained_indexes, writer_config_defaults,
+                "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})",
+                column,
+                num_buckets,
+                fmt_maintained(maintained_indexes),
+                writer_config_defaults,
             ),
             lancedb::table::LsmWriteSpec::Identity {
                 column,
                 maintained_indexes,
                 writer_config_defaults,
             } => format!(
-                "LsmWriteSpec.identity(column={:?}, maintained_indexes={:?}, writer_config_defaults={:?})",
-                column, maintained_indexes, writer_config_defaults,
+                "LsmWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})",
+                column,
+                fmt_maintained(maintained_indexes),
+                writer_config_defaults,
             ),
             lancedb::table::LsmWriteSpec::Unsharded {
                 maintained_indexes,
                 writer_config_defaults,
             } => format!(
-                "LsmWriteSpec.unsharded(maintained_indexes={:?}, writer_config_defaults={:?})",
-                maintained_indexes, writer_config_defaults,
+                "LsmWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})",
+                fmt_maintained(maintained_indexes),
+                writer_config_defaults,
             ),
         }
     }
@@ -307,10 +332,11 @@ impl LsmWriteSpec {
         }
     }
 
-    /// Names of indexes the MemWAL should keep up to date during writes.
+    /// Names of indexes the MemWAL should keep up to date during writes, or
+    /// `None` when the spec asks for every index the MemWAL supports.
     #[getter]
-    pub fn maintained_indexes(&self) -> Vec<String> {
-        self.inner.maintained_indexes().to_vec()
+    pub fn maintained_indexes(&self) -> Option<Vec<String>> {
+        self.inner.maintained_indexes().map(<[String]>::to_vec)
     }
 
     /// Default `ShardWriter` configuration recorded by this spec.