feat: infer maintained indexes when an LsmWriteSpec omits them

`LsmWriteSpec::maintained_indexes` becomes `Option<Vec<String>>`. `None` —
now the default for a freshly constructed spec — maintains every index the
MemWAL supports, resolved when the spec is installed. `Some([])` maintains
none, and an explicit list is still taken verbatim.

Resolution filters on `IndexConfig::is_memwal_maintainable`, which delegates
to lance's `is_maintainable_index_type`. Sweeping up an index the memtable
cannot build (a bitmap, say) would fail every memtable claim and leave the
table unwritable, so the inferred set excludes them and an explicit list
naming one is now rejected up front rather than at claim time.

The resolved set is a snapshot, not a subscription: an index created after
the spec is installed is not maintained until the spec is unset and set
again. That is why the default flipped from "maintain nothing" — an
unwanted maintained index costs memory, while a silently unmaintained one
degrades FTS to an unscored scan.

Note this needs a lance release carrying `is_maintainable_index_type`
before it builds against the pinned tag.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Daniel Rammer
2026-07-31 11:17:14 -05:00
co-authored by Claude Opus 5
parent a6418b6cb9
commit dd197ad0ff
10 changed files with 346 additions and 65 deletions
+13 -1
View File
@@ -197,7 +197,13 @@ export interface LsmWriteSpec {
column?: string;
/** Bucket variant: the number of buckets, in `[1, 1024]`. */
numBuckets?: number;
/** Names of indexes the MemWAL should keep up to date during writes. */
/**
* Names of indexes the MemWAL should keep up to date during writes.
*
* Omit to maintain every index the MemWAL supports, resolved when the spec
* is installed — a snapshot, not a subscription: indexes created afterwards
* are not maintained. Pass `[]` to maintain none.
*/
maintainedIndexes?: string[];
/** Default `ShardWriter` configuration recorded in the MemWAL index. */
writerConfigDefaults?: Record<string, string>;
@@ -595,6 +601,12 @@ export abstract class Table {
* All variants require the table to have an unenforced primary key
* ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally
* requires it to be the single column being bucketed.
*
* Omitting `maintainedIndexes` maintains every index the MemWAL supports,
* resolved here — a snapshot, so an index created afterwards is not
* maintained until the spec is unset and set again. Naming them instead
* pins an exact set, and a still-building index is rejected rather than
* quietly omitted.
* @param {LsmWriteSpec} spec The sharding spec to install.
* @returns {Promise<void>}
* @example
+12 -7
View File
@@ -769,6 +769,8 @@ pub struct LsmWriteSpec {
/// Bucket variant: the number of buckets, in `[1, 1024]`.
pub num_buckets: Option<u32>,
/// Names of indexes the MemWAL should keep up to date during writes.
/// Omitted means every index the MemWAL can maintain, resolved when the
/// spec is installed; an empty array means none.
pub maintained_indexes: Option<Vec<String>>,
/// Default `ShardWriter` configuration recorded in the MemWAL index.
pub writer_config_defaults: Option<HashMap<String, String>>,
@@ -778,7 +780,6 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
type Error = napi::Error;
fn try_from(value: LsmWriteSpec) -> napi::Result<Self> {
let maintained = value.maintained_indexes.unwrap_or_default();
let writer_config_defaults = value.writer_config_defaults.unwrap_or_default();
let spec = match value.spec_type.as_str() {
"bucket" => {
@@ -804,9 +805,13 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
)));
}
};
Ok(spec
.with_maintained_indexes(maintained)
.with_writer_config_defaults(writer_config_defaults))
// An omitted `maintainedIndexes` keeps the spec's default of "every
// index the MemWAL can maintain".
let spec = match value.maintained_indexes {
Some(names) => spec.with_maintained_indexes(names),
None => spec,
};
Ok(spec.with_writer_config_defaults(writer_config_defaults))
}
}
@@ -823,7 +828,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
spec_type: "bucket".to_string(),
column: Some(column),
num_buckets: Some(num_buckets),
maintained_indexes: Some(maintained_indexes),
maintained_indexes,
writer_config_defaults: Some(writer_config_defaults),
},
Native::Identity {
@@ -834,7 +839,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
spec_type: "identity".to_string(),
column: Some(column),
num_buckets: None,
maintained_indexes: Some(maintained_indexes),
maintained_indexes,
writer_config_defaults: Some(writer_config_defaults),
},
Native::Unsharded {
@@ -844,7 +849,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
spec_type: "unsharded".to_string(),
column: None,
num_buckets: None,
maintained_indexes: Some(maintained_indexes),
maintained_indexes,
writer_config_defaults: Some(writer_config_defaults),
},
}