mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-14 14:46:53 +00:00
feat: infer maintained indexes when an LsmWriteSpec omits them
`LsmWriteSpec::maintained_indexes` becomes `Option<Vec<String>>`. `None` — now the default for a freshly constructed spec — maintains every index the MemWAL supports, resolved when the spec is installed. `Some([])` maintains none, and an explicit list is still taken verbatim. Resolution filters on `IndexConfig::is_memwal_maintainable`, which delegates to lance's `is_maintainable_index_type`. Sweeping up an index the memtable cannot build (a bitmap, say) would fail every memtable claim and leave the table unwritable, so the inferred set excludes them and an explicit list naming one is now rejected up front rather than at claim time. The resolved set is a snapshot, not a subscription: an index created after the spec is installed is not maintained until the spec is unset and set again. That is why the default flipped from "maintain nothing" — an unwanted maintained index costs memory, while a silently unmaintained one degrades FTS to an unscored scan. Note this needs a lance release carrying `is_maintainable_index_type` before it builds against the pinned tag. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
a6418b6cb9
commit
dd197ad0ff
+41
-15
@@ -185,12 +185,23 @@ impl From<lancedb::table::MergeResult> for MergeResult {
|
||||
}
|
||||
}
|
||||
|
||||
/// Render a spec's maintained-index selection for `__repr__`, where the
|
||||
/// inferred default reads as Python's `None` rather than Rust's `Some([..])`.
|
||||
fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
|
||||
match maintained {
|
||||
Some(names) => format!("{:?}", names),
|
||||
None => "None".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Specification selecting Lance's MemWAL LSM-style write path for
|
||||
/// `merge_insert`.
|
||||
///
|
||||
/// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()`
|
||||
/// classmethods, then optionally chain `with_maintained_indexes(...)` and
|
||||
/// `with_writer_config_defaults(...)`.
|
||||
/// classmethods, then optionally chain `with_maintained_indexes(...)` /
|
||||
/// `with_no_maintained_indexes()` and `with_writer_config_defaults(...)`.
|
||||
/// A freshly constructed spec maintains every index the MemWAL supports,
|
||||
/// resolved when the spec is installed.
|
||||
#[pyclass(from_py_object)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct LsmWriteSpec {
|
||||
@@ -230,16 +241,24 @@ impl LsmWriteSpec {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace the list of indexes the MemWAL should keep up to date as
|
||||
/// rows are appended. Each name must reference an index that
|
||||
/// already exists on the table at the time `set_lsm_write_spec`
|
||||
/// is called.
|
||||
/// Maintain exactly `indexes`, replacing the default of "every index
|
||||
/// the MemWAL supports". Each name must reference an index that
|
||||
/// already exists on the table, of a type the MemWAL can maintain,
|
||||
/// at the time `set_lsm_write_spec` is called.
|
||||
pub fn with_maintained_indexes(&self, indexes: Vec<String>) -> Self {
|
||||
Self {
|
||||
inner: self.inner.clone().with_maintained_indexes(indexes),
|
||||
}
|
||||
}
|
||||
|
||||
/// Maintain no indexes at all — the memtable serves scans and filters
|
||||
/// without any in-memory index.
|
||||
pub fn with_no_maintained_indexes(&self) -> Self {
|
||||
Self {
|
||||
inner: self.inner.clone().with_no_maintained_indexes(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace the default `ShardWriter` configuration recorded in the
|
||||
/// MemWAL index, so every writer starts from the same defaults.
|
||||
pub fn with_writer_config_defaults(&self, defaults: HashMap<String, String>) -> Self {
|
||||
@@ -256,23 +275,29 @@ impl LsmWriteSpec {
|
||||
maintained_indexes,
|
||||
writer_config_defaults,
|
||||
} => format!(
|
||||
"LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={:?}, writer_config_defaults={:?})",
|
||||
column, num_buckets, maintained_indexes, writer_config_defaults,
|
||||
"LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})",
|
||||
column,
|
||||
num_buckets,
|
||||
fmt_maintained(maintained_indexes),
|
||||
writer_config_defaults,
|
||||
),
|
||||
lancedb::table::LsmWriteSpec::Identity {
|
||||
column,
|
||||
maintained_indexes,
|
||||
writer_config_defaults,
|
||||
} => format!(
|
||||
"LsmWriteSpec.identity(column={:?}, maintained_indexes={:?}, writer_config_defaults={:?})",
|
||||
column, maintained_indexes, writer_config_defaults,
|
||||
"LsmWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})",
|
||||
column,
|
||||
fmt_maintained(maintained_indexes),
|
||||
writer_config_defaults,
|
||||
),
|
||||
lancedb::table::LsmWriteSpec::Unsharded {
|
||||
maintained_indexes,
|
||||
writer_config_defaults,
|
||||
} => format!(
|
||||
"LsmWriteSpec.unsharded(maintained_indexes={:?}, writer_config_defaults={:?})",
|
||||
maintained_indexes, writer_config_defaults,
|
||||
"LsmWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})",
|
||||
fmt_maintained(maintained_indexes),
|
||||
writer_config_defaults,
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -307,10 +332,11 @@ impl LsmWriteSpec {
|
||||
}
|
||||
}
|
||||
|
||||
/// Names of indexes the MemWAL should keep up to date during writes.
|
||||
/// Names of indexes the MemWAL should keep up to date during writes, or
|
||||
/// `None` when the spec asks for every index the MemWAL supports.
|
||||
#[getter]
|
||||
pub fn maintained_indexes(&self) -> Vec<String> {
|
||||
self.inner.maintained_indexes().to_vec()
|
||||
pub fn maintained_indexes(&self) -> Option<Vec<String>> {
|
||||
self.inner.maintained_indexes().map(<[String]>::to_vec)
|
||||
}
|
||||
|
||||
/// Default `ShardWriter` configuration recorded by this spec.
|
||||
|
||||
Reference in New Issue
Block a user