rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-28 14:36:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: let a table's maintained index set be changed in place

Browse Source 
Changing the set meant unsetting the spec and setting it again, which drops
the MemWAL and everything in its fresh tier. `set_lsm_write_spec` now accepts
a repeat that differs only there and updates the set on the live index; the
sharding still cannot move, because the generations already written were
homed under it.

The default set also stops being a snapshot. It is stored as the intent to
maintain everything, which lance re-reads as the table changes, so an index
created later is maintained without another call. An index of a kind the
MemWAL cannot mirror is skipped under that intent rather than refusing the
table, since nobody asked for it by name; a named one is still refused.

The LSM read path asks the spec whether an index is maintained rather than
searching the stored list, which an unnamed set leaves empty -- reading the
list alone would refuse full-text and vector search on every table using the
default.
This commit is contained in:
XYZhan
2026-09-28 11:42:08 -04:00
parent 0327618a62
commit 0654cbd2c0
3 changed files with 168 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/lancedb/src/table.rs
+62 -39
View File
@@ -330,7 +330,8 @@ pub use self::merge::MergeResult;
/// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default
/// `ShardWriter` configuration recorded in the MemWAL index).
///
/// A fresh spec maintains every index on the table, resolved on install.
/// A fresh spec maintains every index on the table, including ones created
/// after it is installed.
///
/// Install a spec with [`Table::set_lsm_write_spec`] and remove it with
/// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch
@@ -348,9 +349,8 @@ pub enum LsmWriteSpec {
num_buckets: u32,
/// Indexes the MemWAL maintains in-memory as rows are appended.
///
/// `None` means every index it can maintain, resolved on install — a
/// snapshot, so indexes created later need the spec unset and re-set.
/// `Some([])` maintains nothing.
/// `None` means every index the table has, including ones created
/// later. `Some([])` maintains nothing.
maintained_indexes: Option<Vec<String>>,
/// Default `ShardWriter` configuration recorded in the MemWAL index.
writer_config_defaults: HashMap<String, String>,
@@ -363,9 +363,8 @@ pub enum LsmWriteSpec {
column: String,
/// Indexes the MemWAL maintains in-memory as rows are appended.
///
/// `None` means every index it can maintain, resolved on install — a
/// snapshot, so indexes created later need the spec unset and re-set.
/// `Some([])` maintains nothing.
/// `None` means every index the table has, including ones created
/// later. `Some([])` maintains nothing.
maintained_indexes: Option<Vec<String>>,
/// Default `ShardWriter` configuration recorded in the MemWAL index.
writer_config_defaults: HashMap<String, String>,
@@ -374,9 +373,8 @@ pub enum LsmWriteSpec {
Unsharded {
/// Indexes the MemWAL maintains in-memory as rows are appended.
///
/// `None` means every index it can maintain, resolved on install — a
/// snapshot, so indexes created later need the spec unset and re-set.
/// `Some([])` maintains nothing.
/// `None` means every index the table has, including ones created
/// later. `Some([])` maintains nothing.
maintained_indexes: Option<Vec<String>>,
/// Default `ShardWriter` configuration recorded in the MemWAL index.
writer_config_defaults: HashMap<String, String>,
@@ -422,14 +420,15 @@ impl LsmWriteSpec {
/// Set which indexes the MemWAL maintains.
///
/// `None` (the default) resolves to every index on the table at install,
/// failing if one cannot be maintained — name the set to install anyway. A
/// list is verbatim: each name must already exist and be maintainable, and
/// an empty list maintains nothing.
/// `None` (the default) is every index on the table, re-read as the table
/// changes, so an index created later is maintained too; one of a kind the
/// MemWAL cannot mirror is skipped rather than failing the table. A list is
/// verbatim: each name must already exist and be maintainable, and an empty
/// list maintains nothing.
///
/// ```
/// # use lancedb::table::LsmWriteSpec;
/// // Every index the table has when the spec is installed:
/// // Every index the table has, now and later:
/// LsmWriteSpec::unsharded().with_maintained_indexes(None);
/// // Exactly these:
/// LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]);
@@ -1985,6 +1984,10 @@ impl Table {
/// - [`LsmWriteSpec::identity`] — shard by the raw value of a scalar column.
/// - [`LsmWriteSpec::unsharded`] — route every write to a single shard.
///
/// Calling this again on a table that already has a spec is how the
/// maintained index set is changed; the sharding cannot move, because the
/// generations already written were homed under it.
///
/// # Example
///
/// ```
@@ -5648,29 +5651,22 @@ mod tests {
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
// Identity sharding round-trips (column recovered from the schema).
// A spec left at its default maintains every index on the table, so it
// reads back naming the one on the table rather than as "infer".
// A spec left at its default round-trips as the default: what is stored
// is the intent to maintain everything, not the set it resolves to now.
let spec = LsmWriteSpec::identity("region");
table.set_lsm_write_spec(spec.clone()).await.unwrap();
assert_eq!(
table.get_lsm_write_spec().await.unwrap(),
Some(spec.with_maintained_indexes(vec![idx_name.clone()]))
);
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
table.unset_lsm_write_spec().await.unwrap();
// Unsharded round-trips (no routing column).
let spec = LsmWriteSpec::unsharded();
table.set_lsm_write_spec(spec.clone()).await.unwrap();
assert_eq!(
table.get_lsm_write_spec().await.unwrap(),
Some(spec.with_maintained_indexes(vec![idx_name]))
);
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
}
/// The maintained set defaults to every index on the table, resolved at
/// install. An index the memtable cannot build fails the install rather
/// than being dropped: maintaining it would take the table offline for
/// writes, dropping it would hide that from the caller.
/// The maintained set defaults to every index on the table. A named index
/// the memtable cannot build fails the install, because the caller asked
/// for it; an unnamed one is skipped, because it arrived by existing.
#[tokio::test]
async fn test_set_lsm_write_spec_infers_maintained_indexes() {
let tmp_dir = tempdir().unwrap();
@@ -5723,19 +5719,25 @@ mod tests {
);
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
// The default covers every index, so the bitmap fails it too.
let err = table
// The default names nothing, so the bitmap is skipped rather than
// refusing the table: it is maintained by existing, not by being asked
// for. The intent is what is stored, not the set it resolves to today.
table
.set_lsm_write_spec(LsmWriteSpec::unsharded())
.await
.unwrap_err();
assert!(
matches!(err, Error::InvalidInput { ref message }
if message.contains("tag_bitmap") && message.contains("maintained_indexes")),
"expected the inferred set to be rejected, got {err:?}"
.unwrap();
assert_eq!(
table
.get_lsm_write_spec()
.await
.unwrap()
.unwrap()
.maintained_indexes(),
None,
"an unnamed set is stored as the intent to maintain everything"
);
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
// Naming the maintainable subset installs.
// Naming the maintainable subset narrows it, with no unset in between.
table
.set_lsm_write_spec(
LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]),
@@ -5753,7 +5755,6 @@ mod tests {
);
// Opting out entirely is distinct from the default.
table.unset_lsm_write_spec().await.unwrap();
table
.set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new()))
.await
@@ -5767,6 +5768,28 @@ mod tests {
.maintained_indexes(),
Some([].as_slice())
);
// The sharding cannot move, so the repeat that changes it is refused
// and the installed spec is left alone.
let err = table
.set_lsm_write_spec(LsmWriteSpec::bucket("id", 4))
.await
.unwrap_err();
assert!(
matches!(err, Error::InvalidInput { ref message }
if message.contains("only its maintained index set can be changed")),
"expected the sharding change to be refused, got {err:?}"
);
assert_eq!(
table
.get_lsm_write_spec()
.await
.unwrap()
.unwrap()
.maintained_indexes(),
Some([].as_slice()),
"a refused call changes nothing"
);
}
#[tokio::test]
rust/lancedb/src/table/merge/lsm.rs
+58 -60
View File
@@ -73,27 +73,57 @@ const SHARD_NAMESPACE: Uuid = Uuid::from_u128(0x4c53_4d57_5249_5445_5f53_4841_52
pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) -> Result<()> {
table.dataset.ensure_mutable()?;
// A named set is checked against the table here, where the caller can still
// be told which names exist. An unnamed set is not resolved at all: it is an
// intent lance re-reads whenever it builds a MemTable, so an index created
// later is maintained without another call.
let maintained_indexes = match spec.maintained_indexes() {
Some(requested) => {
let indices = table.list_indices().await?;
for name in requested {
if !indices.iter().any(|index| &index.name == name) {
return Err(Error::InvalidInput {
message: format!(
"maintained index '{}' does not exist on this table; it has {}",
name,
index_name_list(&indices),
),
});
}
}
// Before the builder borrows the dataset clone.
let dataset = table.dataset.get().await?;
validate_maintained_indexes(&dataset, requested).await?;
Some(requested.to_vec())
}
None => None,
};
// A table that already has a spec accepts one kind of repeat: the same
// sharding with a different maintained set. The rest of a spec describes
// how the generations already written were homed, so it cannot move.
{
let dataset = table.dataset.get().await?;
if dataset.mem_wal_index_details().await?.is_some() {
return Err(Error::InvalidInput {
message: "set_lsm_write_spec: an LSM write spec is already set on this table; mutation is not supported".into(),
});
if let Some(details) = dataset.mem_wal_index_details().await? {
let installed = lsm_write_spec_from_details(&details, dataset.schema())?;
if forget_maintained_indexes(&installed) != forget_maintained_indexes(&spec) {
return Err(Error::InvalidInput {
message: "set_lsm_write_spec: an LSM write spec is already set on this \
table; only its maintained index set can be changed"
.into(),
});
}
drop(dataset);
table.checkout_latest().await?;
let mut dataset = (*table.dataset.get().await?).clone();
dataset
.update_mem_wal_maintained_indexes(maintained_indexes)
.await?;
table.dataset.update(dataset);
return Ok(());
}
}
// Before the builder borrows the dataset clone. `list_indices` merges an
// index's segments into one entry, so the result needs no dedup.
let maintained_indexes = {
let dataset = table.dataset.get().await?;
resolve_maintained_indexes(
&dataset,
&table.list_indices().await?,
spec.maintained_indexes(),
)
.await?
};
table.checkout_latest().await?;
let mut dataset = (*table.dataset.get().await?).clone();
let schema = arrow_schema::Schema::from(dataset.schema());
@@ -138,7 +168,9 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
writer_config_defaults
}
};
builder = builder.maintained_indexes(maintained_indexes);
if let Some(maintained_indexes) = maintained_indexes {
builder = builder.maintained_indexes(maintained_indexes);
}
for (key, value) in writer_config_defaults {
builder = builder.add_writer_config_default(key, value);
}
@@ -147,48 +179,6 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
Ok(())
}
/// Resolve a spec's maintained-index selection against `indices`, as reported
/// by [`Table::list_indices`](crate::Table::list_indices).
///
/// `None` means every index on the table, snapshotted now. Lance validates
/// either selection against its shard-writer rules, so a spec that installs is
/// one the MemWAL can open.
///
/// An unmaintainable index fails an inferred set rather than being dropped from
/// it — dropping would leave the caller believing it is maintained.
async fn resolve_maintained_indexes(
dataset: &Dataset,
indices: &[IndexConfig],
requested: Option<&[String]>,
) -> Result<Vec<String>> {
let Some(requested) = requested else {
let all: Vec<String> = indices.iter().map(|index| index.name.clone()).collect();
validate_maintained_indexes(dataset, &all)
.await
.map_err(|source| Error::InvalidInput {
message: format!(
"cannot maintain every index on this table: {source}. Set \
maintained_indexes explicitly to choose from {}",
index_name_list(indices),
),
})?;
return Ok(all);
};
for name in requested {
if !indices.iter().any(|index| &index.name == name) {
return Err(Error::InvalidInput {
message: format!(
"maintained index '{}' does not exist on this table; it has {}",
name,
index_name_list(indices),
),
});
}
}
validate_maintained_indexes(dataset, requested).await?;
Ok(requested.to_vec())
}
/// Index names for an error message.
fn index_name_list(indices: &[IndexConfig]) -> String {
if indices.is_empty() {
@@ -298,10 +288,18 @@ fn lsm_write_spec_from_details(
};
Ok(base
.with_maintained_indexes(details.maintained_indexes.clone())
.with_maintained_indexes(
(!details.maintain_all_indexes).then(|| details.maintained_indexes.clone()),
)
.with_writer_config_defaults(details.writer_config_defaults.clone()))
}
/// A spec with its maintained set cleared, so two specs compare on everything
/// that cannot be changed after install.
fn forget_maintained_indexes(spec: &LsmWriteSpec) -> LsmWriteSpec {
spec.clone().with_maintained_indexes(None)
}
/// Resolve the single routing column name from a sharding field's source id.
///
/// `set_lsm_write_spec` records the shard column by its Lance field id, so the
rust/lancedb/src/table/query/lsm.rs
+48 -24
View File
@@ -471,14 +471,7 @@ async fn fts_plan(
// Without a maintained in-memory FTS index for this column, the active memtable
// arm produces an empty plan (`active_source_can_execute_fts` returns false), so
// the search silently omits un-compacted documents. Reject rather than mislead.
if !index_maintained(
dataset,
column,
&details.maintained_indexes,
"InvertedIndexDetails",
)
.await?
{
if !index_maintained(dataset, column, details, "InvertedIndexDetails").await? {
return Err(Error::NotSupported {
message: format!(
"the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise un-compacted documents are omitted. set use_lsm(false) to read the base table only"
@@ -493,6 +486,14 @@ async fn fts_plan(
Ok(scanner.create_plan().await?)
}
/// Whether the spec maintains the index named `name`.
///
/// An unnamed set names nothing and covers everything, so the stored list alone
/// cannot answer this.
fn spec_maintains(details: &MemWalIndexDetails, name: &str) -> bool {
details.maintain_all_indexes || details.maintained_indexes.iter().any(|n| n == name)
}
/// Whether an index of `type_url_suffix` covering `column` is in the MemWAL spec's
/// maintained set. Only a maintained index has its catch-up tracked (so exclusion is
/// gated correctly) and its in-memory arm kept current; an unmaintained base index
@@ -502,7 +503,7 @@ async fn fts_plan(
async fn index_maintained(
dataset: &Dataset,
column: &str,
maintained: &[String],
details: &MemWalIndexDetails,
type_url_suffix: &str,
) -> Result<bool> {
use lance::index::DatasetIndexExt;
@@ -512,7 +513,7 @@ async fn index_maintained(
let indices = dataset.load_indices().await?;
Ok(indices.iter().any(|idx| {
idx.fields.contains(&field.id)
&& maintained.iter().any(|m| m == &idx.name)
&& spec_maintains(details, &idx.name)
&& idx
.index_details
.as_ref()
@@ -583,9 +584,7 @@ async fn arm_maintained_index_names(
})
.map(|idx| idx.name.clone())
.collect();
if let Some(name) =
resolve_single_index(segment_names, &details.maintained_indexes, arm, &column)?
{
if let Some(name) = resolve_single_index(segment_names, details, arm, &column)? {
names.push(name);
}
}
@@ -604,7 +603,7 @@ async fn arm_maintained_index_names(
/// compaction watermark).
fn resolve_single_index(
mut names: Vec<String>,
maintained: &[String],
details: &MemWalIndexDetails,
arm: &str,
column: &str,
) -> Result<Option<String>> {
@@ -620,7 +619,7 @@ fn resolve_single_index(
Ok(names
.into_iter()
.next()
.filter(|name| maintained.contains(name)))
.filter(|name| spec_maintains(details, name)))
}
/// Drop the primary-key columns Lance appends internally for dedup when the user's
@@ -692,14 +691,7 @@ async fn vector_plan(
// maintained, its catch-up is untracked and exclusion falls back to the
// compaction watermark — dropping compacted SSTables the (lagging) base index has
// not re-indexed. Reject rather than silently omit rows, mirroring the FTS arm.
if !index_maintained(
dataset,
&column,
&details.maintained_indexes,
"VectorIndexDetails",
)
.await?
{
if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? {
return Err(Error::NotSupported {
message: format!(
"the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only"
@@ -888,7 +880,10 @@ mod tests {
#[test]
fn resolve_single_index_dedupes_segments() {
let maintained = vec!["fts_idx".to_string()];
let maintained = MemWalIndexDetails {
maintained_indexes: vec!["fts_idx".to_string()],
..Default::default()
};
// Two physical segments of ONE logical index must not count as "multiple".
assert_eq!(
resolve_single_index(
@@ -917,4 +912,33 @@ mod tests {
None
);
}
/// A spec that maintains every index names none of them, so the stored list
/// is empty while every index on the table is in fact maintained. Reading
/// the list alone would refuse the read path on the default spec.
#[test]
fn an_unnamed_set_maintains_every_index() {
let all = MemWalIndexDetails {
maintained_indexes: Vec::new(),
maintain_all_indexes: true,
..Default::default()
};
assert!(spec_maintains(&all, "fts_idx"));
assert_eq!(
resolve_single_index(vec!["fts_idx".to_string()], &all, "full-text", "text").unwrap(),
Some("fts_idx".to_string()),
);
// The same empty list without the flag maintains nothing.
let none = MemWalIndexDetails {
maintained_indexes: Vec::new(),
maintain_all_indexes: false,
..Default::default()
};
assert!(!spec_maintains(&none, "fts_idx"));
assert_eq!(
resolve_single_index(vec!["fts_idx".to_string()], &none, "full-text", "text").unwrap(),
None,
);
}
}