diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 3359a2643..a8e661c33 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -197,7 +197,13 @@ export interface LsmWriteSpec { column?: string; /** Bucket variant: the number of buckets, in `[1, 1024]`. */ numBuckets?: number; - /** Names of indexes the MemWAL should keep up to date during writes. */ + /** + * Names of indexes the MemWAL should keep up to date during writes. + * + * Omit to maintain every index the MemWAL supports, resolved when the spec + * is installed — a snapshot, not a subscription: indexes created afterwards + * are not maintained. Pass `[]` to maintain none. + */ maintainedIndexes?: string[]; /** Default `ShardWriter` configuration recorded in the MemWAL index. */ writerConfigDefaults?: Record; @@ -595,6 +601,12 @@ export abstract class Table { * All variants require the table to have an unenforced primary key * ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally * requires it to be the single column being bucketed. + * + * Omitting `maintainedIndexes` maintains every index the MemWAL supports, + * resolved here — a snapshot, so an index created afterwards is not + * maintained until the spec is unset and set again. Naming them instead + * pins an exact set, and a still-building index is rejected rather than + * quietly omitted. * @param {LsmWriteSpec} spec The sharding spec to install. * @returns {Promise} * @example diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 7dd902e19..75213a5ad 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -769,6 +769,8 @@ pub struct LsmWriteSpec { /// Bucket variant: the number of buckets, in `[1, 1024]`. pub num_buckets: Option, /// Names of indexes the MemWAL should keep up to date during writes. + /// Omitted means every index the MemWAL can maintain, resolved when the + /// spec is installed; an empty array means none. pub maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. pub writer_config_defaults: Option>, @@ -778,7 +780,6 @@ impl TryFrom for lancedb::table::LsmWriteSpec { type Error = napi::Error; fn try_from(value: LsmWriteSpec) -> napi::Result { - let maintained = value.maintained_indexes.unwrap_or_default(); let writer_config_defaults = value.writer_config_defaults.unwrap_or_default(); let spec = match value.spec_type.as_str() { "bucket" => { @@ -804,9 +805,13 @@ impl TryFrom for lancedb::table::LsmWriteSpec { ))); } }; - Ok(spec - .with_maintained_indexes(maintained) - .with_writer_config_defaults(writer_config_defaults)) + // An omitted `maintainedIndexes` keeps the spec's default of "every + // index the MemWAL can maintain". + let spec = match value.maintained_indexes { + Some(names) => spec.with_maintained_indexes(names), + None => spec, + }; + Ok(spec.with_writer_config_defaults(writer_config_defaults)) } } @@ -823,7 +828,7 @@ impl From for LsmWriteSpec { spec_type: "bucket".to_string(), column: Some(column), num_buckets: Some(num_buckets), - maintained_indexes: Some(maintained_indexes), + maintained_indexes, writer_config_defaults: Some(writer_config_defaults), }, Native::Identity { @@ -834,7 +839,7 @@ impl From for LsmWriteSpec { spec_type: "identity".to_string(), column: Some(column), num_buckets: None, - maintained_indexes: Some(maintained_indexes), + maintained_indexes, writer_config_defaults: Some(writer_config_defaults), }, Native::Unsharded { @@ -844,7 +849,7 @@ impl From for LsmWriteSpec { spec_type: "unsharded".to_string(), column: None, num_buckets: None, - maintained_indexes: Some(maintained_indexes), + maintained_indexes, writer_config_defaults: Some(writer_config_defaults), }, } diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index f5c59c152..2acf0c945 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -608,8 +608,13 @@ class LsmWriteSpec: @staticmethod def unsharded() -> "LsmWriteSpec": ... def with_maintained_indexes(self, indexes: List[str]) -> "LsmWriteSpec": - """Return a copy of this spec asking the MemWAL to keep the named - indexes up to date as rows are appended.""" + """Return a copy of this spec asking the MemWAL to keep exactly the + named indexes up to date as rows are appended, replacing the default + of every index the MemWAL supports.""" + ... + def with_no_maintained_indexes(self) -> "LsmWriteSpec": + """Return a copy of this spec asking the MemWAL to maintain no + indexes at all.""" ... def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "LsmWriteSpec": """Return a copy of this spec recording the given default @@ -624,7 +629,10 @@ class LsmWriteSpec: @property def num_buckets(self) -> Optional[int]: ... @property - def maintained_indexes(self) -> List[str]: ... + def maintained_indexes(self) -> Optional[List[str]]: + """Names of indexes the MemWAL keeps up to date, or None when the spec + asks for every index the MemWAL supports.""" + ... @property def writer_config_defaults(self) -> Dict[str, str]: ... diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index ea0f97621..0ca8606f8 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -4632,6 +4632,14 @@ class AsyncTable: via [`set_unenforced_primary_key`]; bucket sharding additionally requires it to be the single column being bucketed. + Unless the spec says otherwise, the MemWAL maintains every index on + the table that it supports, resolved here. That is a snapshot, not a + subscription: an index created afterwards is not maintained until the + spec is unset and set again. Use ``with_maintained_indexes`` to pin an + exact set — names must already be committed, so a still-building index + is rejected rather than quietly omitted — or + ``with_no_maintained_indexes`` to maintain none. + Parameters ---------- spec : LsmWriteSpec diff --git a/python/python/tests/test_lsm_write_spec.py b/python/python/tests/test_lsm_write_spec.py index d38918f09..e83ef9643 100644 --- a/python/python/tests/test_lsm_write_spec.py +++ b/python/python/tests/test_lsm_write_spec.py @@ -83,7 +83,9 @@ def test_lsm_write_spec_repr(): assert s.spec_type == "bucket" assert s.column == "id" assert s.num_buckets == 4 - assert s.maintained_indexes == [] + # A fresh spec defers its maintained set to install time. + assert s.maintained_indexes is None + assert s.with_no_maintained_indexes().maintained_indexes == [] assert "bucket" in repr(s) assert "id" in repr(s) assert "4" in repr(s) @@ -169,18 +171,23 @@ def test_get_lsm_write_spec(tmp_path): table.unset_lsm_write_spec() assert table.get_lsm_write_spec() is None - # Identity round-trips (column recovered from the schema). + # Identity round-trips (column recovered from the schema). Leaving the + # maintained set to be inferred picks up the index on the table, so the + # spec reads back naming it rather than as "infer". table.set_lsm_write_spec(LsmWriteSpec.identity("id")) spec = table.get_lsm_write_spec() assert spec.spec_type == "identity" assert spec.column == "id" + assert spec.maintained_indexes == [idx_name] table.unset_lsm_write_spec() - # Unsharded round-trips (no routing column). - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + # Unsharded round-trips (no routing column). Opting out is distinct from + # the inferred default. + table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_no_maintained_indexes()) spec = table.get_lsm_write_spec() assert spec.spec_type == "unsharded" assert spec.column is None + assert spec.maintained_indexes == [] @pytest.mark.asyncio diff --git a/python/src/table.rs b/python/src/table.rs index a4e50c183..aa6f33d47 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -185,12 +185,23 @@ impl From for MergeResult { } } +/// Render a spec's maintained-index selection for `__repr__`, where the +/// inferred default reads as Python's `None` rather than Rust's `Some([..])`. +fn fmt_maintained(maintained: &Option>) -> String { + match maintained { + Some(names) => format!("{:?}", names), + None => "None".to_string(), + } +} + /// Specification selecting Lance's MemWAL LSM-style write path for /// `merge_insert`. /// /// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()` -/// classmethods, then optionally chain `with_maintained_indexes(...)` and -/// `with_writer_config_defaults(...)`. +/// classmethods, then optionally chain `with_maintained_indexes(...)` / +/// `with_no_maintained_indexes()` and `with_writer_config_defaults(...)`. +/// A freshly constructed spec maintains every index the MemWAL supports, +/// resolved when the spec is installed. #[pyclass(from_py_object)] #[derive(Clone, Debug)] pub struct LsmWriteSpec { @@ -230,16 +241,24 @@ impl LsmWriteSpec { } } - /// Replace the list of indexes the MemWAL should keep up to date as - /// rows are appended. Each name must reference an index that - /// already exists on the table at the time `set_lsm_write_spec` - /// is called. + /// Maintain exactly `indexes`, replacing the default of "every index + /// the MemWAL supports". Each name must reference an index that + /// already exists on the table, of a type the MemWAL can maintain, + /// at the time `set_lsm_write_spec` is called. pub fn with_maintained_indexes(&self, indexes: Vec) -> Self { Self { inner: self.inner.clone().with_maintained_indexes(indexes), } } + /// Maintain no indexes at all — the memtable serves scans and filters + /// without any in-memory index. + pub fn with_no_maintained_indexes(&self) -> Self { + Self { + inner: self.inner.clone().with_no_maintained_indexes(), + } + } + /// Replace the default `ShardWriter` configuration recorded in the /// MemWAL index, so every writer starts from the same defaults. pub fn with_writer_config_defaults(&self, defaults: HashMap) -> Self { @@ -256,23 +275,29 @@ impl LsmWriteSpec { maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={:?}, writer_config_defaults={:?})", - column, num_buckets, maintained_indexes, writer_config_defaults, + "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})", + column, + num_buckets, + fmt_maintained(maintained_indexes), + writer_config_defaults, ), lancedb::table::LsmWriteSpec::Identity { column, maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.identity(column={:?}, maintained_indexes={:?}, writer_config_defaults={:?})", - column, maintained_indexes, writer_config_defaults, + "LsmWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})", + column, + fmt_maintained(maintained_indexes), + writer_config_defaults, ), lancedb::table::LsmWriteSpec::Unsharded { maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.unsharded(maintained_indexes={:?}, writer_config_defaults={:?})", - maintained_indexes, writer_config_defaults, + "LsmWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})", + fmt_maintained(maintained_indexes), + writer_config_defaults, ), } } @@ -307,10 +332,11 @@ impl LsmWriteSpec { } } - /// Names of indexes the MemWAL should keep up to date during writes. + /// Names of indexes the MemWAL should keep up to date during writes, or + /// `None` when the spec asks for every index the MemWAL supports. #[getter] - pub fn maintained_indexes(&self) -> Vec { - self.inner.maintained_indexes().to_vec() + pub fn maintained_indexes(&self) -> Option> { + self.inner.maintained_indexes().map(<[String]>::to_vec) } /// Default `ShardWriter` configuration recorded by this spec. diff --git a/rust/lancedb/src/index.rs b/rust/lancedb/src/index.rs index c693dc056..f72514a0a 100644 --- a/rust/lancedb/src/index.rs +++ b/rust/lancedb/src/index.rs @@ -449,6 +449,22 @@ pub struct IndexConfig { pub index_details: Option, } +impl IndexConfig { + /// Whether Lance's MemWAL can maintain this index on a memtable, and so + /// whether it may appear in an + /// [`LsmWriteSpec`](crate::table::LsmWriteSpec)'s maintained set. + /// + /// Support is defined by lance, which rejects an unmaintainable index when + /// a shard writer is opened — long after the spec commits — so callers + /// filter on this first. `false` when [`Self::type_url`] is absent, as it + /// is for remote tables, where the server resolves the set instead. + pub fn is_memwal_maintainable(&self) -> bool { + self.type_url + .as_deref() + .is_some_and(lance::dataset::mem_wal::is_maintainable_index_type) + } +} + #[skip_serializing_none] #[derive(Debug, Deserialize)] pub(crate) struct IndexMetadata { diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index e9c200521..d7a499d5c 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2472,9 +2472,11 @@ impl BaseTable for RemoteTable { self.check_mutable().await?; // Map the spec onto the server's request DTO. `sharding` is internally - // tagged on `mode` to mirror sophon's `Sharding` enum; `maintained_indexes` - // and `writer_config_defaults` are sent verbatim (an empty list means "no - // maintained indexes", not "default to all"). + // tagged on `mode` to mirror sophon's `Sharding` enum; + // `writer_config_defaults` is sent verbatim. `maintained_indexes` is + // sent as null to have the server resolve every index it can maintain + // at HEAD, or as a list taken verbatim — an empty one meaning "no + // maintained indexes". let sharding = match &spec { LsmWriteSpec::Bucket { column, @@ -6552,7 +6554,9 @@ mod tests { body["sharding"], serde_json::json!({ "mode": "bucket", "column": "id", "num_buckets": 16 }) ); - assert_eq!(body["maintained_indexes"], serde_json::json!([])); + // A spec that has not pinned a maintained set sends null, asking + // the server to resolve every index it can maintain at HEAD. + assert_eq!(body["maintained_indexes"], serde_json::Value::Null); http::Response::builder().status(200).body("{}").unwrap() }); table @@ -6561,6 +6565,24 @@ mod tests { .unwrap(); } + /// Opting out of maintained indexes must stay distinguishable on the wire + /// from leaving them to the server: `[]` means none, null means all. + #[tokio::test] + async fn test_set_lsm_write_spec_no_maintained_indexes() { + let table = Table::new_with_handler("my_table", |request| { + let body = request.body().unwrap().as_bytes().unwrap(); + let body: serde_json::Value = serde_json::from_slice(body).unwrap(); + assert_eq!(body["maintained_indexes"], serde_json::json!([])); + http::Response::builder().status(200).body("{}").unwrap() + }); + table + .set_lsm_write_spec( + crate::table::LsmWriteSpec::bucket("id", 16).with_no_maintained_indexes(), + ) + .await + .unwrap(); + } + #[tokio::test] async fn test_set_lsm_write_spec_identity() { let table = Table::new_with_handler("my_table", |request| { @@ -6635,7 +6657,7 @@ mod tests { } => { assert_eq!(column, "id"); assert_eq!(num_buckets, 4); - assert_eq!(maintained_indexes, vec!["id_idx".to_string()]); + assert_eq!(maintained_indexes, Some(vec!["id_idx".to_string()])); assert_eq!( writer_config_defaults .get("durable_write") diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 1c9d68f4c..43c1ce082 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -362,10 +362,14 @@ pub use self::merge::MergeResult; /// /// Construct via [`LsmWriteSpec::bucket`], [`LsmWriteSpec::identity`], or /// [`LsmWriteSpec::unsharded`], then optionally chain -/// [`LsmWriteSpec::with_maintained_indexes`] (indexes the MemWAL keeps up to -/// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default +/// [`LsmWriteSpec::with_maintained_indexes`] / +/// [`LsmWriteSpec::with_no_maintained_indexes`] (indexes the MemWAL keeps up +/// to date) and [`LsmWriteSpec::with_writer_config_defaults`] (default /// `ShardWriter` configuration recorded in the MemWAL index). /// +/// A freshly constructed spec maintains **every** index the MemWAL supports, +/// resolved when the spec is installed. +/// /// Install a spec with [`Table::set_lsm_write_spec`] and remove it with /// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch /// onto the MemWAL writer is a follow-up. @@ -382,7 +386,13 @@ pub enum LsmWriteSpec { num_buckets: u32, /// Names of indexes (already created on the table) that the /// MemWAL should maintain in-memory as rows are appended. - maintained_indexes: Vec, + /// + /// `None` means every index the MemWAL can maintain, resolved when + /// the spec is installed. That is a snapshot, not a subscription: + /// indexes created afterwards are not maintained, and the set can + /// only be changed by unsetting and re-setting the spec. `Some([])` + /// maintains nothing — a scan/filter-only WAL table. + maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, }, @@ -394,7 +404,13 @@ pub enum LsmWriteSpec { column: String, /// Names of indexes (already created on the table) that the /// MemWAL should maintain in-memory as rows are appended. - maintained_indexes: Vec, + /// + /// `None` means every index the MemWAL can maintain, resolved when + /// the spec is installed. That is a snapshot, not a subscription: + /// indexes created afterwards are not maintained, and the set can + /// only be changed by unsetting and re-setting the spec. `Some([])` + /// maintains nothing — a scan/filter-only WAL table. + maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, }, @@ -402,25 +418,31 @@ pub enum LsmWriteSpec { Unsharded { /// Names of indexes (already created on the table) that the /// MemWAL should maintain in-memory as rows are appended. - maintained_indexes: Vec, + /// + /// `None` means every index the MemWAL can maintain, resolved when + /// the spec is installed. That is a snapshot, not a subscription: + /// indexes created afterwards are not maintained, and the set can + /// only be changed by unsetting and re-setting the spec. `Some([])` + /// maintains nothing — a scan/filter-only WAL table. + maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, }, } impl LsmWriteSpec { - /// Construct a hash-bucket sharding spec with no maintained indexes. + /// Construct a hash-bucket sharding spec maintaining every supported index. pub fn bucket(column: impl Into, num_buckets: u32) -> Self { Self::Bucket { column: column.into(), num_buckets, - maintained_indexes: Vec::new(), + maintained_indexes: None, writer_config_defaults: HashMap::new(), } } /// Construct an identity-sharding spec (shard by the raw value of - /// `column`) with no maintained indexes. + /// `column`) maintaining every supported index. /// /// `column` must be a deterministic function of the unenforced primary /// key: every row with a given primary key must always produce the same @@ -432,28 +454,38 @@ impl LsmWriteSpec { pub fn identity(column: impl Into) -> Self { Self::Identity { column: column.into(), - maintained_indexes: Vec::new(), + maintained_indexes: None, writer_config_defaults: HashMap::new(), } } - /// Construct an unsharded spec with no maintained indexes. + /// Construct an unsharded spec maintaining every supported index. pub fn unsharded() -> Self { Self::Unsharded { - maintained_indexes: Vec::new(), + maintained_indexes: None, writer_config_defaults: HashMap::new(), } } - /// Replace the list of indexes the MemWAL should keep up to date as - /// rows are appended. Each name must reference an index that already - /// exists on the table at the time `set_lsm_write_spec` is called. - pub fn with_maintained_indexes(mut self, indexes: I) -> Self + /// Maintain exactly `indexes`, replacing the default of "every supported + /// index". Each name must reference an index that already exists on the + /// table, of a type the MemWAL can maintain, at the time + /// `set_lsm_write_spec` is called. + pub fn with_maintained_indexes(self, indexes: I) -> Self where I: IntoIterator, S: Into, { - let v: Vec = indexes.into_iter().map(Into::into).collect(); + self.set_maintained_indexes(Some(indexes.into_iter().map(Into::into).collect())) + } + + /// Maintain no indexes at all — the memtable serves scans and filters + /// without any in-memory index. + pub fn with_no_maintained_indexes(self) -> Self { + self.set_maintained_indexes(Some(Vec::new())) + } + + fn set_maintained_indexes(mut self, v: Option>) -> Self { match &mut self { Self::Bucket { maintained_indexes, .. @@ -499,8 +531,9 @@ impl LsmWriteSpec { self } - /// Borrow the list of index names this spec asks MemWAL to maintain. - pub fn maintained_indexes(&self) -> &[String] { + /// Borrow the list of index names this spec asks MemWAL to maintain, or + /// `None` when it asks for every supported index. + pub fn maintained_indexes(&self) -> Option<&[String]> { match self { Self::Bucket { maintained_indexes, .. @@ -510,7 +543,7 @@ impl LsmWriteSpec { } | Self::Unsharded { maintained_indexes, .. - } => maintained_indexes, + } => maintained_indexes.as_deref(), } } @@ -4960,7 +4993,7 @@ mod tests { // Bucket spec round-trips exactly, including the routing column (recovered // from its field id), maintained indexes, and writer config defaults. let spec = LsmWriteSpec::bucket("id", 4) - .with_maintained_indexes([idx_name]) + .with_maintained_indexes([idx_name.clone()]) .with_writer_config_defaults([("durable_write", "false")]); table.set_lsm_write_spec(spec.clone()).await.unwrap(); assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); @@ -4970,15 +5003,110 @@ mod tests { assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); // Identity sharding round-trips (column recovered from the schema). + // A spec left at its default maintains every supported index, so it + // reads back naming the one on the table rather than as "infer". let spec = LsmWriteSpec::identity("region"); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); + assert_eq!( + table.get_lsm_write_spec().await.unwrap(), + Some(spec.with_maintained_indexes([idx_name.clone()])) + ); table.unset_lsm_write_spec().await.unwrap(); // Unsharded round-trips (no routing column). let spec = LsmWriteSpec::unsharded(); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); + assert_eq!( + table.get_lsm_write_spec().await.unwrap(), + Some(spec.with_maintained_indexes([idx_name])) + ); + } + + /// The maintained set defaults to every index the MemWAL supports, resolved + /// when the spec is installed. Index types the memtable cannot build are + /// excluded — naming one would make every memtable claim fail, taking the + /// table offline for writes — and naming one explicitly is rejected up + /// front rather than at claim time. + #[tokio::test] + async fn test_set_lsm_write_spec_infers_maintained_indexes() { + let tmp_dir = tempdir().unwrap(); + let uri = tmp_dir.path().to_str().unwrap(); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("tag", DataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(arrow_array::Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + ], + ) + .unwrap(); + let reader: Box = + Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema.clone())); + let conn = ConnectBuilder::new(uri) + .read_consistency_interval(Duration::from_secs(0)) + .execute() + .await + .unwrap(); + let table = conn.create_table("t", reader).execute().await.unwrap(); + + table + .create_index(&["id"], Index::BTree(Default::default())) + .name("id_btree".to_string()) + .execute() + .await + .unwrap(); + table + .create_index(&["tag"], Index::Bitmap(Default::default())) + .name("tag_bitmap".to_string()) + .execute() + .await + .unwrap(); + + // Explicitly naming the bitmap index fails before anything commits. + let err = table + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(["tag_bitmap"])) + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { ref message } if message.contains("tag_bitmap")), + "expected the bitmap index to be rejected, got {err:?}" + ); + assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); + + // The default skips it and keeps the btree. + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + assert_eq!( + table + .get_lsm_write_spec() + .await + .unwrap() + .unwrap() + .maintained_indexes(), + Some(["id_btree".to_string()].as_slice()) + ); + + // Opting out entirely is distinct from the default. + table.unset_lsm_write_spec().await.unwrap(); + table + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_no_maintained_indexes()) + .await + .unwrap(); + assert_eq!( + table + .get_lsm_write_spec() + .await + .unwrap() + .unwrap() + .maintained_indexes(), + Some([].as_slice()) + ); } #[tokio::test] diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index 0eb7c0231..d18deffe2 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -38,7 +38,7 @@ use uuid::Uuid; use crate::error::{Error, Result}; use crate::table::merge::{MergeInsertBuilder, MergeResult}; -use crate::table::{LsmWriteSpec, NativeTable}; +use crate::table::{BaseTable, LsmWriteSpec, NativeTable}; /// Spec id of the sole sharding spec installed by [`set_lsm_write_spec`]. /// Must match Lance's `InitializeMemWalBuilder` (`SHARDING_SPEC_ID`). @@ -80,32 +80,36 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) } } + // Resolved before the builder takes its mutable borrow of the dataset + // clone, and against the same table the spec is about to be committed to. + let maintained_indexes = resolve_maintained_indexes(table, spec.maintained_indexes()).await?; + let mut dataset = (*table.dataset.get().await?).clone(); let mut builder = dataset.initialize_mem_wal(); - let (maintained_indexes, writer_config_defaults) = match spec { + let writer_config_defaults = match spec { LsmWriteSpec::Bucket { column, num_buckets, - maintained_indexes, writer_config_defaults, + .. } => { builder = builder.bucket_sharding(column, num_buckets); - (maintained_indexes, writer_config_defaults) + writer_config_defaults } LsmWriteSpec::Identity { column, - maintained_indexes, writer_config_defaults, + .. } => { builder = builder.identity_sharding(column); - (maintained_indexes, writer_config_defaults) + writer_config_defaults } LsmWriteSpec::Unsharded { - maintained_indexes, writer_config_defaults, + .. } => { builder = builder.unsharded(); - (maintained_indexes, writer_config_defaults) + writer_config_defaults } }; builder = builder.maintained_indexes(maintained_indexes); @@ -117,6 +121,51 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) Ok(()) } +/// Resolve a spec's maintained-index selection against the table's committed +/// indexes. +/// +/// `None` snapshots every index the MemWAL can maintain as of now — indexes +/// created after this call are not picked up. An explicit list is checked both +/// for names that do not exist and for types the memtable cannot build; the +/// latter would otherwise commit cleanly and then fail every memtable claim, +/// leaving the table unwritable far from the call that caused it. +async fn resolve_maintained_indexes( + table: &NativeTable, + requested: Option<&[String]>, +) -> Result> { + // `list_indices` reports one entry per index name, merging an index's + // segments, so the resolved list needs no dedup. + let indices = table.list_indices().await?; + let Some(requested) = requested else { + return Ok(indices + .into_iter() + .filter(|index| index.is_memwal_maintainable()) + .map(|index| index.name) + .collect()); + }; + for name in requested { + let index = indices + .iter() + .find(|index| &index.name == name) + .ok_or_else(|| Error::InvalidInput { + message: format!( + "set_lsm_write_spec: maintained index '{}' does not exist on this table", + name + ), + })?; + if !index.is_memwal_maintainable() { + return Err(Error::InvalidInput { + message: format!( + "set_lsm_write_spec: maintained index '{}' has type {}, which the MemWAL \ + cannot maintain; supported types are BTREE, FTS, and IVF vector indexes", + name, index.index_type + ), + }); + } + } + Ok(requested.to_vec()) +} + // ============================================================================= // unset_lsm_write_spec // =============================================================================