From fbfb53e30fad500e7c4a486816e2beb7b633f0dd Mon Sep 17 00:00:00 2001 From: XY Zhan Date: Fri, 21 Aug 2026 07:35:42 -0400 Subject: [PATCH 01/58] refactor(lsm): remove the index-catchup activation surface (#3980) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follows lance-format/lance#8680, which removes `FLAG_MEM_WAL_INDEX_CATCHUP`. With one set of semantics there is no mode to switch into. ## Removed `require_mem_wal_index_catchup` — the activation entry point — from the trait, from `Table`, and from the LSM merge module. ## The read path `exclusion_watermarks` loses its `catchup_required` argument and keeps the conservative branch: an index with no entry is not known to hold these rows, so every generation stays readable from its SSTable. Nothing is excluded until an index records that it covers those generations, so a table that has never recorded catch-up reads every row from its SSTables rather than assuming the base covers them. ## One guard needed a replacement, not deletion `refresh_column` and computed-column declaration refuse a table whose rows sit in un-compacted tiers, since refresh enumerates base fragments and would silently omit them. They keyed on the feature bit because `unset_lsm_write_spec` **drops the MemWAL index** — after an unset the write spec no longer describes such a table, and the bit was the only marker that outlived it. Two tests covered this, so deleting the term would have dropped a tested property. Both guards now check for MemWAL shard directories on storage, which outlive the index. That is strictly wider than the bit ever was: the bit only marked tables where activation had run. ## Two tests conflated two different things An index that is *caught up* and one that is *untracked* both fell back to the compaction watermark, because absence carried no information without the bit. Absence now means "not caught up", so untracked retains everything. `an_untracked_index_does_not_widen_a_lagging_sibling` becomes `an_untracked_index_retains_everything`, with the genuinely-caught-up case asserted separately. ## Testing 933 `lancedb` lib tests. `cargo fmt` clean. (The pre-existing `Error::Http` build failure in `job.rs` without the `remote` feature is unrelated and untouched.) --- rust/lancedb/src/table.rs | 27 ------- rust/lancedb/src/table/computed_columns.rs | 1 - rust/lancedb/src/table/merge/lsm.rs | 30 -------- rust/lancedb/src/table/query/lsm.rs | 88 ++++++---------------- rust/lancedb/src/table/refresh.rs | 19 +++-- rust/lancedb/src/table/schema_evolution.rs | 18 +++-- 6 files changed, 51 insertions(+), 132 deletions(-) diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 9228b4baf..e1dd942db 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -643,15 +643,6 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { message: "set_lsm_write_spec is not supported on this table type".into(), }) } - /// Switch this table to required index catch-up, one way. - /// - /// The default implementation returns `NotSupported`. Implementations - /// that support the MemWAL LSM write path must override this. - async fn require_mem_wal_index_catchup(&self) -> Result<()> { - Err(Error::NotSupported { - message: "require_mem_wal_index_catchup is not supported on this table type".into(), - }) - } /// Remove the [`LsmWriteSpec`] from this table. /// /// This is a no-op if no spec is currently set. @@ -1793,20 +1784,6 @@ impl Table { self.inner.set_lsm_write_spec(spec).await } - /// Switch this table to required index catch-up, one way. - /// - /// Separate from [`Self::set_lsm_write_spec`] on purpose: a table carrying - /// the bit retains its SSTables until an index records that it holds the - /// compacted rows, so turn it on only once something can repair coverage. - /// A writer that already holds the dataset can call the equivalent on - /// `DatasetMemWalExt` instead; this is the table-level entry point. - /// - /// Errors if no spec is set, or if the table already records SSTable - /// compaction progress from before this protocol. - pub async fn require_mem_wal_index_catchup(&self) -> Result<()> { - self.inner.require_mem_wal_index_catchup().await - } - /// Remove the [`LsmWriteSpec`] from this table, reverting to the standard /// `merge_insert` write path. /// @@ -3354,10 +3331,6 @@ impl BaseTable for NativeTable { merge::lsm::set_lsm_write_spec(self, spec).await } - async fn require_mem_wal_index_catchup(&self) -> Result<()> { - merge::lsm::require_mem_wal_index_catchup(self).await - } - async fn unset_lsm_write_spec(&self) -> Result<()> { merge::lsm::unset_lsm_write_spec(self).await } diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index 841b13856..24cc48658 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -2151,7 +2151,6 @@ mod tests { .set_lsm_write_spec(LsmWriteSpec::unsharded()) .await .unwrap(); - table.require_mem_wal_index_catchup().await.unwrap(); let mut merge = table.merge_insert(&["id"]); merge diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index 5751cd916..44a2874de 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -192,36 +192,6 @@ fn index_name_list(indices: &[IndexConfig]) -> String { format!("[{}]", names.join(", ")) } -// ============================================================================= -// require_mem_wal_index_catchup -// ============================================================================= - -/// Switch this table to required index catch-up, one way. -/// -/// Deliberately **not** part of installing the write spec. Until something can -/// actually repair coverage, a table carrying the bit reports every index as -/// not known to hold the compacted rows, so its SSTables are retained -/// indefinitely -- and the WAL pod trims on the legacy rule meanwhile, leaving -/// readers pointed at files that are gone. Turn this on only once remote -/// maintenance owns the merge and the repair for the table. -/// -/// Lance refuses the activation if the table already records SSTable -/// compaction progress: those numbers predate this protocol and cannot be -/// validated, so such a table must be drained rather than activated. -#[allow(clippy::redundant_pub_crate)] -pub(crate) async fn require_mem_wal_index_catchup(table: &NativeTable) -> Result<()> { - table.dataset.ensure_mutable()?; - let mut dataset = (*table.dataset.get().await?).clone(); - if dataset.mem_wal_index_details().await?.is_none() { - return Err(Error::InvalidInput { - message: "require_mem_wal_index_catchup: no LSM write spec is set on this table".into(), - }); - } - dataset.require_mem_wal_index_catchup().await?; - table.dataset.update(dataset); - Ok(()) -} - // ============================================================================= // unset_lsm_write_spec // ============================================================================= diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 6155ec095..255d649b1 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -36,7 +36,6 @@ use lance::dataset::mem_wal::{ DatasetMemWalExt, LsmScanner, ShardManifestStore, ShardSnapshot, ShardWriterConfig, }; use lance_index::mem_wal::{MemWalIndexDetails, ShardManifest}; -use lance_table::feature_flags::FLAG_MEM_WAL_INDEX_CATCHUP; use uuid::Uuid; use super::NativeTable; @@ -249,7 +248,6 @@ fn pk_columns(dataset: &Dataset) -> Result> { fn exclusion_watermarks( details: &MemWalIndexDetails, index_names: &[String], - catchup_required: bool, ) -> HashMap { let mut exclude: HashMap = HashMap::new(); for entry in &details.compacted_sstables { @@ -262,13 +260,10 @@ fn exclusion_watermarks( .and_then(|icp| icp.caught_up_generation_for_shard(&entry.shard_id)) { Some(caught_up) => watermark = watermark.min(caught_up), - // No entry. On a table that requires catch-up this means the - // index is *not* known to hold these rows, and the base arm is - // index-only -- so every generation stays readable from its - // SSTable. Without the bit the field is not maintained at all, - // and absence carries no information. - None if catchup_required => watermark = 0, - None => {} + // No entry means the index is *not* known to hold these rows, + // and the base arm is index-only -- so every generation stays + // readable from its SSTable. + None => watermark = 0, } } exclude.entry(entry.shard_id).or_insert(watermark); @@ -283,26 +278,13 @@ fn exclusion_watermarks( /// with a live cached `ShardWriter` (this session's in-flight writes) the /// writer's authoritative in-memory manifest and memtables override the /// on-disk view so a read sees data not yet flushed. -/// Whether this table reads a missing `index_catchup` entry as "not caught up". -/// -/// Both words must be set. A reader honouring the bit while a writer does not -/// would retain SSTables the writer had already trimmed, and the reverse would -/// serve rows from files the writer still expects to be excluded -- so a -/// half-set manifest is treated as legacy, which is the conservative side. -fn requires_index_catchup(dataset: &Dataset) -> bool { - let manifest = dataset.manifest(); - manifest.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0 - && manifest.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0 -} - async fn build_read_context( table: &NativeTable, dataset: &Dataset, details: &MemWalIndexDetails, index_names: &[String], ) -> Result<(Vec, HashMap)> { - let catchup_required = requires_index_catchup(dataset); - let exclude = exclusion_watermarks(details, index_names, catchup_required); + let exclude = exclusion_watermarks(details, index_names); let shard_ids = dataset.list_mem_wal_latest_shard_ids().await?; // Use the dataset's own object store (not `ObjectStore::from_uri`, which @@ -789,50 +771,29 @@ mod tests { }; // Plain scan: drop every compacted generation (through 5). - assert_eq!( - exclusion_watermarks(&details, &[], false).get(&shard), - Some(&5) - ); + assert_eq!(exclusion_watermarks(&details, &[]).get(&shard), Some(&5)); // FTS arm with a lagging index: exclusion is capped at the index catch-up // (2), so SSTable generations 3..=5 are retained until the index covers // them — otherwise those documents would silently vanish from FTS results. assert_eq!( - exclusion_watermarks(&details, &["fts_idx".to_string()], false).get(&shard), + exclusion_watermarks(&details, &["fts_idx".to_string()]).get(&shard), Some(&2) ); - // A caught-up index — or one untracked in index_catchup — falls back to the - // compaction watermark. + // An index absent from index_catchup is *not* known to hold these rows, + // so nothing may be excluded and every generation stays readable from + // its SSTable. An indexed query against an index that has not caught up + // must not silently lose rows. assert_eq!( - exclusion_watermarks(&details, &["caught_up_idx".to_string()], false).get(&shard), - Some(&5) - ); - - // The same missing entry, once the table requires catch-up: absence now - // means "not known to hold these rows", so nothing may be excluded and - // every generation stays readable from its SSTable. This is the whole - // point of the protocol -- an indexed query against a table whose index - // has not caught up must not silently lose rows. - assert_eq!( - exclusion_watermarks(&details, &["untracked_idx".to_string()], true).get(&shard), + exclusion_watermarks(&details, &["untracked_idx".to_string()]).get(&shard), Some(&0) ); - // A tracked index is unaffected by the mode: the recorded position is - // information either way, and it still caps the exclusion. - assert_eq!( - exclusion_watermarks(&details, &["fts_idx".to_string()], true).get(&shard), - Some(&2) - ); - // One missing entry is enough to hold everything back, even alongside an // index that has caught up. let mixed = vec!["fts_idx".to_string(), "untracked_idx".to_string()]; - assert_eq!( - exclusion_watermarks(&details, &mixed, true).get(&shard), - Some(&0) - ); + assert_eq!(exclusion_watermarks(&details, &mixed).get(&shard), Some(&0)); } /// A hybrid search reads a vector and a full-text index, and either may lag. @@ -859,31 +820,29 @@ mod tests { // Each index alone stops at its own catch-up. assert_eq!( - exclusion_watermarks(&details, &["vec_idx".to_string()], false).get(&shard), + exclusion_watermarks(&details, &["vec_idx".to_string()]).get(&shard), Some(&7) ); assert_eq!( - exclusion_watermarks(&details, &["fts_idx".to_string()], false).get(&shard), + exclusion_watermarks(&details, &["fts_idx".to_string()]).get(&shard), Some(&4) ); // Used together, the lower one governs regardless of order. let both = ["vec_idx".to_string(), "fts_idx".to_string()]; - assert_eq!( - exclusion_watermarks(&details, &both, false).get(&shard), - Some(&4) - ); + assert_eq!(exclusion_watermarks(&details, &both).get(&shard), Some(&4)); let reversed = ["fts_idx".to_string(), "vec_idx".to_string()]; assert_eq!( - exclusion_watermarks(&details, &reversed, false).get(&shard), + exclusion_watermarks(&details, &reversed).get(&shard), Some(&4) ); } - /// An index with no catch-up entry contributes no cap today, so a lagging - /// sibling must still govern rather than being widened by the untracked one. + /// An index with no catch-up entry is not known to hold these rows, so it + /// retains everything -- it is never widened by a tracked sibling that has + /// caught up further. #[test] - fn an_untracked_index_does_not_widen_a_lagging_sibling() { + fn an_untracked_index_retains_everything() { let shard = Uuid::from_u128(1); let details = MemWalIndexDetails { compacted_sstables: vec![CompactedSsTable::new(shard, 9)], @@ -896,8 +855,11 @@ mod tests { }; let both = ["fts_idx".to_string(), "untracked_idx".to_string()]; + assert_eq!(exclusion_watermarks(&details, &both).get(&shard), Some(&0)); + + // The tracked sibling still caps on its own. assert_eq!( - exclusion_watermarks(&details, &both, false).get(&shard), + exclusion_watermarks(&details, &["fts_idx".to_string()]).get(&shard), Some(&4) ); } diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index e94f20f2b..a3f3da92f 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -141,11 +141,19 @@ pub(crate) async fn execute_refresh_column_async(table: &NativeTable, column: &s /// un-compacted MemWAL tiers it cannot reach -- success would silently omit /// readable rows. async fn ensure_no_lsm_write_spec(table: &NativeTable) -> Result<()> { - // The catch-up flag outlives unset and marks retained SSTable rows. - let catchup = table.dataset.get().await?.manifest().reader_feature_flags - & lance_table::feature_flags::FLAG_MEM_WAL_INDEX_CATCHUP - != 0; - if catchup || table.get_lsm_write_spec().await?.is_some() { + use lance::dataset::mem_wal::DatasetMemWalExt; + + // Unset drops the MemWAL index, so the spec alone stops describing a table + // whose SSTables still hold rows. The shard directories outlive it and are + // the durable evidence. + let retained_sstables = !table + .dataset + .get() + .await? + .list_mem_wal_latest_shard_ids() + .await? + .is_empty(); + if retained_sstables || table.get_lsm_write_spec().await?.is_some() { return Err(Error::NotSupported { message: "refresh_column is not supported on a table with an LSM write \ spec: rows in un-compacted tiers are invisible to refresh" @@ -921,7 +929,6 @@ mod tests { .set_lsm_write_spec(LsmWriteSpec::unsharded()) .await .unwrap(); - table.require_mem_wal_index_catchup().await.unwrap(); let mut merge = table.merge_insert(&["x"]); merge .when_matched_update_all(None) diff --git a/rust/lancedb/src/table/schema_evolution.rs b/rust/lancedb/src/table/schema_evolution.rs index 4e41f0e85..d10a45eea 100644 --- a/rust/lancedb/src/table/schema_evolution.rs +++ b/rust/lancedb/src/table/schema_evolution.rs @@ -124,18 +124,26 @@ pub(crate) async fn execute_declare( table: &NativeTable, columns: &[(String, String)], ) -> Result { + use lance::dataset::mem_wal::DatasetMemWalExt; + // An LSM write spec keeps visible rows in tiers refresh cannot reach; // checked against latest committed state, not this handle's snapshot. - // The catch-up flag outlives unset and marks retained SSTable rows. table.checkout_latest().await?; computed_columns::ensure_no_function_bindings_for_mutation( table.schema().await?.as_ref(), "schema evolution", )?; - let catchup = table.dataset.get().await?.manifest().reader_feature_flags - & lance_table::feature_flags::FLAG_MEM_WAL_INDEX_CATCHUP - != 0; - if catchup || table.get_lsm_write_spec().await?.is_some() { + // Unset drops the MemWAL index, so the spec alone stops describing a table + // whose SSTables still hold rows. The shard directories outlive it and are + // the durable evidence. + let retained_sstables = !table + .dataset + .get() + .await? + .list_mem_wal_latest_shard_ids() + .await? + .is_empty(); + if retained_sstables || table.get_lsm_write_spec().await?.is_some() { return Err(Error::NotSupported { message: "computed columns are not supported on a table with an LSM write \ spec: rows in un-compacted tiers are invisible to refresh" From 6a0df4de474b88fa0e151953540f55cbbe4e0485 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 21 Aug 2026 19:42:37 +0800 Subject: [PATCH 02/58] fix(python): return None from unit jobs (#3999) ## Problem Generic job result propagation exposed the PyO3 representation of Rust's unit value as `()` in Python. Unit jobs therefore returned an empty tuple instead of `None`, breaking the documented `Job.wait()` contract and the Python doctest workflow. ## Behavior Unit job completion now converts explicitly to Python `None`. Typed job results continue to pass through unchanged, with synchronous and asynchronous regression coverage. --- python/python/tests/test_table.py | 4 ++-- python/src/job.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index b28cd9d66..0f98219bf 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -3947,7 +3947,7 @@ def test_refresh_column_async_returns_job(tmp_path): job = table.refresh_column_async("doubled") assert job.id is None # in-process jobs have no server id - job.wait() + assert job.wait() is None assert job.status() == "finished" assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4] @@ -3963,6 +3963,6 @@ async def test_refresh_column_async_job_async_table(tmp_path): await table.add_columns(computed={"tripled": "x * 3"}) job = await table.refresh_column_async("tripled") - await job.wait() + assert await job.wait() is None assert await job.status() == "finished" assert (await table.to_arrow())["tripled"].to_pylist() == [9] diff --git a/python/src/job.rs b/python/src/job.rs index 2755a28c5..a08b958a6 100644 --- a/python/src/job.rs +++ b/python/src/job.rs @@ -93,7 +93,7 @@ impl Job { let inner = self_.inner.clone(); future_into_py(self_.py(), async move { inner.wait().await.infer_error()?; - Ok(()) + Ok(None::<()>) }) } From fd2a202a46ef962d8a7232237a583fb6fbf4e8fd Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Fri, 21 Aug 2026 05:30:24 -0700 Subject: [PATCH 03/58] chore: update lance dependency to v11.0.0-beta.18 (#4000) Updates the Rust workspace Lance dependencies and Java lance-core dependency to v11.0.0-beta.18. Lance tag: https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.18 --- Cargo.lock | 84 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 28 +++++++++--------- java/pom.xml | 2 +- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e0e10bf62..73f5923fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arc-swap", "arrow", @@ -4888,8 +4888,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow-array", "arrow-buffer", @@ -4911,7 +4911,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow-array", "arrow-buffer", @@ -4925,7 +4925,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow-array", "arrow-schema", @@ -4934,8 +4934,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrayref", "crunchy", @@ -4945,8 +4945,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow-array", "arrow-buffer", @@ -4983,8 +4983,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow", "arrow-array", @@ -5013,8 +5013,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow", "arrow-array", @@ -5031,8 +5031,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "proc-macro2", "quote", @@ -5041,8 +5041,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow-arith", "arrow-array", @@ -5075,8 +5075,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow-arith", "arrow-array", @@ -5107,8 +5107,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arc-swap", "arrow", @@ -5172,8 +5172,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow-array", "arrow-schema", @@ -5195,8 +5195,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow", "arrow-array", @@ -5232,8 +5232,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow-array", "arrow-schema", @@ -5247,8 +5247,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow", "async-trait", @@ -5260,8 +5260,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow", "arrow-ipc", @@ -5314,8 +5314,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow-array", "arrow-buffer", @@ -5329,8 +5329,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow", "arrow-array", @@ -5370,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "arrow-array", "arrow-schema", @@ -5384,8 +5384,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "11.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.16#d514a61fa7e449dedcd03571eabd338b3c84b170" +version = "11.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index ac5db7f38..9aeccd76f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0-beta.16", default-features = false, "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=11.0.0-beta.16", "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=11.0.0-beta.16", "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=11.0.0-beta.16", "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=11.0.0-beta.16", default-features = false, "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=11.0.0-beta.16", "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=11.0.0-beta.16", "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=11.0.0-beta.16", "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=11.0.0-beta.16", default-features = false, "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=11.0.0-beta.16", "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=11.0.0-beta.16", "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=11.0.0-beta.16", "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=11.0.0-beta.16", "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=11.0.0-beta.16", "tag" = "v11.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index d4d5c2687..978f7c7c7 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 11.0.0-beta.16 + 11.0.0-beta.18 false 2.30.0 1.7 From 944398d807f55821c29ca410c6990b979ff0411e Mon Sep 17 00:00:00 2001 From: Dan Tasse <105866+dantasse@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:06:20 -0400 Subject: [PATCH 04/58] refactor: make branch ops instructions less redundant, point to docs (#3978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As in https://github.com/lancedb/lancedb/pull/3977, we're trying to reduce anything in the lancedb skill that duplicates other docs. So this shrinks the branch-ops logic down to a few lines that mostly just point the agent to fetch the branching docs from lancedb.github.io. Run stats (2 runs each): Screenshot 2026-08-20 at 5 02 34 PM This is out of order, rearranged: |condition|time (sec)|cost| |---|---|---| |No branch_ops.md|250|1.33| |No branch_ops.md|227|1.26| |Old branch_ops.md|116|0.83| |Old branch_ops.md|127|0.87| |New branch_ops.md|135|0.86| |New branch_ops.md|147|0.93| Averaged between each of the two runs: Screenshot 2026-08-20 at 5 34 15 PM It seems helpful to have *some* doc about branching; otherwise the model gets a little confused about our branch model and what methods to call. But it looks like the new one (in this PR; all just references to current docs) is basically as good as the old one (lots of duplicative text). --------- Co-authored-by: Claude Fable 5 --- .../skills/lancedb/references/branch_ops.md | 184 +----------------- 1 file changed, 9 insertions(+), 175 deletions(-) diff --git a/plugins/lancedb/skills/lancedb/references/branch_ops.md b/plugins/lancedb/skills/lancedb/references/branch_ops.md index e94c7e6db..d79e5c8bc 100644 --- a/plugins/lancedb/skills/lancedb/references/branch_ops.md +++ b/plugins/lancedb/skills/lancedb/references/branch_ops.md @@ -1,182 +1,16 @@ # Branch Operations -Manage branches on a LanceDB table: list what exists, create new ones, delete stale ones, and direct read/write operations at a specific branch without touching main. Use for branch lifecycle tasks, experimental/isolated table versions, targeting an operation at a non-main branch, or confirming a mutation did not affect main. +Branches are isolated, writable lines of history forked from `main` by default (or from another branch/version via `create`'s `from_ref`/`from_version`). There is no global "switch branch" state: `branches.create(...)` / `branches.checkout(...)` return a **table handle scoped to that branch**, and every read/write on that handle lands on the branch while the original main handle is unaffected. Unpinned handles track the branch's latest version and are writable; `checkout(name, version=...)` pins the handle to that version and is read-only. -Works on local/OSS and remote Enterprise/Cloud tables, except merging a branch into main, which is Enterprise-only. +Don't work from memory — read the public docs for the current API: -## The branch model (important) +- **Branching guide (concepts + Python/TypeScript examples):** — covers creating, writing to, reopening, and deleting branches; applying branch-tested changes back to main; diff/merge (Enterprise only); and building indexes on a branch. +- **How branches relate to versions and tags:** +- **Python API reference** (`Table.branches`, `Table.current_branch`, `Branches`/`AsyncBranches` with `list`/`create`/`checkout`/`delete`/`diff`/`merge`): +- **TypeScript API reference** (`Branches` class, same methods; `table.branches()` is async, `table.currentBranch()` returns `null` for main): -Branches are isolated, writable lines of history forked from another branch (or a specific version). Writes on a branch never affect `main`. +Notes the docs may not state prominently: -There is **no global "switch branch" state** — you never repoint the whole table at a branch. Instead, **operations are scoped by which table handle you use**: +- Branch lifecycle works on local/OSS and remote Cloud/Enterprise tables; **merging into main is Enterprise-only** (others raise `NotSupported`). A rejected merge is not an exception — inspect the returned `status` and `diff.mergeBlockers`. +- To verify isolation after a branch write, read through both handles: the branch handle sees the change, the main handle must not. -- The handle you got from `open_table(name)` / `openTable(name)` targets `main`. -- `branches.create(...)` and `branches.checkout(...)` return a **new table handle scoped to that branch**. Every read/write on that handle (add, update, `update_field_metadata`, `create_index`, search, …) lands on the branch. -- The original main handle is unaffected — keep it around to verify isolation. - -`branches.list()` returns only non-main branches. Main always exists and is not listed. - -## Python - -`table.branches` is a property returning the branch manager; `table.current_branch()` tells you what a handle is scoped to (`None` = main). - -```python -table = db.open_table("products") # scoped to main - -# list — dict of name -> metadata (parent_branch, parent_version, ...); {} = only main -table.branches.list() - -# create: forks from main by default and returns a handle scoped to the new branch -exp = table.branches.create("experiment-reindex") -exp = table.branches.create("exp2", from_ref="main", from_version=None) # optional fork point - -# checkout an existing branch -> branch-scoped handle -wip = table.branches.checkout("wip-branch") -# with version= it pins to that version (read-only detached view); omit to track latest, writable - -# operate on the branch simply by using its handle -wip.update_field_metadata( - {"path": "category", "metadata": {"lancedb:description": "Product category label."}} -) -wip.create_scalar_index("category") - -# delete: removes only the branch pointer; main and row data remain intact -table.branches.delete("stale-2024") - -# alternatively, open a branch handle directly from the connection -wip = db.open_table("products", branch="wip-branch") - -exp.current_branch() # "experiment-reindex" -table.current_branch() # None (main) -``` - -Async: same shape — `table.branches` returns `AsyncBranches`; `await table.branches.create(...)` etc. - -## TypeScript - -`table.branches()` is an **async method** returning the `Branches` manager; `table.currentBranch()` returns the scoped branch or `null` for main. - -```typescript -const table = await db.openTable("products"); // scoped to main -const branches = await table.branches(); - -// list — Record; {} = only main -await branches.list(); - -// create: forks from main by default, returns a Table scoped to the new branch -const exp = await branches.create("experiment-reindex"); -const exp2 = await branches.create("exp2", "main" /* fromRef */, undefined /* fromVersion */); - -// checkout an existing branch -> branch-scoped Table -const wip = await branches.checkout("wip-branch"); -// with a version arg it pins (read-only detached view); omit to track latest, writable - -// operate on the branch simply by using its handle -await wip.updateFieldMetadata([ - { path: "category", metadata: { "lancedb:description": "Product category label." } }, -]); -await wip.createIndex("category"); - -// delete: removes only the branch pointer; main and row data remain intact -await branches.delete("stale-2024"); - -// alternatively, open a branch handle directly from the connection -const wip2 = await db.openTable("products", { branch: "wip-branch" }); - -exp.currentBranch(); // "experiment-reindex" -table.currentBranch(); // null (main) -``` - -## Verifying isolation - -After writing to a branch, confirm the change did NOT land on main by reading through both handles: - -```python -wip = table.branches.checkout("wip-branch") -wip.update_field_metadata({"path": "category", "metadata": {"lancedb:description": "..."}}) - -assert b"lancedb:description" in (wip.schema.field("category").metadata or {}) -assert b"lancedb:description" not in (table.schema.field("category").metadata or {}) # main untouched -``` - -Two handles on the same branch see each other's writes (e.g. `table.branches.create("exp")` and `db.open_table(name, branch="exp")`); main stays isolated. - -## Merging a branch into main (Enterprise only) - -Merge is available through the SDKs (`table.branches.merge(...)`) on **Enterprise tables only** — it is not supported on Cloud or local/OSS tables, which raise `NotSupported`. - -`merge` takes the branch to merge **from** and a `dry_run` flag. Both the SDK method and the underlying REST endpoint **actually merge by default** (`dry_run=False`); pass `dry_run=True` to only preview. A rejected merge is **not an exception** — it returns a result with `status="rejected"` rather than raising, so inspect the return value. Use `branches.diff(from_branch)` to inspect a branch's pending diff without attempting a merge. - -```python -exp = "experiment-reindex" - -# preview only — returns status="ready" if it would merge cleanly -preview = table.branches.merge(exp, dry_run=True) - -# actually merge (default) -result = table.branches.merge(exp) -if result["status"] == "merged": - print("landed at", result["mainVersionAfter"]) -elif result["status"] == "rejected": - print(result["diff"]["mergeBlockers"]) # why it was refused - -# inspect a branch's pending diff without merging -diff = table.branches.diff(exp) -``` - -Async: `await table.branches.merge(exp)`, `await table.branches.diff(exp)`. - -```typescript -const branches = await table.branches(); -const exp = "experiment-reindex"; - -// preview only (second arg is dryRun) -const preview = await branches.merge(exp, true); - -// actually merge (default) -const result = await branches.merge(exp); -if (result.status === "merged") { - console.log("landed at", result.mainVersionAfter); -} else if (result.status === "rejected") { - console.log(result.diff.mergeBlockers); -} - -const diff = await branches.diff(exp); -``` - -The result is the wire JSON, containing `status` (`ready` on a passing dry run, `merged` on success, `rejected` when refused — also `notImplemented`/`unknown`), the branch `diff` (including `mergeBlockers` explaining any rejection), a `preview` of the columns that would be promoted, and — after a real merge — `mainVersionAfter`. - -### Merge preconditions - -Merge only **promotes newly added columns** onto main; it does not replay arbitrary commits. Practically, a branch is mergeable only if it has **exactly one commit since it was created, and that commit added a column**. The merge is rejected (`status: "rejected"`, with `mergeBlockers` set) if: - -- the branch was forked from another branch rather than directly from main -- main has advanced since the branch was forked -- the branch's rows changed since the fork (row counts must match main exactly) -- the branch removed columns or changed a column's type/nullability -- the branch added no columns (index-only changes are not merged) - -### Adding a column in a single commit - -Because the branch must contain just one column-adding commit, add the column with its values in one operation rather than add-then-backfill: - -1. **SQL transformation** — `add_columns` with a SQL expression computed from existing columns, so the column lands populated in one commit. -2. **Precompute the values** — compute the column's values externally, then add the fully-populated column in a single operation (e.g. via `merge_insert`/`add_columns` with the data ready). -3. **Lance-format-level data evolution (pylance)** — use Lance's data evolution with backfill, documented at . - -## Quick reference - -| Goal | Python | TypeScript | -|------|--------|------------| -| List branches (non-main) | `table.branches.list()` | `await (await table.branches()).list()` | -| Create branch (off main) | `table.branches.create(name)` → branch handle | `await branches.create(name)` → branch `Table` | -| Create from a fork point | `table.branches.create(name, from_ref=..., from_version=...)` | `await branches.create(name, fromRef, fromVersion)` | -| Get a branch handle | `table.branches.checkout(name)` or `db.open_table(t, branch=name)` | `await branches.checkout(name)` or `await db.openTable(t, { branch: name })` | -| Pin to a branch version (read-only) | `table.branches.checkout(name, version=v)` | `await branches.checkout(name, v)` | -| Delete branch | `table.branches.delete(name)` | `await branches.delete(name)` | -| Which branch is this handle on? | `table.current_branch()` (`None` = main) | `table.currentBranch()` (`null` = main) | -| Target main | use the original (non-branch) handle | use the original (non-branch) handle | -| Merge branch into main (Enterprise only) | `table.branches.merge(from_branch, dry_run=False)` | `await branches.merge(fromBranch, dryRun)` | -| Preview a branch's pending diff (Enterprise only) | `table.branches.diff(from_branch)` | `await branches.diff(fromBranch)` | - -Branch names must be non-empty; empty names raise a validation error. From fe992bf4eee913d0ab12fce6bebf654cb245ffc1 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 22 Aug 2026 00:17:04 +0800 Subject: [PATCH 05/58] fix(python): use canonical remote function endpoints (#4008) Remote Function catalog requests used singular endpoints that are not exposed by Phalanx. Route registration to `POST /v1/functions/create` and exact-version lookup to `POST /v1/functions/get`, while preserving the existing typed Job submission and wait behavior. --- python/python/tests/test_first_class_function_slice2.py | 6 +++--- rust/lancedb/src/remote/db.rs | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 612a711af..99c68876c 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -162,7 +162,7 @@ def _mock_remote_function_catalog(): body = json.loads(self.rfile.read(length) or b"{}") state["requests"].append((self.path, body)) status = 200 - if self.path == "/v1/function/create": + if self.path == "/v1/functions/create": state["version"] = { "name": body["name"], "version": "fv_exact", @@ -187,7 +187,7 @@ def _mock_remote_function_catalog(): "job_state": "DONE", "result": state["version"], } - elif self.path == "/v1/function/describe": + elif self.path == "/v1/functions/get": assert body == { "name": "normalize_score", "version": "fv_exact", @@ -249,6 +249,6 @@ def test_blocking_remote_registration_returns_function_version(): assert created.name == "normalize_score" assert created.version == "fv_exact" assert [path for path, _ in state["requests"]] == [ - "/v1/function/create", + "/v1/functions/create", "/v1/jobs/describe", ] diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 08f71368c..169d0fda5 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -494,7 +494,7 @@ impl Database for RemoteDatabase { &self, request: FunctionRegistrationRequest, ) -> Result> { - let req = self.client.post("/v1/function/create").json(&request); + let req = self.client.post("/v1/functions/create").json(&request); let (request_id, response) = self.client.send(req).await?; let response = self.client.check_response(&request_id, response).await?; let status = response.status(); @@ -513,7 +513,7 @@ impl Database for RemoteDatabase { async fn get_function(&self, name: &str, version: &str) -> Result { let req = self .client - .post("/v1/function/describe") + .post("/v1/functions/get") .json(&serde_json::json!({ "name": name, "version": version, @@ -2489,7 +2489,7 @@ mod tests { include_str!("../../tests/fixtures/first_class_functions/v1/remote_function_job.json"); let expected: serde_json::Value = serde_json::from_str(REQUEST).unwrap(); let conn = Connection::new_with_handler(move |request| match request.url().path() { - "/v1/function/create" => { + "/v1/functions/create" => { assert_eq!(request.method(), &reqwest::Method::POST); let body: serde_json::Value = serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); @@ -2520,7 +2520,7 @@ mod tests { ); let conn = Connection::new_with_handler(|request| { assert_eq!(request.method(), &reqwest::Method::POST); - assert_eq!(request.url().path(), "/v1/function/describe"); + assert_eq!(request.url().path(), "/v1/functions/get"); let body: serde_json::Value = serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); assert_eq!( From 5c3bc7f643ed97da7344b8a228a89bed4d6f0905 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 22 Aug 2026 00:22:28 +0800 Subject: [PATCH 06/58] fix: accept non-null Function inputs for nullable parameters (#4006) Remote Function bindings can expose a nullable parameter schema even when the source table column is non-nullable. Binding validation rebuilt the exact input schema from table nullability and rejected this safe widening. Accept non-null table columns for nullable Function parameters while continuing to reject nullable table columns for non-null parameters. All other input schema fields remain exact, including named multi-input ordering, names, types, and metadata. --- rust/lancedb/src/table/computed_columns.rs | 50 ++++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index 24cc48658..b1c0b8370 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -651,17 +651,20 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding input.field_path ))); } - if field.is_nullable() != input.nullable { + // A non-null source is within a nullable parameter's domain. The + // reverse can pass nulls to a Function that does not accept them. + if field.is_nullable() && !input.nullable { return Err(invalid_function(format!( - "Function input '{}' no longer matches binding '{}'", + "Function input column '{}' is nullable, but parameter '{}' in binding '{}' is non-nullable", input.field_path, + input.parameter, binding.binding_id() ))); } let parameter_field = ArrowField::new( input.parameter.clone(), field.data_type().clone(), - field.is_nullable(), + input.nullable, ) .with_metadata(field.metadata().clone()); let json = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![ @@ -2210,6 +2213,47 @@ mod tests { .unwrap() } + fn function_binding_schema(title_nullable: bool, body_nullable: bool) -> ArrowSchema { + ArrowSchema::new(vec![ + ArrowField::new("title", DataType::Utf8, title_nullable), + ArrowField::new("body", DataType::Utf8, body_nullable), + ArrowField::new("search_text", DataType::Utf8, true), + ArrowField::new("search_token_count", DataType::Int64, true), + ]) + } + + #[test] + fn test_non_nullable_function_inputs_can_bind_to_nullable_parameters() { + let binding = FunctionBinding::from_json(include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json" + )) + .unwrap(); + + ensure_binding_matches_schema(&function_binding_schema(false, false), &binding).unwrap(); + } + + #[test] + fn test_nullable_function_input_cannot_bind_to_non_nullable_parameter() { + let mut raw_binding: Value = serde_json::from_str(include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json" + )) + .unwrap(); + raw_binding["inputs"][0]["nullable"] = Value::Bool(false); + raw_binding["input_schema"]["fields"][0]["nullable"] = Value::Bool(false); + let binding: FunctionBinding = serde_json::from_value(raw_binding).unwrap(); + + let err = ensure_binding_matches_schema(&function_binding_schema(true, false), &binding) + .unwrap_err(); + assert!( + matches!(&err, Error::InvalidInput { message } + if message.contains("input column 'title' is nullable") + && message.contains("parameter 'title'") + && message.contains("binding 'fb_01K3TEXT'") + && message.contains("non-nullable")), + "{err:?}" + ); + } + #[test] fn test_function_binding_metadata_survives_schema_round_trip() { let binding = FunctionBinding::from_json(include_str!( From 7fd881bbe387f8b802d212ab16b141473c217cc0 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 21 Aug 2026 09:23:40 -0700 Subject: [PATCH 07/58] fix(nodejs)!: key parsed embedding configs by vector column (#4003) Two bugs in Node's reading of the embedding_functions schema metadata. First, parseFunctions keyed its result map by function name, so a table whose metadata configures the same function for two vector columns came back with only the last one. It now keys by the vector column, the convention Python's parser already uses. Second, Node could not read metadata written by the Python bindings at all, which spell the keys snake_case: configs parsed with both columns undefined, breaking embedding application on add() and leaving only query-side embedding working. The parse now accepts both spellings. Both fixes land in one shared parser used by every reader -- parseFunctions and the makeArrowTable schema validator, which had its own private camelCase-only parse -- so the wire contract cannot fork between entry points. A config naming no source or vector column is an error at the boundary rather than a default downstream, as are two configs claiming one column. The "vector" fallback remains only on the optional field of user-supplied configs. Breaking: parseFunctions is exported and its map keys change from function name to vector column. Co-authored-by: Claude Opus 5 (1M context) --- docs/src/js/namespaces/embedding/README.md | 3 + .../functions/parseEmbeddingMetadata.md | 22 ++++ .../type-aliases/EmbeddingMetadataEntry.md | 40 +++++++ .../ResolvedEmbeddingFunctionConfig.md | 22 ++++ nodejs/__test__/arrow.test.ts | 30 ++++++ nodejs/__test__/registry.test.ts | 71 ++++++++++++ nodejs/lancedb/arrow.ts | 13 ++- nodejs/lancedb/embedding/registry.ts | 101 ++++++++++++------ 8 files changed, 265 insertions(+), 37 deletions(-) create mode 100644 docs/src/js/namespaces/embedding/functions/parseEmbeddingMetadata.md create mode 100644 docs/src/js/namespaces/embedding/type-aliases/EmbeddingMetadataEntry.md create mode 100644 docs/src/js/namespaces/embedding/type-aliases/ResolvedEmbeddingFunctionConfig.md diff --git a/docs/src/js/namespaces/embedding/README.md b/docs/src/js/namespaces/embedding/README.md index 157018e16..a736674c0 100644 --- a/docs/src/js/namespaces/embedding/README.md +++ b/docs/src/js/namespaces/embedding/README.md @@ -25,9 +25,12 @@ ### Type Aliases - [CreateReturnType](type-aliases/CreateReturnType.md) +- [EmbeddingMetadataEntry](type-aliases/EmbeddingMetadataEntry.md) +- [ResolvedEmbeddingFunctionConfig](type-aliases/ResolvedEmbeddingFunctionConfig.md) ### Functions - [LanceSchema](functions/LanceSchema.md) - [getRegistry](functions/getRegistry.md) +- [parseEmbeddingMetadata](functions/parseEmbeddingMetadata.md) - [register](functions/register.md) diff --git a/docs/src/js/namespaces/embedding/functions/parseEmbeddingMetadata.md b/docs/src/js/namespaces/embedding/functions/parseEmbeddingMetadata.md new file mode 100644 index 000000000..d6c381bb3 --- /dev/null +++ b/docs/src/js/namespaces/embedding/functions/parseEmbeddingMetadata.md @@ -0,0 +1,22 @@ +[**@lancedb/lancedb**](../../../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / parseEmbeddingMetadata + +# Function: parseEmbeddingMetadata() + +```ts +function parseEmbeddingMetadata(json): EmbeddingMetadataEntry[] +``` + +The single parser for `embedding_functions` schema metadata: every reader +goes through here, so the wire contract cannot fork between them. + +## Parameters + +* **json**: `string` + +## Returns + +[`EmbeddingMetadataEntry`](../type-aliases/EmbeddingMetadataEntry.md)[] diff --git a/docs/src/js/namespaces/embedding/type-aliases/EmbeddingMetadataEntry.md b/docs/src/js/namespaces/embedding/type-aliases/EmbeddingMetadataEntry.md new file mode 100644 index 000000000..a1bd247f6 --- /dev/null +++ b/docs/src/js/namespaces/embedding/type-aliases/EmbeddingMetadataEntry.md @@ -0,0 +1,40 @@ +[**@lancedb/lancedb**](../../../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / EmbeddingMetadataEntry + +# Type Alias: EmbeddingMetadataEntry + +```ts +type EmbeddingMetadataEntry: object; +``` + +One entry of the `embedding_functions` schema metadata, with the column +keys normalized across the bindings' spellings. + +## Type declaration + +### model + +```ts +model: EmbeddingFunction["TOptions"]; +``` + +### name + +```ts +name: string; +``` + +### sourceColumn + +```ts +sourceColumn: string; +``` + +### vectorColumn + +```ts +vectorColumn: string; +``` diff --git a/docs/src/js/namespaces/embedding/type-aliases/ResolvedEmbeddingFunctionConfig.md b/docs/src/js/namespaces/embedding/type-aliases/ResolvedEmbeddingFunctionConfig.md new file mode 100644 index 000000000..864dfedfc --- /dev/null +++ b/docs/src/js/namespaces/embedding/type-aliases/ResolvedEmbeddingFunctionConfig.md @@ -0,0 +1,22 @@ +[**@lancedb/lancedb**](../../../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / ResolvedEmbeddingFunctionConfig + +# Type Alias: ResolvedEmbeddingFunctionConfig + +```ts +type ResolvedEmbeddingFunctionConfig: EmbeddingFunctionConfig & object; +``` + +An [EmbeddingFunctionConfig] read back from table metadata, where the +vector column is always recorded. + +## Type declaration + +### vectorColumn + +```ts +vectorColumn: string; +``` diff --git a/nodejs/__test__/arrow.test.ts b/nodejs/__test__/arrow.test.ts index 29030d4f8..160f4d5ef 100644 --- a/nodejs/__test__/arrow.test.ts +++ b/nodejs/__test__/arrow.test.ts @@ -173,6 +173,36 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( } describe("The function makeArrowTable", function () { + it("accepts snake_case embedding metadata like camelCase", function () { + const spellings = [ + // biome-ignore lint/style/useNamingConvention: the Python wire spelling + { source_column: "text", vector_column: "vector" }, + { sourceColumn: "text", vectorColumn: "vector" }, + ]; + for (const columns of spellings) { + const schema = new Schema( + [ + new Field("text", new Utf8(), false), + new Field( + "vector", + new FixedSizeList(3, new Field("item", new Float32(), true)), + false, + ), + ], + new Map([ + [ + "embedding_functions", + JSON.stringify([{ name: "mock", model: {}, ...columns }]), + ], + ]), + ); + // The vector field is non-nullable and absent from the data; only a + // recognized embedding config makes that acceptable. + const table = makeArrowTable([{ text: "hello" }], { schema }); + expect(table.numRows).toBe(1); + } + }); + it("will use data types from a provided schema instead of inference", async function () { const schema = new Schema([ new Field("a", new Int32(), false), diff --git a/nodejs/__test__/registry.test.ts b/nodejs/__test__/registry.test.ts index a5cf73e74..973ad8a25 100644 --- a/nodejs/__test__/registry.test.ts +++ b/nodejs/__test__/registry.test.ts @@ -106,6 +106,77 @@ describe.each([arrow15, arrow16, arrow17, arrow18])("Registry", (arrow) => { 'Embedding function with alias "mock-embedding" already exists', ); }); + test("parseFunctions keeps entries sharing a function name", async () => { + class MockEmbeddingFunction extends EmbeddingFunction { + ndims() { + return 3; + } + embeddingDataType() { + return new arrow.Float32() as apiArrow.Float; + } + async computeSourceEmbeddings(data: string[]) { + return data.map(() => [1, 2, 3]); + } + } + register("mock-embedding")(MockEmbeddingFunction); + const parsed = await getRegistry().parseFunctions( + new Map([ + [ + "embedding_functions", + JSON.stringify([ + { + name: "mock-embedding", + sourceColumn: "text", + vectorColumn: "vector_a", + model: {}, + }, + { + name: "mock-embedding", + sourceColumn: "text", + vectorColumn: "vector_b", + model: {}, + }, + ]), + ], + ]), + ); + expect([...parsed.values()].map((f) => f.vectorColumn)).toEqual([ + "vector_a", + "vector_b", + ]); + + // The Python bindings write snake_case keys. + const snake = await getRegistry().parseFunctions( + new Map([ + [ + "embedding_functions", + JSON.stringify([ + { + name: "mock-embedding", + // biome-ignore lint/style/useNamingConvention: the Python wire spelling + source_column: "text", + // biome-ignore lint/style/useNamingConvention: the Python wire spelling + vector_column: "vector_a", + model: {}, + }, + { + name: "mock-embedding", + // biome-ignore lint/style/useNamingConvention: the Python wire spelling + source_column: "text", + // biome-ignore lint/style/useNamingConvention: the Python wire spelling + vector_column: "vector_b", + model: {}, + }, + ]), + ], + ]), + ); + expect([...snake.keys()]).toEqual(["vector_a", "vector_b"]); + expect([...snake.values()].map((f) => f.sourceColumn)).toEqual([ + "text", + "text", + ]); + }); test("schema should contain correct metadata", async () => { class MockEmbeddingFunction extends EmbeddingFunction { constructor(args: FunctionOptions = {}) { diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts index 587d30b19..8b388d593 100644 --- a/nodejs/lancedb/arrow.ts +++ b/nodejs/lancedb/arrow.ts @@ -48,7 +48,11 @@ import { } from "apache-arrow"; import { Buffers } from "apache-arrow/data"; import { type EmbeddingFunction } from "./embedding/embedding_function"; -import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry"; +import { + EmbeddingFunctionConfig, + getRegistry, + parseEmbeddingMetadata, +} from "./embedding/registry"; import { sanitizeField, sanitizeSchema, @@ -933,7 +937,7 @@ async function applyEmbeddingsFromMetadata( for (const functionEntry of functions.values()) { const sourceColumn = columns[functionEntry.sourceColumn]; - const destColumn = functionEntry.vectorColumn ?? "vector"; + const destColumn = functionEntry.vectorColumn; if (sourceColumn === undefined) { throw new Error( `Cannot apply embedding function because the source column '${functionEntry.sourceColumn}' was not present in the data`, @@ -1385,11 +1389,10 @@ function validateSchemaEmbeddings( // Check schema metadata for embedding functions if (schema.metadata.has("embedding_functions")) { - const embeddings = JSON.parse( + const entries = parseEmbeddingMetadata( schema.metadata.get("embedding_functions")!, ); - // biome-ignore lint/suspicious/noExplicitAny: we don't know the type of `f` - if (embeddings.find((f: any) => f["vectorColumn"] === field.name)) { + if (entries.some((f) => f.vectorColumn === field.name)) { hasEmbeddingFunction = true; } } diff --git a/nodejs/lancedb/embedding/registry.ts b/nodejs/lancedb/embedding/registry.ts index 2eae90ed3..5f32f683c 100644 --- a/nodejs/lancedb/embedding/registry.ts +++ b/nodejs/lancedb/embedding/registry.ts @@ -104,41 +104,29 @@ export class EmbeddingFunctionRegistry { async parseFunctions( this: EmbeddingFunctionRegistry, metadata: Map, - ): Promise> { + ): Promise> { if (!metadata.has("embedding_functions")) { return new Map(); - } else { - type FunctionConfig = { - name: string; - sourceColumn: string; - vectorColumn: string; - model: EmbeddingFunction["TOptions"]; - }; - - const functions = ( - JSON.parse(metadata.get("embedding_functions")!) - ); - - const items: [string, EmbeddingFunctionConfig][] = await Promise.all( - functions.map(async (f) => { - const fn = this.get(f.name); - if (!fn) { - throw new Error(`Function "${f.name}" not found in registry`); - } - const func = await this.get(f.name)!.create(f.model); - return [ - f.name, - { - sourceColumn: f.sourceColumn, - vectorColumn: f.vectorColumn, - function: func, - }, - ]; - }), - ); - - return new Map(items); } + const entries = parseEmbeddingMetadata( + metadata.get("embedding_functions")!, + ); + const items = await Promise.all( + entries.map(async (f): Promise => { + const fn = this.get(f.name); + if (!fn) { + throw new Error(`Function "${f.name}" not found in registry`); + } + const func = await fn.create(f.model); + return { + sourceColumn: f.sourceColumn, + vectorColumn: f.vectorColumn, + function: func, + }; + }), + ); + // Keyed by output column: one function may serve several columns. + return new Map(items.map((config) => [config.vectorColumn, config])); } // biome-ignore lint/suspicious/noExplicitAny: functionToMetadata(conf: EmbeddingFunctionConfig): Record { @@ -218,3 +206,52 @@ export interface EmbeddingFunctionConfig { vectorColumn?: string; function: EmbeddingFunction; } + +/** An [EmbeddingFunctionConfig] read back from table metadata, where the + * vector column is always recorded. */ +export type ResolvedEmbeddingFunctionConfig = EmbeddingFunctionConfig & { + vectorColumn: string; +}; + +/** One entry of the `embedding_functions` schema metadata, with the column + * keys normalized across the bindings' spellings. */ +export type EmbeddingMetadataEntry = { + name: string; + sourceColumn: string; + vectorColumn: string; + model: EmbeddingFunction["TOptions"]; +}; + +/** The single parser for `embedding_functions` schema metadata: every reader + * goes through here, so the wire contract cannot fork between them. */ +export function parseEmbeddingMetadata(json: string): EmbeddingMetadataEntry[] { + // The wire format, honestly: the Python bindings write snake_case keys. + type Raw = { + name: string; + sourceColumn?: string; + // biome-ignore lint/style/useNamingConvention: the Python wire spelling + source_column?: string; + vectorColumn?: string; + // biome-ignore lint/style/useNamingConvention: the Python wire spelling + vector_column?: string; + model: EmbeddingFunction["TOptions"]; + }; + const entries = JSON.parse(json); + const seen = new Set(); + return entries.map((f) => { + const sourceColumn = f.sourceColumn ?? f.source_column; + const vectorColumn = f.vectorColumn ?? f.vector_column; + if (sourceColumn === undefined || vectorColumn === undefined) { + throw new Error( + `Embedding function "${f.name}" metadata names no source or vector column`, + ); + } + if (seen.has(vectorColumn)) { + throw new Error( + `Multiple embedding configs claim vector column "${vectorColumn}"`, + ); + } + seen.add(vectorColumn); + return { name: f.name, sourceColumn, vectorColumn, model: f.model }; + }); +} From bacd0e4c3c9e29870dd823f295d92f5779f16439 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Fri, 21 Aug 2026 09:30:12 -0700 Subject: [PATCH 08/58] ci: group arrow and datafusion dependabot updates into one PR (#3738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The arrow-rs and datafusion crates are released in lockstep, but Dependabot has been opening one PR per sub-crate for them — the 58.3.0 to 58.4.0 wave produced four separate PRs for `arrow`, `arrow-array`, `arrow-schema`, and `arrow-buffer`. The existing `rust-minor-patch` group did not catch them because it only filters on `update-types` and declares no patterns. This PR adds an explicit `arrow-datafusion` group matching `arrow*`, `parquet*`, `datafusion*`, and `object_store`, so those bumps arrive as a single PR. It is listed before `rust-minor-patch` because a dependency joins the first group it matches, and it deliberately omits `update-types` so major bumps are grouped too. Co-authored-by: Claude Opus 5 (1M context) --- .github/dependabot.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d5d4cab08..eee966f76 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,6 +17,18 @@ updates: # newer minimum versions. versioning-strategy: lockfile-only groups: + # The arrow-rs and datafusion crates are released in lockstep and have to + # move together, so keep them in one PR instead of one per sub-crate. + # Listed first: a dependency joins the first group it matches. + arrow-datafusion: + patterns: + - arrow + - arrow-* + - parquet + - parquet-* + - datafusion + - datafusion-* + - object_store rust-minor-patch: update-types: - minor From 217ea1a799e23d2badb02b8e638e2b778ef67769 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Fri, 21 Aug 2026 09:30:20 -0700 Subject: [PATCH 09/58] ci: use thin LTO and a larger runner for the Windows wheel build (#3716) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Windows wheel job is the slowest job in the PyPI release workflow. Fat LTO of the cdylib is single-threaded and the peak-memory step of the build, so it does not get faster with more cores — and it has already caused rustc-LLVM OOM on the Windows runners for the nodejs builds. Switch the job to thin LTO with 16 codegen units on a `windows-2025-8x-x64` runner, trading some runtime performance on our least performance-sensitive platform for build time. This matches what the nodejs Windows builds in `npm-publish.yml` already do. `pypi-publish.yml` is in this workflow's `pull_request` paths filter, so this PR triggers a dry-run build that shows the new timing. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 (1M context) --- .github/workflows/pypi-publish.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 4f5a927dc..b80c1b019 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -129,6 +129,12 @@ jobs: # link.exe is single-threaded and the long pole on Windows builds. Use # rustc's bundled lld-link instead. CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld + # Fat LTO of the cdylib is single-threaded and the peak-memory step of the + # build. ThinLTO parallelizes it across the runner's cores, at some cost + # to runtime performance on our least performance-sensitive platform. + # Matches what the nodejs Windows builds already do in npm-publish.yml. + CARGO_PROFILE_RELEASE_LTO: thin + CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16 steps: - uses: actions/checkout@v6 with: From fa3d9b2ce2937c50e1f10eda8716b10491d84143 Mon Sep 17 00:00:00 2001 From: Dan Tasse <105866+dantasse@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:33:57 -0400 Subject: [PATCH 10/58] refactor: move plugin/skills to lancedb-agent-plugins repo (#4009) Moving the skills and plugins to https://github.com/lancedb/lancedb-agent-plugins --- plugins/lancedb/.claude-plugin/plugin.json | 21 -- plugins/lancedb/.codex-plugin/plugin.json | 33 ---- plugins/lancedb/assets/logo-dark.png | Bin 10619 -> 0 bytes plugins/lancedb/assets/logo.png | Bin 36230 -> 0 bytes plugins/lancedb/skills/lancedb/SKILL.md | 102 ---------- .../lancedb/skills/lancedb/agents/openai.yaml | 6 - .../lancedb/skills/lancedb/assets/icon.png | Bin 23274 -> 0 bytes .../skills/lancedb/references/branch_ops.md | 16 -- .../lancedb/references/column_metadata.md | 183 ------------------ .../lancedb/references/remote_connect.md | 45 ----- .../skills/lancedb/references/remote_jobs.md | 151 --------------- .../lancedb/scripts/check_materialization.py | 137 ------------- 12 files changed, 694 deletions(-) delete mode 100644 plugins/lancedb/.claude-plugin/plugin.json delete mode 100644 plugins/lancedb/.codex-plugin/plugin.json delete mode 100644 plugins/lancedb/assets/logo-dark.png delete mode 100644 plugins/lancedb/assets/logo.png delete mode 100644 plugins/lancedb/skills/lancedb/SKILL.md delete mode 100644 plugins/lancedb/skills/lancedb/agents/openai.yaml delete mode 100644 plugins/lancedb/skills/lancedb/assets/icon.png delete mode 100644 plugins/lancedb/skills/lancedb/references/branch_ops.md delete mode 100644 plugins/lancedb/skills/lancedb/references/column_metadata.md delete mode 100644 plugins/lancedb/skills/lancedb/references/remote_connect.md delete mode 100644 plugins/lancedb/skills/lancedb/references/remote_jobs.md delete mode 100644 plugins/lancedb/skills/lancedb/scripts/check_materialization.py diff --git a/plugins/lancedb/.claude-plugin/plugin.json b/plugins/lancedb/.claude-plugin/plugin.json deleted file mode 100644 index 9fe3d4b67..000000000 --- a/plugins/lancedb/.claude-plugin/plugin.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "lancedb", - "description": "Write, review, debug, and document LanceDB pipelines in Python and TypeScript that work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables, with idiomatic query/search patterns and performance defaults for ingestion, indexing, filtering, and diagnostics.", - "version": "0.1.0", - "author": { - "name": "LanceDB" - }, - "homepage": "https://www.lancedb.com", - "keywords": [ - "lancedb", - "vector-search", - "full-text-search", - "hybrid-search", - "python", - "typescript", - "pipelines", - "ingestion", - "indexing", - "performance" - ] -} diff --git a/plugins/lancedb/.codex-plugin/plugin.json b/plugins/lancedb/.codex-plugin/plugin.json deleted file mode 100644 index bb824ceb4..000000000 --- a/plugins/lancedb/.codex-plugin/plugin.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "lancedb", - "version": "0.1.0", - "description": "Codex plugin for building LanceDB pipelines in Python and TypeScript.", - "author": { - "name": "LanceDB" - }, - "keywords": [ - "lancedb", - "vector-search", - "full-text-search", - "hybrid-search", - "python", - "typescript", - "pipelines" - ], - "skills": "./skills/", - "interface": { - "displayName": "LanceDB", - "shortDescription": "Build LanceDB pipelines in Python and TypeScript.", - "longDescription": "Write, review, debug, and document LanceDB pipelines in Python and TypeScript that work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables, with idiomatic query/search patterns and performance defaults for ingestion, indexing, filtering, and diagnostics.", - "developerName": "LanceDB", - "websiteURL": "https://www.lancedb.com", - "category": "Developer Tools", - "capabilities": [ - "Developer Tools" - ], - "defaultPrompt": "Create a LanceDB table, embed sample text, and run a vector search.", - "composerIcon": "./assets/logo.png", - "logo": "./assets/logo.png", - "logoDark": "./assets/logo-dark.png" - } -} diff --git a/plugins/lancedb/assets/logo-dark.png b/plugins/lancedb/assets/logo-dark.png deleted file mode 100644 index 8fd8f220ed6ed02531d28815e826b6aa56fc7346..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10619 zcmb7}cQ{*L{P=^SHEYFIjoQ2Rro?J(Ay#Ye5j9FtRjn1HYLwcWn4yT;B}QwPP`j!| z3AJngZa>fO`~Cl$Jb9kvoO{ka_vV~?UhmgC)=*!Il8l861Oid&XlodQKm?e7pPM8= ziOiS%1`vq|1F;aooVJd{$7f-wTrtd?fUi zl6~HtdV&ZR#<9oQUe*w@MLqM_IJ1B#K19CNcUev7Y41wdY5*sxJm%cxrl>~mRrWj7 zEYVR)Oq+Xo;3KX-5V1Z$kY3#rQLxM4CCEW zf5g1%)SBs$GP$}u;JwFGJ%0*T;I?}Hh$V}u=`C7@C&`=lcURr5kg|Jl(*?8?Ejg=W z+dXi3ntz90*M!LKeFh===NJw%F?al3aFg;$JiONgX4R}*%?Z}Vbzi!aM-I=5SiQ(+ z$xC~|9`ytJHl%b+U8suXhA9JzYGe13bwcLfY)WUuVap+?4v@U`GSeW zJd-AIPe)mOn{%?z(HUifkZ%#aE8*xm$?uO5LLwnM^R!q(hJl%G_Cw_dHmm&Q2l)YV z3KOyL3sLZK&PRfNUk#ycmy>tztcy|<;yS>qY1NSpnx53AEPK12_#d8Y@EHyz`@Pl^ z32Iqy2l1bym9N5OR7fag4I>`0vn2#I4%hnmKkc<|B z>vq?_NR6gV3^k4a3l|$3pd+Wp{np%_dq8Gv=b>ussoV9Afalv6NXWu2bsJZ4=Ml%eLA9Am zOrJzmmPpX9_rV!PCK4DN)gvmQw8iJeyR~L_!E^m93pddDHjn6#a)OD;rlzFNAdt*a ztE#BHTP?%w_rTwRHWDq1*Yf^xT_u_9p@O!;H@TsKR%ffFhupiK_ zXK6AFB?-dds))51VwU?Zaqo2775SMbdJlS6evqYw_CgGVZkHd^mk8%e=nCh(YiWM| z2o?*?g&TIus|#gdkKAgQ*~mA7^>LiwgXcm0@gLwruV1BHpmMs=yiSjwE%?}ri`pAa z$Fq24I*|19NsS66!dEk~G)N|DIPz=77v}8hlhi9_`17Ukkg%IvGfb#6D~3Dho9a*} zt+>%;S)cpEu4w`|MOf(2;(jlo&ZKOlL%2ci_-`6ml^g2sOgIxO`SD8Nii%&^pjaOE zP^@AFjyenaF-orhtazQ(Q7u+}-3hVaIK8T_2tO*n-p|GkZSBe@$CNk9g`qkuJOr>t zx*a^@3VXT0CYmiqo%Bi|GgoMw%fYMvics6zM$GuQ#klWd>~Wf|(d>hr<_Z;75KnI| zM#Ej#=w0rOx9k+Gw8V}{e|F_7azX$@hOxsFKHpK&?RL3wKyM#%o%GYH}hgmwM$8q3CWI*~$&cd-y_m=g5IQ72azJEWClanaa(qn6|hMlA)pviV+k) z>`1U^iXLX)3o#Os#q;V3z10YikgY6r{K%pXTe_ZzseYbB7<~_I2s^g@1uPu1=OiwC z(y@ZBe*-F`s=IASYz#)zhnYORK77t&L#q->jVzN~-xv4GNZwVmZld0cyNq>nreN%a z=n0LL%Q+*WEUYiKy?TX2M&lzataIU<@M;NJ2VPl2Iq4<7qZ$*(?Hi=dlLy$98dB8o z3b1f!$R%BCn6nGwL`V66eIyoJvuot|#QJWa;)j@-K5uzUuX}Dc^Z7gUb#axsptQYx zGB3DkGh#m692`n>r=J z^JyB&_!zHRlORtek4ATg>G_!^`ckkfUy#f(P_asVhUBvG>4>N$9t1Uxy>xS)ikGQ` z7O9AyISO#c{akT~U~a^9GwT)~w0(0Qy;GH1Nr!YtbLT&vc#u3O>RAkBv6CpH3-s=r z@TlsiL85}uR1z}98_Oi3iTSTQFR=$Q#P)V&N@Ih+Vu`Wvt^QXjc5hLU5|V32EPG#H zq3fchsBvuQx=|jmoP7kD)8;Zof}8;t54g9G$KPAq!m)k`X~*=~z)wmq*0uNeV8r<< zQ;HKDoDJ5cac~wkGrAf~Ms*YwibA=Rji#o1SE5~Vjgz{Tch}tqzjeWF?Ue!$rRSHb zi!piLLAryfG1!xKm;rB%^@6`be}C@DAa@+FC$30Gw9b3uwvWOr$jj_Co_QnD0|?=U zhR$Xmwvj)o?fY;?O!}fj#OnQL2xqp60GN)m3oTD*_*9U*MiN@~5fk!q(cZ4&>@`}| zKGP(1p=40;$lgQoyNhlwB$!U;^mT_3#+ILlrHnw{0K<)SUSB^|1tlz_!1XhVz zqbaNSZzDs|GqsJJt_3!V)onJM5tc59sYL6rzNSrbT@O{Eec3-$yO?*(b8}I1#5?12 z${4*nz>IyK?l!(~($Ic1JS$vtN2FmZt2ZeYny40yQsx0q&Fm4xG0-@bX5P7PyFZROqtf z*~GEYi#?Z#fGzgd6+Y5ZpC-q^=jIMR9tCB*o)Qanm*S`{;d>T} zJJ!(H{+i-;XB4x8TV-@3$1Mk5-b86}^en(M>v9vB6(k;FXH@)>w%i!a$+x_x*|Gmn z5&$Rc0~f~oe7w-$>-THWWAzBwQTwyNcF7Gd;jInKGr>g#SeSQ?)mq>Wi5-K4W^&dp zcJtdvkI|+f4sv8{Zrr#t>oyJoDLyE8f&3CIDLM%f1n;coYv%pCmpFJ%-8Gp_Oc(S*ZPcgxS`%Y_iMU>jKh*t5|p80dRqeHjHAb4?HVv;HZzZYlsC7odia&a z9>3rD%Zkh5aGXa=pSYvohv|i$S4!q5cb#InT^WOoPeT@4_TpKZlAs3sU)mRe4#UF@ zNd0Au`;D77@6KYqT4hhv2@BZ{;#ul@3sF>wq9cU0apwb{tWP{s{scdSy_@@^e|3LD z?^e^IbA_?J_{HR432ThY*7Ek(FN`j7A?&xYykYW{b*3*w0-rn{Vk4iMi2OygXIkt0 zhCSI6|xk$FSoyVrxGo2Uo#osV9Mk>GCnxo z!jK59WS@r{K2TOzied0&I`nd0da@(OUH>pEXT%?)FJ!sB{7E!8_kjGh`@Grc{f#+{ z&9(vN%}e;u$DE=@FTy6Z`txc~R(*6HzWdhmOu;YAq+yPd(msr4<1*MwF7TYW`K*M% zb}JjM`}7vKVs`U2VSAQ{K+GEN*f)_TAI4hVm^V3Kb)hB|XM9yN)lPB$&t}~}T(23DZ(*#Itg#q& zM{rI^U~Nk?$3H#qhg&OCai|iVBYFdJ{>)r zIV54ezagLqYYTqQ ziih;oTd<+E#gLUxN%{x)qENFshuU$0G;7TQ%J$*0r3L%rW^J}Cp+MkToh&jd;0bsq9%>^YtHO}e$ENTA=$!U$CC!U# z(F-Y&e|x@r$Q?G5Yc@XVK_?%ksIMEyF0WQGw&Zk2{*oqoZmp->`KaEzG%#-N^qvUR zY}TjoKgPOfDt%a9>X~QN8_jHTJcIxiHe~6Fg3}ubKy{wqXPiitdz*VBJ%ke?(>SzT zb(f2JC7H+6CU_o}imyXGLH?1(q_rn0YC!w8fdmPtQ~N49=AlRzu2 z&?W{FNY;^PsgJ~BdLhtTL-CN57k^$RU_nH#`H`&!_M(hW7=ID4Zhbg%vz3FjqulPy zuRq8`4v)3`LePW}s2U&nBj_bpArJ~#+S|#Zm0;D}+*;+vLr5#gX43v!8i-Z>f3;2&Z(Ig3=bSgTU^LwU6hkY{g(#nO zffrdGBrfIU*`#SH0e4%)4H_4JkWs_zX_$~7To4Y>UzN<^kCV2q>V@+~9A(H3w#NnV zui*4+VO^f1<#H~FpPlReDTK0(#ois7?TlgU)!*}Nwu6LZ-Ab1$xWAe{Go8NkmV_^I z(s7~6uQk3ShQw<^gikgzqrIhDCg55e+*rn^C0z)C$q5^$95sB-|F(g29V*3tCe!DO ztCW>G{kcvuw5(}1<*x`J=jk$hGKv1m@no+(Mv)7A)LWQF|0~A&By`HIyuYT_q=AHf zNhAi_yRLUU+0Bgm+0+xYBaIK{?Z5kr8222PKV>vXCSG5hvAIac98b!%>n zfUX!3)(hnSG~k$-V#lkpThDF&!iDPj3zsNbAbxc*ut;UC{!LTw(t{`_n(Db-uASI zb8M4X2;^qX0aV*-R}5#%UEnk>kFD-BM%|>oNkzcXBoGw)5}^7aD-gYOj_!VJYMd|jo+!LanaZ6}`tRRItphYb!)32~7tFB2$a4?Z z_rstL;t@Xl-ScoG6-DMOB}H_im*)U?#}L=9WYWd-U&D5|>yVAfl+XD4vGI7kNHaUK z`HhkfFE{Cn>8pb)^fXzl_CaPM&N%hBB&Y^;OpG2kB2zZ5C$LtbI%@@a! zuI(46uw&7=8x2zS$z0$joI}Ogq`WI5w8^v;AkH^8$bj1o0iw?e^|*c znld3JU}58Mp&G`f>3rXb;JByLPmbX>vn8KTtfjINJte__{M`3vBlj#=FZ$$Jiuk@y zG!^t_&YU%Yy2Zhcx0DVUWCjoT{&d~Us8#jBcNc_$vV8l(=p>!Wue;ym(!j{1>SvyQ zoe)R$VHb(3*FA%C+&LwDS>eqSaTS?mxAa$lC~(3tM0sqZ-NrHIKe0LSC0jJaySMN9 z>V1!xh8dEUJn2msZ~r%=S8ONB8xQHpO*tJ%UCO-jnoTLCBc|21sC(@r!g*Clu%B^r zRsOkwdwxQwC4tv3%c|V<^qFvh`0o!A^?m?T*v}GS+o$TF#M{`R#H!u^OKpno2glPaWh8~#5cL+C znb~~56ijx%M;VA%c%SFv7XDfIMsu1Qy|`y++j}KA5QxFyt+wQ9PL^{#wye>uqQQ$% z_~^4=Y0OzHq%5WzUR^t^Xl(|hU-&Z?c}9RnEuw68*E3yJ##+5p`VI1Vfk@hpH}y6W z@Qas)@|bnuwh1mGg_=PUa;s1cx?1zWuB4G!=O<=qyNIs?Nhc0T*6R@5GJI>W0i(=e zkNn$cWG^m``}upKmnFN*b|8rnQ@d58@Yj;~ORxRHH8q;AfkO*b@8I_g!CP$ml-&`8$Kon&B4?>lOw`$Jygw}vRVYziF z50EoMO#4GCF~sN%_6AHbD!W_UJr2FyL5E~QJ#SkLchKYG0h^iz79Uc0_Y3y&X(jF1 z7CeprZj}q5DPUc+3;USoX>zQ}vEu?-s>FK&I{!-9^rmAA0sbt=CH2^LJ&L$Kosi)K zG{m&BU!6+0(=wwLO!VSNQ@gn7q&qok@W{|KmwsJ+~|?9fLJ(liA&3UcZv~WOY42g{FY&9Sp1? zQZOUqArPlRx)FpUW;e4TXUK zqLZMUc(gJqZDU#X8Lg`DKS>FwNh{Qsmu+@{yfoz@D8I>W-J=UI?!XO@`=4ik?qoAg z1wHQr4!|yLP=YLcMO^1}wfc~4#zq?&SN^1B)f z0%_O-WRA2;Y8CDchK5F>}8& z|NcQ9ImpD0a}fh!XM4i~s;8qNSxFE?lel!EsYp3nqdn;ciqfSpzU(`NV=fNEMKLi@ z6x=X9*X1$+E?F~Uk^1QR8-^3?7JB?7M(cS!QHgPCl%x|CU@U|G1lTeix7*xC1{L$B zRPaOHv`t73{4+}8M72#yW3XxcuYR}Bz~AO>Ym86-in+O&p$gfLfUj~wYQa~JT4NHh za|3xv(#EOITCmQzm=dSkSy6Yv=olzrAX8fm9F;+s;^_X+h~@mgxFuOWYB$`i_g_2{ zW%pOSwXaHfyyd9q5I{q}frgk41&W^O{H@LJ_3&-V9t5+$l2Q@oUImK_A}Uk;BHqsN znpGm24UH-M1NhJXv6Q(+0u2~NG!T^Wf^V8_0bL=#O6WFzlY}!4(~t|AyJ+TcfnCR} zC1iU5nF*-Lp}Ji?4bK!MbUDz#^nk)U8=Euo_u%=VJ61Ea0V7KO?cleaee%oj>HpP| z>yW2nN2B z=KZWmDp!`P{MJD|X$g9Lm-h((JeukfDYUI+rs@`pGr*a8(=qj!Hyqy8} zWJel6_Wi$(a(XEmO%=x|X00@cC7BtgBGW=~8D65d?JIi%6}>z|R#k;8T^Y|`++z_K zwkCeI^gX4ZmYY6SzK2&WlPL{$zr6-=dQ^zcCl_&bNWYix9XH(i=0^D;O zS#6xtzJK1A4gy?dA2#5)zj{5bHcO8Ku#NC$_-I9*p^8+9kuVybv>^JJ2WiSazfA4H za_~E6?xB4$57>vSnOs27{EllX(_w+Vg}kgfFXK{+aNjG7fJFGrXnsoTT*GG~N>hP< z(8vV~>yMgV$|jLq`NxmL7ZM)9`6EhJLE;oWTbbd`lo0y}Ld&Ycwk^}>BvbRN@-gJ2 z&fCa5hrP+^eu&uLb2+dXJ@$hYS8);5h=1psu7|mKJg&e*8jU^tmk<&w05{*7{}tHa zT)QiBK&N|5=kM)2FMD3_lTNpE-Vc+o_hZpXXnQw@53DWXN9rO(Y3D&bDnBRq>^C*~ zQTRIRlN1F!H~8vA@yNozwDX{`zc6bhMI0y?p(nqWE(dK9k~*mQWsd(_spLNMwPDMn zI3!^-zihF~=D`8~*EstKVZa_;n7H>j`%Nc|(AO8=GuN|dVnbTug8EKCvqPVvw!GxP z+NGt;^R-1dM&Y)EE(0Z%YFT{!R!Ac%oHe5QBrfLC26(1-4Lz?c^gR3V+wJ`~U1!hg z{?8=LSsnetT=o&a$GR83<-?AA*BWmcw7?l=&ZFd;RIn>LYr|*c=%8mjpM zmS~C;0o^ zC|7um1?#MZ6KK~1J^lF2RT6b+Z38uGa~aprI`$n;?4^0RT=rNaCo~z0|F9`WA>G3R zMsGWQNa=#dM=H%?M?F=q;4u72JMDMs%iKoHKEWcii7JW8`|J3aEJR?uE^N~O2Un>< zV;MP*^T}l_-Y!ntN~G7B4Y9m#@e1qQjA;G4f-ih;aUkHzNWQULDxrVsCFzlH;d8wd zxT{MLrU+ZvNBdRZ(7S+FExwgD`LXW6?eQF3|T^ zrz>OHNAlA}9+ARXVsbT45$NE?*Zsv13f*mTlzBGADf+$F=# zR_}VnUEbzL!CK1Yh0|=EQQeIe<|1XydE7J~lW=|6F)Ji>X8AU+WHZ{%^{kpostGwe zTG$)6*ofn=6O8I+f z2641BiB{nSYXTR$;c^s_>~bV<)K%O1h?gfL${i1JVvzDcU}|YPciU;MR|15$sQLO< zd>Gy^c`L|;D2EQo{w<2^$s3Eg9@6dFTs`y0&4OR1lZN{v;8B{PD)KE?Vr%Gz4Dt&_2N@7RMmpN=p-YD}c z2;Dq~#N@ehbL=ruJF`><()#w1o+Q-T6L*SE0AQb7?U+ zE%im3&|ol${@ZP^ht+m{N3wB`#Q0&CU-%klPJxG{4_T-pS&>a@0gCZ)d?P)SCxezh z<3Z@y+npIm)$2f`n9f*7T%cN9y@BT~Z8Jy$z4rIfZz3EbU%ka5drY6)XDxa9^64q9 z8<{?EGsdUKbXsMz@auY)5p91Yl{7033YUwi=+lR`e$;-abVkgu6io*Ec5bO=&$cZM zEuVef7A9%H%xAQ7XJEx6d(4b~@E0Nd zh@>yeqF0#w(0t2Z6^0aDTX}){V#P{ZfVr1oKguhQQC0-9wDX0QqKwXyx60ko>vSFn ze37mYqe9-kz!{`#vDuL$W?9iWTXq#jg&@Q~ofm0YZ_$oOKnyL<_VG#a zuFzJ_bO^3KAf1-xD3;rMnE-9wxB+08X&R~91JoJc8O!s&%Ux1->$#w}YrS1uFT7aT z^CLZ2Ddl+OWp`AZRe^@`nCzH`oK)}~(zzS6rQ?h*PiHyIG+VV}ya+*c>=QlgccHg> zm(|GW0pR;j1%mQ*{1K`YBZW7}isI_+P-l7?d^)4C0$*fJjd(N|xN%aatWVKOFm919 z3T>mN1X4HsVqT&Ofs;-;e}q@j-SytC$nzxTvF=9v(gt~#@ipSfO3cmwYANYxoDsKZ z-XlID_5alioDtU1`^G{^r*Adg`Z(NlGZeTL7i$Dth#&pMVF~RwF52bP9d`j&iZZ z`fX|auh9~Fw-9-ZK0tWrne%Ib{dEU9n&7YPZW2nt=cgv&`2xGNkK6&q+b5h9Gc{P# zNb|KdQ`e}nnWxRXV($Xxv~Q2A6yU*US*KFuERHy^JmPV?x130(88s`#GunRzic$0g zA3_foMP4(rHTAj%I1eqN_@xVOR(1z=Q`D%mqMi?0&UW9UfXXoK1_V!*p<1`4#zjT&L3E%%;qqc zys(3w$TP0K?W)yFsguiQsphiWk%gOcH*Uj%_LGHgsIctWr5}<%U=M2D=-_>v7I z&~7cTn}rP*P>6C-8d_82c1!c~DXb;l3}_H3Tbg@Qtseei-%rN~V&cBm7{f-HU_3?c z)9a>Hrd@WLS_1ZDEapqbV#+V78h^H5z)QbvS1=Y!=C;n*w_-S=ez&46!Zvzqiyzp^l;!twCG7 z$*(^Gd(fofbEW<9Ej7cRmn%6eZrY9>jQun@IXtY}mn(gb9=SHznc!no=(A6&| zzhBEdwtW-myR7vx&7IM_M=W%3R%By&%>9eA&&2{u^Y!WRbbUbE8`^D;>!T+=AI+~u zD_BJ8psg4$R&xAGlTy`$h{o!ZYd;2j;lIqr|M?|$@6Yv{$v?*P8Ywyd8tXvyHEJH% GzWRSDp*6+; diff --git a/plugins/lancedb/assets/logo.png b/plugins/lancedb/assets/logo.png deleted file mode 100644 index de3e82aeff99fa85f609ac16b32eff999d6b7ed6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36230 zcmcdz1y@yFx2C(hyQI6jq)SS=ySuv^5s+>XX%RRyNJxjg(j|>_H{9j-Bd$ZoP&}No z_u6aC`P3v{L+vd(3JD4n6coCmf~*!46g1@DA0!0u6@k%hLH!y0oY@5e6+dI65lF({FY#U#JplzEC+kJ1#aPI6|Y=!N1Xx zm5iY3Qiz2^s^YnQE!Ydbm!K)I&k^c54K)1_uCkWzduo5X;t3A}Cx!L@{yQx!1{;nj z3hQoM<^B7dv;CQ}@x>lpR$IGo5 zRqrZ=yUvLX+MmWoM?Zc1h-b4inknsf{F5@&>hIxU&T5Bmf9m-4aL<=7NG6PE{SB`v zD+@gO?Os)Gk2l9?!^ui!hw`^>0}JAAxLpRflInb_Fyv$C=hl$Dg|iW%Wy z!iS?>IhvP`wfsV*Ar)JdbABA=p%!tR9nn>qa;sSgRWB0nu&}WBYS8S^ zY+R*X__eC)&#SMZqGEinO_LCePdY!LIO&f4DGEeLZp=2?s!hIPPJ~u%iUHg z_S+P)8Rvxt>h^Enw1OY5>>*x8+6bqX%?49{uOCRNe+MuK3qLsJ@jKsOQi%GQT3Ky# zWsH2k9nax;l3@F`d|dkN8~0mH&2*l$G7-)@ENTuT2-dEj*loK~1&$HKP3pa=Y0=fg zVLcuE#_u z^ZH@JHy{dh?HgYF=-zjRX8V#Slaht7NSj#-;Q>`o^O4ycrsDfI%dQ+eB*~^vZ+sZ^@L1o4`uC+x(?HmUa|N**!*_q5{BOu%YIkhf_)acl}v^%7Rqo4^tEWyKrOT)``n>%8KbF|P{HoHBN!8rKy=c_V%c0FH? zieatpp?O-*j9up!-Rk$m%5)!khAFiy&s<58V~EgV0ul#3hQOyZ=(FN`v;`R}DUFhX zLp5bd$Zj-D6%}_sgmD6YqKHKmX8D!}Usdy)HgmL4+Qp>EDVnmnx+!a>Qs{pU%@T&J zSzT1h86#dMm9s)pLmq8U92bNr(Qvzn!<4&FL_$Qb1c}zMy49;r{CN^@n3)6WIy*Zh zs0WbL=rH-eS$+!3fkE0d=ncaoAc(MV)Ut|7E8Z&%JQ2^ExG}BOeIzBKFHoDO!ij(3 z4$m`A&TgxF5aB#mWq6l4W4BWM+hLvnAAf=>sywn%#awdc!Sf-81e;4j$?%)i$QKN9 z;qmMdD+$BeIa-9sKTkV&c1>gM#P=fgHzLKK&=F*kN353WwV7MJ4u0+Y=ID9)MoAr2 zZ$HSJe3YF_FhtXxa`LgpPe7vtL+v`_kd9MP$d)qyuhBiUm+ zD{P4g2? z{gKZ@gr?)r9H1T)^u*}ec|KI|G>$+b4A&N8UZ#}5hnGe`%w;iv%AEPv7hEcWmcax= z#TEj?xvFvJt*aKKSpPo|&$Zh>_P4_` z@^&b3+0YxQ){&yymlUwEtAA6pE+3nUpAX=sp3%e+7-LCEynQ#If}}V$Ir$=4t4lfK z(Cj^tWRPKg@%_n~n^@L&uSWNHAYu5%R@s2w&3{@?yX2ufM+Ba^_96 zNY&~VMKuJ@%fYcuO48`Qo8|NXBDQ&NZeS%FPyja$T6Jf2&n*7*ciFvx`*0>1wl?$i4AOE*-IZXe~XB|639 zB+d~E`KV-Ft5&&uJ!j~e%n&ua!P6Wijpq<2w8vfr_6H>sUc#u2{aqGSXskA~n7H|m zDEla^ItyHJy+n6jtbxo1Sju04m%9_BE=`L*V>=fJltusP73?=~y7H2!uLa+(PT@#z zcau$=&k4gPU)?Z3ZyRZoy0^)&)9+lP*}&MJoc@LFB4Uaj zC;qQX$8k$^6jpM*-#4QrjL`{P9Gnn>#6dQl9Ft3&#KDyR{`=1x^peq?P#;Qnb`_ae z3}d=N$yIdG)e4?cxag}{1=>38@z2RS(W)eTjOHQuB1HK?6Syj=@)Rd+3C zZ5vLlyDZvlaD4SM?vF#ZR+ZQ8_V)Pv;dS=#K!zms6vTZCAy0*2;U@v-w^-EO zAZ_jcPC-xoqm_lUNhzuGXWlfl0H1;)WO^BQA~73QshcwBtPk1DPLwGGAs=-iWTwuz z1G9K@7SZa@K_VXm)$7yK6MYmGpEJ)67cHV8k$f_VHzC_&j}0SvXjf@zsZYIOD|^8_ zI=qwtdqe%+^X>X+-R!~FPO-MmVB~1&7A3E%{h0f1>n4<6U1Sb_4ZX__+^}@(Ok}jEW5(;p5Mh zHqW2;kHY=QMgTF%y9~r(yLNc0!U!w%=FaBu*!}`n)bgu`ydfP{{iO!&!-o$oE=%9v z;3nmZEC+?tdj2H$9S?V%fjqx@VkHhHClYc{%+$p5M0`ggI-7hsPYq2fJ9G{`U?UnP zth9>7qaOfPFi#Szasql~RzAEMH} zeELMh&o;uT(px#mP8+Jr;L{Q8?d`i+$E>c>WV zQa-atgwv2QcNm&vM1~Yvg4-E;CG|7r%KO7fgci&1yz~(vB`4K+_-&fRAClGX*O$i; zY;5eujqgtR|IKICIfae3Kc4pV{H)c!Ip@c!_Vn-|n2t0tucsD|tsd*4xGAgn{JDFm zS+65nAp)9S%>Gw7v=@sc?C9%~C)742KK}LL#m<=Jy(O_!5Da@Ph9ic#x%slaCNlHf z;q3}38-el!&y;9}dq!$lMo$AciEcy;owmKl&Fwjf@@>}!wOM$0cuxhN6La#sc}+s@ zcT}1d$H~tUE_%B6HlmszK7IOB)W&CTy8jSKrK%eE$LyD8nS2*(_82vNWYq|%UQ5dN z@887)-oJknfwx!lzA1o#j;`0FKR^Z3wCAX*=C36HttUONFE5>2kuC}&(vP(f5fR>I zE1Z$;GL}VRN4gbS#$-G;<0^vaKeF?zXrbZqE#d8R^i2G%zsu58UH{qA33}KptzCBQ z2`-T?fw*91u`u6*^RRW~^N8@Z!F~03mzjk;Cntxr)n%2$llD;Rtj28BdAeB2^|FZG zL_qYV042KA2K0|D`x#7^k20Nz^PO2(I272}jjD7wCGH)3v_Iv$!6h)2(sXB~Z_TZK zeE>)p>z!wN=wJZ8i~9^1#FGVw#lyH*=eGPeAAzDAvX>COi` zlVS--9y;}zpRu>Q0;+{X#B{F~DTr@EB z|8q;!QtK8EB6j1lUnld@XSZvCZgQTS*z1F4As=s z6cMuO_sV1Iy=yp2Vo>=6RiNayy6F8LEmp0vI%p#t@nrLm1QcVhA2L@qNpW%dn0w-X z+9MCq$cAw+!i~EEyeaVgJ!yaH-uGDqxx1h9OiL2Kv#iUII5Lj(a(#Vyri?;g)>D4R zZrD-u;O^&UPF{laSeQ z0No|z$N67^dy#P)&)*A;mZ!1oj6{Rf6yeN<*wl3L0OS~Gsq)$V;b(ix+Xf)ZGT25jz*O#}W`i?6^^U2G;@)xU5rH-JRXa3Qg$U(wEtTzhDVO z$mjA(HU^d-FwZTEH=r2=sOfa;=TD1au^AzxUA~R_MvoH$>u53Fg6yiW2=#%tCmffw z=$%F!uxc%|@=1OPiJOqGqxdYkSCTm$zANs*+7mXH4$^V>DcJ-3CUMLAi%pei-56wo zXPVW&4f->$H)VYf(`i!f?(Q;!FmC`EQ&3e^6+@qd_ZfG4ZGrAs4S|%GN7a5QKk!%7 z()uOesGoFLc#_@I@6cR$IA6csMHQ5^iPgI^p+wF1dB%>U;Bw1 zXz%(dh#vD(;^X6m?l%)TZeqPP)Cj5LSkfg^2hPsU^ogugvb$1UEMiNI-JF} z%b!N~0%~{q}-ms(fGl|}{Vhhhn z-HKlS_s-2`6Z&h$^oYDs1YSGxr-@y?a;-Ozx&`eyR6;V8H`!y3UFWUm7PA8wRoW=% zwVD)#l(^{AAK00g>`0Da;R~YL#oXNAd&#k6k$_)UT2;Iy-Q5%({B;3=+X!yL1rbY2 z20qmYoBqWP-@t!h+{&E?61g$|{reZrp;@@Bu@PHs$*$L7w0F=N!R6pr#hHoK^TYKB z_`hGjPg7SozZj=Wf&#Pc;lKIKdhCN41s)+`8HKb<7ayUlQ|qeFa#V3K)s3XW%1ZJZ z5ta9wL}x?s>p&ZzW@NM#-Dg<2B8Nb@W0AbunrYJQvLvbe{`=$ByT-ygD~B~O%Zn5( z9WP&&qxtR~Pg{4liS!|K8dN`hiOBJzF5+Xow6ydlIR;`~4>i(ev$vprEC%>hUtJy9 zYM0nmEKnHV;llnYXf`GU6~V3EO6GVbK;Ztn5n%8H8S?+QQ<3?%C3 zl)AEJEs@0;sjgM)+q={`_AnD@;hy4JDa&TsDB zrv2{!t9s`oF|C@%Zw^No2C*tKkduQK@$)%Zu`l6}de43nCWAF#C5tfXRulr;B zT;Tbk>#x!R_65M~WZ$hrGs(3k)aEZdYoN9Hrz?2%WuC@UA$icc9ljdW-kgCZ{Tg;! ztII+Hff}GksH>?#ONl2$BjMfmjWNfV$#$0n@5mP={b$5wwY^~AEfxe)u0zOYlQd`{ z#7>Z?K0V*N--w7w1D+y^G?KPj!eoJnuOt=xMw7+E~kSW`V#ECD5Ze@dKXE%(DqBP1_=x32PV;AKuNTdzH*y~bE`F4x5zu=FrPht)v zxbSq^w4g@@;`q2;4bGr3!S? zi-A%%X#KpSR@^w)*ys5w46p@s^0BmT?g!$K4y(IQXz)jK)JGh`Gj0v*HIcg=rK`|dpx-kc@f zUgJLWP`l@UUo@M*&GX1IXBJ8=2J#0)W^`_CZH3Z-)~`krwULtzouo_|;_lAHx|?9l zy=dNS(4||Y{X-ZFgA)__z`V^Ugz3$jgG<`CDK(&}B@6b6Ax2?>4cg}R=SL`jzug{! zWJG=bNOAed)Gz?=bb_zzf&vk^K@}NAoCGzrGgMh#E~{Z`IwktBU$zd!z;|Z?ZfX8- z^6c3n9-f||)yoC3#z=4p6iZ0Z5KW`ozavxvoJ?lY?s2Z}M~Uj+cy zdS_t3Ocynbh)!}P(9RxZuOyHvQKF%bX! z)x79r+T!pVrNycD5=t%%B6AlystaU%uPCfUg4J8Q1KyL#9pCY!2ut_)!fd`j!hQdiw)2E zbm_lOxK?=|K6q?&Kb|DUN>jRIh>61bF(4x&8xC4>;DT|wj^eca<4;9%DURy+drPP?G@4)d<{`F zhh@7L$tr3ZE8{djhWi z4*MK0)rNme(O??TKCd04u+m=#@(McG?mkyLr2-U30sCJx%LpuUe>KnK0Y!|*dFgv& z!_CM6;5bY^9mXj3xj@zqD%NIh8-^`|7-}~VB<>JV%X-!xQ+TNp%mM>yIWjVGF-udIRmqZcNB{P@Pz2@CMP(5jCbXlD8`gj{r(a3ncQd3e))NZ?^Y^_&rWBe%C z6T%{Jn--r)ue+y*SD)Z)t#3zJ>hUzOdFJ1QDgx?4mC)-9%dVihbbMp$Uo~>5RRO3yU-TCGCMztB+Phr-D&+KRF7D&ML6 zWH2l|)J6T5FABrM!(rO=hE(AfJ?2{o+>YZD6VLVlMR5Vq6MF|`^~4oucCTmT_?<3- zduhX?)D)2N79y>`+yC3VQK^+n~E{gNq7~&>WYc^OF^gG*V!xFQ# zvDqvX7_i;=`mo*D*hqS=F`UQaf=a+T42iadx*%+EoIgW@t zW03~|H?fnyhc0v?B9D)`xw&>~G3=iZfBbtPN~VM`v7Wjr4tQ7(t?MCWLyTxZ3ge`r?wMJ@R1{G zN=gy4y?I*F&>=#CTQ`?e5|xojD|kX~j-{i39I-|5X|g`F4M2>aeoDM>rKjkqDATav z_b^!hm{(o3{9|T$aY3Od{mT$d{ybSKbA%y+uIEmSH;EGuh-~?Y@F#AqD93 zKyAzCJuTwf!{V|&lGaOLG4#aFa3u_(+s`y)1g*Av7XZc^qQNO%d?Xs8x#e>XOq%Ej z9F4E@T{(}s8{{8I@-j28a}?-M7PN_pJ^_)M1BtpMXc0m@IjjiGsaqH_Gtw_92;;%P zx!CJC?LI{3mfDKBgYU$-DTY8GY}`3Xq*pJ6+?EoQOuWqo!9GJt#By41NFn~;(Q>nT zP`!l)BU5n3<&Q<HVU^4;Ct^RhThO@?2O)}lyZIT;>qPkST* z-1<{0B*83gpRly;ai31FoV6k)Aub3ZYNLKZMRQxD!%PM$Q{``mdw8alte_P9AYdVq zgH9jFMd~*wPE=SDbq@5Nc>x z+iffE9JA4nZ!iJ8a3)C}n)!uBzTO};s#rEdb^x3|2}p}f8o||Iyy|?h{$YppSQbpv zVyArlyx#fFIu07S>5H2^Z*F2$RaKGGdbe0nh;}0f0Z;FzxWvSanbY3|!rlhuvj-T= zcnJjbp$}-e+x5EDCC77FwuSz?6m{2RC+;#ZIA8DRjWEYRZy%pK@WK}^sI&hut@~Du zlJ!0|(95lg2pGaNLPGarK-4hOw+;77;wY!9m($e5&T9eQVgo-f5Qyq|a#Vt35c5id zCl`xv|N9R?xqf~gtc>%&L){PL-Djui^fkhQ#4*`fC(9^M7p%U6v%g}-Nw!yc60G^+ zrN%#ETA|fPwlEKRP+YF+)s|w=Oq(066AFPikM9huXMMo=tHe`xed_ydgg>M((hC1X z9b3}mA?JXuwMCK`cXvDu{kx~hs&HdHGT z56)MsG<0s%|C$>dQeUV66`Ch3s*191#`zy`msQK;lXF#&?k}N{;IvGYkWEpESXLg2 zLB62#VwC0C($caAFmLhkYQdWavf_&#D~#SZXdnpS>*I4H0WN4cCWKDoLi}`Pl{uel zOKXxnB=9C!sBCZZR$dfcR3QW#ikhzcU8R~QfPMeM0Y9>DFb=$gaq#fI585<62fjSr zM_vWA2%@F6YUDAia*cs%sHKA&y@SLhZTuFx|r+aSw5UF)$o+7AEaeq@#%=g&KC zFk*iV*q<(O8$iE>iekGb<8t_DZ6w=e-XG%vI^L;%?c6g$WZ&UdVd(qXIfpKjo_3(X za9~USvY&%FyZ>^pyHS9Dm+g;%!Op6Bw9>}oHJL9I|NMCCnwOKaXi}x^Lx=>|#qK*e zIEYU`NN6kibk<+RqFWOekuhE}O4#zzXmaDe$qDxN^cw)IX3}-5uP=*>i^)lnKLb1+ zAZq=O@d7sBlPrJfY+|y88kNSh=A8IEXjx%mpHb3eP#b|M@zkIWNbn)JNz#bO$SFP^ zAGh#E#|_dhQm&wt*S#cSG3Y-Z*q_qAp`xRUnKv!YFP$Trf9ZN{5SRT*&e zBYkNA5&*6Xtm&kn?Fz;ex zW9#P*uU*Fr#X68~G%s9mX>6(_>Mp~cfDR9J;reY8+|o3*ofCK<20%j|vGmIaCPe`7 z-FBkVTGHez8a%H6eY-~8J7Jz#Dk9a7miALMG%#2a=H~XW14y(6hz`MdpFa6@zGuz} z4Nr*>>*3Rb)cNJ>czJnwgSVX$3%oJ<54282073%A|7ArmF^gDBrRyw7p<5ZQ{=`z+ ze0TWm;0NAR+07?gtx{yd|E2KL#`GF94UL5bo^3SRP?H5XSZ*n z-rTvh`$@h`C@(9kMTnk$XJ=tyL6b7%&iw15lS{j8=n_INpWH?W!rs4=>S_QSpqQJ# zza13=0v=MvthoMeoEZsPrkTuV{@CLtOv{W|4_~i{J@ZKB%w>;567fBG;^!o&25f{+TGpx_-4BX`6g8-dZ%jYKbO zf`WcFK=3aDTz7LgRpJFQdw;n%Tdc>Z{yCe7FEoqe*$KzwyV=1?+odxAvWYz0+&5C% zhsWnrwNkdWw)F?oCHj1gq1TB;{$C z$;B6BcJ=ojL7kPL-=63Z1qEgI`ee1^4s43V7$G7g;V*L1p0|L(U7xOpYJm9C5#ZYW zoroB)RTQG+t~C<9>*b7+0_eO7Q`XkjZ2znN#)RWH3$&jbL|9&OZe!NX6EIGAR7)c zayxIMmlY@e3?Qwu9xgO)2@a>vrvA*tM1(UtlT-|og0WUbFG?~g?z|6-*a$pXuQ&_+ z(;J?Z(^<4*4^@v(>;aS_V+o@+s%HS&;!y4uK%0h zqQAv>8g5lk0ouVXMn)@R02n^r$4tBxfS7R4*!RIX?z0l0DyphJ-@QDZayV24Zmz($ zhT{u-d+8B{IG7W$SOO?&)w)cbJf`>6B!14t0)v;h9n`E*kTVE2H-KYgUcU5HfCJRhKRS+C| zSEE-`bNM?_-rL*gj|NCzJlq^FOZ*V>?QW4s@1UHP0@+Ro2RPQYCbNeGm633v^4_Z0 z#vpvy2rjz~9Hit*N@D(`GK>ptr3;XE8c|kI2rGp>fxho-^(x57zb-G1-~=wjZxFTd zMQn|wPt(d8-ud%Yf|{J)dK7QL=K}W1wH)PZ9X<1$jc0MyEOPmKTwL7se`{d)aNgC; zKH!S!2ZEBiz_fb}r$-2QMSm%dBqGt2fjtF`+B;SBVi>F0U?L7B(+;v3sbadM*VbHLJf7+dk(@~B5a!|bi;<1U2jV-#^2`Oq7 zL|mZw#`zOSi>(sRSF^3NKWOTk3pX)dqS>i|1@?~%^ze3=hJNUYG!#Y#NQdvAj^n?K zGCkZzJCDAz4n&L`LJ%=Y5=SaBwS;6Q>KaEd9y+bIUuSEO$HKCjnP-lIV@;a*>C+v4 zO{SY=RCeKQb>gJ+oLd-|E7)vg?d;p44eX+Go?*RLUv+3ciFSGB?Bz5*xaC~w`~1)E zbK=hNnYEj6Eo*UPdT)0RpZrY^M;hloJ+otDDGpm&3oP%v$4{(#$Nv~0F&Hqkw#^2< zz!j;l*B&J}FHP@|+nQtM<^BHSU2SyG{pQy#;27*A5%VZXirZ~JtwY3s8+Hp6g{>zp ztYWRv06MppVc;0e%mA8U z`!76@P|fKb{q#)mwVb-LT1iFasZb*A{9z2r)6L{RP_R~F870cC`+XemHzF|u?3$O9 zcwO`g9*@Wx7M&c;`HWLNcu+0=M=L8zU|EVyh-wkl;4>O=epff&v z;QtX1xA3ca2WF-pY`z~LKhTCg@F}ZPO$@5i#Ma8H6GM30W+}j3XJq=j80e5GDh&PC zYJKQ@sf-5)2c#y!*E$Q;iMl5qLZ_iGmqq>F!sH#{IQ66%&}a=NT>)$r2{^NGY#*0F z@=c6saEPy!)b}Du-tzkDVD@b!S=&}*9bujOHSmm1L1@?s8(w}Yg47=YW4K*4m-m2j zq3NQ28T!-c=qPLA;E#fWDEsidD~t7j!`gKAxwbe&s7fo4Pf1m%(DLaa6!sJNErb2* z`Z477`L>f(=x?;}!ViFjcizg%9+gc@aRm^pG+=lPTYZc9$0nh9J-oDJ450WOTb!3~ zEbB-GOawS!b5wkw=eG>V&}upqD}H^pv^;BnL+sYYA`#vEbSU&aqvyu8QD3}-be~J? z4dkB7j+cZZ2p|Y#=pKrn71%?ml&|G#m0M~#a;Mh>_++&&Fr0*I(s3y%yQ4Gb?f%3K z?K1WA0vN>HR@@TTe)J$Yx==6*(&$snnWK{H1FA09o4?ut3teBg%U!77_=U*J=Z}?~ zcZfjz9XA(jZ&)f*R^jtt6aSse_=T#U)+kx3Y%wu0Cjb^12Bw{6YGtl0z51>PXM=!K zbN1__AF|8HaGzfxkfZ$Cy{XIuJo^uzDEpSMF_`Ann41>S3=vtq4xtDN?wEibWd3i% zxIcAtZ%~714Oz)KN}*xXA}a``QvO;;-R8F&0J1@)$m>qFsR6uroZ&k|fN#v^ z3SJ-Pdmcbu10oX~EcWprLIaCXQ%47%-)cCGP_hmb>Q7&<#PlmIHwWS?5HZN&u5Tvp za5XO07Z#X+(@p+Qx+W#6+kIwc1Th;45eBtm;XYF>o*jBvZwOSG02*i^9K@{_6uJGKW7EQmvf|IEGGk&nD z@u|PC0`C|L<4q}cq1@rvlFu>_Qy*x*j@)X@!MbNkA`cdyisTDc&kte z*^e;V_kh5aEqUd$9(;?g&4g#Xza(KJumW-{$-BUb35Mz!muxB8k2SH;(Oq&#(1@g7X@ab* zN6txWa1^AWfT4YmcTX~vcheZu27n zxICN_5)&8WEC9EJ-S!ZwyZ3|AKlBnVfT_^Rc?3TEE)rbCLo_O3%lv^?8&Q{k3dn6- zCvtQPo~TPqVFvC*@Vu}$Qilm|`9K(sfX@$kDJ!MXpcWs} zxXfc&^p~h|#L8iOY%CA$akI3n?6pp-YGEJ4JFVAJ7)-YuzwL1n?^|AA6Udt|{q-Ml z)=mJYYSoNg3a&JFZ`pMR^c>7a=9fxEx(`iHwvpM($z;`zKYsk^j+*TOO;zGE20uYI zlBn>D12Q#rw?-qg*UzDoDT!B6y~a*H`T2Q4{*?(2O*X47=gblb8ev6%1?yt+~$r}scuAqKq6cc%A<1II-mmnr2yv@^Q zwicMT4A9z`*@LPaq@$;=-TZY8BcZIG0ERdiwHGY{e)S_@bUQQ~?5i)L6egPbYp^kj z5i@3tsGSudn=nN}k;49tN~6l9zcN7Rw;px z{(ubJb560{#lMQs+Nbe@77KOZe$WAFZv+YAveU+P#zCla9nnb|aN02eHx|7*tM$Xv z(AU5`U>DHo!%D;Mxi5rk>*-a7fb-hkI|YuxOFE4m^wDZZb^$D~0*%T3rB3|0J`81i z?n<0J_!xXXNO(kICnpzeD!6!b69(`pcZC-6+(dYrYkfHanRLlCi0aIbS{)ImF39yq zI!+fGCChIih&HbszTxhKTg!m1kw8R5bPdl%)6XXGL)LpXZ_!kXPr(gQRiAH<(YFay0#MN8 zUSM#>ZNElV{g++a8>L*!XyRJ#92MbqF(}3sU$Y%{Fh~UMe>|#BS64SCn!)8Wh*E5; zrBF%~0tI=xQ3fu&E)pCMFMdUsqGdy4Rin~7&?zCV6KMVW`wyg>J7jmj4h{O+)GE=B zf7$tX($R2Id~x$k6^B`@|IJE~p$DqEf<*8p*d*Yf0`l}2e-EeejA`JU&0+&r&O>y{fow;%z2eP6e zPmt1*5~-8XTUi$rEmKpDg`UtJ{48lO*p+PI#e4w^hyf?S&+h@xfP~ZVZ$b)e z^z$b+-ol}Zojl;_X!7&(|LA7PUs;P5B*WN!y1!g8VI6=C<8cS|#>_w}o(dYBT2{l* z@bGuoPkLQ%5V|{h1j^y8*U5^|UCRu|6pVwpBp+okMuRnMQEXs1zjqo*Oqs{ZQzcfp zoZw;v$S60y83jbve}}<{gkM)rPbLAUdyx197CiPkBl4nH(vnG7L2sIP*$6{8EZH+# zI6Y(I+Uw7pMg@d}69eK=vj?*MyF!w4(&-9OwKG`TukLrr;bm|8X%%`S&@r$Ee4Qec z(aFl~jP65*YpL;2+*c<&Gl+&lERBr|=9h4&aFZr z%SxQ2#=Cc(n#`HsMhS~z-$n@R_r=-=z+od14bR0!MUg?9TiJX=S2gf~19KFl2RLx@ z7~)X-1_oD=Q~l2k7zMi3wK4Dbu;`}Ei4dHmy97@A=y|{(T|VghiEtA&)Uv(9poL#q5kwrmY2s zG!aC5hCsGxg|nst<+N8aJraBn4IavrWHRze_~nX?tAhGh?XMa7it%vm7!H8g5_}>A z?vqNQfExf_M+ozbML`#gz+*`37ElQoXyOEACJl^zkmnZI0Hb-x5>r!Oe|WTAuiiug zB=SCr;52}Af-O+(_ye1I2P#1Bs>z&*nvS2H-Gv-o^UwBM6>L92zT){q^kw%Lx}G~|cL^_={x&z1cv+9QSCzAid^DmMcdh5h>|RHN6Etes(_<2-Y-Wjl0o`}+ zX!k*u;sL}KIbCp!UeAL9xWQl&OFt*#kxz#gn2!3yeEIQ=IL6Ax=7+G6s*1`udjN0# zx?g=aMx`T|)oKKWwzF2%f7Q;VF!-QZ?yq${&ZG5fgF!yzE_>LhlZ7K;|BLOS9SOf?o=lj?mo&p{ZA}A-|Z3p;kLuaW9?|ng*R;gTR0CS(66S z)!!Vea0KddcMy=GY`uGSohT{!qF+Cc3j7m9=@Kk)vXj_Lvj8n%9lf-2y4i7w9W)C; z+2`m3>VYzAMWL$7A2x}Mg7QE}EsH};e9s?Do~;B|7>EHd(g&>4%jl!%F-%b)BmEd1 zRk8tk|N8q{T|rXiBzvW)3P!+LLF{R_VbvwuEC96pZ*~grP85}tmS_kPpMOSPySMc( zT7rltJ8%Ke@gN{$U*s)pOv^8Pc7zNA&a3exr;3ifebqdA{#Z$5d{uR|GBP^)s*)I! z&n1`1NoF_*pa6O1VHHab4TGjU{TR)~rfFa(=p&|J{>OebE2MILH7mVahtXppkHx=MDzJM8> zsA(an*&k~aAPWMbWE}tGhQqWLsTIV1g8fLxMtzKlB8k_cy{RHXpQ8nAj7Mt*)!gsE zmMJL3U)A^8kB zq!m5_@+6nZ@_BfLHd4OL=mZoRgBsv*mL(DGLTC0&yR>pZlSd^p!kr`)7Xb5?ms*zo zjC!R7nMmnB5bl-(3dZ|#FgVB{EXh7?b~*?BpO&F$aD_5}mMig6d537(jo-Tycc zac5cS6KNbxZ-y3gW&}_kr}UWc8io^1Y7)g51s8bf=&T66rl4H$%LLYQuXsNop##w^ z0y{sjR6wvxoS**&U2p=<3qt_W?}G5(3=aG&W}+&bOZ)~HU`fG|{ttR*cxEjKM_M6a z(P`~x^f$p7WBvyH_v{AW_c6}iGl9L#`xA+}2pI>2z|(LtyY%l>Vy1w_iAX^L!lKewzGAh|g?gHe19 z^FJHy;orjyYGc0l_G=f0=MmVIxJ@~Mg-HFP&CF`p;|}{4aI%!+)eZsj?S)Q<3j?>R z@xesGR&-2E309nV&JP4et?rTa*Ts=w=qjYn0mplK2LVi_^v5t~D#e1p90udO?aG2l z0QeyQIna;6oP3Ml$i?>O=Rqha^sI=@WL_=#4PL_KKR*F3n8Ecwh=!ti54HSAB8fWk%ZH{kUjH-J;99NtsyXjrAz07U0bf`LvY7W@hm6Af3a9$uYCxnxvUn7@!DeAuu!0{45RD>WrtEuQ!7kFU)av>= zn}E-v8 z`Gg_1;tj2v&)~{bVd*g~X~K4)NSi^XVoBt*vx!n$D1e)9ipS&YOxasZP|54iWXa35 znM*7J`K^9K+~ajJ*m*q~-dY3oI~oX+q+d3k1OBhKv+&9~>)t*k-7Q^$G$P$4pro{P zh;)a9bW4kfG=hNANOy`T(o%x7gmk0SZ{PF0f5SU#)|xfWD0iIiIcM+dx;|SibQXgQ zst>o-{^YZHG$eeDu^3~ zK1^YKF!rd8Rn&*d8UN*HOIZ+^^A+PL5765d6s3R2svEe3?}5H5Joxrr!TmoB!}vmK zj~|cj=#{4W-{o!(qUUC@kknI_huvy#0@ksAfF-lktsYUog}@6ZSFj4;Dx38*6UNa> zzeQEkeT9e<>~9n#;)nfUxT(3o)}-chU+t$8$}E2~o3>7HNHJChd8iP{$l>kzwpnb& zUlh3^9fvjp$Z%=T=>^1tbA@`b6`s6G_Uasy*oy`DC{Li@zvHpLDEr`OIF8}68p&=8 zcE2hcXXjl#u{Si_f*FguMyx5x2*V%`H#c4ws!D+8rSN7RDAiGkl=qgMuGeih#qM%) zZlyh`dFJ(-J60gzS#KCBZm@PTLhcn87CxhxXK^9YRX>)RiGhmpiSMxF*<6jwf|R5x z4UJC!$#>-a3hBSABu8L=cGH0PHoI6ol~FfM9+6RsQ|64fy2@zwaug7!VQ zKz63<9H#bsb1NS>I#}{d$QI`W{@c645#f{3 z`zHYK=;NOU6Lq-;RUIV6?uW0p1Qr-@~tLdjvQ*;yt&; z=;?0M&Qiez?*o>!=g>}(A4kzXR1@2n&^OZsZ&QmlJS8(i4y}=c>h!4{sDTJ4)XlV2 zqGY}8Xg)(l62^h7d6Lsy+9RJTsLL91t)pOB!x9^o<0=6U0|Y++V7)wL6bitpU*nn`rC8f z!{EPJm{toOhEuTRBpOv)etrv$+p27=RyB~s0|Ov=nsW`&yY{=hyp>V$Tj8BPII1#? zdkY=Xn`vf&)iTlji^_zd%o2gN1@Hto}0`SaVMd(0?A4;avCj#BV2@GC| zaumeVBHv;sQP6$xjl&P1k3EaqM9-1NOk*0S^<;W;VaojN2O*MYA--LP* zOusgqDO5@O&D*-)T>v4>;pNtGnYp*vR(Ma_VPjl{Z~G~$hyi4-m?Q4a(P)3vp#URA z`z0htX^zG!YQwSi$m zPmMbUWqPIEZ;|dl?SQob!NjKuOryRUXX+%7^0%*RMkLax!nzi@I60X?J|Rp(QDh!G5BoZ{V`R48lj0lrHcYB4SIvKhwKc-q5v*L4_?_Iw3cSLR!f{ z2(k-`{YEJyv54jRiAQb$OljZ-91l3s7uz)zVDeFQ6U(-o|99v$9Q!PR+GZ{bArAqg z=rjN<2}N*T1m!~;T>5m{YWoUJEHF7AYNwYv#pAy(fjLAX1+m~z2=|1KNA1-OV{GBW za^6DNRrDbMO3@RECyt6bv_Q(;kH~8tR|Z`TG=p380>LcG%g<+exQMEts7U9xnyge5 z5gB>SXFHPDmbZVUb}jILJ#gb)6brl>)CIuFLelBuiVq!{2OHKWCwcD~s#qjL_BawXd2IOE4*b9DmfcjMlN`?^aw!FT!Ar*cN z4Y*BTft0~2gHuMH!+%k8w3GX|`srx?gRk%?a)Zr~Ei=g3tQkB<&&4lu9D@;yWS_ zkiSio7G8IZU>}Fa6B4^;-xB3Ed&dwYCocYBQz(SH^&#Gc;|n);;na=3-d-9`ZfP00ETvbz^EM#I6ulAnY7yI0kSImPZ`m9D zIVtqFWY`;q8&-K}QFe{AAvQQU1vY!+Ll^tmland4%$yg8UppQsD!LOH$*W9n3}&Kk z8?fKKYj8HE!Yhk`=X*39!w@2;otXvBMkNAjp=~Gi>iR_?(W?F7)8Mf9Kj0bjL8inH zNgN1*z2qPGxBr=D4tnPVzInrOQ{;>wKjfh^Gje8A|e>C7d!kwy`vg2_xr|2Tl9%z`*R`h&;i`KKwD5%jAtAxMorKw1$q=23g2H#xI}| zf_&#S=?mdH|8#&f zM}zqX+<+>sdJJvFG8w3@QZt|7vUIYp)i*-aFMp^+IMst^F0x+*^_N?O+^A&&vRn15 zh|2AEFX2F0C^K!#nC0U>4!?vRl{vIBvwOO`Pe=SM3!;!(rj-G?_3R9Xw{;{+<+i`f z*H3#LP`&gbGGn`y>|Y&BMndu4Fn2a4(uf|i%VD_Sp}9&>+n86#0o?nLF6oe1-IYF6?!hD^CZlD;&^W%_r5bSwV?4Xag0Kl1RUs7a2tLZ zyGlv%Eq6!YdcTK2gCFj9GVco9S`@Cg_3%X;oDkap)YkeGIaPZ{U)ge>0+-3{KQm5b zf>%AnA#Q}SigpUS>Cu`3Z6h)aN++=Yoot=)2QjR8Su-nPVy!FqaIWByDm;5uqySMU z#3j=L{Gz0S&d86yyWUsif#Y!lJp0ZC-*mT8?hIJ-GKs8^_uDO%nq)asLI>rDH6><( z7T7hpg3fuyWY`hrWA7hYZ6Yat_Vc3Qfv-3lz<+1jzhC0_ zO~lh+5fChxnVROPoY}=Vd*#xQR(V2UFE8LcCE%{+G}#B*8&-*tY}C2@xO?v~WL+)l zdcf!s)s?QBBbHQMXqNSS0pJR^lQ`f(YyS9VyDAd7ZwHFhx~Ry=3fbR&R)>kuAG!i& zi&7Y_i??Jnog6z4xQE|(*LC&p{9_VDgoojd;26}-BTJYYEdInBGAf(kPqXovPoKh5 zR%vATrd2wz3P`(jH22oWw+6n_|I8(WO&=|@ag)`rGN}H|r`xxmzRAqISU5a%Us2Q3 z`(*J2KqB-Zl8JNmXss10=$bxqjzY85kN{>2^B{NDyvAJivmG+VwxNL@;;DbK@{iZv zom-Jl7csH0J{mWB+Rdx=(@*mJK(IBwISR?gRyE7aI{3sY;`cdb139uMiXiB%vqGfp zwxYXrHyruDAvA}>U1w28!K-JZ%tHJsIr)H=k>%|l((k%V^Qi8>A@*iI^(*f~xX{kY z#8D4mZAhW5ydaA?YO;4N5-pH-%Mc36Gw{-JXjXV?mrqG+t2BT3{zqP3os$-T2cw+${_wj&Da*)Hz=4K{;1I2xaOJK(!|)) z)+^_iU{IGtbsB+ZyVHfm`++rQSrk5<#QEmdT<&Lt)hJ>l6uUIiCEzkF3}j%2lmli) z7E)Xq1mXrYwlj%}8nyv}CS&}^sEW=!$8*562?8BT-`FwgAwiTcoO}0qrKGMYYh}8B zymuhw>W5Cw;W4bSTvYyctqj~8WXHXJ=#550>-FQK2)V}#&~UP2uqi81kigb$UWSf3 zGpB&SF;ZC@kZx4Qwba3fAO2Dz>G@GPs({}!@4b1aor%(^4%zF*^x2b--yo*b3iTk5 zEjDgsSyQT2B7+|Y@+!7<91u>u@9vl}^{i$GT;c|t{iGJUcb17dGojN;Dj)R^qGM-w zh_P@L=Z@AQ54~z;Uf}wjbO|8^k|KX27rOHtMDSiIBRW^ei?j!4&=&z zCLkyuMs1XXrMOL(gRFMB+X;C$2pta;q~?5xMi7xbNY6h)FoU9^cd@}NDJ3sU&3mGGN1MA=bd3Y z>#ic2*K}xoVvGu|2g{#%Am_~c?h*OHA~RApGkI(ZTq<56DN5hDb8EaO+F?-(1-4aQ za6ec4F2>(KjKj^#W8~nlH-lh;(Cdz=v8NX)DJelcJuljrf}*<2T_M5p2YLP#7I7Y5 zg{np zqf2HIr03g(dygTrw;dyh~9dMOcqEX~EItG;G=c9<(VM4F@jr;YbuTRWi zM9ZHmAf!~>7hKQ;%o?2UM6TWPfMg-fWk8Nrk)Up#dJh#hj$!5?Ic}xw=NQg;YFWZn z#YImIU=%2LdDUf*x4e_WMi5_o{ME{slB1o3VmBWmpeqe$dMv~dxnf=%xb3_=JbePz zKtzeUMwlYA)!DLQB7m%-OGQns2KPF?Wh+c$_%m{Xu6<7fqySa152PBQ2Qfr6p$zlE zDoUoUKBKMn6C9IEQf|v=MR!pN(-d8S_NO;BH8rr2D3S+=81%B;j};X{IyyQVOBc`! z%KP8z0+!5l7topL^K428J?sjm1A{MC<`4CibAc^7qY52@cwb~c(Bdk zeQ$e<`=1;6_?(N{I%Au-?+44m%ZMh-a!P%q8>PXBz}6H4feNmu+adCm@KY)~`EWNF zsz+@d9U-UU=2Ko(MR>M`j(C;iu@STqko^d<70B?gU4##YKMt6)2?+_$hIi%QrBXJ+ zwd<1^;6sLhIgE2`E#w4nnq>^#fs#_qzkd6=e8i`hkq@3fac=V1;pW5t5JYIZ zcj3TuYlhZA@ngS`SQT93Cr@r?=6@d=n*Z%7MlgmPNk@YC%7XEUu_KgfYHDs4DY_X$ zBBmo^o0-D8ToWwb)`SyMs%|?}_ynQJCT)G*^bFl)RD#CyT|!Lt5N2sWS8?$JLnu9; zG+WaD&FI$21^7h5T^n_gWC;`H8rWhc>;*~Fw?j*IK_K_6MI}wUjX=IA<0OIO#_apE zquc-H-;&Gx_aG1f7{ddk=lL@mz9=*fp8KwFVp%JVL7%QpsZfh{jogoD7&QT1jhcBr zMOy5+sc{e(P&*Ksc*{}*SQ)Z#qE)E4V=)=Av(?Y;oxK0Eso$!JE`sig+Zce$hl?uq z__bt0CXu)`XhQ)WYi9ia}Dv|(usOslYFqQCtM}CGpGw}!xWr<#j8{b6a_NZvy!@}bTdwRDrnzx50XPv-uKqz! z76PtZY;UEHUW2hBT8E4|1fd z`%68~!S5w&X~|TP%D{t)p7NG3f6TsaU{fyyU3JihMe@rV#et`vtiqG#K+cowQkgOL z8$spI`{WHfyaf#-iv|m^?z?Tt+#FmyJQGFC$6=^uhK4Vdd~Weo1fKUgH^^sv5)&2m zjW?}~3W;Jz8SVs6NJy8t2m>KzU`a)V>1}Fie-tgDw?)SWesT9Gak~-0i3l4)TG}g4 z36fr`LkfpF!#-O9XVMr23h=2=o^{+3;yH5$$z$3p3L5j}@SCD+b)xJZ5#(v!wcl;t zsz}TWb%*->f*-DL!Be**-F%k!F!&ifA}zw9V`Oj+;d41*`Y(v?A8G98q%N5`0{@q= zc&<*&z=@4LP1m`}gytrgRP-9{)~x^*QtYnAkkHQp>6zmx;6Y&$x?lc`43YQ)mPbU; z7lFcyaVytd%BHP~7WDq#?r%}$LUTtm-RU(2Ck=FTYQa*R!@{xWPi#Yc%O`8Z}~mAxMd9HeM$jH%~aMX=wP3?h8n2 zd${?JX{MNow5@qwg&1&o&wEXo@P=gpZ=2`i!}U|7a)WB7lBz0KLBYr}xz2$*3AZ>0 zKmtF3Vk#Qxi5aV4->Eny$Yhp=t`TWlSvY!Wz$Vg{f@ZFj%rrQhlJXS@6B!?W)Cq-e zJg(Aw-GGHrm-V**O5dKj<#udvRq<<`VTXBP?Z)A3*Q>&5UCtD&}Z^ldZc73o$}$ zz>sOz|1;g$hR^<;PSTGnq7KWTSmuHMz^D;wzdOjxE+lftI(_=O6~OjB{Pp8&&(zl>l@rheJ`O+w$eN9Xg=U2l zv(?$KNvvGY!2~?X;v8$hlR>BsDAv0fZpEXvSrD^(`*rFbP)Is!pe6|)Tu_CuZbNN1 zmUe2xn~WwP2$?W)&(H`3VBKN^w<{O;FZ1%;xOdlnn_)QQ{{8#+dADBN)(iCQo7p;t zv|pChS#r$JRr&@W3VQ|F^LLSUb`$pwj2wr!f>$vc>}eLxdCLkN2Uz&O9s>VAU>NVN z9*-`T?wd&-ljhg*x}1)EN`Dtzk0+5nZ2@Z`2$QjkVVv4&LmWXgNiyv_2+|(CU#q|j ziJ|{OFcKr|2;NIci97U@drthWZs}AAOLbY`6@$}H#YOSjxa1HN4%M&y{nMdaZo;O5 z4~He^Obrb)deb)XI>qiF^aE-J^OAEgX`I^&P?f?pPl{?aS~n$H1Nq~l4}Z3P)@@0^ znaOGQGB|4vESXz*_J_|Bk&z>~NU<$Y&z+#f;>EA|)l`4Av*JmHYEcO^hw+R2=chlM zE9N3gw7><<9zsVABWGVuMz;LX72auzKyH^D`T!z9#0_W+?YGJWtfp2z!C@P3gi+yO zS<}Z2<>x+LS&NC-Sq)F2-1hlq^&&kq8pyg9thZy+5WYQ zm!-#{FjaggaF5_=)jFrfp9>Fqu%1tp@#Ji4$_2ev0(GWokwFceX|uB9S`1A6|8rW1 z+%$0`C$%NH(!O$j3@sBWTO(Qv6BCn6c$#S6mNdB=emCdS0ZWVn3R$)atB~%<;w=X- zTs8wVMo#w*Ra3{e1;(_2cORZ&{n}gmrk($U8KH_XIjd2{&;8au0x(AhvD_<@Rbz(e zFL5+4)POMXJB~)tJ%~fBK5pSDzb;hzDT2-wIn%ZOa5*@20itIrH=KaVOtHWDh=&`i zuN+9Id!s2-l)^gOSY7>UITC-7;SQ5?vwiUIzI$wR^t3y)=(yU4WRl2{i}^bHBf`TS zy8;7(-sFcVBeXwgY`mU(S~u^7tVA_RuJi_mb+loXC!$DZtl`3??D_Hb%n_dboXC?V zDj^fxVmwv&2~tqoM$XGKxGcH4Cpd$e{7g=nxC5J7*3h1XA_WR`n$~BOiy`C>NAUK` z+!4kyf#$om(aHC>1wo(X6kS&%+q*#o@{RywH;Pqygy~H=X9ynFY5_Fg(oW>o{ z{uD?y__&~pqXH^yZMYw1t6@G4rD?5;A=Ao72??Iq1>$>vA|L{7U^2&fL4m9^j)M$L zmG}xH5B5M^n|kw7PSJWP^%>@VDxu|F4vq{uQc}<4d5`iAQL4GDqP9B?8tMGjcfEM_ z7qxmjgu{ByKst2HhEJgVv*i-&11pu>)!JM-hiM%SGOqWP@c({us&%WJ(&o>bMvRHD zzb&V%2|S7-xTs`1SG>MtuwN@VK7}^Pl^fsP{q&~cT{RJA3>$X1*n5{v*qdmS8qfH< zY$l*DG~r-ly92Miy$RHXp{7o?Ic4)K8JP|W=z$!C01I(&js?bncyYVEIwEs`)%F#T z&Y}pQD9zQhC4!TqTP`ag30}G`g!o-KYP%X|6rdtOn>FttnMNG>VdrCU&L6n~^xTpk z1y&)n1aLY%1e)z%8%IYj|2d*KMrEV1?)T)IbFx34HOvOl@|hzw%*X6BBz1 znq$yM^a~0KnpPC2zb*(-lo}pR z_zPO>dywSRy`@n-`M3F9!J*6QDuII&1TGz*owrv|{0^1h%$?^vf$sTax@jeaYNk+m z5JZ!|C>A7Nyuh1VDwVp%%`=Eel{SVY^<{Zw#mun%g=kC1pqf9f5zT&UrrCTuTkqgV z6xR~ACW2ij#m>&|0(z_75i61Eu_8Uccb*RI4rk4g)^CeNf7l9O4_KJ14(1dRs^^DJ>Jr#YMpHa?QOZxqMYNLGgK;K* zene$1NP8;2aFiiF6sa8T@v&P59$Dbbx zMa%09DtgWHc^lV$Pb4?bzR$_&5FjVlJr08jWYNnUkn^}++_P`Vsg+c?2q!cG8_+o{ zs^L84NVX_34%DcIII8l0j{h#;1NDc4Y%PIC)E%jkUyyLf-HKV8Px(iZ$;ql4;87t_ ziK09scZdB20WAB|Yx7HUuAT+EhyT!;`dEskIl;r~2TYhtMCum$=8GDQe(E#L1B-&0 zBXx6IReEpJ(|z+`iMgC?jH0r75z-vU+C6@Gf}Ma~`nD4%SNyrNH(aynfC_VSmxZO* zhAA5QsR*pWFnms|2u;R&s466FjyQjfQ}zqK_*?S@sD)VI?OFmS& z>;C)OC0J3K7GN>66%`k2z}<)!2QAbWnxV9%6aPZM3)S6xwg>`3Ez&hNT+gV=^wk0F zA#f?h$a2h2y%=GxY#08;=Tw{7`C~1bdDtPyKO|r@6|aTHr3W}njoi|Y*7EowGWQ)H zr4$imPgU~`713<&b?!|*4)cx~18a53A29ew!9^@l0(!$uRnDJVoPz902lmIG5srrB z1D5B~Qk0za`Ix;G7)6 z(Lv93_wG3pBjeJxwc$dS9;#6HN*rBV))duE7GpujfLP+?F|+ZaInYq9$6#}RF6Q+| z0KsB-7}9hbGZIgVLhZNk1G`0Na1tXNEX$Bg*tDCOwOSrzahY3aLpDlfT!J?W1H=MDs(9mnkZA_=-MY4AcJh2;;rhgRq zVSDv#`6bl~-P;=Y@yF%AoTU|wf=zbaHLY|#B5+#A1~`J@T!@5S3J<&zLHB`}_5#c~ zIE;oX4{)UPX*VC4AHo?(g|F(@Wl;S5)gNTKN5ILu}Y5_C+W0+aO0@Y81fS@6qz67gcU7FLRMGT$$ z#mHY?zj?zCy-w>58S}QEkPkS3T7vlWPidv0eueREVyGc2$2Yg(eZ+kzDZvI>*(kh? zNx$Qb$JOIlE8CSa_74iRbOGfh?J4%_F~o9ftE&$df$I^N000j}H-nZ|z;*-+d{$4) z2v+tr%(c?e(!$npP)93~tOYH)(GH;VNZqLOJO4rKKghbUk7taSCADa?qVj&fxX6yMgLa(H1P7#KtcC}267Vo`{%IYATYgbfkc0Em{WC)I!yk$-mts^eCJ96RD+L! z$XiuXVt)Y#?J1z%uQyb9+l~>u7$ z+|t68P)OT@(bWn10?El)S;nqS)x5+peqc)SR##EkLQwI`AOUm_YYqSXeAm9oGY6c^ z^8G2>^1qXK|2#?|m!=OSv=UGyN}jD$E%}XgHu$-*udXH6!^E{|$NE8tMAKivejITu z;5RgK%UEGOEi74h#^g$7=IrUMRBxC>vCP%@ktUJ{~Pd+1qLF zQ=xz@gwBsI1dc|k!?pj-ZwPMgh%Wva9;k3Y2dAdHqai*h72=*7`^__s*V$7;32JA} zvw0>f|6;3C9u@Lo;6oZaun`7Q;cIQ?o~Bof{yp))5ttrK@#MvRE>^SQ$FqBb@e^Iu z?3g(>7iCMgC}ot_8NWcCje2=A zd=f{7hN+`yY;(s6E_&p^Ae7WRA9ch(|1PE3RoFMwRX&xjS|ksLFuksh&gvL9X4aA- z+@Th}B6#)@P@0)-c#2)4Uy!?H;kSKZMRnw(9Qx^4zXy%g3^nPJZZn*aB%?5u-?EQR z!5z`6@=<4Zky^;ibGyQOGAEcytj#p&J(RpSi&a(?7!C}gyUM{mzFtXvFxM*wSiFfx zlhYZjDmX0}xU}0800vZoX)@2;3b!}DGU;zbWxj^O1ax%wC}a$8MDc@mx1nh<@sP4lKz?)Qh}*oQ6>LZZZ0cG4 z!NaO5?a4@yiu{fW(%>??fFvA!YnwgySQQqJ%Z7oH25XA0*UrxqZn@}>g|(IPNz{?# zCeh|Ar|=EY!If9++bq{MLFA5mWJ2~TI39k0%9s1zpK``DNF*GEfJY6()@_D5H-+3B4?TB zTWP>`dWIEH_i(prc4qwLM4J)NW=;z*GzY_XWhi-X1c^SGx@E*t?)ZpFAKP?;dy9KYfD5a%;PSfMK==4d2@%VNG}hWO1Mk6=C-RP z3PSc|5lO(3eL-9i#gIjo3fD8@U#&0uc77_hAwNxp=n&KMm;5}<4XTSxqaMslbA8#@ zwZ5~1`ph&;l5Werrk*&+q35gUdPUF`7`K&{G;<_Wc~NI~pT=;e(o9+;dGXazYZ7@{ z+sHJaj(jfW_KA%bXfUy{OZ>r-FjFCoxvWxE%%PDbQXK@0EIUK?=$x$+ut#wEgJgx^ zyIhQ@Z#U9Sxg*)2B}5T@2Kqzc&iY7GON?kzTH4tFBvdU=Pf;Hvu?~}w9wKrWh|Yh+ z?n(EL)j_gs5xDNPw1I(%te+I{EJJT70SNB{CTT?j2tT&d&`NZvS>HD(_5ojcl~O{v z+yg?F?n3ML;Ir`fm|yX^_WNLAVYnRQYFw>k#KFG9C&=44A=*hNo(f;#bLi^5Em3G~Nj{Oj`KzG?6&kN5UjNQc_tu)V zjP*3)jgJFqkjYk;j!KkXO-+u&7kupU;JW+FRkoe}biBI=gBFxrKEczeQD7&?#pTw9F+iRLE*e|9`g?~n-LKw8 zM`M1V>BEie3{WAt`8ymW=6jLOj9kyXSkba;pABApw0g9~NaraH`y}?VF9^&Ry>#FU za!IV)jKLX$=_UBi)rwIG z3xvU<{#~?Zx+pw_(7F1Te9M>23iy|=D`iF-TixQCwM0k~4s#t$QzaI;5pfvIGRSq$o0Y@?{#DbvI zA4!W`6D3kMLH}5-b7Qc^!41M*ApvexKNHZ0T2fO8+hzcSsXw+JHk#RabtJ<@KIJsu zD*9Q!{=4~u?3*t2U2vKAeu4)~d?QlN@oFcKg<%WwekTXAVd!?7;b$=q>1MB@_0t5G}ZWT#Mx0!8C*%!p48Q7g()|%p#Q-nx-Ady}y&-()+;2$EOmc zXbf-4eV||vj#&;t)wc`i2A{)fir^1Ruxb-mKq#)}NG;I@tj}GOXs%W_abyfICBPWH zMwj-#Ma+^PyGozn3bIr?6DzPjK5x5TH1k$G)CKtJJi^uU>CGwRExa@Gi}QvBhfYM=iAGJdp13>)pI0n z2f&%Gb3U3$mU@1Ch2c$j1$l~}&}2M)zlDGPLXu#1?l}za`U?xoX85JNL=T*2Uvjbz zj>k3FAYt33zK%A8%*&8A@W7?*M9;-_A8DIUh-yq9ls@G6X@kT$#s4 ze9CvQ(_&*osnd-NvRqhTys27DaFCuiiDHJGauYuawLq9a6Q(1%d*?_8nK}z)_Pl%b z>YXlIw0>Vl9soL?Xlq-A$3-?wE{$~lC4xXn)foqB`xlReR(x$EBkEX>?<;7KDd*e0 z?Q=cMOr!Nt2Z~Rgy8?_qkPk#`7cZlMqpS>B6sFBYk)}2b&I^#Qe*)MvmbzV12CNyJ zM-Z3{*VEPgJeu~n!Ho%=U3TxQu1xD}q;_4AI2PbC)z_CV{u<^1e-i_82eEKw| zV`i2}l!bwCeLcAaxdrl+o+YF&Z3%=-N4*P*6|GDD5b3MJ=WcXeLZK2_ot&<mJxo1$kZ6SVVnv6f3#?2tFDL&8B7xCC(bVbST10a`J;klsQDaIre>5HCDnesw zf-a95#o(;Kn)0sugaNwQ%+D4=eU1_fLgWdBL$N7jTQT$=GeuJ)5>*6w!9=~{=4k6m zU`Q9KHROH~J4}XEEAaLlK9?$jU6Biv-NRF%I0G(nIO=G^>7fJC=1%ef^9nQe^D>M@ znAkV2p7}Io0hlIc78bOra0yJB!&u2T0jQA{Xo>~Z#O?Z}6Q=1Hh($0mq8YttE(0_R zUb$=>U{6fTw*11kdBx~lR8^Ivn+Jt}E^7)_J6S=>dH|{oT=spB=nxAI8aZAdZee@~ z>FMtp0G*}=MnaD8V;>O0uTlWCAuA%fd2k>&42I1N`yJ7_5}sp}-J=$9H7}U3mqc4A zIQw(84q>N8#APtgkw_T)yid|8$jE4G47e^fOkF&AS}|!vW+9UWe&;Xac_By4a5w#f zhu#zTk6F=mJ}ea`#YWmvX}_lHbZ)5O--8FbQ0|?`*C71p=M3RL5LrRt>!Sk!c$=6_ z#-_kX3Jlq+e)+w$D&m1+ccE)c0y2y@LC56 zVCadl8ZdO2_K7LG{u1EQXfT4~1%Vvx;^42;(|ie3V)dyXKXgWS3=Agz+-}0*HEqSb zsKnnm@xx$5JIQ<=Y$F3Vu;beEaWUdGyWTJxHRDi$( zRI(Ru^N;gzMbd$n+Jwf8IjjYGw%B$!p#yM}i<3A|AH;gdEV|J$kYe*r!no+*n)RLg zON2Mbj|y&qrQE(y>!bbBrBc>hPWCiRNP@Sl1!Wag;@1N%on&l5Dm>E6?{aJ!i)8EM z`9kG-{xTfa!^Wq7?5NJVGf74g7 zzT9%j*<YH=054Qfv=2{B)w54!By#rk~z zuLK6A0ElS<07a&rrf{ME6O2A|V@`t~WezdO3$QB$g?z>Tc;98=bq6Yc6GzQ{`ugtrjJKjV(gkL^?F2~2h5>?`SIr{a85b|1=2y+j-LyfD4EJ)+85hHA) zHu8Ck*nj2bJQXIBJ|2;`s`eXek(or%0w044Fjj0tFwp2!GyvLP!fjN~(tUs5+mE;t zbLfV_({%zM3d7o3TISM(mvMAunR%bAAtI$t4=!b!4NG9;JOp#yZXI2NLcD+KIVCfce2pWkCa&qHJW>=B)O2 z?_xTB8yI-tiQmh{%6k3XX=(q|;NWZvz!*F0Dz|ZSn}~Y7g4cCmxdkt`fs(2Z3hGEO zd4Y(qq6_#)*=h}U{9#B|94+~&Px|t*r46j?jfP*33f!Ay`^y>i1Ve3M-}7pY1O?4ZUETwCz= z5)!F}dGMabJH@<+g>MCrW04-qIYQ~Gb>@pA5pijqWc)}^#Nhkx2z%*7a}&*GVMIYB z#{$Z|FY%b&oF9>C0}rb50GN6ZA**6HtA^pq&re`4&^w|bf$P)McVrk+-2$$2RTpUo zmSF0Up-f@kdyYgYnTtoE+vsK z)KObHvQ}MRxyj!EfrIN3ZfCkbJRa;#L?N>gv_q=6&t4#-&DakQFKe;rJ|!UicT@FE zH@E()=}L&lFcGLhN#G~zXDteMtRcnzfghYXS!qU(`jM9hFYY|BD+VTlRf14^#jZSM zRRZJQJc8q>1ns5D_N?sb4#=iCieVVUi5g&)@!9Z>MQByWjTd^+pG?5wqeIk>57~r@ zSdC0ewC2D*FAUPBgub$_rEcM3vvvv}1{K_J4_Dd)E<)R`REbvIbyHPAt0@}C!Z*Tj z4K-WHK&_Q#20o2nD2IxwT0Ng_?cn9Vt;#-VmJ7OyaC3Vpv1zylcyt1m8;&PEo%e4u zlwM%)WTR7eBIY0iY00^ZmhG?Z%SU;97%xc>Yy;w~2Bpt2AHTkl#Bc-}N2ng^@;Qa^ zY~$^CMSke263qujV!A(QR9`S(g?(0ch~4$rveHzC800-E(m#uDkZ$gZg$QnfPFkNj z@SCj(ZvY;8eSTLj814c$_xRTXzfGn4No6ZaH&)@;_zL&Q%uR;AZ{Iqog#Vi&tp;Wv z8dXOwj)Po1)2h!a+Z~XHk^3>cV$=z0BDM-4W4g7W=>c913G$!d8 zFFFAoSX1*0KFO^_)_(WAla-B-+iA1skj1|?B^<%UWDAS&Ap^Cl3VPqqc0W%DU%j$8COs_{}xJCxP*ouQu^@Z!^Wz~9OOg5XW$Yux8PWexQB%eOnz zXF$Pv1J@GLyrFE8wP4dq7aB0~v&^$PXQ4(fWHGXR@9+09;z`3WCN0+C$c9Guk?*I@ z7+P~pCEmM0;wpuKJOZdn(3~r|K5gj0i?Dx)cA!LQkd`SObo~L^%~ph?X#KrXQlMJj zt9^?$HH$(cr@Rm89Tiv7c@4bIQiyb>$UX=jqFtU^x5a|uEM)>>vZ?&( z-(8ISfYGP~LQXT>H-8^Wi-#ip2iXCh+WEVJ_0_mcz&cUGo+pjTxGjbB-IyWd$AK)o zo~v?IU@{*Rzs7q(UNCCl3$xB8dem&WSYqNpCr)krjSzG(}2t`{9ZtMLZJ131mBPkyt;CYq^TWOkCI4g97cCMKRl zJQ}o%HP+9bS?YVPzVry@?EVKAlt?J;uQ*`V4!0*hq5X8UUa2Isz(*(=Kg4IQV-mZ% z+KGSQx;$S=i~^knHu_J#qIJCL$3^}q!$u!%myKE< z=SsCc{;}2fHcQ{ww(Ggnc@Me^;FUVmvc*o2MSf)+8Ms+83wux@XQS4Go7YAzPUjxEphf{U>UNMK626isAX^XB{U^ zS51$tKu&91^$KoIn0S7a@V!b0;-kQ*fZVkX9K}KSZ?IdCL2K1-2ei-4avgW>V>IGp z1ES|;8soX2VPq9GG^YPX7?xj~z;bbUww-}X7j(JM`B+!&CYMg)o5`gpin>V{T#)ni zN`o?B^aH7!GA2b~m*IV}=a}xODXq1&I*tvqg{@OhtNJFOteb*k=5E75|t2I z)2+_vk(l|~hg7(O@LJq~V|=b)X}N6ls1QNm`ve}qZRl0l;GxTh>6KcuZT@DJrsNZO zO{P8uXZi%>YKVW{1%U)eppJ05^haq(9Q3}2Dw!Ov>DN0|XQ7dJ2emaYQL?kiQl8b8 zN(QqUki9MPk>F4hgv66oykgBL5a98EAlKt4lnb5W_$1$8x4@=0??uY)4jA17xZ7s) zI>SWJ6$AX*0&8?QuBsv1wChmWCIo)w3a^@7X066t86igl(XdAAd|s1g9Roc*){+k& z79}*{qvaCuJ$(8!AfU|+E~$2z!Y&CgllgVYr%zfn&-YA$J7Xj0nM-sb1x>Uv2x#5Q z9MlD)!Y_JaV)qKrAur&J3ItCp8z(2{-ykZchZc_?C)bpfeeY@`2=$?&Y~i~GnFzb@ zR`VfFsjcKSOWJ^9l1A6<+{CmC2 zf;!+@ryqE|2v=6|{AYd{HYyGt(i-2RXWnyg{295Hi<;NWEY{+H`!JLN@)rNU|7L0w ayTK48KYCGqTDOb@e?5Glu23Oo9{PWJ24)oi diff --git a/plugins/lancedb/skills/lancedb/SKILL.md b/plugins/lancedb/skills/lancedb/SKILL.md deleted file mode 100644 index 47537f31a..000000000 --- a/plugins/lancedb/skills/lancedb/SKILL.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -name: lancedb -description: Use when writing, reviewing, debugging, or documenting LanceDB pipelines in Python or TypeScript, especially code that should work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables. Helps avoid non-portable full-table materialization, choose idiomatic query/search patterns, apply LanceDB performance defaults for ingestion, indexing, filtering, and diagnostics, and resolve connections to the remote server for Enterprise-only operations such as jobs. ---- - -# Building LanceDB Pipelines - -Use this skill to produce LanceDB pipelines that are portable between local and remote tables (for LanceDB Enterprise/Cloud) and idiomatic for the selected SDK. - -## LanceDB Table Modes - -LanceDB has two common execution modes: - -- **Local table**: embedded, open source, in-process LanceDB. The client opens data from a local path or object storage URI and executes queries in the application process. -- **Remote table**: LanceDB Enterprise/Cloud table opened through a `db://...` URI. The data may be very large, commonly backed by object storage, and queried through a remote service. - -Do NOT assume local-only table helpers exist on remote tables. If the user asks for LanceDB Enterprise, Cloud, `db://...`, production remote access, or a remote table, focus on the remote table path: use `search()` / `query()`, keep reads bounded with `select()` and `limit()`, and avoid table-level full materialization APIs. - -## Workflow - -1. Identify the SDK: Python, TypeScript, or both. -2. Identify the table mode: local/embedded OSS, remote Enterprise/Cloud, or portable across both. If the user says "LanceDB Enterprise", choose the remote table path. If the task involves jobs in any way (listing, inspecting, creating, or canceling jobs), it is always the remote path and requires a remote server connection — see "Connecting to the LanceDB remote server" below before doing anything else. -3. Read the matching topic reference before writing or changing code: - - Column metadata authoring (both SDKs): `references/column_metadata.md` - - Branch operations (both SDKs): `references/branch_ops.md` - - Remote server connection resolution (jobs, raw REST): `references/remote_connect.md` - - Job operations REST API (list/describe/cancel/query_events): `references/remote_jobs.md` - - There is no bundled per-language guide. For exact method names, signatures, and options, look them up in the canonical sources instead of relying on memory: - - Python: `docs/src/python/python.md` (the hand-maintained API reference) and the source under `python/python/lancedb/` when working inside the LanceDB repo; otherwise . - - TypeScript: the generated typedoc under `docs/src/js/` and the source under `nodejs/lancedb/` when working inside the LanceDB repo; otherwise . -4. Apply the SDK invariants in "Per-SDK Invariants" below. Read `column_metadata.md` when the task is documenting, tagging, classifying, or grouping table columns (field descriptions, `lancedb:tag:*` tags, logical column families). Read `branch_ops.md` when the task involves branch lifecycle (list/create/delete), writing to a non-main branch, or verifying a change stayed off main. Read `remote_connect.md` when the task involves jobs or direct REST access to an Enterprise deployment, and `remote_jobs.md` for the job REST methods themselves (list, describe, cancel, query_events). -5. For Python schemas, favor Pydantic models and validate records before writing. Use PyArrow schemas when Arrow-native, streaming, or highly dynamic data makes them materially better suited. -6. Prefer `search()` or `query()` builders with explicit `select()` and `limit()` for reads. -7. Avoid table-level full materialization in remote or portable code. This is the main local-vs-remote read pitfall. -8. After a successful embedded OSS ingestion, call `table.optimize()`. Do not call it for Enterprise/Cloud; remote maintenance is automatic. -9. For remote Enterprise/Cloud writes, never drop-then-reuse or `mode="overwrite"` the same table name — see "Enterprise: never drop-then-reuse the same table name" below. This is the main local-vs-remote write pitfall. -10. If reviewing an existing file or repo, run `scripts/check_materialization.py` on the relevant paths and inspect each finding before editing. -11. Cross-check unfamiliar or non-trivial API claims against the source tree instead of relying on memory. - -## Core Portability Rule - -Do not write code that assumes a local table API will exist on a remote table. Remote tables can be very large, so whole-table materialization helpers are intentionally unavailable or unsafe. - -This does **not** mean result conversion is forbidden. Bounded query/search result collection is normal: - -- Python: `table.search(...).select([...]).limit(10).to_pandas()` -- TypeScript: `await table.search(...).select([...]).limit(10).toArray()` - -The unsafe pattern is table-level or unbounded collection, plus local-only dataset escape hatches in remote code: - -- Python: `table.to_pandas()`, `table.to_arrow()`, `table.to_polars()`; `table.to_lance()` is local/OSS-only dataset access, not materialization -- TypeScript: `await table.toArrow()`, `await table.query().toArray()` without `limit()` - -## Per-SDK Invariants - -Python: - -- Result collectors: default to `.to_list()` (plain dicts, no extra dependency) or `.to_arrow()` (PyArrow ships with LanceDB). Use `.to_pandas()` / `.to_polars()` only when the project already declares that dependency — do not assume pandas or polars is installed. -- Plain scans differ by client: the sync client has no `.query()` method — use `table.search()` with no argument; the async client uses `await async_table.query()`. - -TypeScript: - -- Collect bounded results with `.toArray()` (objects) or `.toArrow()` (Arrow) after `select()` and `limit()`. -- For large reads, stream batches instead of collecting: `for await (const batch of table.query().where(...).select(...).limit(...)) { ... }`. - -Both SDKs: - -- Ingest in bulk or in batches of thousands of rows; never write per-row in a loop — each write creates a version and fragment, slowing ingestion and later queries. -- Build a vector index once brute-force search is too slow (rule of thumb: beyond roughly 100K vectors locally), and scalar indexes for filtered columns and merge/upsert keys. Use index defaults unless the task states recall/latency requirements. - -## Enterprise: never drop-then-reuse the same table name - -LanceDB Enterprise/Cloud splits a **control plane** (DDL: create/drop/rename) from a **data plane** (query nodes that serve reads). Query nodes cache the resolved dataset for a table name for up to `table_cache_ttl` — **default 300 seconds (5 minutes)**. After you drop or overwrite a table, the control plane updates immediately but the data plane keeps serving the *old* dataset until that cache entry expires. During the window the two planes disagree. - -The failure this causes: you `drop_table("t")` then immediately `create_table("t", ...)` (or `create_table("t", ..., mode="overwrite")`). The DDL returns success, but every query against `t` returns **`500 Internal Server Error`** (the query node resolves the stale/deleted dataset), and a fresh `describe` may still show the *old* schema/version. It looks like your write silently failed; it didn't — the name is cached. - -**`mode="overwrite"` has the same problem** — it is a drop+create of the same name under the hood. - -Rules for portable Enterprise ingestion: - -1. **Never reuse a table name you just dropped/overwrote within the cache TTL.** Do not use `mode="overwrite"` to replace an existing Enterprise table in place. -2. To (re)load data, **write to a fresh table name** (e.g. `_v2`, or a run-stamped suffix). A brand-new name has no cached data-plane entry, so writes and reads work immediately. -3. Before creating, `list_tables()` and **fail loudly if the name already exists** rather than overwriting — prompt for a new name. -4. To land on a specific final name that is currently occupied by an old table: drop the old table, **wait out the TTL (~5 min), then `rename_table(fresh_name, final_name)`**. Renaming onto a name whose old dataset is still cached hits the same race, so the wait is mandatory. `rename_table` is a supported control-plane op. -5. When you hand a table name back to a human, tell them which step still needs the propagation wait (usually: "the old `t` was dropped; run the rename in ~5 minutes"). - -This is Enterprise/Cloud-specific. Local/OSS tables have no separate data plane, so `mode="overwrite"` and immediate same-name reuse are fine there. - -## Connecting to the LanceDB remote server - -LanceDB Enterprise/Cloud deployments are served by a server implementing the lance-namespace OpenAPI spec (). Every remote (`db://...`) connection talks to such a server, and some operations exist only there. In particular, **all operations around jobs (listing, inspecting, creating, or canceling jobs) run server-side** — there is no local/OSS equivalent. Before any job work, or any direct REST call to an Enterprise deployment, read `references/remote_connect.md` to resolve the base URL, credentials, and database header and to validate the connection. Then use the four job REST methods documented in `references/remote_jobs.md` (list, describe, cancel, query_events). - -## Script - -Run the scanner when reviewing or modifying an existing codebase: - -```bash -python skills/lancedb/scripts/check_materialization.py path/to/file_or_dir -``` - -The script reports likely unsafe full-table materialization in Python and TypeScript. Treat results as review prompts, not automatic proof of a bug. diff --git a/plugins/lancedb/skills/lancedb/agents/openai.yaml b/plugins/lancedb/skills/lancedb/agents/openai.yaml deleted file mode 100644 index 5a5b2d5cd..000000000 --- a/plugins/lancedb/skills/lancedb/agents/openai.yaml +++ /dev/null @@ -1,6 +0,0 @@ -interface: - display_name: "LanceDB" - short_description: "Build LanceDB pipelines in Python and TypeScript" - default_prompt: "Use $lancedb to create a table, embed sample text, and run a vector search." - icon_small: "./assets/icon.png" - icon_large: "./assets/icon.png" diff --git a/plugins/lancedb/skills/lancedb/assets/icon.png b/plugins/lancedb/skills/lancedb/assets/icon.png deleted file mode 100644 index 94cdd637a6b890494db48c78876675f322ffdb4c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23274 zcmced^-~Ug zm-nZhzv1~|t9Ey~duMuQrmDMN6RV-7h=)yu4FCY}K7Ewe0szp!|6MOIpWX(aErcg-*$-E-^5wT z)W>7g1FX>c!b5+{Xlk}!tlKfQAp_DMea1GwyQkzfKtvG;zcYq5+_5RCfZFc)%Er9?9!@O#V)kP(?CD8>h*g9>k}yFYTkT2aUDE&mrs!EIvHQK~ zTJen@+kFsZDbXwz(ZM<>=!5Q9XwNfydk4zA_MD&V0*)u|R}AlSGoDqm zzpk;)IN(WsRn89Z#SWbVaJ;lo1;n3PkNqq@gs-B*UIBd04-NqL)V62M;KP^zt!TbH_bd32_d33^ zI>SJ8$*2P0PsQ@47W6vSP-irWSdhOP?S%rC35b>xdUt)Uy10%3(_-t8eg^i(wH5yT zR^~vq8Pc<|l+=H*518N0=9Y`HGdt-=N8MwPlyf;Duuk-Y@3X^wf_J+v+|Ts>g(4oS zGFo^;%bqR9#!X#4Xx(QH!Flx@_~NT~iUfpcw-2Q(3R43kXa8WQqH5 zQEhTw41@_0Y@|ud9H1LyKl9>K5K4bD>F;UnCY|=fu&D>*L=un@g%n-|*1E)m*pt}Y z{jOV#hFHI3Yl@|qIB@5-c}u&qoO60A4zNeNR(KJ0>p|ySUcV7H^=M{|b^K`0FI!In z>68DOp`Uk%d$G5a<$i~?fP(7;ccZ$cv%#phn*XX`CiQJIe2%9j05fz8N$k0*_X!|j z>?xV51(XQXo?TOsV!nxRnG(=FNJ!woDg0^HmUM*{kVgQc1n3oLO#Pm)e7H-|c?SIy z79bR=U4C)@P$)xds#kBgh_nTGqt%fmh$S*iUf_OheY6f!61YCLvDwlAFoPcL5Q=eX zsj{z0b3t#{mYd}6T>t@m_Y~DS+_#zKvK$0fQHCU7CSMo3y+eKyu&Rb<-j?Uk)fWH= z&AAmb#Vy>peRlf47IS#_!u`g1C7Ij&wIxZzF(%IxfE+29b{}33xbkA%I>y%f!EP zGla2!s+anXHNVs1!LR|YXm$SOtJy40T32_MPz2&L<*t12czBGxhQ`p@QL0<5U+RY; zbBf@#WC%me;~T%mx3QAms64V~b@-u)XxG%C4-OU=Sd)!<35beUWN`+0NGw1L!8qs6 zgY+P_>co`UbC?_l;DE8tTwbms!s6(8-z^*Vd@KLR*3%>~jRP?R%w!Z6isSV@;-uMC z=r#E#eeD{lHsnO=0&+k!_C1&f2!sc|T!lG|-rfv&bJlvIOoiT?Zd^b$uTmLK8^7cNz@QFXQK#oVkhx1drx@ zWED+fXMMd}Ux(L29*UFozLkggi+6eBAW|~E{~t;89hv%`>-qN+e3Qp_e10vZu(s9n zrFmFeo6c7p_S{>(v`|W)@&c;~A}EqZPdb0<#Mbnzpb)dT3dTWmE((BR7i2PRa^hp5 zUB4U%)unT%`+P;U>z}7*Z7H@Vx)@DAv2f;ED+aQS^X9)w>EhC+TpF~m7m|bKTS1Bm z6Lf(G4VJg6u?Bw>h@$53?>l^|ue%nzuO&ij8}sYFL>ZG!@Z+MQ;<^~_*0>AcPI0ZA zi-sZ8=Z9nsB!zC+F6@9Fw7QqeAc9&+;@&<+b|1*T!RYY{0H3uWgi}xg>jIOleTZ!o z$7{d-p?qjA>VU`Q^VVgg!`JYYs3=?^qw;EB^kzn=w0N)Yug+lmeENF+CCnMJTF%XJ zo75p1`$mQy2S&Jbz$>!zz%03E*|c4m95#MJ@s5$@)#2x7XXs1ME3faY@n9ex9kBe% zAqFA5Eoz#Nre64!Ujm=!DiC)QSj30l?c563R5@=0*;)@{s)elluIbPh%l-+!{r71O zdwRgijx$7o7>{-Y$Z{bPMhrb4E=zWj-D!E*9afH%d}(@n-O30$APEq zI_`eQe++4$b|3OmSBTw8t^}i}8JP!rbxJf$z%`Iub<<&8UCNq3VdIx)9CEF7i_Dbi zP0)h6MdGX>@!h;>%PD1B%#V=N1>kn>1Ag`6U@KE#@^Ex&E=7z>)VQnPFBs$0i@%+Re`?w{&NIlBXB2g=M=k#(g$9%m_MlGE|`LOrpA?!c|IEgL6EjjA91@; zEp`Y%U4X%#iW+E&fylPm&-oCbztfsXoMch3F_T5i(U=5Cy( zzjkdR$qL>lk1^;Qo4~?NX@j@CaOGJ{4yT`E_y0E|RfnNCcH^ZOakwMJj#gb)eAE@q z@64G!FyXCV~9ttrB>51a&w|r95^c2H6ng zF^-yGQjjVbFFTcL9P?lFwlXjbkfsZ)$QG;ER|!8EJ|ZX`yN7ky^-tArziEd=aEHur5SYtFA}yh)5c&f6u^K7$vVzX@E}z6 zf0KKRK5a6tPK#GO(}lTP_Nooh+5gT>1FXHG$2!p@0;XoBV@+&U1OkIHEEIPBVGsc5w$7kM0}ezRxEXOSbRx-b-RVzLZRbqMcN|xw zF1IBbh)=b}h^-531do~ZyP5E3tuSLs!(1n6j_3Bfvg!+Dx`%k8h?ICS?!~Nh;@jJ0)RjvJ7^G$h@148$w=<3;YX& z)K0bzQ8dV-lHl> zBI44X2N@E?a`lR{_yq=hV=O_Sx?4xgvLV70&X)-NF*`1#>Hzbm)e2)c~~z_ zTmmyv)1+O!;&9Or)E0zzcKn2GOJ=y*ILXvL4SReDXLT}Qox_BM-AG9oKGhtW?K6ZFG zMNK-&zR<*&s87~vN|lgHwQ!3yREZ|ZB`m}?xsr&bOIGA$M4T-RQIrb&7r^Mk8b&&W zS;1r}t>MOza&}RKg-U^#HPowxQQJQv&NmvUT-YMZZ|$%m{eH%=G=J|T&1Lk$OWVPw zuN;i-raprief_Zbf*s+pL*~Ip1?G_m)^G~dv*nL!8rLIryofeyNZK!m~)jEQ0!Yn zhjssNP)u9_Dr9Tiet*r*@HM3Xu)f@oQbJjkCyD2<^hw7;1y7IZt_SqqgnJ`ur5Bov zJ|}Yja{<|TH(u{ipehFQlZ#Kr#}4-UhL%$Q6R$C9NbqDf*M5xspLZW!l4vr_i)kpw(VJfpjRdzx1UzGsL) z)v0yP660Qsh*DS$UTy4IJv!H~=BNKXH(M0u#u%!6osIh~J-IcUdUqQDH4xj|v4(6| zu8%QSZoJ|QEa98lp4L4K`?J_fmf}N~tFN6M-l6XYR>~xRB?3@`h#iFtPyjX=)t#ir zY~ku9-KIbZyDWnot%)-07qT6M->9=SE6NoA0VWyvs(0z$?7*D1C=kErIWRYxHdQT= zF5~xT%Kc~2-EMeLuYJA$ejvcuk;``9APlnKWW}jcq3k>{@Qy+n4cI0DR@!qvsx1>{ zMz0+rVZ1;5o53XySn*z(#}H0-OwIc%_r1yEIb~S-i>mBEp=zO zEx5-m0*(c9Cq%$3+!CS$y{9H2)S4HJloTbrvWXk)G-591J?3U6`$uT{f9g)`-x{g3 zzOi(aM!Eo`?doHRx`sB*mTLhf?>D|IMCJ~q;|0(vHIai5H#unc*-BG+W77MxpN}RC z9~HA6TOpNsBvEZAMI;T-XD6;x;q2A^XE<5SEx;+(K(fj`c6>UlOT%L%1s{K+5&f+d zC7vWW&gf}>&80gI*wZ!GQ;~EVn%&xv$~cf3zswtnle**Jt}#IL?b7B^bqc$Rpk5Bx zX}J&0+OFRS;0Nft#a?qOFj=m}i(~eB$(+PlT~o;M)8czXY{~-6i$L_;H58Dg`K#y0 z9yWPlk|~K3y)3H%9og$SsW~hPeV6^DV$fYryq(Rn5u$y2z%?a-5BHWrj_sW;ptTal z;Nx%Nq3tA%w@%Fmfnf;<$n!c=CjJi4o@?=?QOKC}H(xz%s?%vPp4^W1(VV>;d6{pj zq`q8M6(8u~YHlMFTPw<^G~L_i-7BpXc7r1~_ z&g~%YYQ2V<5{BeYA+br})Kqu;IDQ|PM)`ffS7c*^H?V+}vwP;KFU~t+Lys&2MGXtc=W7FKHGZgwvOfNU*gg@*(HxG(NBO z^8ehI;vZO8-d#JMpvhHWWzd#q#GbDG-Ig#HEW|uJ6P+tHe||N9hU3$@*caK$Ou^L% zFZe{DZ6E3AL!W?=zskR7*0MfBS5fIc9L2~aCpFYh#4of>8qcg@H#_n|cvCp5@w+j z@UnJP&ayw?@4R>?7^1G;cGO=t={essp|P_HQ}{TF-*S})351Zyr5IN6Y=#n=&EFD4 zLlJgcd*3#L^>ph#e(1Zg6x!%Crf_7yM6j>M+s^X)VO}*#jh$Up_`h+sMO*myT*&3_ zJhR4~kG?C>U#{${E^bdK=~87j3Gwf0o=NL>US?|SdbsIwcfib4sgz5^?_0_2tu7TH zJDso}UG|TiexRn?^dXhFcY-V;$qXnzn4mqnW?05*pa*;1g-8-V8GeAZ{eRW z8(5;>koHTXGClM;?5$u};slFFMu?q5keFXAUR@U9Pu4PL#+N!qiJYe!G{ z7Zr|)abe~Dp6PJjZ|EoZmv)l)$iLsX-?!OzrNkRtz?9S!2on9;YF4?pKZ3nQxG#YI z?$}+Dw8lBD8f~np>|aV?g&9^#$9StV&P?gDA>UGzb~FAi*eV^5r|Y-+)2AkG@a=dT zhUG;d9zpvhfue_DE!0#5?hopzKJ5Q0VVm@iRP4nZ!hiKv3m;iH7Y-wD3sYAK(^m5U zYOZd!CHf!ohMHNl+7e2(@on=r-*E~HpfF%*fNu63rhhe0>QA#@2>0dCtCIA+ zfNYXcIRZV-U@s2YdGQcEuBm7xWZn zK_JEIq{v41uR*E!?jkJ^G7E7F(s>K9yJ{qs#B{-N!40?qy zVQCWBqY?-cw1KlJ6s1_LoN{*h(X{PitnDlY9Zd$(6Fmncs2>l5{(*tsJF=bG@SVNRZuk9gln#*F9sgW2r{4T{IGc(pSZ@ss&TXAV5K`@pv}c*c ze4fn`U$8r7QU_u8MncX2>n~w9fO&$r>39aMN{M*AmL7<>h=V^@ngTO@t@B$uEXXdv z{|EImgSS;Z?1`yB3>bOWH2g=TxxFXgVp)s4ingUCajJr%1@1T6B3vw1yxjIJt(^`4 zBwy$m{|;ONe~oMSLfA%%EHnBPhcQ%1DlL2wZ?8lj z7+4a-O#h0cp1R`~Hv}kYcyi5D3J5i7`tAdu)4`F|FxQ>llIqaHmsqVbMgUn*huVfY}qZX(hjQMdD$bu_>G`*}jp zpR$S4yN+JORfZDko8kL!c4Ou-PPW}hRFnFJ0>DKYU0G={_UFlq0d1gYA{<4vD0(UM z2?(Y5x!r+1n!)SOmR&&^d?1w973uttW8TYIFx5dWH=e~SKs=g3uH~Kre}HhuHv)}g z$*Zu%pBmS)2++UD2;?uU%D%s+JyX+NvZ0PV&~) zqS#XnUEq#P=FSAXbLL2of32CU_I}z!>}4P3A(^vz@<{fOMModTV{7 z10mOLc+xf2b2n32D~9dP`rcl8z-dCsLO)AwLzm42Kpwiql8UBM(uLu!D>CYVr?dM- zl*W(#B<-oh4+YIch`W`q8(VV@o?jIwdu*cICN00&ulZje2;0w$lwFB# zcxZ65IS908AEFAN>>~PC$8PT6YRgWHU+BESh;w#fc=U2|dNO-+1IMBiX{GLN^%x%V z?u~DDrV0Y&W;GDV)LOSpfZMltzW5DmUDiQa^QBRyoh3>-0(;|bpI+iU_r?dKx{#ml z@d^7L;C0s22XYzVZF%w;oW<0FalAeH)QVLx6UTE}bGfi!UiSQlulQK%-N*8j+rw+x zH$KLT*ZYC^ub}aoLN}nJh0_?b0!40LZ&5q}-`c>ch~7X;T0@Zz}X8PL(Oj3OZ%~ zkgjH{Q&G=Xj*rcC)$Kf@+N{U|x=PZjp!ruHLsre+o?q7w!DL^kXRcuxX6O6NzO>#- zevaE24h=vxV!JxCf%UiIj-{r}n?-ZO zilxmzuW%RMw@LnJe8$N@OsDo*XwKkzatwn+2dfUY8BZf*JklA-BIJI`2XM9XVkkec zV+;(&8q4wb^YN?C-<*dzS!7o$I14eNL&3doF=knvv;nr7%{=Dyz{ z`_-QviR}JbXM&QHSY;R5dlOr+A#00AhDP&gN6%K*%uqL$`gD>`e3)aF51RPs!e1jT zx3jCjqnaYK2CG<>3ZV0Fr;fVNaQ?Jk2Oeo)aXpiYJZ>~cWx78LD)l|L1?>!%&HnNz zU|KF&ZLuwPfD^WrH~*yMVdPb!4|v`cTaf(djlgT9$Abb+T+B!Xm!DAzu*;8n+o7%Q z%3#i&xvDGl-RuUl{lYd+$x?!121c*F7{GwSnu2R*h^82KbItors&Fz$YTvM7>pKKR zV?t~+4Jo7Z0(iaohkXoCiP8Sf5u-n+MBbnFK5S(Zv5{u?orZ41MJ78E_#9H-Ep$)bJ8-6% z%wZQ(r>?N()>ZSJjtDI#(-^TcE|_%o%`5FD?((d+z|F0<9!pgW!ORPsqaG{i=i0&( zCZ~V(qQUEW1VEb52RshRHg(%;iB;8NJk-8~M zE43?ML6bsup7s&B@5E_2m%iaas9QNW`N|8r1aLwoQ{;ZwXy_9B7T)h)H8^hjY%O_u zbK!eA{E~*??{M8G;!}GE^!~(`q z3O{I)T!u)s%`r}#ULJ*28LU_^tGU-^mlFi7mjd~^JSlxs^240>$8PFCYWS+Fbe1xS z^C4$CzqP7rOv<<_irGkVK^WbnyZdwfRON(pLY>$nREDh091pj@cN!&rh<;l(7$n_1 zIM<@`=V$HTckk`5&m|%b>CRkCY3xry*d4E^q&Gl+8mrb~Te}WgT%)aqr+E{_U6zb& z(zE~6FmM8{I{M~ygCaF$LfpHv1)E&b`!b&e9l26 z6uA!M{V6nmar5ll&1m3u#&Z3r$m@_*NoB0M@6)@8^ox=Ch<7)S0j?RD>%O&#JaidP zG3$gX^c7XxPSE661<^|dv-d%D_LmdUxVY+JaF1ZePJ5Tanc|0z$B}&+=WGqF=RxB? zxyw59Cp3U^U(;NNZjxJOb^hfM735117CKil! z@e2>e?{7F>JFO=zC}F_kc?N;~l^b431g_|HxnHBV4WrtGlU%tR@YKRi7aHVky`L+> zNO@nCBzRumP^iH^eRKI#Qo_CI87P;ojSBz6^n*nwtA+3yV6PH)zZ4dwA3Shnu%DDM zyTVhvdR7FX$8ePi_f$SxU;o4{S5z9eVau^LcUuZVK9xY(ZfLe)t|Tv zGRPmhMl*)AaQnzF>(<;GRw|L3bliwS1Mh8mr1+qG^EIVGR0Q6IN}MtL)wd*nzmM_0 z=t!P-G!uN0tQh(;NLcQY}+_t1c z;|h$soPr#$;QUELxlOE=%(EA#BPPdT-70P5j5f4+a3`@o^XB)-dU_G^%~nlr+_)|! zZ}t!ZWC$x=HTLV|V&txgi4dO5OXYMBrL4^5AZtxew-D4ec8yS*Dq55qul782X?13a z$m>Jwbftkxitts2D)z~)`F*xnDi}~R=@#oNQle`ajCLPD=lISpg)ESA1NJ6#iIk#Z z0oMizm8;J0=E7=ye*Hf5MrzS<%Hw@q+qR-5W-l+l>;WOJ&tPL&_V41%eTeH;&aV*F z;43ZQP6MNTP2p$kY4%K&w(5-Y%QGvU&g{v25DU5a`2%Imr~YK;mt2nN8!u{A2X;ye zSNVYk*aOUQC$L_^zJJ*2>2bpI#zX7JPP2q_7SAq4iK!WcSZ(x0mHrEeZP zFd|<>8(LVgNEqTN|B2AcY%r0K@A*Jj^Q7H@qAfL-Zn7VZ;G+BW)4bpd;{xYTodqi> zJAtoa3JaIY=QNw>JeeE#SS9jrjppk-AL6;$0`fJ-tM{n0#OTqFN!@ct*}s!b z%?rsJZ8yRbA(fKvo3f;&YsWQ$YFV+TOP9F~n~j&l2eaqX#|#+*A9*z z@7{aNTRs}+NgJ!E&lu6_3`yc7*&t$6kx*N3~ z&p59Cl$F*wIC;ut2M|dcw>8P8&hfMBfdBcC#OI?4&f|oQthxLyCoG{S;KJ)PF@+T6 z(u!!ie0fE<$y;Sw!HcbS4*e|~SH0TYa?U`Y7Bf<}r)f=+`@GUY92)~CDMpQDvtK(2 zpQvT$^(K8ZD?IH&JgOXhKve@+Q-avE*VQWR8<4pk@8Gi6$>_h3g@p`XPN|;3Cc_1{ zMT?{I6i>lury4JNlw;a?dOp#vcHKt^j1?FKKDgFi2Q?+#T^oLgBJ2$Tzj$C`0k4#A zZ#ouIIX%=aG_rVgCFG3XrO7gcFdbK);d$bW-bf^$LrDG*H|gaCgzO4xehY6Dzm9Ye z#{rnkF%TRNSDI>_O4B|ECgdr6z7Fn*i#7{XjS=Fb>+Q+@3F~w}T-FdQ8ZhM~0DzOt ze>K$l2KJbj>7mi1^gSKa)Q?C_^54aAaCocxs-`B^-o0BjG3(}vGe^?<1F`?a3;iLW zAD^gn(jMo>g6mqFE&g2p_3uvo7?Z~syG94Yeq&6W4) zV3;T;Km47ooUxRSrv(72y+kx07w%jDO!{l@o}7qMG|1z0g#dhwU?VYOpprq<{*5)aqnV5NH zc9+M#BX5xZOOhvK?00R~p4+A|(K9y6B28egz!UDhU&T0g7 zb^m&ak~f-&bu*3Q;N7R2E4QU3*50TySST~A>4?4R&DRE}RFxc|l?k4<=W3cI0xduC zD2=AeNl)hAOG#o?2i;tNugV(9HB&pT%q%W#OO-bd3Td2R*OaWIKi^7#xgP7FzjJt^ zn7M0wB}lv(M-&Ww4X`@ed3Kr$RE$y^j|WmkLyF1Wc%>b$1KM)?Xgqf|Zr;+~e=KJ> zmx#po6E(2jfq?5t(ks4cN=0z*iKe#i4l4kH$7>-r0%P1oq(j!2!wXuJJerZ=2eZlT zab~&Colh@rP6@De18UCJD+k_DO5+yA$=8G5KZw7h@DPc=Ts={duV18k#OoE^xKr9E zGvRvB1w8I9Xn788(B`nWGE_02LhUVw@Cyw-?3b*4%-V9V9{BkC2li4%cr}thEBR7^ zRfJopO+be))~bFpneN;%%DS>l@!i&)Q)fn>=Dol1=UY0Kg-!)n6=B0)ewJ@iis-cD zZHTH zrvy@7P?P0SXaR?Sj&U2;h1EelAK)G;_iVzckGISG_k@aDqskv~2qHpVMgg`hl4$PSPmdqe#wf@ z?)!WR0`1znG)+zguU4ZgGc9Gzf6z8PwIlgXB!@fcUm=59{zlCLD?(5C*FxvF2$Ao4 z&%D13qSy}n?kJekakWGbY4f7kr2C8)x)R>II{M#W2tg9;f|@p_SdUlyf>XL#4OH ztR1Y2oGicY1GX64JWk~ad^7caUFuf2b+6xO=UksqO8pt@cigOR4)`9Wo8fl!q15ji zN`=xU;}2D`TtxL6$MH&S2~D+N_a*k6O64lP4J7d_eX5ro?eQfH4|hd*EWs5wB&Xj?DotCOx%NcU|F`)|QBBsi zkxa2N_dDLUKZMdVwWN-f^oH7# z&p%x|n^Oy)IB7wQ4KmZ=lqZf|hbXbB^V>0DW3niQKdlgKZ>gk80b#vC>mEwDp*Z_P6zV52BO(#0e5BQ(Q9LeQ%~Qpv^DjH-S?$qxVu#ML3+&%F&{0d#Jahc|H417|A}8zI{o-5PmY_Vt;oS)6{Z+d@@O=!)km!;`%?>|tDNV4rAq%k0$; zFu6C;OOX#W0O_p|c7Fej55sa!*2@vd>zmsJwO;kGn+=3Sc$Bcuf+=RnW}m1V{gHn> zxySmdlT_$zwAxk+a1@GmI4rS!-|ou6Yt7giiuV91(ie{RYb!m9)kJj-a4bN>$e%Di zenLFw6d4HYe$`N-wg``NH*0GDBew~4q;yAR&u)cDdWD9H2ffEqK>+x z6#@fd)CXS4K!QaxX%OwyNM?qdn(bM|i+TEQtNx@oQ#I-q6an{LyPC9^77IKwJ)aJs z1!pZepYk9bYgF$dwDf_mTV5^N`X4l~u9ElEc4H4E z&;XcHMn6xSUv)emqu6B($|&gjd)2dmnX5o~p<2^J{f4z`L1sTn{H*GWkNvuctE9`v z6G{QSz9Bi3I$OA8KTW&XRKvx|hYY1V9)IUD5D<{vP%qDQZMMhb^nl~<0jjTBdq?FZ zf*!2b#R2`=;e69-P|T#;M`b&c;4}_R(w2<=q8B=g!?#D9{iSO}to4 z(3G3jghG65m#U$VP+h&1-$i!^6#cv<5f1?6N8ydX=?8r2mRmye6!In8j>H7{AqF*G zw!ywu(o{UB10gOedyS_{acJ|F$N$3QktxzXIHyMhu+M<*A!Ph>C_;V^dGqGHNL+i? z^n1C)(aDzG6=}`H?#)Vnh%3jBHjg8t%H_uAp$Q_FTg|wyi+j_YbRHDse!VY?dtHim zT{T|))^}**jEPKBkXOYv}C4e61k|QE+nvH6IImF~MZ9m@qQbd3P$I_qHWTrbtw| zv7(3n=h+wSR0pMEgG>Gw>AgYp9^7U8&%bYyig%x;?+||ebtUnlH-(5)mUwoI;csh! zy+4yAJc`Kp(V+adwb~Vx6zK9s*yhpsH4vi(A8XTWoaw}tNY|wsmE%^MJAr~2L?$f< zaK-sb4LhcshCv<++z;?)*iwGCDaVPifWbGMJy)KzB=LgMjNp@ss}DDr$C~v)NgGk< zH$nG4tXuk-RfAGh(zA}=+J;jXpcvI_ZCAp8e6%{+fnTg_(H{d{TkO|gzyblF=&m%Q zoUgk zM^*Vw+c*DDbb#{hWkZ}l|C~B9gNiRem1GccZpjr2Aq44wt0wtEx zHb^3Ccf8LrwHeh&p;p;wF}0Zof96jdV!h{DL3sAds147J@KhbpDY<%bWV`ECN^JM} zTLa}%Kk}E&Qzi8elb{S~_55p=3nb2hSTSD#1GiX}gh_Ao%(?Hz{$X5dE_~KMm*r9u4MagcdA0DQ^cnc}pAeQp$tNoF_p=(;b%ZIRjV>3|?_-x3uRQ138#+$;Ai zIEGR17*BhJ_OD&f}9Mw*bm*i$)1>4?jvMkLvB)JFP`Fj`4n zoQdunzqU%b;lk45y3bbKaO?Y%l^Si|5LiH+2s@{) zya6X0=S0stMx3$UQaX*-ow(4Xt_*(Pw(&LajnsCK>Z`P773H<9)O(YMFpMT1A+XJ? z`P`OQ!3R31!^6m9Z&Q3qU&%#!NTxv6ZwHp+ z;49>HKej}^H6|w$YZv}gs>cT<(Gtgf;!t5mxxOYlneIp_-#(kf8X7)~@HPJdms#Fx zvpwzDOD`s`ceN{jpNF}=D`0b2Bk&_^WPW(%^VNnzU;K6_s&K=TIHolua=iM;Y(PG5 zQ0NJr#!I>~E7qLmz1INieH=gH4NOoOdp*UjarUq8B0PDH?|%tuh()Ai9bLH}L8(J* zxW~jHAbn`UkwRrGC~spTOK1Q;D&J3FM7p`+r@fexi^6(jr}OA5$aDQvvg{0Z>Pd=V zEV=y_6yKnHGWIma)%u}0-v)i3*kbn5!-yg5J<6UM2l~c!s}u+ofo#`q<3`%NJ>mz$^N6J^O8eJAD9PN zkDFROqdF^r%&NK4m94$?hk*D+p{aMPmBR}_N`E)yuiC&(=n4v2xQ$%22{g4o&3?_# zhf7JY8A8IjCxJN8GqVv%i|nLu3KG`=P9$u|Twe+spAFkkZ*uIU)5+l?yTjYwm?Jot zA``J8D%%mQCI&7ybwJl*!pD*)F4{+QX{bcJ!m3I`&*BLK$0p1_a;G74qUX~++KCSs z5TdyGxEonW4u`;cCspU(_FN+SCL#7wb;DD!kD5(_(&Hxy3E^5+T?SWWg&y2XU9p@7 zU6uXGRyb-cTaxbG$3IS>(3AmRLEdg$YTJ;buzRBzS<+nO-QhEs;TesRCF$-`XJos2 z;u9g|ioHgH_;ffFfM;ED&=)9V^k}u;&@={Dqv)VjEori5@w5aT#{{NXW=?& zY8erThZ2zu8`Z0j@pu;32StbNqdv&lrqL5dy)jk>)GO>^8Xx%Jek6xccF9c+JrL*h zx0KaO1yTQg0$>PwOOmCJN?%|Ij|Y>iSVVWWx}SIn5p1dgXt$BshZ^1CjsG?jN%J^f z*P5)ww1Ji7_6(Z7Z#q|+e`NFEe0&W)R zTAvvvl1+^*Q{sp#mTgT>DGNWRKn~wmoq5vB>-vdU3kHAt6C@wMw|nhDcA{2AdXwEL zOfOIB86Iw|(gi*$$vKZUlETeNdpet`!@8h{w?I-s*BZ6`Fp~k#d`ll$p3QtZ>@}-( zhO5~=R7_a%L_kaUH#zZuNj1OF`vm~w>-$z_qKRa6!Y1-cM3)rL%1b`9p5WZ!!acIqa z%OyejA1qx4`}>3!6TW?UruUKHyir1wULN}+S#xG5Nv^)?It1b!U&F>4MW{Q`;^+VT zh|!h61uXz(BlwcOvT!7w(UBC)*~w(|!3=v^7{Yn6`9GZ7if@k|K(NQ#l+h_EvZ6w8 z<`~8DWPwnd=?VO1a>ei=4-IffWy!waHTAQhD=gW5U3f1ZN`2idA@bxh@Zk-?pZ_Eo z)iHzzzxPV6S}6T{8o^IRN}J)UAE_QsO$-pi9YW%ch0{hOj7UP;KlT{5pjV8--3mDS zyPfriTUt+NIqm-+Hf@`zkbBx&JoNh91Dt zuUYe>SjbaEeE`>v*Je#mIQQxc&T{B8u>7+YqKz+qY}B3uOZxds_?2ppK!EQ4^Eczx zk#HD+z?n)ntLks7Gd&<}X%i(dWE=T~oCT{@rxpu{7Ha+{UM~|&b4!o(lgbaHq%6EY z+0GX&oaoli1P_tCD!jn6#XkNJ#i~4XmS1*8X;1+13pCYpI=%4C>mJ0kHa>6k9f)SJ ze-Ia7o5rn;CDV=!Kj+y&08C7JA>>;i=BY1|v1W~`sW&aSUw>2V_=!b8`nsjQ-MrB> zd%opPmbr-JJA69GpuqIns0O+lq{24GnU0SaPiOOi_ycAcIF@7c!Wg zpgy~tu`3D1f$3Mm$Q0P03LcGs+~IUIUB0xfuAuLbzwy*AMWJ38EOC~R2Pjs6uzkNZ zbG?Zv$y0(JL!go~yx@Tg+ZprtkE2M|5I)tKE-r1{jd!WwK>kr1pa#HWoF`BJ^>^kS z{{mH(J&-gMN`C%JfrYCraX6Q9Az82xh#}k`@wi3UqN#4e3r3R3^`VxS!~)ICObYHg z2Le~7vVxBDp`}(S{U<@euR?1l0*I4bKJnjl_uF)E{IZ(AKG>?Nm4=$vReF8v1h00r zijvgm=|riBtQ%uZMxZ`)_ZUSAF`Tb=6T7?d6)Uh}yKr!;ZO32I1pDU7(n_CrN6nsY z2=aQE_+=&EEj}lIxJ4?~0X}YnsEMVm*%)@8riksETAaTG<=i!d-wI*5P8`cixpkJ9 zpvd>Z&S=n$WvgT>Hk@}o(jX{1Ew}#RaHu+Fgxq=#i*8>+LvLTd2z1l2vT_pC!-M3X zO&AKE7?w7ggTxo9hvPV*$oNO(lBBH=IdSEK#4HC&X&5Axh@D&| zx%BzTvUCa8URt^QGa!A}iykd`6#`@^!LG)EOitVpv@1;*FJ^x$0O<)p6uK3}@)q)8 zz=xQ;d!+}z@JA8b4U7AF+`fq$U+5lKmv8i+V;2b85z2Wt8_YY@0X-BoQI&)AkZTVh z8+OtX@%dC7S@YhvhUX56l1%H-6lm;R)R`>9OMT0`VB6j;BqnOLMmOM!qYX>83M3lX zEBWNaU@`CUEzUcE4SWCbBTImXA*EAvcIm|vBYih-pRfHn)-O#}Js~)A9yltHzs7>m zuUpkA@J{Z@%Ff!cy(I)%TChq;)r%}h;{B07t0(dAoU>)vpf6Q6k=C08K>8X5s-82ZCKP{O$Hzn&% z=R$^|0u~sV@of`i3thSt|ZMcFtIb{^xTe&Oq{3}VhsNo|ys=yN#oN2t76)(8;Q=CEeMweVr1$ zGk*!}TwN+ZXS@4)rF~)=& z#-*Xr+9_cszH0EcHxBHEee@n+$pfSC$MhyY<13KJurN1G&oj)C?HzuBuiHxL?c_2V z!}!+yI#OG4@<((eT5|CVroC0XSlZR_x~L3Df(m_zQ@wDHJkmDp7457ep~jI&F02>6 z4qO_7WMQB46AiYeI)^8pt#e-Ft!mZ!lsrPwr7sQhb+u)w)o8Xltwufj@$@fUnyKk# zsAEA`T<#6?z#{zB&xHteIIj3>Ut^VHAOOyHWfjn6E9PD(8&5perJTcOw;cD(2D(i) zo|Q*=A7Bjh$%6h=m+V`(7zoRtq4pIqYk#YKt>ad74io?y-`u_lzU(7@2`;|q&C}wg zl#9HIbXMU0xPW$$MN5dhxE9U3*k1Ygizp%ZXd*rzdp;8s=5^`fPf4s$_fGG{hQ3S^ zSZ>t0pzujTTqP|>h&qkMa+Amlld3hcM>~QJ9EbR80VuL~v9;t2Zkcupv+c&gzpg~& z1|S(9M%<1lwWK2Zzs!vskDnRNo{O6!!JGFTEnt$lf{_cc(J*q;8wF!Kp>p}iNP1rj znX@zSY-d)(eSQW{aDFiNFL&kTp--yc%#vBaoa=0M}M_h_L+Efi?Ly0jcl5A7UcJ)d+a;V zFME^Wjm;swHIzq#-Y?lY%u-l zP-uE*%WE-%o`mw9_%ke-gs^iz=C_~YwCH{+1PvHkl%Ll*~a@Ss)bftR$n zM6kJPyFk@;?K+yyoB~~XPwIFQl9jOdz6ix~+G5qZwj~7hUI?;a`l=GDgA5iDnncsG z{cZeNI=b0@keVTqm^@he2CCYYVn@b3smaem5?Zj#ViN4{5x`ly4F_co>nIE zt^Y2LxV+;xyRpxrUxwRl;W1vSOLBFukcI(N>!a;BAk*FHGcqflcgp9-)x$yV4p#U; z1M=^-LQ-1JvAl>Fp7&@J?IlQ|XU=PSsWUqt6~Uot%f)qwD_`+wQM}Ig@+}cTd(qdKSG3cPqyp-ved`KM2`#|TtzGLXLMv7i z?en{*x9(rKZ=$#a9}r3OkZ*T4i-=D?9fRdD8M=J89u`9c6wZ@b(S;2SkTfK!4Lkv- zx(`@BhKihhm$$X$_xGw4vTT!=AT@`68ir+3H(lSuWA|^Fpu`tx&kQ>>K{)lj^L{ul z8qB{s%Qz4zWt`=0Y{x`9({wtDAG5Yh&CBJwa=ZtJq8&>3V{24_0#Ch&Z9zk&j#&Hy z_+5`Fmm#?a@^0(+SO3TqQceaXCp6)v{5&!Cl6SV^bR-6s$xkn)$m)X;zjj<Nwlo%cX*^4n5A+RA*7 z2E8GTXkkvCc1B_l2J|Ni^;#_1$@#vd3Ff29$1>NK&Cz$9(o8!@R~xlR;ce&A;J z_{u2q3n*l-ACgc!E(wz#W^Ai}jUDdRo8_YXt2&=(C0cew;ZEw0?|R-u(f89bHKGOB zE6Y?|75e(>&Bg;|jn-k=W|3s#l;&L`2&3m`#^2>wGEs$=gI-hmq@d{7W+rGlHOxg* z=|YtX;n!B^@gzZv3pSjVMk9l)1G)bQHgK%FD4YIHdIq@+I0<(v%y!LYs;`?$0SMT& ztCC02CPj>QPe+S@EeEsfW!C`ACduoxJCE$Q9lR-nT@bI+F8;|5Nq(VNoJpjpa!nC{ zzNJM~b)y?^15*@Vr-*QO*U20-E+Vp=+D?X?{bDIajY*V#!ntY}mH# z4GL_)df2z#)W-dTPl##JT6#DWQjD0g?y&k>v5)g~>m?~Xe5{RX3h^jN%3E$zfvEjH zmDvccD!*O)^|C7qQMkiIIMd^{B>IU&?jp-@DN{ED3az-W;%|PyQLKT82tXo`Q5c+F zKGs#n8ZTo!Bzf33-|4%^l)zHEWfUmw1z|W*jz3--3f-ux=ulTKN)lAVF4e)ZnASyg zq7xM_6jo7T$Jt^!bjz4!UQqq&D7G<>;x^v?3SM7oYNr9}SEQ!*e9BesfE5!_L!0DEysdTFRMqK+ zR+Hv6j~=fto&hW?Vy6OHo)yGWC4c!#Xx<~|PW&e5C+CgHWXs*f6P=DfNE0CZ*9|`` zq-*8d1Vlr(`!21H$+<*6e4;RGj|N#{v`ADD?BxvKg>Wy*5p=BUAa+( zOQr<0qFv6LWzzA@{aSXZIkO7EkLYdE;T1@)#wTXVlQ)i2X2U%3TjBtSz|%LRE#{e` zhX_+xHH#YgNqN$0SRf!YleaHdhS8w$bxbQ@cJA?yrT9POx+MB)J)%tT2nfjHW-^)^ zMNqw;3sG+XYE5Cv-;vRo!syvZOvm)dVD)f{g({2uoJ$9njo6EqO|aI&;FjzSXGhlY z^RlSt?3p#?^%4nBa)m0EyksHl|KvxNIio1d8Zm3G985kEz{^p4a&9H_jYU)w*Y|mT zidz)dK=l2~r;5OuD|)1-YXG*9F=a?Z^}hy+fBt;i1f7@zxECH#{SZoLEC9*r1s4&}8cMB{uPb(qSe2RIo*T1D)_F2Oz zeyoT9k2nuU_l`SbJvqMtV5kqu`M4&|9PKL6C`wCe46Mn%#9RD~c!$Any~2HulBzOMVvn<`MH-8qnj6Lq5XBG-@2Y$4Ik+09ab|$ zO5jZp#j+)bIUjZmZ_x-hhSzi`__e}(_x94+9#zx&E#k~;t(fjQhrHs^wc!cDVe!4b zty@oZ_|DgMYPR&|_Q-*Gq@4Pzi6!~gzXm~^ z7U7b$oCwoO-d|xeo2egSHD~R->baQUT|)w!dXNUX$Q_X3N(M9ToXkPCTYnLLgu{12 zU0GHN>1fYE&*~wOg67&p(QKURmm1u-a}0sx{8(0o!PVQQvWyviR9=NeOU?MTh=pi} z*A4}Ak@Mk3PNJoZ!6!y}E8!evhWkB#`zko@bGRJ-_6&n^R&@EPHJ7`P7BOQlldc2eES{1MMN2~XUlD5xJCu>&+Yot8`6U=)O z$3;klOkoDMvEQXKGk`g{H)f6P2`M0dm`GLG&3JK9pX3-4U`%O)f`pnpxtbM-IT_p* zDfld^hQu;}_oFt#x7E2zm_C7cxaET}7tN9m*yi`;pKOJ=iUwkK#+LIxJr%>SB~i7nxz? zMMiAkh;NTEX1>@hfBrhMndi83e1Y>-&`QVL%!5|wVy|CInca#(1{KeAyBIALE*EkKHB(_)rncY3xTA;u&_J_Abn$?c-#=N!x%8WQE< zRAdrG#$8j}{gJ`=16^G~LCjK!Im7-sTuWKTM{2o` z{`^DmifEok3* z8~w&-3zImPox(xhWTb@}9MJ@TS1M@RQ> z+!@V#tVu&2&C6oX>v)|=hW<%2RTkX2np7w0wZRiQ!V}q# z-R0w4omkF0_cZ80U29bkbWxDke7k{rHQ8GDuDPk>-k3X*;~8d~MO@e%_Qm&~TINW3 z&|EyDoSUs`lOJ}@nX%l)rgYV1qeG^hV>Mn&*#lNLe)j5HpSzN^-^R1SQUM!*Z3+$a zFM9+t{krlCw>}KL`Zg!Rfbt5YuboaI%Dbi&Z1s2?YN~s_LIhg!nIz4Z8xPDQaH0IF zk;vO(Td`)tTK_FS#oKv9XEatwvOyIklnW*6_b$e` zdKKpgD%fGTV%mu~GhnE99~xxmuOiQGX?qdsUd*A=)BHs?|1O;GUuPQjl2|=NhR{gJ zPnLIpQPU}_Doc#(wGUo=HM-O?~sOv??Gp6(%fk*{ze7<)jlmL?U!45V{?--2_{{Zm+9dkX6CAPPV} z0+at%mD76+C+Jq*3mhz@$O8xJVs1C|H8|$n6t??fh}35V*jPP#WG?s>xC3){0FcmX zKt8gGYKDYc;z>-dpCwiP{B+_EEAnkpJ96jpaeUKJn#GHDaF&nho8MWy7L@5%ApvxU_^EtBGhi7MO@LR`f%k4_*%;zXIs zIYkPu@j(_io9NwOz#IS~j*+b18d>CKtwWFN?p#NIx(*y|@ykz^g7I#BFRnV_<?395M7rLf&ug5lQ9ex z$=}SQ$9}u(RVl84n3_DvW0jhr*7UqDjXc@%%cb)L3c@q*CN-^?x%%R zrTJK-f3v6N=`p7$j6Q)Sv|soEX#~|j->*1$sAouj68YABy}!n?G;`O%stKh(`G8=I zCKDMR^M)n0>2Ym?B`PUPWRhlL)Uk{{)xM<=1*?ls5l4l{d^#da|LuA@d;fSXiuvPS ztR`!gVv0k-AQr*uW^f31XPKze0Y9Zo`EK5)7>f1IQ%3hcw70aTmdWp2s-xaDc9DU>K^NV$KxpeeUlUFNe zcaErDrhf=ZWZCJSfi0JpQdbv)+zoPHaBgm6JD0|-BU_>Sj#1jdXQ+Qn`6ot>tTe%- zu;3l;>kE<-$VQ>FDJgPzn+MXR;(qhCyK#(4^kM-;O3Bj-qr^;Wr&)BsMH=?hybO1B zgLz9vxU3$s@8!I*^l`cOUWqC0ys@nE$>V}&U`kguIzU(Kk~PHzy=7KT#hJbl+jr0R l!v8;6n*S$@E{M4e;p5L1-O=*+h;dj0RF$+8YhGK0{U6kFAS3_) diff --git a/plugins/lancedb/skills/lancedb/references/branch_ops.md b/plugins/lancedb/skills/lancedb/references/branch_ops.md deleted file mode 100644 index d79e5c8bc..000000000 --- a/plugins/lancedb/skills/lancedb/references/branch_ops.md +++ /dev/null @@ -1,16 +0,0 @@ -# Branch Operations - -Branches are isolated, writable lines of history forked from `main` by default (or from another branch/version via `create`'s `from_ref`/`from_version`). There is no global "switch branch" state: `branches.create(...)` / `branches.checkout(...)` return a **table handle scoped to that branch**, and every read/write on that handle lands on the branch while the original main handle is unaffected. Unpinned handles track the branch's latest version and are writable; `checkout(name, version=...)` pins the handle to that version and is read-only. - -Don't work from memory — read the public docs for the current API: - -- **Branching guide (concepts + Python/TypeScript examples):** — covers creating, writing to, reopening, and deleting branches; applying branch-tested changes back to main; diff/merge (Enterprise only); and building indexes on a branch. -- **How branches relate to versions and tags:** -- **Python API reference** (`Table.branches`, `Table.current_branch`, `Branches`/`AsyncBranches` with `list`/`create`/`checkout`/`delete`/`diff`/`merge`): -- **TypeScript API reference** (`Branches` class, same methods; `table.branches()` is async, `table.currentBranch()` returns `null` for main): - -Notes the docs may not state prominently: - -- Branch lifecycle works on local/OSS and remote Cloud/Enterprise tables; **merging into main is Enterprise-only** (others raise `NotSupported`). A rejected merge is not an exception — inspect the returned `status` and `diff.mergeBlockers`. -- To verify isolation after a branch write, read through both handles: the branch handle sees the change, the main handle must not. - diff --git a/plugins/lancedb/skills/lancedb/references/column_metadata.md b/plugins/lancedb/skills/lancedb/references/column_metadata.md deleted file mode 100644 index 2fe1c7c5d..000000000 --- a/plugins/lancedb/skills/lancedb/references/column_metadata.md +++ /dev/null @@ -1,183 +0,0 @@ -# Column Metadata Authoring - -Write column-level descriptions, tags, and logical groupings onto a LanceDB table's schema. Use this when the user wants to document, annotate, tag, or classify what their table columns ARE (embeddings vs labels vs eval metrics, model provenance, version families, etc.). - -Works on local/OSS and remote Enterprise/Cloud tables alike — read the schema through the table handle, write through `update_field_metadata` (Python) / `updateFieldMetadata` (TypeScript). - -## Metadata key conventions - -All metadata uses namespaced keys: - -| Key | Purpose | Example value | -|-----|---------|---------------| -| `lancedb:description` | Human-readable explanation of what the column contains | `"CLIP ViT-L/14 image embedding, L2-normalized (768-dim)"` | -| `lancedb:tag:` | Flexible key-value tag; the suffix names the tag category | `lancedb:tag:field_type: "embedding"`, `lancedb:tag:model: "clip"`, `lancedb:tag:project_id: "foo"` | -| `lancedb:logical-column` | Logical group/family this column belongs to | `"clip_features"` | - -Tags are open-ended — use whatever key suffix and value make sense given the user's intent. The tag suffix should describe *what is being classified* (e.g., `field_type`, `model`, `project_id`) and the value describes *how*. Multiple tags on the same column are fine — each is a separate key. All values are strings. - -## Step 1: Read the schema and existing metadata - -Read existing metadata before writing, to avoid redundant updates. - -Python — `table.schema` (sync property; async: `await table.schema()`) returns a `pyarrow.Schema`. **Arrow field metadata is bytes-keyed in Python**: - -```python -schema = table.schema -for field in schema: - meta = field.metadata or {} # dict[bytes, bytes], e.g. {b"lancedb:description": b"..."} - print(field.name, field.type, field.nullable, meta) -``` - -TypeScript — `await table.schema()` returns an Arrow `Schema`; field metadata is a `Map`: - -```typescript -const schema = await table.schema(); -for (const field of schema.fields) { - console.log(field.name, field.type, field.nullable, field.metadata); // Map - // field.metadata.get("lancedb:description") -} -``` - -For struct/nested fields, recurse into the field's children and address them as dot-paths (e.g., `parent.child`). - -If the user hasn't specified which columns to update, work with all columns. - -## Step 2: Generate metadata - -Decide what to generate based on the user's request. - -### Descriptions (`lancedb:description`) - -Base descriptions on: -- The column name and Arrow type (e.g., `FixedSizeList` of floats → likely an embedding) -- User-supplied context (upstream pipeline, sample values, domain knowledge) -- Name patterns: `_embedding`/`_vec`/`_embed` → vector; `_label`/`_class` → label; `_score`/`_eval`/`_metric` → evaluation metric - -Be specific and concise. Good: `"Sentence-BERT embedding of the query text (768-dim)."` Not: `"An embedding column."` - -### Tags (`lancedb:tag:`) - -Choose tag key names that match what the user asked to annotate. Common patterns: - -- Semantic field type → `lancedb:tag:field_type: "embedding"` / `"text"` / `"image"` / `"label"` / `"eval"` / `"id"` / `"metadata"` -- Model or source → `lancedb:tag:model: "clip"` / `"bert"` / `"vit"` -- Project affiliation → `lancedb:tag:project_id: ""` -- Version → `lancedb:tag:version: "v3"` (and `lancedb:tag:latest: "true"` for the newest) - -Use Arrow type as a hint: `FixedSizeList` + float → embedding; `Utf8`/`LargeUtf8` → text; `Binary` → image or blob. - -### Logical groupings (`lancedb:logical-column`) - -Look for naming patterns across columns: -- `clip_v1`, `clip_v2`, `clip_v3` → logical column `"clip"`, latest is `v3` -- `text_embed_20240101`, `text_embed_20240601` → logical column `"text_embed"`, latest is the most recent date suffix - -Write `lancedb:logical-column` on all members of a group. Mark the newest with `lancedb:tag:latest: "true"` (in addition to its version tag). - -## Step 3: Write the metadata - -Each update names a field by dot-path and carries a metadata map. Semantics (identical in both SDKs): - -- **Merge by default** (`replace` omitted/false) — preserves existing metadata the user didn't ask to change -- `replace: true` swaps the field's entire metadata map — only if the user explicitly asks to overwrite -- A value of `None`/`null` deletes that specific key -- Batch all field updates into a single call when possible -- Returns the new table version - -Python (sync and async take one dict per field, as varargs): - -```python -res = table.update_field_metadata( - { - "path": "clip_v3", - "metadata": { - "lancedb:description": "CLIP ViT-L/14 image embedding, L2-normalized (1024-dim).", - "lancedb:tag:field_type": "embedding", - "lancedb:tag:model": "clip", - "lancedb:tag:version": "v3", - "lancedb:tag:latest": "true", - "lancedb:logical-column": "clip", - }, - }, - { - "path": "clip_v2", - "metadata": { - "lancedb:description": "CLIP ViT-B/32 image embedding (768-dim), superseded by v3.", - "lancedb:tag:field_type": "embedding", - "lancedb:tag:model": "clip", - "lancedb:tag:version": "v2", - "lancedb:logical-column": "clip", - }, - }, -) -print(res.version) # new table version - -# merge semantics: add a key, delete one via None, keep the rest -table.update_field_metadata( - {"path": "clip_v2", "metadata": {"lancedb:tag:archived": "true", "lancedb:tag:latest": None}} -) -``` - -(`replace_field_metadata` is deprecated — use `update_field_metadata`.) - -TypeScript (takes an array of `FieldMetadataUpdate`): - -```typescript -const res = await table.updateFieldMetadata([ - { - path: "clip_v3", - metadata: { - "lancedb:description": "CLIP ViT-L/14 image embedding, L2-normalized (1024-dim).", - "lancedb:tag:field_type": "embedding", - "lancedb:tag:model": "clip", - "lancedb:tag:version": "v3", - "lancedb:tag:latest": "true", - "lancedb:logical-column": "clip", - }, - }, - { - path: "clip_v2", - metadata: { - "lancedb:description": "CLIP ViT-B/32 image embedding (768-dim), superseded by v3.", - "lancedb:tag:field_type": "embedding", - "lancedb:tag:model": "clip", - "lancedb:tag:version": "v2", - "lancedb:logical-column": "clip", - }, - }, -]); -console.log(res.version); // new table version - -// merge semantics: add a key, delete one via null, keep the rest -await table.updateFieldMetadata([ - { path: "clip_v2", metadata: { "lancedb:tag:archived": "true", "lancedb:tag:latest": null } }, -]); -``` - -## Step 4: Confirm - -Report back: -- Which columns were updated and what was written -- The new table version number (from the result) -- Any columns skipped (e.g., already had up-to-date metadata) - -## Quick examples - -**"Write descriptions for all columns in the `product_embeddings` table"** -1. Read `table.schema` → all fields + existing metadata -2. Generate a `lancedb:description` for each column based on name + type -3. One `update_field_metadata` call with all descriptions -4. Report - -**"Tag the columns in `model_outputs` with their field type and model"** -1. Read the schema -2. For each field, classify by name + Arrow type → set `lancedb:tag:field_type` and `lancedb:tag:model` where applicable -3. Write in one batched call -4. Report - -**"Group the feature columns in `training_features` into logical families and mark the latest version"** -1. Read the schema -2. Find version patterns → assign `lancedb:logical-column` and `lancedb:tag:version`; mark newest with `lancedb:tag:latest: "true"` -3. Write in one batched call -4. Show the grouping diff --git a/plugins/lancedb/skills/lancedb/references/remote_connect.md b/plugins/lancedb/skills/lancedb/references/remote_connect.md deleted file mode 100644 index 670242c94..000000000 --- a/plugins/lancedb/skills/lancedb/references/remote_connect.md +++ /dev/null @@ -1,45 +0,0 @@ -# Connecting to a LanceDB remote server - -LanceDB Enterprise/Cloud deployments are served by a server implementing the -lance-namespace OpenAPI spec -(). -Every remote (`db://...`) connection talks to such a server, and some operations -exist only there. In particular, all operations around jobs (listing, inspecting, -creating, or canceling jobs) run server-side — there is no local/OSS equivalent, so -resolve a server connection before attempting any job work. The job REST methods -themselves are documented in `references/remote_jobs.md`. - -Every request needs two things: - -1. **Base URL** — the server endpoint -2. **Credentials** — an API key (`x-api-key` header over REST), and usually a database name (`x-lancedb-database` header) - -## Resolution steps - -1. If the user already gave a URL and API key (or said which environment they're working against), use that. -2. Otherwise, look for credentials already available in the environment: - - Env vars like `LANCEDB_URI` / `LANCEDB_HOST` / `LANCEDB_API_KEY` - - A server endpoint already running or port-forwarded locally (the REST default port is 2333, i.e. `http://localhost:2333`) -3. If you didn't find both pieces, ask the user directly: **"What's your LanceDB endpoint's URL, and what's your API key?"** Also ask which database to use if it isn't obvious. Don't guess or probe further — the user knows their deployment. - -## Validating the connection - -Make a cheap authenticated request and check the status before starting real work: - -```bash -curl -s -w "\n%{http_code}" "{base_url}/v1/table/?limit=1" \ - -H "x-api-key: " \ - -H "x-lancedb-database: " -``` - -- `200` — connection, key, and database header all good -- `401` — API key missing or wrong -- `400` mentioning a database header — this deployment expects `x-lancedb-database` - -## Non-REST equivalents - -The same credentials work through the SDKs and CLI: - -- Python SDK: `lancedb.connect("db://", api_key="", host_override="")` -- TypeScript SDK: `await lancedb.connect("db://", { apiKey: "", hostOverride: "" })` -- `lancedb` CLI: a `[profiles.]` entry in `~/.lancedb/config.toml` with `http_server_url`, `api_key`, `database` diff --git a/plugins/lancedb/skills/lancedb/references/remote_jobs.md b/plugins/lancedb/skills/lancedb/references/remote_jobs.md deleted file mode 100644 index 175f0ce51..000000000 --- a/plugins/lancedb/skills/lancedb/references/remote_jobs.md +++ /dev/null @@ -1,151 +0,0 @@ -# Job operations over the LanceDB remote server REST API - -Jobs are server-side background operations on LanceDB Enterprise/Cloud — index builds, -column backfills, materialized view refreshes, and similar async work. Endpoints that -trigger async work (e.g. the column backfill or materialized view refresh endpoints) -return a `job_id`; these four methods are how you track and manage those jobs. - -Resolve the connection first — see `references/remote_connect.md`. All four methods -are **POST** requests under `{base_url}/v1/jobs/` with JSON bodies, and take the usual -`x-api-key` / `x-lancedb-database` headers. If every job call returns `501`, job APIs -are disabled on that deployment (the server has no job registry configured) — report -that rather than retrying. - -## 1. List jobs — `POST /v1/jobs/list` - -The body is optional; an empty body lists everything. All fields are filters: - -```json -{ - "limit": 100, - "table_name": "my_table", - "job_type": "...", - "job_subtype": "...", - "state": "...", - "page_token": "..." -} -``` - -```bash -curl -s -X POST "{base_url}/v1/jobs/list" \ - -H "x-api-key: " -H "x-lancedb-database: " \ - -H "content-type: application/json" \ - -d '{"table_name": "my_table"}' -``` - -Response: - -```json -{ - "jobs": [ - { - "job_id": "...", - "table": "my_table", - "job_type": "...", - "job_subtype": "...", - "state": "done", - "created_at_millis": 1720000000000 - } - ], - "page_token": "..." -} -``` - -A `page_token` in the response means there are more results — pass it back in the next -request to continue. Note list rows use a lowercase `state` string, while describe uses -an uppercase `job_state`. - -## 2. Describe a job — `POST /v1/jobs/describe` - -Body: `{"job_id": ""}`. Returns full detail for one job: - -```json -{ - "job_id": "...", - "job_type": "...", - "job_subtype": "...", - "job_state": "IN_PROGRESS", - "creation_ms": 1720000000000, - "spec": {}, - "status": {} -} -``` - -`job_state` is one of `IN_PROGRESS`, `CANCELLED`, `FAILED`, `DONE`. `spec` and `status` -are job-type-specific JSON objects (the job's input specification and its current -progress/status). Returns `404` for an unknown job id. - -## 3. Cancel a job — `POST /v1/jobs/cancel` - -Body: `{"job_id": ""}`; response echoes `{"job_id": ""}`. Cancellation is a -service-level operation requiring the same administrative authorization as the -`/admin` routes — a database-scoped API key that can list and describe jobs may still -get a permission error here. Other errors: `404` unknown job, `409` state conflict -(e.g. already in a terminal state), `429` too much write contention (safe to retry). - -## 4. Query job event history — `POST /v1/jobs/query_events` - -Returns the event history (state transitions, progress updates) for one or more jobs. -Body: `{"job_id": ""}` for one job, or `{"job_ids": ["", ...]}` for a batch. -Optional fields: `limit` (max event rows), `limit_per_job` (per job in a batch query), -and `filter` — a SQL-like expression over the columns `state`, `updated_by`, -`owner_component`, and `claim_entity`. (`full_text_search` is reserved and currently -rejected as not implemented.) - -The response is **not JSON** — it is an Arrow IPC stream -(`content-type: application/vnd.apache.arrow.stream`). Decode it, e.g. in Python: - -```python -import pyarrow.ipc -import requests - -resp = requests.post( - f"{base_url}/v1/jobs/query_events", - headers={"x-api-key": key, "x-lancedb-database": database}, - json={"job_id": job_id}, -) -resp.raise_for_status() -events = pyarrow.ipc.open_stream(resp.content).read_all() -``` - -## Feature engineering (Geneva) jobs - -Feature engineering jobs — UDF column backfills and materialized view refreshes run -through Geneva — are tracked **separately** from the `/v1/jobs` registry above. Their -records live in a `geneva_jobs` table inside the database itself (in the `__system` -namespace), and you access them through a Python `geneva` connection rather than the -REST endpoints above: - -```python -import geneva -from geneva.jobs import JobStateManager - -# Same credentials as lancedb.connect / the REST API -conn = geneva.connect("db://", api_key="", host_override="") -jsm = JobStateManager(conn) - -# List jobs. NOTE: status defaults to "RUNNING"; pass status=None for all jobs. -# Statuses: PENDING | RUNNING | DONE | FAILED | CANCELLED -jobs = jsm.list_jobs(table_name="my_table", status=None) - -# Fetch one job by id (returns a list of JobRecord) -records = jsm.get("") -``` - -Each `JobRecord` has `table_name`, `column_name`, `job_id`, `job_type`, `status`, -`launched_at`, `completed_at`, `config`, `launched_by`, `manifest_id`, `cluster_name`, -`metrics` (progress counters), `events` (human-readable history), and `updated_at`. -For filters `list_jobs` doesn't support (e.g. time ranges), query the underlying table -directly: `jsm.get_table(True).search().where("launched_at >= TIMESTAMP '...'")` — -pass `True` to check out the latest version, since other processes update job state. - -Stale-status caveat: nothing reaps dead Geneva jobs, so a job can sit in -`RUNNING`/`PENDING` forever if its worker died. Treat a job as effectively `FAILED` -when it has been running longer than ~36 hours, or its `updated_at` is more than ~2 -hours old (this matches the heuristic the Geneva console UI applies on read). - -## Workflow tips - -- To wait for async work (a backfill, an index build), poll `describe` until - `job_state` leaves `IN_PROGRESS`; on `FAILED`, pull `status` and `query_events` for - the failure detail. diff --git a/plugins/lancedb/skills/lancedb/scripts/check_materialization.py b/plugins/lancedb/skills/lancedb/scripts/check_materialization.py deleted file mode 100644 index 6b0f117a1..000000000 --- a/plugins/lancedb/skills/lancedb/scripts/check_materialization.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python3 -"""Scan Python and TypeScript for likely unsafe LanceDB materialization.""" - -from __future__ import annotations - -import argparse -import re -import sys -from dataclasses import dataclass -from pathlib import Path - - -PY_FULL_TABLE = re.compile(r"\b\w+\.(to_pandas|to_arrow|to_polars)\s*\(") -TS_TABLE_TO_ARROW = re.compile(r"\b\w+\.toArrow\s*\(") -TS_QUERY_COLLECTOR = re.compile(r"\.query\s*\(\s*\)[\s\S]*?\.to(Array|Arrow)\s*\(") - - -@dataclass -class Finding: - path: Path - line: int - message: str - text: str - - -def iter_files(paths: list[Path]) -> list[Path]: - files: list[Path] = [] - for path in paths: - if path.is_dir(): - files.extend( - p - for p in path.rglob("*") - if p.suffix in {".py", ".ts", ".tsx"} and "node_modules" not in p.parts - ) - elif path.suffix in {".py", ".ts", ".tsx"}: - files.append(path) - return sorted(set(files)) - - -def line_number(text: str, offset: int) -> int: - return text.count("\n", 0, offset) + 1 - - -def scan_python(path: Path, text: str) -> list[Finding]: - findings: list[Finding] = [] - for match in PY_FULL_TABLE.finditer(text): - line_start = text.rfind("\n", 0, match.start()) + 1 - line_end = text.find("\n", match.start()) - if line_end == -1: - line_end = len(text) - line = text[line_start:line_end].strip() - if ".search(" in line or ".query(" in line: - continue - findings.append( - Finding( - path, - line_number(text, match.start()), - f"Review Python `{match.group(1)}()` call; table-level materialization is not portable to remote tables.", - line, - ) - ) - return findings - - -def statement_around(text: str, start: int, end: int) -> str: - before = max(text.rfind(";", 0, start), text.rfind("\n\n", 0, start)) - after_candidates = [ - pos for pos in (text.find(";", end), text.find("\n\n", end)) if pos != -1 - ] - after = min(after_candidates) if after_candidates else len(text) - return text[before + 1 : after].strip() - - -def scan_typescript(path: Path, text: str) -> list[Finding]: - findings: list[Finding] = [] - for match in TS_TABLE_TO_ARROW.finditer(text): - stmt = statement_around(text, match.start(), match.end()) - if ".query(" in stmt or ".search(" in stmt: - continue - findings.append( - Finding( - path, - line_number(text, match.start()), - "Review TypeScript `table.toArrow()`-style call; table-level materialization is not portable for large/remote tables.", - stmt.splitlines()[0].strip(), - ) - ) - - for match in TS_QUERY_COLLECTOR.finditer(text): - stmt = statement_around(text, match.start(), match.end()) - if ".limit(" in stmt: - continue - findings.append( - Finding( - path, - line_number(text, match.start()), - "Review unbounded TypeScript query collection; add `limit()` or stream batches.", - stmt.splitlines()[0].strip(), - ) - ) - return findings - - -def scan_file(path: Path) -> list[Finding]: - text = path.read_text(encoding="utf-8", errors="replace") - if path.suffix == ".py": - return scan_python(path, text) - if path.suffix in {".ts", ".tsx"}: - return scan_typescript(path, text) - return [] - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("paths", nargs="+", type=Path) - parser.add_argument( - "--no-fail", action="store_true", help="Always exit 0 after reporting findings." - ) - args = parser.parse_args() - - findings: list[Finding] = [] - for path in iter_files(args.paths): - findings.extend(scan_file(path)) - - for finding in findings: - print(f"{finding.path}:{finding.line}: {finding.message}") - print(f" {finding.text}") - - if findings: - print( - f"\n{len(findings)} finding(s). Review manually; bounded query result conversion may be OK." - ) - return 0 if args.no_fail or not findings else 1 - - -if __name__ == "__main__": - sys.exit(main()) From ecf4555cfde8103c143349bb27a6dd5813e3b674 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 22 Aug 2026 00:53:28 +0800 Subject: [PATCH 11/58] fix(remote): fence refresh submissions after add_columns (#4007) A remote backfill submission validates its target column against a table snapshot, but it did not carry the existing read-after-write freshness headers. Immediately after `add_columns`, a stale query node could therefore reject the newly committed column. Route backfill submission through the remote table read fence so it carries the version returned by the preceding write. The shared remote submission path gives synchronous and asynchronous client surfaces the same freshness guarantee. --- rust/lancedb/src/remote/table.rs | 42 ++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 1e5691554..14b869d90 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2866,8 +2866,7 @@ impl BaseTable for RemoteTable { let mut body = serde_json::json!({ "column": column }); self.apply_branch_body(&mut body); let request = self - .client - .post(&format!("/v1/table/{}/backfill_column", self.identifier)) + .post_read(&format!("/v1/table/{}/backfill_column", self.identifier)) .json(&body); let (request_id, response) = self.send(request, true).await?; let response = self.check_table_response(&request_id, response).await?; @@ -6823,6 +6822,45 @@ mod tests { ); } + #[tokio::test] + async fn test_refresh_submission_uses_add_columns_version_fence() { + let table = Table::new_with_handler("my_table", |request| match request.url().path() { + "/v1/table/my_table/describe/" => simple_describe_response(), + "/v1/table/my_table/add_columns/" => http::Response::builder() + .status(200) + .body(r#"{"version": 7}"#.to_string()) + .unwrap(), + "/v1/table/my_table/backfill_column" => { + let min_version = request + .headers() + .get("x-lancedb-min-version") + .and_then(|value| value.to_str().ok()); + if min_version != Some("7") { + return http::Response::builder() + .status(400) + .body(r#"{"error":"Column not found: doubled"}"#.to_string()) + .unwrap(); + } + http::Response::builder() + .status(202) + .body(r#"{"job_id": "j-43"}"#.to_string()) + .unwrap() + } + path => panic!("unexpected request: {path}"), + }); + + let result = table + .add_columns() + .computed("doubled", "a * 2") + .execute() + .await + .unwrap(); + assert_eq!(result.version, 7); + + let job = table.refresh_column_async("doubled").await.unwrap(); + assert_eq!(job.id(), Some("j-43")); + } + /// The gate's reproducer: after a successful wait, a same-handle read /// must carry a freshness baseline so a stale server cache cannot serve /// the pre-backfill snapshot. From 1baada89ef8809cccc45b1dc6360713d28f0db9a Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 22 Aug 2026 02:01:45 +0800 Subject: [PATCH 12/58] feat(python): bind function versions to columns (#4012) A registered `FunctionVersion` has an exact identity and grouped output contract, but the Python SDK cannot currently bind it to table columns without manually constructing wire models. Calling a `FunctionVersion` with named `col(...)` references now returns one immutable `FunctionApplication` pinned to that exact version. The application preserves named-struct outputs as one sibling group, while `rename(columns=...)` defines the result-field to table-column mapping consumed by `Table.add_columns`. Derived expressions and incomplete or unknown input names fail before declaration. --- python/python/lancedb/expr.py | 5 +- python/python/lancedb/functions.py | 68 +++++++++++++++- .../tests/test_first_class_function_slice1.py | 78 +++++++++++++++++++ 3 files changed, 148 insertions(+), 3 deletions(-) diff --git a/python/python/lancedb/expr.py b/python/python/lancedb/expr.py index e8b2d63a4..d16ba95d7 100644 --- a/python/python/lancedb/expr.py +++ b/python/python/lancedb/expr.py @@ -85,8 +85,9 @@ class Expr: # for dict keys / set membership. __hash__ = None # type: ignore[assignment] - def __init__(self, inner: PyExpr) -> None: + def __init__(self, inner: PyExpr, *, column_path: str | None = None) -> None: self._inner = inner + self._column_path = column_path # ── comparisons ────────────────────────────────────────────────────────── @@ -273,7 +274,7 @@ def col(name: str) -> Expr: >>> col("age") > lit(18) Expr((age > 18)) """ - return Expr(expr_col(name)) + return Expr(expr_col(name), column_path=name) def lit(value: Union[bool, int, float, str, bytes, date, datetime, Decimal]) -> Expr: diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 1613e03b4..3b2d05951 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -20,6 +20,7 @@ import re import sys import textwrap import types +import uuid from collections.abc import Mapping from datetime import date, datetime from typing import ( @@ -265,6 +266,64 @@ class FunctionVersion(_RemoteValue): required_secrets: tuple[str, ...] = () created_at: str + def __call__(self, **inputs: Any) -> FunctionApplication: + """Bind this exact version to named table columns. + + Every input must be a direct [lancedb.col][lancedb.expr.col] + reference. The returned application is immutable and retains a + named-struct output as one sibling group, so every row's sibling values + come from one logical Function evaluation. Map result fields to table + columns with + [FunctionApplication.rename][lancedb.functions.FunctionApplication.rename], + then pass the application to + [Table.add_columns][lancedb.table.Table.add_columns]. + + Examples + -------- + >>> from lancedb import col + >>> application = function( + ... title=col("title"), + ... body=col("body"), + ... ).rename(columns={ + ... "normalized_text": "search_text", + ... "token_count": "search_token_count", + ... }) + >>> table.add_columns(application) # doctest: +SKIP + """ + from lancedb.expr import Expr + + parameters = tuple(parameter.name for parameter in self.signature.inputs) + missing = [parameter for parameter in parameters if parameter not in inputs] + unknown = sorted(set(inputs) - set(parameters)) + if missing or unknown: + details = [] + if missing: + details.append(f"missing inputs: {missing!r}") + if unknown: + details.append(f"unknown inputs: {unknown!r}") + raise TypeError("invalid Function inputs (" + "; ".join(details) + ")") + + bindings = [] + for parameter in parameters: + value = inputs[parameter] + if not isinstance(value, Expr) or value._column_path is None: + raise TypeError( + f"Function input {parameter!r} must be a direct col(...) reference" + ) + bindings.append( + ApplicationInput( + parameter=parameter, + kind="column", + value={"path": value._column_path}, + ) + ) + return FunctionApplication( + function=FunctionVersionRef(name=self.name, version=self.version), + inputs=tuple(bindings), + output=self.signature.output, + group_id=f"fg_{uuid.uuid4().hex}", + ) + class FunctionRegistrationRequest(_RemoteValue): """Stable remote registration envelope produced by :func:`udf`. @@ -304,7 +363,14 @@ class ApplicationInput(_OpenRemoteValue): class FunctionApplication(_OpenRemoteValue): - """Immutable pre-declaration application of an exact Function version.""" + """Immutable pre-declaration application of an exact Function version. + + A named-struct output remains one grouped application through table + declaration and execution. + [FunctionApplication.rename][lancedb.functions.FunctionApplication.rename] + records the result-field to table-column mapping without splitting sibling + outputs into separate UDF calls. + """ function: FunctionVersionRef inputs: tuple[ApplicationInput, ...] diff --git a/python/python/tests/test_first_class_function_slice1.py b/python/python/tests/test_first_class_function_slice1.py index fead28bc8..1bb8feaf4 100644 --- a/python/python/tests/test_first_class_function_slice1.py +++ b/python/python/tests/test_first_class_function_slice1.py @@ -6,6 +6,7 @@ from pathlib import Path import pytest +from lancedb import col import lancedb.functions as functions from lancedb.functions import ( FunctionApplication, @@ -120,6 +121,83 @@ def test_function_version_identity_is_immutable_and_exact(): assert FunctionVersion(**changed) != version +def test_function_version_binds_named_columns_as_one_immutable_group(): + version = FunctionVersion.from_json( + json.dumps(job_result("remote_function_job.json")) + ) + + application = version(text=col("documents.body")) + + assert application.function.name == version.name + assert application.function.version == version.version + assert application.output is version.signature.output + assert application.group_id.startswith("fg_") + assert [ + (value.parameter, value.kind, value.value["path"]) + for value in application.inputs + ] == [("text", "column", "documents.body")] + with pytest.raises((TypeError, ValueError)): + application.group_id = "fg_changed" + + +def test_function_version_binding_validates_names_and_direct_columns(): + version = FunctionVersion.from_json( + json.dumps(job_result("remote_function_job.json")) + ) + + with pytest.raises(TypeError, match=r"missing inputs: \['text'\]"): + version() + with pytest.raises(TypeError, match=r"unknown inputs: \['body'\]"): + version(text=col("text"), body=col("body")) + with pytest.raises(TypeError, match="direct col"): + version(text=col("text").lower()) + + +def test_function_version_keeps_named_struct_outputs_in_one_application(): + value = job_result("remote_function_job.json") + value["name"] = "text_features" + value["version"] = "fv_grouped" + value["signature"] = { + "inputs": [ + {"name": "title", "arrow_type": "utf8", "nullable": True}, + {"name": "body", "arrow_type": "utf8", "nullable": True}, + ], + "output": { + "kind": "named_struct", + "fields": [ + { + "name": "normalized_text", + "arrow_type": "utf8", + "nullable": False, + }, + { + "name": "token_count", + "arrow_type": "int64", + "nullable": False, + }, + ], + }, + } + version = FunctionVersion(**value) + + application = version(body=col("body"), title=col("title")).rename( + columns={ + "normalized_text": "search_text", + "token_count": "search_token_count", + } + ) + + assert [value.parameter for value in application.inputs] == ["title", "body"] + assert [field.name for field in application.output.fields] == [ + "normalized_text", + "token_count", + ] + assert dict(application.columns) == { + "normalized_text": "search_text", + "token_count": "search_token_count", + } + + def test_unknown_fields_and_discriminators_are_forward_decodable(): value = job_result("remote_function_job.json") value["future_version_metadata"] = {"retention_class": "catalog"} From f39a7a4dd9f788624f4f1e099f4a4d7765d8702d Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Fri, 21 Aug 2026 13:45:39 -0700 Subject: [PATCH 13/58] feat: support remote tables in the data loader (#3981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `StreamingDataset`, `PermutationBuilder`, and `Permutation` now work against a `RemoteTable` (LanceDB Cloud and Enterprise), which unblocks benchmarking the loader against the enterprise cluster cache. ```python db = lancedb.connect("db://my-db", api_key=..., host_override=...) ds = StreamingDataset(db.open_table("training"), world_size=8, rank=r) ``` Rows are addressed by `_rowid` exactly as before — `PermutationReader::load_batch` already built the same `_rowid IN (...)` filter that `Table::take_row_ids` sends, so the loader's fetch was always the take path. It just was never allowed to run. ### The guard `PermutationBuilder.__init__` rejected anything without `_inner`, so a `RemoteTable` raised `TypeError` before reaching the PyO3 layer — which already unwraps one via `_table._inner`. ### A bounded schema lookup `PermutationReader::output_schema` reads the schema off a query plan, and building a plan on a remote table *executes* the query (`create_plan` → `execute_query`). With no limit that is `k = isize::MAX`, so asking a remote table for its output schema pulled the whole table over HTTP and threw it away — once per assigned split, on every epoch, since `StreamingDataset.__iter__` constructs a `Permutation` per split. One row rather than zero, deliberately: lance gates its limit node on `self.limit.unwrap_or(0) > 0`, so `Some(0)` means *no limit*. ### Tables with an LSM write spec are refused A permutation references rows by row id, and rows that have not been flushed to the base table do not have one yet. The loader could read around them, but they would then be missing from training with nothing said about it, so the build refuses such a table up front instead of half supporting it. ### Fallible identity construction `PermutationReader::identity` resolved `inner_new` with `unwrap`. That was near total against a local dataset, but construction counts the base table — an HTTP round trip for a remote one — so a transient network or auth failure became a panic across the PyO3 boundary. ### Tests End-to-end `permutation_builder` and `StreamingDataset` runs against a mock server, the former torch-free so it runs wherever the suite does, plus a test that a build succeeds without an LSM write spec and is refused once one is installed. --- python/python/lancedb/permutation.py | 18 +- .../python/tests/test_elastic_dataloader.py | 29 +++ python/python/tests/test_permutation.py | 59 +++++ python/python/tests/utils.py | 206 ++++++++++++++++++ python/src/permutation.rs | 4 +- .../src/dataloader/permutation/builder.rs | 104 ++++++++- .../src/dataloader/permutation/reader.rs | 22 +- rust/lancedb/src/remote/table.rs | 13 ++ rust/lancedb/src/table.rs | 13 ++ 9 files changed, 441 insertions(+), 27 deletions(-) diff --git a/python/python/lancedb/permutation.py b/python/python/lancedb/permutation.py index bcf84bf3a..5d7a685ef 100644 --- a/python/python/lancedb/permutation.py +++ b/python/python/lancedb/permutation.py @@ -41,21 +41,15 @@ class PermutationBuilder: The permutation is stored in memory and will be lost when the program exits. """ - def __init__(self, table: LanceTable): + def __init__(self, table: Table): """ Creates a new permutation builder for the given table. By default, the permutation builder will create a single split that contains all rows in the same order as the base table. + + Tables with an LSM write spec are rejected: unflushed rows have no row id. """ - if not hasattr(table, "_inner"): - raise TypeError( - f"PermutationBuilder requires a local LanceTable, " - f"got {type(table).__name__}. " - "The permutation API is not supported on remote tables. " - "Remote tables connect to LanceDB Cloud or Enterprise and do not have " - "direct access to the underlying Lance dataset needed for permutations." - ) self._async = async_permutation_builder(table) def split_random( @@ -231,7 +225,7 @@ class PermutationBuilder: return LOOP.run(do_execute()) -def permutation_builder(table: LanceTable) -> PermutationBuilder: +def permutation_builder(table: Table) -> PermutationBuilder: return PermutationBuilder(table) @@ -248,7 +242,7 @@ class Permutations: Attributes ---------- - base_table: LanceTable + base_table: Table The base table that the permutations are based on. permutation_table: LanceTable The permutation table that defines the splits. @@ -282,7 +276,7 @@ class Permutations: {'train': 0, 'test': 1} """ - def __init__(self, base_table: LanceTable, permutation_table: LanceTable): + def __init__(self, base_table: Table, permutation_table: LanceTable): self.base_table = base_table self.permutation_table = permutation_table diff --git a/python/python/tests/test_elastic_dataloader.py b/python/python/tests/test_elastic_dataloader.py index 734f835c6..0c1a70765 100644 --- a/python/python/tests/test_elastic_dataloader.py +++ b/python/python/tests/test_elastic_dataloader.py @@ -37,6 +37,11 @@ from unittest.mock import patch import lancedb import pyarrow as pa import pytest +from utils import ( + MockPermutationServer, + assert_server_safe_row_id_requests, + mock_remote_table, +) torch = pytest.importorskip("torch") streaming = pytest.importorskip("lancedb.streaming") @@ -2118,3 +2123,27 @@ def test_doc_example_checkpoint(lance_table): assert sorted(consumed + remaining_original) == list(range(NUM_ROWS)), ( "Consumed + remaining must cover every row exactly once" ) + + +# --------------------------------------------------------------------------- +# Remote tables (LanceDB Cloud / Enterprise) +# --------------------------------------------------------------------------- + + +def test_streaming_dataset_over_remote_table(): + """StreamingDataset reads a remote table, with server-safe requests. + + Builds a permutation over a remote table, then fetches batches from it by row id. + """ + server = MockPermutationServer() + + with mock_remote_table(server) as table: + ds = StreamingDataset(table, num_splits=2, shuffle_seed=SHUFFLE_SEED) + ids = [row["id"] for row in ds] + + assert sorted(ids) == list(range(server.num_rows)), ( + "Every row of the remote table must be yielded exactly once" + ) + assert len(server.scans) == 1, "the permutation is built with one row-id scan" + assert server.takes, "rows must be fetched with row-id takes" + assert_server_safe_row_id_requests(server) diff --git a/python/python/tests/test_permutation.py b/python/python/tests/test_permutation.py index 6d8f6f431..135742c84 100644 --- a/python/python/tests/test_permutation.py +++ b/python/python/tests/test_permutation.py @@ -8,6 +8,11 @@ import pytest from lancedb import DBConnection, Table, connect from lancedb.background_loop import LOOP from lancedb.permutation import Permutation, Permutations, permutation_builder +from utils import ( + MockPermutationServer, + assert_server_safe_row_id_requests, + mock_remote_table, +) def test_split_random_ratios(mem_db): @@ -1214,3 +1219,57 @@ def test_remove_rowid_after_select(some_permutation: Permutation): perm_without_rowid = perm_with_rowid.remove_columns(["_rowid"]) assert "_rowid" not in perm_without_rowid.column_names assert perm_without_rowid.column_names == ["id"] + + +def test_permutation_is_stable_when_remote_scan_order_varies(): + """Splits are assigned by scan position, and every rank builds its own + permutation, so two ranks seeing different scan orders must still agree.""" + server = MockPermutationServer(num_rows=16, vary_scan_order=True) + + def split_of_each_row(permutation_tbl): + # Sequential splits are assigned by position, so a reversed scan would put + # the last rows in split 0. Compare the mapping rather than the table order, + # which the split-id sort does not pin down. + rows = permutation_tbl.search(None).to_arrow().to_pydict() + return dict(zip(rows["row_id"], rows["split_id"])) + + with mock_remote_table(server) as table: + first = split_of_each_row( + permutation_builder(table).split_sequential(fixed=2).execute() + ) + second = split_of_each_row( + permutation_builder(table).split_sequential(fixed=2).execute() + ) + + assert server.scan_calls == 2, "both builds must have scanned" + assert first == second + assert first[0] == 0 and first[server.num_rows - 1] == 1, first + + +def test_permutation_over_remote_table(): + """The permutation API accepts a remote table, addressing rows by `_rowid` just + as `take_row_ids` does. Also pins the request shapes sent to the server. + """ + server = MockPermutationServer() + + with mock_remote_table(server) as table: + permutation_tbl = permutation_builder(table).split_sequential(fixed=2).execute() + assert permutation_tbl.count_rows() == server.num_rows + + permutation = Permutation.from_tables(table, permutation_tbl, 0) + assert permutation.num_rows == server.num_rows // 2 + + # Compare against the permutation's own order; the split-id sort is not stable. + rows = permutation_tbl.search(None).to_arrow().to_pydict() + split0 = [ + row_id + for row_id, split in zip(rows["row_id"], rows["split_id"]) + if not split + ] + # The mock table's `id` equals its `_rowid`. + assert permutation.take_offsets([2, 0]) == [ + {"id": split0[2]}, + {"id": split0[0]}, + ] + + assert_server_safe_row_id_requests(server) diff --git a/python/python/tests/utils.py b/python/python/tests/utils.py index 62ec74497..4882f0c28 100644 --- a/python/python/tests/utils.py +++ b/python/python/tests/utils.py @@ -1,7 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import contextlib +import http.server +import json +import re +import threading + +import lancedb +import pyarrow as pa import pytest +ARROW_FILE_CONTENT_TYPE = "application/vnd.apache.arrow.file" + def exception_output(e_info: pytest.ExceptionInfo): import traceback @@ -9,3 +19,199 @@ def exception_output(e_info: pytest.ExceptionInfo): # skip traceback part, since it's not worth checking in tests lines = traceback.format_exception_only(e_info.type, e_info.value) return "".join(lines).strip() + + +def parse_in_list(filter_sql: str) -> list[int]: + """Pull the integers out of a `IN (a, b, c)` predicate. + + Scoped to the parenthesised list so a cast in the SQL adds no phantom values. + """ + match = re.search(r"\bIN\s*\(([^)]*)\)", filter_sql, re.IGNORECASE) + assert match is not None, f"expected an IN list, got: {filter_sql}" + return [int(m) for m in re.findall(r"-?\d+", match.group(1))] + + +def is_row_id_take(body) -> bool: + """True when a query body fetches specific rows by row id.""" + return "_rowid" in (body.get("filter") or "") + + +def arrow_file_bytes(table: pa.Table) -> bytes: + """Serialize to the Arrow IPC *file* framing the /query/ route answers with.""" + sink = pa.BufferOutputStream() + with pa.ipc.new_file(sink, table.schema) as writer: + writer.write_table(table) + return sink.getvalue().to_pybytes() + + +class MockPermutationServer: + """A stand-in LanceDB server hosting one table whose ``id`` equals its ``_rowid``. + + Records every ``/query/`` body so tests can assert on the request shapes sent to + the server, which is the part that has to stay compatible. + """ + + def __init__(self, name="remote_data", num_rows=8, vary_scan_order=False): + self.name = name + self.num_rows = num_rows + self.query_bodies = [] + # Stand in for a distributed scan that answers in no fixed order. + self.vary_scan_order = vary_scan_order + self.scan_calls = 0 + + def __call__(self, request): + path = request.path + if path == f"/v1/table/{self.name}/describe/": + return self._json( + request, + { + "version": 1, + "schema": { + "fields": [ + {"name": "id", "type": {"type": "int64"}, "nullable": False} + ] + }, + }, + ) + if path == f"/v1/table/{self.name}/get_lsm_write_spec/": + self._read_body(request) + # Null spec: this table has no LSM write path. + return self._json(request, {"lsm_write_spec": None}) + if path == f"/v1/table/{self.name}/count_rows/": + self._read_body(request) + return self._json(request, self.num_rows) + if path == f"/v1/table/{self.name}/query/": + return self._query(request, self._read_body(request)) + + # Drain first, so an unexpected route cannot desync a keep-alive connection. + self._read_body(request) + request.send_response(404) + request.end_headers() + + @property + def scans(self): + """Bodies of the permutation build scan: the row id column, nothing else.""" + return [b for b in self.query_bodies if b.get("columns") == ["_rowid"]] + + @property + def takes(self): + """Bodies of the row-id takes the loader fetches batches with. + + Keyed on `_rowid`, not "has a filter": the schema probe also has a predicate. + """ + return [b for b in self.query_bodies if is_row_id_take(b)] + + @staticmethod + def _read_body(request): + content_len = int(request.headers.get("Content-Length") or 0) + return json.loads(request.rfile.read(content_len)) if content_len else {} + + @staticmethod + def _json(request, payload): + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(json.dumps(payload).encode()) + + @staticmethod + def _arrow(request, table): + body = arrow_file_bytes(table) + request.send_response(200) + request.send_header("Content-Type", ARROW_FILE_CONTENT_TYPE) + request.send_header("Content-Length", str(len(body))) + request.end_headers() + request.wfile.write(body) + + def _query(self, request, body): + self.query_bodies.append(body) + + if is_row_id_take(body): + # A row-id take. Answer ascending, so tests prove the client reorders. + row_ids = sorted(parse_in_list(body["filter"])) + return self._arrow( + request, + pa.table( + { + "id": pa.array(row_ids, pa.int64()), + "_rowid": pa.array(row_ids, pa.uint64()), + } + ), + ) + + if body.get("columns") == ["_rowid"]: + # The permutation build scan: row ids and nothing else. + row_ids = list(range(self.num_rows)) + if self.vary_scan_order and self.scan_calls % 2: + row_ids.reverse() + self.scan_calls += 1 + return self._arrow( + request, + pa.table({"_rowid": pa.array(row_ids, pa.uint64())}), + ) + + # The schema probe: filtered to nothing, so it carries schema and no rows. + return self._arrow(request, pa.table({"id": pa.array([], pa.int64())})) + + +def _make_handler(serve): + class MockLanceDBHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + serve(self) + + def do_POST(self): + serve(self) + + def log_message(self, *args): + pass # keep pytest output readable + + return MockLanceDBHandler + + +@contextlib.contextmanager +def mock_remote_table(server): + """Run ``server`` on a local port and yield an open remote table against it. + + Threading: the loader fans out fetch threads a single-threaded server would + serialize, hiding the prefetch overlap under test. + """ + with http.server.ThreadingHTTPServer( + ("localhost", 0), _make_handler(server) + ) as srv: + thread = threading.Thread(target=srv.serve_forever) + thread.start() + try: + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=f"http://localhost:{srv.server_address[1]}", + client_config={"timeout_config": {"connect_timeout": 5}}, + ) + yield db.open_table(server.name) + finally: + srv.shutdown() + thread.join() + + +def assert_server_safe_row_id_requests(server): + """Assert the loader fetched rows by row id and bounded everything else. + + `.get`, not `[...]`, so a dropped field reads as the assertion, not a KeyError. + """ + for body in server.takes: + # The fetch needs the row id back to restore the requested order. + assert body.get("with_row_id") is True, body + assert "_rowid" in body["filter"], body + + # Only the one-off permutation scan may scan the whole table; the schema probe is + # built once per split per epoch. `k == 0` counts as unbounded: lance reads a zero + # limit as "no limit". + def is_unbounded(body): + if is_row_id_take(body): + return False + k = body.get("k") + return k is None or k == 0 or k > server.num_rows + + unbounded = [b for b in server.query_bodies if is_unbounded(b)] + assert unbounded == server.scans, ( + f"only the permutation scan may be unbounded, got {unbounded}" + ) diff --git a/python/src/permutation.rs b/python/src/permutation.rs index 4dc49cfd3..24ca2b5a3 100644 --- a/python/src/permutation.rs +++ b/python/src/permutation.rs @@ -268,7 +268,9 @@ impl PyPermutationReader { .await .infer_error()? } else { - PermutationReader::identity(base_table).await + PermutationReader::identity(base_table) + .await + .infer_error()? }; Ok(Self::from_reader(reader)) }) diff --git a/rust/lancedb/src/dataloader/permutation/builder.rs b/rust/lancedb/src/dataloader/permutation/builder.rs index 6b4ae2303..a3700d0ef 100644 --- a/rust/lancedb/src/dataloader/permutation/builder.rs +++ b/rust/lancedb/src/dataloader/permutation/builder.rs @@ -160,9 +160,10 @@ impl PermutationBuilder { self } - async fn sort_by_split_id( + async fn sort_by_column( &self, data: SendableRecordBatchStream, + column: &str, ) -> Result { let memory_limit = std::env::var("LANCEDB_PERM_BUILDER_MEMORY_LIMIT") .unwrap_or_else(|_| DEFAULT_MEMORY_LIMIT.to_string()) @@ -188,25 +189,26 @@ impl PermutationBuilder { let df = ctx .read_one_shot(data.into_df_stream()) .map_err(|e| Error::Other { - message: format!("Failed to setup sort by split id: {}", e), + message: format!("Failed to setup sort by {}: {}", column, e), source: Some(e.into()), })?; let df_stream = df - .sort_by(vec![col(SPLIT_ID_COLUMN)]) + .sort_by(vec![col(column)]) .map_err(|e| Error::Other { - message: format!("Failed to plan sort by split id: {}", e), + message: format!("Failed to plan sort by {}: {}", column, e), source: Some(e.into()), })? .execute_stream() .await .map_err(|e| Error::Other { - message: format!("Failed to sort by split id: {}", e), + message: format!("Failed to sort by {}: {}", column, e), source: Some(e.into()), })?; + let column = column.to_string(); let schema = df_stream.schema(); - let stream = df_stream.map_err(|e| Error::Other { - message: format!("Failed to execute sort by split id: {}", e), + let stream = df_stream.map_err(move |e| Error::Other { + message: format!("Failed to execute sort by {}: {}", column, e), source: Some(e.into()), }); Ok(Box::pin(SimpleRecordBatchStream { schema, stream })) @@ -238,7 +240,25 @@ impl PermutationBuilder { /// Builds the permutation table and stores it in the given database. pub async fn build(self) -> Result
{ - // First pass, apply filter and load row ids + // Unflushed rows have no row id, so a permutation cannot address them. + match self.base_table.base_table().get_lsm_write_spec().await { + Ok(Some(_)) => { + return Err(Error::NotSupported { + message: "the data loader does not support tables with an LSM write \ + spec: rows that have not been flushed to the base table \ + have no row id, so a permutation cannot reference them" + .to_string(), + }); + } + Ok(None) => {} + // No LSM write path means no spec. + Err(Error::NotSupported { .. }) => {} + Err(err) => return Err(err), + } + + // First pass, apply filter and load row ids. `Shuffler` permutes positions, so + // every rank must scan the rows in the same order to build the same permutation. + // TODO: pin the version resolved here; remote does not implement Lazy. let mut rows = self.base_table.query().select(Select::columns(&[ROW_ID])); if let Some(filter) = &self.config.filter { @@ -263,6 +283,12 @@ impl PermutationBuilder { // Apply splits let rows = rows.execute().await?; + // Splits are assigned by position, so the scan has to arrive in a fixed order. + let rows = if self.base_table.base_table().scan_order_is_deterministic() { + rows + } else { + self.sort_by_column(rows, ROW_ID).await? + }; let split_data = splitter.apply(rows, num_rows).await?; // Shuffle data if requested @@ -284,7 +310,7 @@ impl PermutationBuilder { needs_sort |= !matches!(self.config.shuffle_strategy, ShuffleStrategy::None); let sorted = if needs_sort { - self.sort_by_split_id(shuffled).await? + self.sort_by_column(shuffled, SPLIT_ID_COLUMN).await? } else { shuffled }; @@ -367,6 +393,22 @@ mod tests { ); } + #[tokio::test] + async fn test_native_scan_order_is_deterministic() { + let temp_dir = tempfile::tempdir().unwrap(); + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(10), BatchCount::from(1)); + let table = db.create_table("t", data).execute().await.unwrap(); + + // Native tables skip the canonicalizing sort; remote does not. + assert!(table.base_table().scan_order_is_deterministic()); + } + #[tokio::test] async fn test_permutation_builder() { let temp_dir = tempfile::tempdir().unwrap(); @@ -416,4 +458,48 @@ mod tests { 283 ); } + + /// Rows that have not been flushed to the base table have no row id, so a + /// permutation cannot reference them. Reading the base table alone would drop + /// them from training without saying so, so the table is refused instead. + #[tokio::test] + async fn test_permutation_rejects_lsm_write_spec() { + use crate::table::LsmWriteSpec; + use arrow_array::{Int32Array, RecordBatchIterator}; + use arrow_schema::{DataType, Field, Schema}; + + // MemWAL needs a real dataset directory and a non-nullable primary key. + let temp_dir = tempfile::tempdir().unwrap(); + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let schema = Arc::new(Schema::new(vec![Field::new("idx", DataType::Int32, false)])); + let batch = arrow_array::RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![0, 1, 2, 3]))], + ) + .unwrap(); + let reader: Box = + Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema.clone())); + let table = db.create_table("tbl", reader).execute().await.unwrap(); + + // Without a spec the build succeeds. + PermutationBuilder::new(table.clone()) + .build() + .await + .unwrap(); + + table.set_unenforced_primary_key(["idx"]).await.unwrap(); + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + + let err = PermutationBuilder::new(table).build().await.unwrap_err(); + assert!( + err.to_string().contains("LSM write spec"), + "expected the pre-check to refuse the table, got: {err}" + ); + } } diff --git a/rust/lancedb/src/dataloader/permutation/reader.rs b/rust/lancedb/src/dataloader/permutation/reader.rs index afe79b0ad..6da92e986 100644 --- a/rust/lancedb/src/dataloader/permutation/reader.rs +++ b/rust/lancedb/src/dataloader/permutation/reader.rs @@ -97,8 +97,10 @@ impl PermutationReader { Self::inner_new(base_table, Some(permutation_table), split).await } - pub async fn identity(base_table: Arc) -> Self { - Self::inner_new(base_table, None, 0).await.unwrap() + /// A reader over the base table in storage order, with no permutation. + /// Fallible because construction counts the base table. + pub async fn identity(base_table: Arc) -> Result { + Self::inner_new(base_table, None, 0).await } /// Validates the limit and offset and returns the number of rows that will be read @@ -487,7 +489,13 @@ impl PermutationReader { pub async fn output_schema(&self, selection: Select) -> Result { let table = Table::from(self.base_table.clone()); - table.query().select(selection).output_schema().await + // limit(1) because some table types execute the query to get its schema + table + .query() + .select(selection) + .limit(1) + .output_schema() + .await } pub fn count_rows(&self) -> u64 { @@ -779,7 +787,9 @@ mod tests { .into_mem_table("tbl", RowCount::from(10), BatchCount::from(1)) .await; - let reader = PermutationReader::identity(base_table.base_table().clone()).await; + let reader = PermutationReader::identity(base_table.base_table().clone()) + .await + .unwrap(); // With no permutation table, take_offsets uses the base table directly let offsets = vec![0, 2, 4, 6]; @@ -961,7 +971,9 @@ mod tests { .into_mem_table("tbl", RowCount::from(10), BatchCount::from(1)) .await; - let reader = PermutationReader::identity(base_table.base_table().clone()).await; + let reader = PermutationReader::identity(base_table.base_table().clone()) + .await + .unwrap(); let batch = reader.take_offsets(&[], Select::All).await.unwrap(); diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 14b869d90..f98d4c8dd 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -4303,6 +4303,19 @@ mod tests { write_ipc_stream_uncompressed(&one_row_blob_batch(column)) } + #[tokio::test] + async fn test_remote_scan_order_is_not_deterministic() { + // A distributed scan answers in no fixed order, so callers that assign meaning + // to row position have to sort for themselves. + let table = Table::new_with_handler("my_table", |_| { + http::Response::builder() + .status(200) + .body(Vec::new()) + .unwrap() + }); + assert!(!table.base_table().scan_order_is_deterministic()); + } + #[tokio::test] async fn test_fetch_blobs_sends_the_checked_out_version() { let ipc = one_row_blob_ipc_stream("image"); diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index e1dd942db..096d60345 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -789,6 +789,13 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { async fn checkout_tag(&self, tag: &str) -> Result<()>; /// Checkout the latest version of the table. async fn checkout_latest(&self) -> Result<()>; + /// Whether repeated identical scans return rows in the same order. + /// + /// Callers that assign meaning to a row's position must order the results + /// themselves when this is false. Defaults to false so a table type opts in. + fn scan_order_is_deterministic(&self) -> bool { + false + } /// Restore the table to the currently checked out version. async fn restore(&self) -> Result<()>; /// List the versions of the table. @@ -2996,6 +3003,12 @@ impl BaseTable for NativeTable { self } + /// Lance scans fragments in order (`Scanner::ordered` defaults to true, and we + /// never clear it), so repeated identical scans agree. + fn scan_order_is_deterministic(&self) -> bool { + true + } + fn name(&self) -> &str { self.name.as_str() } From 29822306d2be9b1e8fe683b38b80d2440a708627 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 21 Aug 2026 14:20:03 -0700 Subject: [PATCH 14/58] fix(python): skip the unrunnable FunctionVersion doctest example (#4014) The example binds an undefined `function`; only its last line was skipped, so the doctest suite fails on main and on every PR. --- python/python/lancedb/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 3b2d05951..9532ab8b9 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -281,7 +281,7 @@ class FunctionVersion(_RemoteValue): Examples -------- >>> from lancedb import col - >>> application = function( + >>> application = function( # doctest: +SKIP ... title=col("title"), ... body=col("body"), ... ).rename(columns={ From a35f7044ee8273d3746e7e1256449d6831f4518d Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:43:04 -0700 Subject: [PATCH 15/58] test(rust): cover Azure table URI separators on Windows (#3810) ## Summary - add cross-platform regression coverage for Azure table URI construction - assert that az:// database paths always produce forward-slash blob keys ## Root cause ListingDatabase previously used the host filesystem Path join operation for object-store URIs, which inserted a backslash on Windows. The URI construction was corrected in #2575, but the original Azure report had no regression coverage and remained open. ## Validation - cargo fmt --all - cargo test --quiet --features remote -p lancedb test_table_uri_uses_forward_slashes_for_azure - cargo check --quiet --features remote --tests --examples - cargo clippy --quiet --features remote --tests --examples Fixes #2283 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/database/listing.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index 5ebbe6c5c..d8d687c7c 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -2569,6 +2569,21 @@ mod tests { } } + /// Regression test for https://github.com/lancedb/lancedb/issues/2283. + /// + /// Object-store URIs must use `/` on every platform. In particular, joining + /// with `std::path::Path` used to insert a `\\` into Azure blob keys on + /// Windows. + #[tokio::test] + async fn test_table_uri_uses_forward_slashes_for_azure() { + let (_tempdir, mut db) = setup_database().await; + db.uri = "az://test/db/test".to_string(); + + let uri = db.table_uri("test").unwrap(); + + assert_eq!(uri, "az://test/db/test/test.lance"); + } + /// Regression: connecting via a URL-style URI (which goes through /// `url::Url::parse` and the `query_pairs_mut()` path) must not /// append a trailing `?` to per-table URIs when the input URI has From c7cb0b9afa8f4807fb29c1af370348818416d231 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:13:49 -0700 Subject: [PATCH 16/58] docs(python): clarify threading on two-CPU containers (#3807) ## Summary - document that current LanceDB releases use one compute worker without warning on two-vCPU containers - distinguish compute-worker tuning from storage I/O concurrency - direct users of affected LanceDB 0.21.1 installations to upgrade and link the current threading guidance ## Root cause The Lance version bundled with LanceDB 0.21.1 warned whenever the detected CPU count was less than or equal to its default two-core I/O reservation. A two-vCPU deployment therefore emitted the warning on every query even though falling back to one compute worker was the intended behavior. Lance fixed that warning condition upstream in lance-format/lance#3710, and LanceDB current main already pins a version containing the runtime fix; the Python package documentation did not explain the corrected behavior or the distinct thread controls. ## Validation - `git diff --check` - verified the linked Lance threading-model documentation returns HTTP 200 Fixes #2326 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/python/README.md b/python/README.md index 550698500..3a81c486a 100644 --- a/python/README.md +++ b/python/README.md @@ -38,6 +38,25 @@ Stable releases are created about every 2 weeks. For the latest features and bug pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb ``` +### Threading in CPU-limited containers + +LanceDB uses separate pools for compute work and storage I/O. On a container with +two visible CPUs, current releases intentionally use one compute worker by default; +no manual configuration is needed. If every query logs an I/O core reservation +warning on a two-CPU container, upgrade from LanceDB 0.21.1 or earlier. + +The two commonly tuned environment variables control different resources: + +- `LANCE_CPU_THREADS` overrides the number of compute workers. One worker is the + appropriate setting for a two-CPU container when an explicit override is needed. +- `LANCE_IO_THREADS` controls concurrent storage operations, not reserved CPU + cores. Its default can be greater than the number of CPUs because I/O workers + spend much of their time waiting for storage. + +Keep the defaults unless measurements show that the workload benefits from an +override. See the [Lance threading model](https://lance.org/guide/performance/#threading-model) +for the current defaults and tuning guidance. + ## Usage ### Basic Example From 01679e37fdefcbf12ce42c7b2ac579ad6098d8fd Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 21 Aug 2026 16:17:03 -0700 Subject: [PATCH 17/58] feat: materialized view declarations on local tables (#3930) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A materialized view is a table whose contents are defined by a query over one source table and maintained by refresh rather than by writes. The declaration half: create_materialized_view(name, source) resolves a projected, filtered and limited definition against the source schema -- output types come from the DataFusion planner, never the caller -- and commits an empty table carrying it as kind-tagged JSON in schema metadata. The tag lets a kind added later read back as a view this version cannot refresh rather than as a plain table. Views open and list as ordinary tables. Sources must have stable row ids, checked here because the property cannot be enabled later: each view row records its source row in __source_row_id, and that provenance survives compactions, updates and deletes only when row ids are stable. A view inherits the metadata describing its columns and none governing how a table is written, so blob markers carry through while declarations its always-nullable fields would contradict are stripped. Embedding configuration is rewritten to the view's column names, and dropped where it does not project both ends of a function. Stack created with GitHub Stacks CLI • Give Feedback 💬 --- rust/lancedb/src/connection.rs | 2 +- rust/lancedb/src/database/listing.rs | 302 +++- rust/lancedb/src/database/namespace.rs | 202 ++- rust/lancedb/src/error.rs | 2 + rust/lancedb/src/lib.rs | 2 + rust/lancedb/src/materialized_view.rs | 1992 ++++++++++++++++++++++++ rust/lancedb/src/remote/table.rs | 14 + rust/lancedb/src/table.rs | 5 + rust/lancedb/src/table/refresh.rs | 2 +- 9 files changed, 2406 insertions(+), 117 deletions(-) create mode 100644 rust/lancedb/src/materialized_view.rs diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 8935855a8..187e2df0e 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -41,7 +41,7 @@ use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider}; mod create_table; -fn merge_storage_options( +pub(crate) fn merge_storage_options( store_params: &mut ObjectStoreParams, pairs: impl IntoIterator, ) { diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index d8d687c7c..c9e9bcb22 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -765,60 +765,13 @@ impl ListingDatabase { } } - /// Extract storage option overrides from the request - fn extract_storage_overrides( - &self, - request: &CreateTableRequest, - ) -> Result<(Option, Option, Option)> { - let storage_options = request - .write_options - .lance_write_params - .as_ref() - .and_then(|p| p.store_params.as_ref()) - .and_then(|sp| sp.storage_options()); - - let storage_version_override = storage_options - .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION)) - .map(|s| s.parse::()) - .transpose()?; - - let v2_manifest_override = storage_options - .and_then(|opts| opts.get(OPT_NEW_TABLE_V2_MANIFEST_PATHS)) - .map(|s| s.parse::()) - .transpose() - .map_err(|_| Error::InvalidInput { - message: "enable_v2_manifest_paths must be a boolean".to_string(), - })?; - - let stable_row_ids_override = storage_options - .and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS)) - .map(|s| s.parse::()) - .transpose() - .map_err(|_| Error::InvalidInput { - message: "enable_stable_row_ids must be a boolean".to_string(), - })?; - - Ok(( - storage_version_override, - v2_manifest_override, - stable_row_ids_override, - )) - } - /// Prepare write parameters for table creation fn prepare_write_params( &self, request: &CreateTableRequest, - storage_version_override: Option, - v2_manifest_override: Option, - stable_row_ids_override: Option, + mut write_params: lance::dataset::WriteParams, + overrides: NewTableConfig, ) -> lance::dataset::WriteParams { - let mut write_params = request - .write_options - .lance_write_params - .clone() - .unwrap_or_default(); - // Only modify the storage options if we actually have something to // inherit. There is a difference between storage_options=None and // storage_options=Some({}). Using storage_options=None will cause the @@ -842,18 +795,21 @@ impl ListingDatabase { store_params.storage_options_accessor = Some(Arc::new(accessor)); } - write_params.data_storage_version = storage_version_override + write_params.data_storage_version = overrides + .data_storage_version .or(write_params.data_storage_version) .or(self.new_table_config.data_storage_version); - if let Some(enable_v2_manifest_paths) = - v2_manifest_override.or(self.new_table_config.enable_v2_manifest_paths) + if let Some(enable_v2_manifest_paths) = overrides + .enable_v2_manifest_paths + .or(self.new_table_config.enable_v2_manifest_paths) { write_params.enable_v2_manifest_paths = enable_v2_manifest_paths; } let data_schema = request.data.arrow_schema(); - if let Some(enable_stable_row_ids) = stable_row_ids_override + if let Some(enable_stable_row_ids) = overrides + .enable_stable_row_ids .or(self.new_table_config.enable_stable_row_ids) .or(has_blob_columns(&data_schema).then_some(true)) { @@ -1048,15 +1004,13 @@ impl Database for ListingDatabase { .clone() .unwrap_or_else(|| self.table_uri(&request.name).unwrap()); - let (storage_version_override, v2_manifest_override, stable_row_ids_override) = - self.extract_storage_overrides(&request)?; - - let write_params = self.prepare_write_params( - &request, - storage_version_override, - v2_manifest_override, - stable_row_ids_override, - ); + let mut write_params = request + .write_options + .lance_write_params + .clone() + .unwrap_or_default(); + let overrides = take_request_creation_overrides(&mut write_params)?; + let write_params = self.prepare_write_params(&request, write_params, overrides); let data_schema = request.data.arrow_schema(); @@ -1288,8 +1242,232 @@ impl Database for ListingDatabase { } } +/// Parse the request-level `new_table_*` creation keys into overrides and +/// strip them from the store options in one step: every create path that +/// honors them must also keep them out of the object store. +pub(crate) fn take_request_creation_overrides( + params: &mut lance::dataset::WriteParams, +) -> Result { + let storage_options = params + .store_params + .as_ref() + .and_then(|sp| sp.storage_options()); + let overrides = NewTableConfig { + data_storage_version: storage_options + .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION)) + .map(|s| s.parse::()) + .transpose()?, + enable_v2_manifest_paths: storage_options + .and_then(|opts| opts.get(OPT_NEW_TABLE_V2_MANIFEST_PATHS)) + .map(|s| s.parse::()) + .transpose() + .map_err(|_| Error::InvalidInput { + message: "enable_v2_manifest_paths must be a boolean".to_string(), + })?, + enable_stable_row_ids: storage_options + .and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS)) + .map(|s| s.parse::()) + .transpose() + .map_err(|_| Error::InvalidInput { + message: "enable_stable_row_ids must be a boolean".to_string(), + })?, + }; + if let Some(store_params) = params.store_params.as_mut() { + strip_new_table_creation_keys(store_params); + } + Ok(overrides) +} + +/// Strip the `new_table_*` creation keys from request store options: they are +/// creation config, not credentials, and left in place they fork a fresh +/// store connection for the request. +fn strip_new_table_creation_keys(store_params: &mut ObjectStoreParams) { + let mut options = store_params.storage_options().cloned().unwrap_or_default(); + let mut removed = false; + for key in [ + OPT_NEW_TABLE_STORAGE_VERSION, + OPT_NEW_TABLE_V2_MANIFEST_PATHS, + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, + ] { + removed |= options.remove(key).is_some(); + } + if !removed { + return; + } + let provider = store_params + .storage_options_accessor + .as_ref() + .and_then(|accessor| accessor.provider().cloned()); + store_params.storage_options_accessor = match (options.is_empty(), provider) { + (true, None) => None, + (true, Some(provider)) => Some(Arc::new(StorageOptionsAccessor::with_provider(provider))), + (false, Some(provider)) => Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider(options, provider), + )), + (false, None) => Some(Arc::new(StorageOptionsAccessor::with_static_options( + options, + ))), + }; +} + #[cfg(test)] mod tests { + #[tokio::test] + async fn request_level_creation_keys_do_not_fork_the_store() { + use crate::query::ExecutableQuery; + use futures::TryStreamExt; + + let db = crate::connect("memory://").execute().await.unwrap(); + let batch = arrow_array::record_batch!(("x", Int32, [1, 2])).unwrap(); + let store_params = ObjectStoreParams { + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + HashMap::from([( + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(), + "true".to_string(), + )]), + ))), + ..Default::default() + }; + db.create_table("t", batch) + .write_options(crate::table::WriteOptions { + lance_write_params: Some(lance::dataset::WriteParams { + store_params: Some(store_params), + ..Default::default() + }), + }) + .execute() + .await + .unwrap(); + + let table = db.open_table("t").execute().await.unwrap(); + let rows: usize = table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap() + .iter() + .map(|b| b.num_rows()) + .sum(); + assert_eq!(rows, 2, "the table must live in the session's store"); + } + + mod strip_new_table_creation_keys { + use super::super::*; + + #[derive(Debug)] + struct EmptyProvider; + + #[async_trait::async_trait] + impl StorageOptionsProvider for EmptyProvider { + async fn fetch_storage_options( + &self, + ) -> lance_core::Result>> { + Ok(Some(HashMap::new())) + } + + fn provider_id(&self) -> String { + "empty-test-provider".into() + } + } + + fn params_with_static(options: &[(&str, &str)]) -> ObjectStoreParams { + ObjectStoreParams { + storage_options_accessor: Some(Arc::new( + StorageOptionsAccessor::with_static_options( + options + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ), + )), + ..Default::default() + } + } + + #[test] + fn creation_keys_are_removed_and_store_keys_kept() { + let mut params = params_with_static(&[ + ("region", "us-west-2"), + (OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true"), + ]); + strip_new_table_creation_keys(&mut params); + let options = params.storage_options().cloned().unwrap(); + assert_eq!(options.get("region").map(String::as_str), Some("us-west-2")); + assert!(!options.contains_key(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS)); + + // Creation keys alone: no accessor survives to fork a store. + let mut params = params_with_static(&[(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true")]); + strip_new_table_creation_keys(&mut params); + assert!(params.storage_options_accessor.is_none()); + } + + /// A provider must survive every shape of strip: untouched accessors + /// keep their identity, emptied ones still fetch, and residual + /// statics ride along. + #[test] + fn provider_accessors_survive_the_strip() { + let accessor = Arc::new(StorageOptionsAccessor::with_provider(Arc::new( + EmptyProvider, + ))); + let mut params = ObjectStoreParams { + storage_options_accessor: Some(accessor.clone()), + ..Default::default() + }; + strip_new_table_creation_keys(&mut params); + assert!(Arc::ptr_eq( + params.storage_options_accessor.as_ref().unwrap(), + &accessor + )); + + let mut params = ObjectStoreParams { + storage_options_accessor: Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider( + HashMap::from([ + ("region".to_string(), "us-west-2".to_string()), + ( + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(), + "true".to_string(), + ), + ]), + Arc::new(EmptyProvider), + ), + )), + ..Default::default() + }; + strip_new_table_creation_keys(&mut params); + let accessor = params.storage_options_accessor.unwrap(); + assert!(accessor.has_provider()); + assert_eq!( + accessor + .initial_storage_options() + .and_then(|o| o.get("region").cloned()) + .as_deref(), + Some("us-west-2") + ); + + // Emptied entirely: a first-fetch accessor, not one caching {}. + let mut params = ObjectStoreParams { + storage_options_accessor: Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider( + HashMap::from([( + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(), + "true".to_string(), + )]), + Arc::new(EmptyProvider), + ), + )), + ..Default::default() + }; + strip_new_table_creation_keys(&mut params); + let accessor = params.storage_options_accessor.unwrap(); + assert!(accessor.has_provider()); + assert!(accessor.initial_storage_options().is_none()); + } + } + use super::*; use crate::Table; use crate::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream}; diff --git a/rust/lancedb/src/database/namespace.rs b/rust/lancedb/src/database/namespace.rs index 740e11645..250d933f6 100644 --- a/rust/lancedb/src/database/namespace.rs +++ b/rust/lancedb/src/database/namespace.rs @@ -26,10 +26,7 @@ use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler; use crate::blob::{ensure_blob_storage_version, has_blob_columns}; use crate::connection::NamespaceClientPushdownOperation; use crate::database::ReadConsistency; -use crate::database::listing::{ - NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, OPT_NEW_TABLE_STORAGE_VERSION, - OPT_NEW_TABLE_V2_MANIFEST_PATHS, -}; +use crate::database::listing::{NewTableConfig, take_request_creation_overrides}; use crate::database::read_freshness::{ FreshnessBaselines, ReadFreshnessContextProvider, TableFreshness, }; @@ -197,69 +194,28 @@ impl LanceNamespaceDatabase { TableFreshness::new(self.freshness_baselines.clone(), key) } - fn extract_storage_overrides( - &self, - request: &DbCreateTableRequest, - ) -> Result<( - Option, - Option, - Option, - )> { - let storage_options = request - .write_options - .lance_write_params - .as_ref() - .and_then(|p| p.store_params.as_ref()) - .and_then(|sp| sp.storage_options()); - - let storage_version_override = storage_options - .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION)) - .map(|s| s.parse::()) - .transpose()?; - - let v2_manifest_override = storage_options - .and_then(|opts| opts.get(OPT_NEW_TABLE_V2_MANIFEST_PATHS)) - .map(|s| s.parse::()) - .transpose() - .map_err(|_| Error::InvalidInput { - message: "enable_v2_manifest_paths must be a boolean".to_string(), - })?; - - let stable_row_ids_override = storage_options - .and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS)) - .map(|s| s.parse::()) - .transpose() - .map_err(|_| Error::InvalidInput { - message: "enable_stable_row_ids must be a boolean".to_string(), - })?; - - Ok(( - storage_version_override, - v2_manifest_override, - stable_row_ids_override, - )) - } - fn apply_new_table_config( &self, params: &mut lance::dataset::WriteParams, request: &DbCreateTableRequest, ) -> Result<()> { - let (storage_version_override, v2_manifest_override, stable_row_ids_override) = - self.extract_storage_overrides(request)?; + let overrides = take_request_creation_overrides(params)?; - params.data_storage_version = storage_version_override + params.data_storage_version = overrides + .data_storage_version .or(params.data_storage_version) .or(self.new_table_config.data_storage_version); - if let Some(enable_v2_manifest_paths) = - v2_manifest_override.or(self.new_table_config.enable_v2_manifest_paths) + if let Some(enable_v2_manifest_paths) = overrides + .enable_v2_manifest_paths + .or(self.new_table_config.enable_v2_manifest_paths) { params.enable_v2_manifest_paths = enable_v2_manifest_paths; } let data_schema = request.data.schema(); - if let Some(enable_stable_row_ids) = stable_row_ids_override + if let Some(enable_stable_row_ids) = overrides + .enable_stable_row_ids .or(self.new_table_config.enable_stable_row_ids) .or(has_blob_columns(data_schema.as_ref()).then_some(true)) { @@ -644,6 +600,146 @@ mod tests { RecordBatch::try_new(schema, vec![Arc::new(id_array), Arc::new(name_array)]).unwrap() } + /// The shared parse-and-sanitize boundary is wired into this path: the + /// request-level creation key must act as an override (the strip itself + /// is covered by the listing tests). + #[tokio::test] + async fn request_level_creation_keys_are_taken_as_overrides() { + use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS; + + let tmp_dir = tempdir().unwrap(); + let mut properties = HashMap::new(); + properties.insert( + "root".to_string(), + tmp_dir.path().to_str().unwrap().to_string(), + ); + let db = connect_namespace("dir", properties) + .execute() + .await + .unwrap(); + + let store_params = ObjectStoreParams { + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + HashMap::from([( + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(), + "true".to_string(), + )]), + ))), + ..Default::default() + }; + let table = db + .create_table("t", create_test_data()) + .write_options(crate::table::WriteOptions { + lance_write_params: Some(lance::dataset::WriteParams { + store_params: Some(store_params), + ..Default::default() + }), + }) + .execute() + .await + .unwrap(); + let native = table.as_native().unwrap(); + assert!( + native + .dataset + .get() + .await + .unwrap() + .manifest + .uses_stable_row_ids(), + "the creation key must be honored as an override" + ); + + let table = db.open_table("t").execute().await.unwrap(); + let rows: usize = table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap() + .iter() + .map(|b| b.num_rows()) + .sum(); + assert_eq!(rows, 5); + } + + /// Sanitation on this path: apply must strip the creation keys from the + /// store options while genuine options and the provider survive. + #[tokio::test] + async fn apply_new_table_config_sanitizes_request_store_options() { + use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS; + use lance_io::object_store::StorageOptionsProvider; + + #[derive(Debug)] + struct EmptyProvider; + + #[async_trait::async_trait] + impl StorageOptionsProvider for EmptyProvider { + async fn fetch_storage_options( + &self, + ) -> lance_core::Result>> { + Ok(Some(HashMap::new())) + } + + fn provider_id(&self) -> String { + "empty-test-provider".into() + } + } + + let tmp_dir = tempdir().unwrap(); + let mut properties = HashMap::new(); + properties.insert( + "root".to_string(), + tmp_dir.path().to_str().unwrap().to_string(), + ); + let db = LanceNamespaceDatabase::connect_with_new_table_config( + "dir", + properties, + HashMap::new(), + None, + None, + HashSet::new(), + NewTableConfig::default(), + ) + .await + .unwrap(); + + let request = DbCreateTableRequest::new("t".to_string(), Box::new(create_test_data())); + let mut params = lance::dataset::WriteParams { + store_params: Some(ObjectStoreParams { + storage_options_accessor: Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider( + HashMap::from([ + ("region".to_string(), "us-west-2".to_string()), + ( + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(), + "true".to_string(), + ), + ]), + Arc::new(EmptyProvider), + ), + )), + ..Default::default() + }), + ..Default::default() + }; + db.apply_new_table_config(&mut params, &request).unwrap(); + + assert!(params.enable_stable_row_ids); + let store_params = params.store_params.unwrap(); + let options = store_params.storage_options().cloned().unwrap(); + assert!(!options.contains_key(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS)); + assert_eq!(options.get("region").map(String::as_str), Some("us-west-2")); + assert!( + store_params + .storage_options_accessor + .unwrap() + .has_provider() + ); + } + #[tokio::test] async fn test_namespace_connection_simple() { // Test that namespace connections work with simple connect_namespace(impl_type, properties) diff --git a/rust/lancedb/src/error.rs b/rust/lancedb/src/error.rs index 6bd1ffa2b..be4641388 100644 --- a/rust/lancedb/src/error.rs +++ b/rust/lancedb/src/error.rs @@ -77,6 +77,8 @@ pub enum Error { ColumnAlreadyExists { name: String }, #[snafu(display("Column '{name}' is not a computed column"))] NotAComputedColumn { name: String }, + #[snafu(display("Table '{name}' is not a materialized view"))] + NotAMaterializedView { name: String }, #[snafu(display("Invalid expression for column '{column}': {message}"))] InvalidExpression { column: String, message: String }, diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index 291dcaf65..3a937db5b 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -186,6 +186,7 @@ pub mod index; pub mod io; pub mod ipc; pub mod job; +pub mod materialized_view; #[cfg(feature = "metrics-otel")] pub mod metrics_otel; #[cfg(feature = "polars")] @@ -210,6 +211,7 @@ pub use function::FunctionVersion; pub use job::Job; use lance_index::vector::ApproxMode as LanceApproxMode; use lance_linalg::distance::DistanceType as LanceDistanceType; +pub use materialized_view::{MaterializedView, MaterializedViewDefinition}; /// Re-export of the [`metrics`](https://docs.rs/metrics) crate facade. Enable /// the `metrics` feature to publish LanceDB's internal metrics; install any /// `metrics`-compatible recorder to collect them. See also [`metrics_otel`] for diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs new file mode 100644 index 000000000..f1eddf9da --- /dev/null +++ b/rust/lancedb/src/materialized_view.rs @@ -0,0 +1,1992 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Materialized views. +//! +//! A materialized view is a table whose contents are defined by a query over +//! one source table and maintained by refresh rather than by writes. Creation +//! commits an empty table carrying the kind-tagged definition in schema +//! metadata; a kind added later reads back as unrefreshable, not as a plain +//! table. Queries, indexes and search work on the view unchanged. + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_schema::{DataType, Field as ArrowField, FieldRef, Schema as ArrowSchema, SchemaRef}; +use datafusion_common::ScalarValue; +use lance_core::ROW_ID; +use lance_datafusion::planner::Planner; +use serde::{Deserialize, Serialize}; + +use crate::connection::Connection; +use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS; +use crate::database::{CreateTableRequest, Database, OpenTableRequest}; +use crate::embeddings::EmbeddingDefinition; +use crate::table::Table; +use crate::table::refresh::quote_identifier; +use crate::table::{ColumnDefinition, ColumnKind}; +use crate::{Error, Result}; + +/// Schema metadata key holding the view definition, as kind-tagged JSON. +pub const DEFINITION_META_KEY: &str = "mv.definition"; + +/// Schema metadata key holding the source table version the view was last +/// refreshed to. Absent until the first refresh. +pub const SOURCE_VERSION_META_KEY: &str = "mv.source_version"; + +/// Schema metadata key holding the wall-clock time of the last refresh, +/// in milliseconds since the epoch. +pub const REFRESHED_AT_MS_META_KEY: &str = "mv.refreshed_at_ms"; + +/// Column recording which source row produced each view row: the source's +/// stable `_rowid` at refresh time, which is why sources must keep stable +/// row ids. +pub const SOURCE_ROW_ID_COLUMN: &str = "__source_row_id"; + +/// Field metadata namespace for declarations about schema structure, such as +/// an unenforced primary key. +const SCHEMA_DECLARATION_META_PREFIX: &str = "lance-schema:"; + +/// A field's identity in its own schema, which is not the view's. +const LANCE_FIELD_ID_KEY: &str = "lance:field_id"; + +/// Schema metadata key holding embedding-function configuration. It describes +/// columns rather than storage, so a view carries it through. +const EMBEDDING_FUNCTIONS_META_KEY: &str = "embedding_functions"; + +/// Schema metadata key holding lancedb's own column definitions, one per +/// field in schema order. It marks which columns an embedding function +/// produces, which is what lets a query embed its own text. +const COLUMN_DEFINITIONS_META_KEY: &str = "lancedb::column_definitions"; + +/// Value of the definition's `kind` tag for the projected `select` form. +pub const SELECT_KIND: &str = "select"; + +/// Which view outputs each source column is projected to directly. A column +/// may be projected more than once, so each carries every name the view gives +/// it, in projection order. +type Lineage = HashMap>; + +/// One projected output column of a view. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ViewProjection { + /// Name of the column in the view. + pub output: String, + /// SQL expression over the source table that computes it. + pub expression: String, +} + +/// The query that defines a materialized view. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MaterializedViewDefinition { + /// Name of the source table, in the same database as the view. + pub source_table: String, + /// The projected output columns, in view schema order. + pub projections: Vec, + /// SQL predicate selecting the source rows the view holds. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub filter: Option, + /// Cap on the number of rows the view holds, in materialization order. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, + /// Source columns the projections and filter read, derived at creation. + #[serde(default)] + pub inputs: Vec, +} + +/// A view definition as read back from schema metadata. Non-exhaustive so a +/// kind added later is additive. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum MaterializedViewKind { + /// The projected `select` form. + Select(MaterializedViewDefinition), + /// A kind written by a newer version, reported so a caller can tell an + /// unrefreshable view apart from a plain table. Nothing produces this. + Unrecognized { + /// The kind as it was found in the metadata. + kind: String, + }, +} + +/// Serialize `definition` into the kind-tagged form stored under +/// [`DEFINITION_META_KEY`]. +pub(crate) fn definition_to_metadata(definition: &MaterializedViewDefinition) -> Result { + let mut value = serde_json::to_value(definition).map_err(|e| Error::Runtime { + message: format!("failed to serialize view definition: {e}"), + })?; + value["kind"] = serde_json::Value::String(SELECT_KIND.to_string()); + Ok(value.to_string()) +} + +/// Read a view declaration off a schema metadata map, if it carries one. +/// `Ok(None)` for a plain table; a declaration that does not parse is an +/// error, because treating a view as plain would let it be rewritten. +pub fn materialized_view_kind( + metadata: &HashMap, +) -> Result> { + let Some(raw) = metadata.get(DEFINITION_META_KEY) else { + return Ok(None); + }; + let unreadable = |e: &dyn std::fmt::Display| Error::Runtime { + message: format!("unreadable materialized view definition: {e}"), + }; + let value: serde_json::Value = serde_json::from_str(raw).map_err(|e| unreadable(&e))?; + let kind = value + .get("kind") + .and_then(|k| k.as_str()) + .ok_or_else(|| unreadable(&"missing kind tag"))?; + if kind != SELECT_KIND { + return Ok(Some(MaterializedViewKind::Unrecognized { + kind: kind.to_string(), + })); + } + let definition = serde_json::from_value(value).map_err(|e| unreadable(&e))?; + Ok(Some(MaterializedViewKind::Select(definition))) +} + +/// Resolve a definition against the source schema into the view's projected +/// fields, with `inputs` filled in. Everything statically checkable is +/// checked here rather than at refresh time. Empty `projections` selects +/// every source column as the schema stands now. +pub(crate) fn plan( + source_schema: SchemaRef, + source_table: &str, + projections: &[(String, String)], + filter: Option<&str>, + limit: Option, +) -> Result<(MaterializedViewDefinition, Vec, Lineage)> { + let projections: Vec<(String, String)> = if projections.is_empty() { + source_schema + .fields() + .iter() + // A source that is itself a view carries its own provenance + // column; the new view records its own, not a copy. + .filter(|f| f.name() != SOURCE_ROW_ID_COLUMN) + .map(|f| (f.name().clone(), quote_identifier(f.name()))) + .collect() + } else { + projections.to_vec() + }; + + // A scan takes the cap as i64. Rejecting it here keeps creation and + // refresh from disagreeing about whether a view is valid. + if let Some(limit) = limit + && i64::try_from(limit).is_err() + { + return Err(Error::InvalidInput { + message: format!("view limit {limit} exceeds the maximum of {}", i64::MAX), + }); + } + + let planner = Planner::new(source_schema.clone()); + let mut fields = Vec::with_capacity(projections.len()); + let mut inputs = Vec::new(); + let mut declared: Vec<&str> = Vec::with_capacity(projections.len()); + let mut lineage: Lineage = HashMap::new(); + + for (output, expression) in &projections { + if declared.contains(&output.as_str()) { + return Err(Error::ColumnAlreadyExists { + name: output.clone(), + }); + } + if output == SOURCE_ROW_ID_COLUMN || output == ROW_ID { + return Err(Error::InvalidInput { + message: format!("view column name '{output}' is reserved"), + }); + } + + let parsed = planner + .parse_expr(expression) + .map_err(|e| Error::InvalidExpression { + column: output.clone(), + message: e.to_string(), + })?; + // Before optimization: the simplifier folds a stable-but-not-immutable + // call like now() into a literal, hiding it from the check while the + // stored definition keeps the call. + ensure_immutable(&parsed, |message| Error::InvalidExpression { + column: output.clone(), + message, + })?; + let expr = planner + .optimize_expr(parsed) + .map_err(|e| Error::InvalidExpression { + column: output.clone(), + message: e.to_string(), + })?; + let expr_inputs = + resolve_inputs(&source_schema, &expr, |message| Error::InvalidExpression { + column: output.clone(), + message, + })?; + + // Physical expressions address columns by position, so the planner + // that types the expression is built on the projected schema. + let read_schema = project_schema(&source_schema, &expr_inputs); + let physical = Planner::new(read_schema.clone()) + .create_physical_expr(&expr) + .map_err(|e| Error::InvalidExpression { + column: output.clone(), + message: e.to_string(), + })?; + let data_type = + physical + .data_type(read_schema.as_ref()) + .map_err(|e| Error::InvalidExpression { + column: output.clone(), + message: e.to_string(), + })?; + + // Always nullable: what a refresh appends must fit the declared field + // whatever nullability the evaluator reports for a given batch. + let mut field = ArrowField::new(output, data_type, true); + // Identity projections keep descriptive field metadata (blob markers); + // computed values carry none. Structural declarations never come along. + if let Some(source_field) = projected_field(&expr, &source_schema) { + field = field.with_metadata(source_field.metadata().clone()); + } + if let Some(path) = projected_path(&expr) + && let [column] = path.as_slice() + { + lineage + .entry(column.clone()) + .or_default() + .push(output.clone()); + } + fields.push(without_declarations(&field)); + inputs.extend(expr_inputs); + declared.push(output); + } + + if let Some(filter) = filter { + let expr = planner + .parse_filter(filter) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })?; + ensure_immutable(&expr, |message| Error::InvalidInput { + message: format!("invalid view filter: {message}"), + })?; + let filter_inputs = resolve_inputs(&source_schema, &expr, |message| Error::InvalidInput { + message: format!("invalid view filter: {message}"), + })?; + // A committed filter has to be usable as a predicate. + let read_schema = project_schema(&source_schema, &filter_inputs); + let data_type = Planner::new(read_schema.clone()) + .create_physical_expr(&expr) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })? + .data_type(read_schema.as_ref()) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })?; + if data_type != DataType::Boolean { + return Err(Error::InvalidInput { + message: format!("view filter must be a boolean predicate, not {data_type}"), + }); + } + inputs.extend(filter_inputs); + } + + inputs.sort(); + inputs.dedup(); + + let definition = MaterializedViewDefinition { + source_table: source_table.to_string(), + projections: projections + .into_iter() + .map(|(output, expression)| ViewProjection { output, expression }) + .collect(), + filter: filter.map(String::from), + limit, + inputs, + }; + Ok((definition, fields, lineage)) +} + +/// Reject any function that is not immutable: a view definition has to +/// evaluate identically across refreshes, or incremental maintenance would +/// mix rows from different evaluations of the same definition. +fn ensure_immutable(expr: &datafusion_expr::Expr, error: impl Fn(String) -> Error) -> Result<()> { + use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; + use datafusion_expr::Volatility; + + // Labeled immutable but not determined by row values alone: version() + // depends on the build, the arrow_* introspectors on schema state. + const NOT_VALUE_DETERMINED: &[&str] = + &["version", "arrow_typeof", "arrow_field", "arrow_metadata"]; + + let mut offending: Option = None; + expr.apply(|node| { + if let datafusion_expr::Expr::ScalarFunction(function) = node { + let name = function.func.name(); + if function.func.signature().volatility != Volatility::Immutable + || NOT_VALUE_DETERMINED.contains(&name) + { + offending = Some(name.to_string()); + return Ok(TreeNodeRecursion::Stop); + } + } + Ok(TreeNodeRecursion::Continue) + }) + .map_err(|e| error(e.to_string()))?; + match offending { + Some(name) => Err(error(format!( + "function '{name}' is not immutable and would evaluate differently \ + across refreshes" + ))), + None => Ok(()), + } +} + +/// The root of a possibly-dotted column path: `metadata.age` -> `metadata`. +fn root(path: &str) -> &str { + path.split('.').next().unwrap_or(path) +} + +/// The columns `expr` reads, kept as the planner reports them (a nested +/// reference stays a dotted path) but resolved by root field. +/// Embedding configuration rewritten for the view: entries whose columns the +/// view projects directly are kept under the view's names; the rest describe +/// a table that does not exist and are dropped. +fn embedding_config_for_view(raw: &str, lineage: &Lineage) -> Option { + // Every representation the writers use: the Python bindings name the + // destination `vector_column`, the Rust definition `dest_column`, and the + // Node bindings spell both halves in camelCase. + const SOURCE_KEYS: [&str; 2] = ["source_column", "sourceColumn"]; + const DEST_KEYS: [&str; 4] = ["vector_column", "dest_column", "vectorColumn", "destColumn"]; + + let entries: Vec = serde_json::from_str(raw).ok()?; + let mut kept = Vec::new(); + for entry in &entries { + let Some(object) = entry.as_object() else { + continue; + }; + let named = |keys: &[&str]| { + let key = keys.iter().find(|key| object.contains_key(**key))?; + let outputs = lineage.get(object.get(*key)?.as_str()?)?; + Some(((*key).to_string(), outputs)) + }; + let (Some((source_key, sources)), Some((dest_key, dests))) = + (named(&SOURCE_KEYS), named(&DEST_KEYS)) + else { + continue; + }; + // A projection may give one source column several names, and every + // pairing of the two is a real relationship in the view. + for source in sources { + for dest in dests { + let mut object = object.clone(); + object.insert(source_key.clone(), source.clone().into()); + object.insert(dest_key.clone(), dest.clone().into()); + kept.push(serde_json::Value::Object(object)); + } + } + } + (!kept.is_empty()).then(|| serde_json::Value::Array(kept).to_string()) +} + +/// Lancedb's column definitions rewritten for the view: positional, one per +/// view field. Directly projected embedding columns keep their definition +/// under the view's names; everything else is physical. `None` = no key. +fn column_definitions_for_view( + raw: &str, + source_schema: &ArrowSchema, + view_fields: &[ArrowField], + lineage: &Lineage, +) -> Option { + let source_definitions: Vec = serde_json::from_str(raw).ok()?; + // The definition sits on the column the function writes, so the source + // schema's field name at that position is the embedding's destination. + let embeddings: HashMap<&str, &EmbeddingDefinition> = source_schema + .fields() + .iter() + .zip(&source_definitions) + .filter_map(|(field, definition)| match &definition.kind { + ColumnKind::Embedding(embedding) => Some((field.name().as_str(), embedding)), + ColumnKind::Physical => None, + }) + .collect(); + let sources: HashMap<&str, &str> = lineage + .iter() + .flat_map(|(source, outputs)| outputs.iter().map(move |o| (o.as_str(), source.as_str()))) + .collect(); + + let mut kept = false; + let definitions: Vec = view_fields + .iter() + .map(|field| { + let kind = embedding_for_output(field.name(), &embeddings, &sources, lineage) + .map(|embedding| { + kept = true; + ColumnKind::Embedding(embedding) + }) + .unwrap_or(ColumnKind::Physical); + ColumnDefinition { kind } + }) + .collect(); + kept.then(|| serde_json::to_string(&definitions).ok())? +} + +/// The embedding `output` inherits, renamed to the view's columns. `None` +/// unless the view projects both the function's input and its output +/// directly: anything else advertises a column the view cannot recompute. +fn embedding_for_output( + output: &str, + embeddings: &HashMap<&str, &EmbeddingDefinition>, + sources: &HashMap<&str, &str>, + lineage: &Lineage, +) -> Option { + let embedding = embeddings.get(sources.get(output)?)?; + // The input may be projected several times; the first name the view gives + // it is the one this column is defined against. + let input = lineage.get(&embedding.source_column)?.first()?; + Some(EmbeddingDefinition { + source_column: input.clone(), + dest_column: Some(output.to_string()), + embedding_name: embedding.embedding_name.clone(), + }) +} + +/// The source field a projection reads directly, if it reads one: a bare +/// column, or a path of struct field accesses over one. Anything computed +/// produces a new value and has no source field. +fn projected_field<'a>( + expr: &datafusion_expr::Expr, + schema: &'a ArrowSchema, +) -> Option<&'a ArrowField> { + let path = projected_path(expr)?; + let mut segments = path.iter(); + let mut field = schema.field_with_name(segments.next()?).ok()?; + for segment in segments { + let DataType::Struct(children) = field.data_type() else { + return None; + }; + field = children.iter().find(|c| c.name() == segment)?; + } + Some(field) +} + +/// The dotted path a projection reads directly, root first. +fn projected_path(expr: &datafusion_expr::Expr) -> Option> { + let mut path = Vec::new(); + let mut node = expr; + loop { + match node { + datafusion_expr::Expr::Column(column) => { + path.push(column.name.clone()); + break; + } + // `a.b` parses to get_field(a, "b"), nested for deeper paths. + datafusion_expr::Expr::ScalarFunction(call) if call.func.name() == "get_field" => { + let [ + inner, + datafusion_expr::Expr::Literal(ScalarValue::Utf8(Some(name)), _), + ] = call.args.as_slice() + else { + return None; + }; + path.push(name.clone()); + node = inner; + } + _ => return None, + } + } + + path.reverse(); + Some(path) +} + +/// `field` without the metadata that declares how a column is written, at +/// every depth; descriptive metadata (blob markers) stays. A view is written +/// by refresh alone, and its always-nullable fields contradict declarations. +fn is_declaration(key: &str) -> bool { + key.starts_with(SCHEMA_DECLARATION_META_PREFIX) + || key == LANCE_FIELD_ID_KEY + || crate::table::computed_columns::is_declaration_key(key) +} + +fn without_declarations(field: &ArrowField) -> ArrowField { + let metadata: HashMap = field + .metadata() + .iter() + .filter(|(key, _)| !is_declaration(key)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + let strip = |child: &FieldRef| Arc::new(without_declarations(child)); + // Every Arrow variant that carries a field carries that field's metadata + // with it, so all of them are descended. + let data_type = match field.data_type() { + DataType::Struct(children) => DataType::Struct(children.iter().map(strip).collect()), + DataType::List(child) => DataType::List(strip(child)), + DataType::ListView(child) => DataType::ListView(strip(child)), + DataType::LargeList(child) => DataType::LargeList(strip(child)), + DataType::LargeListView(child) => DataType::LargeListView(strip(child)), + DataType::Map(entries, sorted) => DataType::Map(strip(entries), *sorted), + DataType::FixedSizeList(child, len) => DataType::FixedSizeList(strip(child), *len), + DataType::Union(variants, mode) => DataType::Union( + variants + .iter() + .map(|(id, child)| (id, strip(child))) + .collect(), + *mode, + ), + DataType::RunEndEncoded(run_ends, values) => { + DataType::RunEndEncoded(strip(run_ends), strip(values)) + } + other => other.clone(), + }; + ArrowField::new(field.name(), data_type, field.is_nullable()).with_metadata(metadata) +} + +fn resolve_inputs( + schema: &ArrowSchema, + expr: &datafusion_expr::Expr, + error: impl Fn(String) -> Error, +) -> Result> { + let mut inputs = Planner::column_names_in_expr(expr); + inputs.sort(); + inputs.dedup(); + for input in &inputs { + if schema.field_with_name(root(input)).is_err() { + return Err(error(format!("unknown column '{input}'"))); + } + } + Ok(inputs) +} + +/// Project the root fields of `columns`, deduplicated, in schema order. +fn project_schema(schema: &ArrowSchema, columns: &[String]) -> SchemaRef { + let roots: std::collections::HashSet<&str> = columns.iter().map(|c| root(c)).collect(); + let fields: Vec = schema + .fields() + .iter() + .filter(|f| roots.contains(f.name().as_str())) + .map(|f| f.as_ref().clone()) + .collect(); + Arc::new(ArrowSchema::new(fields)) +} + +/// A validated view declaration, ready to become a table: the projected +/// fields plus [`SOURCE_ROW_ID_COLUMN`], definition stamped in metadata. +/// Produced only by [`prepare_declaration`]. +#[derive(Clone)] +pub struct PreparedDeclaration { + schema: SchemaRef, + definition: MaterializedViewDefinition, + /// The source's own database: the only place + /// [`PreparedDeclaration::create`] will put the view, because refresh + /// resolves the recorded source name through the view's database. + database: Arc, +} + +impl std::fmt::Debug for PreparedDeclaration { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PreparedDeclaration") + .field("definition", &self.definition) + .finish_non_exhaustive() + } +} + +impl PreparedDeclaration { + /// The query the declaration records. + pub fn definition(&self) -> &MaterializedViewDefinition { + &self.definition + } + + /// Create the view table and verify it, consuming the declaration. + /// + /// The view goes in the source's own database, where refresh resolves the + /// recorded source name. Stable row ids are requested at both levels and + /// verified rather than trusted; nothing is rolled back on failure. + pub async fn create(self, name: &str) -> Result { + let empty: Vec> = + vec![]; + let reader: Box = + Box::new(arrow_array::RecordBatchIterator::new(empty, self.schema)); + let mut request = CreateTableRequest::new(name.to_string(), Box::new(reader)); + let write_params = request + .write_options + .lance_write_params + .get_or_insert_with(Default::default); + write_params.enable_stable_row_ids = true; + let store_params = write_params + .store_params + .get_or_insert_with(Default::default); + crate::connection::merge_storage_options( + store_params, + [( + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(), + "true".to_string(), + )], + ); + let table = self.database.clone().create_table(request).await?; + let table = Table::new(table, self.database); + let stable = match table.as_native() { + Some(native) => native.dataset.get().await?.manifest.uses_stable_row_ids(), + None => false, + }; + if !stable { + return Err(Error::Runtime { + message: format!( + "view '{name}' was created without stable row ids: the database \ + ignored the creation option; the table remains and is not \ + usable as a materialized view" + ), + }); + } + Ok(MaterializedView { + table, + definition: self.definition, + }) + } +} + +/// Validate a view declaration against its live source and hold what its +/// creation needs. The declaration is canonicalized through the coordinate a +/// refresh will resolve, so a handle that does not resolve back to itself is +/// rejected, as is a namespaced source. Same creation-time checks as +/// [`Connection::create_materialized_view`]. +/// +/// ```no_run +/// # #![recursion_limit = "256"] +/// # use lancedb::materialized_view::prepare_declaration; +/// # async fn declare(source: &lancedb::Table) -> Result<(), Box> { +/// let prepared = prepare_declaration( +/// source, +/// &[("id".into(), "id".into()), ("double".into(), "value * 2".into())], +/// Some("value > 0"), +/// None, +/// ) +/// .await?; +/// let view = prepared.create("doubles").await?; +/// # Ok(()) +/// # } +/// ``` +pub async fn prepare_declaration( + source: &Table, + projections: &[(String, String)], + filter: Option<&str>, + limit: Option, +) -> Result { + let Some(caller_native) = source.as_native() else { + return Err(Error::NotSupported { + message: "materialized views are supported only on local databases".into(), + }); + }; + // The definition records the source by bare name; any other source + // form would be recorded as a name its refresh cannot resolve. + if !source.namespace().is_empty() { + return Err(Error::NotSupported { + message: format!( + "a namespaced source cannot be recorded in a view definition; \ + '{}' must be a root-namespace table", + source.name() + ), + }); + } + let database = source + .database_opt() + .ok_or_else(|| Error::InvalidInput { + message: "the source was not opened through a database connection".into(), + })? + .clone(); + + // Canonicalize: resolve the recorded coordinate exactly the way a + // refresh will, and plan from what it reaches. A handle that does not + // resolve back to itself must not be declared under this name. + let resolved = database + .open_table(OpenTableRequest { + name: source.name().to_string(), + namespace_path: vec![], + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await?; + let resolved = Table::new(resolved, database.clone()); + let Some(native) = resolved.as_native() else { + return Err(Error::NotSupported { + message: "materialized views are supported only on local databases".into(), + }); + }; + let caller_uri = caller_native.dataset.get().await?.uri().to_string(); + let resolved_uri = native.dataset.get().await?.uri().to_string(); + if caller_uri != resolved_uri { + return Err(Error::InvalidInput { + message: format!( + "the source handle does not resolve to itself through its \ + database: '{}' resolves to '{resolved_uri}', but the handle \ + reads '{caller_uri}'", + source.name() + ), + }); + } + if !native.dataset.get().await?.manifest.uses_stable_row_ids() { + return Err(Error::InvalidInput { + message: format!( + "materialized views require stable row ids on the source table; \ + create '{}' with storage option new_table_enable_stable_row_ids=true", + source.name() + ), + }); + } + let source_schema = resolved.schema().await?; + let source_metadata = source_schema.metadata().clone(); + let (definition, mut fields, lineage) = plan( + source_schema.clone(), + resolved.name(), + projections, + filter, + limit, + )?; + fields.push(ArrowField::new( + SOURCE_ROW_ID_COLUMN, + DataType::UInt64, + false, + )); + // Only column-describing metadata comes along: structural declarations + // describe how a table is written, and a view is written by refresh alone. + let mut metadata: HashMap = HashMap::new(); + if let Some(raw) = source_metadata.get(EMBEDDING_FUNCTIONS_META_KEY) + && let Some(rewritten) = embedding_config_for_view(raw, &lineage) + { + metadata.insert(EMBEDDING_FUNCTIONS_META_KEY.to_string(), rewritten); + } + if let Some(raw) = source_metadata.get(COLUMN_DEFINITIONS_META_KEY) + && let Some(rewritten) = column_definitions_for_view(raw, &source_schema, &fields, &lineage) + { + metadata.insert(COLUMN_DEFINITIONS_META_KEY.to_string(), rewritten); + } + metadata.insert( + DEFINITION_META_KEY.to_string(), + definition_to_metadata(&definition)?, + ); + Ok(PreparedDeclaration { + schema: Arc::new(ArrowSchema::new_with_metadata(fields, metadata)), + definition, + database, + }) +} + +/// One row of [`Connection::list_materialized_views`]: a view's name and its +/// definition kind, which may be one this version cannot refresh. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MaterializedViewEntry { + /// Name of the view's table. + pub name: String, + /// The view's definition as stored. + pub kind: MaterializedViewKind, +} + +/// Materialized views are local-only; refuse a remote connection before any +/// request is made. +fn ensure_local(connection: &Connection) -> Result<()> { + if connection.uri().starts_with("db://") { + return Err(Error::NotSupported { + message: "materialized views are supported only on local databases".into(), + }); + } + Ok(()) +} + +/// Builds a materialized view. Created by +/// [`Connection::create_materialized_view`]. +pub struct CreateMaterializedViewBuilder { + connection: Connection, + name: String, + source: String, + projections: Vec<(String, String)>, + filter: Option, + limit: Option, +} + +impl CreateMaterializedViewBuilder { + pub(crate) fn new(connection: Connection, name: String, source: String) -> Self { + Self { + connection, + name, + source, + projections: Vec::new(), + filter: None, + limit: None, + } + } + + /// The view's columns, as `(name, SQL expression)` pairs. Not calling + /// this selects every source column, expanded at creation time. + pub fn select( + mut self, + columns: impl IntoIterator, impl Into)>, + ) -> Self { + self.projections = columns + .into_iter() + .map(|(output, expression)| (output.into(), expression.into())) + .collect(); + self + } + + /// Only source rows matching the SQL predicate appear in the view. + pub fn only_if(mut self, filter: impl Into) -> Self { + self.filter = Some(filter.into()); + self + } + + /// Cap the view at `limit` rows, in materialization order. + pub fn limit(mut self, limit: u64) -> Self { + self.limit = Some(limit); + self + } + + /// Create the view: an empty table carrying the definition; refresh + /// computes the rows. The source must keep stable row ids -- they hold + /// provenance across compaction, and cannot be enabled later. + pub async fn execute(self) -> Result { + ensure_local(&self.connection)?; + let source = self.connection.open_table(&self.source).execute().await?; + let prepared = prepare_declaration( + &source, + &self.projections, + self.filter.as_deref(), + self.limit, + ) + .await?; + prepared.create(&self.name).await + } +} + +/// A handle on a materialized view: the view table plus its parsed definition. +#[derive(Debug, Clone)] +pub struct MaterializedView { + table: Table, + definition: MaterializedViewDefinition, +} + +impl MaterializedView { + /// Interpret `table` as a materialized view: [`Error::NotAMaterializedView`] + /// for a plain table, [`Error::NotSupported`] for a kind this version + /// cannot refresh. + pub async fn from_table(table: Table) -> Result { + // Same local-only boundary the connection-level entry points hold, + // applied before the schema read so a remote table costs no request. + if table.as_native().is_none() { + return Err(Error::NotSupported { + message: "materialized views are supported only on local databases".into(), + }); + } + let schema = table.schema().await?; + match materialized_view_kind(schema.metadata())? { + Some(MaterializedViewKind::Select(definition)) => Ok(Self { table, definition }), + Some(MaterializedViewKind::Unrecognized { kind }) => Err(Error::NotSupported { + message: format!( + "materialized view '{}' is defined by '{kind}', which this version of \ + lancedb cannot refresh", + table.name() + ), + }), + None => Err(Error::NotAMaterializedView { + name: table.name().to_string(), + }), + } + } + + /// The view, as the table it is. Queries, indexes and search all apply. + pub fn table(&self) -> &Table { + &self.table + } + + /// The view's table name. + pub fn name(&self) -> &str { + self.table.name() + } + + /// The query that defines the view. + pub fn definition(&self) -> &MaterializedViewDefinition { + &self.definition + } +} + +impl Connection { + /// Define a materialized view named `name` over `source`. + /// + /// The view is created empty, with the definition recorded in its schema + /// metadata; refresh computes the rows. Local databases only. + /// + /// ```no_run + /// # use lancedb::Connection; + /// # async fn create(conn: &Connection) -> Result<(), Box> { + /// let view = conn + /// .create_materialized_view("loud_adults", "people") + /// .select([("name", "upper(name)"), ("age", "age")]) + /// .only_if("age >= 18") + /// .execute() + /// .await?; + /// println!("{}", view.definition().source_table); + /// # Ok(()) + /// # } + /// ``` + pub fn create_materialized_view( + &self, + name: impl Into, + source: impl Into, + ) -> CreateMaterializedViewBuilder { + CreateMaterializedViewBuilder::new(self.clone(), name.into(), source.into()) + } + + /// Open the materialized view named `name`. + pub async fn open_materialized_view( + &self, + name: impl Into, + ) -> Result { + ensure_local(self)?; + let table = self.open_table(name).execute().await?; + MaterializedView::from_table(table).await + } + + /// The materialized views in this database, unrefreshable kinds included. + /// Costs a table open per table; one that cannot be opened is skipped + /// rather than failing the listing. + pub async fn list_materialized_views(&self) -> Result> { + ensure_local(self)?; + let names = self.table_names().execute().await?; + let mut views = Vec::new(); + for name in names { + let Ok(table) = self.open_table(&name).execute().await else { + continue; + }; + let schema = table.schema().await?; + if let Some(kind) = materialized_view_kind(schema.metadata())? { + views.push(MaterializedViewEntry { name, kind }); + } + } + Ok(views) + } +} + +#[cfg(test)] +mod tests { + use arrow_array::record_batch; + + use super::*; + use crate::connect; + use crate::table::WriteOptions; + + async fn people_db() -> Connection { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!( + ("name", Utf8, ["ada", "grace", "alan"]), + ("age", Int32, [36, 85, 41]) + ) + .unwrap(); + conn.create_table("people", batch) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + conn + } + + /// Sources must keep stable row ids; see the create-time gate. + pub(super) fn stable_row_ids() -> WriteOptions { + WriteOptions { + lance_write_params: Some(lance::dataset::WriteParams { + enable_stable_row_ids: true, + ..Default::default() + }), + } + } + + /// The error a doomed declaration against `people` produces. + async fn declare_err( + cfg: impl FnOnce(CreateMaterializedViewBuilder) -> CreateMaterializedViewBuilder, + ) -> Error { + let conn = people_db().await; + cfg(conn.create_materialized_view("bad", "people")) + .execute() + .await + .unwrap_err() + } + + #[tokio::test] + async fn test_create_records_the_definition() { + let conn = people_db().await; + let view = conn + .create_materialized_view("adults", "people") + .select([("name", "name"), ("shout", "upper(name)")]) + .only_if("age >= 18") + .limit(10) + .execute() + .await + .unwrap(); + + assert_eq!(view.name(), "adults"); + assert_eq!( + view.definition(), + &MaterializedViewDefinition { + source_table: "people".into(), + projections: vec![ + ViewProjection { + output: "name".into(), + expression: "name".into() + }, + ViewProjection { + output: "shout".into(), + expression: "upper(name)".into() + }, + ], + filter: Some("age >= 18".into()), + limit: Some(10), + inputs: vec!["age".into(), "name".into()], + } + ); + + // The definition round-trips off the stored schema, not the handle. + let reopened = conn.open_materialized_view("adults").await.unwrap(); + assert_eq!(reopened.definition(), view.definition()); + } + + #[tokio::test] + async fn test_view_schema_is_derived_from_the_query() { + let conn = people_db().await; + let view = conn + .create_materialized_view("shapes", "people") + .select([("shout", "upper(name)"), ("next_age", "age + 1")]) + .execute() + .await + .unwrap(); + + let schema = view.table().schema().await.unwrap(); + assert_eq!( + schema.field_with_name("shout").unwrap().data_type(), + &DataType::Utf8 + ); + assert_eq!( + schema.field_with_name("next_age").unwrap().data_type(), + &DataType::Int32 + ); + assert_eq!( + schema + .field_with_name(SOURCE_ROW_ID_COLUMN) + .unwrap() + .data_type(), + &DataType::UInt64 + ); + assert_eq!(view.table().count_rows(None).await.unwrap(), 0); + } + + /// No projection selects every source column, expanded now: the schema + /// captured at creation is the definition. + #[tokio::test] + async fn test_default_projection_captures_the_source_schema() { + let conn = people_db().await; + let view = conn + .create_materialized_view("copy", "people") + .execute() + .await + .unwrap(); + assert_eq!( + view.definition() + .projections + .iter() + .map(|p| p.output.as_str()) + .collect::>(), + vec!["name", "age"] + ); + assert_eq!(view.definition().inputs, vec!["age", "name"]); + } + + #[tokio::test] + async fn test_unknown_column_fails_at_create_time() { + let conn = people_db().await; + let err = conn + .create_materialized_view("bad", "people") + .select([("x", "missing + 1")]) + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidExpression { column, .. } if column == "x")); + let names = conn.table_names().execute().await.unwrap(); + assert!(!names.contains(&"bad".to_string())); + } + + #[tokio::test] + async fn test_unknown_filter_column_fails_at_create_time() { + let err = declare_err(|b| b.only_if("missing > 1")).await; + assert!(matches!(err, Error::InvalidInput { message } if message.contains("missing"))); + } + + #[tokio::test] + async fn test_duplicate_output_is_rejected() { + let err = declare_err(|b| b.select([("dup", "age"), ("dup", "age + 1")])).await; + assert!(matches!(err, Error::ColumnAlreadyExists { name } if name == "dup")); + } + + #[tokio::test] + async fn test_reserved_output_name_is_rejected() { + let err = declare_err(|b| b.select([(SOURCE_ROW_ID_COLUMN, "age")])).await; + assert!(matches!(err, Error::InvalidInput { message } if message.contains("reserved"))); + } + + #[tokio::test] + async fn test_missing_source_fails() { + let conn = connect("memory://").execute().await.unwrap(); + let err = conn + .create_materialized_view("v", "nope") + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::TableNotFound { .. })); + } + + /// Provenance has to survive source compactions and updates, and stable + /// row ids cannot be enabled after a table exists -- so the requirement + /// is checked at the last moment the caller can still act on it. + #[tokio::test] + async fn test_source_without_stable_row_ids_is_refused() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("x", Int32, [1, 2])).unwrap(); + conn.create_table("plain", batch).execute().await.unwrap(); + + let err = conn + .create_materialized_view("v", "plain") + .execute() + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("stable row ids")) + ); + assert!( + !conn + .table_names() + .execute() + .await + .unwrap() + .contains(&"v".to_string()) + ); + } + + #[tokio::test] + async fn test_name_collision_fails() { + let conn = people_db().await; + let err = conn + .create_materialized_view("people", "people") + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::TableAlreadyExists { .. })); + } + + #[tokio::test] + async fn test_a_plain_table_is_not_a_view() { + let conn = people_db().await; + let table = conn.open_table("people").execute().await.unwrap(); + let err = MaterializedView::from_table(table).await.unwrap_err(); + assert!(matches!(err, Error::NotAMaterializedView { name } if name == "people")); + + let err = conn.open_materialized_view("people").await.unwrap_err(); + assert!(matches!(err, Error::NotAMaterializedView { .. })); + } + + /// The reason the kind is tagged: a definition written by a newer version + /// reads back as a view this one cannot refresh, not as a plain table. + #[tokio::test] + async fn test_unrecognized_kind_is_refused_by_name() { + let conn = people_db().await; + conn.create_materialized_view("v", "people") + .execute() + .await + .unwrap(); + let table = conn.open_table("v").execute().await.unwrap(); + table + .as_native() + .unwrap() + .replace_schema_metadata(HashMap::from([( + DEFINITION_META_KEY.to_string(), + r#"{"kind": "join"}"#.to_string(), + )])) + .await + .unwrap(); + + let err = conn.open_materialized_view("v").await.unwrap_err(); + assert!(matches!(err, Error::NotSupported { message } if message.contains("join"))); + } + + #[tokio::test] + async fn test_list_reports_views_and_only_views() { + let conn = people_db().await; + conn.create_materialized_view("adults", "people") + .only_if("age >= 18") + .execute() + .await + .unwrap(); + + let views = conn.list_materialized_views().await.unwrap(); + assert_eq!( + views.iter().map(|v| v.name.as_str()).collect::>(), + vec!["adults"] + ); + let MaterializedViewKind::Select(definition) = &views[0].kind else { + panic!("expected a select view"); + }; + assert_eq!(definition.filter.as_deref(), Some("age >= 18")); + } + + /// The creation option outranks a connection configured to create + /// unstable tables: the view still gets stable row ids, on the same + /// store (no fork -- the table must be reachable through the + /// connection afterwards). + #[tokio::test] + async fn test_view_is_stable_despite_connection_override() { + let conn = connect("memory://") + .storage_options([("new_table_enable_stable_row_ids", "false")]) + .execute() + .await + .unwrap(); + let batch = record_batch!(("x", Int32, [1])).unwrap(); + conn.create_table("src", batch) + .storage_option("new_table_enable_stable_row_ids", "true") + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap(); + let stable = view + .table() + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .manifest + .uses_stable_row_ids(); + assert!(stable); + conn.open_materialized_view("v").await.unwrap(); + } + + /// A committed filter has to be usable as a predicate. + #[tokio::test] + async fn test_non_boolean_filter_is_rejected() { + let err = declare_err(|b| b.only_if("age + 1")).await; + assert!(matches!(err, Error::InvalidInput { message } if message.contains("boolean"))); + } + + /// Nested references stay dotted paths; resolution is by root field. + #[tokio::test] + async fn test_struct_columns_can_be_declared() { + use arrow_array::{ArrayRef, Int32Array, StructArray}; + + let conn = connect("memory://").execute().await.unwrap(); + let ages = StructArray::from(vec![( + Arc::new(ArrowField::new("age", DataType::Int32, false)), + Arc::new(Int32Array::from(vec![36, 17])) as ArrayRef, + )]); + let batch = + arrow_array::RecordBatch::try_from_iter(vec![("metadata", Arc::new(ages) as ArrayRef)]) + .unwrap(); + conn.create_table("people", batch) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("ages", "people") + .select([("age", "metadata.age")]) + .only_if("metadata.age >= 18") + .execute() + .await + .unwrap(); + assert_eq!(view.definition().inputs, vec!["metadata.age"]); + let schema = view.table().schema().await.unwrap(); + assert_eq!( + schema.field_with_name("age").unwrap().data_type(), + &DataType::Int32 + ); + } + + /// A newer-kind view must not disappear from the listing. + #[tokio::test] + async fn test_unrecognized_kind_is_listed_with_its_kind() { + let conn = people_db().await; + conn.create_materialized_view("v", "people") + .execute() + .await + .unwrap(); + let table = conn.open_table("v").execute().await.unwrap(); + table + .as_native() + .unwrap() + .replace_schema_metadata(HashMap::from([( + DEFINITION_META_KEY.to_string(), + r#"{"kind": "join"}"#.to_string(), + )])) + .await + .unwrap(); + + let views = conn.list_materialized_views().await.unwrap(); + assert_eq!(views.len(), 1); + assert_eq!(views[0].name, "v"); + assert_eq!( + views[0].kind, + MaterializedViewKind::Unrecognized { + kind: "join".into() + } + ); + } + + /// Remote connections are refused before any request is made. + #[cfg(feature = "remote")] + #[tokio::test] + async fn test_remote_connection_is_refused_up_front() { + let conn = connect("db://nowhere") + .api_key("sk_test") + .region("us-east-1") + .execute() + .await + .unwrap(); + let err = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. })); + let err = conn.open_materialized_view("v").await.unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. })); + let err = conn.list_materialized_views().await.unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. })); + } + + /// A definition must evaluate identically across refreshes; anything + /// less makes incremental maintenance a mix of evaluations. + #[tokio::test] + async fn test_volatile_and_unstable_expressions_are_rejected() { + let conn = people_db().await; + for expression in [ + "random()", + "now()", + "version()", + "arrow_typeof(age)", + "arrow_metadata(age, 'k')", + ] { + let err = conn + .create_materialized_view("bad", "people") + .select([("x", expression)]) + .execute() + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidExpression { message, .. } + if message.contains("not immutable")), + "{expression} was not rejected" + ); + } + for filter in ["age > random() * 100", "age >= 0 and now() is not null"] { + let err = conn + .create_materialized_view("bad", "people") + .only_if(filter) + .execute() + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("not immutable")), + "{filter} was not rejected" + ); + } + } + + /// A column projected as itself stays the column it was: blob discovery + /// and the blob APIs key off field metadata, which a bare rebuild of the + /// field would drop. + #[tokio::test] + async fn test_identity_projection_keeps_field_metadata() { + let conn = connect("memory://").execute().await.unwrap(); + let schema = Arc::new(ArrowSchema::new_with_metadata( + vec![ + ArrowField::new("id", DataType::Int32, true), + crate::blob("payload", true), + ], + HashMap::new(), + )); + conn.create_empty_table("src", schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap(); + let view_schema = view.table().schema().await.unwrap(); + + let payload = view_schema.field_with_name("payload").unwrap(); + assert!( + crate::blob::is_blob(payload), + "default projection dropped the blob marker: {:?}", + payload.metadata() + ); + assert_eq!( + view.table().blob_columns().await.unwrap(), + vec!["payload".to_string()], + "blob discovery no longer finds the projected column" + ); + assert!(view_schema.metadata().contains_key(DEFINITION_META_KEY)); + // Structural declarations describe how a table is written; a view is + // written by refresh, and its fields are always nullable. + assert!(!view_schema.metadata().contains_key("lance:primary_key")); + + // A computed column is a new value and carries no source metadata. + let computed = conn + .create_materialized_view("c", "src") + .select([("payload", "payload"), ("n", "id + 1")]) + .execute() + .await + .unwrap(); + let computed_schema = computed.table().schema().await.unwrap(); + assert!(crate::blob::is_blob( + computed_schema.field_with_name("payload").unwrap() + )); + assert!( + computed_schema + .field_with_name("n") + .unwrap() + .metadata() + .is_empty() + ); + } + + /// A nested column projected straight through is still that column, and a + /// declaration buried in a struct child binds as hard as one on top. + #[tokio::test] + async fn test_nested_projection_metadata_and_declarations() { + let conn = connect("memory://").execute().await.unwrap(); + let payload = crate::blob("payload", true).with_metadata(HashMap::from([ + ("lance-encoding:blob".to_string(), "true".to_string()), + ( + "lance-schema:unenforced-primary-key".to_string(), + "0".to_string(), + ), + ])); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("meta", DataType::Struct(vec![payload].into()), true), + ])); + conn.create_empty_table("src", schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + // A nested path is a direct projection: the leaf's metadata comes with + // it, so the blob stays a blob rather than a plain struct. + let lifted = conn + .create_materialized_view("lifted", "src") + .select([("payload", "meta.payload")]) + .execute() + .await + .unwrap(); + let field = lifted.table().schema().await.unwrap(); + let field = field.field_with_name("payload").unwrap().clone(); + assert_eq!( + field.metadata().get("lance-encoding:blob"), + Some(&"true".to_string()), + "nested projection lost the leaf's metadata" + ); + assert!( + !field + .metadata() + .contains_key("lance-schema:unenforced-primary-key"), + "a structural declaration rode along" + ); + + // Projecting the struct whole must not carry the child's declaration + // out to a view whose fields are nullable. + let whole = conn + .create_materialized_view("whole", "src") + .select([("meta", "meta")]) + .execute() + .await + .unwrap(); + let schema = whole.table().schema().await.unwrap(); + let DataType::Struct(children) = schema.field_with_name("meta").unwrap().data_type() else { + panic!("meta is not a struct"); + }; + let child = children.iter().find(|c| c.name() == "payload").unwrap(); + assert!( + !child + .metadata() + .contains_key("lance-schema:unenforced-primary-key"), + "a nested declaration survived: {:?}", + child.metadata() + ); + assert_eq!( + child.metadata().get("lance-encoding:blob"), + Some(&"true".to_string()) + ); + } + + /// A map's entries are fields like any other, and a declaration on one + /// binds the view's writes just as hard as one on top. + #[tokio::test] + async fn test_map_declarations_are_stripped() { + let conn = connect("memory://").execute().await.unwrap(); + let value = + ArrowField::new("value", DataType::Utf8, false).with_metadata(HashMap::from([( + "lance-schema:unenforced-clustering-key:position".to_string(), + "1".to_string(), + )])); + let entries = ArrowField::new( + "entries", + DataType::Struct(vec![ArrowField::new("key", DataType::Utf8, false), value].into()), + false, + ); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "props", + DataType::Map(Arc::new(entries), false), + true, + )])); + conn.create_empty_table("src", schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("view", "src") + .execute() + .await + .unwrap(); + let schema = view.table().schema().await.unwrap(); + let DataType::Map(entries, _) = schema.field_with_name("props").unwrap().data_type() else { + panic!("props is not a map"); + }; + let DataType::Struct(children) = entries.data_type() else { + panic!("map entries are not a struct"); + }; + let value = children.iter().find(|c| c.name() == "value").unwrap(); + assert!( + !value + .metadata() + .contains_key("lance-schema:unenforced-clustering-key:position"), + "a declaration survived inside a map: {:?}", + value.metadata() + ); + } + + /// A computed column is declared by field metadata. Projecting one -- + /// as itself or under an alias -- must carry its description without its + /// declaration, which the target table would reject as foreign. + #[tokio::test] + async fn test_view_over_a_computed_column() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("id", Int32, [1, 2])).unwrap(); + let source = conn + .create_table("src", batch) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + source + .add_columns() + .computed("doubled", "id * 2") + .execute() + .await + .unwrap(); + + // Default projection reaches the computed column too. + let whole = conn + .create_materialized_view("whole", "src") + .execute() + .await + .unwrap(); + let schema = whole.table().schema().await.unwrap(); + let field = schema.field_with_name("doubled").unwrap(); + assert!( + !field + .metadata() + .keys() + .any(|k| k.starts_with("computed_column")), + "a computed-column declaration rode along: {:?}", + field.metadata() + ); + + // And under an alias. + conn.create_materialized_view("aliased", "src") + .select([("twice", "doubled")]) + .execute() + .await + .unwrap(); + } + + /// Embedding configuration names columns. It comes along only for the + /// columns a view actually projects, under the names the view gives them. + #[tokio::test] + async fn test_embedding_config_follows_the_projection() { + let config = r#"[{"name":"f","model":{},"source_column":"text","vector_column":"vec"}]"#; + let conn = connect("memory://").execute().await.unwrap(); + let schema = Arc::new(ArrowSchema::new_with_metadata( + vec![ + ArrowField::new("text", DataType::Utf8, true), + ArrowField::new("vec", DataType::Float32, true), + ], + HashMap::from([("embedding_functions".to_string(), config.to_string())]), + )); + conn.create_empty_table("src", schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let carried = |view: &MaterializedView| { + let view = view.table().clone(); + async move { + view.schema() + .await + .unwrap() + .metadata() + .get("embedding_functions") + .cloned() + } + }; + + // Both columns projected as themselves: kept as it stands. + let whole = conn + .create_materialized_view("whole", "src") + .execute() + .await + .unwrap(); + let kept = carried(&whole).await.expect("config dropped"); + assert!(kept.contains(r#""source_column":"text""#), "{kept}"); + assert!(kept.contains(r#""vector_column":"vec""#), "{kept}"); + + // Only the source column: the configuration names a vector column the + // view does not have, so it describes nothing and goes. + let partial = conn + .create_materialized_view("partial", "src") + .select([("text", "text")]) + .execute() + .await + .unwrap(); + assert_eq!(carried(&partial).await, None); + + // Renamed: the configuration follows the names the view uses. + let renamed = conn + .create_materialized_view("renamed", "src") + .select([("body", "text"), ("embedding", "vec")]) + .execute() + .await + .unwrap(); + let remapped = carried(&renamed).await.expect("config dropped"); + assert!(remapped.contains(r#""source_column":"body""#), "{remapped}"); + assert!( + remapped.contains(r#""vector_column":"embedding""#), + "{remapped}" + ); + + // The Node bindings spell the same configuration in camelCase, and + // the Rust definition names the destination `dest_column`. + for (config, source_key, dest_key) in [ + ( + r#"[{"name":"f","model":{},"sourceColumn":"text","vectorColumn":"vec"}]"#, + "sourceColumn", + "vectorColumn", + ), + ( + r#"[{"name":"f","model":{},"source_column":"text","dest_column":"vec"}]"#, + "source_column", + "dest_column", + ), + ] { + let schema = Arc::new(ArrowSchema::new_with_metadata( + vec![ + ArrowField::new("text", DataType::Utf8, true), + ArrowField::new("vec", DataType::Float32, true), + ], + HashMap::from([("embedding_functions".to_string(), config.to_string())]), + )); + let name = format!("src_{source_key}"); + conn.create_empty_table(&name, schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + let view = conn + .create_materialized_view(format!("v_{source_key}"), &name) + .select([("body", "text"), ("embedding", "vec")]) + .execute() + .await + .unwrap(); + let carried = carried(&view).await.expect("config dropped"); + assert!( + carried.contains(&format!(r#""{source_key}":"body""#)), + "{carried}" + ); + assert!( + carried.contains(&format!(r#""{dest_key}":"embedding""#)), + "{carried}" + ); + } + + // A computed column is not the source column under another name. + let computed = conn + .create_materialized_view("computed", "src") + .select([("body", "upper(text)"), ("embedding", "vec")]) + .execute() + .await + .unwrap(); + assert_eq!(carried(&computed).await, None); + + // One column projected twice is two columns in the view, and the + // configuration has to describe both rather than whichever came last. + let twice = conn + .create_materialized_view("twice", "src") + .select([("body", "text"), ("a", "vec"), ("b", "vec")]) + .execute() + .await + .unwrap(); + let carried = carried(&twice).await.expect("config dropped"); + let entries: Vec = serde_json::from_str(&carried).unwrap(); + let mut vectors: Vec<&str> = entries + .iter() + .filter_map(|e| e["vector_column"].as_str()) + .collect(); + vectors.sort_unstable(); + assert_eq!(vectors, ["a", "b"], "{carried}"); + } + + /// The native Rust producer records embeddings as column definitions + /// rather than as `embedding_functions`, and a query embeds its own text + /// through them. They are positional, so the view's list covers every one + /// of its fields. + #[tokio::test] + async fn test_native_column_definitions_follow_the_projection() { + let conn = connect("memory://").execute().await.unwrap(); + let rich = crate::table::TableDefinition::new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("text", DataType::Utf8, true), + ArrowField::new("vector", DataType::Float32, true), + ])), + vec![ + ColumnDefinition { + kind: ColumnKind::Physical, + }, + ColumnDefinition { + kind: ColumnKind::Embedding(EmbeddingDefinition::new( + "text", + "model", + Some("vector"), + )), + }, + ], + ) + .into_rich_schema(); + conn.create_empty_table("src", rich) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("view", "src") + .select([("body", "text"), ("embedding", "vector")]) + .execute() + .await + .unwrap(); + let schema = view.table().schema().await.unwrap(); + let raw = schema + .metadata() + .get(COLUMN_DEFINITIONS_META_KEY) + .expect("the view dropped the native column definitions"); + let definitions: Vec = serde_json::from_str(raw).unwrap(); + assert_eq!( + definitions.len(), + schema.fields().len(), + "column definitions are positional" + ); + let ColumnKind::Embedding(embedding) = &definitions[1].kind else { + panic!("the embedding column came back physical: {raw}"); + }; + assert_eq!(embedding.source_column, "body"); + assert_eq!(embedding.dest_column.as_deref(), Some("embedding")); + assert_eq!(embedding.embedding_name, "model"); + assert!(matches!(definitions[0].kind, ColumnKind::Physical)); + assert!(matches!(definitions[2].kind, ColumnKind::Physical)); + + // Without the column the function reads, the view cannot recompute + // the embedding, so it carries no definition for it. + let partial = conn + .create_materialized_view("partial", "src") + .select([("embedding", "vector")]) + .execute() + .await + .unwrap(); + assert_eq!( + partial + .table() + .schema() + .await + .unwrap() + .metadata() + .get(COLUMN_DEFINITIONS_META_KEY), + None + ); + } + + /// A scan takes the cap as i64, so a larger one is refused where it is + /// declared rather than at the refresh that cannot run it. What a cap of + /// zero means is a refresh question, tested there. + #[tokio::test] + async fn test_limit_above_i64_max_is_refused_at_creation() { + let conn = people_db().await; + let err = conn + .create_materialized_view("too_big", "people") + .limit(i64::MAX as u64 + 1) + .execute() + .await + .unwrap_err(); + assert!( + matches!(&err, Error::InvalidInput { message } if message.contains("exceeds the maximum")), + "got {err:?}" + ); + + // The boundary itself is accepted. + conn.create_materialized_view("at_max", "people") + .limit(i64::MAX as u64) + .execute() + .await + .unwrap(); + } + + #[tokio::test] + async fn test_drop_is_drop_table() { + let conn = people_db().await; + conn.create_materialized_view("v", "people") + .execute() + .await + .unwrap(); + conn.drop_table("v", &[]).await.unwrap(); + assert!(conn.list_materialized_views().await.unwrap().is_empty()); + } + + /// The public declaration contract: prepare validates the source and + /// create consumes the declaration into a verified view table; a + /// source that cannot anchor refresh and a target outside the + /// source's database are both refused. + #[tokio::test] + async fn prepare_and_create_bind_the_declaration_lifecycle() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("id", Int32, [1, 2]), ("value", Int32, [3, 4])).unwrap(); + let source = conn + .create_table("src", batch.clone()) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let projections = [ + ("id".to_string(), "id".to_string()), + ("double".to_string(), "value * 2".to_string()), + ]; + let prepared = prepare_declaration(&source, &projections, Some("value > 0"), None) + .await + .unwrap(); + assert_eq!(prepared.definition().source_table, "src"); + + let view = prepared.create("v").await.unwrap(); + let schema = view.table().schema().await.unwrap(); + let names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert_eq!(names, ["id", "double", SOURCE_ROW_ID_COLUMN]); + assert!(schema.metadata().contains_key(DEFINITION_META_KEY)); + + // The same call rejects a source without stable row ids, so an + // external creation path cannot skip the check. + conn.create_table("plain", batch).execute().await.unwrap(); + let plain = conn.open_table("plain").execute().await.unwrap(); + let err = prepare_declaration(&plain, &[], None, None) + .await + .unwrap_err(); + assert!(err.to_string().contains("stable row ids"), "{err}"); + + // A handle whose location does not resolve back through its name is + // refused: the definition would record a name reaching other data. + let plain_uri = plain + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .uri() + .to_string(); + let masquerade = conn + .open_table("src") + .location(plain_uri) + .execute() + .await + .unwrap(); + let err = prepare_declaration(&masquerade, &[], None, None) + .await + .unwrap_err(); + assert!( + err.to_string().contains("does not resolve to itself"), + "{err}" + ); + + // A table created at a custom location is refused outright: its + // recorded name reaches nothing at the database root, so the + // canonical reopen fails before any URI comparison. + let custom = conn + .create_table( + "custom_loc", + record_batch!(("id", Int32, [1, 2]), ("value", Int32, [3, 4])).unwrap(), + ) + .location("memory://elsewhere/custom_loc") + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + let err = prepare_declaration(&custom, &[], None, None) + .await + .unwrap_err(); + assert!(err.to_string().contains("custom_loc"), "{err}"); + + // A namespaced source cannot be recorded in the definition: the + // bare name refresh resolves would reach a different table or none. + let namespaced = crate::table::NativeTable::create( + "memory://ns_src", + "ns_src", + vec!["ns".to_string()], + Box::new(arrow_array::RecordBatchIterator::new( + vec![], + std::sync::Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new( + "id", + arrow_schema::DataType::Int32, + true, + )])), + )) as Box, + None, + None, + None, + None, + std::collections::HashSet::new(), + ) + .await + .unwrap(); + let namespaced = Table::new(std::sync::Arc::new(namespaced), conn.database().clone()); + let err = prepare_declaration(&namespaced, &[], None, None) + .await + .unwrap_err(); + assert!(err.to_string().contains("namespaced source"), "{err}"); + } +} diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index f98d4c8dd..6c446907d 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -10410,6 +10410,20 @@ mod tests { ); } + #[tokio::test] + async fn test_materialized_view_refused_without_a_request() { + // Materialized views are local-only. The table-level entry the + // bindings use must refuse a remote table before reading its schema, + // so the panicking handler is the assertion. + let table = Table::new_with_handler("my_table", |request| -> http::Response { + panic!("unexpected request: {}", request.url().path()) + }); + let err = crate::MaterializedView::from_table(table) + .await + .unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. }), "got {err:?}"); + } + #[tokio::test] async fn test_create_branch_empty_name_rejected_client_side() { use lance::dataset::refs::Ref; diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 096d60345..ca9ed8f26 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -1066,6 +1066,11 @@ impl Table { self.database.as_ref().unwrap() } + /// The database this handle was opened through, when it was. + pub fn database_opt(&self) -> Option<&Arc> { + self.database.as_ref() + } + pub fn embedding_registry(&self) -> &Arc { &self.embedding_registry } diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index a3f3da92f..2715a163c 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -193,7 +193,7 @@ fn declared_expression(dataset: &Dataset, column: &str) -> Result { /// /// Lance's dialect delimits with backticks, so a double-quoted name would /// parse as a string literal rather than a column. -fn quote_identifier(name: &str) -> String { +pub(crate) fn quote_identifier(name: &str) -> String { format!("`{}`", name.replace('`', "``")) } From 9e8f1c1a6dffeafcdebf218d4f9e7212f90917f6 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:31:34 -0700 Subject: [PATCH 18/58] fix(python): expose FTS build memory limits (#3796) ## Summary - expose `memory_limit` and `num_workers` on the Python FTS configuration for local builds - forward both build-only settings to the Lance inverted-index builder - add an end-to-end regression proving the configured memory budget reaches the native build ## Root cause LanceDB 0.26.1 pinned Lance 1.0.1. That Lance version used an FTS partition-merge path whose retained data made memory grow with merge progress on very large indexes. Upstream Lance [#5754](https://github.com/lance-format/lance/pull/5754) changed partition merging to stream its inputs, reducing peak memory by about 25%. Lance [#6174](https://github.com/lance-format/lance/pull/6174) then removed the old merge phase, compressed posting lists during construction, reduced indexing memory by about 60%, and introduced a total build `memory_limit` for bounded workers. Current `main` pins Lance 11.0.0-beta.3, which contains those architectural fixes. This PR does not duplicate or claim the upstream leak fix; it addresses the remaining Python API gap. ## This repair LanceDB Python did not expose the native FTS builder resource controls. `memory_limit` now sets the total local-build budget in MiB, divided among effective workers, and `num_workers` controls build parallelism. Both are build-only settings and do not affect remote builds or persisted index configuration. ## Validation - `cargo check --quiet --features remote --tests --examples` - `cargo fmt --all` - `uv run --project python --extra tests --extra dev ruff check .` - `uv run --project python --extra tests --extra dev ruff format --check python/python/lancedb/index.py python/python/tests/test_fts.py` - `uv run --project python --extra tests pytest python/tests/test_fts.py -q` (51 passed) Fixes #2923 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/Cargo.toml | 9 +++-- python/pyproject.toml | 2 +- python/python/lancedb/index.py | 11 +++++++ python/python/tests/test_fts.py | 8 +++++ python/src/index.rs | 58 ++++++++++++++++++++++++++++++++- 5 files changed, 81 insertions(+), 7 deletions(-) diff --git a/python/Cargo.toml b/python/Cargo.toml index 6181f704f..ddc13b69f 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -26,7 +26,9 @@ lance-namespace-impls.workspace = true lance-io.workspace = true env_logger.workspace = true log.workspace = true -pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "chrono"] } +# Maturin enables extension-module mode for Python builds. Keeping it out of +# Cargo features lets Rust unit tests link against libpython. +pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] } chrono.workspace = true pyo3-async-runtimes = { version = "0.28", features = [ "attributes", @@ -41,10 +43,7 @@ tokio.workspace = true libc = "0.2" [build-dependencies] -pyo3-build-config = { version = "0.28", features = [ - "extension-module", - "abi3-py310", -] } +pyo3-build-config = { version = "0.28", features = ["abi3-py310"] } [features] default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"] diff --git a/python/pyproject.toml b/python/pyproject.toml index ae42172c0..fad3b1001 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -103,7 +103,7 @@ python-source = "python" module-name = "lancedb._lancedb" [build-system] -requires = ["maturin>=1.4"] +requires = ["maturin>=1.9.4"] build-backend = "maturin" [tool.ruff.lint] diff --git a/python/python/lancedb/index.py b/python/python/lancedb/index.py index aa7846892..d2b63baf6 100644 --- a/python/python/lancedb/index.py +++ b/python/python/lancedb/index.py @@ -163,6 +163,15 @@ class FTS: The number of documents per compressed posting block. Supported values are 128 and 256. A value of 256 uses the experimental FTS V3 format and may introduce breaking changes. + memory_limit : int, optional + The total memory limit in MiB for the local FTS build stage. The limit + is divided evenly among indexing workers. This build-only setting is + not persisted with the index and does not apply to remote tables. + num_workers : int, optional + The number of workers for a local FTS build. By default Lance uses + roughly half of the available CPU cores. The effective value is + limited by the available compute capacity. This build-only setting is + not persisted with the index and does not apply to remote tables. Notes ----- @@ -185,6 +194,8 @@ class FTS: prefix_only: bool = False block_size: int = 128 custom_stop_words: Optional[List[str]] = None + memory_limit: Optional[int] = None + num_workers: Optional[int] = None @dataclass diff --git a/python/python/tests/test_fts.py b/python/python/tests/test_fts.py index f791f9886..625198d92 100644 --- a/python/python/tests/test_fts.py +++ b/python/python/tests/test_fts.py @@ -245,6 +245,14 @@ def test_create_inverted_index_rejects_invalid_block_size(table): table.create_index("text", config=FTS(block_size=129)) +def test_create_inverted_index_respects_build_memory_limit(table): + with pytest.raises(ValueError, match="exceeds worker memory limit"): + table.create_index( + "text", + config=FTS(memory_limit=0, num_workers=1), + ) + + def test_custom_stop_words_list(table): table.create_index( "text", diff --git a/python/src/index.rs b/python/src/index.rs index dd362373e..a5ca63c68 100644 --- a/python/src/index.rs +++ b/python/src/index.rs @@ -42,7 +42,7 @@ pub fn extract_index_params(source: &Option>) -> PyResult Ok(LanceDbIndex::Fm(FmIndexBuilder::default())), "FTS" => { let params = source.extract::()?; - let inner_opts = FtsIndexBuilder::default() + let mut inner_opts = FtsIndexBuilder::default() .base_tokenizer(params.base_tokenizer) .language(¶ms.language) .map_err(|_| { @@ -61,6 +61,12 @@ pub fn extract_index_params(source: &Option>) -> PyResult, + num_workers: Option, } #[derive(FromPyObject)] @@ -444,3 +452,51 @@ impl IndexConfig { } } } + +#[cfg(test)] +mod tests { + use super::*; + use pyo3::types::{PyDict, PyDictMethods}; + use serde_json::json; + + #[test] + fn fts_build_controls_are_forwarded() { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c"class FTS: + with_position = True + base_tokenizer = 'simple' + language = 'English' + max_token_length = None + lower_case = True + stem = False + remove_stop_words = False + custom_stop_words = None + ascii_folding = False + ngram_min_length = 3 + ngram_max_length = 3 + prefix_only = False + block_size = 128 + memory_limit = 2048 + num_workers = 7 + +config = FTS()", + None, + Some(&locals), + ) + .unwrap(); + + let config = locals.get_item("config").unwrap().unwrap(); + let index = extract_index_params(&Some(config)).unwrap(); + let LanceDbIndex::FTS(params) = index else { + panic!("expected FTS index parameters"); + }; + let training_json = params.to_training_json().unwrap(); + + assert_eq!(training_json.get("memory_limit"), Some(&json!(2048))); + assert_eq!(training_json.get("num_workers"), Some(&json!(7))); + }); + } +} From c0df2c63b6f14c29bef699f93b33153c39dfbc96 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:39:13 -0700 Subject: [PATCH 19/58] test(rust): cover fixed-size-list merge overflow (#3907) ## Summary - add a merge-insert regression test whose fixed-size-list child count crosses `u32::MAX` - verify delete-by-source updates the matching row, deletes every other row, and completes without an Arrow panic - use a null child array so the boundary case avoids allocating a real vector payload ## Root cause and fix The affected Lance merge fallback carried the target payload through a full outer hash join. Arrow's fixed-size-list take kernel uses `u32` child indices, so taking a target row whose child offset crossed `u32::MAX` wrapped the offset and produced child data shorter than the parent array, triggering the reported `ArrayData::slice` assertion. The projection-aware merge path in the Lance version now used by `main` avoids materializing the target fixed-size-list payload in that join. This regression test locks in that production behavior at the exact child-index boundary. ## Validation - `cargo fmt --all` - `cargo test --quiet --features remote -p lancedb test_merge_insert_fixed_size_list_above_u32_child_count` - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` Fixes #2874 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/table/merge.rs | 71 ++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index ea68e99df..b9dd5732b 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -321,7 +321,8 @@ pub(crate) async fn execute_merge_insert( mod tests { use arrow_array::builder::FixedSizeBinaryBuilder; use arrow_array::{ - Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, UInt64Array, + FixedSizeListArray, Int32Array, NullArray, RecordBatch, RecordBatchIterator, + RecordBatchReader, StringArray, UInt32Array, UInt64Array, }; use arrow_schema::{DataType, Field, Schema}; use std::sync::Arc; @@ -529,6 +530,74 @@ mod tests { assert_eq!(result.num_deleted_rows, 5); assert_eq!(table.count_rows(None).await.unwrap(), 5); } + + #[tokio::test] + async fn test_merge_insert_fixed_size_list_above_u32_child_count() { + // Arrow's FixedSizeList take kernel uses u32 child indices. Previously, + // delete-by-source materialized the target payload in a full outer join, + // causing the final list below to overflow those indices and panic. + // A Null child keeps this boundary test small in memory. + const LIST_SIZE: i32 = 65_536; + const ROW_COUNT: usize = (u32::MAX as usize / LIST_SIZE as usize) + 1; + const BATCH_SIZE: usize = 8_192; + + let item = Arc::new(Field::new("item", DataType::Null, true)); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false), + Field::new( + "vector", + DataType::FixedSizeList(item.clone(), LIST_SIZE), + false, + ), + ])); + let batch = |start: usize, len: usize| { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values( + start as u32..(start + len) as u32, + )), + Arc::new(FixedSizeListArray::new( + item.clone(), + LIST_SIZE, + Arc::new(NullArray::new(len * LIST_SIZE as usize)), + None, + )), + ], + ) + .unwrap() + }; + + let target_batches = (0..ROW_COUNT) + .step_by(BATCH_SIZE) + .map(|start| { + let len = (ROW_COUNT - start).min(BATCH_SIZE); + Ok(batch(start, len)) + }) + .collect::>(); + let target_data: Box = + Box::new(RecordBatchIterator::new(target_batches, schema.clone())); + let conn = connect("memory://").execute().await.unwrap(); + let table = conn + .create_table("fixed_size_list_overflow", target_data) + .execute() + .await + .unwrap(); + + let source = batch(ROW_COUNT - 1, 1); + let mut merge = table.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_by_source_delete(None); + let result = merge + .execute(Box::new(RecordBatchIterator::new([Ok(source)], schema))) + .await + .unwrap(); + + assert_eq!(result.num_updated_rows, 1); + assert_eq!(result.num_deleted_rows, (ROW_COUNT - 1) as u64); + assert_eq!(table.count_rows(None).await.unwrap(), 1); + } } #[cfg(test)] From 5468f3d490229ab0dc18e4dd3e6639779f377ad3 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:44:21 -0700 Subject: [PATCH 20/58] fix(rust): reject bitmap indexes on JSON fields (#3895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - reject whole-document `lance.json` fields during native BITMAP index preparation - preserve BITMAP support for raw `LargeBinary` fields - return guidance to use a JSON-path scalar index or FTS instead - add regression coverage for the logical JSON type while retaining the existing raw binary coverage ## Root cause Native scalar-index validation resolved the complete Arrow field but checked BITMAP compatibility only against its physical data type. Because `lance.json` is stored as `LargeBinary`, it was incorrectly accepted under the raw binary compatibility rule. The fix reuses Lance’s `lance_arrow::json::is_json_field` helper before physical type validation. Remote serialization is unchanged, so remote clients continue to send the requested BITMAP type for server-side validation. ## Validation - `cargo fmt --all -- --check` - `cargo test --quiet --features remote -p lancedb test_create_bitmap_index -- --nocapture` - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` - `cargo test --quiet --features remote --tests` Fixes #3889 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/table/create_index.rs | 38 ++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 144c6dbfb..e373522bc 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -12,6 +12,7 @@ use arrow_schema::{DataType, Field}; use lance::index::DatasetIndexExt; use lance::index::vector::VectorIndexParams; use lance::index::vector::utils::infer_vector_dim; +use lance_arrow::json::is_json_field; use lance_index::IndexType; use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; use lance_index::vector::bq::RQBuildParams; @@ -219,6 +220,14 @@ impl NativeTable { ))) } Index::Bitmap(_) => { + if is_json_field(field) { + return Err(Error::Schema { + message: format!( + "A BITMAP index cannot be created on the whole-document lance.json field `{}`. Create a JSON-path scalar index for structured equality or range predicates, or use FTS for document search", + field.name() + ), + }); + } Self::validate_index_type(field, "Bitmap", supported_bitmap_data_type)?; Ok(Box::new(ScalarIndexParams::for_builtin( BuiltinIndexType::Bitmap, @@ -1465,6 +1474,35 @@ mod tests { assert_eq!(stats.distance_type, None); } + #[tokio::test] + async fn test_create_bitmap_index_rejects_lance_json() { + let conn = connect("memory://").execute().await.unwrap(); + let schema = Arc::new(Schema::new(vec![lance_arrow::json::json_field( + "metadata", true, + )])); + let table = conn + .create_empty_table("json_bitmap", schema) + .execute() + .await + .unwrap(); + + let err = table + .create_index(&["metadata"], Index::Bitmap(Default::default())) + .execute() + .await + .expect_err("a whole-document lance.json field must not support a bitmap index"); + let message = err.to_string(); + assert!( + message.contains("lance.json"), + "unexpected error: {message}" + ); + assert!( + message.contains("JSON-path scalar index"), + "unexpected error: {message}" + ); + assert!(message.contains("FTS"), "unexpected error: {message}"); + } + #[tokio::test] async fn test_create_label_list_index() { let conn = connect("memory://").execute().await.unwrap(); From 7801e2746aae9438b7c87e814fe52d5631d0fd9c Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Fri, 21 Aug 2026 21:43:45 -0700 Subject: [PATCH 21/58] chore: update lance dependency to v11.0.0-beta.19 (#4025) Updates the Lance dependencies and Java lance-core dependency to v11.0.0-beta.19. No compatibility fixes were required; workspace clippy with all features passes. Triggering tag: https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.19 --- Cargo.lock | 84 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 28 +++++++++--------- java/pom.xml | 2 +- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73f5923fe..5947187cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arc-swap", "arrow", @@ -4888,8 +4888,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-buffer", @@ -4911,7 +4911,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-buffer", @@ -4925,7 +4925,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-schema", @@ -4934,8 +4934,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrayref", "crunchy", @@ -4945,8 +4945,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-buffer", @@ -4983,8 +4983,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow", "arrow-array", @@ -5013,8 +5013,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow", "arrow-array", @@ -5031,8 +5031,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "proc-macro2", "quote", @@ -5041,8 +5041,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-arith", "arrow-array", @@ -5075,8 +5075,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-arith", "arrow-array", @@ -5107,8 +5107,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arc-swap", "arrow", @@ -5172,8 +5172,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-schema", @@ -5195,8 +5195,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow", "arrow-array", @@ -5232,8 +5232,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-schema", @@ -5247,8 +5247,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow", "async-trait", @@ -5260,8 +5260,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow", "arrow-ipc", @@ -5314,8 +5314,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-buffer", @@ -5329,8 +5329,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow", "arrow-array", @@ -5370,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-schema", @@ -5384,8 +5384,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 9aeccd76f..c147b06db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 978f7c7c7..36ec01d9e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 11.0.0-beta.18 + 11.0.0-beta.19 false 2.30.0 1.7 From a578e9ff7f63f0ef534b088c2f9fc9c4deaa0cb6 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 21 Aug 2026 22:39:47 -0700 Subject: [PATCH 22/58] feat: refresh materialized views (#4010) A declared view holds no rows; refresh computes them. It pins one source version, brings the view to exactly the definition's result at that version, and records the version as a watermark in the view's schema metadata. It is incremental when it can reconcile what changed: appended rows are computed and appended, and rows the source deleted or updated are found by the lance delta and evicted by their __source_row_id provenance, the updated ones recomputed in the same commit. Compaction rearranges rows without changing them, so its outputs cost nothing -- which is what keeps routine background compaction from rebuilding the view. A vacuumed watermark, a delta the transaction-log walk cannot classify, a Legacy-storage source, or more staged ids than a fixed cap all fall back to a rebuild; rebuilding an indexed view swaps every fragment in one Update, so readers never see it unindexed or empty. Concurrent refreshes serialize at commit -- each carries the same sentinel row id in its inserted-rows filter, so the loser lands nothing. On the append path the watermark moves in a follow-up commit, so a crash between the two re-appends those rows. Bumps lance to v11.0.0-beta.19 for the delta reader. --- rust/lancedb/src/lib.rs | 4 +- rust/lancedb/src/materialized_view.rs | 60 +- rust/lancedb/src/materialized_view/refresh.rs | 2719 +++++++++++++++++ rust/lancedb/src/table/merge/lsm.rs | 7 + rust/lancedb/src/table/refresh.rs | 2 +- 5 files changed, 2789 insertions(+), 3 deletions(-) create mode 100644 rust/lancedb/src/materialized_view/refresh.rs diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index 3a937db5b..9c3c199ff 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -211,7 +211,9 @@ pub use function::FunctionVersion; pub use job::Job; use lance_index::vector::ApproxMode as LanceApproxMode; use lance_linalg::distance::DistanceType as LanceDistanceType; -pub use materialized_view::{MaterializedView, MaterializedViewDefinition}; +pub use materialized_view::{ + MaterializedView, MaterializedViewDefinition, RefreshMaterializedViewResult, RefreshMode, +}; /// Re-export of the [`metrics`](https://docs.rs/metrics) crate facade. Enable /// the `metrics` feature to publish LanceDB's internal metrics; install any /// `metrics`-compatible recorder to collect them. See also [`metrics_otel`] for diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index f1eddf9da..8b553165b 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -9,6 +9,7 @@ //! metadata; a kind added later reads back as unrefreshable, not as a plain //! table. Queries, indexes and search work on the view unchanged. +pub mod refresh; use std::collections::HashMap; use std::sync::Arc; @@ -27,6 +28,8 @@ use crate::table::refresh::quote_identifier; use crate::table::{ColumnDefinition, ColumnKind}; use crate::{Error, Result}; +pub use refresh::{RefreshMaterializedViewResult, RefreshMode}; + /// Schema metadata key holding the view definition, as kind-tagged JSON. pub const DEFINITION_META_KEY: &str = "mv.definition"; @@ -736,6 +739,12 @@ pub async fn prepare_declaration( ), }); } + refresh::ensure_no_mem_wal( + native.dataset.get().await?.as_ref(), + "source table", + resolved.name(), + ) + .await?; let source_schema = resolved.schema().await?; let source_metadata = source_schema.metadata().clone(); let (definition, mut fields, lineage) = plan( @@ -909,6 +918,54 @@ impl MaterializedView { pub fn definition(&self) -> &MaterializedViewDefinition { &self.definition } + + /// Recompute the view from its source. + /// + /// By default the refresh is incremental when the source's changes can be + /// reconciled into the view, and otherwise rebuilds; see + /// [`RefreshMaterializedViewBuilder`]. + /// + /// ```no_run + /// # #![recursion_limit = "256"] + /// # use lancedb::materialized_view::MaterializedView; + /// # async fn refresh(view: &MaterializedView) -> Result<(), Box> { + /// let result = view.refresh().execute().await?; + /// println!("{:?}: {} rows", result.mode, result.rows_written); + /// # Ok(()) + /// # } + /// ``` + pub fn refresh(&self) -> RefreshMaterializedViewBuilder { + RefreshMaterializedViewBuilder { + view: self.clone(), + full: false, + source_version: None, + } + } +} + +/// Builds a refresh. Created by [`MaterializedView::refresh`]. +pub struct RefreshMaterializedViewBuilder { + view: MaterializedView, + full: bool, + source_version: Option, +} + +impl RefreshMaterializedViewBuilder { + /// Rebuild the view even where an incremental refresh would do. + pub fn full(mut self, full: bool) -> Self { + self.full = full; + self + } + + /// Refresh to this source table version instead of the latest. + pub fn source_version(mut self, version: u64) -> Self { + self.source_version = Some(version); + self + } + + pub async fn execute(self) -> Result { + refresh::execute_refresh(&self.view.table, self.full, self.source_version).await + } } impl Connection { @@ -918,6 +975,7 @@ impl Connection { /// metadata; refresh computes the rows. Local databases only. /// /// ```no_run + /// # #![recursion_limit = "256"] /// # use lancedb::Connection; /// # async fn create(conn: &Connection) -> Result<(), Box> { /// let view = conn @@ -926,7 +984,7 @@ impl Connection { /// .only_if("age >= 18") /// .execute() /// .await?; - /// println!("{}", view.definition().source_table); + /// view.refresh().execute().await?; /// # Ok(()) /// # } /// ``` diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs new file mode 100644 index 000000000..7ac39a9f5 --- /dev/null +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -0,0 +1,2719 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Refreshing materialized views. +//! +//! A refresh pins one source version and brings the view to exactly the +//! definition's result at that version: added rows are computed and appended, +//! removed or changed rows are evicted by provenance id and recomputed in the +//! same pass. Compaction outputs cost nothing, which is only sound while +//! [`SOURCE_ROW_ID_COLUMN`] stays valid across the rewrite. Anything the +//! classifier cannot prove intact rebuilds; an indexed rebuild swaps all +//! fragments in one commit that retains index definitions. +//! +//! The watermark ([`SOURCE_VERSION_META_KEY`]) lands in a follow-up commit; a +//! crash or race between the two leaves the view visibly unstamped and the +//! next refresh rebuilds. In-process refreshes serialize on a per-view lock; +//! across processes the commit's inserted-rows filter carries a shared token, +//! so two refreshes of one view conflict and only one lands. + +use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex as StdMutex, OnceLock}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use arrow_array::cast::AsArray; +use arrow_array::types::UInt64Type; +use arrow_array::{RecordBatch, UInt64Array}; +use arrow_schema::{Schema as ArrowSchema, SchemaRef}; +use datafusion::common::ScalarValue; +use datafusion::error::DataFusionError; +use datafusion::physical_plan::SendableRecordBatchStream; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::prelude::{col, lit}; +use futures::{StreamExt, TryStreamExt}; +use lance::Dataset; +use lance::dataset::mem_wal::DatasetMemWalExt; +use lance::dataset::transaction::{Operation, Transaction}; +use lance::dataset::write::delete::DeleteBuilder; +use lance::dataset::write::merge_insert::inserted_rows::{ + KeyExistenceFilter, KeyExistenceFilterBuilder, KeyValue, +}; +use lance::dataset::{CommitBuilder, InsertBuilder, WriteDestination, WriteMode, WriteParams}; +use lance_core::{ROW_CREATED_AT_VERSION, ROW_ID, ROW_LAST_UPDATED_AT_VERSION}; +use lance_file::version::ConcreteFileVersion; +use lance_table::format::Fragment; +use serde::{Deserialize, Serialize}; + +use super::{ + MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY, SOURCE_ROW_ID_COLUMN, + SOURCE_VERSION_META_KEY, +}; +use crate::database::OpenTableRequest; +use crate::table::{NativeTable, NativeTableExt, Table}; +use crate::{Error, Result}; + +/// How a refresh brought the view up to date. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RefreshMode { + /// The view was recomputed from scratch. + Rebuild, + /// Rows from source fragments added since the last refresh were appended. + Incremental, + /// The view was already at the requested source version. + NoOp, +} + +/// The result of refreshing a materialized view. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RefreshMaterializedViewResult { + /// How the view was brought up to date. + pub mode: RefreshMode, + /// Rows written to the view: everything on a rebuild, and on an + /// incremental refresh both the rows added and the rows recomputed in + /// place of ones the source changed. + pub rows_written: u64, + /// The source table version the view now reflects. + pub source_version: u64, + /// The view table version after the refresh. + pub version: u64, +} + +/// Schema metadata key holding the view table version a successful refresh +/// left behind. Any other commit on the view is drift, and refresh rebuilds. +pub const VIEW_VERSION_META_KEY: &str = "mv.view_version"; + +/// Schema metadata key holding the commit timestamp of the watermark's source +/// manifest. A dropped and recreated source reuses version numbers but never +/// their timestamps, so a mismatch means the watermark describes a different +/// incarnation and refresh rebuilds. +pub const SOURCE_VERSION_TS_META_KEY: &str = "mv.source_version_ts"; + +/// One refresh per view at a time within this process. +fn refresh_lock(uri: &str) -> Arc> { + static LOCKS: OnceLock>>>> = + OnceLock::new(); + LOCKS + .get_or_init(Default::default) + .lock() + .expect("refresh lock registry poisoned") + .entry(uri.to_string()) + .or_default() + .clone() +} + +/// Internal implementation of the refresh logic. +pub(crate) async fn execute_refresh( + view: &Table, + full: bool, + pinned: Option, +) -> Result { + let view_native = view.as_native().ok_or_else(|| Error::NotSupported { + message: "materialized views are supported only on local tables".into(), + })?; + view_native.dataset.ensure_mutable()?; + let lock = refresh_lock(view_native.dataset.get().await?.uri()); + let _guard = lock.lock().await; + // Force-load the latest view state under the lock: each handle caches + // lazily, and a second handle would otherwise plan from a snapshot taken + // before another handle's commit -- appending the same rows again or + // reporting NoOp over a mutated view. + view_native.dataset.reload().await?; + let view_ds = view_native.dataset.get().await?.as_ref().clone(); + + // The definition a handle cached at open may since have been replaced; + // what refresh executes and what it stamps must be one generation. + let definition = match super::materialized_view_kind(&view_ds.schema().metadata)? { + Some(super::MaterializedViewKind::Select(definition)) => definition, + Some(super::MaterializedViewKind::Unrecognized { kind }) => { + return Err(Error::NotSupported { + message: format!( + "materialized view '{}' is defined by '{kind}', which this \ + version of lancedb cannot refresh", + view.name() + ), + }); + } + None => { + return Err(Error::NotAMaterializedView { + name: view.name().to_string(), + }); + } + }; + let definition = &definition; + ensure_no_mem_wal(&view_ds, "materialized view", view.name()).await?; + + let source_ds = open_source(view, definition).await?; + let source_ds = match pinned { + Some(version) => source_ds.checkout_version(version).await?, + None => source_ds, + }; + ensure_no_mem_wal(&source_ds, "source table", &definition.source_table).await?; + let source_version = source_ds.version().version; + let source_ts = source_ds.manifest.timestamp_nanos; + + // Re-plan the persisted definition against the current source schema and + // require its planned output to be exactly the view's physical schema: a + // definition the stored table cannot represent must not be certified. + let source_schema = Arc::new(ArrowSchema::from(source_ds.schema())); + let projections: Vec<(String, String)> = definition + .projections + .iter() + .map(|p| (p.output.clone(), p.expression.clone())) + .collect(); + validate_inputs(&source_ds, definition)?; + let (replanned, mut planned_fields, _renames) = super::plan( + source_schema, + &definition.source_table, + &projections, + definition.filter.as_deref(), + definition.limit, + )?; + planned_fields.push(arrow_schema::Field::new( + SOURCE_ROW_ID_COLUMN, + arrow_schema::DataType::UInt64, + false, + )); + let physical = ArrowSchema::from(view_ds.schema()); + let planned_shape: Vec<_> = planned_fields + .iter() + .map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable())) + .collect(); + let physical_shape: Vec<_> = physical + .fields() + .iter() + .map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable())) + .collect(); + if planned_shape != physical_shape { + return Err(Error::Schema { + message: format!( + "the stored definition of view '{}' does not produce this \ + view's schema; recreate the view", + view.name() + ), + }); + } + let definition = &replanned; + + let metadata = &view_ds.schema().metadata; + let watermark: Option = metadata + .get(SOURCE_VERSION_META_KEY) + .and_then(|raw| raw.parse().ok()); + let recorded_ts: Option = metadata + .get(SOURCE_VERSION_TS_META_KEY) + .and_then(|raw| raw.parse().ok()); + // The watermark speaks only for the view state its refresh left behind; + // any other commit on the view since then is drift. + let view_intact = metadata + .get(VIEW_VERSION_META_KEY) + .and_then(|raw| raw.parse::().ok()) + == Some(view_ds.version().version); + + if !full && watermark == Some(source_version) && view_intact && recorded_ts == Some(source_ts) { + return Ok(RefreshMaterializedViewResult { + mode: RefreshMode::NoOp, + rows_written: 0, + source_version, + version: view_ds.version().version, + }); + } + + let watermark = watermark.filter(|_| view_intact); + match plan_increment( + &source_ds, + source_version, + watermark, + recorded_ts, + full, + definition, + ) + .await + { + Some(increment) => { + let reconciled = incremental( + view_native, + &view_ds, + &source_ds, + source_version, + source_ts, + increment, + definition, + watermark, + ) + .await?; + match reconciled { + Some(result) => Ok(result), + // The delta was too large to reconcile in bounded memory. + None => { + rebuild( + view_native, + &view_ds, + &source_ds, + source_version, + source_ts, + definition, + ) + .await + } + } + } + None => { + rebuild( + view_native, + &view_ds, + &source_ds, + source_version, + source_ts, + definition, + ) + .await + } + } +} + +/// The source fragments whose rows are new since the watermark, or `None` +/// where the view has to rebuild. Two tiers: the transaction walk is exact +/// where it applies; the fragment-signature check is the fallback for deltas +/// the walk cannot read, and under it any fragment churn rebuilds. +async fn plan_increment( + source_ds: &Dataset, + source_version: u64, + watermark: Option, + recorded_ts: Option, + full: bool, + definition: &MaterializedViewDefinition, +) -> Option { + if full { + return None; + } + let watermark = watermark?; + if watermark > source_version { + return None; + } + let old = source_ds.checkout_version(watermark).await.ok()?; + // A recreated source reuses version numbers, never their timestamps: a + // mismatch means the watermark describes a different incarnation. + if recorded_ts != Some(old.manifest.timestamp_nanos) { + return None; + } + let old_ids: HashSet = old.get_fragments().iter().map(|f| f.id() as u64).collect(); + let live: Vec = source_ds + .get_fragments() + .iter() + .map(|f| f.metadata().clone()) + .collect(); + + if let Some(delta) = appends_and_rewrites(source_ds, watermark, source_version).await { + // A rewrite that consumed a fragment neither present at the watermark + // nor produced by an earlier rewrite swallowed a mid-delta append; + // its rows cannot be told apart from already-materialized ones. + let folded = delta + .rewritten + .iter() + .any(|id| !old_ids.contains(id) && !delta.produced.contains(id)); + if folded { + return None; + } + // An update rewrites a whole fragment. If it touched one the watermark + // never saw, that fragment holds rows appended since -- new rows the + // update did not change, which the recompute does not cover and which + // this fragment's exclusion from the append set would drop. + if delta + .updated_in_place + .iter() + .any(|id| !old_ids.contains(id)) + { + return None; + } + // Rows past the cap left every delta when the watermark advanced, so + // a capped view cannot reconcile a removal incrementally. The rebuild + // is cheap for the same reason it is capped: the scan stops there. + if definition.limit.is_some() && (delta.deleted_rows || delta.updated_rows) { + return None; + } + // Legacy storage cannot serve the row-version columns update + // discovery scans; deletes and appends need none of them. + if delta.updated_rows + && source_ds.manifest.data_storage_format.lance_file_format() == ConcreteFileVersion::V1 + { + return None; + } + // Every other fragment new at head is an append: appends and rewrites + // are the only operations in the delta that add fragments, and the + // rewrite outputs are already-materialized rows rearranged. + return Some(Increment { + appended: live + .into_iter() + .filter(|f| !old_ids.contains(&f.id) && !delta.produced.contains(&f.id)) + .collect(), + evict_deleted: delta.deleted_rows, + replace_updated: delta.updated_rows, + }); + } + + is_pure_append(&old, source_ds, &relevant_field_ids(source_ds, definition)).then(|| Increment { + appended: live + .into_iter() + .filter(|f| !old_ids.contains(&f.id)) + .collect(), + evict_deleted: false, + replace_updated: false, + }) +} + +/// Version gap beyond which per-version transaction reads stop being cheaper +/// than one fragment scan. +const MAX_TRANSACTION_WALK: u64 = 512; + +/// Fragment ids moved by the `Rewrite` operations of the delta. Only rewrite +/// ids are real in transaction files (an Append's are placeholders assigned +/// at commit), so appends are derived as new-at-head minus rewrite outputs. +/// What an incremental refresh must do to bring the view up to date. +struct Increment { + /// Source fragments whose rows are not in the view yet. + appended: Vec, + /// Source rows left the range, so the view holds rows to evict. + evict_deleted: bool, + /// Source rows changed in the range, so the view holds rows to replace. + replace_updated: bool, +} + +struct TxnDelta { + /// Fragments consumed by `Rewrite` operations. + rewritten: HashSet, + /// Fragments produced by `Rewrite` operations. + produced: HashSet, + /// The delta removed source rows, so the view holds rows to evict. + deleted_rows: bool, + /// The delta changed source rows in place, so the view holds rows to + /// recompute. + updated_rows: bool, + /// Fragments an update modified in place, as opposed to produced. + updated_in_place: HashSet, +} + +/// Read the delta from the transaction log, `None` where it holds anything +/// but appends and rewrites or cannot be read; `None` only sends the caller +/// to a slower check. `ReserveFragments` moves no rows and rides along. +async fn appends_and_rewrites(cur: &Dataset, from: u64, to: u64) -> Option { + if to <= from || to - from > MAX_TRANSACTION_WALK { + return None; + } + let mut delta = TxnDelta { + rewritten: HashSet::new(), + produced: HashSet::new(), + deleted_rows: false, + updated_rows: false, + updated_in_place: HashSet::new(), + }; + for version in (from + 1)..=to { + let Ok(Some(txn)) = cur.read_transaction_by_version(version).await else { + return None; + }; + match txn.operation { + Operation::Append { .. } | Operation::ReserveFragments { .. } => {} + // A delete removes source rows without changing the ones that + // remain, so the view's other rows stay valid: the refresh + // evicts exactly the ids that left. + Operation::Delete { .. } => delta.deleted_rows = true, + // Update outputs carry no new rows, so they are excluded from + // the append set like rewrite outputs; the changed rows are + // replaced individually below. + Operation::Update { + removed_fragment_ids, + new_fragments, + updated_fragments, + .. + } => { + delta.updated_rows = true; + // merge_insert reaches here too, and its by-source arm deletes + // rows rather than changing them. + delta.deleted_rows = true; + delta.rewritten.extend(removed_fragment_ids.iter().copied()); + // Only pre-existing fragment ids are real here; created ones + // are placeholders. Rewritten rows are excluded by creation + // version below, not by fragment identity. + delta + .produced + .extend(updated_fragments.iter().map(|f| f.id)); + delta + .updated_in_place + .extend(updated_fragments.iter().map(|f| f.id)); + let _ = new_fragments; + } + Operation::Rewrite { groups, .. } => { + for group in groups { + delta + .rewritten + .extend(group.old_fragments.iter().map(|f| f.id)); + delta + .produced + .extend(group.new_fragments.iter().map(|f| f.id)); + } + } + _ => return None, + } + } + Some(delta) +} + +/// Fallback pure-append check: every old fragment still present with an +/// identical signature over the columns the view reads. Compaction, deletes +/// and updates each break it and force a rebuild; a change to a column the +/// view does not read leaves it alone, which is what lets this tier pass +/// deltas the transaction walk cannot. +fn is_pure_append(old: &Dataset, cur: &Dataset, relevant: &HashSet) -> bool { + let signature = |fragment: &lance::dataset::fragment::FileFragment| { + fragment_signature(fragment.metadata(), relevant) + }; + let current: HashSet<(u64, String)> = cur.get_fragments().iter().map(signature).collect(); + old.get_fragments() + .iter() + .all(|fragment| current.contains(&signature(fragment))) +} + +/// A fragment's identity as the view observes it: data files and overlays +/// touching the columns it reads, plus the deletion file. Overlays change no +/// file path, so they must be part of the signature. +fn fragment_signature(metadata: &Fragment, relevant: &HashSet) -> (u64, String) { + let touches_relevant = + |fields: &[i32]| relevant.is_empty() || fields.iter().any(|id| relevant.contains(id)); + let mut files: Vec<&str> = metadata + .files + .iter() + .filter(|file| touches_relevant(&file.fields)) + .map(|file| file.path.as_str()) + .collect(); + files.sort_unstable(); + let mut overlays: Vec = metadata + .overlays + .iter() + .filter(|overlay| touches_relevant(&overlay.data_file.fields)) + .map(|overlay| format!("{}@{}", overlay.data_file.path, overlay.committed_version)) + .collect(); + overlays.sort_unstable(); + ( + metadata.id, + format!( + "{}|{}|{:?}", + files.join(","), + overlays.join(","), + metadata.deletion_file + ), + ) +} + +/// Field ids (with struct descendants) of the source columns the view reads. +fn relevant_field_ids(source: &Dataset, definition: &MaterializedViewDefinition) -> HashSet { + fn collect(field: &lance_core::datatypes::Field, ids: &mut HashSet) { + ids.insert(field.id); + for child in &field.children { + collect(child, ids); + } + } + let mut ids = HashSet::new(); + for input in &definition.inputs { + if let Some(field) = source.schema().field(input) { + collect(field, &mut ids); + } + } + ids +} + +/// Error if a column the view reads no longer exists in the source. +fn validate_inputs(source: &Dataset, definition: &MaterializedViewDefinition) -> Result<()> { + for input in &definition.inputs { + if source.schema().field(input).is_none() { + return Err(Error::Schema { + message: format!( + "source column '{input}' read by the view no longer exists \ + (dropped or renamed in '{}')", + definition.source_table + ), + }); + } + } + Ok(()) +} + +/// Reject MemWAL/LSM state on a refresh participant: un-compacted tiers are +/// invisible to the fragment-planned refresh scan. An active write spec and +/// retained rows both disqualify; shard directories on storage are the +/// durable evidence of the latter. +pub(crate) async fn ensure_no_mem_wal(dataset: &Dataset, role: &str, name: &str) -> Result<()> { + let retained = !dataset.list_mem_wal_latest_shard_ids().await?.is_empty(); + if retained || dataset.mem_wal_index_details().await?.is_some() { + return Err(Error::NotSupported { + message: format!( + "{role} '{name}' has an LSM write spec or retained un-compacted \ + rows: rows in un-compacted tiers are invisible to refresh" + ), + }); + } + Ok(()) +} + +async fn open_source(view: &Table, definition: &MaterializedViewDefinition) -> Result { + let database = view.database_opt().ok_or_else(|| Error::InvalidInput { + message: "the view was not opened through a database connection".into(), + })?; + let source = database + .open_table(OpenTableRequest { + name: definition.source_table.clone(), + namespace_path: Vec::new(), + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await?; + let native = source.as_native().ok_or_else(|| Error::NotSupported { + message: "materialized views are supported only on local tables".into(), + })?; + let dataset = native.dataset.get().await?.as_ref().clone(); + if !dataset.manifest.uses_stable_row_ids() { + return Err(Error::InvalidInput { + message: format!( + "source table '{}' does not have stable row ids; it is not the \ + table this view was declared over", + definition.source_table + ), + }); + } + Ok(dataset) +} + +#[allow(clippy::too_many_arguments)] +async fn incremental( + view_native: &NativeTable, + view_ds: &Dataset, + source_ds: &Dataset, + source_version: u64, + source_ts: u128, + increment: Increment, + definition: &MaterializedViewDefinition, + watermark: Option, +) -> Result> { + let new_fragments = increment.appended; + let watermark_version = watermark.unwrap_or(0); + // Provenance ids this refresh removes: dropped rows, plus changed rows + // recomputed in the same commit. One view row per source row makes the + // eviction exact. Staged, not committed: removals ride with the rows + // that replace them, so a reader never sees the view without either. + let mut eviction = Eviction::new(view_ds, EVICTION_CHUNK); + let mut updated_rows = false; + if (increment.evict_deleted || increment.replace_updated) + && let Some(watermark) = watermark + { + let delta = source_ds + .delta() + .with_begin_version(watermark) + .with_end_version(source_version) + .build()?; + // Reconciling holds the delta's provenance ids in staged deletion + // vectors; past the cap, the streamed rebuild is the bounded path. + let cap = eviction_rebuild_cap(); + let mut evicted = 0usize; + if increment.evict_deleted { + let mut stream = delta.get_deleted_row_ids().await?; + while let Some(batch) = stream.try_next().await? { + let ids = row_ids_of(&batch)?; + evicted += ids.len(); + if evicted > cap { + return Ok(None); + } + eviction.push(ids).await?; + } + } + if increment.replace_updated { + // Ids only: `get_updated_rows` carries every column of every + // updated row, and discovery needs none of them. + let mut scanner = source_ds.scan(); + scanner.with_row_id().project(&[ROW_CREATED_AT_VERSION])?; + // A fixed bound: the configured default could make one discovery + // batch arbitrarily large before the fallback cap is consulted. + scanner.batch_size(8192); + scanner.filter(&format!( + "{ROW_CREATED_AT_VERSION} <= {watermark} + AND {ROW_LAST_UPDATED_AT_VERSION} > {watermark} + AND {ROW_LAST_UPDATED_AT_VERSION} <= {source_version}" + ))?; + let mut stream = scanner.try_into_stream().await?; + while let Some(batch) = stream.try_next().await? { + let ids = row_ids_of(&batch)?; + evicted += ids.len(); + if evicted > cap { + return Ok(None); + } + updated_rows |= !ids.is_empty(); + eviction.push(ids).await?; + } + } + } + let eviction = eviction.finish().await?; + + // The cap counts rows already materialized, in first-materialized order. + let remaining = match definition.limit { + Some(limit) => { + let held = view_ds.count_rows(None).await? as u64; + Some(limit.saturating_sub(held)) + } + None => None, + }; + + let mut result = RefreshMaterializedViewResult { + mode: RefreshMode::Incremental, + rows_written: 0, + source_version, + version: view_ds.version().version, + }; + let nothing_to_add = (new_fragments.is_empty() && !updated_rows) || remaining == Some(0); + if nothing_to_add && eviction.is_none() { + result.version = + stamp_watermark(view_native, view_ds.clone(), source_version, source_ts).await?; + return Ok(Some(result)); + } + // Rows left but none arrive: the removals still have to be published. + if nothing_to_add { + let filter = refresh_filter(&empty_keys(view_ds)?)?; + let published = publish(view_ds, eviction, Vec::new(), Some(filter)).await?; + result.version = stamp_watermark(view_native, published, source_version, source_ts).await?; + return Ok(Some(result)); + } + + // Appends carry the view's schema as it stands; the watermark moves in a + // follow-up commit (see the module docs for the crash window). + let schema = Arc::new(ArrowSchema::from(view_ds.schema())); + let rows_written = Arc::new(AtomicU64::new(0)); + // compute_stream counts what it produces; the truncation below can drop + // some of that, so the written count comes from the tee instead. + let computed = Arc::new(AtomicU64::new(0)); + let mut stream = compute_stream( + source_ds, + definition, + RowScope { + fragments: Some(new_fragments), + // An update rewrites whole fragments, so a fragment new at head + // can hold rows the view already has. Their creation version + // does not change, so it -- not fragment identity -- says which + // rows are new. + created_after: increment.replace_updated.then_some(watermark_version), + limit: remaining, + ..Default::default() + }, + schema.clone(), + computed.clone(), + ) + .await?; + + // The updated rows' current values, computed the same way and appended + // in the same commit as the new fragments' rows. + if updated_rows { + let recomputed = compute_stream( + source_ds, + definition, + RowScope { + updated_between: Some((watermark_version, source_version)), + ..Default::default() + }, + schema.clone(), + computed.clone(), + ) + .await?; + stream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + recomputed.chain(stream), + )); + } + + // Nothing survived the filter: the watermark still has to advance or the + // same fragments would be rescanned forever, but any removals still do. + let Some(first) = stream.try_next().await? else { + let published = if eviction.is_some() { + publish( + view_ds, + eviction, + Vec::new(), + Some(refresh_filter(&empty_keys(view_ds)?)?), + ) + .await? + } else { + view_ds.clone() + }; + result.version = stamp_watermark(view_native, published, source_version, source_ts).await?; + return Ok(Some(result)); + }; + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::iter([Ok(first)]).chain(stream), + )); + + // Any two refreshes of one view must not both commit: the filter's + // shared token makes lance reject the loser on key overlap however + // their planned rows relate. + let keys = Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new(vec![ + source_row_id_field_id(view_ds)?, + ]))); + let stream = collect_source_row_ids(stream, keys.clone(), rows_written.clone()); + + let ds = Arc::new(view_ds.clone()); + let write_txn = InsertBuilder::new(WriteDestination::Dataset(ds.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await?; + let Operation::Append { + fragments: new_fragments, + } = write_txn.operation + else { + return Err(Error::Runtime { + message: "expected an append when staging the view's new rows".into(), + }); + }; + let filter = refresh_filter(&keys)?; + let appended = publish(view_ds, eviction, new_fragments, Some(filter)).await?; + result.rows_written = rows_written.load(Ordering::Relaxed); + result.version = stamp_watermark(view_native, appended, source_version, source_ts).await?; + Ok(Some(result)) +} + +async fn rebuild( + view_native: &NativeTable, + view_ds: &Dataset, + source_ds: &Dataset, + source_version: u64, + source_ts: u128, + definition: &MaterializedViewDefinition, +) -> Result { + let rows_written = Arc::new(AtomicU64::new(0)); + let schema = Arc::new(ArrowSchema::from(view_ds.schema())); + let stream = compute_stream( + source_ds, + definition, + RowScope { + limit: definition.limit, + ..Default::default() + }, + schema, + rows_written.clone(), + ) + .await?; + let keys = Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new(vec![ + source_row_id_field_id(view_ds)?, + ]))); + let stream = collect_source_row_ids(stream, keys.clone(), Arc::new(AtomicU64::new(0))); + // Every rebuild is one fragment swap, indexed or not: an Update commit + // carries no schema metadata, so it cannot erase a definition update + // that raced in the way an overwrite (which adopts its stream's schema) + // durably would -- and it must land on the planned generation or abort. + let replaced = replace_retaining_indices(view_ds.clone(), stream, keys).await?; + let version = stamp_watermark(view_native, replaced, source_version, source_ts).await?; + Ok(RefreshMaterializedViewResult { + mode: RefreshMode::Rebuild, + rows_written: rows_written.load(Ordering::Relaxed), + source_version, + version, + }) +} + +/// Replace all of the view's data in one commit that retains its index +/// definitions: new fragments staged uncommitted, one `Update` removing every +/// old fragment. `Update` prunes index bitmaps only for modified fields and +/// none are modified here, so readers never see the view unindexed or empty. +async fn replace_retaining_indices( + view_ds: Dataset, + stream: SendableRecordBatchStream, + keys: Arc>, +) -> Result { + let ds = Arc::new(view_ds); + let read_version = ds.version().version; + #[cfg(test)] + tests::hold_before_publish(ds.uri()).await; + let removed_fragment_ids: Vec = ds.get_fragments().iter().map(|f| f.id() as u64).collect(); + + let write_txn = InsertBuilder::new(WriteDestination::Dataset(ds.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await?; + let Operation::Append { + fragments: new_fragments, + } = write_txn.operation + else { + return Err(Error::Runtime { + message: "expected an append when staging the view's replacement rows".into(), + }); + }; + + // Built only now: the tee fills as the staging drains the stream. + let filter = refresh_filter(&keys)?; + let transaction = Transaction::new( + read_version, + Operation::Update { + removed_fragment_ids, + updated_fragments: Vec::new(), + new_fragments, + fields_modified: Vec::new(), + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: Vec::new(), + update_mode: None, + // Two refreshes that materialized the same source rows must not + // both land -- a raced first rebuild would double the view. + inserted_rows_filter: Some(filter), + updated_fragment_offsets: None, + }, + None, + ); + let committed = CommitBuilder::new(WriteDestination::Dataset(ds)) + .execute(transaction) + .await?; + if committed.version().version != read_version + 1 { + return Err(Error::Runtime { + message: format!( + "a concurrent commit raced this refresh (view version {}); the + refresh is unrecorded and the next one will rebuild", + committed.version().version + ), + }); + } + Ok(committed) +} + +/// Record that the view now reflects `source_version`, including the view +/// version this very commit produces. The version is predicted and then +/// verified; on a mismatch another commit raced in between, and the stamp +/// ABORTS rather than certify that commit as the refresh's own generation. +/// The view is left visibly unstamped, so the next refresh rebuilds. +async fn stamp_watermark( + view_native: &NativeTable, + mut dataset: Dataset, + source_version: u64, + source_ts: u128, +) -> Result { + let predicted = dataset.version().version + 1; + dataset + .update_schema_metadata([ + ( + SOURCE_VERSION_META_KEY.to_string(), + Some(source_version.to_string()), + ), + ( + SOURCE_VERSION_TS_META_KEY.to_string(), + Some(source_ts.to_string()), + ), + ( + REFRESHED_AT_MS_META_KEY.to_string(), + Some(now_ms().to_string()), + ), + ( + VIEW_VERSION_META_KEY.to_string(), + Some(predicted.to_string()), + ), + ]) + .await?; + let actual = dataset.version().version; + if actual != predicted { + return Err(Error::Runtime { + message: format!( + "a concurrent commit raced this refresh (view version {actual}, \ + expected {predicted}); the refresh is unrecorded and the next \ + one will rebuild" + ), + }); + } + view_native.dataset.update(dataset); + Ok(predicted) +} + +fn now_ms() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} + +/// Evaluate the definition over `source`, restricted to `fragments` when +/// given, as batches in the view's schema. The filter is pushed into the +/// scan; projections are `(name, expression)` pairs, never spliced into SQL; +/// [`SOURCE_ROW_ID_COLUMN`] is filled by the scan's row id. +/// Which source rows a compute pass reads. +#[derive(Default)] +struct RowScope { + /// Read only these fragments. + fragments: Option>, + /// Read only rows created after this version. + created_after: Option, + /// Read only rows changed in place after the first version and no later + /// than the second. + updated_between: Option<(u64, u64)>, + /// Stop after this many rows. + limit: Option, +} + +async fn compute_stream( + source: &Dataset, + definition: &MaterializedViewDefinition, + scope: RowScope, + schema: SchemaRef, + rows_written: Arc, +) -> Result { + let RowScope { + fragments, + created_after, + updated_between, + limit, + } = scope; + let mut scanner = source.scan(); + if let Some(fragments) = fragments { + scanner.with_fragments(fragments); + } + scanner.with_row_id(); + // Narrowing keeps the definition's filter, so a row updated out of the + // view simply does not come back. Changed rows are named by the predicate + // `DatasetDelta::get_updated_rows` uses, not its streamed ids: an id list + // grows with the delta, this does not. + let updated_filter = updated_between.map(|(from, to)| { + format!( + "{ROW_CREATED_AT_VERSION} <= {from} \ + AND {ROW_LAST_UPDATED_AT_VERSION} > {from} \ + AND {ROW_LAST_UPDATED_AT_VERSION} <= {to}" + ) + }); + let created_filter = + created_after.map(|version| format!("{ROW_CREATED_AT_VERSION} > {version}")); + let clauses: Vec = definition + .filter + .clone() + .map(|f| format!("({f})")) + .into_iter() + .chain(updated_filter) + .chain(created_filter) + .collect(); + if !clauses.is_empty() { + scanner.filter(&clauses.join(" AND "))?; + } + let transforms: Vec<(&str, &str)> = definition + .projections + .iter() + .map(|p| (p.output.as_str(), p.expression.as_str())) + .collect(); + scanner.project_with_transform(&transforms)?; + // A scan reads a limit of zero as no limit at all, so a view capped at + // nothing is answered without one. + if limit == Some(0) { + return Ok(Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::empty(), + ))); + } + if let Some(limit) = limit { + let limit = i64::try_from(limit).map_err(|_| Error::InvalidInput { + message: format!("view limit {limit} exceeds the maximum of {}", i64::MAX), + })?; + scanner.limit(Some(limit), None)?; + } + + let out_schema = schema.clone(); + let mapped = scanner.try_into_stream().await?.map(move |batch| { + let batch = batch.map_err(|e| DataFusionError::External(Box::new(e)))?; + let mut columns = Vec::with_capacity(out_schema.fields().len()); + for field in out_schema.fields() { + let name = if field.name() == SOURCE_ROW_ID_COLUMN { + ROW_ID + } else { + field.name() + }; + let column = batch.column_by_name(name).ok_or_else(|| { + DataFusionError::Internal(format!( + "view column '{}' is not produced by the view's definition", + field.name() + )) + })?; + columns.push(column.clone()); + } + rows_written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + Ok(RecordBatch::try_new(out_schema.clone(), columns)?) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new(schema, mapped))) +} + +/// Commit the view's removals and additions as one change, on the exact +/// generation the refresh planned from. Lance rejects an overlapping +/// provenance key, but an unrelated write to the view is not a key conflict, +/// so the generation is checked here too. +async fn publish( + view_ds: &Dataset, + eviction: Option<(Vec, Vec)>, + new_fragments: Vec, + keys: Option, +) -> Result { + let planned = view_ds.version().version; + #[cfg(test)] + tests::hold_before_publish(view_ds.uri()).await; + let (updated_fragments, removed_fragment_ids) = eviction.unwrap_or_default(); + let committed = CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .execute(Transaction::new( + planned, + Operation::Update { + removed_fragment_ids, + updated_fragments, + new_fragments, + fields_modified: Vec::new(), + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: Vec::new(), + update_mode: None, + inserted_rows_filter: keys, + updated_fragment_offsets: None, + }, + None, + )) + .await?; + if committed.version().version != planned + 1 { + return Err(Error::Runtime { + message: format!( + "a concurrent commit raced this refresh (view version {}); + the refresh is unrecorded and the next one will rebuild", + committed.version().version + ), + }); + } + Ok(committed) +} + +/// A delta batch's row id column, borrowed: a delta batch is as large as one +/// fragment's deletions, so copying it out would be the unbounded step the +/// chunking exists to avoid. +fn row_ids_of(batch: &RecordBatch) -> Result<&UInt64Array> { + let column = batch.column_by_name(ROW_ID).ok_or_else(|| Error::Runtime { + message: format!("'{ROW_ID}' is missing from a delta batch"), + })?; + column + .as_primitive_opt::() + .ok_or_else(|| Error::Runtime { + message: "row ids are not UInt64".into(), + }) +} + +/// A provenance id no source row can hold, carried in every refresh's +/// inserted-rows filter: any two refreshes of one view overlap on it, so +/// the loser of a race conflicts at commit whatever rows each planned. +const REFRESH_TOKEN_ID: u64 = u64::MAX; + +/// A builder with no collected keys, for publishes that only remove rows. +fn empty_keys(view_ds: &Dataset) -> Result>> { + Ok(Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new( + vec![source_row_id_field_id(view_ds)?], + )))) +} + +/// An inserted-rows filter holding the refresh token plus whatever the +/// tee collected. +fn refresh_filter(keys: &Arc>) -> Result { + let mut keys = keys.lock().map_err(|_| Error::Runtime { + message: "the provenance key filter was poisoned mid-refresh".into(), + })?; + keys.insert(KeyValue::UInt64(REFRESH_TOKEN_ID)) + .map_err(|e| Error::Runtime { + message: format!("failed to mark the refresh's filter: {e}"), + })?; + Ok(keys.build()) +} + +/// Reconciled ids past this fall back to the streamed rebuild, whose memory +/// does not grow with the delta. +fn eviction_rebuild_cap() -> usize { + #[cfg(test)] + if let Some(cap) = tests::eviction_cap_override() { + return cap; + } + 4 * 1024 * 1024 +} + +/// Provenance ids per staged eviction: the delete predicate carries one +/// literal per id, so a whole delta at once is unbounded. Each chunk costs a +/// pass over the view's provenance column, which is why it is not smaller. +const EVICTION_CHUNK: usize = 64 * 1024; + +/// Accumulates the view's removals from bounded chunks of provenance ids into +/// the one set of fragment changes the refresh publishes. +struct Eviction { + /// The view as the chunks staged so far leave it. A chunk's delete has to + /// see the deletion vectors the earlier ones wrote, or it stages a + /// fragment that drops them. + snapshot: Dataset, + chunk: usize, + updated: HashMap, + removed: Vec, + pending: Vec, + staged: bool, + /// Largest the buffer ever got, which is the bound under test. + #[cfg(test)] + peak: usize, +} + +impl Eviction { + fn new(view_ds: &Dataset, chunk: usize) -> Self { + Self { + snapshot: view_ds.clone(), + chunk, + updated: HashMap::new(), + removed: Vec::new(), + pending: Vec::with_capacity(chunk), + staged: false, + #[cfg(test)] + peak: 0, + } + } + + /// Fill a chunk at a time. Taking the batch whole and splitting it would + /// hold a fragment's worth of ids and recopy the tail per chunk. + async fn push(&mut self, ids: &UInt64Array) -> Result<()> { + for id in ids.values() { + self.pending.push(*id); + #[cfg(test)] + { + self.peak = self.peak.max(self.pending.len()); + } + if self.pending.len() == self.chunk { + self.flush().await?; + } + } + Ok(()) + } + + /// Stage what is buffered, keeping the buffer's allocation. + async fn flush(&mut self) -> Result<()> { + let mut chunk = std::mem::take(&mut self.pending); + let staged = self.stage(&chunk).await; + chunk.clear(); + self.pending = chunk; + staged + } + + /// The staged fragment changes, or `None` where nothing was evicted. + async fn finish(mut self) -> Result, Vec)>> { + if !self.pending.is_empty() { + self.flush().await?; + } + if !self.staged { + return Ok(None); + } + let mut updated: Vec = self.updated.into_values().collect(); + updated.sort_unstable_by_key(|f| f.id); + Ok(Some((updated, self.removed))) + } + + async fn stage(&mut self, ids: &[u64]) -> Result<()> { + let (updated, removed) = stage_eviction(&self.snapshot, ids).await?; + self.snapshot = advance(&self.snapshot, &updated); + for fragment in updated { + self.updated.insert(fragment.id, fragment); + } + self.removed.extend(removed); + self.staged = true; + Ok(()) + } +} + +/// The view as staged removals leave it, without committing them. Fragments +/// are replaced in place so the fragment bitmap stays in step; an emptied one +/// is left alone, since the delta names each provenance id only once. +fn advance(view_ds: &Dataset, updated: &[Fragment]) -> Dataset { + if updated.is_empty() { + return view_ds.clone(); + } + let by_id: HashMap = updated.iter().map(|f| (f.id, f)).collect(); + let fragments = view_ds + .manifest + .fragments + .iter() + .map(|f| by_id.get(&f.id).map_or_else(|| f.clone(), |u| (*u).clone())) + .collect(); + let mut manifest = view_ds.manifest.as_ref().clone(); + manifest.fragments = Arc::new(fragments); + let mut snapshot = view_ds.clone(); + snapshot.manifest = Arc::new(manifest); + snapshot +} + +/// The fragment changes that remove the view's rows for `ids`, staged rather +/// than committed so they can ride in the refresh's single data commit. +async fn stage_eviction(view_ds: &Dataset, ids: &[u64]) -> Result<(Vec, Vec)> { + // An expression rather than SQL text: the id list is a value here, not a + // predicate string that grows with the delta and has to be parsed. + let predicate = col(SOURCE_ROW_ID_COLUMN).in_list( + ids.iter() + .map(|id| lit(ScalarValue::UInt64(Some(*id)))) + .collect(), + false, + ); + let staged = DeleteBuilder::from_expr(Arc::new(view_ds.clone()), predicate) + .execute_uncommitted() + .await?; + let Operation::Delete { + updated_fragments, + deleted_fragment_ids, + .. + } = staged.transaction.operation + else { + return Err(Error::Runtime { + message: "expected a delete when staging the view's evictions".into(), + }); + }; + Ok((updated_fragments, deleted_fragment_ids)) +} + +fn source_row_id_field_id(view_ds: &Dataset) -> Result { + view_ds + .schema() + .field(SOURCE_ROW_ID_COLUMN) + .map(|f| f.id) + .ok_or_else(|| Error::Runtime { + message: format!("the view has no '{SOURCE_ROW_ID_COLUMN}' column"), + }) +} + +/// Tee the provenance ids of everything written into `keys`. +fn collect_source_row_ids( + stream: SendableRecordBatchStream, + keys: Arc>, + written: Arc, +) -> SendableRecordBatchStream { + let schema = stream.schema(); + let mapped = stream.map(move |batch| { + let batch = batch?; + let column = batch + .column_by_name(SOURCE_ROW_ID_COLUMN) + .ok_or_else(|| { + DataFusionError::Internal(format!( + "'{SOURCE_ROW_ID_COLUMN}' is missing from the rows being written" + )) + })? + .as_primitive_opt::() + .ok_or_else(|| { + DataFusionError::Internal(format!("'{SOURCE_ROW_ID_COLUMN}' is not a uint64")) + })?; + let mut keys = keys + .lock() + .map_err(|_| DataFusionError::Internal("provenance key filter poisoned".into()))?; + for id in column.values() { + keys.insert(KeyValue::UInt64(*id)) + .map_err(|e| DataFusionError::Internal(e.to_string()))?; + } + written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + Ok(batch) + }); + Box::pin(RecordBatchStreamAdapter::new(schema, mapped)) +} + +#[cfg(test)] +mod tests { + + /// Park a refresh between planning and publication so a test can move + /// the view underneath it. Inert unless [`DRIFT_TARGET`] names this view. + pub(super) async fn hold_before_publish(uri: &str) { + { + let mut target = DRIFT_TARGET.lock().unwrap(); + if target.as_deref() != Some(uri) { + return; + } + // Take it: memory:// uris are relative and repeat across tests, so + // leaving it armed would park an unrelated refresh forever. + *target = None; + } + DRIFT_PLANNED.notify_one(); + DRIFT_RELEASED.notified().await; + } + + /// The rendezvous below is one global pair, so the cases that use it run + /// one at a time rather than trading each other's signals. + pub(super) static EVICTION_CAP: StdMutex> = StdMutex::new(None); + pub(super) fn eviction_cap_override() -> Option { + *EVICTION_CAP.lock().unwrap() + } + + pub(super) static DRIFT_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + pub(super) static DRIFT_TARGET: StdMutex> = StdMutex::new(None); + pub(super) static DRIFT_PLANNED: tokio::sync::Notify = tokio::sync::Notify::const_new(); + pub(super) static DRIFT_RELEASED: tokio::sync::Notify = tokio::sync::Notify::const_new(); + use arrow_array::{Int32Array, record_batch}; + use futures::TryStreamExt; + use lance::dataset::NewColumnTransform; + use lance_file::version::LanceFileVersion; + + use super::*; + use crate::connect; + use crate::connection::Connection; + use crate::index::Index; + use crate::index::scalar::BTreeIndexBuilder; + use crate::materialized_view::MaterializedView; + use crate::query::{ExecutableQuery, QueryBase, Select}; + use crate::table::{CompactionOptions, OptimizeAction}; + + async fn db_with_source(values: Vec) -> (Connection, Table) { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("x", Int32, values)).unwrap(); + let table = conn + .create_table("src", batch) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + (conn, table) + } + + async fn doubled_view(conn: &Connection) -> MaterializedView { + conn.create_materialized_view("doubled", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap() + } + + /// A refreshed doubled view over a source holding `values`. + async fn refreshed_doubled(values: Vec) -> (Connection, Table, MaterializedView) { + let (conn, source) = db_with_source(values).await; + let view = doubled_view(&conn).await; + view.refresh().execute().await.unwrap(); + (conn, source, view) + } + + async fn read(table: &Table, column: &str) -> Vec { + let batches = table + .query() + .select(Select::columns(&[column])) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut values: Vec = batches + .iter() + .flat_map(|batch| { + batch[column] + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .flatten() + .collect::>() + }) + .collect(); + values.sort(); + values + } + + async fn append(table: &Table, values: Vec) { + let batch = record_batch!(("x", Int32, values)).unwrap(); + table.add(batch).execute().await.unwrap(); + } + + #[tokio::test] + async fn test_first_refresh_materializes_the_view() { + let (conn, _) = db_with_source(vec![1, 2, 3]).await; + let view = doubled_view(&conn).await; + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(result.rows_written, 3); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]); + + // The watermark survives on the stored schema, not just the handle. + let reopened = conn.open_materialized_view("doubled").await.unwrap(); + let again = reopened.refresh().execute().await.unwrap(); + assert_eq!(again.mode, RefreshMode::NoOp); + assert_eq!(again.rows_written, 0); + } + + #[tokio::test] + async fn test_filter_selects_the_source_rows() { + let (conn, _) = db_with_source(vec![1, 20, 3, 40]).await; + let view = conn + .create_materialized_view("big", "src") + .select([("x", "x")]) + .only_if("x > 10") + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 2); + assert_eq!(read(view.table(), "x").await, vec![20, 40]); + } + + #[tokio::test] + async fn test_append_refreshes_incrementally() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await; + + append(&source, vec![5]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 10]); + } + + #[tokio::test] + async fn test_incremental_applies_the_filter() { + let (conn, source) = db_with_source(vec![1, 20]).await; + let view = conn + .create_materialized_view("big", "src") + .select([("x", "x")]) + .only_if("x > 10") + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + append(&source, vec![3, 30]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1); + assert_eq!(read(view.table(), "x").await, vec![20, 30]); + } + + /// The watermark has to advance even when no appended row matches, or the + /// same fragments would be rescanned by every later refresh. + #[tokio::test] + async fn test_incremental_with_nothing_matching_advances_the_watermark() { + let (conn, source) = db_with_source(vec![20]).await; + let view = conn + .create_materialized_view("big", "src") + .select([("x", "x")]) + .only_if("x > 10") + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + append(&source, vec![1, 2]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + + let again = view.refresh().execute().await.unwrap(); + assert_eq!(again.mode, RefreshMode::NoOp); + } + + /// Unlike a computed column, a view reflects source mutation: an update + /// rebuilds rather than going stale. + #[tokio::test] + async fn test_update_replaces_the_rows_it_changed() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + + // An update changes rows in place, so the view replaces exactly + // those rows and leaves the rest of what it holds alone. + source + .update() + .column("x", "20") + .only_if("x = 2") + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1, "only the changed row is recomputed"); + assert_eq!(read(view.table(), "twice").await, vec![2, 6, 40]); + } + + /// Legacy storage cannot serve the row-version columns update discovery + /// scans; appends stay incremental, updates rebuild rather than fail. + #[tokio::test] + async fn test_a_legacy_storage_source_is_reconciled() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("x", Int32, [1, 2, 3])).unwrap(); + let source = conn + .create_table("legacy_src", batch) + .write_options(crate::table::WriteOptions { + lance_write_params: Some(lance::dataset::WriteParams { + enable_stable_row_ids: true, + data_storage_version: Some(LanceFileVersion::Legacy), + ..Default::default() + }), + }) + .execute() + .await + .unwrap(); + let view = conn + .create_materialized_view("legacy_doubled", "legacy_src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + source + .add(record_batch!(("x", Int32, [4])).unwrap()) + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6, 8]); + + source + .update() + .column("x", "20") + .only_if("x = 2") + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![2, 6, 8, 40]); + } + + #[tokio::test] + async fn test_delete_evicts_the_view_rows_it_removed() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + + // A delete removes source rows without changing the ones that + // remain, so the view evicts exactly those rows and keeps the rest + // rather than recomputing every row it already held. + source.delete("x = 2").await.unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![2, 6]); + + // A delete and an append in one span: both are applied. + source.delete("x = 1").await.unwrap(); + source + .add(record_batch!(("x", Int32, vec![4])).unwrap()) + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![6, 8]); + } + + /// merge_insert commits an `Update`, and its by-source arm removes rows + /// rather than changing them, so the classifier must treat that + /// transaction form as a source of deletions. + #[tokio::test] + async fn test_merge_insert_by_source_delete_evicts_the_view_rows() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + + let batch = record_batch!(("x", Int32, vec![1, 3])).unwrap(); + let reader = arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut merge = source.merge_insert(&["x"]); + merge.when_not_matched_by_source_delete(None); + merge.execute(Box::new(reader)).await.unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![2, 6]); + } + + /// A refresh may only certify the generation it planned from. A write + /// that lands between planning and publication is drift the refresh did + /// not account for, so it aborts rather than stamp it as materialized. + #[tokio::test(flavor = "multi_thread")] + async fn test_refresh_aborts_rather_than_certify_view_drift() { + let _serial = DRIFT_LOCK.lock().await; + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("drifting_view", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + let certified = source_version_of(view.table()).await; + + // The source loses a row, so the next refresh plans an eviction. + source.delete("x = 2").await.unwrap(); + + let uri = view + .table() + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .uri() + .to_string(); + *DRIFT_TARGET.lock().unwrap() = Some(uri); + let refreshing = tokio::spawn(async move { view.refresh().execute().await }); + + // Move the view once the refresh has planned against it. + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("the refresh never reached the publication boundary"); + let drifted = conn.open_table("drifting_view").execute().await.unwrap(); + drifted.delete("twice = 6").await.unwrap(); + DRIFT_RELEASED.notify_one(); + + // Publishing removals and additions as one change makes the drift a + // conflict lance itself rejects; a pure append, which touches no + // existing fragment, still relies on the generation check. + let err = refreshing.await.unwrap().unwrap_err(); + let message = err.to_string(); + assert!( + message.contains("raced this refresh") || message.contains("preempted by concurrent"), + "got {err:?}" + ); + // The watermark still names the generation that was actually proven. + assert_eq!(source_version_of(&drifted).await, certified); + } + + async fn source_version_of(table: &Table) -> Option { + table + .schema() + .await + .unwrap() + .metadata() + .get(SOURCE_VERSION_META_KEY) + .cloned() + } + + /// A transaction file carries placeholder ids for the fragments it + /// creates. Into an empty source those collide with the ids the commit + /// assigns, so treating them as already-materialized drops the very + /// first rows while the watermark still advances past them. + #[tokio::test] + async fn test_merge_into_an_empty_source_is_materialized() { + let (_conn, source, view) = refreshed_doubled(vec![]).await; + assert_eq!(read(view.table(), "twice").await, Vec::::new()); + + let batch = record_batch!(("x", Int32, vec![1, 2])).unwrap(); + let reader = arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut merge = source.merge_insert(&["x"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all(); + merge.execute(Box::new(reader)).await.unwrap(); + + view.refresh().execute().await.unwrap(); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + // A second refresh must not double them either. + view.refresh().execute().await.unwrap(); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// A refresh publishes what it removes and what it adds as one change, + /// so an update never exposes the view without the rows it is replacing. + #[tokio::test(flavor = "multi_thread")] + async fn test_an_update_is_never_visible_as_a_gap() { + let _serial = DRIFT_LOCK.lock().await; + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("atomic_view", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + source + .update() + .column("x", "x + 10") + .execute() + .await + .unwrap(); + + let uri = view + .table() + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .uri() + .to_string(); + *DRIFT_TARGET.lock().unwrap() = Some(uri); + let refreshing = tokio::spawn(async move { view.refresh().execute().await }); + + // Read the view while the refresh is staged but not yet published. + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("the refresh never reached the publication boundary"); + let midway = conn.open_table("atomic_view").execute().await.unwrap(); + assert_eq!( + read(&midway, "twice").await, + vec![2, 4, 6], + "the pre-refresh rows must still be there in full" + ); + DRIFT_RELEASED.notify_one(); + + refreshing.await.unwrap().unwrap(); + let after = conn.open_table("atomic_view").execute().await.unwrap(); + assert_eq!(read(&after, "twice").await, vec![22, 24, 26]); + } + + async fn provenance_by_x(table: &Table) -> HashMap { + let batches = table + .query() + .select(Select::columns(&["x", SOURCE_ROW_ID_COLUMN])) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + batches + .iter() + .flat_map(|batch| { + let xs = batch["x"].as_any().downcast_ref::().unwrap(); + let ids = batch[SOURCE_ROW_ID_COLUMN].as_primitive::(); + (0..batch.num_rows()) + .map(|i| (xs.value(i), ids.value(i))) + .collect::>() + }) + .collect() + } + + /// One delta batch is as large as a fragment's deletions, so it arrives + /// well past a chunk: it has to be drained a chunk at a time without ever + /// holding the batch, and the passes' deletion vectors have to accumulate + /// rather than replace each other. + #[tokio::test] + async fn test_a_chunked_eviction_removes_every_row_it_names() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2, 3, 4, 5, 6]).await; + + let native = view.table().as_native().unwrap(); + let view_ds = native.dataset.get().await.unwrap().as_ref().clone(); + let provenance = provenance_by_x(view.table()).await; + + let mut eviction = Eviction::new(&view_ds, 2); + let batch = UInt64Array::from_iter_values([1, 2, 3, 4].map(|x| provenance[&x])); + eviction.push(&batch).await.unwrap(); + assert_eq!( + eviction.peak, 2, + "a batch past the chunk was buffered whole" + ); + let staged = eviction.finish().await.unwrap(); + assert!(staged.is_some(), "four ids over a chunk of two stage twice"); + publish(&view_ds, staged, Vec::new(), None).await.unwrap(); + native.dataset.reload().await.unwrap(); + + assert_eq!(read(view.table(), "x").await, vec![5, 6]); + } + + /// Two first rebuilds materialize the same source rows; the loser must + /// key-conflict at commit and land nothing, or the view doubles. + #[tokio::test] + async fn test_a_raced_first_rebuild_lands_nothing() { + let _guard = DRIFT_LOCK.lock().await; + let (conn, _) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("raced_rebuild", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + let native = view.table().as_native().unwrap(); + let view_ds = native.dataset.get().await.unwrap().as_ref().clone(); + let uri = view_ds.uri().to_string(); + *DRIFT_TARGET.lock().unwrap() = Some(uri); + let racing = tokio::spawn(async move { view.refresh().execute().await }); + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("the rebuild never reached the publication boundary"); + + // The other process's first rebuild, reduced to its commit. Its + // source rows are DISJOINT from ours -- the sentinel alone must make + // the two rebuilds conflict. + let batch = record_batch!( + ("x", Int32, [8, 9]), + ("twice", Int32, [16, 18]), + ("__source_row_id", UInt64, [7u64, 8]) + ) + .unwrap(); + let schema = Arc::new(ArrowSchema::from(view_ds.schema())); + let batch = RecordBatch::try_new( + schema.clone(), + schema + .fields() + .iter() + .map(|f| batch.column_by_name(f.name()).unwrap().clone()) + .collect(), + ) + .unwrap(); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter([Ok(batch)]), + )); + let keys = Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new(vec![ + source_row_id_field_id(&view_ds).unwrap(), + ]))); + let stream = collect_source_row_ids(stream, keys.clone(), Arc::new(AtomicU64::new(0))); + let write_txn = InsertBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await + .unwrap(); + let Operation::Append { fragments } = write_txn.operation else { + panic!("expected an append"); + }; + let filter = { + let mut keys = keys.lock().unwrap(); + keys.insert(KeyValue::UInt64(super::REFRESH_TOKEN_ID)) + .unwrap(); + keys.build() + }; + CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .execute(Transaction::new( + view_ds.version().version, + Operation::Update { + removed_fragment_ids: Vec::new(), + updated_fragments: Vec::new(), + new_fragments: fragments, + fields_modified: Vec::new(), + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: Vec::new(), + update_mode: None, + inserted_rows_filter: Some(filter), + updated_fragment_offsets: None, + }, + None, + )) + .await + .unwrap(); + DRIFT_RELEASED.notify_one(); + + let err = racing.await.unwrap().unwrap_err(); + // The loser lands nothing: only the winner's rows remain. + let raced = conn.open_table("raced_rebuild").execute().await.unwrap(); + assert_eq!( + read(&raced, "twice").await, + vec![16, 18], + "the losing rebuild must not union with the winner ({err})" + ); + } + + /// An incremental refresh racing any other refresh must land nothing, + /// even when their planned rows are disjoint: the shared token makes the + /// commits conflict. + #[tokio::test] + async fn test_a_raced_incremental_lands_nothing() { + let _guard = DRIFT_LOCK.lock().await; + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("raced_incremental", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + append(&source, vec![4]).await; + + let native = view.table().as_native().unwrap(); + native.dataset.reload().await.unwrap(); + let view_ds = native.dataset.get().await.unwrap().as_ref().clone(); + *DRIFT_TARGET.lock().unwrap() = Some(view_ds.uri().to_string()); + let racing = tokio::spawn(async move { view.refresh().execute().await }); + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("the refresh never reached the publication boundary"); + + // The concurrent refresh, reduced to its commit: disjoint rows, the + // shared token alone must collide. + let batch = record_batch!( + ("x", Int32, [9]), + ("twice", Int32, [18]), + ("__source_row_id", UInt64, [8u64]) + ) + .unwrap(); + let schema = Arc::new(ArrowSchema::from(view_ds.schema())); + let batch = RecordBatch::try_new( + schema.clone(), + schema + .fields() + .iter() + .map(|f| batch.column_by_name(f.name()).unwrap().clone()) + .collect(), + ) + .unwrap(); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter([Ok(batch)]), + )); + let keys = empty_keys(&view_ds).unwrap(); + let stream = collect_source_row_ids(stream, keys.clone(), Arc::new(AtomicU64::new(0))); + let write_txn = InsertBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await + .unwrap(); + let Operation::Append { fragments } = write_txn.operation else { + panic!("expected an append"); + }; + let filter = refresh_filter(&keys).unwrap(); + CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .execute(Transaction::new( + view_ds.version().version, + Operation::Update { + removed_fragment_ids: Vec::new(), + updated_fragments: Vec::new(), + new_fragments: fragments, + fields_modified: Vec::new(), + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: Vec::new(), + update_mode: None, + inserted_rows_filter: Some(filter), + updated_fragment_offsets: None, + }, + None, + )) + .await + .unwrap(); + DRIFT_RELEASED.notify_one(); + + let err = racing.await.unwrap().unwrap_err(); + let raced = conn + .open_table("raced_incremental") + .execute() + .await + .unwrap(); + assert_eq!( + read(&raced, "twice").await, + vec![2, 4, 6, 18], + "the losing increment must not land its rows ({err})" + ); + } + + /// Past the eviction cap the refresh falls back to the streamed rebuild, + /// and the result is identical either way. + #[tokio::test] + async fn test_oversized_delta_falls_back_to_rebuild() { + let (conn, source) = db_with_source((1..=20).collect()).await; + let view = doubled_view(&conn).await; + view.refresh().execute().await.unwrap(); + + source.delete("x <= 10").await.unwrap(); + *tests::EVICTION_CAP.lock().unwrap() = Some(4); + let result = view.refresh().execute().await; + *tests::EVICTION_CAP.lock().unwrap() = None; + let result = result.unwrap(); + assert_eq!( + result.mode, + RefreshMode::Rebuild, + "ten evictions, cap of four" + ); + assert_eq!(read(view.table(), "x").await, (11..=20).collect::>()); + + // Under the cap the same shape stays incremental. + source.delete("x = 11").await.unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "x").await, (12..=20).collect::>()); + } + + /// A cap of zero is a view that holds nothing, not a view without a cap. + #[tokio::test] + async fn test_zero_limit_holds_no_rows() { + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("empty", "src") + .select([("x", "x")]) + .limit(0) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 0); + assert_eq!(view.table().count_rows(None).await.unwrap(), 0); + + // Still nothing after the source grows, and the watermark advances. + append(&source, vec![4]).await; + view.refresh().execute().await.unwrap(); + assert_eq!(view.table().count_rows(None).await.unwrap(), 0); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// An update rewrites a whole fragment. When it lands on one appended + /// since the watermark, that fragment also holds rows the update never + /// touched -- rows the recompute does not cover and the append set no + /// longer reaches. + #[tokio::test] + async fn test_update_touching_a_new_fragment_keeps_its_untouched_rows() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await; + + // One fragment, appended after the watermark, holding both a row the + // update will change and a row it will not. + append(&source, vec![3, 40]).await; + source + .update() + .column("x", "99") + .only_if("x = 3") + .execute() + .await + .unwrap(); + + view.refresh().execute().await.unwrap(); + assert_eq!( + read(view.table(), "twice").await, + vec![2, 4, 80, 198], + "a row appended into the updated fragment went missing" + ); + } + + async fn compact(source: &Table) { + source + .optimize(OptimizeAction::Compact { + options: CompactionOptions::default(), + remap_options: None, + }) + .await + .unwrap(); + } + + /// A compaction rearranges rows without changing them, so it costs the + /// view nothing: the watermark advances and no row is recomputed. + #[tokio::test] + async fn test_compaction_alone_refreshes_incrementally() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await; + append(&source, vec![3]).await; + view.refresh().execute().await.unwrap(); + + compact(&source).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// Rows appended after a compaction are separable through the transaction + /// log: only the appended fragments are computed. + #[tokio::test] + async fn test_append_after_compaction_stays_incremental() { + let (_conn, source, view) = refreshed_doubled(vec![1]).await; + append(&source, vec![2]).await; + view.refresh().execute().await.unwrap(); + + compact(&source).await; + append(&source, vec![3]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]); + } + + /// An append swallowed by a later compaction cannot be told apart from + /// the rows the view already holds, so the refresh rebuilds -- once -- + /// rather than duplicate or drop. + #[tokio::test] + async fn test_append_folded_into_compaction_rebuilds() { + let (_conn, source, view) = refreshed_doubled(vec![1]).await; + + append(&source, vec![2]).await; + compact(&source).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// The create-time gate holds across a drop-and-recreate of the source + /// under the same name. + #[tokio::test] + async fn test_refresh_refuses_a_recreated_source_without_stable_row_ids() { + let (conn, _, view) = refreshed_doubled(vec![1]).await; + + conn.drop_table("src", &[]).await.unwrap(); + let batch = record_batch!(("x", Int32, [9])).unwrap(); + conn.create_table("src", batch).execute().await.unwrap(); + + let err = view.refresh().execute().await.unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("stable row ids")) + ); + } + + /// A change to a column the view does not read is not a reason to + /// rebuild: the exact signature check is scoped to the view's inputs. + #[tokio::test] + async fn test_unrelated_column_change_does_not_rebuild() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await; + + source + .add_columns() + .transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new( + vec![arrow_schema::Field::new( + "unrelated", + arrow_schema::DataType::Int32, + true, + )], + )))) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + #[tokio::test] + async fn test_full_forces_a_rebuild() { + let (_conn, source, view) = refreshed_doubled(vec![1]).await; + + append(&source, vec![2]).await; + let result = view.refresh().full(true).execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(result.rows_written, 2); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// A cap and incremental reconciliation do not compose: rows skipped at + /// the cap fall behind the watermark, so room freed later cannot be + /// refilled from any delta. A capped view rebuilds instead, which its + /// own cap keeps cheap. + #[tokio::test] + async fn test_limited_view_rebuilds_rather_than_reconcile() { + let (conn, source) = db_with_source(vec![1, 2]).await; + let view = conn + .create_materialized_view("capped", "src") + .select([("x", "x")]) + .limit(2) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + assert_eq!(read(view.table(), "x").await, vec![1, 2]); + + append(&source, vec![3, 4]).await; + source + .update() + .column("x", "11") + .only_if("x = 1") + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + let held = read(view.table(), "x").await; + assert_eq!(held.len(), 2, "the cap holds: {held:?}"); + let selectable = read(&source, "x").await; + assert!( + held.iter().all(|x| selectable.contains(x)), + "{held:?} is not a subset of {selectable:?}" + ); + } + + #[tokio::test] + async fn test_limit_caps_the_view() { + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("capped", "src") + .select([("x", "x")]) + .limit(4) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 3); + + // The cap counts already-held rows, so only one appended row lands. + append(&source, vec![4, 5, 6]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1); + assert_eq!(view.table().count_rows(None).await.unwrap(), 4); + + // At the cap, later appends only move the watermark. + append(&source, vec![7]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 0); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// The whole point of the fragment-swap commit: a rebuild must never + /// leave the view without its index definitions. + #[tokio::test] + async fn test_rebuild_retains_indexes() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + + view.table() + .create_index(&["twice"], Index::BTree(BTreeIndexBuilder::default())) + .execute() + .await + .unwrap(); + assert_eq!(view.table().list_indices().await.unwrap().len(), 1); + + source + .update() + .column("x", "x + 10") + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(view.table().list_indices().await.unwrap().len(), 1); + assert_eq!(read(view.table(), "twice").await, vec![22, 24, 26]); + + // The swapped-in rows are reachable through an indexed query. + let batches = view + .table() + .query() + .only_if("twice = 24") + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 1); + } + + #[tokio::test] + async fn test_rebuild_of_an_empty_result_is_an_empty_view() { + let (conn, _) = db_with_source(vec![1, 2]).await; + let view = conn + .create_materialized_view("none", "src") + .select([("x", "x")]) + .only_if("x > 100") + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(result.rows_written, 0); + assert_eq!(view.table().count_rows(None).await.unwrap(), 0); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// Provenance: every view row records the source row that produced it. + #[tokio::test] + async fn test_source_row_ids_are_recorded() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2, 3]).await; + + let batches = view + .table() + .query() + .select(Select::columns(&[SOURCE_ROW_ID_COLUMN])) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let total: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total, 3); + for batch in &batches { + assert_eq!(batch[SOURCE_ROW_ID_COLUMN].null_count(), 0); + } + } + + #[tokio::test] + async fn test_dropping_a_source_input_fails_the_refresh() { + let (conn, source) = db_with_source(vec![1]).await; + let view = conn + .create_materialized_view("v", "src") + .select([("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + append(&source, vec![2]).await; + source + .add_columns() + .transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new( + vec![arrow_schema::Field::new( + "y", + arrow_schema::DataType::Int32, + true, + )], + )))) + .execute() + .await + .unwrap(); + source.drop_columns(&["x"]).await.unwrap(); + + let err = view.refresh().execute().await.unwrap_err(); + assert!(matches!(err, Error::Schema { message } if message.contains("'x'"))); + } + + /// A pinned refresh materializes the source as of `version`; catching up + /// to the appends beyond it stays incremental. + #[tokio::test] + async fn test_pinned_refresh_and_catch_up() { + let (conn, source) = db_with_source(vec![1]).await; + let view = doubled_view(&conn).await; + let pinned = source.version().await.unwrap(); + + append(&source, vec![2]).await; + let result = view + .refresh() + .source_version(pinned) + .execute() + .await + .unwrap(); + assert_eq!(result.source_version, pinned); + assert_eq!(read(view.table(), "twice").await, vec![2]); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// Views chain: a view's stable row ids and provenance column make it a + /// source like any other, and the default projection takes its declared + /// columns without copying its provenance. + #[tokio::test] + async fn test_a_view_can_source_another_view() { + let (conn, source) = db_with_source(vec![1, 2, 30]).await; + let first = doubled_view(&conn).await; + first.refresh().execute().await.unwrap(); + + let second = conn + .create_materialized_view("second", "doubled") + .only_if("twice > 10") + .execute() + .await + .unwrap(); + assert!( + second + .definition() + .projections + .iter() + .all(|p| p.output != SOURCE_ROW_ID_COLUMN) + ); + let result = second.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 1); + assert_eq!(read(second.table(), "twice").await, vec![60]); + + append(&source, vec![50]).await; + first.refresh().execute().await.unwrap(); + let result = second.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(second.table(), "twice").await, vec![60, 100]); + } + + /// The watermark speaks only for the state a refresh left behind: a + /// direct write to the view is drift, and the next refresh rebuilds + /// rather than preserving it as current. + #[tokio::test] + async fn test_direct_view_mutation_forces_a_rebuild() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; + + view.table().delete("x = 1").await.unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// A dropped and recreated source reuses version numbers but never their + /// timestamps; the watermark must not vouch for the replacement's rows. + #[tokio::test] + async fn test_source_recreation_forces_a_rebuild() { + let (conn, _, view) = refreshed_doubled(vec![1]).await; + + conn.drop_table("src", &[]).await.unwrap(); + let batch = record_batch!(("x", Int32, [7])).unwrap(); + conn.create_table("src", batch) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![14]); + } + + /// In-process refreshes of one view serialize: the loser of the race + /// observes the winner's watermark instead of appending the same rows. + #[tokio::test(flavor = "multi_thread")] + async fn test_concurrent_refreshes_do_not_duplicate() { + let (_conn, source, view) = refreshed_doubled(vec![1]).await; + + append(&source, vec![2, 3]).await; + let (a, b) = tokio::join!(view.refresh().execute(), view.refresh().execute()); + let (a, b) = (a.unwrap(), b.unwrap()); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]); + let modes = [a.mode, b.mode]; + assert!(modes.contains(&RefreshMode::Incremental)); + assert!(modes.contains(&RefreshMode::NoOp)); + } + + /// A second handle's lazy cache must not defeat the lock: after another + /// handle's refresh commits, the stale handle plans from the reloaded + /// state and no-ops instead of appending the same fragments again. + #[tokio::test] + async fn test_a_second_handle_does_not_double_append() { + let (conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + let stale = conn.open_materialized_view("doubled").await.unwrap(); + + append(&source, vec![4]).await; + view.refresh().execute().await.unwrap(); + + let result = stale.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::NoOp); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6, 8]); + } + + /// A commit racing between a refresh's data commit and its stamp must + /// not be certified as the refresh's generation: the stamp aborts, and + /// the next refresh rebuilds from the drifted state. + #[tokio::test] + async fn test_stamp_aborts_on_a_racing_commit() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; + + let view_native = view.table().as_native().unwrap(); + let stale = view_native.dataset.get().await.unwrap().as_ref().clone(); + view.table().delete("x = 1").await.unwrap(); + + let err = stamp_watermark(view_native, stale, 99, 99).await; + assert!(err.is_err()); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// What refresh executes and what it stamps must be one generation: a + /// replaced definition wins over whatever a stale handle cached. + #[tokio::test] + async fn test_refresh_uses_the_latest_persisted_definition() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; + + let replacement = crate::materialized_view::MaterializedViewDefinition { + source_table: "src".into(), + projections: vec![ + crate::materialized_view::ViewProjection { + output: "x".into(), + expression: "x".into(), + }, + crate::materialized_view::ViewProjection { + output: "twice".into(), + expression: "x * 3".into(), + }, + ], + filter: None, + limit: None, + inputs: vec!["x".into()], + }; + let mut metadata = HashMap::new(); + metadata.insert( + crate::materialized_view::DEFINITION_META_KEY.to_string(), + crate::materialized_view::definition_to_metadata(&replacement).unwrap(), + ); + view.table() + .as_native() + .unwrap() + .replace_schema_metadata(metadata) + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![3, 6]); + } + + /// A persisted definition that does not produce this view's schema must + /// not refresh at all, let alone be certified. + #[tokio::test] + async fn test_definition_view_schema_mismatch_is_refused() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; + + let narrower = crate::materialized_view::MaterializedViewDefinition { + source_table: "src".into(), + projections: vec![crate::materialized_view::ViewProjection { + output: "x".into(), + expression: "x".into(), + }], + filter: None, + limit: None, + inputs: vec!["x".into()], + }; + let mut metadata = HashMap::new(); + metadata.insert( + crate::materialized_view::DEFINITION_META_KEY.to_string(), + crate::materialized_view::definition_to_metadata(&narrower).unwrap(), + ); + view.table() + .as_native() + .unwrap() + .replace_schema_metadata(metadata) + .await + .unwrap(); + + let err = view.refresh().execute().await.unwrap_err(); + assert!(matches!(err, Error::Schema { message } if message.contains("does not produce")),); + } + + /// An overlay replaces cell values without changing any file path; the + /// signature must see it, scoped to the columns the view reads like + /// data files are. + #[test] + fn test_fragment_signature_sees_overlays() { + use lance_file::version::ConcreteFileVersion; + use lance_table::format::DataFile; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + + let base = Fragment::new(7); + let mut file = DataFile::new_unstarted("f0.lance", ConcreteFileVersion::V2_1); + file.fields = vec![0, 1].into(); + let mut with_file = base.clone(); + with_file.files.push(file.clone()); + + let overlay = |field: i32| { + let mut data_file = DataFile::new_unstarted("o0.lance", ConcreteFileVersion::V2_1); + data_file.fields = vec![field].into(); + DataOverlayFile { + data_file, + coverage: OverlayCoverage::PerField(Vec::new()), + committed_version: 9, + } + }; + let relevant: HashSet = [0].into_iter().collect(); + + let mut overlaid_relevant = with_file.clone(); + overlaid_relevant.overlays.push(overlay(0)); + assert_ne!( + fragment_signature(&with_file, &relevant), + fragment_signature(&overlaid_relevant, &relevant), + ); + + let mut overlaid_unrelated = with_file.clone(); + overlaid_unrelated.overlays.push(overlay(5)); + assert_eq!( + fragment_signature(&with_file, &relevant), + fragment_signature(&overlaid_unrelated, &relevant), + ); + } + + /// An output whose name needs quoting flows through as a projection + /// alias, never spliced into SQL text. + #[tokio::test] + async fn test_output_names_needing_quotes() { + let (conn, _) = db_with_source(vec![1, 2]).await; + let view = conn + .create_materialized_view("v", "src") + .select([("double value", "x * 2")]) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 2); + assert_eq!(read(view.table(), "double value").await, vec![2, 4]); + } + + /// MemWAL tiers are visible to reads but not to the refresh scan, so LSM + /// state disqualifies every participant: the source at create, either + /// side at refresh, and the view can never accept a spec. Retained + /// un-compacted rows (the catch-up flag outlives unset) count as state. + #[tokio::test] + async fn lsm_state_disqualifies_source_and_view() { + use crate::table::LsmWriteSpec; + use arrow_array::RecordBatchIterator; + + let tmp_dir = tempfile::tempdir().unwrap(); + let conn = connect(tmp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + // Hand-rolled: the LSM primary key must be non-nullable, which + // record_batch! cannot express. + let schema = Arc::new(ArrowSchema::new(vec![ + arrow_schema::Field::new("id", arrow_schema::DataType::Int64, false), + arrow_schema::Field::new("x", arrow_schema::DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(arrow_array::Int64Array::from(vec![1, 2])) as _, + Arc::new(Int32Array::from(vec![1, 2])) as _, + ], + ) + .unwrap(); + let table = conn + .create_table("src", batch.clone()) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + + // An active-LSM source is refused at create. + let err = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap_err(); + assert!(err.to_string().contains("un-compacted"), "{err}"); + + // Unset with nothing written clears the state: the view creates, + // and a spec can never be installed over it. + table.unset_lsm_write_spec().await.unwrap(); + let view = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap(); + let err = view + .table() + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap_err(); + assert!(err.to_string().contains("materialized view"), "{err}"); + + // A source that acquires a spec after create fails refresh. + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + let err = view.refresh().execute().await.unwrap_err(); + assert!(err.to_string().contains("source table 'src'"), "{err}"); + + // Retained rows outlive unset: write through the WAL, unset, and + // refresh still refuses. + let mut merge = table.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all() + .use_lsm(true); + merge + .execute(Box::new(RecordBatchIterator::new( + vec![Ok(batch.clone())], + batch.schema(), + ))) + .await + .unwrap(); + table.unset_lsm_write_spec().await.unwrap(); + let err = view.refresh().execute().await.unwrap_err(); + assert!(err.to_string().contains("source table 'src'"), "{err}"); + } +} diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index 44a2874de..a06507ba0 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -104,6 +104,13 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) .into(), }); } + if crate::materialized_view::materialized_view_kind(&dataset.schema().metadata)?.is_some() { + return Err(Error::NotSupported { + message: "an LSM write spec cannot be installed on a materialized view: \ + rows in un-compacted tiers are invisible to refresh" + .into(), + }); + } let mut builder = dataset.initialize_mem_wal(); let writer_config_defaults = match spec { LsmWriteSpec::Bucket { diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 2715a163c..7d74bbae7 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -597,7 +597,7 @@ mod tests { /// A fragment spanning several scan batches exercises the streamed fill: /// the probe buffers only until the first gained value and the rest flows - /// through write_column a batch at a time. + /// through write_columns a batch at a time. #[tokio::test] async fn test_refresh_streams_a_multi_batch_fragment() { let values: Vec = (0..20_000).collect(); From d04ac7ed202181049cb871c2d6b97e5e5d9d54f0 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 21 Aug 2026 23:04:04 -0700 Subject: [PATCH 23/58] test: differential refresh harness for materialized views (#3932) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Example tests pin behaviors; the refresh contract is a property: after any sequence of source mutations, a view maintained by default refreshes equals the definition evaluated against the source directly, and so does a forced rebuild. This drives every mutation sequence up to length three -- appends, deletes, updates crossing the filter, compactions, unrelated column adds -- over an identity and a filtered view shape, checking against an oracle that shares nothing with the refresh path: a plain column scan with the filter applied in Rust. The oracle runs after every step because a later rebuild-forcing mutation silently heals an incremental error; end-state checks miss exactly the transient bugs that matter. A length-four sweep runs behind ignore. Named regressions additionally assert the refresh mode, which value comparison cannot: a wrongly rebuilding classifier still matches the oracle, so the append, unrelated-column and compaction cases pin that the incremental path actually ran. Stack created with GitHub Stacks CLI • Give Feedback 💬 --- rust/lancedb/src/materialized_view.rs | 4 + .../src/materialized_view/differential.rs | 730 ++++++++++++++++++ rust/lancedb/src/materialized_view/refresh.rs | 38 + 3 files changed, 772 insertions(+) create mode 100644 rust/lancedb/src/materialized_view/differential.rs diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 8b553165b..1d7cbb5e7 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -10,6 +10,10 @@ //! table. Queries, indexes and search work on the view unchanged. pub mod refresh; + +#[cfg(test)] +mod differential; + use std::collections::HashMap; use std::sync::Arc; diff --git a/rust/lancedb/src/materialized_view/differential.rs b/rust/lancedb/src/materialized_view/differential.rs new file mode 100644 index 000000000..396b4e65b --- /dev/null +++ b/rust/lancedb/src/materialized_view/differential.rs @@ -0,0 +1,730 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Differential refresh testing. +//! +//! The refresh contract is a property: after any sequence of source +//! mutations, a view maintained by default (incremental-where-possible) +//! refreshes equals the definition evaluated against the source directly, +//! and so does a forced rebuild. The oracle is an independent read of the +//! source -- plain column scan, filter applied in Rust -- so it shares +//! nothing with the refresh path it checks. +//! +//! The oracle runs after every step, not just at the end: a later mutation +//! that forces a rebuild would silently heal an incremental error, and those +//! transient errors are exactly the bugs this exists to catch. + +use arrow_array::{Float32Array, Int32Array, RecordBatch}; +use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; +use futures::{StreamExt, TryStreamExt}; +use lance::dataset::NewColumnTransform; +use std::sync::Arc; + +use super::MaterializedView; +use super::refresh::RefreshMode; +use crate::connect; +use crate::connection::Connection; +use crate::query::{ExecutableQuery, QueryBase, Select}; +use crate::table::{CompactionOptions, OptimizeAction, Table}; + +/// One source mutation, one per correctness-relevant class. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SrcOp { + /// Fresh non-colliding ids of both parities, so every other op has + /// view-resident rows to act on: the only op that should refresh + /// incrementally. + AppendNew, + /// Deletion in surviving fragments must break the pure-append check. + DeleteEven, + /// An in-place update; on the filtered shape it crosses the predicate, + /// so rows must leave the view. + UpdateOddScore, + /// Fragment rewrite/renumber must break the pure-append check. + Compact, + /// A column the view does not read must NOT force a rebuild. + AddColumn, + /// merge_insert commits an Update whose by-source arm deletes rows, so a + /// classifier that reads Update as "changed only" loses those deletions. + MergeDropLargest, + /// merge_insert that both changes existing rows and inserts new ones in + /// one transaction. + MergeUpsert, +} + +const ALL_OPS: [SrcOp; 7] = [ + SrcOp::AppendNew, + SrcOp::DeleteEven, + SrcOp::UpdateOddScore, + SrcOp::Compact, + SrcOp::AddColumn, + SrcOp::MergeDropLargest, + SrcOp::MergeUpsert, +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Shape { + /// SELECT id, score. + Identity, + /// SELECT id, score WHERE score > 50: additionally sensitive to rows + /// crossing the predicate. + Filtered, + /// SELECT id, score LIMIT 4. Which rows are held depends on the order + /// they were first materialized, so the oracle checks containment and + /// the cap rather than equality. + Limited, +} + +impl Shape { + fn filter(&self) -> Option<&'static str> { + match self { + Self::Identity | Self::Limited => None, + Self::Filtered => Some("score > 50"), + } + } + + fn matches(&self, score: f32) -> bool { + match self { + Self::Identity | Self::Limited => true, + Self::Filtered => score > 50.0, + } + } + + fn limit(&self) -> Option { + match self { + Self::Limited => Some(4), + _ => None, + } + } +} + +struct Case { + conn: Connection, + source: Table, + view: MaterializedView, + shape: Shape, + next_id: i32, + added_columns: u32, +} + +fn rows_batch(ids: &[i32]) -> RecordBatch { + let scores: Vec = ids.iter().map(|id| (*id * 10) as f32).collect(); + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("score", DataType::Float32, true), + ])), + vec![ + Arc::new(Int32Array::from(ids.to_vec())), + Arc::new(Float32Array::from(scores)), + ], + ) + .unwrap() +} + +fn merge_batch(ids: &[i32]) -> RecordBatch { + let scores: Vec = ids.iter().map(|id| (*id * 10 + 5) as f32).collect(); + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("score", DataType::Float32, true), + ])), + vec![ + Arc::new(Int32Array::from(ids.to_vec())), + Arc::new(Float32Array::from(scores)), + ], + ) + .unwrap() +} + +impl Case { + async fn new(shape: Shape) -> Self { + let conn = connect("memory://").execute().await.unwrap(); + let source = conn + .create_table("src", rows_batch(&[1, 2, 3, 4])) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + let mut builder = conn + .create_materialized_view("view", "src") + .select([("id", "id"), ("score", "score")]); + if let Some(filter) = shape.filter() { + builder = builder.only_if(filter); + } + if let Some(limit) = shape.limit() { + builder = builder.limit(limit as u64); + } + let view = builder.execute().await.unwrap(); + Self { + conn, + source, + view, + shape, + next_id: 100, + added_columns: 0, + } + } + + async fn apply(&mut self, op: SrcOp) { + match op { + SrcOp::AppendNew => { + // Mixed parity: the middle id is odd, so UpdateOddScore always + // has a filter-matching appended row to evict. + let ids = vec![self.next_id, self.next_id + 101, self.next_id + 202]; + self.next_id += 303; + self.source.add(rows_batch(&ids)).execute().await.unwrap(); + } + SrcOp::DeleteEven => { + self.source.delete("id % 2 = 0").await.unwrap(); + } + SrcOp::UpdateOddScore => { + self.source + .update() + .column("score", "-1.0") + .only_if("id % 2 = 1") + .execute() + .await + .unwrap(); + } + SrcOp::Compact => { + self.source + .optimize(OptimizeAction::Compact { + options: CompactionOptions::default(), + remap_options: None, + }) + .await + .unwrap(); + } + SrcOp::MergeDropLargest => { + let mut ids = self.source_ids().await; + ids.sort_unstable(); + ids.pop(); + if ids.is_empty() { + return; + } + let batch = rows_batch(&ids); + let reader = + arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut merge = self.source.merge_insert(&["id"]); + merge.when_not_matched_by_source_delete(None); + merge.execute(Box::new(reader)).await.unwrap(); + } + SrcOp::MergeUpsert => { + let mut ids = self.source_ids().await; + ids.sort_unstable(); + // One row that exists (updated in place) and one that does not. + let existing = ids.first().copied().unwrap_or(self.next_id); + let fresh = self.next_id; + self.next_id += 1; + let batch = merge_batch(&[existing, fresh]); + let reader = + arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut merge = self.source.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all(); + merge.execute(Box::new(reader)).await.unwrap(); + } + SrcOp::AddColumn => { + self.added_columns += 1; + let field = ArrowField::new( + format!("extra_{}", self.added_columns), + DataType::Int32, + true, + ); + self.source + .add_columns() + .transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new( + vec![field], + )))) + .execute() + .await + .unwrap(); + } + } + } + + async fn source_ids(&self) -> Vec { + read_rows( + self.source + .query() + .select(Select::columns(&["id", "score"])), + ) + .await + .into_iter() + .map(|(id, _)| id) + .collect() + } + + /// The definition's result, read independently of the refresh path: + /// plain column scan, filter applied here, sorted. + async fn oracle(&self) -> Vec<(i32, i32)> { + let mut rows = read_rows( + self.source + .query() + .select(Select::columns(&["id", "score"])), + ) + .await + .into_iter() + .filter(|(_, score)| self.shape.matches(*score as f32)) + .collect::>(); + rows.sort_unstable(); + rows + } + + async fn view_rows(&self) -> Vec<(i32, i32)> { + let mut rows = read_rows( + self.view + .table() + .query() + .select(Select::columns(&["id", "score"])), + ) + .await; + rows.sort_unstable(); + rows + } + + async fn check(&self, label: &str) -> Result<(), String> { + let expected = self.oracle().await; + let actual = self.view_rows().await; + let Some(cap) = self.shape.limit() else { + if expected != actual { + return Err(format!( + "{label}: view diverged from oracle\n expected: {expected:?}\n actual: {actual:?}" + )); + } + return Ok(()); + }; + // A capped view holds some subset of the definition's result, never + // more than the cap, and never the same row twice. + if actual.len() > cap { + return Err(format!( + "{label}: view holds {} rows, over its cap of {cap}: {actual:?}", + actual.len() + )); + } + let mut unique = actual.clone(); + unique.dedup(); + if unique.len() != actual.len() { + return Err(format!("{label}: view holds a row twice: {actual:?}")); + } + if let Some(stray) = actual.iter().find(|row| !expected.contains(row)) { + return Err(format!( + "{label}: view holds {stray:?}, which the definition does not select: {expected:?}" + )); + } + // Below the cap the view must be complete, or a row was lost. + if actual.len() < cap.min(expected.len()) { + return Err(format!( + "{label}: view holds {} of {} selectable rows under a cap of {cap}: {actual:?}", + actual.len(), + expected.len() + )); + } + Ok(()) + } +} + +async fn read_rows(query: impl ExecutableQuery) -> Vec<(i32, i32)> { + let batches = query + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + batches + .iter() + .flat_map(|batch| { + let ids = batch["id"].as_any().downcast_ref::().unwrap(); + let scores = batch["score"] + .as_any() + .downcast_ref::() + .unwrap(); + // Scores are integer-valued by construction; compare exactly. + (0..batch.num_rows()) + .map(|i| (ids.value(i), scores.value(i) as i32)) + .collect::>() + }) + .collect() +} + +/// Drive one mutation sequence: refresh + oracle-check after every step, +/// then a forced rebuild checked against the same oracle. +async fn run_sequence(ops: &[SrcOp], shape: Shape) -> Result<(), String> { + let label = format!("{shape:?} {ops:?}"); + let mut case = Case::new(shape).await; + case.view + .refresh() + .execute() + .await + .map_err(|e| format!("{label}: initial refresh failed: {e}"))?; + case.check(&format!("{label} (initial)")).await?; + + for (step, op) in ops.iter().enumerate() { + case.apply(*op).await; + case.view + .refresh() + .execute() + .await + .map_err(|e| format!("{label}: refresh at step {step} failed: {e}"))?; + case.check(&format!("{label} (step {step}, {op:?})")) + .await?; + } + + case.view + .refresh() + .full(true) + .execute() + .await + .map_err(|e| format!("{label}: final full refresh failed: {e}"))?; + case.check(&format!("{label} (final rebuild)")).await?; + // Silence the unused-connection lint without dropping it mid-case. + let _ = &case.conn; + Ok(()) +} + +/// Every op sequence up to `max_len`. +fn all_sequences(max_len: u32) -> Vec> { + let mut sequences = Vec::new(); + for len in 1..=max_len { + for mut index in 0..ALL_OPS.len().pow(len) { + let mut ops = Vec::with_capacity(len as usize); + for _ in 0..len { + ops.push(ALL_OPS[index % ALL_OPS.len()]); + index /= ALL_OPS.len(); + } + sequences.push(ops); + } + } + sequences +} + +async fn run_exhaustive(max_len: u32) { + let mut cases = Vec::new(); + for shape in [Shape::Identity, Shape::Filtered, Shape::Limited] { + for ops in all_sequences(max_len) { + cases.push((ops, shape)); + } + } + let failures: Vec = futures::stream::iter(cases) + .map(|(ops, shape)| async move { run_sequence(&ops, shape).await.err() }) + .buffer_unordered(8) + .filter_map(|failure| async move { failure }) + .collect() + .await; + assert!( + failures.is_empty(), + "{} sequences diverged; first: {}", + failures.len(), + failures[0] + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn differential_exhaustive() { + run_exhaustive(3).await; +} + +#[tokio::test(flavor = "multi_thread")] +#[ignore = "longer sweep; run manually"] +async fn differential_exhaustive_deep() { + run_exhaustive(4).await; +} + +/// Named interleavings that double as repro handles. The mode assertions pin +/// the classifier, which value comparison alone cannot: a wrongly rebuilt +/// view still matches the oracle. +#[tokio::test(flavor = "multi_thread")] +async fn differential_named_regressions() { + // An append is the one op that must stay incremental. + let mut case = Case::new(Shape::Identity).await; + case.view.refresh().execute().await.unwrap(); + case.apply(SrcOp::AppendNew).await; + let result = case.view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + case.check("append stays incremental").await.unwrap(); + + // A column the view does not read must not force a rebuild. + let mut case = Case::new(Shape::Identity).await; + case.view.refresh().execute().await.unwrap(); + case.apply(SrcOp::AddColumn).await; + let result = case.view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + + // Compaction rearranges rows without changing them: the watermark + // advances and nothing rebuilds. + let mut case = Case::new(Shape::Identity).await; + case.view.refresh().execute().await.unwrap(); + case.apply(SrcOp::AppendNew).await; + case.view.refresh().execute().await.unwrap(); + case.apply(SrcOp::Compact).await; + let result = case.view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + case.check("compaction alone").await.unwrap(); + + // Fragment bookkeeping stays coherent across the compaction: the next + // append is separable and computed alone. + case.apply(SrcOp::AppendNew).await; + let result = case.view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 3); + case.check("compact then append").await.unwrap(); + + // A row updated to no longer match the filter must leave the view -- + // and the fixture must prove the eviction happened, not merely that the + // end state matches: an update that never touched a view-resident row + // would also "match". + let mut case = Case::new(Shape::Filtered).await; + case.apply(SrcOp::AppendNew).await; + case.view.refresh().execute().await.unwrap(); + let before = case.view_rows().await.len(); + case.apply(SrcOp::UpdateOddScore).await; + case.view.refresh().execute().await.unwrap(); + let after = case.view_rows().await.len(); + assert!( + after < before, + "no view-resident row was evicted ({before} -> {after}); the fixture \ + no longer exercises the filtered-update transition" + ); + case.check("update crosses the filter").await.unwrap(); +} + +// --------------------------------------------------------------------------- +// Concurrency +// --------------------------------------------------------------------------- +// +// The sequential cases above cannot observe a cross-process race: the +// per-view refresh lock is process-local, so a second refresh in this +// process queues behind the first. What is missing is not more op +// sequences but a second process. These cases add one, and assert the same +// property the harness always asserts -- the view holds each row once. + +/// Rows the definition selects from the source: every id but the first, +/// read straight from the source, sharing nothing with the refresh path. +async fn concurrency_oracle(conn: &Connection) -> Vec { + let batches: Vec = conn + .open_table("src") + .execute() + .await + .unwrap() + .query() + .select(Select::columns(&["id"])) + .execute() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let mut ids = Vec::new(); + for batch in &batches { + let column = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + if column.value(i) > 1 { + ids.push(column.value(i)); + } + } + } + ids.sort_unstable(); + ids +} + +/// The view's ids, sorted. +async fn concurrency_view_ids(conn: &Connection) -> Vec { + let batches: Vec = conn + .open_table("mv") + .execute() + .await + .unwrap() + .query() + .select(Select::columns(&["id"])) + .execute() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let mut ids = Vec::new(); + for batch in &batches { + let column = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + ids.push(column.value(i)); + } + } + ids.sort_unstable(); + ids +} + +/// One refresh of the view at `MV_RACE_DIR`, in its own process. +/// +/// Setup happens before the start barrier so warm-up does not stagger the +/// two processes. What makes the race certain rather than likely is the +/// second barrier inside `refresh()` itself, which holds every participant +/// between staging and commit. +#[tokio::test] +#[ignore = "spawned as a child process by the concurrency cases"] +async fn cross_process_refresh_child() { + let Ok(dir) = std::env::var("MV_RACE_DIR") else { + return; + }; + let dir = std::path::PathBuf::from(dir); + let tag = std::env::var("MV_RACE_TAG").unwrap(); + + let conn = connect(dir.to_str().unwrap()).execute().await.unwrap(); + let table = conn.open_table("mv").execute().await.unwrap(); + let _ = table.schema().await.unwrap(); + let _ = table.count_rows(None).await.unwrap(); + let source = conn.open_table("src").execute().await.unwrap(); + let _ = source.count_rows(None).await.unwrap(); + let view = MaterializedView::from_table(table).await.unwrap(); + + std::fs::write(dir.join(format!("ready-{tag}")), b"1").unwrap(); + while !dir.join("START").exists() { + std::thread::sleep(std::time::Duration::from_millis(2)); + } + + let outcome = match view.refresh().execute().await { + Ok(result) => format!("committed rows={}", result.rows_written), + Err(err) if is_commit_conflict(&err) => "conflicted".to_string(), + Err(err) => format!("failed {err}"), + }; + std::fs::write(dir.join(format!("outcome-{tag}")), outcome).unwrap(); +} + +/// Whether a refresh lost its commit to a concurrent one, as opposed to +/// failing for any other reason. +fn is_commit_conflict(err: &crate::Error) -> bool { + let text = err.to_string(); + text.contains("Retryable commit conflict") || text.contains("preempted by concurrent") +} + +/// Two processes refreshing one view concurrently must leave the view +/// equal to the oracle: each selected row present exactly once. +/// +/// Both plan the same incremental delta from one watermark. A refresh is +/// meant to land on the generation it planned or leave nothing behind, so +/// at most one of them may write. +#[tokio::test] +async fn concurrent_refreshes_hold_each_row_once() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().to_str().unwrap().to_string(); + let conn = connect(&path).execute().await.unwrap(); + conn.create_table("src", rows_batch(&[1, 2, 3, 4])) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + let view = conn + .create_materialized_view("mv", "src") + .select([("id", "id"), ("score", "score")]) + .only_if("id > 1") + .execute() + .await + .unwrap(); + // Seed the watermark so the racing refreshes are both incremental. + view.refresh().execute().await.unwrap(); + + // Large enough that a refresh is real work rather than a formality. + let ids: Vec = (100..200_100).collect(); + conn.open_table("src") + .execute() + .await + .unwrap() + .add(rows_batch(&ids)) + .execute() + .await + .unwrap(); + + let tags = ["a", "b"]; + let exe = std::env::current_exe().unwrap(); + let children: Vec = tags + .iter() + .map(|tag| { + std::process::Command::new(&exe) + .args([ + "--exact", + "materialized_view::differential::cross_process_refresh_child", + "--ignored", + "--nocapture", + ]) + .env("MV_RACE_DIR", dir.path()) + .env("MV_RACE_SYNC", dir.path()) + .env("MV_RACE_PEERS", "2") + .env("MV_RACE_TAG", tag) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap() + }) + .collect(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(180); + while tags + .iter() + .any(|tag| !dir.path().join(format!("ready-{tag}")).exists()) + { + assert!( + std::time::Instant::now() < deadline, + "children never became ready" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + std::fs::write(dir.path().join("START"), b"1").unwrap(); + for (tag, mut child) in tags.iter().zip(children) { + let status = loop { + match child.try_wait().unwrap() { + Some(status) => break status, + None if std::time::Instant::now() >= deadline => { + child.kill().unwrap(); + panic!("child {tag} never finished"); + } + None => std::thread::sleep(std::time::Duration::from_millis(10)), + } + }; + assert!(status.success(), "child {tag} exited {status}"); + } + + // Both refreshes reached the commit boundary before either committed -- + // the in-refresh barrier guarantees it -- so exactly one may win. + let outcomes: Vec = tags + .iter() + .map(|tag| { + std::fs::read_to_string(dir.path().join(format!("outcome-{tag}"))) + .unwrap_or_else(|_| panic!("child {tag} recorded no outcome")) + }) + .collect(); + for tag in tags { + assert!( + dir.path().join(format!("planned-{tag}")).exists(), + "child {tag} never reached the commit boundary, so nothing was synchronized" + ); + } + let committed = outcomes.iter().filter(|o| o.contains("committed")).count(); + let conflicted = outcomes.iter().filter(|o| o.contains("conflicted")).count(); + assert_eq!( + (committed, conflicted), + (1, 1), + "exactly one refresh may win the generation both planned: {outcomes:?}" + ); + + let expected = concurrency_oracle(&conn).await; + let actual = concurrency_view_ids(&conn).await; + assert_eq!( + actual.len(), + expected.len(), + "the view holds {} rows, the oracle {}: a losing refresh left rows behind", + actual.len(), + expected.len() + ); + assert_eq!(actual, expected, "the view does not match the oracle"); +} diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index 7ac39a9f5..24d828e01 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -1056,6 +1056,8 @@ async fn publish( let planned = view_ds.version().version; #[cfg(test)] tests::hold_before_publish(view_ds.uri()).await; + #[cfg(test)] + tests::hold_until_peers_planned(); let (updated_fragments, removed_fragment_ids) = eviction.unwrap_or_default(); let committed = CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) .execute(Transaction::new( @@ -1342,6 +1344,42 @@ mod tests { pub(super) static DRIFT_TARGET: StdMutex> = StdMutex::new(None); pub(super) static DRIFT_PLANNED: tokio::sync::Notify = tokio::sync::Notify::const_new(); pub(super) static DRIFT_RELEASED: tokio::sync::Notify = tokio::sync::Notify::const_new(); + + /// Block until every participant in a cross-process race has planned and + /// staged its write, so the commits they then attempt genuinely contend + /// rather than depending on the scheduler to overlap them. Inert unless + /// `MV_RACE_SYNC` names a directory shared by the participants. + pub(super) fn hold_until_peers_planned() { + let (Ok(dir), Ok(tag), Ok(peers)) = ( + std::env::var("MV_RACE_SYNC"), + std::env::var("MV_RACE_TAG"), + std::env::var("MV_RACE_PEERS"), + ) else { + return; + }; + let dir = std::path::PathBuf::from(dir); + let peers: usize = peers.parse().unwrap(); + std::fs::write(dir.join(format!("planned-{tag}")), b"1").unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120); + while planned_count(&dir) < peers { + assert!( + std::time::Instant::now() < deadline, + "peers never reached the commit boundary" + ); + std::thread::sleep(std::time::Duration::from_millis(2)); + } + } + + fn planned_count(dir: &std::path::Path) -> usize { + std::fs::read_dir(dir) + .map(|entries| { + entries + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().starts_with("planned-")) + .count() + }) + .unwrap_or(0) + } use arrow_array::{Int32Array, record_batch}; use futures::TryStreamExt; use lance::dataset::NewColumnTransform; From 851fa16b47ecdde2b9aa116ad4100adfaf68279c Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 21 Aug 2026 23:08:41 -0700 Subject: [PATCH 24/58] feat(python): materialized view bindings (#3933) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposes materialized views to Python in both the async and sync clients: create_materialized_view / open_materialized_view / list_materialized_views on the connections, and MaterializedView / AsyncMaterializedView handles carrying the parsed definition and refresh(full=, source_version=), which returns the typed refresh result. select accepts column names, (alias, expression) pairs, or a dict of the same; the definition reads back off the stored schema, so a reopened handle needs no side channel. Remote connections raise NotImplementedError up front rather than failing deep in a request, matching the computed-column convention. Stack created with GitHub Stacks CLI • Give Feedback 💬 --- docs/src/python/python.md | 10 + python/python/lancedb/__init__.py | 8 + python/python/lancedb/_lancedb.pyi | 18 ++ python/python/lancedb/db.py | 166 +++++++++++ python/python/lancedb/materialized_view.py | 178 ++++++++++++ python/python/lancedb/namespace.py | 68 +++++ python/python/lancedb/remote/db.py | 27 ++ .../python/tests/test_materialized_views.py | 268 ++++++++++++++++++ python/src/connection.rs | 34 +++ python/src/lib.rs | 5 +- python/src/table.rs | 55 ++++ 11 files changed, 835 insertions(+), 2 deletions(-) create mode 100644 python/python/lancedb/materialized_view.py create mode 100644 python/python/tests/test_materialized_views.py diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 70b2a7207..8a24ea199 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -102,6 +102,12 @@ listing a storage directory. ::: lancedb.job.AsyncJob +## Materialized Views (Synchronous) + +::: lancedb.materialized_view.MaterializedView + +::: lancedb.materialized_view.MaterializedViewDefinition + ## Expressions Type-safe expression builder for filters and projections. Use these instead @@ -295,6 +301,10 @@ Table hold your actual data as a collection of records / rows. ::: lancedb.table.AsyncBranches +## Materialized Views (Asynchronous) + +::: lancedb.materialized_view.AsyncMaterializedView + ## Indices (Asynchronous) Indices can be created on a table to speed up queries. This section diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 0ceda4558..aa473c5f8 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -32,6 +32,11 @@ from .functions import ( UdfDefinition as UdfDefinition, udf as udf, ) +from .materialized_view import ( + AsyncMaterializedView, + MaterializedView, + MaterializedViewDefinition, +) from .table import AsyncTable, Table from .types import BaseTokenizerType from ._lancedb import Session @@ -506,6 +511,9 @@ async def connect_async( __all__ = [ + "AsyncMaterializedView", + "MaterializedView", + "MaterializedViewDefinition", "connect", "connect_async", "tokenize", diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 59537f45a..b23d79c85 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -197,6 +197,15 @@ class Connection(object): cur_namespace_path: Optional[List[str]] = None, new_namespace_path: Optional[List[str]] = None, ) -> None: ... + async def create_materialized_view( + self, + name: str, + source: str, + projections: Optional[List[Tuple[str, str]]] = None, + filter: Optional[str] = None, + limit: Optional[int] = None, + ) -> Table: ... + async def list_materialized_views(self) -> List[str]: ... async def drop_table( self, name: str, namespace_path: Optional[List[str]] = None ) -> None: ... @@ -355,6 +364,9 @@ class Table: ) -> AddColumnsResult: ... async def refresh_column(self, column: str) -> RefreshColumnResult: ... async def refresh_column_async(self, column: str) -> Job: ... + async def refresh_materialized_view( + self, full: bool = False, source_version: Optional[int] = None + ) -> RefreshMaterializedViewResult: ... async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ... async def alter_columns( self, columns: list[dict[str, Any]] @@ -704,6 +716,12 @@ class RefreshColumnResult: rows_filled: int version: int +class RefreshMaterializedViewResult: + mode: str + rows_written: int + source_version: int + version: int + class AlterColumnsResult: version: int diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index af18b6944..7ad749920 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -47,6 +47,12 @@ from . import __version__ from ._lancedb import connect as lancedb_connect # type: ignore from .functions import FunctionVersion, UdfDefinition from .job import AsyncJob, Job, _function_job +from .materialized_view import ( + AsyncMaterializedView, + MaterializedView, + SelectArg, + normalize_select, +) from .table import ( AsyncTable, LanceTable, @@ -510,6 +516,70 @@ class DBConnection(EnforceOverrides): """ raise NotImplementedError + def create_materialized_view( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> MaterializedView: + """Define a materialized view named ``name`` over the table ``source``. + + The view is created empty, with the query recorded in its schema + metadata; ``view.refresh()`` computes the rows. The view is a normal + table: it can be queried, indexed and searched, and it appears in + ``table_names``. Local databases only. + + The source table must have stable row ids (create it with the + ``new_table_enable_stable_row_ids`` storage option): they keep the + view's provenance valid across source compactions, and cannot be + enabled after a table exists. + + Parameters + ---------- + name: str + The name of the view. + source: str + The name of the source table, in this database. + select: list or dict, optional + The view's columns: column names, ``(alias, SQL expression)`` + pairs, or a dict of the same. Omitting it selects every source + column, expanded against the source schema at creation time. + where: str, optional + SQL predicate; only matching source rows appear in the view. + limit: int, optional + Cap the view at this many rows, in materialization order. + + Returns + ------- + MaterializedView + """ + raise NotImplementedError( + "materialized views are not supported on this connection type" + ) + + def open_materialized_view(self, name: str) -> MaterializedView: + """Open the materialized view named ``name``. + + Raises ``ValueError`` if the table exists but is not a materialized + view. + """ + raise NotImplementedError( + "materialized views are not supported on this connection type" + ) + + def list_materialized_views(self) -> List[str]: + """The names of the materialized views in this database. + + Found by reading every table's schema, so this costs an open per + table. + """ + raise NotImplementedError( + "materialized views are not supported on this connection type" + ) + def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): """Drop a table from the database. @@ -1136,6 +1206,58 @@ class LanceDBConnection(DBConnection): tbl.checkout(version) return tbl + @override + def create_materialized_view( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> MaterializedView: + """Define a materialized view named ``name`` over the table ``source``. + See + [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. + + Examples + -------- + >>> import lancedb + >>> db = lancedb.connect( + ... "./.lancedb", + ... storage_options={"new_table_enable_stable_row_ids": "true"}, + ... ) + >>> data = [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}] + >>> table = db.create_table("people", data) + >>> view = db.create_materialized_view( + ... "adults", + ... "people", + ... select=["name", ("shout", "upper(name)")], + ... where="age >= 18", + ... ) + >>> result = view.refresh() + >>> result.rows_written + 1 + """ + LOOP.run( + self._conn.create_materialized_view( + name, source, select=select, where=where, limit=limit + ) + ) + return MaterializedView(self.open_table(name)) + + @override + def open_materialized_view(self, name: str) -> MaterializedView: + """Open the materialized view named ``name``.""" + view = MaterializedView(self.open_table(name)) + view.definition + return view + + @override + def list_materialized_views(self) -> List[str]: + """The names of the materialized views in this database.""" + return LOOP.run(self._conn.list_materialized_views()) + def clone_table( self, target_table_name: str, @@ -1906,6 +2028,50 @@ class AsyncConnection(object): await tbl.checkout(version) return tbl + async def create_materialized_view( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> AsyncMaterializedView: + """Define a materialized view named ``name`` over the table ``source``. + See + [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. + """ + inner = await self._inner.create_materialized_view( + name, + source, + projections=normalize_select(select), + filter=where, + limit=limit, + ) + return AsyncMaterializedView(AsyncTable(inner)) + + async def open_materialized_view(self, name: str) -> AsyncMaterializedView: + """Open the materialized view named ``name``. + + Raises ``ValueError`` if the table exists but is not a materialized + view. + """ + if self.uri.startswith("db://"): + raise NotImplementedError( + "materialized views are supported only on local databases" + ) + view = AsyncMaterializedView(await self.open_table(name)) + await view.definition() + return view + + async def list_materialized_views(self) -> List[str]: + """The names of the materialized views in this database. + + Found by reading every table's schema, so this costs an open per + table. + """ + return await self._inner.list_materialized_views() + async def clone_table( self, target_table_name: str, diff --git a/python/python/lancedb/materialized_view.py b/python/python/lancedb/materialized_view.py new file mode 100644 index 000000000..5abb44dc0 --- /dev/null +++ b/python/python/lancedb/materialized_view.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Materialized views: tables defined by a query over a source table and +maintained by refresh. See ``DBConnection.create_materialized_view``.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union + +from .background_loop import LOOP + +if TYPE_CHECKING: + import pyarrow as pa + + from ._lancedb import RefreshMaterializedViewResult + from .table import AsyncTable, LanceTable + +DEFINITION_META_KEY = b"mv.definition" + +SelectArg = Union[ + str, + Sequence[Union[str, Tuple[str, str]]], + Dict[str, str], + None, +] + + +@dataclass +class MaterializedViewDefinition: + """The query that defines a materialized view.""" + + source_table: str + """Name of the source table, in the same database as the view.""" + projections: List[Tuple[str, str]] + """``(output column, SQL expression)`` pairs, in view schema order.""" + filter: Optional[str] = None + """SQL predicate selecting the source rows the view holds.""" + limit: Optional[int] = None + """Cap on the number of rows the view holds.""" + inputs: List[str] = field(default_factory=list) + """Source columns the projections and filter read.""" + + +def _definition_from_schema( + schema: "pa.Schema", name: str +) -> MaterializedViewDefinition: + metadata = schema.metadata or {} + raw = metadata.get(DEFINITION_META_KEY) + if raw is None: + raise ValueError(f"Table '{name}' is not a materialized view") + value = json.loads(raw) + kind = value.get("kind") + if kind != "select": + raise NotImplementedError( + f"materialized view '{name}' is defined by '{kind}', which this " + "version of lancedb cannot refresh" + ) + return MaterializedViewDefinition( + source_table=value["source_table"], + projections=[ + (p["output"], p["expression"]) for p in value.get("projections", []) + ], + filter=value.get("filter"), + limit=value.get("limit"), + inputs=value.get("inputs", []), + ) + + +def _quote_identifier(name: str) -> str: + """Quote a column name as a Lance SQL identifier (backticks).""" + escaped = name.replace("`", "``") + return f"`{escaped}`" + + +def normalize_select(select: SelectArg) -> Optional[List[Tuple[str, str]]]: + """``select`` items may be a column name, an ``(alias, expression)`` pair, + or a dict of the same. A bare name projects itself and is quoted, so any + valid column name works; dict and pair entries are kept verbatim because + their right side is an expression. + + A lone string is one column, not a sequence of its characters.""" + if select is None: + return None + if isinstance(select, str): + select = [select] + if isinstance(select, dict): + return list(select.items()) + normalized = [] + for item in select: + if isinstance(item, str): + normalized.append((item, _quote_identifier(item))) + else: + alias, expression = item + normalized.append((alias, expression)) + return normalized + + +class AsyncMaterializedView: + """A handle on a materialized view: its table plus its definition. + + Obtained from ``AsyncConnection.create_materialized_view`` or + ``AsyncConnection.open_materialized_view``. + """ + + def __init__(self, table: "AsyncTable"): + self._table = table + + def __repr__(self) -> str: + return f"AsyncMaterializedView(name={self.name!r})" + + @property + def name(self) -> str: + return self._table.name + + @property + def table(self) -> "AsyncTable": + """The view, as the table it is. Queries, indexes and search all + apply; writes are not blocked, but a rebuild replaces them.""" + return self._table + + async def definition(self) -> MaterializedViewDefinition: + """The query that defines the view, read from its stored schema.""" + return _definition_from_schema(await self._table.schema(), self.name) + + async def refresh( + self, *, full: bool = False, source_version: Optional[int] = None + ) -> "RefreshMaterializedViewResult": + """Recompute the view from its source. + + The refresh is incremental when the source's changes can be + reconciled into the view -- rows added, changed or removed since the + last one -- and otherwise rebuilds. ``full=True`` forces a rebuild; + ``source_version`` refreshes to that source version instead of the + latest. + + Concurrent refreshes of one view do not duplicate its rows. Two that + plan the same source rows conflict on commit, and the loser raises + rather than writing them a second time. + """ + return await self._table._inner.refresh_materialized_view( + full=full, source_version=source_version + ) + + +class MaterializedView: + """Synchronous variant of + [AsyncMaterializedView][lancedb.materialized_view.AsyncMaterializedView].""" + + def __init__(self, table: "LanceTable"): + self._table = table + self._async = AsyncMaterializedView(table._table) + + def __repr__(self) -> str: + return f"MaterializedView(name={self.name!r})" + + @property + def name(self) -> str: + return self._table.name + + @property + def table(self) -> "LanceTable": + """The view, as the table it is.""" + return self._table + + @property + def definition(self) -> MaterializedViewDefinition: + """The query that defines the view, read from its stored schema.""" + return _definition_from_schema(self._table.schema, self.name) + + def refresh( + self, *, full: bool = False, source_version: Optional[int] = None + ) -> "RefreshMaterializedViewResult": + """Recompute the view from its source. See + [AsyncMaterializedView.refresh][lancedb.materialized_view.AsyncMaterializedView.refresh].""" + return LOOP.run(self._async.refresh(full=full, source_version=source_version)) diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index 0e60bd218..f2e553321 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -61,6 +61,11 @@ from lance_namespace import ( NamespaceExistsRequest, TableExistsRequest, ) +from lancedb.materialized_view import ( + AsyncMaterializedView, + MaterializedView, + SelectArg, +) from lancedb.table import AsyncTable, LanceTable, Table from lancedb.util import validate_table_name from lancedb.common import DATA @@ -619,6 +624,42 @@ class LanceNamespaceDBConnection(DBConnection): tbl.checkout(version) return tbl + @override + def create_materialized_view( + self, + name: str, + source: str, + *, + select: "SelectArg" = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> "MaterializedView": + """Define a materialized view over a table in the root namespace. + See + [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. + """ + return MaterializedView( + self.open_table( + LOOP.run( + self._inner.create_materialized_view( + name, source, select=select, where=where, limit=limit + ) + ).name + ) + ) + + @override + def open_materialized_view(self, name: str) -> "MaterializedView": + """Open the materialized view named ``name``.""" + view = MaterializedView(self.open_table(name)) + view.definition + return view + + @override + def list_materialized_views(self) -> List[str]: + """The names of the materialized views in the root namespace.""" + return LOOP.run(self._inner.list_materialized_views()) + @override def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): if namespace_path is None: @@ -1141,6 +1182,33 @@ class AsyncLanceNamespaceDBConnection: route_pushdown_to_rust=self._route_pushdown_to_rust, ) + async def create_materialized_view( + self, + name: str, + source: str, + *, + select: "SelectArg" = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> "AsyncMaterializedView": + """Define a materialized view over a table in the root namespace.""" + view = await self._inner.create_materialized_view( + name, source, select=select, where=where, limit=limit + ) + # Reopen through the namespace so the view's table carries the + # namespace client and pushdown configuration a bare inner table lacks. + return AsyncMaterializedView(await self.open_table(view.name)) + + async def open_materialized_view(self, name: str) -> "AsyncMaterializedView": + """Open the materialized view named ``name``.""" + view = AsyncMaterializedView(await self.open_table(name)) + await view.definition() + return view + + async def list_materialized_views(self) -> List[str]: + """The names of the materialized views in the root namespace.""" + return await self._inner.list_materialized_views() + async def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): """Drop a table from the namespace.""" if namespace_path is None: diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index 822756a34..b228cfb5b 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -25,6 +25,7 @@ from ..common import DATA from ..db import DBConnection, LOOP from ..functions import FunctionVersion, UdfDefinition from ..job import AsyncJob, Job +from ..materialized_view import MaterializedView, SelectArg if TYPE_CHECKING: from .._lancedb import JobDescription, JobInfo @@ -648,6 +649,32 @@ class RemoteDBConnection(DBConnection): namespace_path=namespace_path, ) + @override + def create_materialized_view( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> MaterializedView: + raise NotImplementedError( + "materialized views are supported only on local databases" + ) + + @override + def open_materialized_view(self, name: str) -> MaterializedView: + raise NotImplementedError( + "materialized views are supported only on local databases" + ) + + @override + def list_materialized_views(self) -> List[str]: + raise NotImplementedError( + "materialized views are supported only on local databases" + ) + @override def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): """Drop a table from the database. diff --git a/python/python/tests/test_materialized_views.py b/python/python/tests/test_materialized_views.py new file mode 100644 index 000000000..5fa3aa4fb --- /dev/null +++ b/python/python/tests/test_materialized_views.py @@ -0,0 +1,268 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import lancedb +import pytest +from lancedb.materialized_view import MaterializedViewDefinition + + +STABLE_ROW_IDS = {"new_table_enable_stable_row_ids": "true"} + + +def make_db(tmp_path): + db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS) + db.create_table( + "people", + [ + {"name": "ada", "age": 36}, + {"name": "kid", "age": 7}, + {"name": "grace", "age": 85}, + ], + ) + return db + + +def test_create_refresh_and_query(tmp_path): + db = make_db(tmp_path) + view = db.create_materialized_view( + "adults", + "people", + select=["name", ("shout", "upper(name)")], + where="age >= 18", + ) + assert view.name == "adults" + assert view.table.count_rows() == 0 + + result = view.refresh() + assert result.mode == "rebuild" + assert result.rows_written == 2 + + rows = view.table.search().to_list() + assert sorted(row["shout"] for row in rows) == ["ADA", "GRACE"] + + +def test_definition_round_trips(tmp_path): + db = make_db(tmp_path) + db.create_materialized_view("adults", "people", where="age >= 18") + + view = db.open_materialized_view("adults") + assert view.definition == MaterializedViewDefinition( + source_table="people", + projections=[("name", "`name`"), ("age", "`age`")], + filter="age >= 18", + inputs=["age", "name"], + ) + + +def test_incremental_refresh_after_append(tmp_path): + db = make_db(tmp_path) + view = db.create_materialized_view("copy", "people") + view.refresh() + + db.open_table("people").add([{"name": "alan", "age": 41}]) + result = view.refresh() + assert result.mode == "incremental" + assert result.rows_written == 1 + assert view.table.count_rows() == 4 + + assert view.refresh().mode == "no_op" + + +def test_incremental_refresh_after_update(tmp_path): + db = make_db(tmp_path) + view = db.create_materialized_view("copy", "people") + view.refresh() + + db.open_table("people").update(where="name = 'kid'", values={"age": 8}) + result = view.refresh() + assert result.mode == "incremental" + assert result.rows_written == 1 + rows = view.table.search().to_list() + assert sorted(row["age"] for row in rows) == [8, 36, 85] + + +def test_legacy_storage_source_update_rebuilds(tmp_path): + db = lancedb.connect( + tmp_path, + storage_options={**STABLE_ROW_IDS, "new_table_data_storage_version": "legacy"}, + ) + db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]) + view = db.create_materialized_view("copy", "people") + view.refresh() + + db.open_table("people").update(where="name = 'kid'", values={"age": 8}) + result = view.refresh() + assert result.mode == "rebuild" + rows = view.table.search().to_list() + assert sorted(row["age"] for row in rows) == [8, 36] + + +def test_list_and_not_a_view(tmp_path): + db = make_db(tmp_path) + db.create_materialized_view("adults", "people", where="age >= 18") + + assert db.list_materialized_views() == ["adults"] + with pytest.raises(ValueError, match="not a materialized view"): + db.open_materialized_view("people") + + +def test_invalid_expression_fails_at_create(tmp_path): + db = make_db(tmp_path) + with pytest.raises(Exception, match="missing"): + db.create_materialized_view("bad", "people", select=[("x", "missing + 1")]) + assert "bad" not in db.list_tables().tables + + +@pytest.mark.asyncio +async def test_async_create_refresh_and_open(tmp_path): + db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) + await db.create_table("people", [{"name": "ada", "age": 36}]) + + view = await db.create_materialized_view( + "shouts", "people", select=[("shout", "upper(name)")] + ) + result = await view.refresh() + assert result.mode == "rebuild" + assert result.rows_written == 1 + + reopened = await db.open_materialized_view("shouts") + definition = await reopened.definition() + assert definition.projections == [("shout", "upper(name)")] + assert await db.list_materialized_views() == ["shouts"] + + +@pytest.mark.asyncio +async def test_async_incremental(tmp_path): + db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) + await db.create_table("people", [{"name": "ada", "age": 36}]) + view = await db.create_materialized_view("copy", "people") + await view.refresh() + + table = await db.open_table("people") + await table.add([{"name": "alan", "age": 41}]) + result = await view.refresh() + assert result.mode == "incremental" + assert result.rows_written == 1 + + +def test_source_requires_stable_row_ids(tmp_path): + db = lancedb.connect(tmp_path) + db.create_table("plain", [{"x": 1}]) + with pytest.raises(Exception, match="stable row ids"): + db.create_materialized_view("v", "plain") + + +def test_bare_select_names_are_quoted(tmp_path): + db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS) + db.create_table("odd_names", [{"order item": "widget", "select": 2}]) + + view = db.create_materialized_view( + "quoted", "odd_names", select=["order item", "select"] + ) + result = view.refresh() + assert result.rows_written == 1 + rows = view.table.search().to_list() + assert rows[0]["order item"] == "widget" + assert rows[0]["select"] == 2 + + +@pytest.mark.asyncio +async def test_async_remote_is_refused_without_network(): + db = await lancedb.connect_async( + "db://nowhere", api_key="sk_test", region="us-east-1" + ) + with pytest.raises(NotImplementedError, match="local"): + await db.create_materialized_view("v", "src") + with pytest.raises(NotImplementedError, match="local"): + await db.open_materialized_view("v") + with pytest.raises(NotImplementedError, match="local"): + await db.list_materialized_views() + + +def test_scalar_select_is_one_column(tmp_path): + db = make_db(tmp_path) + view = db.create_materialized_view("just_name", "people", select="name") + view.refresh() + rows = view.table.search().to_list() + assert set(rows[0]) - {"__source_row_id"} == {"name"} + assert sorted(row["name"] for row in rows) == ["ada", "grace", "kid"] + + +@pytest.mark.asyncio +async def test_async_scalar_select_is_one_column(tmp_path): + db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) + await db.create_table("people", [{"name": "ada", "age": 36}]) + view = await db.create_materialized_view("just_name", "people", select="name") + await view.refresh() + rows = await view.table.query().to_list() + assert set(rows[0]) - {"__source_row_id"} == {"name"} + + +def test_limit_above_i64_max_is_refused(tmp_path): + db = make_db(tmp_path) + with pytest.raises(ValueError, match="exceeds the maximum"): + db.create_materialized_view("too_big", "people", limit=2**63) + # The boundary is fine, and zero still means an empty view. + db.create_materialized_view("at_max", "people", limit=2**63 - 1) + empty = db.create_materialized_view("none", "people", limit=0) + empty.refresh() + assert empty.table.count_rows() == 0 + + +def _namespace_db(tmp_path): + return lancedb.connect_namespace( + "dir", + {"root": str(tmp_path)}, + storage_options=STABLE_ROW_IDS, + ) + + +def test_namespace_connection_materialized_views(tmp_path): + db = _namespace_db(tmp_path) + db.create_table( + "people", + [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}], + storage_options=STABLE_ROW_IDS, + ) + + view = db.create_materialized_view("adults", "people", where="age >= 18") + view.refresh() + assert view.table.count_rows() == 1 + assert db.list_materialized_views() == ["adults"] + + reopened = db.open_materialized_view("adults") + assert reopened.definition.source_table == "people" + with pytest.raises(ValueError, match="not a materialized view"): + db.open_materialized_view("people") + + +@pytest.mark.asyncio +async def test_async_namespace_connection_materialized_views(tmp_path): + db = lancedb.connect_namespace_async( + "dir", + {"root": str(tmp_path)}, + storage_options=STABLE_ROW_IDS, + ) + await db.create_table( + "people", + [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}], + storage_options=STABLE_ROW_IDS, + ) + + view = await db.create_materialized_view("adults", "people", where="age >= 18") + await view.refresh() + assert await view.table.count_rows() == 1 + assert await db.list_materialized_views() == ["adults"] + + reopened = await db.open_materialized_view("adults") + assert (await reopened.definition()).source_table == "people" + + # The view's table came through the namespace, not straight from the + # inner connection: a bare inner table carries no namespace context, so + # its pushdown routing differs from a table the namespace opened. + through_namespace = await db.open_table("adults") + for handle in (view.table, reopened.table): + assert ( + handle._route_pushdown_to_rust == through_namespace._route_pushdown_to_rust + ) + assert handle._namespace_path == through_namespace._namespace_path diff --git a/python/src/connection.rs b/python/src/connection.rs index 87870b800..4143ac9c9 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -333,6 +333,40 @@ impl Connection { }) } + #[pyo3(signature = (name, source, projections=None, filter=None, limit=None))] + pub fn create_materialized_view( + self_: PyRef<'_, Self>, + name: String, + source: String, + projections: Option>, + filter: Option, + limit: Option, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + future_into_py(self_.py(), async move { + let mut builder = inner.create_materialized_view(name, source); + if let Some(projections) = projections { + builder = builder.select(projections); + } + if let Some(filter) = filter { + builder = builder.only_if(filter); + } + if let Some(limit) = limit { + builder = builder.limit(limit); + } + let view = builder.execute().await.infer_error()?; + Ok(Table::new(view.table().clone())) + }) + } + + pub fn list_materialized_views(self_: PyRef<'_, Self>) -> PyResult> { + let inner = self_.get_inner()?.clone(); + future_into_py(self_.py(), async move { + let views = inner.list_materialized_views().await.infer_error()?; + Ok(views.into_iter().map(|view| view.name).collect::>()) + }) + } + #[pyo3(signature = (name, namespace_path=None))] pub fn drop_table( self_: PyRef<'_, Self>, diff --git a/python/src/lib.rs b/python/src/lib.rs index 756b3557f..c1dfbc02b 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -16,8 +16,8 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery}; use session::Session; use table::{ AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken, - LsmWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, Table, UpdateFieldMetadataResult, - UpdateResult, + LsmWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult, + Table, UpdateFieldMetadataResult, UpdateResult, }; pub mod arrow; @@ -60,6 +60,7 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/src/table.rs b/python/src/table.rs index cb4752cce..a3a7c9d68 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -441,6 +441,41 @@ impl From for RefreshColumnResult { } } +#[pyclass(get_all, from_py_object)] +#[derive(Clone, Debug)] +pub struct RefreshMaterializedViewResult { + pub mode: String, + pub rows_written: u64, + pub source_version: u64, + pub version: u64, +} + +#[pymethods] +impl RefreshMaterializedViewResult { + pub fn __repr__(&self) -> String { + format!( + "RefreshMaterializedViewResult(mode={}, rows_written={}, source_version={}, version={})", + self.mode, self.rows_written, self.source_version, self.version + ) + } +} + +impl From for RefreshMaterializedViewResult { + fn from(result: lancedb::RefreshMaterializedViewResult) -> Self { + let mode = match result.mode { + lancedb::RefreshMode::Rebuild => "rebuild", + lancedb::RefreshMode::Incremental => "incremental", + lancedb::RefreshMode::NoOp => "no_op", + }; + Self { + mode: mode.to_string(), + rows_written: result.rows_written, + source_version: result.source_version, + version: result.version, + } + } +} + #[pymethods] impl AddColumnsResult { pub fn __repr__(&self) -> String { @@ -1588,6 +1623,26 @@ impl Table { }) } + #[pyo3(signature = (full=false, source_version=None))] + pub fn refresh_materialized_view( + self_: PyRef<'_, Self>, + full: bool, + source_version: Option, + ) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + future_into_py(self_.py(), async move { + let view = lancedb::MaterializedView::from_table(inner) + .await + .infer_error()?; + let mut builder = view.refresh().full(full); + if let Some(version) = source_version { + builder = builder.source_version(version); + } + let result = builder.execute().await.infer_error()?; + Ok(RefreshMaterializedViewResult::from(result)) + }) + } + pub fn add_columns_with_schema( self_: PyRef<'_, Self>, schema: PyArrowType, From e98d8ac6857fa058fa07e005e0796c2a9a940d12 Mon Sep 17 00:00:00 2001 From: Drew Gallardo Date: Fri, 21 Aug 2026 23:37:12 -0700 Subject: [PATCH 25/58] feat!: rename branch merge to cherry_pick (#3986) This PR is a **breaking** rename of #3686. merge reads like git merge w/ three-way, replay history, combine two lines of work. That is not this API. This call takes one additive change on a branch and lands it on main. New column, including a blob column. Main's existing columns are not rewritten. If it cannot land, you get `status="failed"` and `diff.errors`, not a merge conflict to resolve. Cherry-pick is terminology that aligns more with that. ```python table = db.open_table("images") table.branches.create("exp") exp = table.branches.checkout("exp") exp.add_columns({"tag": "cast('draft' as string)"}) diff = table.branches.diff("exp") preview = table.branches.cherry_pick("exp", dry_run=True) result = table.branches.cherry_pick("exp") if result["status"] == "cherryPicked": print("landed at", result["mainVersionAfter"]) elif result["status"] == "failed": print(result["diff"]["errors"]) ``` ### Behavior - Remote / Enterprise only. Local still NotSupported. - HTTP 409 is not an exception. It is Ok with status="failed" and diff.errors (CherryPickError). - Unknown error / status codes still parse as Unknown. - Requests are not retried. 409 is final and carries the body. - Endpoint is POST /v1/table/{id}/branches/cherry_pick/. - merge_insert and Table.merge are unchanged. ### Testing - `cargo test -p lancedb --features remote diff_branch` - `cargo test -p lancedb --features remote cherry_pick` - `pytest python/python/tests/test_remote_db.py -k cherry_pick` - node `remote.test.ts` diffs / cherry-picks path --- docs/src/js/classes/Branches.md | 50 ++++++------ docs/src/js/globals.md | 6 +- docs/src/js/interfaces/BranchDiff.md | 24 ++---- .../{MergeBlocker.md => CherryPickError.md} | 6 +- docs/src/js/interfaces/CherryPickPreview.md | 17 +++++ ...rgeBranchResult.md => CherryPickResult.md} | 12 +-- docs/src/js/interfaces/MergePreview.md | 17 ----- nodejs/__test__/remote.test.ts | 26 +++---- nodejs/lancedb/index.ts | 6 +- nodejs/lancedb/table.ts | 37 +++++---- nodejs/src/table.rs | 6 +- python/python/lancedb/_lancedb.pyi | 2 +- python/python/lancedb/table.py | 22 +++--- python/python/tests/test_remote_db.py | 17 ++--- python/src/table.rs | 4 +- rust/lancedb/src/remote/table.rs | 76 ++++++++++--------- rust/lancedb/src/table.rs | 30 ++++---- .../table/{branch_merge.rs => cherry_pick.rs} | 27 ++++--- 18 files changed, 188 insertions(+), 197 deletions(-) rename docs/src/js/interfaces/{MergeBlocker.md => CherryPickError.md} (54%) create mode 100644 docs/src/js/interfaces/CherryPickPreview.md rename docs/src/js/interfaces/{MergeBranchResult.md => CherryPickResult.md} (60%) delete mode 100644 docs/src/js/interfaces/MergePreview.md rename rust/lancedb/src/table/{branch_merge.rs => cherry_pick.rs} (86%) diff --git a/docs/src/js/classes/Branches.md b/docs/src/js/classes/Branches.md index 5296d0d08..6680fbfee 100644 --- a/docs/src/js/classes/Branches.md +++ b/docs/src/js/classes/Branches.md @@ -37,6 +37,31 @@ latest and stays writable. *** +### cherryPick() + +```ts +cherryPick(fromBranch, dryRun): Promise +``` + +Cherry-pick a branch onto main. + +Set `dryRun` to `true` to preview. A failed cherry-pick resolves +with `status: "failed"` instead of throwing. + +#### Parameters + +* **fromBranch**: `string` + Branch to cherry-pick from. + +* **dryRun**: `boolean` = `false` + When true, only preview. Defaults to false. + +#### Returns + +`Promise`<[`CherryPickResult`](../interfaces/CherryPickResult.md)> + +*** + ### create() ```ts @@ -112,28 +137,3 @@ List all branches, mapping name to branch metadata. #### Returns `Promise`<`Record`<`string`, [`BranchContents`](BranchContents.md)>> - -*** - -### merge() - -```ts -merge(fromBranch, dryRun): Promise -``` - -Merge a branch into main. - -Set `dryRun` to `true` to preview the merge. A rejected merge resolves -with `status: "rejected"` instead of throwing. - -#### Parameters - -* **fromBranch**: `string` - Branch to merge from. - -* **dryRun**: `boolean` = `false` - When true, only preview the merge. Defaults to false. - -#### Returns - -`Promise`<[`MergeBranchResult`](../interfaces/MergeBranchResult.md)> diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 462907cfd..4deeed1bc 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -59,6 +59,9 @@ - [BranchIndexSummary](interfaces/BranchIndexSummary.md) - [BranchRowCountSummary](interfaces/BranchRowCountSummary.md) - [BucketStats](interfaces/BucketStats.md) +- [CherryPickError](interfaces/CherryPickError.md) +- [CherryPickPreview](interfaces/CherryPickPreview.md) +- [CherryPickResult](interfaces/CherryPickResult.md) - [ClientConfig](interfaces/ClientConfig.md) - [ColumnAlteration](interfaces/ColumnAlteration.md) - [ColumnOrdering](interfaces/ColumnOrdering.md) @@ -99,9 +102,6 @@ - [LsmStats](interfaces/LsmStats.md) - [LsmWriteSpec](interfaces/LsmWriteSpec.md) - [MemtableStats](interfaces/MemtableStats.md) -- [MergeBlocker](interfaces/MergeBlocker.md) -- [MergeBranchResult](interfaces/MergeBranchResult.md) -- [MergePreview](interfaces/MergePreview.md) - [MergeResult](interfaces/MergeResult.md) - [NativeOAuthConfig](interfaces/NativeOAuthConfig.md) - [OAuthConfig](interfaces/OAuthConfig.md) diff --git a/docs/src/js/interfaces/BranchDiff.md b/docs/src/js/interfaces/BranchDiff.md index 224be1991..b408ebe11 100644 --- a/docs/src/js/interfaces/BranchDiff.md +++ b/docs/src/js/interfaces/BranchDiff.md @@ -50,6 +50,14 @@ changedColumns: BranchColumnChange[]; *** +### errors + +```ts +errors: CherryPickError[]; +``` + +*** + ### fromBranch ```ts @@ -66,22 +74,6 @@ mainVersion: number; *** -### mergeBlockers - -```ts -mergeBlockers: MergeBlocker[]; -``` - -*** - -### mergeable - -```ts -mergeable: boolean; -``` - -*** - ### parentVersion ```ts diff --git a/docs/src/js/interfaces/MergeBlocker.md b/docs/src/js/interfaces/CherryPickError.md similarity index 54% rename from docs/src/js/interfaces/MergeBlocker.md rename to docs/src/js/interfaces/CherryPickError.md index 6c8f84b37..84f8fe012 100644 --- a/docs/src/js/interfaces/MergeBlocker.md +++ b/docs/src/js/interfaces/CherryPickError.md @@ -2,11 +2,11 @@ *** -[@lancedb/lancedb](../globals.md) / MergeBlocker +[@lancedb/lancedb](../globals.md) / CherryPickError -# Interface: MergeBlocker +# Interface: CherryPickError -A reason why a branch cannot currently be merged. +A reason why a cherry-pick cannot currently land. ## Properties diff --git a/docs/src/js/interfaces/CherryPickPreview.md b/docs/src/js/interfaces/CherryPickPreview.md new file mode 100644 index 000000000..9620068a2 --- /dev/null +++ b/docs/src/js/interfaces/CherryPickPreview.md @@ -0,0 +1,17 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / CherryPickPreview + +# Interface: CherryPickPreview + +Changes that would be, or were, promoted by a cherry-pick. + +## Properties + +### promotedColumns + +```ts +promotedColumns: string[]; +``` diff --git a/docs/src/js/interfaces/MergeBranchResult.md b/docs/src/js/interfaces/CherryPickResult.md similarity index 60% rename from docs/src/js/interfaces/MergeBranchResult.md rename to docs/src/js/interfaces/CherryPickResult.md index 61c2d0d33..c837ab143 100644 --- a/docs/src/js/interfaces/MergeBranchResult.md +++ b/docs/src/js/interfaces/CherryPickResult.md @@ -2,11 +2,11 @@ *** -[@lancedb/lancedb](../globals.md) / MergeBranchResult +[@lancedb/lancedb](../globals.md) / CherryPickResult -# Interface: MergeBranchResult +# Interface: CherryPickResult -Result of previewing or attempting a branch merge. +Result of previewing or attempting a cherry-pick. ## Properties @@ -29,7 +29,7 @@ optional mainVersionAfter: number; ### preview ```ts -preview: MergePreview; +preview: CherryPickPreview; ``` *** @@ -38,9 +38,9 @@ preview: MergePreview; ```ts status: + | "failed" | "unknown" - | "rejected" | "ready" | "notImplemented" - | "merged"; + | "cherryPicked"; ``` diff --git a/docs/src/js/interfaces/MergePreview.md b/docs/src/js/interfaces/MergePreview.md deleted file mode 100644 index 0d9717289..000000000 --- a/docs/src/js/interfaces/MergePreview.md +++ /dev/null @@ -1,17 +0,0 @@ -[**@lancedb/lancedb**](../README.md) • **Docs** - -*** - -[@lancedb/lancedb](../globals.md) / MergePreview - -# Interface: MergePreview - -Changes that would be, or were, promoted by a branch merge. - -## Properties - -### promotedColumns - -```ts -promotedColumns: string[]; -``` diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index e766b3d2a..01320d2b0 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -311,7 +311,7 @@ describe("remote connection", () => { expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]); }); - it("diffs and merges remote branches", async () => { + it("diffs and cherry-picks remote branches", async () => { const sampleDiff = { fromBranch: "exp", parentVersion: 1, @@ -333,10 +333,9 @@ describe("remote connection", () => { changedColumns: [], addedIndexes: [], removedIndexes: [], - mergeable: true, - mergeBlockers: [], + errors: [], }; - const mergeBodies: Record[] = []; + const cherryPickBodies: Record[] = []; await withMockDatabase( (req, res) => { @@ -366,17 +365,16 @@ describe("remote connection", () => { .end(JSON.stringify(sampleDiff)); return; } - if (path.endsWith("/branches/merge/")) { - mergeBodies.push(body); + if (path.endsWith("/branches/cherry_pick/")) { + cherryPickBodies.push(body); const dryRun = body["dry_run"] === true; const response = { - status: dryRun ? "ready" : "rejected", + status: dryRun ? "ready" : "failed", diff: dryRun ? sampleDiff : { ...sampleDiff, - mergeable: false, - mergeBlockers: [ + errors: [ { code: "baseMoved", message: "main has advanced" }, ], }, @@ -398,19 +396,19 @@ describe("remote connection", () => { await expect(branches.diff("exp")).resolves.toEqual(sampleDiff); - const rejected = await branches.merge("exp"); - expect(rejected.status).toBe("rejected"); - expect(rejected.diff.mergeBlockers).toEqual([ + const failed = await branches.cherryPick("exp"); + expect(failed.status).toBe("failed"); + expect(failed.diff.errors).toEqual([ { code: "baseMoved", message: "main has advanced" }, ]); - const preview = await branches.merge("exp", true); + const preview = await branches.cherryPick("exp", true); expect(preview.status).toBe("ready"); expect(preview.preview.promotedColumns).toEqual(["tag"]); }, ); - expect(mergeBodies).toEqual([ + expect(cherryPickBodies).toEqual([ // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format { from_branch: "exp", dry_run: false }, // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 6a5bfe3b4..aa9c7ef13 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -135,10 +135,10 @@ export { BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, - MergeBlocker, + CherryPickError, BranchDiff, - MergePreview, - MergeBranchResult, + CherryPickPreview, + CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 964c2cea3..4e323ec64 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -1557,8 +1557,8 @@ export interface BranchRowCountSummary { deltaAvailable: boolean; } -/** A reason why a branch cannot currently be merged. */ -export interface MergeBlocker { +/** A reason why a cherry-pick cannot currently land. */ +export interface CherryPickError { code: string; message: string; } @@ -1578,20 +1578,19 @@ export interface BranchDiff { changedColumns: BranchColumnChange[]; addedIndexes: BranchIndexSummary[]; removedIndexes: BranchIndexSummary[]; - mergeable: boolean; - mergeBlockers: MergeBlocker[]; + errors: CherryPickError[]; } -/** Changes that would be, or were, promoted by a branch merge. */ -export interface MergePreview { +/** Changes that would be, or were, promoted by a cherry-pick. */ +export interface CherryPickPreview { promotedColumns: string[]; } -/** Result of previewing or attempting a branch merge. */ -export interface MergeBranchResult { - status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown"; +/** Result of previewing or attempting a cherry-pick. */ +export interface CherryPickResult { + status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown"; diff: BranchDiff; - preview: MergePreview; + preview: CherryPickPreview; mainVersionAfter?: number; } @@ -1654,21 +1653,21 @@ export class Branches { } /** - * Merge a branch into main. + * Cherry-pick a branch onto main. * - * Set `dryRun` to `true` to preview the merge. A rejected merge resolves - * with `status: "rejected"` instead of throwing. + * Set `dryRun` to `true` to preview. A failed cherry-pick resolves + * with `status: "failed"` instead of throwing. * - * @param fromBranch Branch to merge from. - * @param dryRun When true, only preview the merge. Defaults to false. + * @param fromBranch Branch to cherry-pick from. + * @param dryRun When true, only preview. Defaults to false. */ - async merge( + async cherryPick( fromBranch: string, dryRun: boolean = false, - ): Promise { - return (await this.#inner.merge( + ): Promise { + return (await this.#inner.cherryPick( fromBranch, dryRun, - )) as unknown as MergeBranchResult; + )) as unknown as CherryPickResult; } } diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index b15491202..694b6a704 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -1605,18 +1605,18 @@ impl Branches { } #[napi(ts_return_type = "Promise>")] - pub async fn merge( + pub async fn cherry_pick( &self, from_branch: String, dry_run: Option, ) -> napi::Result { let result = self .inner - .merge_branch(&from_branch, dry_run.unwrap_or(false)) + .cherry_pick(&from_branch, dry_run.unwrap_or(false)) .await .default_error()?; serde_json::to_value(result).map_err(|err| { - napi::Error::from_reason(format!("failed to serialize branch merge result: {err}")) + napi::Error::from_reason(format!("failed to serialize cherry-pick result: {err}")) }) } } diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index b23d79c85..648469579 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -432,7 +432,7 @@ class Branches: async def checkout(self, name: str, version: Optional[int] = None) -> Table: ... async def delete(self, name: str) -> None: ... async def diff(self, from_branch: str) -> Dict[str, Any]: ... - async def merge( + async def cherry_pick( self, from_branch: str, dry_run: bool = False ) -> Dict[str, Any]: ... diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 913ab5289..0c84b4036 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -6801,21 +6801,21 @@ class Branches: """Diff a branch against main.""" return LOOP.run(self._table.branches.diff(from_branch)) - def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]: - """Merge a branch into main, or dry-run. + def cherry_pick(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]: + """Cherry-pick a branch onto main, or dry-run. Parameters ---------- from_branch: str - Branch to merge from. + Branch to cherry-pick from. dry_run: bool, default False - When True, only preview. When False, attempt the merge. + When True, only preview. When False, attempt the cherry-pick. Notes ----- - A rejected merge returns ``status="rejected"`` instead of raising. + A failed cherry-pick returns ``status="failed"`` instead of raising. """ - return LOOP.run(self._table.branches.merge(from_branch, dry_run)) + return LOOP.run(self._table.branches.cherry_pick(from_branch, dry_run)) def _wrap( self, async_table: "AsyncTable", version: Optional[int] = None @@ -6951,9 +6951,11 @@ class AsyncBranches: """Diff a branch against main.""" return await self._table.branches.diff(from_branch) - async def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]: - """Merge a branch into main, or dry-run. + async def cherry_pick( + self, from_branch: str, dry_run: bool = False + ) -> Dict[str, Any]: + """Cherry-pick a branch onto main, or dry-run. - A rejected merge returns ``status="rejected"`` instead of raising. + A failed cherry-pick returns ``status="failed"`` instead of raising. """ - return await self._table.branches.merge(from_branch, dry_run) + return await self._table.branches.cherry_pick(from_branch, dry_run) diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 13ffc4415..08f5550eb 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -242,8 +242,8 @@ def test_remote_table_branches_sync(): table.branches.delete("exp") -def test_remote_table_branch_merge_defaults_to_execute(): - merge_bodies = [] +def test_remote_table_cherry_pick_defaults_to_execute(): + cherry_pick_bodies = [] diff = { "fromBranch": "exp", "parentVersion": 1, @@ -265,8 +265,7 @@ def test_remote_table_branch_merge_defaults_to_execute(): "changedColumns": [], "addedIndexes": [], "removedIndexes": [], - "mergeable": True, - "mergeBlockers": [], + "errors": [], } def handler(request): @@ -276,11 +275,11 @@ def test_remote_table_branch_merge_defaults_to_execute(): else: content_len = int(request.headers.get("Content-Length")) request_body = json.loads(request.rfile.read(content_len)) - merge_bodies.append(request_body) + cherry_pick_bodies.append(request_body) dry_run = request_body["dry_run"] status = 200 if dry_run else 409 body = { - "status": "ready" if dry_run else "rejected", + "status": "ready" if dry_run else "failed", "diff": diff, "preview": {"promotedColumns": []}, } @@ -292,10 +291,10 @@ def test_remote_table_branch_merge_defaults_to_execute(): with mock_lancedb_connection(handler) as db: branches = db.open_table("test").branches - assert branches.merge("exp")["status"] == "rejected" - assert branches.merge("exp", dry_run=True)["status"] == "ready" + assert branches.cherry_pick("exp")["status"] == "failed" + assert branches.cherry_pick("exp", dry_run=True)["status"] == "ready" - assert merge_bodies == [ + assert cherry_pick_bodies == [ {"from_branch": "exp", "dry_run": False}, {"from_branch": "exp", "dry_run": True}, ] diff --git a/python/src/table.rs b/python/src/table.rs index a3a7c9d68..5cdcc3653 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -1940,7 +1940,7 @@ impl Branches { } #[pyo3(signature = (from_branch, dry_run=false))] - pub fn merge( + pub fn cherry_pick( self_: PyRef<'_, Self>, from_branch: String, dry_run: bool, @@ -1948,7 +1948,7 @@ impl Branches { let inner = self_.inner.clone(); future_into_py(self_.py(), async move { let result = inner - .merge_branch(&from_branch, dry_run) + .cherry_pick(&from_branch, dry_run) .await .infer_error()?; Python::attach(|py| struct_to_wire_py(py, &result)) diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 6c446907d..fb7278db9 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -21,11 +21,11 @@ use crate::remote::job::RemoteJob; use crate::table::AddColumnsResult; use crate::table::AddResult; use crate::table::BranchDiff; +use crate::table::CherryPickResult; use crate::table::DeleteResult; use crate::table::DropColumnsResult; use crate::table::LsmStats; use crate::table::LsmWriteSpec; -use crate::table::MergeBranchResult; use crate::table::MergeResult; use crate::table::Tags; use crate::table::UpdateResult; @@ -2031,7 +2031,7 @@ impl BaseTable for RemoteTable { async fn diff_branch(&self, from_branch: &str) -> Result { if from_branch.trim().is_empty() { return Err(Error::InvalidInput { - message: "from_branch must be a non-empty string".into(), + message: "Branch name cannot be empty.".into(), }); } let request = self @@ -2058,20 +2058,23 @@ impl BaseTable for RemoteTable { }) } - async fn merge_branch(&self, from_branch: &str, dry_run: bool) -> Result { + async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result { if from_branch.trim().is_empty() { return Err(Error::InvalidInput { - message: "from_branch must be a non-empty string".into(), + message: "Branch name cannot be empty.".into(), }); } let request = self .client - .post(&format!("/v1/table/{}/branches/merge/", self.identifier)) + .post(&format!( + "/v1/table/{}/branches/cherry_pick/", + self.identifier + )) .json(&serde_json::json!({ "from_branch": from_branch, "dry_run": dry_run, })); - // No retry. 409 rejected merge is final and carries a body. + // No retry. HTTP 409 is CherryPickStatus::Failed with a body, not a transport error. let (request_id, response) = self.send(request, false).await?; let status = response.status(); if status == StatusCode::NOT_FOUND { @@ -2080,11 +2083,11 @@ impl BaseTable for RemoteTable { source: format!("branch '{}' does not exist", from_branch).into(), }); } - // 200 and 409 both carry MergeBranchResult. + // 200 and 409 both carry CherryPickResult. if status != StatusCode::OK && status != StatusCode::CONFLICT { let body = response.text().await.unwrap_or_default(); return Err(Error::Http { - source: format!("unexpected status {status} from merge_branch: {body}").into(), + source: format!("unexpected status {status} from cherry_pick: {body}").into(), request_id, status_code: Some(status), }); @@ -2092,7 +2095,7 @@ impl BaseTable for RemoteTable { let body = response.text().await.err_to_http(request_id.clone())?; serde_json::from_str(&body).map_err(|err| Error::Http { source: format!( - "Failed to parse merge_branch response: {}, body: {}", + "Failed to parse cherry_pick response: {}, body: {}", err, body ) .into(), @@ -10513,8 +10516,7 @@ mod tests { "changedColumns":[], "addedIndexes":[], "removedIndexes":[], - "mergeable":true, - "mergeBlockers":[] + "errors":[] }"# } @@ -10532,15 +10534,18 @@ mod tests { }); let diff = table.diff_branch("exp").await.unwrap(); assert_eq!(diff.from_branch, "exp"); - assert!(diff.mergeable); + assert!(diff.errors.is_empty()); assert_eq!(diff.added_columns.len(), 1); assert_eq!(diff.added_columns[0].name, "tag"); } #[tokio::test] - async fn test_merge_branch_dry_run() { + async fn test_cherry_pick_dry_run() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/"); + assert_eq!( + request.url().path(), + "/v1/table/my_table/branches/cherry_pick/" + ); let body = request_body_json(&request); assert_eq!(body["from_branch"], "exp"); assert_eq!(body["dry_run"], true); @@ -10550,27 +10555,29 @@ mod tests { ); http::Response::builder().status(200).body(resp).unwrap() }); - let result = table.merge_branch("exp", true).await.unwrap(); - assert_eq!(result.status, crate::table::MergeBranchStatus::Ready); + let result = table.cherry_pick("exp", true).await.unwrap(); + assert_eq!(result.status, crate::table::CherryPickStatus::Ready); assert_eq!(result.preview.promoted_columns, vec!["tag".to_string()]); assert!(result.main_version_after.is_none()); } #[tokio::test] - async fn test_merge_branch_rejected_returns_ok_with_body() { + async fn test_cherry_pick_failed_returns_ok_with_body() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/"); + assert_eq!( + request.url().path(), + "/v1/table/my_table/branches/cherry_pick/" + ); let body = request_body_json(&request); assert_eq!(body["dry_run"], false); let mut diff: serde_json::Value = serde_json::from_str(sample_branch_diff_json()).unwrap(); - diff["mergeable"] = serde_json::json!(false); - diff["mergeBlockers"] = serde_json::json!([{ + diff["errors"] = serde_json::json!([{ "code": "baseMoved", "message": "main has advanced" }]); let resp = serde_json::json!({ - "status": "rejected", + "status": "failed", "diff": diff, "preview": { "promotedColumns": [] } }); @@ -10579,24 +10586,23 @@ mod tests { .body(resp.to_string()) .unwrap() }); - let result = table.merge_branch("exp", false).await.unwrap(); - assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected); - assert!(!result.diff.mergeable); - assert_eq!(result.diff.merge_blockers.len(), 1); + let result = table.cherry_pick("exp", false).await.unwrap(); + assert_eq!(result.status, crate::table::CherryPickStatus::Failed); + assert!(!result.diff.errors.is_empty()); + assert_eq!(result.diff.errors.len(), 1); } #[tokio::test] - async fn test_merge_branch_unknown_blocker_code_parses() { + async fn test_cherry_pick_unknown_error_code_parses() { let table = Table::new_with_handler("my_table", |_| { let mut diff: serde_json::Value = serde_json::from_str(sample_branch_diff_json()).unwrap(); - diff["mergeable"] = serde_json::json!(false); - diff["mergeBlockers"] = serde_json::json!([{ + diff["errors"] = serde_json::json!([{ "code": "multipleCommits", "message": "branch has more than one data commit" }]); let resp = serde_json::json!({ - "status": "rejected", + "status": "failed", "diff": diff, "preview": { "operation": "append", "rowsAdded": 2 } }); @@ -10605,24 +10611,24 @@ mod tests { .body(resp.to_string()) .unwrap() }); - let result = table.merge_branch("exp", false).await.unwrap(); - assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected); + let result = table.cherry_pick("exp", false).await.unwrap(); + assert_eq!(result.status, crate::table::CherryPickStatus::Failed); assert_eq!( - result.diff.merge_blockers[0].code, - crate::table::MergeBlockerCode::Unknown + result.diff.errors[0].code, + crate::table::CherryPickErrorCode::Unknown ); assert!(result.preview.promoted_columns.is_empty()); } #[tokio::test] - async fn test_merge_branch_unexpected_2xx_is_error() { + async fn test_cherry_pick_unexpected_2xx_is_error() { let table = Table::new_with_handler("my_table", |_| { http::Response::builder() .status(204) .body(String::new()) .unwrap() }); - let err = table.merge_branch("exp", false).await.unwrap_err(); + let err = table.cherry_pick("exp", false).await.unwrap_err(); match err { Error::Http { status_code: Some(code), diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index ca9ed8f26..c2c12fff5 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -66,8 +66,8 @@ use self::merge::MergeInsertBuilder; pub mod add_columns; mod add_data; -pub mod branch_merge; pub mod checkpoint; +pub mod cherry_pick; pub mod computed_columns; mod create_index; pub mod datafusion; @@ -87,9 +87,9 @@ pub use add_columns::AddColumnsBuilder; #[cfg(feature = "remote")] pub(crate) use add_data::PreprocessingOutput; pub use add_data::{AddDataBuilder, AddDataMode, AddResult, NaNVectorBehavior}; -pub use branch_merge::{ - BranchDiff, ColumnChange, ColumnSummary, IndexSummary, MergeBlocker, MergeBlockerCode, - MergeBranchResult, MergeBranchStatus, MergePreview, RowCountSummary, +pub use cherry_pick::{ + BranchDiff, CherryPickError, CherryPickErrorCode, CherryPickPreview, CherryPickResult, + CherryPickStatus, ColumnChange, ColumnSummary, IndexSummary, RowCountSummary, }; pub use chrono::Duration; pub use computed_columns::{ @@ -832,14 +832,14 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { /// Diff a branch against main. Remote only. async fn diff_branch(&self, _from_branch: &str) -> Result { Err(Error::NotSupported { - message: "diff_branch is only supported on remote tables".into(), + message: "Branch diffs are only supported on Enterprise tables.".into(), }) } - /// Merge a branch into main, or dry-run. Remote only. - /// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`]. - async fn merge_branch(&self, _from_branch: &str, _dry_run: bool) -> Result { + /// Cherry-pick a branch onto main, or dry-run. Remote only. + /// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`]. + async fn cherry_pick(&self, _from_branch: &str, _dry_run: bool) -> Result { Err(Error::NotSupported { - message: "merge_branch is only supported on remote tables".into(), + message: "Cherry-picking branches is only supported on Enterprise tables.".into(), }) } /// The branch this handle is scoped to, or `None` for `main`. @@ -2263,14 +2263,10 @@ impl Table { self.inner.diff_branch(from_branch).await } - /// Merge a branch into main, or dry-run. Remote only. - /// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`]. - pub async fn merge_branch( - &self, - from_branch: &str, - dry_run: bool, - ) -> Result { - self.inner.merge_branch(from_branch, dry_run).await + /// Cherry-pick a branch onto main, or dry-run. Remote only. + /// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`]. + pub async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result { + self.inner.cherry_pick(from_branch, dry_run).await } /// The branch this handle is scoped to, or `None` for `main`. diff --git a/rust/lancedb/src/table/branch_merge.rs b/rust/lancedb/src/table/cherry_pick.rs similarity index 86% rename from rust/lancedb/src/table/branch_merge.rs rename to rust/lancedb/src/table/cherry_pick.rs index ad81ab64c..93bc76637 100644 --- a/rust/lancedb/src/table/branch_merge.rs +++ b/rust/lancedb/src/table/cherry_pick.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Types for remote branch diff / merge against main. +//! Types for remote branch diff / cherry-pick onto main. use serde::{Deserialize, Serialize}; @@ -44,13 +44,13 @@ pub struct RowCountSummary { #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub enum MergeBlockerCode { +pub enum CherryPickErrorCode { BaseMoved, RowCountMismatch, RowsChanged, ColumnRemoved, ColumnChanged, - NoMergeableChanges, + NothingToApply, NoColumnChanges, InputColumnDependency, ParentNotMain, @@ -60,8 +60,8 @@ pub enum MergeBlockerCode { #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct MergeBlocker { - pub code: MergeBlockerCode, +pub struct CherryPickError { + pub code: CherryPickErrorCode, pub message: String, } @@ -81,34 +81,33 @@ pub struct BranchDiff { pub changed_columns: Vec, pub added_indexes: Vec, pub removed_indexes: Vec, - pub mergeable: bool, - pub merge_blockers: Vec, + pub errors: Vec, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct MergePreview { +pub struct CherryPickPreview { #[serde(default)] pub promoted_columns: Vec, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub enum MergeBranchStatus { +pub enum CherryPickStatus { Ready, - Rejected, + Failed, NotImplemented, - Merged, + CherryPicked, #[serde(other)] Unknown, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct MergeBranchResult { - pub status: MergeBranchStatus, +pub struct CherryPickResult { + pub status: CherryPickStatus, pub diff: BranchDiff, - pub preview: MergePreview, + pub preview: CherryPickPreview, #[serde(default, skip_serializing_if = "Option::is_none")] pub main_version_after: Option, } From 68749ecfa38f966bec6e650bc41c3f6b8e498818 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 21 Aug 2026 23:48:43 -0700 Subject: [PATCH 26/58] feat(nodejs): materialized view bindings (#3935) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposes materialized views to TypeScript: createMaterializedView, openMaterializedView and listMaterializedViews on Connection, and a MaterializedView handle carrying the parsed definition and refresh({full, sourceVersion}), which returns the typed refresh result. select accepts column names, [alias, expression] pairs, or a record of the same; the definition reads back off the stored schema, so a reopened handle needs no side channel. Remote connections surface the core's not-supported error up front. The napi crate needed the same recursion-limit raise as the core crate: the refresh future's type graph overflows the default trait-recursion depth. Stack created with GitHub Stacks CLI • Give Feedback 💬 --- docs/src/js/classes/Connection.md | 80 ++++++++- docs/src/js/classes/MaterializedView.md | 101 +++++++++++ docs/src/js/globals.md | 4 + .../interfaces/MaterializedViewDefinition.md | 59 +++++++ .../RefreshMaterializedViewResult.md | 41 +++++ .../js/type-aliases/MaterializedViewSelect.md | 14 ++ nodejs/__test__/embedding.test.ts | 48 ++++++ nodejs/__test__/materialized_view.test.ts | 147 ++++++++++++++++ nodejs/__test__/remote.test.ts | 19 +++ nodejs/__test__/table.test.ts | 2 +- nodejs/lancedb/connection.ts | 70 ++++++++ nodejs/lancedb/index.ts | 6 + nodejs/lancedb/materialized_view.ts | 161 ++++++++++++++++++ nodejs/lancedb/table.ts | 20 +++ nodejs/src/connection.rs | 52 ++++++ nodejs/src/lib.rs | 4 + nodejs/src/table.rs | 45 +++++ 17 files changed, 867 insertions(+), 6 deletions(-) create mode 100644 docs/src/js/classes/MaterializedView.md create mode 100644 docs/src/js/interfaces/MaterializedViewDefinition.md create mode 100644 docs/src/js/interfaces/RefreshMaterializedViewResult.md create mode 100644 docs/src/js/type-aliases/MaterializedViewSelect.md create mode 100644 nodejs/__test__/materialized_view.test.ts create mode 100644 nodejs/lancedb/materialized_view.ts diff --git a/docs/src/js/classes/Connection.md b/docs/src/js/classes/Connection.md index e4cbc1e96..92cfd2568 100644 --- a/docs/src/js/classes/Connection.md +++ b/docs/src/js/classes/Connection.md @@ -169,6 +169,45 @@ Creates a new empty Table *** +### createMaterializedView() + +```ts +abstract createMaterializedView( + name, + source, + options?): Promise +``` + +Define a materialized view named `name` over the table `source`. + +The view is created empty, with the query recorded in its schema +metadata; `view.refresh()` computes the rows. The view is a normal +table: it can be queried, indexed and searched, and it appears in +`tableNames`. The source table must have stable row ids (create it with +the `newTableEnableStableRowIds` storage option); they keep the view's +provenance valid across source compactions and cannot be enabled after +a table exists. Local databases only. + +#### Parameters + +* **name**: `string` + +* **source**: `string` + +* **options?** + +* **options.limit?**: `number` + +* **options.select?**: [`MaterializedViewSelect`](../type-aliases/MaterializedViewSelect.md) + +* **options.where?**: `string` + +#### Returns + +`Promise`<[`MaterializedView`](MaterializedView.md)> + +*** + ### createNamespace() ```ts @@ -499,6 +538,22 @@ List server-side jobs across the database's tables. *** +### listMaterializedViews() + +```ts +abstract listMaterializedViews(): Promise +``` + +The names of the materialized views in this database. + +Found by reading every table's schema, so this costs an open per table. + +#### Returns + +`Promise`<`string`[]> + +*** + ### listNamespaces() ```ts @@ -529,6 +584,26 @@ Child namespace names and *** +### openMaterializedView() + +```ts +abstract openMaterializedView(name): Promise +``` + +Open the materialized view named `name`. + +Rejects a table that exists but is not a materialized view. + +#### Parameters + +* **name**: `string` + +#### Returns + +`Promise`<[`MaterializedView`](MaterializedView.md)> + +*** + ### openTable() ```ts @@ -538,18 +613,13 @@ abstract openTable( options?): Promise
``` -Open a table in the database. - #### Parameters * **name**: `string` - The name of the table * **namespacePath?**: `string`[] - The namespace path of the table (defaults to root namespace) * **options?**: `Partial`<[`OpenTableOptions`](../interfaces/OpenTableOptions.md)> - Additional options #### Returns diff --git a/docs/src/js/classes/MaterializedView.md b/docs/src/js/classes/MaterializedView.md new file mode 100644 index 000000000..e6ff66142 --- /dev/null +++ b/docs/src/js/classes/MaterializedView.md @@ -0,0 +1,101 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MaterializedView + +# Class: MaterializedView + +A handle on a materialized view: its table plus its definition. + +Obtained from [Connection#createMaterializedView](Connection.md#creatematerializedview) or +[Connection#openMaterializedView](Connection.md#openmaterializedview). The view is a normal table -- +queries, indexes and search all apply through [MaterializedView#table](MaterializedView.md#table) +-- whose contents are maintained by [MaterializedView#refresh](MaterializedView.md#refresh). + +## Constructors + +### new MaterializedView() + +```ts +new MaterializedView(table): MaterializedView +``` + +#### Parameters + +* **table**: [`Table`](Table.md) + +#### Returns + +[`MaterializedView`](MaterializedView.md) + +## Accessors + +### name + +```ts +get name(): string +``` + +#### Returns + +`string` + +## Methods + +### definition() + +```ts +definition(): Promise +``` + +The query that defines the view, read from its stored schema. + +#### Returns + +`Promise`<[`MaterializedViewDefinition`](../interfaces/MaterializedViewDefinition.md)> + +*** + +### refresh() + +```ts +refresh(options?): Promise +``` + +Recompute the view from its source. + +The refresh is incremental when the source's changes can be reconciled +into the view -- rows added, changed or removed since the last one -- +and otherwise rebuilds. `full` forces a rebuild; `sourceVersion` +refreshes to that source version instead of the latest. + +Concurrent refreshes of one view do not duplicate its rows. Two that +plan the same source rows conflict on commit, and the loser throws +rather than writing them a second time. + +#### Parameters + +* **options?** + +* **options.full?**: `boolean` + +* **options.sourceVersion?**: `number` + +#### Returns + +`Promise`<[`RefreshMaterializedViewResult`](../interfaces/RefreshMaterializedViewResult.md)> + +*** + +### table() + +```ts +table(): Table +``` + +The view, as the table it is. + +#### Returns + +[`Table`](Table.md) diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 4deeed1bc..6a8f644eb 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -28,6 +28,7 @@ - [Job](classes/Job.md) - [MakeArrowTableOptions](classes/MakeArrowTableOptions.md) - [MatchQuery](classes/MatchQuery.md) +- [MaterializedView](classes/MaterializedView.md) - [MergeInsertBuilder](classes/MergeInsertBuilder.md) - [MultiMatchQuery](classes/MultiMatchQuery.md) - [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md) @@ -101,6 +102,7 @@ - [ListNamespacesResponse](interfaces/ListNamespacesResponse.md) - [LsmStats](interfaces/LsmStats.md) - [LsmWriteSpec](interfaces/LsmWriteSpec.md) +- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md) - [MemtableStats](interfaces/MemtableStats.md) - [MergeResult](interfaces/MergeResult.md) - [NativeOAuthConfig](interfaces/NativeOAuthConfig.md) @@ -110,6 +112,7 @@ - [OptimizeStats](interfaces/OptimizeStats.md) - [QueryExecutionOptions](interfaces/QueryExecutionOptions.md) - [RefreshColumnResult](interfaces/RefreshColumnResult.md) +- [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md) - [RemovalStats](interfaces/RemovalStats.md) - [RenameTableOptions](interfaces/RenameTableOptions.md) - [RestNamespaceConfig](interfaces/RestNamespaceConfig.md) @@ -142,6 +145,7 @@ - [FieldLike](type-aliases/FieldLike.md) - [IntoSql](type-aliases/IntoSql.md) - [IntoVector](type-aliases/IntoVector.md) +- [MaterializedViewSelect](type-aliases/MaterializedViewSelect.md) - [MultiVector](type-aliases/MultiVector.md) - [RecordBatchLike](type-aliases/RecordBatchLike.md) - [SchemaLike](type-aliases/SchemaLike.md) diff --git a/docs/src/js/interfaces/MaterializedViewDefinition.md b/docs/src/js/interfaces/MaterializedViewDefinition.md new file mode 100644 index 000000000..741bbba31 --- /dev/null +++ b/docs/src/js/interfaces/MaterializedViewDefinition.md @@ -0,0 +1,59 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MaterializedViewDefinition + +# Interface: MaterializedViewDefinition + +The query that defines a materialized view. + +## Properties + +### filter? + +```ts +optional filter: string; +``` + +SQL predicate selecting the source rows the view holds. + +*** + +### inputs + +```ts +inputs: string[]; +``` + +Source columns the projections and filter read. + +*** + +### limit? + +```ts +optional limit: number; +``` + +Cap on the number of rows the view holds. + +*** + +### projections + +```ts +projections: [string, string][]; +``` + +`[output column, SQL expression]` pairs, in view schema order. + +*** + +### sourceTable + +```ts +sourceTable: string; +``` + +Name of the source table, in the same database as the view. diff --git a/docs/src/js/interfaces/RefreshMaterializedViewResult.md b/docs/src/js/interfaces/RefreshMaterializedViewResult.md new file mode 100644 index 000000000..cb7100cd8 --- /dev/null +++ b/docs/src/js/interfaces/RefreshMaterializedViewResult.md @@ -0,0 +1,41 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / RefreshMaterializedViewResult + +# Interface: RefreshMaterializedViewResult + +## Properties + +### mode + +```ts +mode: string; +``` + +How the view was brought up to date: "rebuild", "incremental" or "no_op". + +*** + +### rowsWritten + +```ts +rowsWritten: number; +``` + +*** + +### sourceVersion + +```ts +sourceVersion: number; +``` + +*** + +### version + +```ts +version: number; +``` diff --git a/docs/src/js/type-aliases/MaterializedViewSelect.md b/docs/src/js/type-aliases/MaterializedViewSelect.md new file mode 100644 index 000000000..7b246e945 --- /dev/null +++ b/docs/src/js/type-aliases/MaterializedViewSelect.md @@ -0,0 +1,14 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MaterializedViewSelect + +# Type Alias: MaterializedViewSelect + +```ts +type MaterializedViewSelect: (string | [string, string])[] | Record; +``` + +The view's columns: column names, `[alias, SQL expression]` pairs, or a +record of the same. A bare name projects itself. diff --git a/nodejs/__test__/embedding.test.ts b/nodejs/__test__/embedding.test.ts index 06184751e..2a8494e0f 100644 --- a/nodejs/__test__/embedding.test.ts +++ b/nodejs/__test__/embedding.test.ts @@ -487,4 +487,52 @@ describe("embedding functions", () => { expect(stringSchema3).toEqual(stringExpectedSchema); }, ); + test("parses one function writing several vector columns", async () => { + class MockEmbeddingFunction extends EmbeddingFunction { + ndims() { + return 3; + } + embeddingDataType(): Float { + return new Float32(); + } + async computeQueryEmbeddings(_data: string) { + return [1, 2, 3]; + } + async computeSourceEmbeddings(data: string[]) { + return Array.from({ length: data.length }).fill([ + 1, 2, 3, + ]) as number[][]; + } + } + const registry = getRegistry(); + registry.register("multi_output_mock")(MockEmbeddingFunction); + + // A materialized view can project one source vector column under two + // names, so a table's configuration names the same function twice. + const parsed = await registry.parseFunctions( + new Map([ + [ + "embedding_functions", + JSON.stringify([ + { + name: "multi_output_mock", + sourceColumn: "text", + vectorColumn: "vector_a", + model: {}, + }, + { + name: "multi_output_mock", + sourceColumn: "text", + vectorColumn: "vector_b", + model: {}, + }, + ]), + ], + ]), + ); + + expect( + [...parsed.values()].map(({ vectorColumn }) => vectorColumn).sort(), + ).toEqual(["vector_a", "vector_b"]); + }); }); diff --git a/nodejs/__test__/materialized_view.test.ts b/nodejs/__test__/materialized_view.test.ts new file mode 100644 index 000000000..2e7b2ec4d --- /dev/null +++ b/nodejs/__test__/materialized_view.test.ts @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import * as tmp from "tmp"; + +import { Connection, connect } from "../lancedb"; +import { + DEFINITION_META_KEY, + definitionFromMetadata, +} from "../lancedb/materialized_view"; + +describe("materialized views", () => { + let tmpDir: tmp.DirResult; + let db: Connection; + + beforeEach(async () => { + tmpDir = tmp.dirSync({ unsafeCleanup: true }); + db = await connect(tmpDir.name); + await db.createTable( + "people", + [ + { name: "ada", age: 36 }, + { name: "kid", age: 7 }, + { name: "grace", age: 85 }, + ], + { storageOptions: { newTableEnableStableRowIds: "true" } }, + ); + }); + afterEach(() => tmpDir.removeCallback()); + + it("rejects a stored limit a number cannot carry", () => { + const big = new Map([ + [ + DEFINITION_META_KEY, + '{"kind":"select","source_table":"people","limit":9007199254740993}', + ], + ]); + expect(() => definitionFromMetadata(big, "v")).toThrow( + /too large to represent exactly/, + ); + + const safe = new Map([ + [ + DEFINITION_META_KEY, + '{"kind":"select","source_table":"people","limit":42}', + ], + ]); + expect(definitionFromMetadata(safe, "v").limit).toBe(42); + }); + + it("creates, refreshes and queries a view", async () => { + const view = await db.createMaterializedView("adults", "people", { + select: ["name", ["shout", "upper(name)"]], + where: "age >= 18", + }); + expect(view.name).toBe("adults"); + expect(await view.table().countRows()).toBe(0); + + const result = await view.refresh(); + expect(result.mode).toBe("rebuild"); + expect(Number(result.rowsWritten)).toBe(2); + + const rows = await view.table().query().toArray(); + expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]); + }); + + it("round-trips the definition", async () => { + await db.createMaterializedView("adults", "people", { + where: "age >= 18", + }); + const view = await db.openMaterializedView("adults"); + const definition = await view.definition(); + expect(definition.sourceTable).toBe("people"); + expect(definition.filter).toBe("age >= 18"); + expect(definition.projections).toEqual([ + ["name", "`name`"], + ["age", "`age`"], + ]); + expect(definition.inputs).toEqual(["age", "name"]); + }); + + it("refreshes incrementally after an append", async () => { + const view = await db.createMaterializedView("copy", "people"); + await view.refresh(); + + const people = await db.openTable("people"); + await people.add([{ name: "alan", age: 41 }]); + const result = await view.refresh(); + expect(result.mode).toBe("incremental"); + expect(Number(result.rowsWritten)).toBe(1); + expect(await view.table().countRows()).toBe(4); + + expect((await view.refresh()).mode).toBe("no_op"); + }); + + it("lists views and rejects non-views", async () => { + await db.createMaterializedView("adults", "people", { + where: "age >= 18", + }); + expect(await db.listMaterializedViews()).toEqual(["adults"]); + await expect(db.openMaterializedView("people")).rejects.toThrow( + "not a materialized view", + ); + }); + + it("rejects an invalid expression at create time", async () => { + await expect( + db.createMaterializedView("bad", "people", { + select: [["x", "missing + 1"]], + }), + ).rejects.toThrow("missing"); + }); + + it("rejects invalid numeric options before creating anything", async () => { + for (const limit of [-5, 1.5, Infinity, NaN]) { + await expect( + db.createMaterializedView("bad", "people", { limit }), + ).rejects.toThrow("non-negative integer"); + } + expect(await db.listMaterializedViews()).toEqual([]); + + const view = await db.createMaterializedView("copy", "people"); + for (const sourceVersion of [-1, 1.5, Infinity, NaN]) { + await expect(view.refresh({ sourceVersion })).rejects.toThrow( + "non-negative integer", + ); + } + }); + + it("quotes bare select names", async () => { + await db.createTable("odd_names", [{ "order item": "widget" }], { + storageOptions: { newTableEnableStableRowIds: "true" }, + }); + const view = await db.createMaterializedView("quoted", "odd_names", { + select: ["order item"], + }); + const result = await view.refresh(); + expect(Number(result.rowsWritten)).toBe(1); + }); + + it("requires stable row ids on the source", async () => { + await db.createTable("plain", [{ x: 1 }]); + await expect(db.createMaterializedView("v", "plain")).rejects.toThrow( + "stable row ids", + ); + }); +}); diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 01320d2b0..c51cbbbb7 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -75,6 +75,25 @@ async function withMockDatabase( } describe("remote connection", () => { + it("refuses materialized views before issuing any request", async () => { + const paths: string[] = []; + await withMockDatabase( + (req, res) => { + paths.push(req.url ?? ""); + res.writeHead(404).end(); + }, + async (db) => { + await expect(db.openMaterializedView("secret_table")).rejects.toThrow( + /only on local databases/, + ); + await expect(db.listMaterializedViews()).rejects.toThrow( + /only on local databases/, + ); + expect(paths).toEqual([]); + }, + ); + }); + it("should accept partial connection options", async () => { await connect("db://test", { apiKey: "fake", diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 80c50f1ac..0f6ed3615 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2953,7 +2953,7 @@ describe("column name options", () => { .limit(10) .toArray(); expect(results2.length).toBe(10); - }); + }, 30_000); }); describe("when creating an empty table", () => { diff --git a/nodejs/lancedb/connection.ts b/nodejs/lancedb/connection.ts index a81dc0442..e5528f8c6 100644 --- a/nodejs/lancedb/connection.ts +++ b/nodejs/lancedb/connection.ts @@ -16,6 +16,12 @@ import { makeEmptyTable, } from "./arrow"; import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry"; +import { + MaterializedView, + MaterializedViewSelect, + normalizeSelect, + validateNonNegativeInteger, +} from "./materialized_view"; import { Connection as LanceDbConnection } from "./native"; import type { CreateNamespaceResponse, @@ -247,6 +253,41 @@ export abstract class Connection { * @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace) * @param {Partial} options - Additional options */ + /** + * Define a materialized view named `name` over the table `source`. + * + * The view is created empty, with the query recorded in its schema + * metadata; `view.refresh()` computes the rows. The view is a normal + * table: it can be queried, indexed and searched, and it appears in + * `tableNames`. The source table must have stable row ids (create it with + * the `newTableEnableStableRowIds` storage option); they keep the view's + * provenance valid across source compactions and cannot be enabled after + * a table exists. Local databases only. + */ + abstract createMaterializedView( + name: string, + source: string, + options?: { + select?: MaterializedViewSelect; + where?: string; + limit?: number; + }, + ): Promise; + + /** + * Open the materialized view named `name`. + * + * Rejects a table that exists but is not a materialized view. + */ + abstract openMaterializedView(name: string): Promise; + + /** + * The names of the materialized views in this database. + * + * Found by reading every table's schema, so this costs an open per table. + */ + abstract listMaterializedViews(): Promise; + abstract openTable( name: string, namespacePath?: string[], @@ -531,6 +572,35 @@ export class LocalConnection extends Connection { ); } + async createMaterializedView( + name: string, + source: string, + options?: { + select?: MaterializedViewSelect; + where?: string; + limit?: number; + }, + ): Promise { + validateNonNegativeInteger(options?.limit, "limit"); + const innerTable = await this.inner.createMaterializedView( + name, + source, + normalizeSelect(options?.select), + options?.where, + options?.limit, + ); + return new MaterializedView(new LocalTable(innerTable)); + } + + async openMaterializedView(name: string): Promise { + const innerTable = await this.inner.openMaterializedView(name); + return new MaterializedView(new LocalTable(innerTable)); + } + + async listMaterializedViews(): Promise { + return await this.inner.listMaterializedViews(); + } + async openTable( name: string, namespacePath?: string[], diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index aa9c7ef13..da13b434f 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -21,6 +21,11 @@ import type { BaseTokenizer } from "./indices"; import type { FtsToken } from "./table"; // Re-export native header provider for use with connectWithHeaderProvider +export { + MaterializedView, + MaterializedViewDefinition, + MaterializedViewSelect, +} from "./materialized_view"; export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js"; // OpenTelemetry metrics bridge. Only the high-level entry point is public; the @@ -51,6 +56,7 @@ export { AddResult, AddColumnsResult, RefreshColumnResult, + RefreshMaterializedViewResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, diff --git a/nodejs/lancedb/materialized_view.ts b/nodejs/lancedb/materialized_view.ts new file mode 100644 index 000000000..b26dee59e --- /dev/null +++ b/nodejs/lancedb/materialized_view.ts @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { RefreshMaterializedViewResult } from "./native"; +import { Table } from "./table"; + +/** Schema metadata key holding a materialized view's definition. */ +export const DEFINITION_META_KEY = "mv.definition"; + +/** The query that defines a materialized view. */ +export interface MaterializedViewDefinition { + /** Name of the source table, in the same database as the view. */ + sourceTable: string; + /** `[output column, SQL expression]` pairs, in view schema order. */ + projections: [string, string][]; + /** SQL predicate selecting the source rows the view holds. */ + filter?: string; + /** Cap on the number of rows the view holds. */ + limit?: number; + /** Source columns the projections and filter read. */ + inputs: string[]; +} + +/** + * The view's columns: column names, `[alias, SQL expression]` pairs, or a + * record of the same. A bare name projects itself. + */ +export type MaterializedViewSelect = + | (string | [string, string])[] + | Record; + +/** + * @internal Reject a numeric option N-API would otherwise silently coerce: + * `Infinity` reaches Rust as 0, `1.5` as 1. + */ +export function validateNonNegativeInteger( + value: number | undefined, + name: string, +): void { + if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) { + throw new Error(`${name} must be a non-negative integer`); + } +} + +/** @internal Quote a column name as a Lance SQL identifier (backticks). */ +function quoteIdentifier(name: string): string { + return "`" + name.replace(/`/g, "``") + "`"; +} + +/** + * @internal Normalize a select argument into `[alias, expression]` pairs. + * A bare name projects itself and is quoted, so any valid column name works; + * pair and record entries are kept verbatim because their right side is an + * expression. + */ +export function normalizeSelect( + select?: MaterializedViewSelect, +): [string, string][] | undefined { + if (select === undefined) { + return undefined; + } + if (Array.isArray(select)) { + return select.map((item) => + typeof item === "string" ? [item, quoteIdentifier(item)] : item, + ); + } + return Object.entries(select); +} + +/** @internal Parse a definition off a table's stored schema metadata. */ +export function definitionFromMetadata( + metadata: Map, + name: string, +): MaterializedViewDefinition { + const raw = metadata.get(DEFINITION_META_KEY); + if (raw === undefined) { + throw new Error(`Table '${name}' is not a materialized view`); + } + // biome-ignore lint/suspicious/noExplicitAny: raw JSON + const value: any = JSON.parse(raw); + if (value.kind !== "select") { + throw new Error( + `materialized view '${name}' is defined by '${value.kind}', which this ` + + "version of lancedb cannot refresh", + ); + } + const limit = value.limit ?? undefined; + // JSON.parse rounds integers past 2^53; every exact u64 parses to a safe + // integer and every rounded one does not, so this rejects precisely the + // values a number cannot carry. + if (limit !== undefined && !Number.isSafeInteger(limit)) { + throw new Error( + `materialized view '${name}' has a stored limit too large to represent exactly`, + ); + } + return { + sourceTable: value.source_table, + // biome-ignore lint/suspicious/noExplicitAny: raw JSON + projections: (value.projections ?? []).map((p: any) => [ + p.output, + p.expression, + ]), + filter: value.filter ?? undefined, + limit, + inputs: value.inputs ?? [], + }; +} + +/** + * A handle on a materialized view: its table plus its definition. + * + * Obtained from {@link Connection#createMaterializedView} or + * {@link Connection#openMaterializedView}. The view is a normal table -- + * queries, indexes and search all apply through {@link MaterializedView#table} + * -- whose contents are maintained by {@link MaterializedView#refresh}. + */ +export class MaterializedView { + private readonly inner: Table; + + constructor(table: Table) { + this.inner = table; + } + + get name(): string { + return this.inner.name; + } + + /** The view, as the table it is. */ + table(): Table { + return this.inner; + } + + /** The query that defines the view, read from its stored schema. */ + async definition(): Promise { + const schema = await this.inner.schema(); + return definitionFromMetadata(schema.metadata, this.name); + } + + /** + * Recompute the view from its source. + * + * The refresh is incremental when the source's changes can be reconciled + * into the view -- rows added, changed or removed since the last one -- + * and otherwise rebuilds. `full` forces a rebuild; `sourceVersion` + * refreshes to that source version instead of the latest. + * + * Concurrent refreshes of one view do not duplicate its rows. Two that + * plan the same source rows conflict on commit, and the loser throws + * rather than writing them a second time. + */ + async refresh(options?: { + full?: boolean; + sourceVersion?: number; + }): Promise { + validateNonNegativeInteger(options?.sourceVersion, "sourceVersion"); + return await this.inner.refreshMaterializedView( + options?.full, + options?.sourceVersion, + ); + } +} diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 4e323ec64..28603cca9 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -35,6 +35,7 @@ import { Branches as NativeBranches, OptimizeStats, RefreshColumnResult, + RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, @@ -602,6 +603,18 @@ export abstract class Table { */ abstract refreshColumnAsync(column: string): Promise; + /** + * Recompute this table's contents from its materialized-view definition. + * + * Plumbing for {@link MaterializedView.refresh}, which is the way to call + * it: rejects tables that carry no view definition. Local tables only. + * @ignore + */ + abstract refreshMaterializedView( + full?: boolean, + sourceVersion?: number, + ): Promise; + /** * Alter the name or nullability of columns. * @param {ColumnAlteration[]} columnAlterations One or more alterations to @@ -1264,6 +1277,13 @@ export class LocalTable extends Table { return await this.inner.refreshColumnAsync(column); } + async refreshMaterializedView( + full?: boolean, + sourceVersion?: number, + ): Promise { + return await this.inner.refreshMaterializedView(full, sourceVersion); + } + async alterColumns( columnAlterations: ColumnAlteration[], ): Promise { diff --git a/nodejs/src/connection.rs b/nodejs/src/connection.rs index c9f5e10ea..44eb68f32 100644 --- a/nodejs/src/connection.rs +++ b/nodejs/src/connection.rs @@ -266,6 +266,58 @@ impl Connection { Ok(Table::new(tbl)) } + #[napi(catch_unwind)] + pub async fn create_materialized_view( + &self, + name: String, + source: String, + projections: Option>>, + filter: Option, + limit: Option, + ) -> napi::Result
{ + let mut builder = self.get_inner()?.create_materialized_view(name, source); + if let Some(projections) = projections { + let mut pairs = Vec::with_capacity(projections.len()); + for pair in projections { + let [output, expression]: [String; 2] = pair.try_into().map_err(|_| { + napi::Error::from_reason("each projection must be an [output, expression] pair") + })?; + pairs.push((output, expression)); + } + builder = builder.select(pairs); + } + if let Some(filter) = filter { + builder = builder.only_if(filter); + } + if let Some(limit) = limit { + let limit = u64::try_from(limit) + .map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?; + builder = builder.limit(limit); + } + let view = builder.execute().await.default_error()?; + Ok(Table::new(view.table().clone())) + } + + #[napi(catch_unwind)] + pub async fn open_materialized_view(&self, name: String) -> napi::Result
{ + let view = self + .get_inner()? + .open_materialized_view(&name) + .await + .default_error()?; + Ok(Table::new(view.table().clone())) + } + + #[napi(catch_unwind)] + pub async fn list_materialized_views(&self) -> napi::Result> { + let views = self + .get_inner()? + .list_materialized_views() + .await + .default_error()?; + Ok(views.into_iter().map(|v| v.name).collect()) + } + #[napi(catch_unwind)] pub async fn open_table( &self, diff --git a/nodejs/src/lib.rs b/nodejs/src/lib.rs index 312b675bd..1110f6203 100644 --- a/nodejs/src/lib.rs +++ b/nodejs/src/lib.rs @@ -1,6 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors +// The materialized-view refresh future deepens the type graph past the +// default trait-recursion depth; same raise as the core crate applies. +#![recursion_limit = "256"] + use std::collections::HashMap; use env_logger::Env; diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 694b6a704..9d60b2056 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -381,6 +381,26 @@ impl Table { Ok(crate::job::Job::new(job)) } + #[napi(catch_unwind)] + pub async fn refresh_materialized_view( + &self, + full: Option, + source_version: Option, + ) -> napi::Result { + let view = lancedb::MaterializedView::from_table(self.inner_ref()?.clone()) + .await + .default_error()?; + let mut builder = view.refresh().full(full.unwrap_or(false)); + if let Some(version) = source_version { + let version = u64::try_from(version).map_err(|_| { + napi::Error::from_reason("sourceVersion must be a non-negative integer") + })?; + builder = builder.source_version(version); + } + let result = builder.execute().await.default_error()?; + Ok(result.into()) + } + #[napi(catch_unwind)] pub async fn add_columns_with_schema( &self, @@ -1387,6 +1407,31 @@ pub struct RefreshColumnResult { pub version: i64, } +#[napi(object)] +pub struct RefreshMaterializedViewResult { + /// How the view was brought up to date: "rebuild", "incremental" or "no_op". + pub mode: String, + pub rows_written: i64, + pub source_version: i64, + pub version: i64, +} + +impl From for RefreshMaterializedViewResult { + fn from(value: lancedb::RefreshMaterializedViewResult) -> Self { + let mode = match value.mode { + lancedb::RefreshMode::Rebuild => "rebuild", + lancedb::RefreshMode::Incremental => "incremental", + lancedb::RefreshMode::NoOp => "no_op", + }; + Self { + mode: mode.to_string(), + rows_written: value.rows_written as i64, + source_version: value.source_version as i64, + version: value.version as i64, + } + } +} + impl From for RefreshColumnResult { fn from(value: lancedb::table::RefreshColumnResult) -> Self { Self { From 45cd05347846bc2280103680e497e8fc0cdf480e Mon Sep 17 00:00:00 2001 From: Lance Release Date: Sat, 22 Aug 2026 16:38:08 +0000 Subject: [PATCH 27/58] =?UTF-8?q?Bump=20version:=200.38.0-beta.3=20?= =?UTF-8?q?=E2=86=92=200.38.0-beta.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index b1d2bf1dc..bba238f96 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.3" +current_version = "0.38.0-beta.4" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 5947187cc..87fec3378 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5398,7 +5398,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.3" +version = "0.38.0-beta.4" dependencies = [ "ahash", "anyhow", @@ -5486,7 +5486,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.3" +version = "0.38.0-beta.4" dependencies = [ "arrow-array", "arrow-buffer", @@ -5511,7 +5511,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.3" +version = "0.38.0-beta.4" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index a8869d319..cff4cf2f5 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.3 + 0.38.0-beta.4 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 9de14d1f9..98c10ca38 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.3 + 0.38.0-beta.4 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 36ec01d9e..56366bb15 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.3 + 0.38.0-beta.4 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index cba7012d1..99aed2e71 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.3" +version = "0.38.0-beta.4" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 3c76481fc..c84b83c45 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 7a1a3af03..52f2b7901 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 7175233af..afa7fac32 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index add5a1da2..36cfa0343 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 72a82bf6d..23ea473ef 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index c69beeb8a..880b65540 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index d21fc1eb3..8cc40de7b 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 1c368dab5..dba3ede7c 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index 7f72eb8b0..c2f5be2c8 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index ddc13b69f..dace5aa7c 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.3" +version = "0.38.0-beta.4" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index f785e6743..2920498b1 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.3" +version = "0.38.0-beta.4" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 1f1d03f306c322c42df93a42eb064225b0206f20 Mon Sep 17 00:00:00 2001 From: Weston Pace Date: Sun, 23 Aug 2026 00:28:07 -0700 Subject: [PATCH 28/58] feat(python): add backpressure to StreamingDataset post-transform queue (#3897) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename prefetch_batches → io_queue_depth and introduce transform_queue_depth as a symmetric pair: both express "number of batches to buffer per split at this pipeline stage." The old names are still accepted as keyword arguments but log a deprecation warning redirecting callers to the new names. transform_queue_depth caps how many transform-result batches can accumulate per split in the post-transform queue. Without this limit a slow consumer (e.g. a GPU training step) causes cooked rows to pile up unboundedly. The backpressure check in _try_submit_tx counts both already-cooked rows and rows expected from in-flight transforms; it skips proactive transform submission when the combined total reaches the limit. The reactive _ensure_cooked path bypasses the check so the consumer never stalls. --------- Co-authored-by: Claude Sonnet 4.6 --- python/python/lancedb/streaming.py | 65 +++++- .../python/tests/test_elastic_dataloader.py | 186 +++++++++++++++++- 2 files changed, 239 insertions(+), 12 deletions(-) diff --git a/python/python/lancedb/streaming.py b/python/python/lancedb/streaming.py index b27e606a4..6951fa0bc 100644 --- a/python/python/lancedb/streaming.py +++ b/python/python/lancedb/streaming.py @@ -61,7 +61,7 @@ class StreamingDataset(IterableDataset): Internally ``__iter__`` runs a two-stage pipeline: - - **Stage 1 (I/O)**: one thread pool with ``num_splits * prefetch_batches`` + - **Stage 1 (I/O)**: one thread pool with ``num_splits * io_queue_depth`` workers fetches raw ``RecordBatch`` objects from LanceDB in parallel across all splits and places them in a per-split raw-batch queue. - **Stage 2 (transform)**: a second thread pool with @@ -104,11 +104,11 @@ class StreamingDataset(IterableDataset): call. Larger values amortise per-request overhead (critical on object storage) at the cost of higher memory usage per split buffer. Defaults to ``DEFAULT_READ_BATCH_SIZE`` (64). - prefetch_batches: + io_queue_depth: Number of I/O batches to keep in flight per split. Higher values overlap storage latency with transform and training compute at the cost - of more memory and threads. Defaults to ``DEFAULT_PREFETCH_BATCHES`` - (4). + of more memory and threads. Must be greater than zero. Defaults to + ``DEFAULT_PREFETCH_BATCHES`` (4). columns: Optional list of column names to read. When set, only those columns are fetched from storage; all others are omitted. ``None`` (the @@ -175,6 +175,16 @@ class StreamingDataset(IterableDataset): Prefer the ``filter`` parameter when bad rows can be expressed as a SQL predicate (e.g. ``"col IS NOT NULL"``) — filtering happens before splits are built, so every guarantee is fully preserved. + transform_queue_depth: + Number of transform-result batches to buffer per split in the + post-transform queue before backpressure is applied to the transform + stage. When the combined count of in-flight transform futures and + already-buffered rows for a split reaches + ``transform_queue_depth * read_batch_size``, no new transforms are + submitted for that split until the consumer catches up. Useful for + capping peak memory when the consumer (e.g. a GPU training step) is + slower than the transform stage. Must be greater than zero. + ``None`` (the default) imposes no limit. worker_info_override: If set, used in place of ``torch.utils.data.get_worker_info()`` to determine the DataLoader worker assignment. Intended for unit tests @@ -194,17 +204,26 @@ class StreamingDataset(IterableDataset): rank: int = 0, world_size: int = 1, read_batch_size: int = DEFAULT_READ_BATCH_SIZE, - prefetch_batches: int = DEFAULT_PREFETCH_BATCHES, + io_queue_depth: int = DEFAULT_PREFETCH_BATCHES, columns: Optional[list[str]] = None, shuffle_clump_size: Optional[int] = None, filter: Optional[str] = None, transform: Optional[Callable] = None, transform_parallelism: Optional[int] = None, on_transform_error: Union[str, Callable[[Exception], bool]] = "raise", + transform_queue_depth: Optional[int] = None, connection_factory: Optional[Callable[[str], Any]] = None, worker_info_override=None, + # Deprecated; use io_queue_depth instead. + prefetch_batches: Optional[int] = None, ): super().__init__() + if prefetch_batches is not None: + logger.warning( + "prefetch_batches is deprecated and will be removed in a future " + "version; use io_queue_depth instead" + ) + io_queue_depth = prefetch_batches if num_splits is None: num_splits = world_size if shuffle_seed is None: @@ -214,6 +233,8 @@ class StreamingDataset(IterableDataset): f"num_splits ({num_splits}) must be divisible by " f"world_size ({world_size})" ) + if io_queue_depth <= 0: + raise ValueError("io_queue_depth must be greater than 0") if transform_parallelism is not None and transform_parallelism <= 0: raise ValueError("transform_parallelism must be greater than 0") if on_transform_error not in ("raise", "skip", "warn") and not callable( @@ -223,6 +244,8 @@ class StreamingDataset(IterableDataset): "on_transform_error must be 'raise', 'skip', 'warn', or a " f"callable, got {on_transform_error!r}" ) + if transform_queue_depth is not None and transform_queue_depth <= 0: + raise ValueError("transform_queue_depth must be greater than 0") self._table = table self._num_splits = num_splits @@ -232,13 +255,14 @@ class StreamingDataset(IterableDataset): self._rank = rank self._world_size = world_size self._read_batch_size = read_batch_size - self._prefetch_batches = prefetch_batches + self._io_queue_depth = io_queue_depth self._columns = columns self._shuffle_clump_size = shuffle_clump_size self._filter = filter self._transform = transform self._transform_parallelism = transform_parallelism self._on_transform_error = on_transform_error + self._transform_queue_depth = transform_queue_depth self._connection_factory = connection_factory self._worker_info_override = worker_info_override @@ -365,7 +389,7 @@ class StreamingDataset(IterableDataset): pos_consumed = list(initial_positions) batch_size = self._read_batch_size - max_prefetch = self._prefetch_batches + io_queue_depth = self._io_queue_depth transform_workers = ( self._transform_parallelism if self._transform_parallelism is not None @@ -374,6 +398,13 @@ class StreamingDataset(IterableDataset): final_transform = ( self._transform if self._transform is not None else Transforms.arrow2python ) + # None means no limit; otherwise cap rows per split to + # transform_queue_depth batches worth (including in-flight transforms). + max_cooked_rows = ( + self._transform_queue_depth * batch_size + if self._transform_queue_depth is not None + else None + ) # Per-split pipeline state. Batches are paired with the absolute # permutation position of their first row so that skipped rows can be @@ -409,7 +440,9 @@ class StreamingDataset(IterableDataset): io_pending[i].append((abs_start, io_pool.submit(_io_call, perm_i, indices))) def _fill_io(i: int) -> None: - while len(io_pending[i]) < max_prefetch and fetch_head[i] < split_sizes[i]: + while ( + len(io_pending[i]) < io_queue_depth and fetch_head[i] < split_sizes[i] + ): _submit_io(i) def _drain_io(i: int) -> None: @@ -487,7 +520,19 @@ class StreamingDataset(IterableDataset): def _try_submit_tx(i: int) -> None: """Submit transforms for raw_batches[i] up to available capacity.""" - while raw_batches[i] and tx_semaphore.acquire(blocking=False): + while raw_batches[i]: + # Backpressure: only submit a new transform when there is room + # for a full batch in the post-transform queue. Checking for + # a full batch prevents submitting a transform that would + # overflow the limit mid-batch (e.g. 990 rows queued with a + # capacity of 1000 and a batch_size of 128 must wait until + # 128 rows have been consumed, not just 1). + if max_cooked_rows is not None: + in_pipeline = len(cooked[i]) + len(tx_pending[i]) * batch_size + if in_pipeline + batch_size > max_cooked_rows: + break + if not tx_semaphore.acquire(blocking=False): + break abs_start, batch = raw_batches[i].popleft() tx_pending[i].append(tx_pool.submit(_tx_call_guarded, abs_start, batch)) @@ -531,7 +576,7 @@ class StreamingDataset(IterableDataset): # ── Main loop ───────────────────────────────────────────────────────── - with ThreadPoolExecutor(max_workers=n * max_prefetch) as io_pool: + with ThreadPoolExecutor(max_workers=n * io_queue_depth) as io_pool: with ThreadPoolExecutor(max_workers=transform_workers) as tx_pool: self._raw_batches_ref = raw_batches self._cooked_ref = cooked diff --git a/python/python/tests/test_elastic_dataloader.py b/python/python/tests/test_elastic_dataloader.py index 0c1a70765..4d860a5de 100644 --- a/python/python/tests/test_elastic_dataloader.py +++ b/python/python/tests/test_elastic_dataloader.py @@ -1374,6 +1374,188 @@ def test_transform_parallelism_must_be_positive(lance_table, transform_paralleli ) +# --------------------------------------------------------------------------- +# Backpressure / transform_queue_depth tests +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("transform_queue_depth", [0, -1]) +def test_transform_queue_depth_must_be_positive(lance_table, transform_queue_depth): + """transform_queue_depth=0 or negative must raise ValueError.""" + with pytest.raises( + ValueError, match="transform_queue_depth must be greater than 0" + ): + StreamingDataset( + lance_table, + num_splits=NUM_SPLITS, + transform_queue_depth=transform_queue_depth, + ) + + +@pytest.mark.parametrize("transform_queue_depth", [1, 2, 4]) +def test_transform_queue_depth_correctness(lance_table, transform_queue_depth): + """With backpressure enabled, every row is still yielded exactly once.""" + ds = StreamingDataset( + lance_table, + num_splits=NUM_SPLITS, + shuffle_seed=SHUFFLE_SEED, + transform_queue_depth=transform_queue_depth, + read_batch_size=8, + ) + items = list(ds) + assert sorted(item["id"] for item in items) == list(range(NUM_ROWS)) + + +def test_transform_queue_depth_matches_no_backpressure(lance_table): + """With backpressure enabled the same samples are produced as without it.""" + ds_unlimited = StreamingDataset( + lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED + ) + ds_limited = StreamingDataset( + lance_table, + num_splits=NUM_SPLITS, + shuffle_seed=SHUFFLE_SEED, + transform_queue_depth=1, + ) + assert [item["id"] for item in ds_unlimited] == [ + item["id"] for item in ds_limited + ], "transform_queue_depth must not affect the sample ordering or set" + + +def test_transform_queue_depth_bounds_cooked_rows(lance_table): + """prefetch_queue_depth stays within transform_queue_depth * read_batch_size + per split when observed from the main thread during iteration.""" + n_splits = 4 + batch_size = 8 + cooked_depth = 2 + # max cooked rows across all 4 splits: 4 * 2 * 8 = 64 + max_allowed = n_splits * cooked_depth * batch_size + + ds = StreamingDataset( + lance_table, + num_splits=n_splits, + shuffle_seed=SHUFFLE_SEED, + transform_queue_depth=cooked_depth, + read_batch_size=batch_size, + transform_parallelism=1, + world_size=1, + ) + + peak = 0 + for _ in ds: + depth = ds.prefetch_queue_depth + if depth > peak: + peak = depth + + # The main thread observes depth *after* popping a row, so the peak is at + # most max_allowed (one row already popped from the split just served). + assert peak <= max_allowed, ( + f"prefetch_queue_depth peaked at {peak}, expected <= {max_allowed}" + ) + + +def test_transform_queue_depth_does_not_admit_at_capacity_minus_one(tmp_path): + """Admission requires a full read_batch_size of free space, not just one slot. + + The test intercepts ThreadPoolExecutor.submit to make I/O calls execute + synchronously on the main thread. This ensures all raw batches land in + raw_batches (via _drain_io) before _try_submit_tx evaluates the admission + predicate for the first time. Without this, the I/O future for batch N+1 + might still be in io_pending at the capacity-minus-one transition, leaving + raw_batches empty and causing _try_submit_tx to skip the admission check + entirely — so both the correct and the broken predicate produce depth=0 + observations and the test cannot distinguish them. + + With all raw batches pre-loaded in raw_batches the 4→3 cooked transition + (consuming one row from a full cooked queue) always triggers _try_submit_tx + against a non-empty raw_batches. + + With transform_queue_depth=1 and batch_size=4, max_cooked_rows=4. + A transform may only be submitted when in_pipeline + batch_size <= 4, i.e. + when in_pipeline == 0 (cooked is completely empty). Under the old broken + predicate (in_pipeline >= max_cooked_rows) the second transform would be + admitted with cooked containing batch_size-1 rows still unconsumed. + """ + import concurrent.futures as cf + from concurrent.futures import ThreadPoolExecutor + from unittest.mock import patch + + db = lancedb.connect(tmp_path) + batch_size = 4 + # Four full batches → four transform submissions to observe. + table = db.create_table("t", pa.table({"id": list(range(batch_size * 4))})) + + cooked_at_submit: list[int] = [] + + original_submit = ThreadPoolExecutor.submit + + def tracking_submit(self, fn, *args, **kwargs): + name = getattr(fn, "__name__", "") + if name == "_io_call": + # Run I/O synchronously on the calling (main) thread and return an + # already-completed Future. _drain_io checks fut.done(), so a + # completed Future is moved to raw_batches immediately on the next + # _advance call — making raw-batch readiness deterministic at the + # capacity-minus-one transition instead of depending on I/O thread + # scheduling. + fut = cf.Future() + try: + fut.set_result(fn(*args, **kwargs)) + except Exception as exc: + fut.set_exception(exc) + return fut + if name == "_tx_call_guarded": + # Capture cooked depth synchronously on the main thread before the + # transform worker can drain the queue. + ref = ds._cooked_ref + cooked_at_submit.append(len(ref[0]) if ref is not None else -1) + return original_submit(self, fn, *args, **kwargs) + + with patch.object(ThreadPoolExecutor, "submit", tracking_submit): + ds = StreamingDataset( + table, + num_splits=1, + shuffle_seed=42, + read_batch_size=batch_size, + transform_queue_depth=1, + transform_parallelism=1, + ) + list(ds) + + assert len(cooked_at_submit) == 4, ( + f"Expected 4 transform submissions (one per batch), got {len(cooked_at_submit)}" + ) + # With full-batch backpressure each transform is only admitted when the + # cooked queue is completely empty (depth == 0). The old broken predicate + # would admit at depth == batch_size - 1 == 3. + assert all(depth == 0 for depth in cooked_at_submit), ( + "Transform admitted with non-empty cooked queue; full-batch backpressure " + "requires in_pipeline + batch_size <= max_cooked_rows before admission. " + f"Cooked depths at each submission: {cooked_at_submit}" + ) + + +# --------------------------------------------------------------------------- +# Deprecated parameter name tests +# --------------------------------------------------------------------------- + + +def test_prefetch_batches_deprecated_warns(lance_table, caplog): + """prefetch_batches logs a deprecation warning and behaves like io_queue_depth.""" + with caplog.at_level(logging.WARNING, logger="lancedb.streaming"): + ds = StreamingDataset( + lance_table, + num_splits=NUM_SPLITS, + shuffle_seed=SHUFFLE_SEED, + prefetch_batches=2, + ) + messages = [r.message for r in caplog.records if r.levelno >= logging.WARNING] + assert any("deprecated" in m.lower() and "io_queue_depth" in m for m in messages), ( + f"Expected deprecation warning mentioning io_queue_depth; got: {messages}" + ) + assert sorted(item["id"] for item in ds) == list(range(NUM_ROWS)) + + def test_filter_limits_rows(tmp_path): """A filter expression is applied to the permutation so only matching rows are yielded. IDs 0..59 pass ``id < 60``; the other 60 are excluded.""" @@ -1954,7 +2136,7 @@ def test_doc_example_basic(tmp_path): def test_doc_example_prefetch_params(tmp_path): - """doc: Prefetching — read_batch_size and prefetch_batches still cover all rows.""" + """doc: Prefetching — read_batch_size and io_queue_depth still cover all rows.""" db = lancedb.connect(tmp_path) table = db.create_table("t", pa.table({"id": list(range(NUM_ROWS))})) @@ -1963,7 +2145,7 @@ def test_doc_example_prefetch_params(tmp_path): num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED, read_batch_size=8, - prefetch_batches=2, + io_queue_depth=2, ) assert sorted(s["id"] for s in ds) == list(range(NUM_ROWS)) From 6cc77b573c0013e4d36205d1fee42f03972ae29a Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Sun, 23 Aug 2026 00:53:14 -0700 Subject: [PATCH 29/58] chore: update lance dependency to v11.0.0-beta.21 (#4029) Updates the Rust workspace and Java `lance-core` dependency to Lance v11.0.0-beta.21. No compatibility fixes were required; workspace Clippy passes with warnings denied. Triggering tag: https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.21 --- Cargo.lock | 84 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 28 +++++++++--------- java/pom.xml | 2 +- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87fec3378..804b5c8d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arc-swap", "arrow", @@ -4888,8 +4888,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-buffer", @@ -4911,7 +4911,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-buffer", @@ -4925,7 +4925,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-schema", @@ -4934,8 +4934,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrayref", "crunchy", @@ -4945,8 +4945,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-buffer", @@ -4983,8 +4983,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow", "arrow-array", @@ -5013,8 +5013,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow", "arrow-array", @@ -5031,8 +5031,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "proc-macro2", "quote", @@ -5041,8 +5041,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-arith", "arrow-array", @@ -5075,8 +5075,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-arith", "arrow-array", @@ -5107,8 +5107,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arc-swap", "arrow", @@ -5172,8 +5172,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-schema", @@ -5195,8 +5195,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow", "arrow-array", @@ -5232,8 +5232,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-schema", @@ -5247,8 +5247,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow", "async-trait", @@ -5260,8 +5260,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow", "arrow-ipc", @@ -5314,8 +5314,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-buffer", @@ -5329,8 +5329,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow", "arrow-array", @@ -5370,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-schema", @@ -5384,8 +5384,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index c147b06db..d45fe9a68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=11.0.0-beta.21", default-features = false, "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=11.0.0-beta.21", default-features = false, "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=11.0.0-beta.21", default-features = false, "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 56366bb15..e53ad869f 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 11.0.0-beta.19 + 11.0.0-beta.21 false 2.30.0 1.7 From 1b950188c3dc73383707fbab1ce85d4679787e07 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Sun, 23 Aug 2026 08:07:09 +0000 Subject: [PATCH 30/58] =?UTF-8?q?Bump=20version:=200.38.0-beta.4=20?= =?UTF-8?q?=E2=86=92=200.38.0-beta.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index bba238f96..daaec8171 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.4" +current_version = "0.38.0-beta.5" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 804b5c8d6..f83e81dc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5398,7 +5398,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.4" +version = "0.38.0-beta.5" dependencies = [ "ahash", "anyhow", @@ -5486,7 +5486,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.4" +version = "0.38.0-beta.5" dependencies = [ "arrow-array", "arrow-buffer", @@ -5511,7 +5511,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.4" +version = "0.38.0-beta.5" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index cff4cf2f5..b39c2dafb 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.4 + 0.38.0-beta.5 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 98c10ca38..7ebcb5743 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.4 + 0.38.0-beta.5 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index e53ad869f..10dc4c78c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.4 + 0.38.0-beta.5 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 99aed2e71..527715d8c 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.4" +version = "0.38.0-beta.5" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index c84b83c45..42e5bd09c 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 52f2b7901..d01e72d4e 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index afa7fac32..6c57484cf 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 36cfa0343..50a02b0de 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 23ea473ef..c8257452f 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 880b65540..cd98d84d4 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 8cc40de7b..1fda855df 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index dba3ede7c..429845ba8 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index c2f5be2c8..a82757bff 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index dace5aa7c..d88a25257 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.4" +version = "0.38.0-beta.5" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 2920498b1..439c9613b 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.4" +version = "0.38.0-beta.5" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 000e3b506b58ba71e4f0d43d1f5812006c696257 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Sun, 23 Aug 2026 10:31:32 -0700 Subject: [PATCH 31/58] chore: update lance dependency to v11.0.0-beta.22 (#4036) Updates the Rust workspace Lance dependencies and Java lance-core dependency to v11.0.0-beta.22, including the refreshed Cargo lockfile. No compatibility fixes were required; see the [Lance tag](https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.22). --- Cargo.lock | 88 +++++++++++++++++++++++++++------------------------- Cargo.toml | 28 ++++++++--------- java/pom.xml | 2 +- 3 files changed, 61 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f83e81dc3..89e4a1e30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arc-swap", "arrow", @@ -4888,8 +4888,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-buffer", @@ -4911,7 +4911,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-buffer", @@ -4925,7 +4925,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-schema", @@ -4934,8 +4934,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrayref", "crunchy", @@ -4945,8 +4945,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-buffer", @@ -4983,8 +4983,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-array", @@ -5013,8 +5013,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-array", @@ -5031,8 +5031,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "proc-macro2", "quote", @@ -5041,8 +5041,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-arith", "arrow-array", @@ -5075,8 +5075,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-arith", "arrow-array", @@ -5107,8 +5107,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arc-swap", "arrow", @@ -5172,8 +5172,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-schema", @@ -5195,8 +5195,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-array", @@ -5222,7 +5222,11 @@ dependencies = [ "pin-project", "prost", "rand 0.9.5", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", "serde", + "serde_json", "tempfile", "tokio", "tracing", @@ -5232,8 +5236,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-schema", @@ -5247,8 +5251,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "async-trait", @@ -5260,8 +5264,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-ipc", @@ -5314,8 +5318,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-buffer", @@ -5329,8 +5333,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-array", @@ -5370,8 +5374,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-schema", @@ -5384,8 +5388,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index d45fe9a68..716b14d7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0-beta.21", default-features = false, "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=11.0.0-beta.21", default-features = false, "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=11.0.0-beta.21", default-features = false, "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 10dc4c78c..a6b1ced24 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 11.0.0-beta.21 + 11.0.0-beta.22 false 2.30.0 1.7 From 40d4d012e760ba0cc0bb9033ac9bf67fd1b83c43 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Sun, 23 Aug 2026 17:32:40 +0000 Subject: [PATCH 32/58] =?UTF-8?q?Bump=20version:=200.38.0-beta.5=20?= =?UTF-8?q?=E2=86=92=200.38.0-beta.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index daaec8171..f5f981b2c 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.5" +current_version = "0.38.0-beta.6" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 89e4a1e30..6c6dd4aea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5402,7 +5402,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.5" +version = "0.38.0-beta.6" dependencies = [ "ahash", "anyhow", @@ -5490,7 +5490,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.5" +version = "0.38.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.5" +version = "0.38.0-beta.6" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index b39c2dafb..d6c285fc3 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.5 + 0.38.0-beta.6 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 7ebcb5743..23d58124d 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.5 + 0.38.0-beta.6 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index a6b1ced24..a07414ec0 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.5 + 0.38.0-beta.6 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 527715d8c..ead873855 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.5" +version = "0.38.0-beta.6" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 42e5bd09c..d89eb5ad1 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index d01e72d4e..1721c970e 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 6c57484cf..177804dce 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 50a02b0de..9af443583 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index c8257452f..c98aa3812 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index cd98d84d4..05b1086f5 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 1fda855df..50432bdea 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 429845ba8..219025534 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index a82757bff..63c51a799 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index d88a25257..2e4f8996f 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.5" +version = "0.38.0-beta.6" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 439c9613b..d32d88ecd 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.5" +version = "0.38.0-beta.6" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 242ade8017e6935485f37f18a1bd5050434fec06 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Mon, 24 Aug 2026 13:26:36 +0530 Subject: [PATCH 33/58] feat(python): support sequence packing in streaming dataset (#3920) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## How packing works Consider four tokenized documents: [1] [2] [10, 11, 12, 13, 14, 15, 16, 17] [20] With: ``` StreamingDataset( table, shuffle=False, columns=["tokens"], num_splits=2, pack_sequences=5, eos_id=9, pad_id=0, blocks_per_epoch=6, ) ``` the documents are assigned to two fixed logical splits. Each split maintains an independent token buffer, appends eos_id after every document, and emits blocks of five tokens. Because blocks_per_epoch=6, each split emits exactly three blocks: Cycl/e 1: Split 0: [1, 9, 2, 9, 0] # 9 is eos, 0 is padding Split 1: [10, 11, 12, 13, 14] Cycle 2: Split 0: [0, 0, 0, 0, 0] Split 1: [15, 16, 17, 9, 20] Cycle 3: Split 0: [0, 0, 0, 0, 0] Split 1: [9, 0, 0, 0, 0] If a split runs out of tokens early, it emits padded blocks through the fixed budget. This prevents one rank from finishing before another. Logical splits are independent of rank and worker ownership. A checkpoint records each split’s consumed-document count, emitted-block count, remaining tokens, and document boundaries. Merging those per-split states allows the same packed stream to resume after the topology changes. doc_ids identifies document segments, including continuations across block boundaries. It is not a padding mask: padding retains the preceding document ID, so callers must mask padding using a reserved pad_id. blocks_per_epoch="auto" is also available. It estimates the budget from a deterministic bounded sample and warns that the result is approximate. WIP pre-training tests: ``` ┌────────────────────────────────────┬────────────────────────┬─────────────────────────────────────┐ │ │ GPT-2 124M │ GPT-2 medium 354M │ ├────────────────────────────────────┼────────────────────────┼─────────────────────────────────────┤ │ Corpus │ 2.4M docs / 12GB table │ 9.67M docs / 45GB table │ │ Tokens (Chinchilla) │ 2.43B │ 7.0B │ ├────────────────────────────────────┼────────────────────────┼─────────────────────────────────────┤ │ Data prep (ingest→curate→tokenize) │ ~12 min │ ~51 min │ ├────────────────────────────────────┼────────────────────────┼─────────────────────────────────────┤ │ Training wall time │ ~50 min │ 3h 06m │ ├────────────────────────────────────┼────────────────────────┼─────────────────────────────────────┤ │ Throughput / MFU │ 1.60M tok/s / 35% │ 684k tok/s / 42.0%, │ ├────────────────────────────────────┼────────────────────────┼─────────────────────────────────────┤ │ Final val loss │ 3.230 │ 2.840 │ └────────────────────────────────────┴────────────────────────┴─────────────────────────────────────┘ ``` --------- Co-authored-by: OpenAI Codex --- python/python/lancedb/streaming.py | 491 +++++++++++++++--- .../python/tests/test_elastic_dataloader.py | 208 ++++++++ 2 files changed, 638 insertions(+), 61 deletions(-) diff --git a/python/python/lancedb/streaming.py b/python/python/lancedb/streaming.py index 6951fa0bc..c54940ab7 100644 --- a/python/python/lancedb/streaming.py +++ b/python/python/lancedb/streaming.py @@ -24,11 +24,16 @@ import os import random import threading import time +import warnings from collections import deque from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy from multiprocessing import RawArray -from typing import Any, Callable, Iterator, Optional, Union +from typing import Any, Callable, cast, Iterator, Literal, Optional, Union +import pyarrow as pa +import pyarrow.compute as pc +import torch from torch.utils.data import IterableDataset, get_worker_info from .permutation import ( @@ -132,6 +137,39 @@ class StreamingDataset(IterableDataset): Maximum number of transforms to run concurrently. Must be greater than zero. When ``None`` (the default), uses ``os.cpu_count()`` or 1 when the CPU count is unavailable. + pack_sequences: + Sequence-packing mode: token lists from consecutive documents are + joined with ``eos_id`` and sliced into blocks of this many tokens. + Each item is then a dict of two ``(pack_sequences,)`` LongTensors — + ``input_ids`` and ``doc_ids`` (per-position document index within + the block, for block-diagonal masks or position-id resets). + * Packing happens independently per owned split and preserves per-split + resume state. + * When a split cannot fill a real block for a cycle but an owned sibling + still can, or when only a short tail remains at epoch end, the short buffer + is padded to ``pack_sequences`` with ``pad_id`` so every local cycle emits + one block per owned split. + * ``eos_id``, ``pad_id``, and ``columns`` naming a single integer-list + column are required; incompatible with ``transform``. + eos_id: + Separator token id between packed documents. Required with + ``pack_sequences``, ignored otherwise. + pad_id: + Padding token id used to complete blocks when a split runs out of + real tokens mid-cycle or at epoch end. Required with + ``pack_sequences``, ignored otherwise. It must be reserved for padding: + padding positions retain the preceding document's ``doc_id`` (or zero + in an all-padding block), so callers must mask them separately using + ``input_ids == pad_id``. + blocks_per_epoch: + Total number of packed blocks emitted globally per epoch. Required with + ``pack_sequences``. An integer must be divisible by ``num_splits``. + Every logical split emits exactly ``blocks_per_epoch / num_splits`` + blocks: exhausted splits emit padding, while tokens beyond the budget + are left out of the epoch. This fixed per-split budget keeps packed + iteration and checkpoints independent of rank topology. + Pass ``"auto"`` to estimate a corpus-level budget from a bounded sample + of token lists. The estimate may be inaccurate. on_transform_error: What to do when the transform raises an exception: @@ -210,6 +248,10 @@ class StreamingDataset(IterableDataset): filter: Optional[str] = None, transform: Optional[Callable] = None, transform_parallelism: Optional[int] = None, + pack_sequences: Optional[int] = None, + eos_id: Optional[int] = None, + pad_id: Optional[int] = None, + blocks_per_epoch: Optional[Union[int, Literal["auto"]]] = None, on_transform_error: Union[str, Callable[[Exception], bool]] = "raise", transform_queue_depth: Optional[int] = None, connection_factory: Optional[Callable[[str], Any]] = None, @@ -237,6 +279,55 @@ class StreamingDataset(IterableDataset): raise ValueError("io_queue_depth must be greater than 0") if transform_parallelism is not None and transform_parallelism <= 0: raise ValueError("transform_parallelism must be greater than 0") + if pack_sequences is not None: + if pack_sequences <= 0: + raise ValueError("pack_sequences must be greater than 0") + if eos_id is None: + raise ValueError("eos_id is required when pack_sequences is set") + if pad_id is None: + raise ValueError("pad_id is required when pack_sequences is set") + if blocks_per_epoch is None: + raise ValueError( + "blocks_per_epoch is required when pack_sequences is set" + ) + if blocks_per_epoch != "auto": + if not isinstance(blocks_per_epoch, int) or isinstance( + blocks_per_epoch, bool + ): + raise ValueError( + "blocks_per_epoch must be a positive integer or 'auto'" + ) + if blocks_per_epoch <= 0: + raise ValueError("blocks_per_epoch must be greater than 0") + if blocks_per_epoch % num_splits != 0: + raise ValueError( + f"blocks_per_epoch ({blocks_per_epoch}) must be divisible by " + f"num_splits ({num_splits})" + ) + if transform is not None: + raise ValueError("transform cannot be combined with pack_sequences") + if columns is None or len(columns) != 1: + raise ValueError( + "pack_sequences requires columns to name exactly one " + "list-typed column of token ids" + ) + field = table.schema.field(columns[0]) + if not ( + pa.types.is_list(field.type) + or pa.types.is_large_list(field.type) + or pa.types.is_fixed_size_list(field.type) + ): + raise ValueError( + f"pack_sequences requires a list-typed token column; " + f"{columns[0]} has type {field.type}" + ) + if not pa.types.is_integer(field.type.value_type): + raise ValueError( + "pack_sequences requires a token column with integer values; " + f"{columns[0]} has value type {field.type.value_type}" + ) + elif blocks_per_epoch is not None: + raise ValueError("blocks_per_epoch requires pack_sequences") if on_transform_error not in ("raise", "skip", "warn") and not callable( on_transform_error ): @@ -261,11 +352,20 @@ class StreamingDataset(IterableDataset): self._filter = filter self._transform = transform self._transform_parallelism = transform_parallelism + self._pack_sequences = pack_sequences + self._eos_id = eos_id + self._pad_id = pad_id + self._blocks_per_epoch = blocks_per_epoch self._on_transform_error = on_transform_error self._transform_queue_depth = transform_queue_depth self._connection_factory = connection_factory self._worker_info_override = worker_info_override + # Packing resume state: permutation positions and partial-block buffers. + self._pack_consumed: list[int] = [0] * num_splits + self._pack_buffers: dict[int, dict[str, list[int]]] = {} + self._pack_blocks_emitted: list[int] = [0] * num_splits + # Live references to pipeline state, set only while __iter__ is running # in the same process. Used by the observability properties when the # DataLoader runs with num_workers=0. @@ -315,6 +415,9 @@ class StreamingDataset(IterableDataset): else: self._perm_table = builder.split_sequential(fixed=num_splits).execute() + if self._blocks_per_epoch == "auto": + self._blocks_per_epoch = self._estimate_blocks_per_epoch() + # Contiguous block of global split indices assigned to this rank. splits_per_rank = num_splits // world_size rank_start = rank * splits_per_rank @@ -322,6 +425,71 @@ class StreamingDataset(IterableDataset): range(rank_start, rank_start + splits_per_rank) ) + def _estimate_blocks_per_epoch(self) -> int: + """Estimate a fixed packed-block budget from a bounded token sample.""" + # TODO: Replace this fallback with Lance's dedicated exact token-count + # estimation API once it is available. + if self._pack_sequences is None or not self._columns: + raise RuntimeError( + "packing must be configured before estimating its budget" + ) + + pack_len = self._pack_sequences + token_column = self._columns[0] + sample_cap_per_split = max(1, 100_000 // self._num_splits) + sampled_tokens = 0 + total_sampled = 0 + total_rows = 0 + rng = random.Random(self._shuffle_seed) + + warnings.warn( + "blocks_per_epoch='auto' uses an approximate token-count sample; " + "pass an explicit value for exact epoch sizing", + ) + + for split in range(self._num_splits): + permutation = Permutation.from_tables( + self._table, self._perm_table, split=split + ) + permutation = permutation.select_columns([token_column]) + permutation = permutation.with_transform(Transforms.arrow2arrow) + split_rows = permutation.num_rows + if split_rows == 0: + raise ValueError( + "blocks_per_epoch='auto' cannot estimate an empty dataset" + ) + + # Sample roughly 1% from each logical split, with at least one row + # per split and a global target cap of 100,000 rows. + sample_rows = min( + split_rows, + max(1, min((split_rows + 99) // 100, sample_cap_per_split)), + ) + sample_offsets = sorted(rng.sample(range(split_rows), sample_rows)) + sample_batch_size = max(1, self._read_batch_size) + for start in range(0, sample_rows, sample_batch_size): + batch = permutation.__getitems__( + sample_offsets[start : start + sample_batch_size] + ) + lengths = pc.list_value_length(batch.column(0)) + if lengths.null_count: + raise ValueError("pack_sequences does not support null token lists") + sampled_tokens += int(pc.sum(lengths).as_py()) + + total_sampled += sample_rows + total_rows += split_rows + + # Pool the samples into one global average. Each document contributes + # one EOS token. + estimated_tokens = ( + (sampled_tokens + total_sampled) * total_rows // total_sampled + ) + blocks = estimated_tokens // pack_len + return max( + self._num_splits, + blocks - blocks % self._num_splits, + ) + def _resolve_my_splits(self) -> list[int]: """Return the split indices this instance should read in __iter__.""" torch_worker_info = get_worker_info() @@ -372,8 +540,14 @@ class StreamingDataset(IterableDataset): ) if self._columns is not None: perm = perm.select_columns(self._columns) - perm = perm.with_transform(lambda batch: batch) - start_pos = self._resume_positions.get(split_idx, self._resume_offset) + perm = perm.with_transform(Transforms.arrow2arrow) + # Both modes resume from absolute permutation positions. Packing + # stores them separately because it also checkpoints partial blocks. + start_pos = ( + self._pack_consumed[split_idx] + if self._pack_sequences is not None + else self._resume_positions.get(split_idx, self._resume_offset) + ) if start_pos > 0: perm = perm.with_skip(start_pos) initial_positions.append(start_pos) @@ -395,9 +569,24 @@ class StreamingDataset(IterableDataset): if self._transform_parallelism is not None else (os.cpu_count() or 1) ) - final_transform = ( - self._transform if self._transform is not None else Transforms.arrow2python - ) + final_transform: Callable[[pa.RecordBatch], Any] + if self._pack_sequences is not None: + # Packing consumes raw token lists, one per document. + def arrow_tokens(batch: pa.RecordBatch) -> list[list[int]]: + token_column = batch.column(0) + if token_column.null_count or token_column.flatten().null_count: + raise ValueError( + "pack_sequences does not support null token lists or values" + ) + return cast(list[list[int]], token_column.to_pylist()) + + final_transform = arrow_tokens + else: + final_transform = ( + self._transform + if self._transform is not None + else Transforms.arrow2python + ) # None means no limit; otherwise cap rows per split to # transform_queue_depth batches worth (including in-flight transforms). max_cooked_rows = ( @@ -574,6 +763,82 @@ class StreamingDataset(IterableDataset): else: break # split exhausted + def _update_stats(*, idle: bool = False) -> None: + """Refresh pipeline statistics visible to the parent process.""" + ws = self._worker_stats + ws[0] = sum(split_sizes[j] - fetch_head[j] for j in range(n)) + ws[1] = ( + 0 + if idle + else sum(batch.num_rows for q in raw_batches for _, batch in q) + ) + ws[2] = 0 if idle else sum(len(q) for q in cooked) + ws[3] = sum(local_consumed) + ws[4] = self._bytes_loaded + ws[5] = int(self._fetch_time * 1_000_000) + ws[6] = int(self._transform_time * 1_000_000) + ws[7] = self._rows_skipped + + # Sequence-packing helpers + pack_len = cast(int, self._pack_sequences) + eos_id = cast(int, self._eos_id) + pad_id = cast(int, self._pad_id) + blocks_per_split = ( + cast(int, self._blocks_per_epoch) // self._num_splits + if self._pack_sequences is not None + else 0 + ) + pack_consumed = list(self._pack_consumed) + pack_buffers = deepcopy(self._pack_buffers) + pack_blocks_emitted = list(self._pack_blocks_emitted) + + def _pack_buffer(i: int) -> dict[str, list[int]]: + return pack_buffers.setdefault(my_splits[i], {"tokens": [], "starts": []}) + + def _fill_block(i: int) -> None: + """Fill split i's buffer to one block or exhaust the split.""" + buf = _pack_buffer(i) + while len(buf["tokens"]) < pack_len: + _ensure_cooked(i) + if not cooked[i]: + return + buf["starts"].append(len(buf["tokens"])) + pos, tokens = cooked[i].popleft() + buf["tokens"].extend(tokens) + buf["tokens"].append(eos_id) + pack_consumed[my_splits[i]] = pos + 1 + local_consumed[i] += 1 + _advance(i) + + def _emit_block(i: int) -> dict[str, Any]: + buf = _pack_buffer(i) + tokens, starts = buf["tokens"], buf["starts"] + # doc_ids label document segments within the block; 0 also covers + # the continuation of a document begun in a prior block. + doc_ids = torch.zeros(pack_len, dtype=torch.int64) + doc_starts = [s for s in starts if 0 < s < pack_len] + doc_ids[doc_starts] = 1 + doc_ids.cumsum_(dim=0) # cumulative sum marks document boundaries + block = { + "input_ids": torch.tensor(tokens[:pack_len], dtype=torch.int64), + "doc_ids": doc_ids, + } + del tokens[:pack_len] + # Shift start boundaries for the next call. + buf["starts"] = [s - pack_len for s in starts if s >= pack_len] + return block + + def _commit_pack_state() -> None: + self._pack_consumed = list(pack_consumed) + self._pack_buffers = { + split: { + "tokens": list(buffer["tokens"]), + "starts": list(buffer["starts"]), + } + for split, buffer in pack_buffers.items() + } + self._pack_blocks_emitted = list(pack_blocks_emitted) + # ── Main loop ───────────────────────────────────────────────────────── with ThreadPoolExecutor(max_workers=n * io_queue_depth) as io_pool: @@ -583,10 +848,42 @@ class StreamingDataset(IterableDataset): self._fetch_head_ref = fetch_head self._split_sizes_ref = split_sizes self._local_consumed_ref = local_consumed + try: for i in range(n): _fill_io(i) + if self._pack_sequences is not None: + first_count = pack_blocks_emitted[my_splits[0]] + if any( + pack_blocks_emitted[split] != first_count + for split in my_splits[1:] + ): + raise ValueError( + "Packed checkpoint is not aligned across the splits " + "owned by this iterator; merge every rank " + "state with merge_state_dicts before resuming on a " + "different topology" + ) + + while pack_blocks_emitted[my_splits[0]] < blocks_per_split: + # Each logical split gets one block per cycle. Exhausted + # splits are padded through the fixed global budget. + for i in range(n): + _fill_block(i) + + for i in range(n): + tokens = _pack_buffer(i)["tokens"] + if len(tokens) < pack_len: + tokens.extend([pad_id] * (pack_len - len(tokens))) + block = _emit_block(i) + pack_blocks_emitted[my_splits[i]] += 1 + if i == n - 1: + _commit_pack_state() + _update_stats() + yield block + return + while True: # A cycle only runs if every split can still produce a # row. Without skips all splits exhaust simultaneously @@ -620,21 +917,7 @@ class StreamingDataset(IterableDataset): self._resume_offset = initial_offset + local_consumed[i] for j, split_idx in enumerate(my_splits): self._resume_positions[split_idx] = pos_consumed[j] - ws = self._worker_stats - ws[0] = sum( - split_sizes[j] - fetch_head[j] for j in range(n) - ) - ws[1] = sum( - batch.num_rows - for q in raw_batches - for _, batch in q - ) - ws[2] = sum(len(q) for q in cooked) - ws[3] = sum(local_consumed) - ws[4] = self._bytes_loaded - ws[5] = int(self._fetch_time * 1_000_000) - ws[6] = int(self._transform_time * 1_000_000) - ws[7] = self._rows_skipped + _update_stats() yield row finally: @@ -642,15 +925,7 @@ class StreamingDataset(IterableDataset): # when iteration ends mid-cycle (e.g. a split whose rows # were all skipped before completing a single cycle), so # counters like rows_skipped would otherwise be stale. - ws = self._worker_stats - ws[0] = sum(split_sizes[j] - fetch_head[j] for j in range(n)) - ws[1] = 0 # queue-depth properties document 0 when idle - ws[2] = 0 - ws[3] = sum(local_consumed) - ws[4] = self._bytes_loaded - ws[5] = int(self._fetch_time * 1_000_000) - ws[6] = int(self._transform_time * 1_000_000) - ws[7] = self._rows_skipped + _update_stats(idle=True) self._raw_batches_ref = None self._cooked_ref = None self._fetch_head_ref = None @@ -808,21 +1083,31 @@ class StreamingDataset(IterableDataset): def state_dict(self) -> dict: """Snapshot the dataset's consumption state. - The returned dict is topology-independent: at global step boundaries - every split has been consumed the same number of times (by the - round-robin design), so the per-split count is a single uniform value - that is identical across all ranks and DataLoader workers. - - ``positions_consumed_per_split`` records how far into each split's - permutation iteration has advanced. It only differs from - ``samples_consumed_per_split`` when ``on_transform_error`` skipped - rows, in which case entries are exact for the splits this instance - iterated and a lower bound (the sample count) for splits owned by - other ranks or workers. Combine the state dicts from all ranks with + In row mode, the returned dict is topology-independent at global step + boundaries. ``positions_consumed_per_split`` records how far each + split's permutation has advanced, which can differ from the sample + count when ``on_transform_error`` skips rows. Combine state dicts from + every rank with [merge_state_dicts][lancedb.streaming.StreamingDataset.merge_state_dicts] - to recover the exact value for every split before resuming on a - different topology. + before resuming on a different topology. + + Packed state includes partial token buffers and emitted block counts + for every logical split. When packing is sharded, merge every rank + state with ``merge_state_dicts`` before loading it. """ + if self._pack_sequences is not None: + return { + "shuffle_seed": self._shuffle_seed, + "num_splits": self._num_splits, + "epoch": self._epoch, + "pack_sequences": self._pack_sequences, + "eos_id": self._eos_id, + "pad_id": self._pad_id, + "blocks_per_epoch": self._blocks_per_epoch, + "samples_consumed_per_split": list(self._pack_consumed), + "blocks_emitted_per_split": list(self._pack_blocks_emitted), + "pack_buffers": deepcopy(self._pack_buffers), + } positions = [ self._resume_positions.get(split, self._resume_offset) for split in range(self._num_splits) @@ -840,7 +1125,9 @@ class StreamingDataset(IterableDataset): Raises ``ValueError`` if ``num_splits`` or ``shuffle_seed`` differ from the checkpoint, since a different split structure or shuffle order - makes mid-epoch resumption meaningless. + makes mid-epoch resumption meaningless. Packed checkpoints + pin ``pack_sequences``, ``eos_id``, ``pad_id``, + ``blocks_per_epoch``, and ``epoch``. """ if state["num_splits"] != self._num_splits: raise ValueError( @@ -852,6 +1139,31 @@ class StreamingDataset(IterableDataset): f"shuffle_seed mismatch: checkpoint has {state['shuffle_seed']}, " f"current dataset has {self._shuffle_seed}" ) + + if "pack_buffers" in state or self._pack_sequences is not None: + for key in ( + "pack_sequences", + "eos_id", + "pad_id", + "blocks_per_epoch", + "epoch", + ): + ours = getattr(self, f"_{key}") + if state.get(key) != ours: + raise ValueError( + f"{key} mismatch: checkpoint has {state.get(key)}, " + f"current dataset has {ours}" + ) + self._pack_consumed = [int(c) for c in state["samples_consumed_per_split"]] + self._pack_blocks_emitted = [ + int(c) for c in state["blocks_emitted_per_split"] + ] + self._pack_buffers = { + int(g): {"tokens": list(b["tokens"]), "starts": list(b["starts"])} + for g, b in state["pack_buffers"].items() + } + return + consumed = state["samples_consumed_per_split"] # All entries are equal at step boundaries; use the first. if isinstance(consumed, list): @@ -873,25 +1185,22 @@ class StreamingDataset(IterableDataset): def merge_state_dicts(states: list[dict]) -> dict: """Merge state dicts saved by different ranks into one exact state. - Only needed when ``on_transform_error`` skips rows in multi-rank - training: each rank then knows the exact permutation position only for - its own splits, and records a lower bound for the rest. Because - exactly one rank owns each split, the elementwise maximum across all - ranks' ``positions_consumed_per_split`` recovers the exact position of - every split. Without skipped rows every rank's state is already - identical and merging is a no-op. + For row mode, the elementwise maximum of permutation positions recovers + splits advanced by different ranks after transform failures. For packed + mode, the state that emitted the most blocks for each logical split + supplies that split's permutation position and partial token buffer. Packed + states must cover every rank at the same global step. - Raises ``ValueError`` if the states are empty or were not produced by - the same run (mismatched seed, split count, epoch, or sample counts). + Raises ``ValueError`` if the states are empty, were not produced by + the same run, or do not represent the same global step. The merge is always all-to-all and topology-agnostic: collect the - ``state_dict()`` from every rank of the *previous* run into one list, - merge that whole list, and hand the identical merged result to every - rank of the *next* run — regardless of whether the rank count grew, - shrank, or stayed the same. There is no pairwise or subset merging - step, because each split's exact position is only known to whichever - rank owned that split, and the elementwise maximum needs every rank's - contribution to be correct. + ``state_dict()`` from every rank of the *previous* run into + one list, merge that whole list, and hand the identical merged result + to every rank of the *next* run — regardless of whether the + topology grew, shrank, or stayed the same. There is no pairwise or + subset merging step, because each split's exact state is only known to + whichever iterator owned that split. For example, checkpointing 8 ranks and resuming on 4 (the same pattern applies when growing, e.g. 4 ranks resuming on 8):: @@ -924,13 +1233,73 @@ class StreamingDataset(IterableDataset): if not states: raise ValueError("merge_state_dicts requires at least one state dict") first = states[0] + packed = "pack_buffers" in first + config_keys = ["shuffle_seed", "num_splits", "epoch"] + if packed: + config_keys.extend( + ["pack_sequences", "eos_id", "pad_id", "blocks_per_epoch"] + ) + for state in states[1:]: - for key in ("shuffle_seed", "num_splits", "epoch"): + if ("pack_buffers" in state) != packed: + raise ValueError("cannot merge packed and unpacked state dicts") + for key in config_keys: if state[key] != first[key]: raise ValueError( f"{key} mismatch across state dicts: " f"{state[key]} != {first[key]}" ) + + if packed: + num_splits = first["num_splits"] + for state in states: + for key in ( + "samples_consumed_per_split", + "blocks_emitted_per_split", + ): + if len(state[key]) != num_splits: + raise ValueError( + f"{key} must contain one entry per logical split" + ) + + merged_consumed = [] + merged_emitted = [] + merged_buffers = {} + for split in range(num_splits): + owner = states[0] + owner_progress = ( + owner["blocks_emitted_per_split"][split], + owner["samples_consumed_per_split"][split], + ) + for state in states[1:]: + progress = ( + state["blocks_emitted_per_split"][split], + state["samples_consumed_per_split"][split], + ) + if progress > owner_progress: + owner = state + owner_progress = progress + merged_consumed.append(owner["samples_consumed_per_split"][split]) + merged_emitted.append(owner["blocks_emitted_per_split"][split]) + buffer = owner["pack_buffers"].get( + split, owner["pack_buffers"].get(str(split)) + ) + if buffer is not None: + merged_buffers[split] = deepcopy(buffer) + + if len(set(merged_emitted)) > 1: + raise ValueError( + "packed state dicts were not captured at the same global " + "step or do not cover every rank" + ) + + merged = dict(first) + merged["samples_consumed_per_split"] = merged_consumed + merged["blocks_emitted_per_split"] = merged_emitted + merged["pack_buffers"] = merged_buffers + return merged + + for state in states[1:]: if ( state["samples_consumed_per_split"] != first["samples_consumed_per_split"] diff --git a/python/python/tests/test_elastic_dataloader.py b/python/python/tests/test_elastic_dataloader.py index 4d860a5de..ff65d100c 100644 --- a/python/python/tests/test_elastic_dataloader.py +++ b/python/python/tests/test_elastic_dataloader.py @@ -2113,6 +2113,214 @@ def test_shuffle_seed_none_generates_stable_seed(lance_table): assert first == second, "Same resolved seed must produce the same ordering" +# Sequence packing tests + + +def _create_token_table(tmp_path, documents): + db = lancedb.connect(tmp_path) + tokens = pa.array(documents, type=pa.list_(pa.int64())) + return db.create_table("tokens", pa.table({"tokens": tokens})) + + +def _packed_dataset(table, pack_sequences, *, blocks_per_epoch, pad_id=0, **kwargs): + return StreamingDataset( + table, + shuffle=False, + columns=["tokens"], + pack_sequences=pack_sequences, + eos_id=9, + pad_id=pad_id, + blocks_per_epoch=blocks_per_epoch, + **kwargs, + ) + + +def test_pack_sequences_emits_blocks_and_pads_final_tail(tmp_path): + table = _create_token_table(tmp_path, [[1, 2], [3, 4], [5]]) + dataset = _packed_dataset(table, 6, blocks_per_epoch=2) + + blocks = list(dataset) + + assert len(blocks) == 2 + assert blocks[0]["input_ids"].tolist() == [1, 2, 9, 3, 4, 9] + assert blocks[0]["doc_ids"].tolist() == [0, 0, 0, 1, 1, 1] + assert blocks[1]["input_ids"].tolist() == [5, 9, 0, 0, 0, 0] + assert blocks[1]["doc_ids"].tolist() == [0, 0, 0, 0, 0, 0] + assert blocks[0]["input_ids"].dtype == torch.int64 + assert blocks[0]["doc_ids"].dtype == torch.int64 + + +def test_pack_sequences_pads_lagging_splits(tmp_path): + table = _create_token_table( + tmp_path, + [[1], [2], [10, 11, 12, 13, 14, 15, 16, 17], [20]], + ) + dataset = _packed_dataset(table, 5, blocks_per_epoch=6, num_splits=2) + input_ids = [block["input_ids"].tolist() for block in dataset] + # Split 0 has four real tokens including EOS markers, while split 1 has + # eleven. Packing must emit three complete two-split cycles. + assert input_ids == [ + [1, 9, 2, 9, 0], + [10, 11, 12, 13, 14], + [0, 0, 0, 0, 0], + [15, 16, 17, 9, 20], + [0, 0, 0, 0, 0], + [9, 0, 0, 0, 0], + ] + + per_rank = [] + for rank in range(2): + rank_dataset = _packed_dataset( + table, + 5, + blocks_per_epoch=6, + num_splits=2, + world_size=2, + rank=rank, + ) + per_rank.append([block["input_ids"].tolist() for block in rank_dataset]) + + assert [len(blocks) for blocks in per_rank] == [3, 3] + sharded = [block for cycle in zip(*per_rank) for block in cycle] + assert sharded == input_ids + + +def test_pack_sequences_auto_estimates_filtered_token_column(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table( + "tokens", + pa.table( + { + "tokens": pa.array([[1] * 4, [2] * 9], type=pa.list_(pa.int64())), + "keep": [True, False], + } + ), + ) + table.add( + pa.table( + { + "tokens": pa.array([[3] * 4, [4] * 9], type=pa.list_(pa.int64())), + "keep": [True, False], + } + ) + ) + + with pytest.warns(UserWarning, match="approximate token-count sample"): + dataset = _packed_dataset( + table, + 5, + blocks_per_epoch="auto", + num_splits=2, + filter="keep", + ) + + # Two kept documents contain 8 tokens plus 2 EOS tokens: two blocks. + assert dataset.state_dict()["blocks_per_epoch"] == 2 + + +def test_pack_sequences_checkpoint_resumes_on_new_topology(tmp_path): + table = _create_token_table( + tmp_path, + [[1], [2], [10, 11, 12, 13, 14, 15, 16, 17], [20]], + ) + kwargs = dict(pack_sequences=5, blocks_per_epoch=6, num_splits=2) + reference = list(_packed_dataset(table, **kwargs)) + + datasets = [ + _packed_dataset(table, world_size=2, rank=rank, **kwargs) for rank in range(2) + ] + iterators = [iter(dataset) for dataset in datasets] + first_cycle = [next(iterator) for iterator in iterators] + checkpoint = StreamingDataset.merge_state_dicts( + [dataset.state_dict() for dataset in datasets] + ) + for iterator in iterators: + iterator.close() + + resumed = _packed_dataset(table, **kwargs) + resumed.load_state_dict(checkpoint) + actual_remaining = list(resumed) + + assert [block["input_ids"].tolist() for block in first_cycle] == [ + [1, 9, 2, 9, 0], + [10, 11, 12, 13, 14], + ] + assert checkpoint["blocks_emitted_per_split"] == [1, 1] + assert [block["input_ids"].tolist() for block in actual_remaining] == [ + block["input_ids"].tolist() for block in reference[2:] + ] + assert [block["doc_ids"].tolist() for block in actual_remaining] == [ + block["doc_ids"].tolist() for block in reference[2:] + ] + + +def test_pack_sequences_validates_configuration_and_tokens(tmp_path): + table = _create_token_table(tmp_path, [[1, 2]]) + + with pytest.raises(ValueError, match="pad_id is required"): + StreamingDataset( + table, + shuffle=False, + columns=["tokens"], + pack_sequences=4, + eos_id=9, + ) + + with pytest.raises(ValueError, match="blocks_per_epoch is required"): + StreamingDataset( + table, + shuffle=False, + columns=["tokens"], + pack_sequences=4, + eos_id=9, + pad_id=0, + ) + + with pytest.raises(ValueError, match="must be divisible"): + _packed_dataset(table, 4, blocks_per_epoch=3, num_splits=2) + + with pytest.raises(ValueError, match="positive integer or 'auto'"): + _packed_dataset(table, 4, blocks_per_epoch="estimate") + + checkpoint = _packed_dataset(table, 4, blocks_per_epoch=1).state_dict() + resumed = _packed_dataset(table, 4, blocks_per_epoch=1, pad_id=8) + with pytest.raises(ValueError, match="pad_id mismatch"): + resumed.load_state_dict(checkpoint) + + float_db = lancedb.connect(tmp_path / "float") + float_table = float_db.create_table( + "tokens", + pa.table({"tokens": pa.array([[1.5, 2.5]], type=pa.list_(pa.float64()))}), + ) + with pytest.raises(ValueError, match="token column with integer values"): + _packed_dataset(float_table, 4, blocks_per_epoch=1) + + null_db = lancedb.connect(tmp_path / "null") + null_table = null_db.create_table( + "tokens", + pa.table({"tokens": pa.array([None], type=pa.list_(pa.int64()))}), + ) + with pytest.raises(ValueError, match="does not support null token lists"): + list(_packed_dataset(null_table, 4, blocks_per_epoch=1)) + + null_value_db = lancedb.connect(tmp_path / "null_value") + null_value_table = null_value_db.create_table( + "tokens", + pa.table( + {"tokens": pa.array([[1], [2, None], [3]], type=pa.list_(pa.int64()))} + ), + ) + blocks = list( + _packed_dataset( + null_value_table, + 2, + blocks_per_epoch=2, + on_transform_error="skip", + ) + ) + assert [block["input_ids"].tolist() for block in blocks] == [[1, 9], [3, 9]] + + # --------------------------------------------------------------------------- # Doc examples — each test mirrors the code snippet in index.mdx so that # broken doc examples are caught before they ship. From b0dae5eb0b468a89b2f353a548e49011ce4881d6 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 25 Aug 2026 00:01:32 +0800 Subject: [PATCH 34/58] feat: return typed refresh job results (#4013) ## Problem `refresh_column_async` returned a unit-result job even though durable refresh jobs carry a canonical terminal result. Python callers could not obtain row counts or source and published versions through the public `Job` API, and local and remote refresh jobs exposed different result semantics. ## Behavior `refresh_column_async` now returns `Job[RefreshColumnResult]` for local and remote tables. The general typed-job bridge binds each endpoint to its public result model while preserving unit-result jobs and existing status, wait, cancel, and timeout behavior. A local no-op refresh reports no published version. The Node.js API continues to resolve `wait()` as `void`; its binding erases the Rust result type internally to preserve the existing public contract. ## Ownership and integration boundary LanceDB owns the language-neutral `Job` contract and language-binding decode. Sophon owns production and durable persistence of terminal payloads. Sophon #7348 and #7378 now publish the canonical refresh result for Function-backed and expression-backed refresh jobs, respectively. The remote client fixture matches the merged server schema; live deployment and end-to-end demo acceptance remain separate rollout checks. --- nodejs/src/job.rs | 7 +- python/python/lancedb/__init__.py | 1 + python/python/lancedb/_lancedb.pyi | 11 +- python/python/lancedb/db.py | 4 +- python/python/lancedb/functions.py | 10 +- python/python/lancedb/job.py | 58 ++++--- python/python/lancedb/remote/table.py | 4 +- python/python/lancedb/table.py | 39 ++++- python/python/tests/test_db.py | 4 +- .../tests/test_first_class_function_slice2.py | 2 +- python/python/tests/test_index.py | 2 +- python/python/tests/test_remote_db.py | 76 ++++++++- python/python/tests/test_table.py | 21 ++- python/src/connection.rs | 2 +- python/src/job.rs | 73 +++------ python/src/lib.rs | 1 - python/src/table.rs | 2 +- rust/lancedb/src/function.rs | 14 +- rust/lancedb/src/job.rs | 152 +++++++++++++----- rust/lancedb/src/remote/db.rs | 4 +- rust/lancedb/src/remote/table.rs | 36 ++++- rust/lancedb/src/table.rs | 22 ++- rust/lancedb/src/table/refresh.rs | 71 ++++++-- 23 files changed, 429 insertions(+), 187 deletions(-) diff --git a/nodejs/src/job.rs b/nodejs/src/job.rs index 6aaeee174..14013fd27 100644 --- a/nodejs/src/job.rs +++ b/nodejs/src/job.rs @@ -14,9 +14,12 @@ pub struct Job { } impl Job { - pub(crate) fn new(inner: lancedb::Job) -> Self { + pub(crate) fn new(inner: lancedb::Job) -> Self + where + T: Clone + Send + Sync + 'static, + { Self { - inner: Arc::new(inner), + inner: Arc::new(inner.map(|_| ())), } } } diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index aa473c5f8..df8950686 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -29,6 +29,7 @@ from .functions import ( FunctionRegistrationRequest as FunctionRegistrationRequest, FunctionVersion as FunctionVersion, PythonRuntimeSpec as PythonRuntimeSpec, + RefreshColumnResult as RefreshColumnResult, UdfDefinition as UdfDefinition, udf as udf, ) diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 648469579..1e314ede8 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -147,7 +147,7 @@ class Connection(object): limit: Optional[int], ) -> list[str]: ... # Deprecated: Use list_tables instead def job(self, job_id: str) -> Job: ... - async def create_function_async(self, request_json: str) -> FunctionJob: ... + async def create_function_async(self, request_json: str) -> Job: ... async def get_function(self, name: str, version: str) -> str: ... async def list_jobs(self) -> List[JobInfo]: ... async def get_job(self, job_id: str) -> Optional[JobDescription]: ... @@ -234,14 +234,7 @@ class Job: @property def id(self) -> Optional[str]: ... async def status(self) -> str: ... - async def wait(self) -> None: ... - async def cancel(self) -> None: ... - -class FunctionJob: - @property - def id(self) -> Optional[str]: ... - async def status(self) -> str: ... - async def wait(self) -> str: ... + async def wait(self) -> Optional[str]: ... async def cancel(self) -> None: ... class JobInfo: diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 7ad749920..51b8d9993 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -46,7 +46,7 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError from . import __version__ from ._lancedb import connect as lancedb_connect # type: ignore from .functions import FunctionVersion, UdfDefinition -from .job import AsyncJob, Job, _function_job +from .job import AsyncJob, Job, _typed_job from .materialized_view import ( AsyncMaterializedView, MaterializedView, @@ -2237,7 +2237,7 @@ class AsyncConnection(object): inner = await self._inner.create_function_async( definition.registration_request.to_canonical_json() ) - return _function_job(inner) + return _typed_job(inner, FunctionVersion.from_json) async def get_function(self, name: str, *, version: str) -> FunctionVersion: """Open one exact immutable Function version from the remote catalog.""" diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 9532ab8b9..c4ff9a9b3 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -1,10 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors -"""Canonical values exchanged with LanceDB Enterprise Function services. +"""Canonical Function values exchanged with LanceDB Enterprise services. These immutable models contain client/wire state only. Catalog persistence, environment bake, secret resolution, and execution are owned by Sophon. +``RefreshColumnResult`` is also the backend-neutral result of a local +expression-backed refresh job. """ from __future__ import annotations @@ -460,7 +462,11 @@ class FunctionBinding(_RemoteValue): class RefreshColumnResult(_RemoteValue): - """Terminal result of a remote Function-column refresh Job.""" + """Terminal result of an expression-backed or Function-backed refresh Job. + + Local jobs produce this value in process. LanceDB Cloud and Enterprise + decode the same value from the durable server-job terminal payload. + """ rows_assigned: _UInt64 rows_failed: _UInt64 diff --git a/python/python/lancedb/job.py b/python/python/lancedb/job.py index 7bd600a74..f688768cb 100644 --- a/python/python/lancedb/job.py +++ b/python/python/lancedb/job.py @@ -5,12 +5,11 @@ import asyncio from datetime import timedelta -from typing import Any, Generic, Optional, TypeVar, cast +from typing import Any, Callable, Generic, Optional, TypeVar, cast from lancedb.background_loop import LOOP from . import _lancedb -from .functions import FunctionVersion T = TypeVar("T") @@ -18,11 +17,18 @@ T = TypeVar("T") class AsyncJob(Generic[T]): """A handle to an operation that may still be running. - The operation may already be complete when the handle is created. + The operation may already be complete when the handle is created. ``T`` + is the endpoint's terminal result type; unit-result jobs resolve to + ``None``. """ - def __init__(self, inner: Optional[Any]): + def __init__( + self, + inner: Optional[Any], + result_decoder: Optional[Callable[[Any], T]] = None, + ): self._inner = inner + self._result_decoder = result_decoder @property def id(self) -> Optional[str]: @@ -50,17 +56,21 @@ class AsyncJob(Generic[T]): async def wait(self, timeout: Optional[timedelta] = None) -> T: """Wait until the operation reaches a terminal state. + Returns the endpoint's typed result, or ``None`` for a unit-result + job. + Raises `JobFailedError` if the operation failed, `JobCancelledError` if it was cancelled, and `TimeoutError` if `timeout` elapses first. """ if self._inner is None: return cast(T, None) if timeout is None: - return cast(T, await self._inner.wait()) - return cast( - T, - await asyncio.wait_for(self._inner.wait(), timeout.total_seconds()), - ) + result = await self._inner.wait() + else: + result = await asyncio.wait_for(self._inner.wait(), timeout.total_seconds()) + if self._result_decoder is not None: + return self._result_decoder(result) + return cast(T, result) async def cancel(self): """Request cancellation. Cancelling a finished operation is a no-op.""" @@ -70,7 +80,7 @@ class AsyncJob(Generic[T]): class Job(Generic[T]): - """Synchronous counterpart of `AsyncJob`.""" + """Synchronous counterpart of `AsyncJob` with the same result type.""" def __init__(self, inner: Optional[AsyncJob[T]]): self._inner = inner @@ -96,6 +106,9 @@ class Job(Generic[T]): def wait(self, timeout: Optional[timedelta] = None) -> T: """Block until the operation reaches a terminal state. + Returns the endpoint's typed result, or ``None`` for a unit-result + job. + Raises `JobFailedError` if the operation failed, `JobCancelledError` if it was cancelled, and `TimeoutError` if `timeout` elapses first. """ @@ -110,23 +123,8 @@ class Job(Generic[T]): LOOP.run(self._inner.cancel()) -class _FunctionJobAdapter: - def __init__(self, inner: "_lancedb.FunctionJob"): - self._inner = inner - - @property - def id(self) -> Optional[str]: - return self._inner.id - - async def status(self) -> str: - return await self._inner.status() - - async def wait(self) -> FunctionVersion: - return FunctionVersion.from_json(await self._inner.wait()) - - async def cancel(self): - await self._inner.cancel() - - -def _function_job(inner: "_lancedb.FunctionJob") -> AsyncJob[FunctionVersion]: - return AsyncJob(_FunctionJobAdapter(inner)) +def _typed_job( + inner: "_lancedb.Job", result_decoder: Callable[[str], T] +) -> AsyncJob[T]: + """Bind an internal JSON-producing job to its public result model.""" + return AsyncJob(inner, result_decoder) diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index b97f8f194..41394ed71 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -49,7 +49,7 @@ from lancedb.index import ( LabelList, ) from lancedb.job import Job -from lancedb.functions import FunctionApplication +from lancedb.functions import FunctionApplication, RefreshColumnResult from lancedb.remote.db import LOOP from lancedb.table import IndexConfigType, KNOWN_METRICS import pyarrow as pa @@ -972,7 +972,7 @@ class RemoteTable(Table): def refresh_column(self, column: str): return LOOP.run(self._table.refresh_column(column)) - def refresh_column_async(self, column: str) -> Job: + def refresh_column_async(self, column: str) -> Job[RefreshColumnResult]: return Job(LOOP.run(self._table.refresh_column_async(column))) def alter_columns( diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 0c84b4036..75b7b1db8 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -40,7 +40,7 @@ from ._blob import ( from .types import BlobMode from lancedb.arrow import peek_reader from lancedb.background_loop import LOOP, embedding_executor -from lancedb.job import AsyncJob, Job +from lancedb.job import AsyncJob, Job, _typed_job from .dependencies import ( _check_for_hugging_face, _check_for_lance, @@ -72,7 +72,10 @@ from .index import ( FTS, ) from .expr import Expr -from .functions import FunctionApplication +from .functions import ( + FunctionApplication, + RefreshColumnResult as RefreshColumnJobResult, +) from .merge import LanceMergeInsertBuilder from .pydantic import LanceModel, model_to_dict from .query import ( @@ -2039,7 +2042,7 @@ class Table(ABC): """ @abstractmethod - def refresh_column_async(self, column: str) -> Job: + def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]: """ Like :meth:`refresh_column`, but returns a handle to the refresh job instead of blocking until it completes. @@ -2050,6 +2053,12 @@ class Table(ABC): than failing the job. On local tables the job runs in-process; on LanceDB Cloud and Enterprise it is the server's backfill job. + Returns + ------- + Job[RefreshColumnResult] + A job whose successful ``wait`` returns row counts plus the source + and published table versions. + Examples -------- >>> import lancedb @@ -2058,7 +2067,9 @@ class Table(ABC): >>> table.add_columns(computed={"doubled": "x * 2"}) AddColumnsResult(version=2) >>> job = table.refresh_column_async("doubled") - >>> job.wait() + >>> result = job.wait() + >>> result.rows_assigned + 2 >>> job.status() 'finished' """ @@ -4082,7 +4093,7 @@ class LanceTable(Table): [`AsyncTable.refresh_column`][lancedb.AsyncTable.refresh_column].""" return LOOP.run(self._table.refresh_column(column)) - def refresh_column_async(self, column: str) -> Job: + def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]: """Fill a computed column's unfilled rows, returning a handle to the refresh job. See [`Table.refresh_column_async`][lancedb.table.Table.refresh_column_async]. @@ -6122,7 +6133,9 @@ class AsyncTable: """ return await self._inner.refresh_column(column) - async def refresh_column_async(self, column: str) -> AsyncJob: + async def refresh_column_async( + self, column: str + ) -> AsyncJob[RefreshColumnJobResult]: """ Like :meth:`refresh_column`, but returns a handle to the refresh job instead of blocking until it completes. @@ -6134,6 +6147,12 @@ class AsyncTable: in-process; on LanceDB Cloud and Enterprise it is the server's backfill job. + Returns + ------- + AsyncJob[RefreshColumnResult] + A job whose successful ``wait`` returns row counts plus the source + and published table versions. + Examples -------- >>> import asyncio @@ -6143,12 +6162,16 @@ class AsyncTable: ... table = await db.create_table("computed_job_async_demo", [{"x": 1}]) ... await table.add_columns(computed={"doubled": "x * 2"}) ... job = await table.refresh_column_async("doubled") - ... await job.wait() + ... result = await job.wait() + ... assert result.rows_assigned == 1 ... return await job.status() >>> asyncio.run(refresh_in_background()) 'finished' """ - return AsyncJob(await self._inner.refresh_column_async(column)) + return _typed_job( + await self._inner.refresh_column_async(column), + RefreshColumnJobResult.from_json, + ) async def alter_columns( self, *alterations: Iterable[dict[str, Any]] diff --git a/python/python/tests/test_db.py b/python/python/tests/test_db.py index 38bbb53fb..aeb9feeb8 100644 --- a/python/python/tests/test_db.py +++ b/python/python/tests/test_db.py @@ -774,7 +774,7 @@ def test_drop_table_async(tmp_db: lancedb.DBConnection): job = tmp_db.drop_table_async("test") assert job.id is None assert job.status() == "finished" - job.wait() + assert job.wait() is None assert tmp_db.table_names() == [] tmp_db.create_table("test", data=data) @@ -790,7 +790,7 @@ async def test_drop_table_async_connection(tmp_db_async: lancedb.AsyncConnection job = await tmp_db_async.drop_table_async("test") assert job.id is None assert await job.status() == "finished" - await job.wait() + assert await job.wait() is None assert await tmp_db_async.table_names() == [] diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 99c68876c..a20347771 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -187,7 +187,7 @@ def _mock_remote_function_catalog(): "job_state": "DONE", "result": state["version"], } - elif self.path == "/v1/functions/get": + elif self.path == "/v1/functions/describe": assert body == { "name": "normalize_score", "version": "fv_exact", diff --git a/python/python/tests/test_index.py b/python/python/tests/test_index.py index 94268a53e..fe6ebe87a 100644 --- a/python/python/tests/test_index.py +++ b/python/python/tests/test_index.py @@ -88,7 +88,7 @@ async def binary_table(db_async): async def test_create_index_async_returns_done_job(some_table: AsyncTable): job = await some_table.create_index_async("id", config=BTree()) assert job.id is None - await job.wait() + assert await job.wait() is None assert len(await some_table.list_indices()) == 1 await job.cancel() diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 08f5550eb..2952f00a4 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -875,11 +875,85 @@ def test_remote_create_index_async_returns_job(): table = db.create_table("test", [{"id": 1}]) job = table.create_index_async("id", config=BTree()) assert job.id == "job-1" - job.wait(timeout=timedelta(seconds=30)) + assert job.wait(timeout=timedelta(seconds=30)) is None assert len(describe_calls) == 2 job.cancel() +def test_remote_refresh_async_returns_typed_terminal_result(): + terminal_result = { + "rows_assigned": 12, + "rows_failed": 0, + "rows_remaining": 0, + "source_version": 7, + "published_version": 8, + } + + def handler(request): + content_len = int(request.headers.get("Content-Length", 0)) + body = request.rfile.read(content_len) if content_len > 0 else b"" + if request.path == "/v1/table/test/backfill_column": + assert json.loads(body)["column"] == "derived" + request.send_response(202) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b'{"job_id": "refresh-1"}') + elif request.path == "/v1/jobs/describe": + assert json.loads(body)["job_id"] == "refresh-1" + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write( + json.dumps( + { + "job_id": "refresh-1", + "job_type": "function_refresh", + "job_state": "DONE", + "result": terminal_result, + } + ).encode() + ) + elif request.path == "/v1/table/test/create/?mode=create": + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b"{}") + elif request.path == "/v1/table/test/describe/": + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write( + json.dumps( + { + "version": 1, + "schema": { + "fields": [ + { + "name": "id", + "type": {"type": "int64"}, + "nullable": False, + } + ] + }, + } + ).encode() + ) + else: + request.send_response(404) + request.end_headers() + + with mock_lancedb_connection(handler) as db: + table = db.create_table("test", [{"id": 1}]) + job = table.refresh_column_async("derived") + assert job.id == "refresh-1" + result = job.wait(timeout=timedelta(seconds=30)) + + assert isinstance(result, lancedb.RefreshColumnResult) + assert result.model_dump() == terminal_result + assert result.rows_filled == 12 + assert result.version == 8 + + def test_remote_job_wait_raises_on_failure(): from lancedb.exceptions import JobFailedError from lancedb.index import BTree diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 0f98219bf..56e0eacfd 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -1467,7 +1467,7 @@ def test_create_index_async_returns_done_job(mem_db: DBConnection): table = mem_db.create_table("job_test", [{"id": i} for i in range(10)]) job = table.create_index_async("id", config=BTree()) assert job.id is None - job.wait() + assert job.wait() is None assert len(table.list_indices()) == 1 job.cancel() @@ -3947,10 +3947,21 @@ def test_refresh_column_async_returns_job(tmp_path): job = table.refresh_column_async("doubled") assert job.id is None # in-process jobs have no server id - assert job.wait() is None + result = job.wait() + assert isinstance(result, lancedb.RefreshColumnResult) + assert result.rows_assigned == 2 + assert result.rows_failed == 0 + assert result.rows_remaining == 0 + assert result.source_version == 2 + assert result.published_version == 3 assert job.status() == "finished" assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4] + no_op = table.refresh_column_async("doubled").wait() + assert no_op.rows_assigned == 0 + assert no_op.source_version == 3 + assert no_op.published_version is None + # Bad input raises at the call, not through the job. with pytest.raises(Exception, match="not a computed column"): table.refresh_column_async("x") @@ -3963,6 +3974,10 @@ async def test_refresh_column_async_job_async_table(tmp_path): await table.add_columns(computed={"tripled": "x * 3"}) job = await table.refresh_column_async("tripled") - assert await job.wait() is None + result = await job.wait() + assert isinstance(result, lancedb.RefreshColumnResult) + assert result.rows_assigned == 1 + assert result.source_version == 2 + assert result.published_version == 3 assert await job.status() == "finished" assert (await table.to_arrow())["tripled"].to_pylist() == [9] diff --git a/python/src/connection.rs b/python/src/connection.rs index 4143ac9c9..902489f4f 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -609,7 +609,7 @@ impl Connection { .create_function_async(request) .await .infer_error() - .map(crate::job::FunctionJob::new) + .map(crate::job::Job::new_typed) }) } diff --git a/python/src/job.rs b/python/src/job.rs index a08b958a6..688cba7f9 100644 --- a/python/src/job.rs +++ b/python/src/job.rs @@ -5,72 +5,33 @@ use std::sync::Arc; use crate::runtime::future_into_py; use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods}; +use serde::Serialize; use crate::error::PythonErrorExt; #[pyclass] pub struct Job { - inner: Arc, -} - -/// Python bridge for a typed remote Function registration job. -/// -/// The public Python layer decodes the canonical JSON returned by `wait` -/// into its immutable `FunctionVersion` model. -#[pyclass] -pub struct FunctionJob { - inner: Arc>, -} - -impl FunctionJob { - pub(crate) fn new(inner: lancedb::Job) -> Self { - Self { - inner: Arc::new(inner), - } - } + inner: Arc, String>>>, } impl Job { pub(crate) fn new(inner: lancedb::Job) -> Self { Self { - inner: Arc::new(inner), + inner: Arc::new(inner.map(|()| Ok(None))), } } -} -#[pymethods] -impl FunctionJob { - #[getter] - pub fn id(&self) -> Option { - self.inner.id().map(str::to_string) - } - - pub fn status(self_: PyRef<'_, Self>) -> PyResult> { - let inner = self_.inner.clone(); - future_into_py( - self_.py(), - async move { inner.status().await.infer_error() }, - ) - } - - pub fn wait(self_: PyRef<'_, Self>) -> PyResult> { - let inner = self_.inner.clone(); - future_into_py(self_.py(), async move { - inner - .wait() - .await - .infer_error()? - .to_canonical_json() - .infer_error() - }) - } - - pub fn cancel(self_: PyRef<'_, Self>) -> PyResult> { - let inner = self_.inner.clone(); - future_into_py(self_.py(), async move { - inner.cancel().await.infer_error()?; - Ok(()) - }) + pub(crate) fn new_typed(inner: lancedb::Job) -> Self + where + T: Clone + Serialize + Send + Sync + 'static, + { + Self { + inner: Arc::new(inner.map(|result| { + serde_json::to_string(&result) + .map(Some) + .map_err(|error| format!("failed to serialize typed job result: {error}")) + })), + } } } @@ -92,8 +53,10 @@ impl Job { pub fn wait(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner.clone(); future_into_py(self_.py(), async move { - inner.wait().await.infer_error()?; - Ok(None::<()>) + let result = inner.wait().await.infer_error()?; + result + .map_err(|message| lancedb::Error::Runtime { message }) + .infer_error() }) } diff --git a/python/src/lib.rs b/python/src/lib.rs index c1dfbc02b..8d3eab787 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -47,7 +47,6 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::
()?; m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/src/table.rs b/python/src/table.rs index 5cdcc3653..b225b191f 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -1619,7 +1619,7 @@ impl Table { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { let job = inner.refresh_column_async(column).await.infer_error()?; - Ok(crate::job::Job::new(job)) + Ok(crate::job::Job::new_typed(job)) }) } diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index 835fca9a2..f02158871 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Canonical values exchanged with the Enterprise Function service. +//! Canonical Function values exchanged with the Enterprise service, plus the +//! backend-neutral terminal result of a computed-column refresh. //! //! This module contains client/wire values only. Catalog persistence, //! environment bake, secret resolution, and execution are owned by Sophon. @@ -580,13 +581,22 @@ impl FunctionBinding { impl_json!(FunctionBinding); -/// Stable terminal result of a remote Function-column refresh Job. +/// Stable terminal result of an expression-backed or Function-backed column +/// refresh [`crate::Job`]. +/// +/// Local refresh jobs produce this value in process. LanceDB Cloud and +/// Enterprise decode the same value from the durable job's terminal payload. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct RefreshColumnResult { + /// Rows assigned a value by this refresh. pub rows_assigned: u64, + /// Rows whose computation failed. pub rows_failed: u64, + /// Rows that still need a value when the job completes. pub rows_remaining: u64, + /// Exact table version the refresh read. pub source_version: u64, + /// Table version made visible by the refresh, when one was published. #[serde(default, skip_serializing_if = "Option::is_none")] pub published_version: Option, } diff --git a/rust/lancedb/src/job.rs b/rust/lancedb/src/job.rs index 1a76c7683..94d1ba2b6 100644 --- a/rust/lancedb/src/job.rs +++ b/rust/lancedb/src/job.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use async_trait::async_trait; -use serde::de::DeserializeOwned; +use serde::{Serialize, de::DeserializeOwned}; use serde_json::Value; use tokio::sync::watch; use tokio::task::{AbortHandle, JoinHandle}; @@ -26,20 +26,16 @@ pub(crate) trait JobHandle: Send + Sync { } /// A backend-neutral successful terminal result. -/// -/// Local operations do not carry a value. Remote operations may carry JSON -/// that the public [`Job`] decodes according to its result type. +#[derive(Clone)] pub(crate) struct TerminalResult { - #[allow(dead_code)] // Typed remote submit endpoints consume this after Slice 1. value: Option, - #[allow(dead_code)] // Preserved so typed decode errors retain request correlation. request_id: Option, } impl TerminalResult { - pub(crate) fn local() -> Self { + fn local(value: Value) -> Self { Self { - value: None, + value: Some(value), request_id: None, } } @@ -51,23 +47,31 @@ impl TerminalResult { } } - #[allow(dead_code)] // Exercised by the remote typed-result fixtures in Slice 1. fn decode(self) -> Result { - let request_id = self.request_id.unwrap_or_default(); - let value = self.value.ok_or_else(|| Error::Http { - source: "successful typed job response did not contain a result".into(), - request_id: request_id.clone(), - status_code: None, + let value = self.value.ok_or_else(|| match &self.request_id { + Some(request_id) => Error::Http { + source: "successful typed job response did not contain a result".into(), + request_id: request_id.clone(), + status_code: None, + }, + None => Error::Runtime { + message: "successful typed job did not contain a result".to_string(), + }, })?; - serde_json::from_value(value).map_err(|error| Error::Http { - source: format!("failed to parse typed job result: {error}").into(), - request_id, - status_code: None, + serde_json::from_value(value).map_err(|error| match self.request_id { + Some(request_id) => Error::Http { + source: format!("failed to parse typed job result: {error}").into(), + request_id, + status_code: None, + }, + None => Error::Runtime { + message: format!("failed to parse typed job result: {error}"), + }, }) } } -type ResultDecoder = fn(TerminalResult) -> Result; +type ResultDecoder = Arc Result + Send + Sync>; enum JobInner { Handle { @@ -79,7 +83,9 @@ enum JobInner { /// A handle to an operation that may still be running. /// -/// The operation may already be complete when the handle is created. +/// The operation may already be complete when the handle is created. `T` is +/// the endpoint's successful terminal result; unit-result operations use the +/// default `Job<()>`. pub struct Job where T: Clone + Send + Sync + 'static, @@ -111,15 +117,10 @@ impl Job<()> { Self { inner: JobInner::Handle { handle, - decode: |_| Ok(()), + decode: Arc::new(|_| Ok(())), }, } } - - /// A unit-result job running as a task in this process. - pub(crate) fn spawned(task: JoinHandle>) -> Self { - Self::new(Box::new(SpawnedJob::new(task))) - } } impl Job @@ -131,12 +132,22 @@ where Self { inner: JobInner::Handle { handle, - decode: TerminalResult::decode::, + decode: Arc::new(TerminalResult::decode::), }, } } } +impl Job +where + T: Clone + Serialize + DeserializeOwned + Send + Sync + 'static, +{ + /// A typed job running as a task in this process. + pub(crate) fn spawned(task: JoinHandle>) -> Self { + Self::new_typed(Box::new(SpawnedJob::new(task))) + } +} + impl Job where T: Clone + Send + Sync + 'static, @@ -169,11 +180,13 @@ where /// Waits until the operation reaches a terminal state. /// + /// Returns the endpoint's typed result. Unit-result jobs return `()`. + /// /// Returns [`crate::Error::JobFailed`] if the operation failed and /// [`crate::Error::JobCancelled`] if it was cancelled. pub async fn wait(&self) -> Result { match &self.inner { - JobInner::Handle { handle, decode } => decode(handle.wait().await?), + JobInner::Handle { handle, decode } => (decode)(handle.wait().await?), JobInner::Completed(result) => Ok(result.clone()), } } @@ -187,21 +200,53 @@ where JobInner::Completed(_) => Ok(()), } } + + /// Maps a successful terminal result without changing the job lifecycle. + /// The mapping may run once for each call to [`Job::wait`], so it should + /// be deterministic and free of externally visible side effects. + /// + /// ``` + /// use lancedb::{Job, function::RefreshColumnResult}; + /// + /// # async fn rows_assigned( + /// # job: Job, + /// # ) -> lancedb::Result { + /// let job = job.map(|result| result.rows_assigned); + /// job.wait().await + /// # } + /// ``` + pub fn map(self, map: F) -> Job + where + U: Clone + Send + Sync + 'static, + F: Fn(T) -> U + Send + Sync + 'static, + { + match self.inner { + JobInner::Handle { handle, decode } => Job { + inner: JobInner::Handle { + handle, + decode: Arc::new(move |result| Ok(map((decode)(result)?))), + }, + }, + JobInner::Completed(result) => Job { + inner: JobInner::Completed(map(result)), + }, + } + } } /// How an in-process operation ended. Cloneable so every waiter can be given /// the outcome; [`Error`] is not, so failures share one behind an [`Arc`]. #[derive(Clone)] enum Outcome { - Succeeded, + Succeeded(TerminalResult), Failed(Arc), Cancelled, } impl Outcome { - fn into_result(self) -> Result<()> { + fn into_result(self) -> Result { match self { - Self::Succeeded => Ok(()), + Self::Succeeded(result) => Ok(result), Self::Failed(source) => Err(Error::JobFailed { job_id: None, failure: JobFailure::from_source(source), @@ -220,12 +265,20 @@ struct SpawnedJob { } impl SpawnedJob { - fn new(task: JoinHandle>) -> Self { + fn new(task: JoinHandle>) -> Self + where + T: Serialize + Send + 'static, + { let abort = task.abort_handle(); let (tx, outcome) = watch::channel(None); tokio::spawn(async move { let outcome = match task.await { - Ok(Ok(())) => Outcome::Succeeded, + Ok(Ok(result)) => match serde_json::to_value(result) { + Ok(value) => Outcome::Succeeded(TerminalResult::local(value)), + Err(err) => Outcome::Failed(Arc::new(Error::Runtime { + message: format!("failed to serialize job result: {err}"), + })), + }, Ok(Err(err)) => Outcome::Failed(Arc::new(err)), Err(err) if err.is_cancelled() => Outcome::Cancelled, Err(err) => Outcome::Failed(Arc::new(Error::Runtime { @@ -243,7 +296,7 @@ impl JobHandle for SpawnedJob { async fn status(&self) -> Result { let label = match &*self.outcome.borrow() { None => "running", - Some(Outcome::Succeeded) => "finished", + Some(Outcome::Succeeded(_)) => "finished", Some(Outcome::Failed(_)) => "failed", Some(Outcome::Cancelled) => "cancelled", }; @@ -256,12 +309,11 @@ impl JobHandle for SpawnedJob { .wait_for(|outcome| outcome.is_some()) .await .map_err(|_| Error::Runtime { - message: "index job outcome was dropped before it completed".to_string(), + message: "job outcome was dropped before it completed".to_string(), })? .clone() .expect("wait_for returns once an outcome is set"); - settled.into_result()?; - Ok(TerminalResult::local()) + settled.into_result() } async fn cancel(&self) -> Result<()> { @@ -269,3 +321,29 @@ impl JobHandle for SpawnedJob { Ok(()) } } + +#[cfg(test)] +mod tests { + use std::future::pending; + + use super::*; + + #[tokio::test] + async fn mapped_spawned_job_reuses_outcome() { + let job = Job::spawned(tokio::spawn(async { Ok(41_u64) })).map(|value| value + 1); + + assert_eq!(job.wait().await.unwrap(), 42); + assert_eq!(job.wait().await.unwrap(), 42); + assert_eq!(job.status().await.unwrap(), "finished"); + } + + #[tokio::test] + async fn mapped_spawned_job_preserves_cancellation() { + let job = Job::spawned(tokio::spawn(async { pending::>().await })) + .map(|value| value.to_string()); + + job.cancel().await.unwrap(); + assert!(matches!(job.wait().await, Err(Error::JobCancelled { .. }))); + assert_eq!(job.status().await.unwrap(), "cancelled"); + } +} diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 169d0fda5..8265d22c0 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -513,7 +513,7 @@ impl Database for RemoteDatabase { async fn get_function(&self, name: &str, version: &str) -> Result { let req = self .client - .post("/v1/functions/get") + .post("/v1/functions/describe") .json(&serde_json::json!({ "name": name, "version": version, @@ -2520,7 +2520,7 @@ mod tests { ); let conn = Connection::new_with_handler(|request| { assert_eq!(request.method(), &reqwest::Method::POST); - assert_eq!(request.url().path(), "/v1/functions/get"); + assert_eq!(request.url().path(), "/v1/functions/describe"); let body: serde_json::Value = serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); assert_eq!( diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index fb7278db9..c6b078282 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2864,7 +2864,10 @@ impl BaseTable for RemoteTable { }) } - async fn refresh_column_async(&self, column: &str) -> Result { + async fn refresh_column_async( + &self, + column: &str, + ) -> Result> { self.check_mutable().await?; let mut body = serde_json::json!({ "column": column }); self.apply_branch_body(&mut body); @@ -2885,7 +2888,7 @@ impl BaseTable for RemoteTable { status_code: None, })?; - Ok(Job::new(Box::new(FreshnessJob { + Ok(Job::new_typed(Box::new(FreshnessJob { inner: RemoteJob::new(self.client.clone(), response.job_id), freshness: self.freshness.clone(), version: self.version.clone(), @@ -3280,6 +3283,21 @@ mod tests { }, }; + fn refresh_done(job_id: &str) -> String { + json!({ + "job_id": job_id, + "job_state": "DONE", + "result": { + "rows_assigned": 12, + "rows_failed": 0, + "rows_remaining": 0, + "source_version": 7, + "published_version": 8, + } + }) + .to_string() + } + #[tokio::test] async fn test_not_found() { let table = Table::new_with_handler("my_table", |_| { @@ -6892,7 +6910,7 @@ mod tests { .unwrap(), "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-7", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-7")) .unwrap(), "/v1/table/my_table/count_rows/" => { saw.store( @@ -6908,7 +6926,9 @@ mod tests { }); let job = table.refresh_column_async("doubled").await.unwrap(); - job.wait().await.unwrap(); + let result = job.wait().await.unwrap(); + assert_eq!(result.rows_assigned, 12); + assert_eq!(result.published_version, Some(8)); table.count_rows(None).await.unwrap(); assert!( saw_min_timestamp.load(std::sync::atomic::Ordering::SeqCst), @@ -6930,7 +6950,7 @@ mod tests { .unwrap(), "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-8", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-8")) .unwrap(), "/v1/table/my_table/describe/" => { let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]); @@ -6976,7 +6996,7 @@ mod tests { .unwrap(), "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-9", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-9")) .unwrap(), "/v1/table/my_table/tags/version/" => http::Response::builder() .status(200) @@ -7040,7 +7060,7 @@ mod tests { } "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-10", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-10")) .unwrap(), "/v1/table/my_table/describe/" => { let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]); @@ -7113,7 +7133,7 @@ mod tests { } "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-11", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-11")) .unwrap(), "/v1/table/my_table/count_rows/" => { *saw.lock().unwrap() = request diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index c2c12fff5..41551bd37 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -771,7 +771,10 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { } /// Fill a computed column's unfilled rows, returning a [`Job`] tracking /// the operation. - async fn refresh_column_async(&self, _column: &str) -> Result { + async fn refresh_column_async( + &self, + _column: &str, + ) -> Result> { Err(Error::NotSupported { message: "computed columns are supported only on local tables".into(), }) @@ -1708,7 +1711,9 @@ impl Table { /// operation instead of blocking until it completes. /// /// The job may already be complete when returned, and callers must not - /// assume the column is filled until [`Job::wait`] returns. Invalid input + /// assume the column is filled until [`Job::wait`] returns. A successful + /// wait returns the durable [`crate::function::RefreshColumnResult`] for + /// both expression-backed and Function-backed columns. Invalid input /// -- an unknown column, or one that is not computed -- is reported by /// this call rather than by the job. On local tables the job runs as an /// in-process task; on LanceDB Cloud and Enterprise it is the server's @@ -1719,11 +1724,15 @@ impl Table { /// # async fn refresh_in_background(table: &Table) -> Result<(), Box> { /// let job = table.refresh_column_async("doubled").await?; /// println!("refresh running: {:?}", job.status().await?); - /// job.wait().await?; + /// let result = job.wait().await?; + /// println!("assigned {} rows", result.rows_assigned); /// # Ok(()) /// # } /// ``` - pub async fn refresh_column_async(&self, column: impl AsRef) -> Result { + pub async fn refresh_column_async( + &self, + column: impl AsRef, + ) -> Result> { self.inner.refresh_column_async(column.as_ref()).await } @@ -3425,7 +3434,10 @@ impl BaseTable for NativeTable { Ok(result) } - async fn refresh_column_async(&self, column: &str) -> Result { + async fn refresh_column_async( + &self, + column: &str, + ) -> Result> { refresh::execute_refresh_column_async(self, column).await } diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 7d74bbae7..35f883411 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -49,11 +49,25 @@ pub struct RefreshColumnResult { pub version: u64, } +struct RefreshExecution { + result: RefreshColumnResult, + source_version: u64, +} + /// Internal implementation of the refresh logic. pub(crate) async fn execute_refresh_column( table: &NativeTable, column: &str, ) -> Result { + Ok(execute_refresh_column_with_source(table, column) + .await? + .result) +} + +async fn execute_refresh_column_with_source( + table: &NativeTable, + column: &str, +) -> Result { table.dataset.ensure_mutable()?; ensure_no_lsm_write_spec(table).await?; let dataset = table.dataset.get().await?; @@ -87,9 +101,13 @@ pub(crate) async fn execute_refresh_column( } if replacements.is_empty() { - return Ok(RefreshColumnResult { - rows_filled: 0, - version: dataset.version().version, + let source_version = dataset.version().version; + return Ok(RefreshExecution { + result: RefreshColumnResult { + rows_filled: 0, + version: source_version, + }, + source_version, }); } @@ -110,14 +128,20 @@ pub(crate) async fn execute_refresh_column( let version = new_dataset.version().version; table.dataset.update(new_dataset); - Ok(RefreshColumnResult { - rows_filled, - version, + Ok(RefreshExecution { + result: RefreshColumnResult { + rows_filled, + version, + }, + source_version: read_version, }) } /// Run the refresh as a [`Job`] in this process. -pub(crate) async fn execute_refresh_column_async(table: &NativeTable, column: &str) -> Result { +pub(crate) async fn execute_refresh_column_async( + table: &NativeTable, + column: &str, +) -> Result> { // Validate before spawning so bad input is reported by this call rather // than only by the job. table.dataset.ensure_mutable()?; @@ -129,9 +153,16 @@ pub(crate) async fn execute_refresh_column_async(table: &NativeTable, column: &s let table = table.clone(); let column = column.to_string(); Ok(Job::spawned(tokio::spawn(async move { - execute_refresh_column(&table, &column).await?; + let execution = execute_refresh_column_with_source(&table, &column).await?; table.bump_freshness(); - Ok(()) + Ok(crate::function::RefreshColumnResult { + rows_assigned: execution.result.rows_filled, + rows_failed: 0, + rows_remaining: 0, + source_version: execution.source_version, + published_version: (execution.result.rows_filled > 0) + .then_some(execution.result.version), + }) }))) } @@ -396,6 +427,17 @@ mod tests { read(&table, "doubled").await, vec![Some(2), Some(4), Some(6)] ); + + let no_op = table + .refresh_column_async("doubled") + .await + .unwrap() + .wait() + .await + .unwrap(); + assert_eq!(no_op.rows_assigned, 0); + assert_eq!(no_op.source_version, 3); + assert_eq!(no_op.published_version, None); } /// Values written after the last refresh must be reachable by another one. @@ -646,7 +688,12 @@ mod tests { let job = table.refresh_column_async("doubled").await.unwrap(); assert!(job.id().is_none(), "in-process jobs have no server id"); - job.wait().await.unwrap(); + let result = job.wait().await.unwrap(); + assert_eq!(result.rows_assigned, 3); + assert_eq!(result.rows_failed, 0); + assert_eq!(result.rows_remaining, 0); + assert_eq!(result.source_version, 2); + assert_eq!(result.published_version, Some(3)); assert_eq!(job.status().await.unwrap(), "finished"); assert_eq!( read(&table, "doubled").await, @@ -672,9 +719,9 @@ mod tests { declare_doubled(&table).await.unwrap(); let job = table.refresh_column_async("doubled").await.unwrap(); - job.wait().await.unwrap(); + let first = job.wait().await.unwrap(); // A second wait after completion observes the same outcome. - job.wait().await.unwrap(); + assert_eq!(job.wait().await.unwrap(), first); assert_eq!(job.status().await.unwrap(), "finished"); } From 94d484f539048ada0b92deb289251d6e5ed7211e Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 24 Aug 2026 12:00:48 -0700 Subject: [PATCH 35/58] fix(listing): don't drop a table at a page boundary (#4040) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Listing tables a page at a time against a local database silently skipped one table at every page boundary. `ListingDatabase::list_tables` returned the first name of the *next* page as that page's token, but resuming from a token drops every name at or before it — so the table the token named was never handed to the caller. Walking `[a, b, c, d, e]` with a limit of 2 returned `[a, b, d, e]`. This PR returns the last name of the page as the token instead, which is what resuming after the token expects. This is reachable from Python today through `db.list_tables(page_token=...)` on a local connection; it also affects `len(db)` and `name in db`, which walk the pages. Remote and namespace-backed connections page on the server and were never affected. ## Example ```python db = lancedb.connect(tmp_path) for name in ["a", "b", "c", "d", "e"]: db.create_table(name, [{"id": 1}]) names, token = [], None while True: page = db.list_tables(page_token=token, limit=2) names += page.tables token = page.page_token if not token: break # before: ['a', 'b', 'd', 'e'] # after: ['a', 'b', 'c', 'd', 'e'] ``` Co-authored-by: Claude Opus 5 (1M context) --- rust/lancedb/src/connection.rs | 44 ++++++++++++++++++++++++++++ rust/lancedb/src/database/listing.rs | 16 +++++----- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 187e2df0e..5f66d9dee 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -1679,6 +1679,50 @@ mod tests { assert_eq!(tables, names[..7]); } + #[tokio::test] + async fn test_list_tables_walks_page_boundaries() { + let tc = new_test_connection().await.unwrap(); + if tc.is_remote { + // What resumes a page is the server's to decide, and asserting it here would be + // asserting the server's contract rather than this one. + return; + } + let db = tc.connection; + let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)])); + let mut names = Vec::with_capacity(5); + for _ in 0..5 { + let name = uuid::Uuid::new_v4().to_string(); + names.push(name.clone()); + db.create_empty_table(name, schema.clone()) + .execute() + .await + .unwrap(); + } + names.sort(); + + // Walking in pages has to reach every table exactly once, with nothing lost at a + // page boundary. + let mut seen = Vec::with_capacity(names.len()); + let mut page_token = None; + loop { + let page = db + .list_tables(ListTablesRequest { + id: Some(Vec::new()), + limit: Some(2), + page_token, + ..Default::default() + }) + .await + .unwrap(); + seen.extend(page.tables); + page_token = page.page_token.filter(|token| !token.is_empty()); + if page_token.is_none() { + break; + } + } + assert_eq!(seen, names); + } + #[tokio::test] async fn test_open_table() { let tc = new_test_connection().await.unwrap(); diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index c9e9bcb22..17dc82756 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -974,17 +974,15 @@ impl Database for ListingDatabase { f.drain(0..index); } - // Determine if there's a next page - let next_page_token = if let Some(limit) = request.limit { - if f.len() > limit as usize { - let token = f[limit as usize].clone(); + // Determine if there's a next page. The token is the last name of this page, + // not the first of the next one: the next page resumes strictly after the + // token, so naming the next page's first entry would skip it. + let next_page_token = match request.limit { + Some(limit) if f.len() > limit as usize => { f.truncate(limit as usize); - Some(token) - } else { - None + f.last().cloned() } - } else { - None + _ => None, }; Ok(ListTablesResponse { From 105fd73bc6da822e40f2c7206c8612bd009023bd Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 04:22:22 +0800 Subject: [PATCH 36/58] fix(python): commit streaming worker checkpoints on consumption (#4023) ## Summary - add `StreamingDataLoader`, which transports worker snapshots with prefetched batches and commits them to the parent dataset only when the trainer receives each batch - preserve exact non-uniform per-split progress and resume lagging splits without replaying already-consumed rows - reject stale parent checkpoints after a standard multi-process `DataLoader` has started, with guidance to use the consumer-aware loader - document the new public loader and merge non-uniform state across ranks ## Root cause PyTorch runs `StreamingDataset.__iter__` in private worker-process copies, while callers invoke `state_dict()` on the parent dataset. Sharing producer counters would still be incorrect because DataLoader prefetch can advance workers beyond batches returned to the trainer. ## Validation - `uv run --extra tests pytest python/tests/test_elastic_dataloader.py -q` (154 passed) - focused non-uniform merge regression (1 passed) - `uv run --project python --extra tests --extra dev ruff format .` - `uv run --project python --extra tests --extra dev ruff check .` - `cd docs && PYTHONPATH=. ../python/.venv/bin/mkdocs build` Fixes #3967 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo --- docs/src/python/python.md | 2 + python/python/lancedb/streaming.py | 594 +++++++++++++++- .../python/tests/test_elastic_dataloader.py | 658 ++++++++++++++++++ 3 files changed, 1221 insertions(+), 33 deletions(-) diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 8a24ea199..3cbeee6f0 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -261,6 +261,8 @@ instead of being materialized with the rest of the row. ::: lancedb.streaming.StreamingDataset +::: lancedb.streaming.StreamingDataLoader + ::: lancedb.permutation.permutation_builder ::: lancedb.permutation.PermutationBuilder diff --git a/python/python/lancedb/streaming.py b/python/python/lancedb/streaming.py index c54940ab7..2c0e2d5c1 100644 --- a/python/python/lancedb/streaming.py +++ b/python/python/lancedb/streaming.py @@ -19,6 +19,7 @@ above. """ import ctypes +import heapq import logging import os import random @@ -29,12 +30,12 @@ from collections import deque from concurrent.futures import ThreadPoolExecutor from copy import deepcopy from multiprocessing import RawArray -from typing import Any, Callable, cast, Iterator, Literal, Optional, Union +from typing import Any, Callable, cast, Iterator, Literal, NamedTuple, Optional, Union import pyarrow as pa import pyarrow.compute as pc import torch -from torch.utils.data import IterableDataset, get_worker_info +from torch.utils.data import DataLoader, IterableDataset, get_worker_info from .permutation import ( Permutation, @@ -55,6 +56,155 @@ DEFAULT_READ_BATCH_SIZE = 64 DEFAULT_PREFETCH_BATCHES = 4 +class _WorkerSample(NamedTuple): + data: Any + dataset: "StreamingDataset" + + +class _WorkerBatch(NamedTuple): + data: Any + state: dict + + +class _ConsumerIteratorLease(NamedTuple): + owner_token: int + owner_thread: int + + +class _CheckpointCollate: + """Attach the worker's post-fetch state to a collated batch.""" + + def __init__(self, collate_fn: Callable): + self._collate_fn = collate_fn + + def __call__(self, samples): + try: + if isinstance(samples, list): + if not samples: + return _WorkerBatch(self._collate_fn(samples), {}) + worker_samples = samples + data = self._collate_fn([sample.data for sample in worker_samples]) + dataset = worker_samples[-1].dataset + else: + data = self._collate_fn(samples.data) + dataset = samples.dataset + except StopIteration as exc: + raise RuntimeError( + "collate_fn raised StopIteration before returning a batch" + ) from exc + return _WorkerBatch(data, dataset._checkpoint_snapshot()) + + +class _StreamingDatasetAdapter(IterableDataset): + """Yield private sample wrappers for :class:`StreamingDataLoader`.""" + + def __init__(self, dataset: "StreamingDataset"): + super().__init__() + self.dataset = dataset + + def __iter__(self): + for sample in self.dataset._iter(consumer_checkpoint_transport=True): + yield _WorkerSample(sample, self.dataset) + + def __getattr__(self, name): + dataset = self.__dict__.get("dataset") + if dataset is None: + raise AttributeError(name) + return getattr(dataset, name) + + +class _ConsumerCommitIterator: + def __init__( + self, + iterator, + dataset: "StreamingDataset", + *, + owner_token: int, + require_uniform: bool, + ): + self._iterator = iterator + self._dataset = dataset + self._owner_token = owner_token + self._require_uniform = require_uniform + self._released = False + self._terminal = False + + def __iter__(self): + return self + + def __next__(self): + if self._terminal: + raise StopIteration + try: + batch = next(self._iterator) + except StopIteration: + self._terminal = True + self._release() + raise + except BaseException as exc: + self._dataset._invalidate_checkpoint( + f"a DataLoader batch failed before it was returned: {exc}" + ) + raise + try: + if not isinstance(batch, _WorkerBatch): + raise RuntimeError( + "StreamingDataLoader did not receive worker checkpoint metadata" + ) + self._dataset._commit_worker_state( + batch.state, require_uniform=self._require_uniform + ) + return batch.data + except BaseException as exc: + self._dataset._invalidate_checkpoint( + f"a DataLoader batch failed before it was returned: {exc}" + ) + raise + + def _release(self) -> None: + if self.__dict__.get("_released", True): + return + self._released = True + dataset = self.__dict__.get("_dataset") + if dataset is not None: + dataset._release_consumer_iterator(self._owner_token) + + def _shutdown_workers(self): + if self.__dict__.get("_released", True): + return None + self._terminal = True + iterator = self.__dict__.get("_iterator") + shutdown = getattr(iterator, "_shutdown_workers", None) + try: + if shutdown is not None: + shutdown() + else: + fetcher = getattr(iterator, "_dataset_fetcher", None) + dataset_iterator = getattr(fetcher, "dataset_iter", None) + close = getattr(dataset_iterator, "close", None) + if close is None: + raise RuntimeError( + "StreamingDataLoader could not close its inner iterator" + ) + close() + except BaseException as exc: + self._dataset._invalidate_checkpoint( + f"a DataLoader iterator could not be shut down safely: {exc}" + ) + raise + else: + self._release() + + def __del__(self): + try: + self._shutdown_workers() + except BaseException: + pass + + def __getattr__(self, name): + return getattr(self._iterator, name) + + class StreamingDataset(IterableDataset): """An elastic, resumable PyTorch IterableDataset backed by a LanceDB table. @@ -384,6 +534,22 @@ class StreamingDataset(IterableDataset): # rows_skipped] self._worker_stats: RawArray = RawArray(ctypes.c_int64, 8) + # A standard multi-process DataLoader cannot report which prefetched + # batches were actually returned to its consumer. Workers set this + # shared flag so state_dict() can reject a stale parent checkpoint + # unless StreamingDataLoader installed the consumer-commit transport. + self._untracked_worker_iteration: RawArray = RawArray(ctypes.c_int64, 1) + + # Parent-side checkpoint lifecycle. A failed DataLoader task creates + # a permanent hole in that iterator's delivery stream, while a + # multi-worker checkpoint is safe to restore only after all splits + # reach the same logical step boundary. + self._checkpoint_invalid_reason: Optional[str] = None + self._consumer_checkpoint_requires_uniform = False + self._consumer_iterator_lock = threading.Lock() + self._consumer_iterator_generation = 0 + self._consumer_iterator_lease: Optional[_ConsumerIteratorLease] = None + # Cumulative bytes of Arrow buffer data fetched across all iterations. self._bytes_loaded: int = 0 # Cumulative seconds spent in LanceDB I/O and in transform functions. @@ -396,6 +562,10 @@ class StreamingDataset(IterableDataset): # step boundaries all splits have consumed this many samples, so a # single scalar captures the topology-independent checkpoint state. self._resume_offset: int = 0 + # Exact yielded-sample counts for splits this process has advanced. + # Missing entries use _resume_offset, which remains the lower-bound + # checkpoint inherited from an earlier uniform/global state. + self._resume_samples: dict[int, int] = {} # Permutation position each split has consumed through, keyed by # global split index. Equal to _resume_offset for every split unless # on_transform_error skipped rows, in which case skipped positions @@ -521,11 +691,45 @@ class StreamingDataset(IterableDataset): return self._rank_splits[start : start + splits_per_worker] def __iter__(self) -> Iterator[dict[str, Any]]: + return self._iter() + + def _iter( + self, *, consumer_checkpoint_transport: bool = False + ) -> Iterator[dict[str, Any]]: + owner_token = None + previous_lease = self._consumer_iterator_lease + if consumer_checkpoint_transport: + if not self._consumer_iterator_active: + raise RuntimeError( + "StreamingDataLoader worker transport requires an active " + "parent iterator reservation" + ) + else: + try: + owner_token = self._acquire_consumer_iterator() + except BaseException: + self._release_consumer_iterator_after_failed_acquire(previous_lease) + raise + try: + yield from self._iter_owned( + consumer_checkpoint_transport=consumer_checkpoint_transport + ) + finally: + if owner_token is not None: + self._release_consumer_iterator(owner_token) + + def _iter_owned( + self, *, consumer_checkpoint_transport: bool + ) -> Iterator[dict[str, Any]]: if self._raw_batches_ref is not None: raise RuntimeError( "StreamingDataset does not support concurrent iteration. " "Only one active iterator per dataset instance is allowed." ) + real_worker = get_worker_info() is not None + if real_worker and not consumer_checkpoint_transport: + self._untracked_worker_iteration[0] = 1 + my_splits = self._resolve_my_splits() if not my_splits: return @@ -533,6 +737,7 @@ class StreamingDataset(IterableDataset): # Set identity transform on each Permutation so __getitems__ returns # the raw RecordBatch. Stage 2 applies the real transform. permutations: list[Permutation] = [] + initial_samples: list[int] = [] initial_positions: list[int] = [] for split_idx in my_splits: perm = Permutation.from_tables( @@ -541,21 +746,22 @@ class StreamingDataset(IterableDataset): if self._columns is not None: perm = perm.select_columns(self._columns) perm = perm.with_transform(Transforms.arrow2arrow) + sample_count = self._resume_samples.get(split_idx, self._resume_offset) # Both modes resume from absolute permutation positions. Packing # stores them separately because it also checkpoints partial blocks. start_pos = ( self._pack_consumed[split_idx] if self._pack_sequences is not None - else self._resume_positions.get(split_idx, self._resume_offset) + else self._resume_positions.get(split_idx, sample_count) ) if start_pos > 0: perm = perm.with_skip(start_pos) + initial_samples.append(sample_count) initial_positions.append(start_pos) permutations.append(perm) n = len(permutations) split_sizes = [perm.num_rows for perm in permutations] - initial_offset = self._resume_offset local_consumed = [0] * n # Permutation position each split has consumed through (absolute, # i.e. counted from the start of the unskipped split). Runs ahead of @@ -853,6 +1059,27 @@ class StreamingDataset(IterableDataset): for i in range(n): _fill_io(i) + def _yield_row(i: int): + pos, row = cooked[i].popleft() + # Surface any completed prefetched failure before the + # current row becomes durable checkpoint progress. + _advance(i) + local_consumed[i] += 1 + pos_consumed[i] = pos + 1 + split_idx = my_splits[i] + self._resume_samples[split_idx] = ( + initial_samples[i] + local_consumed[i] + ) + self._resume_positions[split_idx] = pos_consumed[i] + return row + + def _update_progress_stats() -> None: + if not real_worker: + self._resume_offset = min( + initial_samples[j] + local_consumed[j] for j in range(n) + ) + _update_stats() + if self._pack_sequences is not None: first_count = pack_blocks_emitted[my_splits[0]] if any( @@ -878,12 +1105,38 @@ class StreamingDataset(IterableDataset): tokens.extend([pad_id] * (pack_len - len(tokens))) block = _emit_block(i) pack_blocks_emitted[my_splits[i]] += 1 + # Checkpoint state must advance before yielding so + # StreamingDataLoader can attach the exact state to + # the batch it transports to the parent process. + _commit_pack_state() if i == n - 1: - _commit_pack_state() _update_stats() yield block return + # A checkpoint taken between round-robin split turns has + # non-uniform counts. Resume lagging splits first so the + # exact canonical sequence continues without replaying + # already-consumed rows. + if len(set(initial_samples)) > 1: + catch_up_to = max(initial_samples) + pending = [ + (initial_samples[i], my_splits[i], i) + for i in range(n) + if initial_samples[i] < catch_up_to + ] + heapq.heapify(pending) + while pending: + consumed, _, i = heapq.heappop(pending) + _ensure_cooked(i) + if not cooked[i]: + return + row = _yield_row(i) + if consumed + 1 < catch_up_to: + heapq.heappush(pending, (consumed + 1, my_splits[i], i)) + _update_progress_stats() + yield row + while True: # A cycle only runs if every split can still produce a # row. Without skips all splits exhaust simultaneously @@ -904,20 +1157,14 @@ class StreamingDataset(IterableDataset): break for i in range(n): - pos, row = cooked[i].popleft() - local_consumed[i] += 1 - pos_consumed[i] = pos + 1 - _advance(i) + row = _yield_row(i) # After the last split in each cycle: update the # global offset and refresh the shared-memory stats # so the main process can observe pipeline depth # even when __iter__ runs in a worker process. if i == n - 1: - self._resume_offset = initial_offset + local_consumed[i] - for j, split_idx in enumerate(my_splits): - self._resume_positions[split_idx] = pos_consumed[j] - _update_stats() + _update_progress_stats() yield row finally: @@ -1064,6 +1311,7 @@ class StreamingDataset(IterableDataset): "_local_consumed_ref", ): state[key] = None + state["_consumer_iterator_lock"] = None return state def __setstate__(self, state): @@ -1074,6 +1322,7 @@ class StreamingDataset(IterableDataset): table_state = state.pop("_table") perm_name, perm_data = state.pop("_perm_table") self.__dict__.update(state) + self._consumer_iterator_lock = threading.Lock() if self._connection_factory is not None: self._table = self._connection_factory(table_name) else: @@ -1083,10 +1332,18 @@ class StreamingDataset(IterableDataset): def state_dict(self) -> dict: """Snapshot the dataset's consumption state. + When using DataLoader workers, construct a + [StreamingDataLoader][lancedb.streaming.StreamingDataLoader]. It + commits worker state only when a prefetched batch is returned to the + trainer. A standard multi-process ``DataLoader`` cannot expose that + boundary, so calling this method after one has started raises + ``RuntimeError`` instead of returning stale producer state. + In row mode, the returned dict is topology-independent at global step boundaries. ``positions_consumed_per_split`` records how far each split's permutation has advanced, which can differ from the sample - count when ``on_transform_error`` skips rows. Combine state dicts from + count when ``on_transform_error`` skips rows. ``StreamingDataLoader`` + combines worker state in its parent process. Combine state dicts from every rank with [merge_state_dicts][lancedb.streaming.StreamingDataset.merge_state_dicts] before resuming on a different topology. @@ -1095,6 +1352,43 @@ class StreamingDataset(IterableDataset): for every logical split. When packing is sharded, merge every rank state with ``merge_state_dicts`` before loading it. """ + if self._untracked_worker_iteration[0] and get_worker_info() is None: + raise RuntimeError( + "StreamingDataset cannot checkpoint a standard DataLoader with " + "num_workers > 0 because prefetched worker progress is not " + "consumer-committed. Use StreamingDataLoader instead." + ) + if self._checkpoint_invalid_reason is not None: + raise RuntimeError( + "StreamingDataset checkpointing is invalid because " + f"{self._checkpoint_invalid_reason}. Load the last valid " + "checkpoint into a fresh dataset before continuing." + ) + state = self._checkpoint_snapshot() + if self._pack_sequences is not None: + rank_blocks = [ + state["blocks_emitted_per_split"][split] for split in self._rank_splits + ] + if len(set(rank_blocks)) > 1: + raise RuntimeError( + "Packed StreamingDataset checkpointing is only safe at a " + "complete logical step boundary, when every split assigned " + "to this rank has emitted the same block count. Consume more " + "batches before calling state_dict()." + ) + elif self._consumer_checkpoint_requires_uniform: + samples = state["samples_consumed_per_split"] + rank_samples = [samples[split] for split in self._rank_splits] + if len(set(rank_samples)) > 1: + raise RuntimeError( + "StreamingDataLoader checkpointing with multiple workers is " + "only safe at a complete logical step boundary, when every " + "split assigned to this rank has the same consumed-sample " + "count. Consume more batches before calling state_dict()." + ) + return state + + def _checkpoint_snapshot(self) -> dict: if self._pack_sequences is not None: return { "shuffle_seed": self._shuffle_seed, @@ -1108,18 +1402,141 @@ class StreamingDataset(IterableDataset): "blocks_emitted_per_split": list(self._pack_blocks_emitted), "pack_buffers": deepcopy(self._pack_buffers), } + samples = [ + self._resume_samples.get(split, self._resume_offset) + for split in range(self._num_splits) + ] positions = [ - self._resume_positions.get(split, self._resume_offset) + self._resume_positions.get(split, samples[split]) for split in range(self._num_splits) ] return { "shuffle_seed": self._shuffle_seed, "num_splits": self._num_splits, "epoch": self._epoch, - "samples_consumed_per_split": [self._resume_offset] * self._num_splits, + "samples_consumed_per_split": samples, "positions_consumed_per_split": positions, } + def _invalidate_checkpoint(self, reason: str) -> None: + if self._checkpoint_invalid_reason is None: + self._checkpoint_invalid_reason = reason + + @property + def _consumer_iterator_active(self) -> bool: + return self._consumer_iterator_lease is not None + + @property + def _consumer_iterator_owner(self) -> Optional[int]: + lease = self._consumer_iterator_lease + return lease.owner_token if lease is not None else None + + @property + def _consumer_iterator_owner_thread(self) -> Optional[int]: + lease = self._consumer_iterator_lease + return lease.owner_thread if lease is not None else None + + def _acquire_consumer_iterator(self) -> int: + """Reserve this parent dataset for one checkpoint-aware iterator.""" + with self._consumer_iterator_lock: + if self._consumer_iterator_active or self._raw_batches_ref is not None: + raise RuntimeError( + "StreamingDataset does not support concurrent iteration. " + "Only one active iterator per dataset instance is allowed." + ) + owner_thread = threading.get_ident() + owner_token = self._consumer_iterator_generation + 1 + lease = _ConsumerIteratorLease(owner_token, owner_thread) + self._consumer_iterator_generation = owner_token + self._consumer_iterator_lease = lease + return owner_token + + def _release_consumer_iterator(self, owner_token: int) -> None: + with self._consumer_iterator_lock: + lease = self._consumer_iterator_lease + if lease is not None and lease.owner_token == owner_token: + self._consumer_iterator_lease = None + + def _release_consumer_iterator_after_failed_acquire( + self, previous_lease: Optional[_ConsumerIteratorLease] + ) -> None: + """Clean up when an interrupted acquire set a lease but did not return it.""" + owner_thread = threading.current_thread().ident + with self._consumer_iterator_lock: + lease = self._consumer_iterator_lease + if ( + lease is not None + and lease is not previous_lease + and lease.owner_thread == owner_thread + ): + self._consumer_iterator_lease = None + + def _commit_worker_state(self, state: dict, *, require_uniform: bool) -> None: + """Merge one trainer-consumed worker batch into parent state.""" + for key, expected in ( + ("shuffle_seed", self._shuffle_seed), + ("num_splits", self._num_splits), + ("epoch", self._epoch), + ): + if state.get(key) != expected: + raise ValueError( + f"{key} mismatch in worker checkpoint: " + f"{state.get(key)} != {expected}" + ) + packed = "pack_buffers" in state + if packed != (self._pack_sequences is not None): + raise ValueError("worker checkpoint mode does not match the dataset") + if packed: + for key in ("pack_sequences", "eos_id", "pad_id", "blocks_per_epoch"): + expected = getattr(self, f"_{key}") + if state.get(key) != expected: + raise ValueError( + f"{key} mismatch in worker checkpoint: " + f"{state.get(key)} != {expected}" + ) + samples = state["samples_consumed_per_split"] + emitted = state["blocks_emitted_per_split"] + if len(samples) != self._num_splits or len(emitted) != self._num_splits: + raise ValueError( + "packed worker checkpoint must contain one entry per split" + ) + buffers = state["pack_buffers"] + for split, (count, blocks) in enumerate(zip(samples, emitted)): + incoming = (int(blocks), int(count)) + current = ( + self._pack_blocks_emitted[split], + self._pack_consumed[split], + ) + if incoming > current: + self._pack_blocks_emitted[split] = incoming[0] + self._pack_consumed[split] = incoming[1] + buffer = buffers.get(split, buffers.get(str(split))) + if buffer is None: + self._pack_buffers.pop(split, None) + else: + self._pack_buffers[split] = { + "tokens": list(buffer["tokens"]), + "starts": list(buffer["starts"]), + } + self._consumer_checkpoint_requires_uniform |= require_uniform + return + + samples = state["samples_consumed_per_split"] + positions = state.get("positions_consumed_per_split", samples) + for split, count in enumerate(samples): + current = self._resume_samples.get(split, self._resume_offset) + self._resume_samples[split] = max(current, int(count)) + for split, position in enumerate(positions): + current = self._resume_positions.get( + split, self._resume_samples.get(split, self._resume_offset) + ) + self._resume_positions[split] = max(current, int(position)) + self._resume_offset = min( + self._resume_samples.get(split, self._resume_offset) + for split in range(self._num_splits) + ) + self._consumer_checkpoint_requires_uniform |= require_uniform + def load_state_dict(self, state: dict) -> None: """Resume from a previously snapshotted state. @@ -1139,6 +1556,7 @@ class StreamingDataset(IterableDataset): f"shuffle_seed mismatch: checkpoint has {state['shuffle_seed']}, " f"current dataset has {self._shuffle_seed}" ) + self._consumer_checkpoint_requires_uniform = False if "pack_buffers" in state or self._pack_sequences is not None: for key in ( @@ -1165,14 +1583,17 @@ class StreamingDataset(IterableDataset): return consumed = state["samples_consumed_per_split"] - # All entries are equal at step boundaries; use the first. if isinstance(consumed, list): - self._resume_offset = consumed[0] if consumed else 0 + self._resume_offset = min(consumed) if consumed else 0 + self._resume_samples = { + split: int(count) for split, count in enumerate(consumed) + } else: self._resume_offset = int(consumed) + self._resume_samples = {} # Older checkpoints predate positions_consumed_per_split; without # skipped rows positions equal sample counts, so falling back to - # _resume_offset (the .get default in __iter__) is exact. + # the per-split sample count (the .get default in __iter__) is exact. positions = state.get("positions_consumed_per_split") if positions is None: self._resume_positions = {} @@ -1185,10 +1606,11 @@ class StreamingDataset(IterableDataset): def merge_state_dicts(states: list[dict]) -> dict: """Merge state dicts saved by different ranks into one exact state. - For row mode, the elementwise maximum of permutation positions recovers - splits advanced by different ranks after transform failures. For packed - mode, the state that emitted the most blocks for each logical split - supplies that split's permutation position and partial token buffer. Packed + In row mode, each rank records exact consumer-committed progress for + its own splits and lower bounds for the rest, so elementwise maxima + recover both sample counts and permutation positions. In packed mode, + the state that emitted the most blocks for each logical split supplies + that split's permutation position and partial token buffer. Packed states must cover every rank at the same global step. Raises ``ValueError`` if the states are empty, were not produced by @@ -1299,17 +1721,13 @@ class StreamingDataset(IterableDataset): merged["pack_buffers"] = merged_buffers return merged - for state in states[1:]: - if ( - state["samples_consumed_per_split"] - != first["samples_consumed_per_split"] - ): - raise ValueError( - "samples_consumed_per_split mismatch across state dicts; " - "state_dict() must be called at the same global step " - "boundary on every rank" - ) merged = dict(first) + merged["samples_consumed_per_split"] = [ + max(per_split) + for per_split in zip( + *(state["samples_consumed_per_split"] for state in states) + ) + ] all_positions = [ state.get( "positions_consumed_per_split", state["samples_consumed_per_split"] @@ -1320,3 +1738,113 @@ class StreamingDataset(IterableDataset): max(per_split) for per_split in zip(*all_positions) ] return merged + + +class StreamingDataLoader(DataLoader): + """A PyTorch DataLoader with consumer-committed dataset checkpoints. + + PyTorch workers prefetch batches ahead of the trainer, so worker-local + producer progress is not a safe checkpoint. This loader carries a state + snapshot alongside every internal batch and applies it to the parent + [StreamingDataset][lancedb.streaming.StreamingDataset] only when that batch + is returned by ``next()``. + The trainer receives the same collated batch it would receive from a + standard ``torch.utils.data.DataLoader``. + + With more than one worker, row-mode ``state_dict()`` is available only at + complete logical step boundaries, when every split assigned to the rank has + the same consumed-sample count. Packed checkpoints require equal emitted-block + counts across the rank's splits for any worker count. ``persistent_workers=True`` + is not supported because prefetched worker copies cannot be restored from + parent-committed state. If batch collation raises, checkpointing remains + invalid for that dataset instance; restore the last valid checkpoint into a + fresh dataset before continuing. + Only one active iterator may own a dataset at a time, including when worker + processes are used. Exhausting or explicitly shutting down the iterator + releases that ownership. ``drop_last=True`` is not supported because worker + replicas discard incomplete tails independently, which cannot produce a + topology-independent checkpoint. + + Parameters are the same as ``torch.utils.data.DataLoader`` except that + ``dataset`` must be a + [StreamingDataset][lancedb.streaming.StreamingDataset]. + Subclasses that override ``StreamingDataset.__iter__`` are not supported + because the custom iterator cannot provide the exact per-yield checkpoint + snapshots required by this loader. + + Examples + -------- + >>> # dataset = StreamingDataset(table, num_splits=2) + >>> # loader = StreamingDataLoader(dataset, batch_size=8, num_workers=2) + >>> # batch = next(iter(loader)) + >>> # checkpoint = dataset.state_dict() + """ + + def __init__(self, dataset: StreamingDataset, *args, **kwargs): + if not isinstance(dataset, StreamingDataset): + raise TypeError("StreamingDataLoader requires a StreamingDataset") + if type(dataset).__iter__ is not StreamingDataset.__iter__: + raise TypeError( + "StreamingDataLoader does not support StreamingDataset subclasses " + "that override __iter__ because they cannot provide exact " + "per-yield checkpoint state" + ) + if kwargs.get("in_order", True) is False: + raise ValueError( + "StreamingDataLoader requires in_order=True for deterministic " + "consumer checkpoints" + ) + if kwargs.get("persistent_workers", False): + raise ValueError( + "StreamingDataLoader does not support persistent_workers=True " + "because worker prefetch state cannot be reset from a checkpoint" + ) + self._streaming_dataset = dataset + super().__init__(_StreamingDatasetAdapter(dataset), *args, **kwargs) + if self.drop_last: + raise ValueError( + "StreamingDataLoader does not support drop_last=True because " + "discarded worker tails cannot be checkpointed " + "topology-independently" + ) + self.collate_fn = _CheckpointCollate(self.collate_fn) + + def __iter__(self): + dataset = self._streaming_dataset + previous_lease = dataset._consumer_iterator_lease + owner_token = None + try: + owner_token = dataset._acquire_consumer_iterator() + state = dataset._checkpoint_snapshot() + packed = dataset._pack_sequences is not None + if packed: + blocks = state["blocks_emitted_per_split"] + rank_blocks = [blocks[split] for split in dataset._rank_splits] + if len(set(rank_blocks)) > 1: + raise RuntimeError( + "StreamingDataLoader cannot start from a partial packed " + "logical step; resume from a checkpoint whose splits " + "assigned to this rank have equal emitted-block counts" + ) + elif self.num_workers > 1: + samples = state["samples_consumed_per_split"] + rank_samples = [samples[split] for split in dataset._rank_splits] + if len(set(rank_samples)) > 1: + raise RuntimeError( + "StreamingDataLoader cannot start multiple workers from a " + "partial logical step; resume from a checkpoint whose " + "splits assigned to this rank have equal consumed-sample " + "counts" + ) + return _ConsumerCommitIterator( + super().__iter__(), + dataset, + owner_token=owner_token, + require_uniform=self.num_workers > 1 or packed, + ) + except BaseException: + if owner_token is not None: + dataset._release_consumer_iterator(owner_token) + else: + dataset._release_consumer_iterator_after_failed_acquire(previous_lease) + raise diff --git a/python/python/tests/test_elastic_dataloader.py b/python/python/tests/test_elastic_dataloader.py index ff65d100c..da27e5bfc 100644 --- a/python/python/tests/test_elastic_dataloader.py +++ b/python/python/tests/test_elastic_dataloader.py @@ -32,6 +32,7 @@ Parameters used throughout: import dataclasses import logging +import threading from unittest.mock import patch import lancedb @@ -46,6 +47,7 @@ from utils import ( torch = pytest.importorskip("torch") streaming = pytest.importorskip("lancedb.streaming") StreamingDataset = streaming.StreamingDataset +StreamingDataLoader = streaming.StreamingDataLoader # --------------------------------------------------------------------------- # Dataset parameters @@ -92,6 +94,27 @@ class FakeWorkerInfo: num_workers: int +def _collate_with_first_batch_error(samples): + ids = [sample["id"] for sample in samples] + if ids == [0, 1]: + raise ValueError("first batch fails") + return ids + + +def _collate_with_first_batch_stop(samples): + ids = [sample["id"] for sample in samples] + if ids == [0, 1]: + raise StopIteration("first batch stopped") + return ids + + +def _collate_with_first_batch_interrupt(samples): + ids = [sample["id"] for sample in samples] + if ids == [0, 1]: + raise KeyboardInterrupt("first batch interrupted") + return ids + + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -1008,6 +1031,565 @@ def test_multi_worker_elastic_det_across_worker_counts(lance_table): # ── Resumability with num_workers ───────────────────────────────────────────── +def test_streaming_dataloader_commits_only_consumed_worker_batches(tmp_path): + """Prefetched worker state is committed only as the trainer receives it.""" + db = lancedb.connect(tmp_path) + table = db.create_table( + "worker_commit", pa.table({"id": [1, 2, 3, 4, 10, 20, 30, 40]}) + ) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=2, + multiprocessing_context="spawn", + prefetch_factor=4, + ) + iterator = iter(loader) + try: + first = next(iterator)["id"].tolist() + + assert first == [1, 2] + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2, 0] + with pytest.raises(RuntimeError, match="complete logical step boundary"): + dataset.state_dict() + + second = next(iterator)["id"].tolist() + assert second == [10, 20] + checkpoint = dataset.state_dict() + assert checkpoint["samples_consumed_per_split"] == [2, 2] + uninterrupted = [batch["id"].tolist() for batch in iterator] + finally: + iterator._shutdown_workers() + + resumed = StreamingDataset(table, num_splits=2, shuffle=False) + resumed.load_state_dict(checkpoint) + resumed_loader = StreamingDataLoader( + resumed, + batch_size=2, + num_workers=2, + multiprocessing_context="spawn", + prefetch_factor=4, + ) + resumed_iterator = iter(resumed_loader) + try: + remaining = [batch["id"].tolist() for batch in resumed_iterator] + finally: + resumed_iterator._shutdown_workers() + assert remaining == uninterrupted == [[3, 4], [30, 40]] + + +def test_distributed_checkpoint_uses_rank_local_worker_boundary(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("rank_boundary", pa.table({"id": list(range(8))})) + dataset = StreamingDataset( + table, + num_splits=4, + shuffle=False, + rank=0, + world_size=2, + ) + loader = StreamingDataLoader( + dataset, + batch_size=1, + num_workers=2, + multiprocessing_context="spawn", + ) + iterator = iter(loader) + try: + assert next(iterator)["id"].tolist() == [0] + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [ + 1, + 0, + 0, + 0, + ] + with pytest.raises(RuntimeError, match="complete logical step boundary"): + dataset.state_dict() + + assert next(iterator)["id"].tolist() == [2] + checkpoint = dataset.state_dict() + remaining = [batch["id"].tolist() for batch in iterator] + finally: + iterator._shutdown_workers() + + assert checkpoint["samples_consumed_per_split"] == [1, 1, 0, 0] + assert remaining == [[1], [3]] + + +def test_standard_dataloader_rejects_stale_parent_checkpoint(tmp_path): + """A standard DataLoader must not expose prefetched producer progress.""" + db = lancedb.connect(tmp_path) + table = db.create_table("untracked_workers", pa.table({"id": [1, 2, 10, 20]})) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + # Merely constructing the checkpoint-aware loader must not authorize a + # later plain DataLoader's worker progress. + StreamingDataLoader(dataset, batch_size=2, num_workers=0) + loader = torch.utils.data.DataLoader( + dataset, + batch_size=2, + num_workers=2, + multiprocessing_context="spawn", + ) + iterator = iter(loader) + try: + assert next(iterator)["id"].tolist() == [1, 2] + with pytest.raises(RuntimeError, match="Use StreamingDataLoader"): + dataset.state_dict() + list(iterator) + finally: + iterator._shutdown_workers() + + +def test_streaming_dataloader_rejects_persistent_workers(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("persistent_workers", pa.table({"id": [1, 2]})) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + + with pytest.raises(ValueError, match="persistent_workers=True"): + StreamingDataLoader( + dataset, + batch_size=1, + num_workers=2, + persistent_workers=True, + ) + + +def test_collate_failure_invalidates_consumer_checkpoint(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table( + "collate_failure", pa.table({"id": [0, 1, 2, 3, 100, 101, 102, 103]}) + ) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=2, + multiprocessing_context="spawn", + collate_fn=_collate_with_first_batch_error, + prefetch_factor=2, + ) + iterator = iter(loader) + try: + with pytest.raises(ValueError, match="first batch fails"): + next(iterator) + assert next(iterator) == [100, 101] + assert next(iterator) == [2, 3] + with pytest.raises(RuntimeError, match="failed before it was returned"): + dataset.state_dict() + list(iterator) + finally: + iterator._shutdown_workers() + + +def test_collate_stop_iteration_invalidates_consumer_checkpoint(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("collate_stop", pa.table({"id": list(range(6))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=0, + collate_fn=_collate_with_first_batch_stop, + ) + iterator = iter(loader) + + with pytest.raises(RuntimeError, match="collate_fn raised StopIteration"): + next(iterator) + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2] + with pytest.raises(RuntimeError, match="failed before it was returned"): + dataset.state_dict() + assert list(iterator) == [[2, 3], [4, 5]] + + +def test_batch_base_exception_invalidates_consumer_checkpoint(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("collate_interrupt", pa.table({"id": list(range(6))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=0, + collate_fn=_collate_with_first_batch_interrupt, + ) + iterator = iter(loader) + + with pytest.raises(KeyboardInterrupt, match="first batch interrupted"): + next(iterator) + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2] + with pytest.raises(RuntimeError, match="failed before it was returned"): + dataset.state_dict() + assert list(iterator) == [[2, 3], [4, 5]] + + +def test_parent_commit_base_exception_invalidates_consumer_checkpoint(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("commit_interrupt", pa.table({"id": list(range(4))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0) + iterator = iter(loader) + real_commit = dataset._commit_worker_state + + def interrupt_after_commit(state, *, require_uniform): + real_commit(state, require_uniform=require_uniform) + raise KeyboardInterrupt("after parent commit") + + with patch.object( + dataset, "_commit_worker_state", side_effect=interrupt_after_commit + ): + with pytest.raises(KeyboardInterrupt, match="after parent commit"): + next(iterator) + + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2] + with pytest.raises(RuntimeError, match="failed before it was returned"): + dataset.state_dict() + + +def test_direct_iteration_surfaces_prefetch_failure_before_committing_row( + tmp_path, monkeypatch +): + db = lancedb.connect(tmp_path) + table = db.create_table("prefetch_failure", pa.table({"id": list(range(4))})) + release = threading.Event() + failed = threading.Event() + real_getitems = streaming.Permutation.__getitems__ + + def controlled_getitems(permutation, indices): + if indices and indices[0] >= 2: + assert release.wait(timeout=5) + failed.set() + raise RuntimeError("later prefetched I/O failed") + return real_getitems(permutation, indices) + + class SignalDict(dict): + def __setitem__(self, key, value): + super().__setitem__(key, value) + release.set() + assert failed.wait(timeout=5) + + monkeypatch.setattr(streaming.Permutation, "__getitems__", controlled_getitems) + dataset = StreamingDataset( + table, + num_splits=1, + shuffle=False, + read_batch_size=2, + io_queue_depth=2, + ) + dataset._resume_positions = SignalDict() + iterator = iter(dataset) + + assert next(iterator)["id"] == 0 + with pytest.raises(RuntimeError, match="later prefetched I/O failed"): + next(iterator) + + checkpoint = dataset.state_dict() + assert checkpoint["samples_consumed_per_split"] == [1] + assert checkpoint["positions_consumed_per_split"] == [1] + + +@pytest.mark.parametrize("workers", [0, 1, 2]) +def test_streaming_dataloader_rejects_drop_last(tmp_path, workers): + db = lancedb.connect(tmp_path) + table = db.create_table("drop_last", pa.table({"id": [0, 1, 2]})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + worker_options = {"multiprocessing_context": "spawn"} if workers else {} + + with pytest.raises(ValueError, match="drop_last=True"): + StreamingDataLoader( + dataset, + batch_size=2, + num_workers=workers, + drop_last=True, + **worker_options, + ) + + +def test_streaming_dataloader_owns_one_iterator_until_teardown(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("iterator_owner", pa.table({"id": list(range(4))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=1, + multiprocessing_context="spawn", + ) + + first = iter(loader) + try: + assert next(first)["id"].tolist() == [0, 1] + with pytest.raises(RuntimeError, match="concurrent iteration"): + iter(loader) + finally: + first._shutdown_workers() + + second = iter(loader) + try: + assert [batch["id"].tolist() for batch in second] == [[2, 3]] + except BaseException: + second._shutdown_workers() + raise + + # Natural exhaustion releases ownership too. + third = iter(loader) + try: + assert list(third) == [] + finally: + third._shutdown_workers() + + +def test_zero_worker_shutdown_closes_inner_iterator_before_release(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("zero_worker_shutdown", pa.table({"id": list(range(6))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0) + + first = iter(loader) + assert next(first)["id"].tolist() == [0, 1] + first._shutdown_workers() + + assert dataset._consumer_iterator_active is False + assert dataset._raw_batches_ref is None + second = iter(loader) + try: + with pytest.raises(StopIteration): + next(first) + assert next(second)["id"].tolist() == [2, 3] + finally: + second._shutdown_workers() + + +def test_direct_and_loader_admission_share_one_atomic_lease(tmp_path, monkeypatch): + db = lancedb.connect(tmp_path) + table = db.create_table("direct_loader_lease", pa.table({"id": list(range(4))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0) + entered = threading.Event() + release = threading.Event() + direct_result = [] + direct_error = [] + contender = [] + real_resolve = dataset._resolve_my_splits + + def controlled_resolve(): + if threading.current_thread().name == "direct-start": + entered.set() + assert release.wait(timeout=5) + return real_resolve() + + def advance_direct(iterator): + try: + direct_result.append(next(iterator)["id"]) + except BaseException as exc: + direct_error.append(exc) + + monkeypatch.setattr(dataset, "_resolve_my_splits", controlled_resolve) + direct = iter(dataset) + thread = threading.Thread( + target=advance_direct, args=(direct,), name="direct-start" + ) + thread.start() + assert entered.wait(timeout=5) + try: + with pytest.raises(RuntimeError, match="concurrent iteration"): + contender.append(iter(loader)) + finally: + release.set() + thread.join(timeout=5) + if contender: + contender[0]._shutdown_workers() + direct.close() + + assert not thread.is_alive() + assert direct_error == [] + assert direct_result == [0] + + +def test_loader_acquires_before_snapshot_and_cleans_interrupted_acquire( + tmp_path, monkeypatch +): + db = lancedb.connect(tmp_path) + table = db.create_table("lease_snapshot", pa.table({"id": list(range(4))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0) + first = iter(loader) + assert next(first)["id"].tolist() == [0, 1] + + entered = threading.Event() + release = threading.Event() + pending = [] + pending_errors = [] + observed_snapshots = [] + real_acquire = dataset._acquire_consumer_iterator + real_snapshot = dataset._checkpoint_snapshot + + def controlled_acquire(): + if threading.current_thread().name == "stale-start": + entered.set() + assert release.wait(timeout=5) + return real_acquire() + + def recording_snapshot(): + state = real_snapshot() + if threading.current_thread().name == "stale-start": + observed_snapshots.append(state["samples_consumed_per_split"]) + return state + + def create_pending_iterator(): + try: + pending.append(iter(loader)) + except BaseException as exc: + pending_errors.append(exc) + + monkeypatch.setattr(dataset, "_acquire_consumer_iterator", controlled_acquire) + monkeypatch.setattr(dataset, "_checkpoint_snapshot", recording_snapshot) + thread = threading.Thread(target=create_pending_iterator, name="stale-start") + thread.start() + assert entered.wait(timeout=5) + assert next(first)["id"].tolist() == [2, 3] + with pytest.raises(StopIteration): + next(first) + release.set() + thread.join(timeout=5) + + assert not thread.is_alive() + assert pending_errors == [] + assert observed_snapshots == [[4]] + assert len(pending) == 1 + assert list(pending[0]) == [] + assert dataset.state_dict()["samples_consumed_per_split"] == [4] + + def interrupted_acquire(): + real_acquire() + raise KeyboardInterrupt("after acquire") + + monkeypatch.setattr(dataset, "_acquire_consumer_iterator", interrupted_acquire) + with pytest.raises(KeyboardInterrupt, match="after acquire"): + iter(loader) + assert dataset._consumer_iterator_active is False + + +def test_consumer_iterator_lease_publication_is_atomic(tmp_path, monkeypatch): + db = lancedb.connect(tmp_path) + table = db.create_table("atomic_lease", pa.table({"id": [0, 1]})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=1, num_workers=0) + real_get_ident = streaming.threading.get_ident + calls = 0 + + def interrupt_during_publication(): + nonlocal calls + calls += 1 + if calls == 1: + raise KeyboardInterrupt("during lease mutation") + return real_get_ident() + + monkeypatch.setattr(streaming.threading, "get_ident", interrupt_during_publication) + with pytest.raises(KeyboardInterrupt, match="during lease mutation"): + iter(loader) + monkeypatch.setattr(streaming.threading, "get_ident", real_get_ident) + + assert dataset._consumer_iterator_active is False + iterator = iter(loader) + try: + assert next(iterator)["id"].tolist() == [0] + finally: + iterator._shutdown_workers() + + +def test_streaming_dataloader_rejects_dataset_iter_override(tmp_path): + class CustomizedDataset(StreamingDataset): + def __iter__(self): + return iter([1000, 1001]) + + db = lancedb.connect(tmp_path) + table = db.create_table("custom_iteration", pa.table({"id": [0, 1, 2]})) + dataset = CustomizedDataset(table, num_splits=1, shuffle=False) + + assert list(dataset) == [1000, 1001] + with pytest.raises(TypeError, match="override __iter__"): + StreamingDataLoader( + dataset, + batch_size=2, + num_workers=0, + collate_fn=list, + ) + + +def test_interleaved_adapters_do_not_authorize_plain_iteration(tmp_path): + db = lancedb.connect(tmp_path) + table_a = db.create_table("adapter_a", pa.table({"id": [0, 1]})) + table_b = db.create_table("adapter_b", pa.table({"id": [10, 11]})) + dataset_a = StreamingDataset(table_a, num_splits=1, shuffle=False) + dataset_b = StreamingDataset(table_b, num_splits=1, shuffle=False) + initial_state = dataset_a.state_dict() + + owner_a = dataset_a._acquire_consumer_iterator() + owner_b = dataset_b._acquire_consumer_iterator() + try: + iterator_a = iter(streaming._StreamingDatasetAdapter(dataset_a)) + iterator_b = iter(streaming._StreamingDatasetAdapter(dataset_b)) + assert next(iterator_a).data["id"] == 0 + assert next(iterator_b).data["id"] == 10 + assert [sample.data["id"] for sample in iterator_a] == [1] + assert [sample.data["id"] for sample in iterator_b] == [11] + finally: + dataset_a._release_consumer_iterator(owner_a) + dataset_b._release_consumer_iterator(owner_b) + + dataset_a.load_state_dict(initial_state) + with patch( + "lancedb.streaming.get_worker_info", + return_value=FakeWorkerInfo(id=0, num_workers=1), + ): + plain_iterator = iter(dataset_a) + assert next(plain_iterator)["id"] == 0 + plain_iterator.close() + + assert dataset_a._untracked_worker_iteration[0] == 1 + with pytest.raises(RuntimeError, match="Use StreamingDataLoader"): + dataset_a.state_dict() + + +def test_resume_from_partial_split_cycle_preserves_remaining_order(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("partial_cycle", pa.table({"id": [1, 2, 10, 20]})) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + iterator = iter(dataset) + + assert next(iterator)["id"] == 1 + checkpoint = dataset.state_dict() + iterator.close() + assert checkpoint["samples_consumed_per_split"] == [1, 0] + + resumed = StreamingDataset(table, num_splits=2, shuffle=False) + resumed.load_state_dict(checkpoint) + assert [row["id"] for row in resumed] == [10, 2, 20] + + +def test_partial_cycle_resume_preserves_skip_truncation(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table( + "partial_skip", pa.table({"id": [0, 1, 2, 3, 100, 101, 102, 103]}) + ) + kwargs = dict( + num_splits=2, + shuffle=False, + transform=_failing_transform({1, 2, 3}), + on_transform_error="skip", + ) + dataset = StreamingDataset(table, **kwargs) + iterator = iter(dataset) + + assert next(iterator)["id"] == 0 + checkpoint = dataset.state_dict() + uninterrupted = [row["id"] for row in iterator] + + resumed = StreamingDataset(table, **kwargs) + resumed.load_state_dict(checkpoint) + assert [row["id"] for row in resumed] == uninterrupted == [100] + + def test_multi_worker_resumability_same_topology(lance_table): """Checkpoint with num_workers=2, resume with num_workers=2: exact continuation.""" world_size = 1 @@ -2018,6 +2600,23 @@ def test_merge_state_dicts_validates_consistency(lance_table): StreamingDataset.merge_state_dicts([]) +def test_merge_state_dicts_combines_nonuniform_consumer_progress(lance_table): + dataset = StreamingDataset( + lance_table, num_splits=2, shuffle=False, shuffle_seed=SHUFFLE_SEED + ) + rank0 = dataset.state_dict() + rank0["samples_consumed_per_split"] = [2, 0] + rank0["positions_consumed_per_split"] = [2, 0] + rank1 = dataset.state_dict() + rank1["samples_consumed_per_split"] = [0, 2] + rank1["positions_consumed_per_split"] = [0, 2] + + merged = StreamingDataset.merge_state_dicts([rank0, rank1]) + + assert merged["samples_consumed_per_split"] == [2, 2] + assert merged["positions_consumed_per_split"] == [2, 2] + + def test_load_state_dict_without_positions_key(lance_table): """Checkpoints from before positions_consumed_per_split existed still resume exactly (positions equal sample counts when nothing is skipped).""" @@ -2254,6 +2853,65 @@ def test_pack_sequences_checkpoint_resumes_on_new_topology(tmp_path): ] +def test_packed_checkpoint_requires_complete_split_cycle(tmp_path): + table = _create_token_table(tmp_path, [[1], [2], [10], [20]]) + dataset = _packed_dataset(table, pack_sequences=3, blocks_per_epoch=4, num_splits=2) + iterator = iter(dataset) + + next(iterator) + with pytest.raises(RuntimeError, match="complete logical step boundary"): + dataset.state_dict() + + next(iterator) + assert dataset.state_dict()["blocks_emitted_per_split"] == [1, 1] + iterator.close() + + +def test_streaming_dataloader_commits_consumed_packed_batches(tmp_path): + table = _create_token_table( + tmp_path, + [[1], [2], [3], [4], [10], [20], [30], [40]], + ) + kwargs = dict(pack_sequences=4, blocks_per_epoch=4, num_splits=2) + dataset = _packed_dataset(table, **kwargs) + loader = StreamingDataLoader( + dataset, + batch_size=1, + num_workers=2, + multiprocessing_context="spawn", + prefetch_factor=2, + ) + iterator = iter(loader) + try: + next(iterator) + with pytest.raises(RuntimeError, match="complete logical step boundary"): + dataset.state_dict() + + next(iterator) + checkpoint = dataset.state_dict() + uninterrupted = [batch["input_ids"].tolist() for batch in iterator] + finally: + iterator._shutdown_workers() + + resumed = _packed_dataset(table, **kwargs) + resumed.load_state_dict(checkpoint) + resumed_loader = StreamingDataLoader( + resumed, + batch_size=1, + num_workers=2, + multiprocessing_context="spawn", + prefetch_factor=2, + ) + resumed_iterator = iter(resumed_loader) + try: + remaining = [batch["input_ids"].tolist() for batch in resumed_iterator] + finally: + resumed_iterator._shutdown_workers() + + assert checkpoint["blocks_emitted_per_split"] == [1, 1] + assert remaining == uninterrupted + + def test_pack_sequences_validates_configuration_and_tokens(tmp_path): table = _create_token_table(tmp_path, [[1, 2]]) From 93f47b8aab4f888ab6cd4da75403e2d41be80da4 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 24 Aug 2026 13:35:31 -0700 Subject: [PATCH 37/58] fix(remote): stop table_names inventing a page token for a namespace (#4039) `table_names` paginates using a `start_after` table name. This works for the `/v1/table` endpoint, which guarantees table-order. But the `/v1/namespace/{id}/table/list` does not. We change that caller to instead collect all table names, sort, and apply the pagination locally. We are deprecating this API, so this is just an interim fix. For good performance, users should move to the `list_tables` API instead, which uses opaque tokens that don't rely on lexical sorting. --------- Co-authored-by: Claude Opus 5 (1M context) --- rust/lancedb/src/remote/db.rs | 191 ++++++++++++++++++++++++++++++---- 1 file changed, 171 insertions(+), 20 deletions(-) diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 8265d22c0..3f4216bc8 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -344,6 +344,62 @@ impl RemoteDatabase { self.table_cache.remove(&cache_key).await; Ok((request_id, resp)) } + + /// Collect the tables of a namespace in name order, for `table_names`. + /// + /// `table_names` promises name order and resumes after a table name, but the namespace + /// route's `page_token` is opaque -- it belongs to the store the listing walks, and a + /// token this client invented would resume from the wrong place. So the whole namespace is + /// walked by handing each response's token straight back, and the name semantics are + /// applied here. Constructing no token is what makes this work against a server on either + /// side of the change: it only ever repeats what the server said. + /// + /// This is the cost `table_names` already paid -- the server used to enumerate and sort the + /// namespace on every request -- and it is why `list_tables` replaces it. + async fn table_names_in_namespace( + &self, + request: &TableNamesRequest, + ) -> Result<(Vec, ServerVersion)> { + let namespace_id = + build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter); + let path = format!("/v1/namespace/{}/table/list", namespace_id); + + let mut names = Vec::new(); + // Every page reports the same server, so keep the first page's version. + let mut version: Option = None; + let mut page_token: Option = None; + loop { + let mut req = self.client.get(&path); + if let Some(ref token) = page_token { + req = req.query(&[("page_token", token)]); + } + let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?; + let rsp = self.client.check_response(&request_id, rsp).await?; + if version.is_none() { + version = Some(parse_server_version(&request_id, &rsp)?); + } + let response: ListTablesResponse = rsp.json().await.err_to_http(request_id)?; + names.extend(response.tables); + // An empty token is the end of the listing, not a token to send back: a server + // that reads an empty token as "start from the beginning" would hand back the + // first page again. + match response.page_token.filter(|token| !token.is_empty()) { + // A server that repeated a token would never finish; treat that as the end + // rather than looping on it. + Some(token) if Some(&token) != page_token.as_ref() => page_token = Some(token), + _ => break, + } + } + + names.sort(); + if let Some(ref start_after) = request.start_after { + names.retain(|name| name > start_after); + } + if let Some(limit) = request.limit { + names.truncate(limit as usize); + } + Ok((names, version.unwrap_or_default())) + } } #[cfg(all(test, feature = "remote"))] @@ -621,29 +677,29 @@ impl Database for RemoteDatabase { } async fn table_names(&self, request: TableNamesRequest) -> Result> { - let mut req = if !request.namespace_path.is_empty() { - let namespace_id = - build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter); - self.client - .get(&format!("/v1/namespace/{}/table/list", namespace_id)) + let (tables, version) = if request.namespace_path.is_empty() { + // The flat route resumes after a table name and orders by name, which is exactly + // what `start_after` means, so the server does the paging. + let mut req = self.client.get("/v1/table/"); + if let Some(limit) = request.limit { + req = req.query(&[("limit", limit)]); + } + if let Some(ref start_after) = request.start_after { + req = req.query(&[("page_token", start_after)]); + } + let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?; + let rsp = self.client.check_response(&request_id, rsp).await?; + let version = parse_server_version(&request_id, &rsp)?; + let tables = rsp + .json::() + .await + .err_to_http(request_id)? + .tables; + (tables, version) } else { - self.client.get("/v1/table/") + self.table_names_in_namespace(&request).await? }; - if let Some(limit) = request.limit { - req = req.query(&[("limit", limit)]); - } - if let Some(start_after) = request.start_after { - req = req.query(&[("page_token", start_after)]); - } - let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?; - let rsp = self.client.check_response(&request_id, rsp).await?; - let version = parse_server_version(&request_id, &rsp)?; - let tables = rsp - .json::() - .await - .err_to_http(request_id)? - .tables; for table in &tables { let table_identifier = build_table_identifier(table, &request.namespace_path, &self.client.id_delimiter); @@ -1227,6 +1283,101 @@ mod tests { assert_eq!(names, vec!["table1", "table2"]); } + #[tokio::test] + async fn test_table_names_in_a_namespace_never_invents_a_page_token() { + // The namespace route's token belongs to the store, so `table_names` cannot build one + // from `start_after`. It walks the namespace on the server's own tokens and applies the + // name semantics itself, which is what keeps it working either side of the change. + let page = Arc::new(AtomicUsize::new(0)); + let conn = Connection::new_with_handler(move |request| { + assert_eq!(request.url().path(), "/v1/namespace/ns/table/list"); + let query = request.url().query().unwrap_or(""); + assert!( + !query.contains("page_token=users"), + "a table name must never be sent as a page token: {query}" + ); + match page.fetch_add(1, Ordering::SeqCst) { + 0 => { + assert!( + !query.contains("page_token"), + "the walk starts with no token" + ); + http::Response::builder() + .status(200) + .body(r#"{"tables": ["users", "orders"], "page_token": "opaque-1"}"#) + .unwrap() + } + _ => { + assert!(query.contains("page_token=opaque-1")); + http::Response::builder() + .status(200) + .body(r#"{"tables": ["widgets"]}"#) + .unwrap() + } + } + }); + + let names = conn + .table_names() + .namespace(vec!["ns".to_string()]) + .start_after("users") + .execute() + .await + .unwrap(); + // Name order, resumed after "users": "orders" sorts before it and is dropped. + assert_eq!(names, vec!["widgets"]); + } + + #[tokio::test] + async fn test_table_names_in_a_namespace_stops_on_a_repeated_token() { + // A server that handed back the token it was given would never finish the walk. + let conn = Connection::new_with_handler(|_request| { + http::Response::builder() + .status(200) + .body(r#"{"tables": ["a"], "page_token": "same"}"#) + .unwrap() + }); + + let names = conn + .table_names() + .namespace(vec!["ns".to_string()]) + .execute() + .await + .unwrap(); + // The guard bounds the walk instead of letting it run forever. The repeat is the + // server breaking the token contract and is not papered over here. + assert_eq!(names, vec!["a", "a"]); + } + + #[tokio::test] + async fn test_table_names_in_a_namespace_stops_on_an_empty_token() { + // An empty token ends the listing. Sending it back would ask a server that reads it + // as "start from the beginning" for the first page a second time, and every name on + // that page would be collected twice. + let requests = Arc::new(AtomicUsize::new(0)); + let seen = requests.clone(); + let conn = Connection::new_with_handler(move |request| { + seen.fetch_add(1, Ordering::SeqCst); + assert!( + !request.url().query().unwrap_or("").contains("page_token"), + "an empty token must never be sent back" + ); + http::Response::builder() + .status(200) + .body(r#"{"tables": ["a"], "page_token": ""}"#) + .unwrap() + }); + + let names = conn + .table_names() + .namespace(vec!["ns".to_string()]) + .execute() + .await + .unwrap(); + assert_eq!(names, vec!["a"]); + assert_eq!(requests.load(Ordering::SeqCst), 1); + } + #[tokio::test] async fn test_table_names_pagination() { let conn = Connection::new_with_handler(|request| { From c72f5b2960d08a1d6caee703191c360ecfe6f3ff Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Mon, 24 Aug 2026 14:06:40 -0700 Subject: [PATCH 38/58] feat: bind a materialized view refresh to the view incarnation (#4043) A caller that queues a refresh and executes it later can only tell the view it captured from a drop-and-recreate by comparing the definition and version. A recreated view with the same definition and an equal or higher version passes that check, and the check runs before the refresh reloads the view, so it never sees the state it commits against. This mints an `mv.incarnation` token in the schema metadata at each physical creation of a view table. `RefreshMaterializedViewBuilder::expect_incarnation` carries the captured token into the refresh, which compares it against the latest stored manifest before planning and again immediately before each commit (publish, fragment swap, watermark stamp), refusing to land in a different incarnation. A view with no token -- declared before tokens existed, or its metadata replaced wholesale -- is refused under a bound refresh with its own wording and is minted one by its next unbound refresh. The token is exposed through `MaterializedView::incarnation`; refreshes without an expectation are unchanged. This is best effort: the token is not part of lance's commit condition, so a recreation landing between the final pre-commit read and the commit itself is not caught. Closing that window needs a base-manifest precondition in lance's commit path. --- rust/lancedb/src/materialized_view.rs | 58 +++- rust/lancedb/src/materialized_view/refresh.rs | 265 +++++++++++++++++- 2 files changed, 307 insertions(+), 16 deletions(-) diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 1d7cbb5e7..b28d52931 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -37,6 +37,13 @@ pub use refresh::{RefreshMaterializedViewResult, RefreshMode}; /// Schema metadata key holding the view definition, as kind-tagged JSON. pub const DEFINITION_META_KEY: &str = "mv.definition"; +/// Schema metadata key holding the view's incarnation: a token minted at each +/// physical creation of a view table, so a view dropped and recreated under +/// the same name and definition is still told apart from the one a caller +/// captured. A view whose metadata was replaced wholesale, or one declared +/// before tokens existed, carries none until its next refresh mints one. +pub const INCARNATION_META_KEY: &str = "mv.incarnation"; + /// Schema metadata key holding the source table version the view was last /// refreshed to. Absent until the first refresh. pub const SOURCE_VERSION_META_KEY: &str = "mv.source_version"; @@ -612,8 +619,17 @@ impl PreparedDeclaration { pub async fn create(self, name: &str) -> Result { let empty: Vec> = vec![]; + // Minted here, not at preparation: a declaration can be cloned and + // create more than one physical table, and each needs its own token. + let incarnation = uuid::Uuid::new_v4().to_string(); + let mut metadata = self.schema.metadata().clone(); + metadata.insert(INCARNATION_META_KEY.to_string(), incarnation.clone()); + let schema = Arc::new(ArrowSchema::new_with_metadata( + self.schema.fields().clone(), + metadata, + )); let reader: Box = - Box::new(arrow_array::RecordBatchIterator::new(empty, self.schema)); + Box::new(arrow_array::RecordBatchIterator::new(empty, schema)); let mut request = CreateTableRequest::new(name.to_string(), Box::new(reader)); let write_params = request .write_options @@ -648,6 +664,7 @@ impl PreparedDeclaration { Ok(MaterializedView { table, definition: self.definition, + incarnation: Some(incarnation), }) } } @@ -878,6 +895,7 @@ impl CreateMaterializedViewBuilder { pub struct MaterializedView { table: Table, definition: MaterializedViewDefinition, + incarnation: Option, } impl MaterializedView { @@ -893,8 +911,13 @@ impl MaterializedView { }); } let schema = table.schema().await?; + let incarnation = schema.metadata().get(INCARNATION_META_KEY).cloned(); match materialized_view_kind(schema.metadata())? { - Some(MaterializedViewKind::Select(definition)) => Ok(Self { table, definition }), + Some(MaterializedViewKind::Select(definition)) => Ok(Self { + table, + definition, + incarnation, + }), Some(MaterializedViewKind::Unrecognized { kind }) => Err(Error::NotSupported { message: format!( "materialized view '{}' is defined by '{kind}', which this version of \ @@ -923,6 +946,13 @@ impl MaterializedView { &self.definition } + /// The view's incarnation token as of when this handle was opened; see + /// [`RefreshMaterializedViewBuilder::expect_incarnation`]. `None` for a + /// view that has none yet (see [`INCARNATION_META_KEY`]). + pub fn incarnation(&self) -> Option<&str> { + self.incarnation.as_deref() + } + /// Recompute the view from its source. /// /// By default the refresh is incremental when the source's changes can be @@ -943,6 +973,7 @@ impl MaterializedView { view: self.clone(), full: false, source_version: None, + expected_incarnation: None, } } } @@ -952,6 +983,7 @@ pub struct RefreshMaterializedViewBuilder { view: MaterializedView, full: bool, source_version: Option, + expected_incarnation: Option, } impl RefreshMaterializedViewBuilder { @@ -967,8 +999,28 @@ impl RefreshMaterializedViewBuilder { self } + /// Refresh only if the view is still the incarnation that minted `token` + /// (see [`MaterializedView::incarnation`]): a refresh requested against + /// one declaration must not land in a view dropped and recreated since, + /// even under the same name and definition. + /// + /// Best effort. The token is read from the latest stored manifest before + /// planning and again immediately before every commit, but it is not part + /// of the commit's own condition, so a recreation that lands between that + /// final read and the commit is not caught. + pub fn expect_incarnation(mut self, token: impl Into) -> Self { + self.expected_incarnation = Some(token.into()); + self + } + pub async fn execute(self) -> Result { - refresh::execute_refresh(&self.view.table, self.full, self.source_version).await + refresh::execute_refresh( + &self.view.table, + self.full, + self.source_version, + self.expected_incarnation.as_deref(), + ) + .await } } diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index 24d828e01..735751c27 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -46,8 +46,8 @@ use lance_table::format::Fragment; use serde::{Deserialize, Serialize}; use super::{ - MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY, SOURCE_ROW_ID_COLUMN, - SOURCE_VERSION_META_KEY, + INCARNATION_META_KEY, MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY, + SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY, }; use crate::database::OpenTableRequest; use crate::table::{NativeTable, NativeTableExt, Table}; @@ -108,6 +108,7 @@ pub(crate) async fn execute_refresh( view: &Table, full: bool, pinned: Option, + expected_incarnation: Option<&str>, ) -> Result { let view_native = view.as_native().ok_or_else(|| Error::NotSupported { message: "materialized views are supported only on local tables".into(), @@ -122,6 +123,8 @@ pub(crate) async fn execute_refresh( view_native.dataset.reload().await?; let view_ds = view_native.dataset.get().await?.as_ref().clone(); + ensure_incarnation(&view_ds, expected_incarnation, view.name()).await?; + // The definition a handle cached at open may since have been replaced; // what refresh executes and what it stamps must be one generation. let definition = match super::materialized_view_kind(&view_ds.schema().metadata)? { @@ -240,6 +243,7 @@ pub(crate) async fn execute_refresh( increment, definition, watermark, + expected_incarnation, ) .await?; match reconciled { @@ -253,6 +257,7 @@ pub(crate) async fn execute_refresh( source_version, source_ts, definition, + expected_incarnation, ) .await } @@ -266,6 +271,7 @@ pub(crate) async fn execute_refresh( source_version, source_ts, definition, + expected_incarnation, ) .await } @@ -595,6 +601,7 @@ async fn incremental( increment: Increment, definition: &MaterializedViewDefinition, watermark: Option, + expected_incarnation: Option<&str>, ) -> Result> { let new_fragments = increment.appended; let watermark_version = watermark.unwrap_or(0); @@ -671,15 +678,35 @@ async fn incremental( }; let nothing_to_add = (new_fragments.is_empty() && !updated_rows) || remaining == Some(0); if nothing_to_add && eviction.is_none() { - result.version = - stamp_watermark(view_native, view_ds.clone(), source_version, source_ts).await?; + result.version = stamp_watermark( + view_native, + view_ds.clone(), + source_version, + source_ts, + expected_incarnation, + ) + .await?; return Ok(Some(result)); } // Rows left but none arrive: the removals still have to be published. if nothing_to_add { let filter = refresh_filter(&empty_keys(view_ds)?)?; - let published = publish(view_ds, eviction, Vec::new(), Some(filter)).await?; - result.version = stamp_watermark(view_native, published, source_version, source_ts).await?; + let published = publish( + view_ds, + eviction, + Vec::new(), + Some(filter), + expected_incarnation, + ) + .await?; + result.version = stamp_watermark( + view_native, + published, + source_version, + source_ts, + expected_incarnation, + ) + .await?; return Ok(Some(result)); } @@ -737,12 +764,20 @@ async fn incremental( eviction, Vec::new(), Some(refresh_filter(&empty_keys(view_ds)?)?), + expected_incarnation, ) .await? } else { view_ds.clone() }; - result.version = stamp_watermark(view_native, published, source_version, source_ts).await?; + result.version = stamp_watermark( + view_native, + published, + source_version, + source_ts, + expected_incarnation, + ) + .await?; return Ok(Some(result)); }; let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( @@ -775,9 +810,23 @@ async fn incremental( }); }; let filter = refresh_filter(&keys)?; - let appended = publish(view_ds, eviction, new_fragments, Some(filter)).await?; + let appended = publish( + view_ds, + eviction, + new_fragments, + Some(filter), + expected_incarnation, + ) + .await?; result.rows_written = rows_written.load(Ordering::Relaxed); - result.version = stamp_watermark(view_native, appended, source_version, source_ts).await?; + result.version = stamp_watermark( + view_native, + appended, + source_version, + source_ts, + expected_incarnation, + ) + .await?; Ok(Some(result)) } @@ -788,6 +837,7 @@ async fn rebuild( source_version: u64, source_ts: u128, definition: &MaterializedViewDefinition, + expected_incarnation: Option<&str>, ) -> Result { let rows_written = Arc::new(AtomicU64::new(0)); let schema = Arc::new(ArrowSchema::from(view_ds.schema())); @@ -810,8 +860,16 @@ async fn rebuild( // carries no schema metadata, so it cannot erase a definition update // that raced in the way an overwrite (which adopts its stream's schema) // durably would -- and it must land on the planned generation or abort. - let replaced = replace_retaining_indices(view_ds.clone(), stream, keys).await?; - let version = stamp_watermark(view_native, replaced, source_version, source_ts).await?; + let replaced = + replace_retaining_indices(view_ds.clone(), stream, keys, expected_incarnation).await?; + let version = stamp_watermark( + view_native, + replaced, + source_version, + source_ts, + expected_incarnation, + ) + .await?; Ok(RefreshMaterializedViewResult { mode: RefreshMode::Rebuild, rows_written: rows_written.load(Ordering::Relaxed), @@ -828,11 +886,13 @@ async fn replace_retaining_indices( view_ds: Dataset, stream: SendableRecordBatchStream, keys: Arc>, + expected_incarnation: Option<&str>, ) -> Result { let ds = Arc::new(view_ds); let read_version = ds.version().version; #[cfg(test)] tests::hold_before_publish(ds.uri()).await; + ensure_incarnation(&ds, expected_incarnation, ds.uri()).await?; let removed_fragment_ids: Vec = ds.get_fragments().iter().map(|f| f.id() as u64).collect(); let write_txn = InsertBuilder::new(WriteDestination::Dataset(ds.clone())) @@ -886,6 +946,32 @@ async fn replace_retaining_indices( } /// Record that the view now reflects `source_version`, including the view +/// Refuse to act on a view that is not `expected`'s incarnation, judged from +/// the latest stored manifest. Not a commit condition; see +/// `RefreshMaterializedViewBuilder::expect_incarnation`. +async fn ensure_incarnation(view_ds: &Dataset, expected: Option<&str>, what: &str) -> Result<()> { + let Some(expected) = expected else { + return Ok(()); + }; + let mut latest = view_ds.clone(); + latest.checkout_latest().await?; + match latest.schema().metadata.get(INCARNATION_META_KEY) { + Some(actual) if actual == expected => Ok(()), + Some(_) => Err(Error::Runtime { + message: format!( + "materialized view '{what}' is not the incarnation this refresh was \ + requested for: it was dropped and recreated" + ), + }), + None => Err(Error::Runtime { + message: format!( + "materialized view '{what}' carries no incarnation token: its schema \ + metadata was replaced since the token was captured" + ), + }), + } +} + /// version this very commit produces. The version is predicted and then /// verified; on a mismatch another commit raced in between, and the stamp /// ABORTS rather than certify that commit as the refresh's own generation. @@ -895,10 +981,21 @@ async fn stamp_watermark( mut dataset: Dataset, source_version: u64, source_ts: u128, + expected_incarnation: Option<&str>, ) -> Result { + ensure_incarnation(&dataset, expected_incarnation, dataset.uri()).await?; let predicted = dataset.version().version + 1; + // A view with no token (declared before tokens existed, or its metadata + // replaced wholesale) starts a new incarnation here. + let incarnation = dataset + .schema() + .metadata + .get(INCARNATION_META_KEY) + .cloned() + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); dataset .update_schema_metadata([ + (INCARNATION_META_KEY.to_string(), Some(incarnation)), ( SOURCE_VERSION_META_KEY.to_string(), Some(source_version.to_string()), @@ -1052,12 +1149,14 @@ async fn publish( eviction: Option<(Vec, Vec)>, new_fragments: Vec, keys: Option, + expected_incarnation: Option<&str>, ) -> Result { let planned = view_ds.version().version; #[cfg(test)] tests::hold_before_publish(view_ds.uri()).await; #[cfg(test)] tests::hold_until_peers_planned(); + ensure_incarnation(view_ds, expected_incarnation, view_ds.uri()).await?; let (updated_fragments, removed_fragment_ids) = eviction.unwrap_or_default(); let committed = CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) .execute(Transaction::new( @@ -1830,7 +1929,9 @@ mod tests { ); let staged = eviction.finish().await.unwrap(); assert!(staged.is_some(), "four ids over a chunk of two stage twice"); - publish(&view_ds, staged, Vec::new(), None).await.unwrap(); + publish(&view_ds, staged, Vec::new(), None, None) + .await + .unwrap(); native.dataset.reload().await.unwrap(); assert_eq!(read(view.table(), "x").await, vec![5, 6]); @@ -2486,6 +2587,144 @@ mod tests { assert_eq!(read(view.table(), "twice").await, vec![14]); } + /// A refresh bound to an incarnation refuses a view dropped and recreated + /// since, even under the same name and definition; the recreated view's + /// own token is accepted, and the token survives a refresh's stamp. + #[tokio::test] + async fn test_refresh_refuses_a_recreated_view_incarnation() { + let (conn, _, view) = refreshed_doubled(vec![1]).await; + let token = view.incarnation().unwrap().to_string(); + view.refresh() + .expect_incarnation(&token) + .execute() + .await + .unwrap(); + let reopened = conn.open_materialized_view("doubled").await.unwrap(); + assert_eq!(reopened.incarnation(), Some(token.as_str())); + + conn.drop_table("doubled", &[]).await.unwrap(); + let recreated = doubled_view(&conn).await; + assert_ne!(recreated.incarnation(), Some(token.as_str())); + + let err = recreated + .refresh() + .expect_incarnation(&token) + .execute() + .await + .unwrap_err(); + assert!(err.to_string().contains("dropped and recreated"), "{err}"); + assert_eq!(read(recreated.table(), "twice").await, Vec::::new()); + + recreated + .refresh() + .expect_incarnation(recreated.incarnation().unwrap()) + .execute() + .await + .unwrap(); + assert_eq!(read(recreated.table(), "twice").await, vec![2]); + } + + /// A cloned declaration creates two physical tables; each gets its own + /// token. + #[tokio::test] + async fn test_cloned_declaration_mints_a_fresh_incarnation_per_create() { + let (conn, source) = db_with_source(vec![1]).await; + let prepared = crate::materialized_view::prepare_declaration( + &source, + &[("x".into(), "x".into()), ("twice".into(), "x * 2".into())], + None, + None, + ) + .await + .unwrap(); + let replacement = prepared.clone(); + let first = prepared.create("cloned").await.unwrap(); + let first_token = first.incarnation().unwrap().to_string(); + + conn.drop_table("cloned", &[]).await.unwrap(); + let second = replacement.create("cloned").await.unwrap(); + assert_ne!(second.incarnation(), Some(first_token.as_str())); + } + + /// A recreation that lands after planning but before publication is + /// caught by the pre-commit read: the stale refresh fails and the + /// replacement stays empty under its own token. + #[tokio::test(flavor = "multi_thread")] + async fn test_bound_refresh_cannot_publish_into_a_raced_recreation() { + let _serial = DRIFT_LOCK.lock().await; + let (conn, _) = db_with_source(vec![1]).await; + let view = doubled_view(&conn).await; + let token = view.incarnation().unwrap().to_string(); + let uri = view + .table() + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .uri() + .to_string(); + + *DRIFT_TARGET.lock().unwrap() = Some(uri); + let refreshing = + tokio::spawn(async move { view.refresh().expect_incarnation(token).execute().await }); + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("refresh never reached publication"); + + conn.drop_table("doubled", &[]).await.unwrap(); + let replacement = doubled_view(&conn).await; + let replacement_token = replacement.incarnation().unwrap().to_string(); + DRIFT_RELEASED.notify_one(); + + let result = refreshing.await.unwrap(); + assert!(result.is_err(), "the stale refresh unexpectedly succeeded"); + let reopened = conn.open_materialized_view("doubled").await.unwrap(); + assert_eq!(reopened.incarnation(), Some(replacement_token.as_str())); + assert_eq!(read(reopened.table(), "twice").await, Vec::::new()); + } + + /// Replacing the schema metadata wholesale drops the token. A refresh + /// bound to the old token is refused for that reason, not as a + /// recreation; an unbound refresh mints the view a fresh one. + #[tokio::test] + async fn test_a_view_whose_metadata_was_replaced_starts_a_new_incarnation() { + let (conn, _, view) = refreshed_doubled(vec![1]).await; + let token = view.incarnation().unwrap().to_string(); + let mut metadata = HashMap::new(); + metadata.insert( + crate::materialized_view::DEFINITION_META_KEY.to_string(), + crate::materialized_view::definition_to_metadata(view.definition()).unwrap(), + ); + view.table() + .as_native() + .unwrap() + .replace_schema_metadata(metadata) + .await + .unwrap(); + assert_eq!( + conn.open_materialized_view("doubled") + .await + .unwrap() + .incarnation(), + None + ); + + let err = view + .refresh() + .expect_incarnation(&token) + .execute() + .await + .unwrap_err(); + assert!(err.to_string().contains("no incarnation token"), "{err}"); + + view.refresh().execute().await.unwrap(); + let reopened = conn.open_materialized_view("doubled").await.unwrap(); + assert!(reopened.incarnation().is_some()); + assert_ne!(reopened.incarnation(), Some(token.as_str())); + } + /// In-process refreshes of one view serialize: the loser of the race /// observes the winner's watermark instead of appending the same rows. #[tokio::test(flavor = "multi_thread")] @@ -2528,7 +2767,7 @@ mod tests { let stale = view_native.dataset.get().await.unwrap().as_ref().clone(); view.table().delete("x = 1").await.unwrap(); - let err = stamp_watermark(view_native, stale, 99, 99).await; + let err = stamp_watermark(view_native, stale, 99, 99, None).await; assert!(err.is_err()); let result = view.refresh().execute().await.unwrap(); From 71f85a8d9fab89c238648c9963e7c89e1452e74d Mon Sep 17 00:00:00 2001 From: Lance Release Date: Mon, 24 Aug 2026 21:08:39 +0000 Subject: [PATCH 39/58] =?UTF-8?q?Bump=20version:=200.38.0-beta.6=20?= =?UTF-8?q?=E2=86=92=200.38.0-beta.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index f5f981b2c..bcf714002 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.6" +current_version = "0.38.0-beta.7" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 6c6dd4aea..a60a5ddf1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5402,7 +5402,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.6" +version = "0.38.0-beta.7" dependencies = [ "ahash", "anyhow", @@ -5490,7 +5490,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.6" +version = "0.38.0-beta.7" dependencies = [ "arrow-array", "arrow-buffer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.6" +version = "0.38.0-beta.7" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index d6c285fc3..59ee8a7d6 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.6 + 0.38.0-beta.7 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 23d58124d..b24ad61ab 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.6 + 0.38.0-beta.7 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index a07414ec0..0ee59bcb7 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.6 + 0.38.0-beta.7 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index ead873855..8732d1965 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.6" +version = "0.38.0-beta.7" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index d89eb5ad1..56480c0cb 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 1721c970e..a1102ee29 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 177804dce..695862baf 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 9af443583..9099ee4de 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index c98aa3812..f04c2a884 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 05b1086f5..f9b35af32 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 50432bdea..cc5340105 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 219025534..6604da6cd 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index 63c51a799..66777d60c 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 2e4f8996f..d2ccf3064 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.6" +version = "0.38.0-beta.7" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index d32d88ecd..071c876d9 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.6" +version = "0.38.0-beta.7" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 5013c176dd9a664813888c8bd8cf99161c047ec7 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:14:33 +0800 Subject: [PATCH 40/58] fix: pin remote snapshots during permutation construction (#4022) Fixes #4015 ## Root cause PermutationBuilder issued count_rows and the projected row-ID scan through an unpinned RemoteTable handle. Each request could independently resolve latest, so a concurrent table update could make the count and scanned rows come from different snapshots. ## Fix - add a backend hook for obtaining an independent handle pinned to the currently selected version - resolve latest once for remote tables while preserving an explicit checkout and leaving the caller handle unchanged - build the count, filtered projection, and scan from that pinned handle while retaining native-table behavior - add a remote mock regression that advances latest between count and scan and covers explicit checkout preservation ## Validation - cargo test --quiet --features remote -p lancedb --lib test_remote_permutation_builder_pins_snapshot - cargo test --quiet --features remote -p lancedb --lib dataloader::permutation::builder::tests - cargo check --quiet --features remote --tests --examples - cargo clippy --quiet --features remote --tests --examples - cargo fmt --all -- --check Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo --- .../src/dataloader/permutation/builder.rs | 128 +++++++++++++++++- rust/lancedb/src/remote/table.rs | 12 ++ rust/lancedb/src/table.rs | 8 ++ 3 files changed, 146 insertions(+), 2 deletions(-) diff --git a/rust/lancedb/src/dataloader/permutation/builder.rs b/rust/lancedb/src/dataloader/permutation/builder.rs index a3700d0ef..641c191d5 100644 --- a/rust/lancedb/src/dataloader/permutation/builder.rs +++ b/rust/lancedb/src/dataloader/permutation/builder.rs @@ -239,7 +239,20 @@ impl PermutationBuilder { } /// Builds the permutation table and stores it in the given database. - pub async fn build(self) -> Result
{ + pub async fn build(mut self) -> Result
{ + // Remote tables resolve latest independently for each request. Use a + // separate pinned handle so count, projection, and scan all refer to one + // snapshot without changing the caller's table checkout state. Native + // tables return `None` here and retain their existing behavior. + if let Some(snapshot) = self + .base_table + .base_table() + .snapshot_at_current_version() + .await? + { + self.base_table = Table::from(snapshot); + } + // Unflushed rows have no row id, so a permutation cannot address them. match self.base_table.base_table().get_lsm_write_spec().await { Ok(Some(_)) => { @@ -258,7 +271,6 @@ impl PermutationBuilder { // First pass, apply filter and load row ids. `Shuffler` permutes positions, so // every rank must scan the rows in the same order to build the same permutation. - // TODO: pin the version resolved here; remote does not implement Lazy. let mut rows = self.base_table.query().select(Select::columns(&[ROW_ID])); if let Some(filter) = &self.config.filter { @@ -409,6 +421,118 @@ mod tests { assert!(table.base_table().scan_order_is_deterministic()); } + #[cfg(feature = "remote")] + #[tokio::test] + async fn test_remote_permutation_builder_pins_snapshot() { + use std::sync::{ + Mutex, + atomic::{AtomicU64, Ordering}, + }; + + use arrow_array::{RecordBatch, UInt64Array}; + use arrow_schema::{DataType, Field, Schema}; + + let row_ids = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + ROW_ID, + DataType::UInt64, + false, + )])), + vec![Arc::new(UInt64Array::from(vec![100]))], + ) + .unwrap(); + let mut query_body = Vec::new(); + { + let mut writer = + arrow_ipc::writer::FileWriter::try_new(&mut query_body, &row_ids.schema()).unwrap(); + writer.write(&row_ids).unwrap(); + writer.finish().unwrap(); + } + + let latest = Arc::new(AtomicU64::new(7)); + let expected_snapshot = Arc::new(AtomicU64::new(7)); + let planning_versions = Arc::new(Mutex::new(Vec::new())); + let latest_ref = latest.clone(); + let expected_snapshot_ref = expected_snapshot.clone(); + let planning_versions_ref = planning_versions.clone(); + let table = Table::new_with_handler("remote_base", move |request| { + let path = request.url().path(); + let body = request + .body() + .and_then(|body| body.as_bytes()) + .map(|body| serde_json::from_slice::(body).unwrap()); + + match path { + "/v1/table/remote_base/describe/" => { + let requested = body.as_ref().and_then(|body| body["version"].as_u64()); + let version = requested.unwrap_or_else(|| latest_ref.load(Ordering::SeqCst)); + http::Response::builder() + .status(200) + .body( + format!(r#"{{"version":{version},"schema":{{"fields":[]}}}}"#) + .into_bytes(), + ) + .unwrap() + } + "/v1/table/remote_base/get_lsm_write_spec/" => http::Response::builder() + .status(200) + .body(br#"{"lsm_write_spec":null}"#.to_vec()) + .unwrap(), + "/v1/table/remote_base/count_rows/" => { + let body = body.unwrap(); + let version = body["version"].as_u64().unwrap(); + assert_eq!(version, expected_snapshot_ref.load(Ordering::SeqCst)); + assert_eq!(body["predicate"], "value > 0"); + planning_versions_ref.lock().unwrap().push(version); + + // Simulate a concurrent append after count_rows. An unpinned + // scan would now resolve version 8 and include different rows. + latest_ref.store(8, Ordering::SeqCst); + http::Response::builder() + .status(200) + .body(b"1".to_vec()) + .unwrap() + } + "/v1/table/remote_base/query/" => { + let body = body.unwrap(); + let version = body["version"].as_u64().unwrap(); + assert_eq!(version, expected_snapshot_ref.load(Ordering::SeqCst)); + assert_eq!(body["filter"], "value > 0"); + assert_eq!(body["columns"], serde_json::json!([ROW_ID])); + planning_versions_ref.lock().unwrap().push(version); + http::Response::builder() + .status(200) + .header("content-type", "application/vnd.apache.arrow.file") + .body(query_body.clone()) + .unwrap() + } + _ => panic!("unexpected request: {path}"), + } + }); + + let permutation = PermutationBuilder::new(table.clone()) + .with_filter("value > 0".to_string()) + .build() + .await + .unwrap(); + assert_eq!(permutation.count_rows(None).await.unwrap(), 1); + + // Building uses a separate handle and must not pin the caller's table. + assert_eq!(table.version().await.unwrap(), 8); + + // An explicit checkout is copied as-is and remains checked out afterward. + expected_snapshot.store(6, Ordering::SeqCst); + table.checkout(6).await.unwrap(); + let permutation = PermutationBuilder::new(table.clone()) + .with_filter("value > 0".to_string()) + .build() + .await + .unwrap(); + assert_eq!(permutation.count_rows(None).await.unwrap(), 1); + assert_eq!(table.version().await.unwrap(), 6); + assert_eq!(*planning_versions.lock().unwrap(), vec![7, 7, 6, 6]); + } + #[tokio::test] async fn test_permutation_builder() { let temp_dir = tempfile::tempdir().unwrap(); diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index c6b078282..633d0ffce 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -1775,6 +1775,18 @@ impl BaseTable for RemoteTable { Ok(()) } + async fn snapshot_at_current_version(&self) -> Result>> { + // A checked-out handle already names its snapshot. Otherwise resolve + // latest exactly once before creating the independent pinned handle. + let version = match self.current_version().await { + Some(version) => version, + None => self.describe().await?.version, + }; + + let snapshot = self.with_branch(self.branch.clone()); + *snapshot.version.write().await = Some(version); + Ok(Some(Arc::new(snapshot))) + } async fn restore(&self) -> Result<()> { let mut request = self .client diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 41551bd37..356c87bbe 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -792,6 +792,14 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { async fn checkout_tag(&self, tag: &str) -> Result<()>; /// Checkout the latest version of the table. async fn checkout_latest(&self) -> Result<()>; + /// Return an independent handle pinned to the version currently selected. + /// + /// Backends that can advance between requests should override this for + /// multi-request operations that need snapshot consistency. Backends whose + /// existing handles already provide the desired behavior return `None`. + async fn snapshot_at_current_version(&self) -> Result>> { + Ok(None) + } /// Whether repeated identical scans return rows in the same order. /// /// Callers that assign meaning to a row's position must order the results From fce45ba9fc5254d3e7fff8fc90c89b99706f120c Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 24 Aug 2026 16:25:14 -0700 Subject: [PATCH 41/58] feat(nodejs): add listTables, deprecate tableNames (#4041) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `table_names` is being replaced by `list_tables` across the SDKs, but TypeScript only had `tableNames`. This PR adds `listTables`, which returns a page of table names together with the token that resumes after it, and marks `tableNames` and `TableNamesOptions` deprecated in favor of it. It binds the `Connection::list_tables` that already exists, so nothing in the Rust API changes and nothing existing breaks. `pageToken` is documented as opaque rather than as a table name, since what resumes a listing is the database's to decide — that keeps callers off a detail that is going to change. Stacked on #4040, which fixes a table being dropped at every page boundary. The page-walking test here needs that fix to pass. Review the last commit only until #4040 lands. ## Example ```ts const names = []; let pageToken = undefined; do { const page = await conn.listTables({ pageToken, limit: 100 }); names.push(...page.tables); pageToken = page.pageToken; } while (pageToken); ``` A namespace can be listed by passing its path first, mirroring `tableNames`: ```ts const page = await conn.listTables(["analytics"], { limit: 100 }); ``` Co-authored-by: Claude Opus 5 (1M context) --- docs/src/js/classes/Connection.md | 74 ++++++++++++++++- docs/src/js/globals.md | 2 + docs/src/js/interfaces/ListTablesOptions.md | 34 ++++++++ docs/src/js/interfaces/ListTablesResponse.md | 23 ++++++ docs/src/js/interfaces/TableNamesOptions.md | 11 ++- nodejs/__test__/connection.test.ts | 69 +++++++++++++++- nodejs/lancedb/connection.ts | 85 ++++++++++++++++++++ nodejs/lancedb/index.ts | 2 + nodejs/src/connection.rs | 34 ++++++++ 9 files changed, 329 insertions(+), 5 deletions(-) create mode 100644 docs/src/js/interfaces/ListTablesOptions.md create mode 100644 docs/src/js/interfaces/ListTablesResponse.md diff --git a/docs/src/js/classes/Connection.md b/docs/src/js/classes/Connection.md index 92cfd2568..1c5abd89f 100644 --- a/docs/src/js/classes/Connection.md +++ b/docs/src/js/classes/Connection.md @@ -584,6 +584,70 @@ Child namespace names and *** +### listTables() + +#### listTables(options) + +```ts +abstract listTables(options?): Promise +``` + +List a page of the tables in this database. + +To retrieve the tables after the page, pass the `pageToken` the response +carries back in. A page can be shorter than `limit` without being the last +one, so walk until a response carries no page token: + +```ts +const names = []; +let pageToken = undefined; +do { + const page = await conn.listTables({ pageToken, limit: 100 }); + names.push(...page.tables); + pageToken = page.pageToken; +} while (pageToken); +``` + +##### Parameters + +* **options?**: `Partial`<[`ListTablesOptions`](../interfaces/ListTablesOptions.md)> + Pagination options + (`pageToken`, `limit`). + +##### Returns + +`Promise`<[`ListTablesResponse`](../interfaces/ListTablesResponse.md)> + +A page of table names and an + optional token for the tables after it. + +#### listTables(namespacePath, options) + +```ts +abstract listTables(namespacePath?, options?): Promise +``` + +List a page of the tables in this database. + +##### Parameters + +* **namespacePath?**: `string`[] + The namespace path to list tables from + (defaults to root namespace) + +* **options?**: `Partial`<[`ListTablesOptions`](../interfaces/ListTablesOptions.md)> + Pagination options + (`pageToken`, `limit`). + +##### Returns + +`Promise`<[`ListTablesResponse`](../interfaces/ListTablesResponse.md)> + +A page of table names and an + optional token for the tables after it. + +*** + ### openMaterializedView() ```ts @@ -660,7 +724,7 @@ a "not supported" error. *** -### tableNames() +### ~~tableNames()~~ #### tableNames(options) @@ -682,6 +746,10 @@ Tables will be returned in lexicographical order. `Promise`<`string`[]> +##### Deprecated + +Use [Connection.listTables](Connection.md#listtables) instead. + #### tableNames(namespacePath, options) ```ts @@ -704,3 +772,7 @@ Tables will be returned in lexicographical order. ##### Returns `Promise`<`string`[]> + +##### Deprecated + +Use [Connection.listTables](Connection.md#listtables) instead. diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 6a8f644eb..e0635ab65 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -100,6 +100,8 @@ - [JobInfo](interfaces/JobInfo.md) - [ListNamespacesOptions](interfaces/ListNamespacesOptions.md) - [ListNamespacesResponse](interfaces/ListNamespacesResponse.md) +- [ListTablesOptions](interfaces/ListTablesOptions.md) +- [ListTablesResponse](interfaces/ListTablesResponse.md) - [LsmStats](interfaces/LsmStats.md) - [LsmWriteSpec](interfaces/LsmWriteSpec.md) - [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md) diff --git a/docs/src/js/interfaces/ListTablesOptions.md b/docs/src/js/interfaces/ListTablesOptions.md new file mode 100644 index 000000000..52ace47e5 --- /dev/null +++ b/docs/src/js/interfaces/ListTablesOptions.md @@ -0,0 +1,34 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / ListTablesOptions + +# Interface: ListTablesOptions + +## Properties + +### limit? + +```ts +optional limit: number; +``` + +An upper bound on how many tables to return. + +A page may hold fewer than this and still not be the last one, so keep +going while the response carries a page token rather than while pages are +full. + +*** + +### pageToken? + +```ts +optional pageToken: string; +``` + +Token from a previous response, to resume listing where it left off. + +The token is opaque: it carries whatever the database needs to resume, and +callers should not construct or interpret one. diff --git a/docs/src/js/interfaces/ListTablesResponse.md b/docs/src/js/interfaces/ListTablesResponse.md new file mode 100644 index 000000000..76cac2b23 --- /dev/null +++ b/docs/src/js/interfaces/ListTablesResponse.md @@ -0,0 +1,23 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / ListTablesResponse + +# Interface: ListTablesResponse + +## Properties + +### pageToken? + +```ts +optional pageToken: string; +``` + +*** + +### tables + +```ts +tables: string[]; +``` diff --git a/docs/src/js/interfaces/TableNamesOptions.md b/docs/src/js/interfaces/TableNamesOptions.md index 45fa3d1b0..9254e9fa7 100644 --- a/docs/src/js/interfaces/TableNamesOptions.md +++ b/docs/src/js/interfaces/TableNamesOptions.md @@ -4,11 +4,16 @@ [@lancedb/lancedb](../globals.md) / TableNamesOptions -# Interface: TableNamesOptions +# Interface: ~~TableNamesOptions~~ + +## Deprecated + +Use [ListTablesOptions](ListTablesOptions.md) with [Connection.listTables](../classes/Connection.md#listtables) +instead. ## Properties -### limit? +### ~~limit?~~ ```ts optional limit: number; @@ -18,7 +23,7 @@ An optional limit to the number of results to return. *** -### startAfter? +### ~~startAfter?~~ ```ts optional startAfter: string; diff --git a/nodejs/__test__/connection.test.ts b/nodejs/__test__/connection.test.ts index af471b478..816f8c3de 100644 --- a/nodejs/__test__/connection.test.ts +++ b/nodejs/__test__/connection.test.ts @@ -4,7 +4,13 @@ import { readdirSync } from "fs"; import { Field, Float64, Schema } from "apache-arrow"; import * as tmp from "tmp"; -import { Connection, Table, connect, connectNamespace } from "../lancedb"; +import { + Connection, + ListTablesResponse, + Table, + connect, + connectNamespace, +} from "../lancedb"; import { LocalTable } from "../lancedb/table"; describe("when connecting", () => { @@ -47,6 +53,7 @@ describe("given a connection", () => { await db.close(); expect(db.isOpen()).toBe(false); await expect(db.tableNames()).rejects.toThrow("Connection is closed"); + await expect(db.listTables()).rejects.toThrow("Connection is closed"); await expect(db.renameTable("a", "b")).rejects.toThrow( "Connection is closed", ); @@ -129,6 +136,66 @@ describe("given a connection", () => { expect(tables).toEqual(["b", "c"]); }); + it("should respect limit and page token when listing tables", async () => { + const db = await connect(tmpDir.name); + + await db.createTable("b", [{ id: 1 }]); + await db.createTable("a", [{ id: 1 }]); + await db.createTable("c", [{ id: 1 }]); + + const all = await db.listTables(); + expect(all.tables).toEqual(["a", "b", "c"]); + expect(all.pageToken).toBeUndefined(); + + const first = await db.listTables({ limit: 1 }); + expect(first.tables).toEqual(["a"]); + expect(first.pageToken).toBeDefined(); + + const second = await db.listTables({ + limit: 1, + pageToken: first.pageToken, + }); + expect(second.tables).toEqual(["b"]); + }); + + it("should visit every table exactly once when walking pages", async () => { + const db = await connect(tmpDir.name); + + const created = ["a", "b", "c", "d", "e"]; + for (const name of created) { + await db.createTable(name, [{ id: 1 }]); + } + + const seen: string[] = []; + let pageToken: string | undefined = undefined; + do { + const page: ListTablesResponse = await db.listTables({ + limit: 2, + pageToken, + }); + seen.push(...page.tables); + pageToken = page.pageToken; + } while (pageToken); + + expect(seen).toEqual(created); + }); + + it("should list tables in a namespace", async () => { + const db = await connect(tmpDir.name, { + // biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust + namespaceClientProperties: { manifest_enabled: "true" }, + }); + await db.createNamespace(["child"]); + await db.createTable("nested", [{ id: 1 }], ["child"]); + + await expect(db.listTables(["child"])).resolves.toEqual( + expect.objectContaining({ tables: ["nested"] }), + ); + await expect(db.listTables()).resolves.toEqual( + expect.objectContaining({ tables: [] }), + ); + }); + it("should create tables in v2 mode", async () => { const db = await connect(tmpDir.name); const data = [...Array(10000).keys()].map((i) => ({ id: i })); diff --git a/nodejs/lancedb/connection.ts b/nodejs/lancedb/connection.ts index e5528f8c6..263a338ab 100644 --- a/nodejs/lancedb/connection.ts +++ b/nodejs/lancedb/connection.ts @@ -31,12 +31,14 @@ import type { JobDescription, JobInfo, ListNamespacesResponse, + ListTablesResponse, } from "./native"; export type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, ListNamespacesResponse, + ListTablesResponse, }; import { sanitizeTable } from "./sanitize"; import { LocalTable, Table } from "./table"; @@ -134,6 +136,10 @@ export interface OpenTableOptions { indexCacheSize?: number; } +/** + * @deprecated Use {@link ListTablesOptions} with {@link Connection.listTables} + * instead. + */ export interface TableNamesOptions { /** * If present, only return names that come lexicographically after the @@ -147,6 +153,24 @@ export interface TableNamesOptions { limit?: number; } +export interface ListTablesOptions { + /** + * Token from a previous response, to resume listing where it left off. + * + * The token is opaque: it carries whatever the database needs to resume, and + * callers should not construct or interpret one. + */ + pageToken?: string; + /** + * An upper bound on how many tables to return. + * + * A page may hold fewer than this and still not be the last one, so keep + * going while the response carries a page token rather than while pages are + * full. + */ + limit?: number; +} + export interface ListNamespacesOptions { /** Token from a previous response for pagination. */ pageToken?: string; @@ -231,6 +255,7 @@ export abstract class Connection { * @param {Partial} options - options to control the * paging / start point (backwards compatibility) * + * @deprecated Use {@link Connection.listTables} instead. */ abstract tableNames(options?: Partial): Promise; /** @@ -241,12 +266,53 @@ export abstract class Connection { * @param {Partial} options - options to control the * paging / start point * + * @deprecated Use {@link Connection.listTables} instead. */ abstract tableNames( namespacePath?: string[], options?: Partial, ): Promise; + /** + * List a page of the tables in this database. + * + * To retrieve the tables after the page, pass the `pageToken` the response + * carries back in. A page can be shorter than `limit` without being the last + * one, so walk until a response carries no page token: + * + * ```ts + * const names = []; + * let pageToken = undefined; + * do { + * const page = await conn.listTables({ pageToken, limit: 100 }); + * names.push(...page.tables); + * pageToken = page.pageToken; + * } while (pageToken); + * ``` + * + * @param {Partial} options - Pagination options + * (`pageToken`, `limit`). + * @returns {Promise} A page of table names and an + * optional token for the tables after it. + */ + abstract listTables( + options?: Partial, + ): Promise; + /** + * List a page of the tables in this database. + * + * @param {string[]} namespacePath - The namespace path to list tables from + * (defaults to root namespace) + * @param {Partial} options - Pagination options + * (`pageToken`, `limit`). + * @returns {Promise} A page of table names and an + * optional token for the tables after it. + */ + abstract listTables( + namespacePath?: string[], + options?: Partial, + ): Promise; + /** * Open a table in the database. * @param {string} name - The name of the table @@ -601,6 +667,25 @@ export class LocalConnection extends Connection { return await this.inner.listMaterializedViews(); } + async listTables( + namespacePathOrOptions?: string[] | Partial, + options?: Partial, + ): Promise { + // Detect if first argument is namespacePath array or options object + const namespacePath = Array.isArray(namespacePathOrOptions) + ? namespacePathOrOptions + : undefined; + const listTablesOptions = Array.isArray(namespacePathOrOptions) + ? options + : namespacePathOrOptions; + + return this.inner.listTables( + namespacePath ?? [], + listTablesOptions?.pageToken, + listTablesOptions?.limit, + ); + } + async openTable( name: string, namespacePath?: string[], diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index da13b434f..ebc8cda8d 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -81,11 +81,13 @@ export { Connection, CreateTableOptions, TableNamesOptions, + ListTablesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, + ListTablesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, diff --git a/nodejs/src/connection.rs b/nodejs/src/connection.rs index 44eb68f32..5cf676256 100644 --- a/nodejs/src/connection.rs +++ b/nodejs/src/connection.rs @@ -17,6 +17,7 @@ use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, conne use lance_namespace::models::{ CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest, + ListTablesRequest, }; use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema}; @@ -36,6 +37,12 @@ pub struct ListNamespacesResponse { pub page_token: Option, } +#[napi(object)] +pub struct ListTablesResponse { + pub tables: Vec, + pub page_token: Option, +} + #[napi(object)] pub struct CreateNamespaceResponse { pub properties: Option>, @@ -206,6 +213,33 @@ impl Connection { op.execute().await.default_error() } + /// List a page of tables in the database. + #[napi(catch_unwind)] + pub async fn list_tables( + &self, + namespace_path: Option>, + page_token: Option, + limit: Option, + ) -> napi::Result { + let request = ListTablesRequest { + // The root namespace is an empty path, not an absent one: a namespace-backed + // database rejects a request that names no namespace. + id: Some(namespace_path.unwrap_or_default()), + page_token, + limit: limit.map(|limit| i32::try_from(limit).unwrap_or(i32::MAX)), + ..Default::default() + }; + let response = self + .get_inner()? + .list_tables(request) + .await + .default_error()?; + Ok(ListTablesResponse { + tables: response.tables, + page_token: response.page_token, + }) + } + /// Create table from a Apache Arrow IPC (file) buffer. /// /// Parameters: From c1a8c3f089fbab95883bd58d9a568d4c30d41074 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:41:05 -0700 Subject: [PATCH 42/58] fix(node): validate inferred types across records (#3786) ## Summary - compare inferred Arrow types by their semantic representation across records - throw the schema inference error when a later record has an incompatible type - cover compatible and incompatible multi-record inference across supported Arrow versions ## Root cause Schema inference compared newly allocated Arrow DataType objects by identity, so equivalent inferred types did not compare equal. The mismatch path also constructed an Error without throwing it, which silently accepted incompatible values. ## Validation - pnpm test __test__/arrow.test.ts --runInBand (176 tests passed) - pnpm lint - pnpm build - pnpm run docs Fixes #3781 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/arrow.test.ts | 131 ++++++++ nodejs/lancedb/arrow.ts | 340 ++------------------ nodejs/lancedb/arrow_type.ts | 40 +++ nodejs/lancedb/schema.ts | 566 ++++++++++++++++++++++++++++++++++ 4 files changed, 770 insertions(+), 307 deletions(-) create mode 100644 nodejs/lancedb/arrow_type.ts create mode 100644 nodejs/lancedb/schema.ts diff --git a/nodejs/__test__/arrow.test.ts b/nodejs/__test__/arrow.test.ts index 160f4d5ef..c5bbbf169 100644 --- a/nodejs/__test__/arrow.test.ts +++ b/nodejs/__test__/arrow.test.ts @@ -515,6 +515,137 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( ); }); + it("will allow matching inferred types across records", function () { + expect(() => + makeArrowTable([{ value: 1 }, { value: 2 }]), + ).not.toThrow(); + }); + + it("will reject mismatched inferred types across records", function () { + expect(() => makeArrowTable([{ value: 1 }, { value: "two" }])).toThrow( + "Failed to infer schema for data. Previously inferred type Float64 but found Utf8 for field value at row 1. Consider providing an explicit schema.", + ); + }); + + it("will ignore generated dictionary IDs when comparing inferred types", function () { + const table = makeArrowTable([{ str: "a" }, { str: "b" }], { + dictionaryEncodeStrings: true, + }); + + expect(table.getChild("str")?.toJSON()).toEqual(["a", "b"]); + }); + + it("will preserve null values without treating them as type mismatches", function () { + for (const records of [ + [{ vector: [1, 2, 3] }, { vector: null }], + [{ vector: null }, { vector: [1, 2, 3] }], + ]) { + const table = makeArrowTable(records); + + expect(table.numRows).toBe(2); + expect(table.getChild("vector")?.nullCount).toBe(1); + } + }); + + it("will preserve empty variable-size lists", function () { + for (const records of [ + [{ items: [1] }, { items: [] }], + [{ items: [] }, { items: [1] }], + ]) { + const table = makeArrowTable(records); + expect( + table + .getChild("items") + ?.toJSON() + .map((value) => value.toJSON()), + ).toEqual(records.map((record) => record.items)); + } + }); + + it("will propagate deferred evidence through nested lists", function () { + for (const records of [ + [{ items: [1] }, { items: [null] }], + [{ items: [null] }, { items: [1] }], + [{ items: [null, 1] }, { items: [2, null] }], + ]) { + const table = makeArrowTable(records); + expect( + table + .getChild("items") + ?.toJSON() + .map((value) => value.toJSON()), + ).toEqual(records.map((record) => record.items)); + } + + const nestedRecords = [{ items: [[1]] }, { items: [[null]] }]; + const nestedTable = makeArrowTable(nestedRecords); + expect( + nestedTable + .getChild("items") + ?.toJSON() + .map((value) => + value + .toJSON() + .map((nestedValue: { toJSON: () => unknown[] }) => + nestedValue.toJSON(), + ), + ), + ).toEqual(nestedRecords.map((record) => record.items)); + }); + + it("will reject incompatible deferred evidence within a list", function () { + for (const items of [ + [[], 1], + [1, []], + [[null], 1], + [1, [null]], + ]) { + expect(() => makeArrowTable([{ items }])).toThrow( + "Failed to infer data type for field items at row 0.", + ); + } + }); + + it("will reject empty fixed-size lists", function () { + expect(() => + makeArrowTable([{ vector: [1, 2, 3] }, { vector: [] }]), + ).toThrow( + "Failed to infer schema for data. Previously inferred type FixedSizeList[3] but found List[0] for field vector at row 1.", + ); + }); + + it("will reject inferred leaf and branch shape changes", function () { + expect(() => + makeArrowTable([{ value: 1 }, { value: { nested: 2 } }]), + ).toThrow( + "Failed to infer schema for data. Previously inferred type Float64 but found Struct for field value at row 1.", + ); + expect(() => + makeArrowTable([{ value: { nested: 1 } }, { value: 2 }]), + ).toThrow( + "Failed to infer schema for data. Previously inferred type Struct but found Float64 for field value at row 1.", + ); + }); + + it("will allow null values around inferred struct values", function () { + for (const { records, nullIndex } of [ + { + records: [{ value: null }, { value: { nested: 2 } }], + nullIndex: 0, + }, + { + records: [{ value: { nested: 1 } }, { value: null }], + nullIndex: 1, + }, + ]) { + const table = makeArrowTable(records); + const values = table.getChild("value"); + + expect(values?.nullCount).toBe(1); + expect(values?.get(nullIndex)).toBeNull(); + } + }); + it("will allow a schema to be provided", async function () { await checkTableCreation( async (records, _, schema) => diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts index 8b388d593..b52ab50ef 100644 --- a/nodejs/lancedb/arrow.ts +++ b/nodejs/lancedb/arrow.ts @@ -5,7 +5,6 @@ import { Data as ArrowData, Table as ArrowTable, Binary, - Bool, BufferType, DataType, DateUnit, @@ -18,12 +17,7 @@ import { FixedSizeList, Float, Float32, - Float64, Int, - Int8, - Int16, - Int32, - Int64, LargeBinary, List, Null, @@ -36,17 +30,16 @@ import { Struct, Timestamp, Type, - Uint8, - Uint16, - Uint32, Utf8, Vector, makeVector as arrowMakeVector, + util as arrowUtil, vectorFromArray as badVectorFromArray, makeBuilder, makeData, } from "apache-arrow"; import { Buffers } from "apache-arrow/data"; +import { typedArrayToArrowType } from "./arrow_type"; import { type EmbeddingFunction } from "./embedding/embedding_function"; import { EmbeddingFunctionConfig, @@ -59,14 +52,7 @@ import { sanitizeTable, sanitizeType, } from "./sanitize"; - -/** - * Check if a field name indicates a vector column. - */ -function nameSuggestsVectorColumn(fieldName: string): boolean { - const nameLower = fieldName.toLowerCase(); - return nameLower.includes("vector") || nameLower.includes("embedding"); -} +import { inferSchema } from "./schema"; export * from "apache-arrow"; export type SchemaLike = @@ -459,110 +445,6 @@ export function makeArrowTable( return new ArrowTable(inferredSchema, finalColumns); } -function inferSchema( - data: Array>, - schema: Schema | undefined, - opts: MakeArrowTableOptions, -): Schema { - // We will collect all fields we see in the data. - const pathTree = new PathTree(); - - for (const [rowI, row] of data.entries()) { - for (const [path, value] of rowPathsAndValues(row)) { - if (!pathTree.has(path)) { - // First time seeing this field. - if (schema !== undefined) { - const field = getFieldForPath(schema, path); - if (field === undefined) { - throw new Error( - `Found field not in schema: ${path.join(".")} at row ${rowI}`, - ); - } else { - pathTree.set(path, field.type); - } - } else { - const inferredType = inferType(value, path, opts); - if (inferredType === undefined) { - throw new Error(`Failed to infer data type for field ${path.join( - ".", - )} at row ${rowI}. \ - Consider providing an explicit schema.`); - } - pathTree.set(path, inferredType); - } - } else if (schema === undefined) { - const currentType = pathTree.get(path); - const newType = inferType(value, path, opts); - if (currentType !== newType) { - new Error(`Failed to infer schema for data. Previously inferred type \ - ${currentType} but found ${newType} at row ${rowI}. Consider \ - providing an explicit schema.`); - } - } - } - } - - if (schema === undefined) { - function fieldsFromPathTree(pathTree: PathTree): Field[] { - const fields = []; - for (const [name, value] of pathTree.map.entries()) { - if (value instanceof PathTree) { - const children = fieldsFromPathTree(value); - fields.push(new Field(name, new Struct(children), true)); - } else { - fields.push(new Field(name, value, true)); - } - } - return fields; - } - const fields = fieldsFromPathTree(pathTree); - return new Schema(fields); - } else { - function takeMatchingFields( - fields: Field[], - pathTree: PathTree, - ): Field[] { - const outFields = []; - for (const field of fields) { - if (pathTree.map.has(field.name)) { - const value = pathTree.get([field.name]); - if (value instanceof PathTree) { - const struct = field.type as Struct; - const children = takeMatchingFields(struct.children, value); - outFields.push( - new Field(field.name, new Struct(children), field.nullable), - ); - } else { - outFields.push( - new Field(field.name, value as DataType, field.nullable), - ); - } - } - } - return outFields; - } - const fields = takeMatchingFields(schema.fields, pathTree); - return new Schema(fields); - } -} - -function* rowPathsAndValues( - row: Record, - basePath: string[] = [], -): Generator<[string[], unknown]> { - for (const [key, value] of Object.entries(row)) { - if (isObject(value)) { - yield* rowPathsAndValues(value, [...basePath, key]); - } else { - // Skip undefined values - they should be treated the same as missing fields - // for embedding function purposes - if (value !== undefined) { - yield [[...basePath, key], value]; - } - } - } -} - function isObject(value: unknown): value is Record { return ( typeof value === "object" && @@ -577,146 +459,19 @@ function isObject(value: unknown): value is Record { ); } -function getFieldForPath(schema: Schema, path: string[]): Field | undefined { - let current: Field | Schema = schema; +function valueAtPath(datum: Record, path: string[]): unknown { + let current: unknown = datum; for (const key of path) { - if (current instanceof Schema) { - const field: Field | undefined = current.fields.find( - (f) => f.name === key, - ); - if (field === undefined) { - return undefined; - } - current = field; - } else if (current instanceof Field && DataType.isStruct(current.type)) { - const struct: Struct = current.type; - const field = struct.children.find((f) => f.name === key); - if (field === undefined) { - return undefined; - } - current = field; + if (current == null) { + return null; + } + if (isObject(current) && (Object.hasOwn(current, key) || key in current)) { + current = current[key]; } else { return undefined; } } - if (current instanceof Field) { - return current; - } else { - return undefined; - } -} - -/** - * Try to infer which Arrow type to use for a given value. - * - * May return undefined if the type cannot be inferred. - */ -function inferType( - value: unknown, - path: string[], - opts: MakeArrowTableOptions, -): DataType | undefined { - if (typeof value === "bigint") { - return new Int64(); - } else if (typeof value === "number") { - // Even if it's an integer, it's safer to assume Float64. Users can - // always provide an explicit schema or use BigInt if they mean integer. - return new Float64(); - } else if (typeof value === "string") { - if (opts.dictionaryEncodeStrings) { - return new Dictionary(new Utf8(), new Int32()); - } else { - return new Utf8(); - } - } else if (typeof value === "boolean") { - return new Bool(); - } else if (value instanceof Buffer) { - return new Binary(); - } else if (ArrayBuffer.isView(value) && !(value instanceof DataView)) { - const info = typedArrayToArrowType(value); - if (info !== undefined) { - const child = new Field("item", info.elementType, true); - return new FixedSizeList(info.length, child); - } - return undefined; - } else if (Array.isArray(value)) { - if (value.length === 0) { - return undefined; // Without any values we can't infer the type - } - if (path.length === 1 && Object.hasOwn(opts.vectorColumns, path[0])) { - const floatType = sanitizeType(opts.vectorColumns[path[0]].type); - return new FixedSizeList( - value.length, - new Field("item", floatType, true), - ); - } - const valueType = inferType(value[0], path, opts); - if (valueType === undefined) { - return undefined; - } - // Try to automatically detect embedding columns. - if (nameSuggestsVectorColumn(path[path.length - 1])) { - // Check if value is a Uint8Array for integer vector type determination - if (value instanceof Uint8Array) { - // For integer vectors, we default to Uint8 (matching Python implementation) - const child = new Field("item", new Uint8(), true); - return new FixedSizeList(value.length, child); - } else { - // For float vectors, we default to Float32 - const child = new Field("item", new Float32(), true); - return new FixedSizeList(value.length, child); - } - } else { - const child = new Field("item", valueType, true); - return new List(child); - } - } else { - // TODO: timestamp - return undefined; - } -} - -class PathTree { - map: Map>; - - constructor(entries?: [string[], V][]) { - this.map = new Map(); - if (entries !== undefined) { - for (const [path, value] of entries) { - this.set(path, value); - } - } - } - has(path: string[]): boolean { - let ref: PathTree = this; - for (const part of path) { - if (!(ref instanceof PathTree) || !ref.map.has(part)) { - return false; - } - ref = ref.map.get(part) as PathTree; - } - return true; - } - get(path: string[]): V | undefined { - let ref: PathTree = this; - for (const part of path) { - if (!(ref instanceof PathTree) || !ref.map.has(part)) { - return undefined; - } - ref = ref.map.get(part) as PathTree; - } - return ref as V; - } - set(path: string[], value: V): void { - let ref: PathTree = this; - for (const part of path.slice(0, path.length - 1)) { - if (!ref.map.has(part)) { - ref.map.set(part, new PathTree()); - } - ref = ref.map.get(part) as PathTree; - } - ref.map.set(path[path.length - 1], value); - } + return current; } function transposeData( @@ -724,37 +479,26 @@ function transposeData( field: Field, path: string[] = [], ): Vector { + const valuesPath = [...path, field.name]; + const values = data.map((datum) => valueAtPath(datum, valuesPath)); if (field.type instanceof Struct) { const childFields = field.type.children; - const fullPath = [...path, field.name]; const childVectors = childFields.map((child) => { - return transposeData(data, child, fullPath); + return transposeData(data, child, valuesPath); }); + const nullCount = values.filter((value) => value === null).length; const structData = makeData({ type: field.type, + length: values.length, + nullCount, + nullBitmap: + nullCount > 0 + ? arrowUtil.packBools(values.map((value) => value !== null)) + : undefined, children: childVectors as unknown as ArrowData[], }); return arrowMakeVector(structData); } else { - const valuesPath = [...path, field.name]; - const values = data.map((datum) => { - let current: unknown = datum; - for (const key of valuesPath) { - if (current == null) { - return null; - } - - if ( - isObject(current) && - (Object.hasOwn(current, key) || key in current) - ) { - current = current[key]; - } else { - return null; - } - } - return current; - }); return makeVector(values, field.type, undefined, field.nullable); } } @@ -797,32 +541,6 @@ function makeListVector(lists: unknown[][]): Vector { return listBuilder.finish().toVector(); } -/** - * Map a JS TypedArray instance to the corresponding Arrow element DataType - * and its length. Returns undefined if the value is not a recognized TypedArray. - */ -function typedArrayToArrowType( - value: ArrayBufferView, -): { elementType: DataType; length: number } | undefined { - if (value instanceof Float32Array) - return { elementType: new Float32(), length: value.length }; - if (value instanceof Float64Array) - return { elementType: new Float64(), length: value.length }; - if (value instanceof Uint8Array) - return { elementType: new Uint8(), length: value.length }; - if (value instanceof Uint16Array) - return { elementType: new Uint16(), length: value.length }; - if (value instanceof Uint32Array) - return { elementType: new Uint32(), length: value.length }; - if (value instanceof Int8Array) - return { elementType: new Int8(), length: value.length }; - if (value instanceof Int16Array) - return { elementType: new Int16(), length: value.length }; - if (value instanceof Int32Array) - return { elementType: new Int32(), length: value.length }; - return undefined; -} - /** Helper function to convert an Array of JS values to an Arrow Vector */ function makeVector( values: unknown[], @@ -1462,8 +1180,12 @@ export function ensureNestedFieldsExist( completeRow[field.name] = row[field.name]; } } else { - // Field is missing from the data - set to null - completeRow[field.name] = null; + // Keep a missing struct valid while filling each of its children with + // null. This is distinct from an explicitly null struct value. + completeRow[field.name] = + field.type.constructor.name === "Struct" + ? ensureStructFieldsExist({}, field.type as Struct) + : null; } } @@ -1498,8 +1220,12 @@ function ensureStructFieldsExist( completeStruct[childField.name] = data[childField.name]; } } else { - // Field is missing - set to null - completeStruct[childField.name] = null; + // Keep a missing struct valid while filling each of its children with + // null. This is distinct from an explicitly null struct value. + completeStruct[childField.name] = + childField.type.constructor.name === "Struct" + ? ensureStructFieldsExist({}, childField.type as Struct) + : null; } } diff --git a/nodejs/lancedb/arrow_type.ts b/nodejs/lancedb/arrow_type.ts new file mode 100644 index 000000000..35da346ef --- /dev/null +++ b/nodejs/lancedb/arrow_type.ts @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { + type DataType, + Float32, + Float64, + Int8, + Int16, + Int32, + Uint8, + Uint16, + Uint32, +} from "apache-arrow"; + +/** + * Map a JS TypedArray instance to the corresponding Arrow element type and + * length. Returns undefined when the view is not a supported TypedArray. + */ +export function typedArrayToArrowType( + value: ArrayBufferView, +): { elementType: DataType; length: number } | undefined { + if (value instanceof Float32Array) + return { elementType: new Float32(), length: value.length }; + if (value instanceof Float64Array) + return { elementType: new Float64(), length: value.length }; + if (value instanceof Uint8Array) + return { elementType: new Uint8(), length: value.length }; + if (value instanceof Uint16Array) + return { elementType: new Uint16(), length: value.length }; + if (value instanceof Uint32Array) + return { elementType: new Uint32(), length: value.length }; + if (value instanceof Int8Array) + return { elementType: new Int8(), length: value.length }; + if (value instanceof Int16Array) + return { elementType: new Int16(), length: value.length }; + if (value instanceof Int32Array) + return { elementType: new Int32(), length: value.length }; + return undefined; +} diff --git a/nodejs/lancedb/schema.ts b/nodejs/lancedb/schema.ts new file mode 100644 index 000000000..e4749ef37 --- /dev/null +++ b/nodejs/lancedb/schema.ts @@ -0,0 +1,566 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { + Binary, + Bool, + DataType, + Dictionary, + Field, + FixedSizeList, + Float32, + Float64, + Int32, + Int64, + List, + Schema, + Struct, + Utf8, + util as arrowUtil, +} from "apache-arrow"; +import { typedArrayToArrowType } from "./arrow_type"; +import { sanitizeType } from "./sanitize"; + +type InferenceOptions = { + dictionaryEncodeStrings: boolean; + vectorColumns: Record; +}; + +/** + * Infer the Arrow schema represented by a set of records. + * + * This is the intentionally small interface to schema inference. The stateful + * details of combining partial type evidence are encapsulated below so callers + * only need to provide records, an optional schema, and inference options. + */ +export function inferSchema( + data: Array>, + schema: Schema | undefined, + options: InferenceOptions, +): Schema { + return new SchemaInferrer(schema, options).infer(data); +} + +class SchemaInferrer { + private readonly fields = new FieldTree(); + + constructor( + private readonly providedSchema: Schema | undefined, + private readonly options: InferenceOptions, + ) {} + + infer(data: Array>): Schema { + for (const [row, record] of data.entries()) { + for (const [path, value] of recordPathsAndValues(record)) { + this.observe(path, value, row); + } + } + + return this.providedSchema === undefined + ? new Schema(fieldsFromTree(this.fields)) + : new Schema(matchingFields(this.providedSchema.fields, this.fields)); + } + + private observe(path: string[], value: unknown, row: number): void { + const current = this.fields.get(path); + if (current === undefined) { + this.addField(path, value, row); + } else if (this.providedSchema === undefined) { + this.updateInferredField(path, value, row, current); + } + } + + private addField(path: string[], value: unknown, row: number): void { + if (this.providedSchema !== undefined) { + this.addSchemaField(this.providedSchema, path, row); + return; + } + + const evidence = + this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row); + if (evidence === undefined) { + throw typeInferenceError(path, row); + } + + const conflict = this.fields.set( + path, + evidence, + (existing) => + existing instanceof DeferredTypeEvidence && existing.isOnlyNulls(), + ); + if (conflict !== undefined) { + throw branchConflictError(conflict, row, "Struct"); + } + } + + private addSchemaField(schema: Schema, path: string[], row: number): void { + const field = fieldAtPath(schema, path); + if (field === undefined) { + throw new Error( + `Found field not in schema: ${path.join(".")} at row ${row}`, + ); + } + + const conflict = this.fields.set(path, field.type); + if (conflict !== undefined) { + throw branchConflictError(conflict, row, "Struct"); + } + } + + private updateInferredField( + path: string[], + value: unknown, + row: number, + current: FieldNode, + ): void { + const newType = this.inferType(value, path); + const deferred = DeferredTypeEvidence.from(value, row); + + if (current instanceof FieldTree) { + if (deferred?.isOnlyNulls()) { + return; + } + throw schemaInferenceError( + path, + row, + "Struct", + describeEvidence(newType ?? deferred), + ); + } + + if (current instanceof DeferredTypeEvidence) { + this.resolveDeferredField(path, row, current, newType, deferred); + return; + } + + if (newType !== undefined) { + if (!inferredTypesEqual(current, newType)) { + throw schemaInferenceError( + path, + row, + describeEvidence(current), + describeEvidence(newType), + ); + } + return; + } + + if (deferred === undefined || !deferred.matches(current)) { + throw schemaInferenceError( + path, + row, + describeEvidence(current), + describeEvidence(deferred), + ); + } + } + + private resolveDeferredField( + path: string[], + row: number, + current: DeferredTypeEvidence, + newType: DataType | undefined, + deferred: DeferredTypeEvidence | undefined, + ): void { + if (newType !== undefined) { + if (!current.matches(newType)) { + throw schemaInferenceError( + path, + row, + current.describe(), + describeEvidence(newType), + ); + } + this.fields.set(path, newType); + return; + } + + if (deferred !== undefined) { + this.fields.set(path, current.merge(deferred)); + return; + } + + throw schemaInferenceError( + path, + row, + current.describe(), + describeEvidence(newType), + ); + } + + private inferType(value: unknown, path: string[]): DataType | undefined { + if (typeof value === "bigint") { + return new Int64(); + } + if (typeof value === "number") { + return new Float64(); + } + if (typeof value === "string") { + return this.options.dictionaryEncodeStrings + ? new Dictionary(new Utf8(), new Int32()) + : new Utf8(); + } + if (typeof value === "boolean") { + return new Bool(); + } + if (value instanceof Buffer) { + return new Binary(); + } + if (ArrayBuffer.isView(value) && !(value instanceof DataView)) { + const typedArray = typedArrayToArrowType(value); + return typedArray === undefined + ? undefined + : new FixedSizeList( + typedArray.length, + new Field("item", typedArray.elementType, true), + ); + } + if (!Array.isArray(value) || value.length === 0) { + return undefined; + } + + const configuredVector = + path.length === 1 ? this.options.vectorColumns[path[0]] : undefined; + if (configuredVector !== undefined) { + return new FixedSizeList( + value.length, + new Field("item", sanitizeType(configuredVector.type), true), + ); + } + + const itemType = this.inferArrayItemType(value, path); + if (itemType === undefined) { + return undefined; + } + + return nameSuggestsVectorColumn(path[path.length - 1]) + ? new FixedSizeList(value.length, new Field("item", new Float32(), true)) + : new List(new Field("item", itemType, true)); + } + + private inferArrayItemType( + values: unknown[], + path: string[], + ): DataType | undefined { + let itemType: DataType | undefined; + const deferredItems: unknown[] = []; + + for (const value of values) { + const candidate = this.inferType(value, path); + if (candidate === undefined) { + if (!isDeferredValue(value)) { + return undefined; + } + deferredItems.push(value); + } else if (itemType === undefined) { + itemType = candidate; + } else if (!inferredTypesEqual(itemType, candidate)) { + return undefined; + } + } + + if (itemType === undefined) { + return undefined; + } + return deferredItems.every((value) => + deferredValueMatchesType(value, itemType), + ) + ? itemType + : undefined; + } +} + +/** Nulls and empty/all-null lists that do not determine a type by themselves. */ +class DeferredTypeEvidence { + private constructor( + private readonly values: Array<{ value: unknown; row: number }>, + ) {} + + static from(value: unknown, row: number): DeferredTypeEvidence | undefined { + return isDeferredValue(value) + ? new DeferredTypeEvidence([{ value, row }]) + : undefined; + } + + isOnlyNulls(): boolean { + return this.values.every(({ value }) => value == null); + } + + matches(type: DataType): boolean { + return this.values.every(({ value }) => + deferredValueMatchesType(value, type), + ); + } + + merge(other: DeferredTypeEvidence): DeferredTypeEvidence { + return new DeferredTypeEvidence([...this.values, ...other.values]); + } + + describe(): string { + const list = this.values.find(({ value }) => Array.isArray(value)); + return list === undefined + ? "null" + : `List[${(list.value as unknown[]).length}]`; + } + + firstRow(): number { + return this.values[0].row; + } +} + +type FieldNode = DataType | DeferredTypeEvidence | FieldTree; +type LeafNode = Exclude; +type FieldConflict = { path: string[]; value: FieldNode }; + +/** Nested field state, kept separate from Arrow's eventual Struct types. */ +class FieldTree { + private readonly children = new Map(); + + get(path: string[]): FieldNode | undefined { + let current: FieldNode = this; + for (const part of path) { + if (!(current instanceof FieldTree)) { + return undefined; + } + const child = current.children.get(part); + if (child === undefined) { + return undefined; + } + current = child; + } + return current; + } + + set( + path: string[], + value: LeafNode, + canReplaceLeaf: (value: LeafNode) => boolean = () => false, + ): FieldConflict | undefined { + let branch: FieldTree = this; + for (const [index, part] of path.slice(0, -1).entries()) { + const child = branch.children.get(part); + if (child === undefined || (isLeaf(child) && canReplaceLeaf(child))) { + const nextBranch = new FieldTree(); + branch.children.set(part, nextBranch); + branch = nextBranch; + } else if (child instanceof FieldTree) { + branch = child; + } else { + return { path: path.slice(0, index + 1), value: child }; + } + } + + const name = path[path.length - 1]; + const current = branch.children.get(name); + if (current instanceof FieldTree) { + return { path, value: current }; + } + branch.children.set(name, value); + return undefined; + } + + entries(): IterableIterator<[string, FieldNode]> { + return this.children.entries(); + } + + has(name: string): boolean { + return this.children.has(name); + } +} + +function isLeaf(value: FieldNode): value is LeafNode { + return !(value instanceof FieldTree); +} + +function fieldsFromTree(tree: FieldTree, path: string[] = []): Field[] { + const fields: Field[] = []; + for (const [name, value] of tree.entries()) { + if (value instanceof FieldTree) { + fields.push( + new Field( + name, + new Struct(fieldsFromTree(value, [...path, name])), + true, + ), + ); + } else if (value instanceof DeferredTypeEvidence) { + throw typeInferenceError([...path, name], value.firstRow()); + } else { + fields.push(new Field(name, value, true)); + } + } + return fields; +} + +function matchingFields(fields: Field[], tree: FieldTree): Field[] { + const matches: Field[] = []; + for (const field of fields) { + if (!tree.has(field.name)) { + continue; + } + const value = tree.get([field.name]); + if (value instanceof FieldTree) { + const struct = field.type as Struct; + matches.push( + new Field( + field.name, + new Struct(matchingFields(struct.children, value)), + field.nullable, + ), + ); + } else { + matches.push(new Field(field.name, value as DataType, field.nullable)); + } + } + return matches; +} + +function* recordPathsAndValues( + record: Record, + path: string[] = [], +): Generator<[string[], unknown]> { + for (const [name, value] of Object.entries(record)) { + if (isRecord(value)) { + yield* recordPathsAndValues(value, [...path, name]); + } else if (value !== undefined) { + yield [[...path, name], value]; + } + } +} + +function isRecord(value: unknown): value is Record { + return ( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + !(value instanceof RegExp) && + !(value instanceof Date) && + !(value instanceof Set) && + !(value instanceof Map) && + !(value instanceof Buffer) && + !ArrayBuffer.isView(value) + ); +} + +function fieldAtPath(schema: Schema, path: string[]): Field | undefined { + let fields = schema.fields; + let field: Field | undefined; + for (const [index, name] of path.entries()) { + field = fields.find((candidate) => candidate.name === name); + if (field === undefined || index === path.length - 1) { + return field; + } + if (!DataType.isStruct(field.type)) { + return undefined; + } + fields = field.type.children; + } + return field; +} + +function isDeferredValue(value: unknown): boolean { + return ( + value == null || (Array.isArray(value) && value.every(isDeferredValue)) + ); +} + +function deferredValueMatchesType(value: unknown, type: DataType): boolean { + if (value == null) { + return true; + } + if (!Array.isArray(value)) { + return false; + } + if (DataType.isList(type)) { + return value.every((item) => + deferredValueMatchesType(item, type.valueType), + ); + } + if (DataType.isFixedSizeList(type)) { + return ( + value.length === type.listSize && + value.every((item) => deferredValueMatchesType(item, type.valueType)) + ); + } + return false; +} + +function inferredTypesEqual(current: DataType, candidate: DataType): boolean { + if (DataType.isDictionary(current)) { + return ( + DataType.isDictionary(candidate) && + current.isOrdered === candidate.isOrdered && + inferredTypesEqual(current.indices, candidate.indices) && + inferredTypesEqual(current.dictionary, candidate.dictionary) + ); + } + if (DataType.isList(current)) { + return ( + DataType.isList(candidate) && + current.valueField.name === candidate.valueField.name && + current.valueField.nullable === candidate.valueField.nullable && + inferredTypesEqual(current.valueType, candidate.valueType) + ); + } + if (DataType.isFixedSizeList(current)) { + return ( + DataType.isFixedSizeList(candidate) && + current.listSize === candidate.listSize && + current.valueField.name === candidate.valueField.name && + current.valueField.nullable === candidate.valueField.nullable && + inferredTypesEqual(current.valueType, candidate.valueType) + ); + } + return arrowUtil.compareTypes(current, candidate); +} + +function describeEvidence( + evidence: DataType | DeferredTypeEvidence | undefined, +): string { + if (evidence === undefined) { + return "an unsupported value"; + } + return evidence instanceof DeferredTypeEvidence + ? evidence.describe() + : evidence.toString(); +} + +function branchConflictError( + conflict: FieldConflict, + row: number, + candidate: string, +): Error { + return schemaInferenceError( + conflict.path, + row, + conflict.value instanceof FieldTree + ? "Struct" + : describeEvidence(conflict.value), + candidate, + ); +} + +function schemaInferenceError( + path: string[], + row: number, + currentType: string, + newType: string, +): Error { + return new Error( + `Failed to infer schema for data. Previously inferred type ${currentType} ` + + `but found ${newType} for field ${path.join(".")} at row ${row}. ` + + "Consider providing an explicit schema.", + ); +} + +function typeInferenceError(path: string[], row: number): Error { + return new Error( + `Failed to infer data type for field ${path.join(".")} at row ${row}. ` + + "Consider providing an explicit schema.", + ); +} + +function nameSuggestsVectorColumn(name: string): boolean { + const normalized = name.toLowerCase(); + return normalized.includes("vector") || normalized.includes("embedding"); +} From 6ed3074d4cf98b08bec11269cd2c1fba5bf8f7d3 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Mon, 24 Aug 2026 17:16:42 -0700 Subject: [PATCH 43/58] feat: pin the base table version for data loader reads (#3982) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A permutation stores `_rowid`s, which are row addresses unless stable row ids are enabled. Nothing in the data loader pinned a table version, so a compaction between building a permutation and reading it can resolve those ids to different rows. The exposure differs by backend but exists on both: - Remote never pins. `prepare_query_bodies` stamps `"version": current_version()` on every request, but `current_version()` is `None` unless `checkout` was called, so every request means "latest". - Native pins implicitly by holding an `Arc` under `ConsistencyMode::Lazy`, but `StreamingDataset.__setstate__` reopens the table in each DataLoader worker, so each worker pins to whatever is latest at fork time. ## Changes `Table::at_version` returns an independent handle pinned to a version without mutating the receiver. `checkout` cannot serve this: on remote the version cell is an `Arc>>` shared across clones, so pinning through it would silently pin the caller's table too. `PermutationBuilder::build` pins for the whole build and records the version in the permutation table's schema metadata, alongside the existing split names. `PermutationReader` pins the base table to that version before any take. Because the reader pins on construction, the Python worker fork is covered without touching the pickle format — `Permutation.__setstate__` drops the reader and `_ensure_open` rebuilds it, which re-pins. ## Behaviour change A permutation is now bound to the version it was built against, so rows appended to the base table afterwards are not visible through an existing permutation. That is the intended semantics — the permutation only addresses rows that existed when it was built — but it is a change worth flagging. Permutations written before this carry no version key and read exactly as they did before. --- python/python/lancedb/permutation.py | 26 ++- python/python/lancedb/streaming.py | 4 + python/python/tests/test_permutation.py | 25 +++ .../src/dataloader/permutation/builder.rs | 186 ++++++++++++++++-- .../src/dataloader/permutation/reader.rs | 93 ++++++++- rust/lancedb/src/remote/table.rs | 37 ++++ rust/lancedb/src/table/query.rs | 3 + 7 files changed, 350 insertions(+), 24 deletions(-) diff --git a/python/python/lancedb/permutation.py b/python/python/lancedb/permutation.py index 5d7a685ef..8287ef2fe 100644 --- a/python/python/lancedb/permutation.py +++ b/python/python/lancedb/permutation.py @@ -391,6 +391,15 @@ def _table_to_pickle_state(table: Table) -> dict[str, Any]: } +def _drop_base_version(permutation_data: pa.Table) -> pa.Table: + """Strip the recorded base version so the reader leaves the base table unpinned.""" + metadata = dict(permutation_data.schema.metadata or {}) + if metadata.pop(b"base_version", None) is None: + return permutation_data + metadata.pop(b"base_branch", None) + return permutation_data.replace_schema_metadata(metadata) + + def _table_from_pickle_state(state: dict[str, Any]) -> Table: from . import connect @@ -679,11 +688,15 @@ class Permutation: from . import connect connection_factory = state["connection_factory"] + rebuilt_base = False if connection_factory is not None: base_table = connection_factory(state["base_table_name"]) elif "base_table_state" in state: - base_table = _table_from_pickle_state(state["base_table_state"]) + base_state = state["base_table_state"] + rebuilt_base = base_state["kind"] == "memory" + base_table = _table_from_pickle_state(base_state) elif "base_table_data" in state: + rebuilt_base = True # In-memory base table inlined into the pickle; rebuild the same # way we rebuild the in-memory permutation table. mem_db = connect("memory://") @@ -701,11 +714,14 @@ class Permutation: ) permutation_table: Optional[Table] = None - if state["permutation_data"] is not None: + permutation_data = state["permutation_data"] + if permutation_data is not None: + if rebuilt_base: + # The base table was materialized from Arrow, so it is a fresh + # single-version dataset and the recorded pin cannot resolve on it. + permutation_data = _drop_base_version(permutation_data) mem_db = connect("memory://") - permutation_table = mem_db.create_table( - "permutation", state["permutation_data"] - ) + permutation_table = mem_db.create_table("permutation", permutation_data) self.base_table = base_table self.permutation_table = permutation_table diff --git a/python/python/lancedb/streaming.py b/python/python/lancedb/streaming.py index 2c0e2d5c1..76b3702a1 100644 --- a/python/python/lancedb/streaming.py +++ b/python/python/lancedb/streaming.py @@ -41,6 +41,7 @@ from .permutation import ( Permutation, Transforms, permutation_builder, + _drop_base_version, _table_from_pickle_state, _table_to_pickle_state, ) @@ -1327,6 +1328,9 @@ class StreamingDataset(IterableDataset): self._table = self._connection_factory(table_name) else: self._table = _table_from_pickle_state(table_state) + if table_state["kind"] == "memory": + # Rebuilt from Arrow, so the recorded pin cannot resolve on it. + perm_data = _drop_base_version(perm_data) self._perm_table = _connect("memory://").create_table(perm_name, perm_data) def state_dict(self) -> dict: diff --git a/python/python/tests/test_permutation.py b/python/python/tests/test_permutation.py index 135742c84..142fc84f1 100644 --- a/python/python/tests/test_permutation.py +++ b/python/python/tests/test_permutation.py @@ -56,6 +56,31 @@ def test_execute_does_not_reenter_background_loop(tmp_path, monkeypatch): assert permutation_tbl._conn.read_consistency_interval is None +def test_pickled_permutation_reads_pinned_version(tmp_path): + """An unpickled copy must still read the pinned version, which also covers the + version surviving the ``to_arrow()`` round trip in ``__getstate__``.""" + import pickle + + db = connect(tmp_path) + tbl = db.create_table("base", pa.table({"idx": range(20)})) + permutation_tbl = permutation_builder(tbl).execute() + perm = Permutation.from_tables(tbl, permutation_tbl) + + payload = pickle.dumps(perm) + + # Compact so the stored row addresses no longer describe these rows at latest. + tbl.delete("true") + tbl.optimize() + assert tbl.count_rows() == 0 + + # Unpickle after the mutation: __setstate__ reopens at latest, so this only + # passes if the recorded version is applied on reopen. + restored = pickle.loads(payload) + assert len(restored) == 20 + rows = restored.__getitems__(list(range(20))) + assert sorted(row["idx"] for row in rows) == list(range(20)) + + def test_split_random_counts(mem_db): """Test random splitting with absolute counts.""" tbl = mem_db.create_table( diff --git a/rust/lancedb/src/dataloader/permutation/builder.rs b/rust/lancedb/src/dataloader/permutation/builder.rs index 641c191d5..ba0d0e180 100644 --- a/rust/lancedb/src/dataloader/permutation/builder.rs +++ b/rust/lancedb/src/dataloader/permutation/builder.rs @@ -27,6 +27,12 @@ pub const SRC_ROW_ID_COL: &str = "row_id"; pub const SPLIT_NAMES_CONFIG_KEY: &str = "split_names"; +/// Base table version the permutation was built against. +pub const BASE_VERSION_CONFIG_KEY: &str = "base_version"; + +/// Base table branch the permutation was built against. Absent means main. +pub const BASE_BRANCH_CONFIG_KEY: &str = "base_branch"; + pub const DEFAULT_MEMORY_LIMIT: usize = 100 * 1024 * 1024; /// Where to store the permutation table @@ -214,21 +220,11 @@ impl PermutationBuilder { Ok(Box::pin(SimpleRecordBatchStream { schema, stream })) } - fn add_split_names( + fn add_config_metadata( data: SendableRecordBatchStream, - split_names: &[String], + metadata: HashMap, ) -> Result { - let schema = data - .schema() - .as_ref() - .clone() - .with_metadata(HashMap::from([( - SPLIT_NAMES_CONFIG_KEY.to_string(), - serde_json::to_string(split_names).map_err(|e| Error::Other { - message: format!("Failed to serialize split names: {}", e), - source: Some(e.into()), - })?, - )])); + let schema = data.schema().as_ref().clone().with_metadata(metadata); let schema = Arc::new(schema); let schema_clone = schema.clone(); let stream = data.map_ok(move |batch| batch.with_schema(schema.clone()).unwrap()); @@ -269,6 +265,12 @@ impl PermutationBuilder { Err(err) => return Err(err), } + // The handle above is already pinned to one version. Record which one, so a + // reader -- in a DataLoader worker, against a table that has since moved -- + // resolves these row addresses against the same snapshot. + let base_version = self.base_table.version().await?; + let base_branch = self.base_table.current_branch(); + // First pass, apply filter and load row ids. `Shuffler` permutes positions, so // every rank must scan the rows in the same order to build the same permutation. let mut rows = self.base_table.query().select(Select::columns(&[ROW_ID])); @@ -330,11 +332,24 @@ impl PermutationBuilder { // Rename _rowid to row_id let renamed = rename_column(sorted, ROW_ID, SRC_ROW_ID_COL)?; - let streaming_data = if let Some(split_names) = &self.config.split_names { - Self::add_split_names(renamed, split_names)? - } else { - renamed - }; + let mut metadata = HashMap::from([( + BASE_VERSION_CONFIG_KEY.to_string(), + base_version.to_string(), + )]); + // Version numbers are per-branch, so the branch is part of the coordinate. + if let Some(branch) = &base_branch { + metadata.insert(BASE_BRANCH_CONFIG_KEY.to_string(), branch.clone()); + } + if let Some(split_names) = &self.config.split_names { + metadata.insert( + SPLIT_NAMES_CONFIG_KEY.to_string(), + serde_json::to_string(split_names).map_err(|e| Error::Other { + message: format!("Failed to serialize split names: {}", e), + source: Some(e.into()), + })?, + ); + } + let streaming_data = Self::add_config_metadata(renamed, metadata)?; let (name, database) = match &self.config.destination { PermutationDestination::Permanent(database, table_name) => { @@ -533,6 +548,141 @@ mod tests { assert_eq!(*planning_versions.lock().unwrap(), vec![7, 7, 6, 6]); } + #[tokio::test] + async fn test_permutation_records_base_version() { + let temp_dir = tempfile::tempdir().unwrap(); + + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let initial_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(100), BatchCount::from(2)); + let data_table = db + .create_table("base_tbl", initial_data) + .execute() + .await + .unwrap(); + + let build_version = data_table.version().await.unwrap(); + let permutation_table = PermutationBuilder::new(data_table.clone()) + .build() + .await + .unwrap(); + + let recorded = permutation_table + .schema() + .await + .unwrap() + .metadata + .get(BASE_VERSION_CONFIG_KEY) + .expect("permutation should record the base version") + .parse::() + .unwrap(); + assert_eq!(recorded, build_version); + + // Advancing the base table must not move the recorded version. + let more_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(50), BatchCount::from(1)); + data_table.add(more_data).execute().await.unwrap(); + assert!(data_table.version().await.unwrap() > recorded); + assert_eq!( + permutation_table + .schema() + .await + .unwrap() + .metadata + .get(BASE_VERSION_CONFIG_KEY) + .unwrap() + .parse::() + .unwrap(), + recorded, + ); + } + + /// Version numbers are per-branch, so a permutation built on a branch must record + /// it -- a worker reopens by name and lands on main at the same number. + #[tokio::test] + async fn test_permutation_records_base_branch() { + let temp_dir = tempfile::tempdir().unwrap(); + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let initial_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(10), BatchCount::from(1)); + let data_table = db + .create_table("base_tbl", initial_data) + .execute() + .await + .unwrap(); + + let branch = data_table + .create_branch("exp", lance::dataset::refs::Ref::from(("main", 1))) + .await + .unwrap(); + let permutation_table = PermutationBuilder::new(branch.clone()) + .build() + .await + .unwrap(); + + let metadata = permutation_table.schema().await.unwrap().metadata.clone(); + assert_eq!( + metadata.get(BASE_BRANCH_CONFIG_KEY).map(String::as_str), + Some("exp") + ); + + // Main records nothing, so an absent key keeps meaning main. + let main_permutation = PermutationBuilder::new(data_table.clone()) + .build() + .await + .unwrap(); + assert!( + !main_permutation + .schema() + .await + .unwrap() + .metadata + .contains_key(BASE_BRANCH_CONFIG_KEY) + ); + } + + #[tokio::test] + async fn test_build_does_not_pin_the_callers_table() { + let temp_dir = tempfile::tempdir().unwrap(); + + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let initial_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(100), BatchCount::from(1)); + let data_table = db + .create_table("base_tbl", initial_data) + .execute() + .await + .unwrap(); + + PermutationBuilder::new(data_table.clone()) + .build() + .await + .unwrap(); + + // The builder pins its own handle; the caller's must still track latest. + let more_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(50), BatchCount::from(1)); + data_table.add(more_data).execute().await.unwrap(); + assert_eq!(data_table.count_rows(None).await.unwrap(), 150); + } + #[tokio::test] async fn test_permutation_builder() { let temp_dir = tempfile::tempdir().unwrap(); diff --git a/rust/lancedb/src/dataloader/permutation/reader.rs b/rust/lancedb/src/dataloader/permutation/reader.rs index 6da92e986..9757dc552 100644 --- a/rust/lancedb/src/dataloader/permutation/reader.rs +++ b/rust/lancedb/src/dataloader/permutation/reader.rs @@ -8,7 +8,9 @@ //! the rows from a source table that correspond to row IDs stored in a separate table. use crate::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream}; -use crate::dataloader::permutation::builder::SRC_ROW_ID_COL; +use crate::dataloader::permutation::builder::{ + BASE_BRANCH_CONFIG_KEY, BASE_VERSION_CONFIG_KEY, SRC_ROW_ID_COL, +}; use crate::dataloader::permutation::split::SPLIT_ID_COLUMN; use crate::error::Error; use crate::query::{ @@ -23,6 +25,7 @@ use arrow_array::{RecordBatch, UInt64Array}; use arrow_schema::SchemaRef; use datafusion_expr::{Expr, col, lit}; use futures::{StreamExt, TryStreamExt}; +use lance::dataset::refs::MAIN_BRANCH; use lance::dataset::scanner::DatasetRecordBatchStream; use lance::io::RecordBatchStream; use lance_arrow::RecordBatchExt; @@ -69,6 +72,10 @@ impl PermutationReader { permutation_table: Option>, split: u64, ) -> Result { + let base_table = match &permutation_table { + Some(permutation_table) => Self::pin_base_table(base_table, permutation_table).await?, + None => base_table, + }; let mut slf = Self { base_table, permutation_table, @@ -89,6 +96,34 @@ impl PermutationReader { Ok(slf) } + /// Pins the base table to the version the permutation was built against. + /// Permutations written before that was recorded carry no key and stay unpinned. + async fn pin_base_table( + base_table: Arc, + permutation_table: &Arc, + ) -> Result> { + let schema = permutation_table.schema().await?; + let Some(raw) = schema.metadata.get(BASE_VERSION_CONFIG_KEY) else { + return Ok(base_table); + }; + let version = raw.parse::().map_err(|e| Error::InvalidInput { + message: format!( + "Permutation table has an unreadable {} of {:?}: {}", + BASE_VERSION_CONFIG_KEY, raw, e + ), + })?; + // The recorded branch, not the handle's: a worker reopens by name and lands + // on main, and version numbers are per-branch. + let branch = schema + .metadata + .get(BASE_BRANCH_CONFIG_KEY) + .map(String::as_str) + .unwrap_or(MAIN_BRANCH); + base_table + .checkout_branch_version(branch, Some(version)) + .await + } + pub async fn try_from_tables( base_table: Arc, permutation_table: Arc, @@ -511,9 +546,13 @@ mod tests { use lance_datagen::{BatchCount, RowCount}; use rand::seq::SliceRandom; + // Aliased: `test_utils::datagen` exports a trait of the same name. + use crate::arrow::LanceDbDatagenExt as _; use crate::{ Table, arrow::SendableRecordBatchStream, + connect, + dataloader::permutation::builder::PermutationBuilder, query::{ExecutableQuery, QueryBase}, test_utils::datagen::{LanceDbDatagenExt, virtual_table}, }; @@ -545,6 +584,58 @@ mod tests { .await } + /// Compaction moves row addresses, so the reader must read the pinned version. + #[tokio::test] + async fn test_reader_pins_base_version() { + let temp_dir = tempfile::tempdir().unwrap(); + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let data = lance_datagen::gen_batch() + .col("idx", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(20), BatchCount::from(1)); + let base_table = db.create_table("base_tbl", data).execute().await.unwrap(); + + let permutation_table = PermutationBuilder::new(base_table.clone()) + .build() + .await + .unwrap(); + + base_table.delete("true").await.unwrap(); + base_table + .optimize(crate::table::OptimizeAction::All) + .await + .unwrap(); + assert_eq!(base_table.count_rows(None).await.unwrap(), 0); + + let reader = PermutationReader::try_from_tables( + base_table.base_table().clone(), + permutation_table.base_table().clone(), + 0, + ) + .await + .unwrap(); + + let values = collect_from_stream::( + reader + .read( + Select::Columns(vec!["idx".to_string()]), + QueryExecutionOptions::default(), + ) + .await + .unwrap(), + "idx", + ) + .await; + assert_eq!( + values.len(), + 20, + "reader should still see the pinned version" + ); + } + #[tokio::test] async fn test_permutation_reader() { let base_table = lance_datagen::gen_batch() diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 633d0ffce..10980d55a 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -4349,6 +4349,43 @@ mod tests { assert!(!table.base_table().scan_order_is_deterministic()); } + #[tokio::test] + async fn test_checkout_branch_pins_without_touching_the_original() { + let seen = Arc::new(std::sync::Mutex::new(Vec::new())); + let recorder = seen.clone(); + let table = Table::new_with_handler_version( + "my_table", + semver::Version::new(0, 5, 0), + move |request| match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(br#"{"version": 42, "schema": {"fields": []}}"#.to_vec()) + .unwrap(), + "/v1/table/my_table/count_rows/" => { + let body = request_body_json(&request); + recorder.lock().unwrap().push(body["version"].clone()); + http::Response::builder() + .status(200) + .body(b"0".to_vec()) + .unwrap() + } + path => panic!("unexpected request path: {path}"), + }, + ); + + let pinned = table.checkout_branch("main", Some(42)).await.unwrap(); + pinned.count_rows(None).await.unwrap(); + table.count_rows(None).await.unwrap(); + + let seen = seen.lock().unwrap(); + assert_eq!(seen[0], 42, "the pinned handle must send its version"); + assert!( + seen[1].is_null(), + "the original handle must still track latest, got {:?}", + seen[1] + ); + } + #[tokio::test] async fn test_fetch_blobs_sends_the_checked_out_version() { let ipc = one_row_blob_ipc_stream("image"); diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index 9feb9d5ab..9b81786cd 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -70,6 +70,9 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R .contains(&NamespaceClientPushdownOperation::QueryTable) && table.namespace_client.is_some() && table.dataset.current_branch().is_none() + // NsQueryTableRequest has no version field, so a pushed-down query would + // read latest and ignore the pin. + && table.dataset.time_travel_version().is_none() && !requires_local_namespace_execution(query)) { return Ok(false); From 0e65123bd8910548c1ad60b4038f0674c33ed940 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Mon, 24 Aug 2026 23:15:07 -0700 Subject: [PATCH 44/58] fix: package ordinary @udf bodies and emit only the V1 type grammar (#4044) Registering a real (embedding) Function failed on the client for three reasons: - `_package_source` treated `inspect.getclosurevars().unbound` as "unresolved globals"; CPython puts attribute names there, so any body with `np.linalg.norm(...)` or `body.split()` was rejected. Module-scope references now come from Python's own scope analysis (`symtable`) over the function source, recursively, and each is resolved the way the interpreter would: the function's globals first (a module global may shadow a builtin), then builtins. Free variables of nested scopes stay lexical; postponed annotations are not runtime loads. A genuinely missing global still fails. - `_canonical_arrow_type` emitted spellings the server's frozen grammar rejects (`fixed_size_list[n]`, `timestamp[us]`, `struct<...>`, zero-sized lists). It now emits exactly the grammar, with the server's `fixed_size_list` form, and the Rust declaration planner parses that form too. A shared golden (`tests/fixtures/first_class_functions/v1/arrow_types.json`) enumerates every grammar type, nested forms and rejected spellings; the Python emitter and Rust parser are tested against it, and the same file is under test in sophon. Packaging tests execute the shipped artifact in a fresh namespace. Contract changes (hence `breaking-change`): - `@udf` now rejects namespace acquisition structurally (`globals()`/`eval`/... by name, plus `import sys`/`builtins`/`importlib`/`inspect` inside the body), requires the function's captured `__builtins__` to be the standard mapping itself (identity, so neither lookups nor implicit hooks such as `__import__` can differ), rejects module globals that are namespace-bearing modules (`builtins`, `sys`, ...), and treats the function's own name as recursion only when the module binds it to the function or to the exact `UdfDefinition` the decorator produced; it resolves module globals through the function's real namespace (a module global may shadow a builtin) and ships importable classes/functions as imports. - List outputs must declare a non-nullable, metadata-free child named `item` (`pa.list_(pa.field("item", t, nullable=False))`); that is what the grammar means, and pyarrow's default nullable child was being silently collapsed into it. Contract, stated in the `udf` docstring: the artifact is a snapshot of the function source plus exactly the module names it references. Reaching the module namespace by another route is rejected where a static packager can see it and is otherwise unsupported; there is no dynamic-access detection beyond that. --- python/python/lancedb/functions.py | 248 ++++++++--- .../tests/test_first_class_function_slice2.py | 399 +++++++++++++++++- rust/lancedb/src/remote/table.rs | 47 +++ rust/lancedb/src/table/computed_columns.rs | 55 +++ .../first_class_functions/v1/arrow_types.json | 333 +++++++++++++++ ...remote_fixed_size_declaration_request.json | 79 ++++ 6 files changed, 1100 insertions(+), 61 deletions(-) create mode 100644 rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json create mode 100644 rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index c4ff9a9b3..33a67b413 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -12,10 +12,13 @@ expression-backed refresh job. from __future__ import annotations import ast +import builtins import base64 import functools import hashlib +import importlib import inspect +import symtable import json import math import re @@ -489,59 +492,58 @@ _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$") _SECRET_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_GRAMMAR_PRIMITIVES = ( + (pa.bool_(), "bool"), + (pa.int8(), "int8"), + (pa.int16(), "int16"), + (pa.int32(), "int32"), + (pa.int64(), "int64"), + (pa.uint8(), "uint8"), + (pa.uint16(), "uint16"), + (pa.uint32(), "uint32"), + (pa.uint64(), "uint64"), + (pa.float16(), "float16"), + (pa.float32(), "float32"), + (pa.float64(), "float64"), + (pa.string(), "utf8"), + (pa.binary(), "binary"), + (pa.date32(), "date32"), + (pa.date64(), "date64"), +) + + def _canonical_arrow_type(data_type: pa.DataType) -> str: - primitive_types = ( - (pa.bool_(), "bool"), - (pa.int8(), "int8"), - (pa.int16(), "int16"), - (pa.int32(), "int32"), - (pa.int64(), "int64"), - (pa.uint8(), "uint8"), - (pa.uint16(), "uint16"), - (pa.uint32(), "uint32"), - (pa.uint64(), "uint64"), - (pa.float16(), "float16"), - (pa.float32(), "float32"), - (pa.float64(), "float64"), - (pa.string(), "utf8"), - (pa.large_utf8(), "large_utf8"), - (pa.binary(), "binary"), - (pa.large_binary(), "large_binary"), - (pa.date32(), "date32"), - (pa.date64(), "date64"), - ) - for candidate, name in primitive_types: + """The server's V1 Function type grammar. Anything outside it is rejected + here rather than at registration.""" + for candidate, name in _GRAMMAR_PRIMITIVES: if data_type == candidate: return name - if pa.types.is_fixed_size_binary(data_type): - return f"fixed_size_binary[{data_type.byte_width}]" - if pa.types.is_list(data_type): - return f"list<{_canonical_arrow_type(data_type.value_type)}>" - if pa.types.is_large_list(data_type): - return f"large_list<{_canonical_arrow_type(data_type.value_type)}>" - if pa.types.is_fixed_size_list(data_type): + if pa.types.is_list(data_type) or pa.types.is_large_list(data_type): + prefix = "list" if pa.types.is_list(data_type) else "large_list" + return f"{prefix}<{_canonical_list_item(data_type)}>" + if pa.types.is_fixed_size_list(data_type) and data_type.list_size > 0: return ( - f"fixed_size_list<{_canonical_arrow_type(data_type.value_type)}>" - f"[{data_type.list_size}]" + f"fixed_size_list<{_canonical_list_item(data_type)}, {data_type.list_size}>" ) - if pa.types.is_struct(data_type): - fields = ",".join( - f"{field.name}:{_canonical_arrow_type(field.type)}" for field in data_type - ) - return f"struct<{fields}>" - if pa.types.is_timestamp(data_type): - timezone = f",tz={data_type.tz}" if data_type.tz is not None else "" - return f"timestamp[{data_type.unit}{timezone}]" - if pa.types.is_time32(data_type) or pa.types.is_time64(data_type): - return f"time[{data_type.unit}]" - if pa.types.is_duration(data_type): - return f"duration[{data_type.unit}]" - if pa.types.is_decimal(data_type): - bit_width = data_type.bit_width - return f"decimal{bit_width}({data_type.precision},{data_type.scale})" raise TypeError(f"unsupported Arrow type for Function signature: {data_type}") +def _canonical_list_item(data_type: pa.DataType) -> str: + """The grammar names only the item type; it always means a non-nullable + child called `item`, so any other child metadata cannot be represented.""" + child = data_type.value_field + if child.name != "item" or child.nullable or child.metadata: + raise TypeError( + "unsupported Arrow type for Function signature: list items must be a " + f"non-nullable field named 'item', got {child}" + ) + return _canonical_arrow_type(child.type) + + +def _list_of(item: pa.DataType) -> pa.DataType: + return pa.list_(pa.field("item", item, nullable=False)) + + def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]: nullable = False origin = get_origin(annotation) @@ -589,7 +591,7 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]: value_type, value_nullable = _annotation_type(arguments[0]) if value_nullable: raise TypeError("nullable Function list elements are not supported") - return pa.list_(value_type), nullable + return _list_of(value_type), nullable raise TypeError(f"unsupported Function annotation: {annotation!r}") @@ -736,6 +738,104 @@ def _literal_source(value: Any) -> str: ) +_DYNAMIC_NAMESPACE_ACCESS = frozenset( + {"globals", "locals", "vars", "eval", "exec", "compile", "__import__"} +) +# Modules that hand out namespaces (`sys.modules`, `builtins`, importers, +# introspection). The artifact's module namespace holds only the names it was +# packaged with, so reaching around it cannot be represented. +_NAMESPACE_MODULES = frozenset( + {"sys", "builtins", "importlib", "inspect", "gc", "ctypes", "types"} +) + + +def _namespace_acquisition( + definition: ast.FunctionDef, references: set[str] +) -> list[str]: + found = set(references & _DYNAMIC_NAMESPACE_ACCESS) + for node in ast.walk(definition): + if isinstance(node, ast.Import): + found.update( + alias.name + for alias in node.names + if alias.name.split(".")[0] in _NAMESPACE_MODULES + ) + elif isinstance(node, ast.ImportFrom) and node.module: + if node.module.split(".")[0] in _NAMESPACE_MODULES: + found.add(node.module) + return sorted(found) + + +def _module_references(module_source: str) -> set[str]: + """Names any scope in `module_source` binds or loads at module scope. + Python's own scope analysis on the exact text that ships: free variables + belong to an enclosing scope inside the function, and postponed + annotations are not runtime loads.""" + + def visit(table: symtable.SymbolTable, found: set[str]) -> None: + for symbol in table.get_symbols(): + if symbol.is_global() and ( + symbol.is_referenced() or symbol.is_declared_global() + ): + found.add(symbol.get_name()) + for child in table.get_children(): + visit(child, found) + + found: set[str] = set() + for table in symtable.symtable(module_source, "", "exec").get_children(): + visit(table, found) + return found + + +def _global_source(name: str, value: Any) -> str: + """One module-level line that rebinds `name` to `value` in the artifact: + an import for modules and importable classes/functions, a literal otherwise.""" + if isinstance(value, types.ModuleType): + if value.__name__.split(".")[0] in _NAMESPACE_MODULES: + raise ValueError( + f"@udf cannot package dynamic namespace access: {value.__name__!r}" + ) + try: + imported = importlib.import_module(value.__name__) + except ImportError: + imported = None + if imported is not value: + raise TypeError( + f"Function source references module {name!r} that does not import " + f"as {value.__name__!r}" + ) + return f"import {value.__name__} as {name}" + module_name = getattr(value, "__module__", None) + qualname = getattr(value, "__qualname__", None) + if ( + isinstance(module_name, str) + and isinstance(qualname, str) + and module_name != "__main__" + and "." not in qualname + and "<" not in qualname + ): + try: + imported = getattr(importlib.import_module(module_name), qualname) + except (ImportError, AttributeError): + imported = None + if imported is value: + return f"from {module_name} import {qualname} as {name}" + return f"{name} = {_literal_source(value)}" + + +def _is_recursive_reference(function: Callable[..., Any], name: str) -> bool: + """`name` inside the body means the function itself unless the module has + since bound it to something else.""" + if name != function.__name__: + return False + bound = function.__globals__.get(name, function) + if bound is function: + return True + # The decorator's own result is the one wrapper known to call `function` + # unchanged; any other binding may behave differently from a self-call. + return type(bound) is UdfDefinition and bound._function is function + + def _package_source(function: Callable[..., Any]) -> bytes: if not inspect.isfunction(function) or inspect.iscoroutinefunction(function): raise TypeError("@udf requires a synchronous Python function") @@ -760,23 +860,46 @@ def _package_source(function: Callable[..., Any]) -> bytes: closure = inspect.getclosurevars(function) if closure.nonlocals: raise ValueError("@udf cannot package functions that capture closure values") - if closure.unbound: - raise ValueError( - f"@udf source contains unresolved global names: {sorted(closure.unbound)!r}" - ) - globals_source = [] - for name, value in sorted(closure.globals.items()): - if isinstance(value, types.ModuleType): - globals_source.append(f"import {value.__name__} as {name}") - else: - globals_source.append(f"{name} = {_literal_source(value)}") - function_source = ast.unparse(definition) - parts = ["from __future__ import annotations"] + module_header = "from __future__ import annotations" + references = _module_references(f"{module_header}\n\n{function_source}\n") + dynamic = _namespace_acquisition(definition, references) + if dynamic: + raise ValueError(f"@udf cannot package dynamic namespace access: {dynamic!r}") + # Resolve every module-scope reference the way the interpreter would: the + # function's own globals first (a module global may shadow a builtin, and + # nested scopes are not visible to getclosurevars), then its builtins. + # The artifact runs under the standard builtins; only the exact mapping is + # provably equivalent (a subclass or copy can change lookups and hooks). + if function.__builtins__ is not vars(builtins): + raise ValueError("@udf cannot package a non-standard builtins environment") + globals_source = [] + unresolved = [] + for name in sorted(references): + if name == function.__name__: + if not _is_recursive_reference(function, name): + raise ValueError( + f"@udf cannot package {name!r}: the module binds that name to " + "another value, which the artifact's own definition would shadow" + ) + continue + if name in function.__globals__: + globals_source.append(_global_source(name, function.__globals__[name])) + elif hasattr(builtins, name): + pass + else: + unresolved.append(name) + if unresolved: + raise ValueError( + f"@udf source contains unresolved global names: {unresolved!r}" + ) + + parts = [module_header] if globals_source: parts.extend(["", *globals_source]) parts.extend(["", function_source, ""]) - return "\n".join(parts).encode("utf-8") + packaged = "\n".join(parts) + return packaged.encode("utf-8") class UdfDefinition: @@ -923,6 +1046,13 @@ def udf( python_version : str, optional Remote Python major/minor version. Defaults to the client version. + The packaged artifact is a snapshot: the function source plus exactly + the module-level names it references (modules as imports, importable + classes and functions as imports, literals inline). Code that reaches the + module namespace another way -- ``globals()``/``eval``, ``sys.modules``, + ``builtins`` -- is rejected where it can be seen and otherwise + unsupported; closures and a non-standard ``__builtins__`` are rejected. + Returns ------- UdfDefinition diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index a20347771..c67e17520 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -3,7 +3,12 @@ from __future__ import annotations +import base64 import contextlib +import functools +import importlib.util +import types +from datetime import date import http.server import json from pathlib import Path @@ -16,6 +21,9 @@ import pytest import lancedb from lancedb.functions import UdfDefinition, udf +THRESHOLD = 20 +_CACHE = None + FIXTURES = ( Path(__file__).parents[3] @@ -71,9 +79,396 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable(): _assert_no_secret_values(request) +def _run_packaged(definition, *args): + """Execute the shipped artifact in a fresh namespace, as a worker would.""" + source = base64.b64decode(definition.registration_request.artifact.content.data) + namespace: dict = {} + exec(compile(source, "", "exec"), namespace) + return namespace[definition.registration_request.artifact.entrypoint](*args) + + +def test_udf_packages_attribute_access_and_body_imports(): + @udf + def word_norm(body: str) -> float: + import numpy as np + + try: + words = body.split() + except AttributeError as error: + raise ValueError(str(error)) from error + return float(np.linalg.norm([len(w) for w in words])) + + assert _run_packaged(word_norm, "aa bb") == pytest.approx(8**0.5) + + +def test_udf_packages_module_globals_and_global_caches(): + @udf + def label(value: int) -> str: + return "big" if value >= THRESHOLD else "small" + + assert _run_packaged(label, 21) == "big" + + @udf + def cached(value: int) -> int: + global _CACHE + if _CACHE is None: + _CACHE = 40 + return _CACHE + value + + assert _run_packaged(cached, 2) == 42 + + +def test_udf_annotations_are_not_runtime_names(): + @udf + def identity(value: date) -> date: + return value + + assert _run_packaged(identity, date(2026, 8, 25)) == date(2026, 8, 25) + + +def test_udf_nested_scopes_resolve_lexically(): + @udf + def score(value: int) -> int: + offset = 2 + + def add_offset() -> int: + return value + offset + + return add_offset() + sum(v for v in [0]) + + assert _run_packaged(score, 3) == 5 + + +def test_udf_resolves_module_globals_before_builtins(tmp_path): + module_path = tmp_path / "shadowing_udfs.py" + module_path.write_text( + "max = 7\n" + "len = lambda _: 99\n" + "\n" + "def uses_literal_shadow(value: int) -> int:\n" + " def nested() -> int:\n" + " return max\n" + " return nested() + value\n" + "\n" + "def uses_callable_shadow(value: int) -> int:\n" + " def nested() -> int:\n" + " return len([1])\n" + " return nested() + value\n" + ) + spec = importlib.util.spec_from_file_location("shadowing_udfs", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # The module's `max = 7` is what the interpreter would use, so it ships. + assert _run_packaged(udf(module.uses_literal_shadow), 1) == 8 + # A callable global cannot ship; it must not be silently swapped for the builtin. + with pytest.raises(TypeError, match="unsupported global value of type function"): + udf(module.uses_callable_shadow) + + +def test_canonical_arrow_type_is_exactly_the_grammar(): + from lancedb.functions import _GRAMMAR_PRIMITIVES, _canonical_arrow_type + + golden = json.loads( + ( + Path(__file__).parents[3] + / "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json" + ).read_text() + ) + primitives = [ + case["arrow_type"] for case in golden["valid"] if "<" not in case["arrow_type"] + ] + assert [name for _, name in _GRAMMAR_PRIMITIVES] == primitives + for outside in [ + pa.timestamp("us"), + pa.decimal128(10, 2), + pa.large_string(), + pa.large_binary(), + pa.binary(4), + pa.duration("s"), + pa.struct([pa.field("a", pa.int32())]), + pa.list_(pa.float32(), 0), + pa.list_(pa.timestamp("us")), + ]: + with pytest.raises(TypeError, match="unsupported Arrow type"): + _canonical_arrow_type(outside) + + +def test_udf_nested_annotations_are_postponed_in_the_artifact(): + @udf + def score(value: int) -> int: + def identity(item: date) -> date: + return item + + identity(date(2026, 8, 25)) + return value + + assert _run_packaged(score, 3) == 3 + + +def test_udf_ships_globals_the_body_deletes(): + @udf + def clear(value: int) -> int: + global _CACHE + del _CACHE + return value + + assert _run_packaged(clear, 3) == 3 + + +def test_udf_rejects_a_module_global_that_does_not_import_as_itself(tmp_path): + module_path = tmp_path / "fake_module_udfs.py" + module_path.write_text( + "import types\n" + "np = types.ModuleType('numpy')\n" + "np.sqrt = lambda x: 0\n" + "\n" + "def score(value: int) -> int:\n" + " return int(np.sqrt(value))\n" + ) + spec = importlib.util.spec_from_file_location("fake_module_udfs", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + with pytest.raises(TypeError, match="does not import as 'numpy'"): + udf(module.score) + + +def test_udf_rejects_a_module_level_namespace_alias(tmp_path): + module_path = tmp_path / "aliasing_udfs.py" + module_path.write_text( + "import builtins as b\n" + "THRESHOLD = 5\n" + "\n" + "def score(value: int) -> int:\n" + " return value + b.vars(b.__import__('aliasing_udfs'))['THRESHOLD']\n" + ) + spec = importlib.util.spec_from_file_location("aliasing_udfs", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + with pytest.raises(ValueError, match="dynamic namespace access"): + udf(module.score) + + +@pytest.mark.parametrize( + "access", + [ + "globals()['THRESHOLD']", + "eval('THRESHOLD')", + "(lambda g: g()['THRESHOLD'])(globals)", + "__import__('sys').modules[__name__].THRESHOLD", + "sys.modules[__name__].THRESHOLD", + ], +) +def test_udf_rejects_dynamic_namespace_access(access): + namespace: dict = {} + exec( + f"def score(value: int) -> int:\n return value + {access}\n", + {"THRESHOLD": 5}, + namespace, + ) + with pytest.raises(ValueError, match="dynamic namespace access"): + _package_from_text( + "def score(value: int) -> int:\n" + " import sys\n" + f" return value + {access}\n" + ) + + +def _package_from_text(source: str, module_globals: dict | None = None): + """Load `source` as a real module file so the packager can inspect it.""" + import tempfile + + directory = tempfile.mkdtemp() + path = Path(directory) / "generated_udf_module.py" + path.write_text(source) + spec = importlib.util.spec_from_file_location(f"generated_udf_{id(source)}", path) + module = importlib.util.module_from_spec(spec) + if module_globals: + module.__dict__.update(module_globals) + spec.loader.exec_module(module) + functions = [ + value + for value in vars(module).values() + if callable(value) and getattr(value, "__module__", None) == module.__name__ + ] + return udf(functions[0]) + + +def test_udf_rejects_a_non_standard_builtins_environment(): + def score(value: int) -> int: + return len([1]) + value + + score.__globals__ # noqa: B018 -- real function, real globals + import builtins + + patched = types.FunctionType( + score.__code__, + {"__builtins__": {**vars(builtins), "len": lambda _: 99}}, + "score", + ) + patched.__annotations__ = score.__annotations__ + assert patched(3) == 102 + with pytest.raises(ValueError, match="non-standard builtins environment"): + udf(patched) + + class ReportingDict(dict): # reports standard entries, resolves differently + def __missing__(self, key): + return vars(builtins)[key] + + disguised = types.FunctionType( + score.__code__, {"__builtins__": ReportingDict(len=lambda _: 99)}, "score" + ) + disguised.__annotations__ = score.__annotations__ + assert disguised(3) == 102 + with pytest.raises(ValueError, match="non-standard builtins environment"): + udf(disguised) + + hooked = types.FunctionType( + score.__code__, + {"__builtins__": {**vars(builtins), "__import__": lambda *a, **k: None}}, + "score", + ) + hooked.__annotations__ = score.__annotations__ + with pytest.raises(ValueError, match="non-standard builtins environment"): + udf(hooked) + + +def test_udf_recursion_versus_a_rebound_module_name(tmp_path): + module_path = tmp_path / "rebound_udfs.py" + module_path.write_text( + "def fact(value: int) -> int:\n" + " return 1 if value <= 1 else value * fact(value - 1)\n" + "\n" + "def score(value: int) -> int:\n" + " return score + value\n" + ) + spec = importlib.util.spec_from_file_location("rebound_udfs", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + assert _run_packaged(udf(module.fact), 5) == 120 + raw = module.score + module.score = 10 + with pytest.raises(ValueError, match="binds that name to another value"): + udf(raw) + # A wrapper that merely exposes __wrapped__ is not the function. + module.score = functools.wraps(raw)(lambda value: 41) + with pytest.raises(ValueError, match="binds that name to another value"): + udf(raw) + # The decorator's own result is; a subclass of it is not. + module.fact = udf(module.fact) + assert _run_packaged(module.fact, 4) == 24 + + class Twisted(UdfDefinition): + def __call__(self, *args, **kwargs): + return 41 + + raw_fact = module.fact._function + module.fact = Twisted( + raw_fact, + name=None, + input_schema=None, + output_schema=None, + pip=(), + env={}, + secrets=(), + python_version=None, + ) + with pytest.raises(ValueError, match="binds that name to another value"): + udf(raw_fact) + + +def test_canonical_arrow_type_rejects_unrepresentable_list_children(): + from lancedb.functions import _canonical_arrow_type + + for outside in [ + pa.list_(pa.float32()), # pyarrow default: nullable child + pa.list_(pa.field("custom", pa.float32(), nullable=False)), + pa.list_(pa.field("item", pa.float32(), nullable=False, metadata={"k": "v"})), + pa.list_(pa.field("item", pa.float32(), nullable=False), 0), + ]: + with pytest.raises(TypeError, match="unsupported Arrow type"): + _canonical_arrow_type(outside) + assert ( + _canonical_arrow_type( + pa.list_(pa.field("item", pa.float32(), nullable=False), 3) + ) + == "fixed_size_list" + ) + + +def _calls_missing(value: int) -> int: + return missing(value) # noqa: F821 + + +def _shadows_missing_in_a_comprehension(value: int) -> int: + return missing(value) + sum(missing for missing in ()) # noqa: F821 + + +def _shadows_missing_in_a_lambda(value: int) -> int: + return (lambda missing: missing)(value) + missing # noqa: F821 + + +@pytest.mark.parametrize( + "function", + [_calls_missing, _shadows_missing_in_a_comprehension, _shadows_missing_in_a_lambda], +) +def test_udf_rejects_a_truly_unresolved_global(function): + with pytest.raises(ValueError, match=r"unresolved global names: \['missing'\]"): + udf(function) + + +def _arrow_type_from_golden(spec: dict) -> pa.DataType: + kind = spec["type"] + if kind in ("list", "large_list", "fixed_size_list"): + item = _arrow_type_from_golden(spec["fields"][0]["type"]) + field = pa.field("item", item, nullable=False) + if kind == "list": + return pa.list_(field) + if kind == "large_list": + return pa.large_list(field) + return pa.list_(field, spec["length"]) + return { + "null": pa.null(), + "bool": pa.bool_(), + "utf8": pa.string(), + "binary": pa.binary(), + "float16": pa.float16(), + "float32": pa.float32(), + "float64": pa.float64(), + "date32": pa.date32(), + "date64": pa.date64(), + }.get(kind) or getattr(pa, kind)() + + +def test_arrow_type_grammar_matches_the_shared_golden(): + golden = json.loads( + ( + Path(__file__).parents[3] + / "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json" + ).read_text() + ) + from lancedb.functions import _canonical_arrow_type + + emitted = { + case["arrow_type"]: _canonical_arrow_type(_arrow_type_from_golden(case["json"])) + for case in golden["valid"] + } + assert emitted == { + case["arrow_type"]: case["arrow_type"] for case in golden["valid"] + } + assert not set(emitted) & set(golden["invalid"]) + for case in golden["server_only"]: + with pytest.raises(TypeError, match="unsupported Arrow type"): + _canonical_arrow_type(_arrow_type_from_golden(case["json"])) + + def test_explicit_arrow_schema_is_deterministic(): input_schema = pa.schema([pa.field("value", pa.float32(), nullable=True)]) - output_schema = pa.field("embedding", pa.list_(pa.float32(), 3), nullable=False) + output_schema = pa.field( + "embedding", + pa.list_(pa.field("item", pa.float32(), nullable=False), 3), + nullable=False, + ) @udf(input_schema=input_schema, output_schema=output_schema) def explicit(value): @@ -82,7 +477,7 @@ def test_explicit_arrow_schema_is_deterministic(): signature = explicit.registration_request.signature assert signature.inputs[0].arrow_type == "float32" assert signature.inputs[0].nullable is True - assert signature.output.arrow_type == "fixed_size_list[3]" + assert signature.output.arrow_type == "fixed_size_list" assert signature.output.nullable is False diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 10980d55a..35ac56970 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -6802,6 +6802,53 @@ mod tests { assert_eq!(result.version, 8); } + #[tokio::test] + async fn test_add_fixed_size_list_function_column_declares_the_vector_type() { + let table = Table::new_with_handler("my_table", |request| { + match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body( + r#"{"version":1,"schema":{"fields":[{"name":"description","nullable":true,"type":{"type":"string"}}]}}"#, + ) + .unwrap(), + "/v1/table/my_table/add_columns/" => { + let actual: serde_json::Value = serde_json::from_slice( + request.body().unwrap().as_bytes().unwrap(), + ) + .unwrap(); + let expected: serde_json::Value = serde_json::from_str(include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json" + )) + .unwrap(); + assert_eq!(actual, expected); + http::Response::builder() + .status(200) + .body(r#"{"version":8}"#) + .unwrap() + } + path => panic!("Unexpected path: {path}"), + } + }); + let application = crate::function::FunctionApplication::from_json( + r#"{ + "function":{"name":"embed","version":"fv_01K3EXACT"}, + "inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}], + "output":{"kind":"scalar","arrow_type":"fixed_size_list","nullable":false}, + "group_id":"fg_fixed" + }"#, + ) + .unwrap(); + + let result = table + .add_columns() + .function_as("embedding", application) + .execute() + .await + .unwrap(); + assert_eq!(result.version, 8); + } + #[tokio::test] async fn test_add_named_struct_function_expands_one_atomic_sibling_group() { let table = Table::new_with_handler("my_table", |request| match request.url().path() { diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index b1c0b8370..133f37a55 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -586,6 +586,25 @@ fn canonical_input_arrow_type(field: &JsonArrowField) -> Result { } } +/// `fixed_size_list` -> (`item`, `size`); the comma must sit outside +/// any nested `<...>`. +fn split_fixed_size_list(raw: &str) -> Option<(&str, i32)> { + let inner = raw.strip_prefix("fixed_size_list<")?.strip_suffix('>')?; + let mut depth = 0_u32; + let mut separator = None; + for (index, byte) in inner.bytes().enumerate() { + match byte { + b'<' => depth += 1, + b'>' => depth = depth.checked_sub(1)?, + b',' if depth == 0 => separator = Some(index), + _ => {} + } + } + let (item, size) = inner.split_at(separator?); + let size: i32 = size[1..].trim().parse().ok()?; + (size > 0).then_some((item.trim(), size)) +} + fn parse_output_arrow_type(raw: &str) -> Result { fn parse(raw: &str) -> Result { let raw = raw.trim(); @@ -618,6 +637,16 @@ fn parse_output_arrow_type(raw: &str) -> Result { )]); return Ok(data_type); } + if let Some((inner, size)) = split_fixed_size_list(raw) { + let mut data_type = JsonArrowDataType::new("fixed_size_list".to_string()); + data_type.fields = Some(vec![JsonArrowField::new( + "item".to_string(), + false, + parse(inner)?, + )]); + data_type.length = Some(i64::from(size)); + return Ok(data_type); + } let normalized = match raw { "boolean" => "bool", "string" => "utf8", @@ -1322,6 +1351,32 @@ pub(super) async fn add_foreign_kind(table: &crate::Table, name: &str, kind: &st #[cfg(test)] mod tests { + #[test] + fn output_arrow_type_grammar_matches_the_shared_golden() { + let golden: serde_json::Value = serde_json::from_str(include_str!( + "../../tests/fixtures/first_class_functions/v1/arrow_types.json" + )) + .unwrap(); + let valid = golden["valid"].as_array().unwrap().iter(); + for case in valid.chain(golden["server_only"].as_array().unwrap()) { + let raw = case["arrow_type"].as_str().unwrap(); + let parsed = super::parse_output_arrow_type(raw) + .unwrap_or_else(|error| panic!("{raw}: {error}")); + assert_eq!( + serde_json::to_value(&parsed).unwrap(), + case["json"], + "{raw}" + ); + } + for raw in golden["invalid"].as_array().unwrap() { + let raw = raw.as_str().unwrap(); + assert!( + super::parse_output_arrow_type(raw).is_err(), + "{raw:?} should be rejected" + ); + } + } + use arrow_array::record_batch; use arrow_schema::DataType; use futures::TryStreamExt; diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json new file mode 100644 index 000000000..ff26e4c08 --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json @@ -0,0 +1,333 @@ +{ + "valid": [ + { + "arrow_type": "bool", + "json": { + "type": "bool" + } + }, + { + "arrow_type": "int8", + "json": { + "type": "int8" + } + }, + { + "arrow_type": "int16", + "json": { + "type": "int16" + } + }, + { + "arrow_type": "int32", + "json": { + "type": "int32" + } + }, + { + "arrow_type": "int64", + "json": { + "type": "int64" + } + }, + { + "arrow_type": "uint8", + "json": { + "type": "uint8" + } + }, + { + "arrow_type": "uint16", + "json": { + "type": "uint16" + } + }, + { + "arrow_type": "uint32", + "json": { + "type": "uint32" + } + }, + { + "arrow_type": "uint64", + "json": { + "type": "uint64" + } + }, + { + "arrow_type": "float16", + "json": { + "type": "float16" + } + }, + { + "arrow_type": "float32", + "json": { + "type": "float32" + } + }, + { + "arrow_type": "float64", + "json": { + "type": "float64" + } + }, + { + "arrow_type": "utf8", + "json": { + "type": "utf8" + } + }, + { + "arrow_type": "binary", + "json": { + "type": "binary" + } + }, + { + "arrow_type": "date32", + "json": { + "type": "date32" + } + }, + { + "arrow_type": "date64", + "json": { + "type": "date64" + } + }, + { + "arrow_type": "list", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + }, + { + "arrow_type": "large_list", + "json": { + "type": "large_list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + }, + { + "arrow_type": "list", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "int64" + } + } + ] + } + }, + { + "arrow_type": "large_list", + "json": { + "type": "large_list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "int64" + } + } + ] + } + }, + { + "arrow_type": "list", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "utf8" + } + } + ] + } + }, + { + "arrow_type": "large_list", + "json": { + "type": "large_list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "utf8" + } + } + ] + } + }, + { + "arrow_type": "fixed_size_list", + "json": { + "type": "fixed_size_list", + "length": 384, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + }, + { + "arrow_type": "fixed_size_list", + "json": { + "type": "fixed_size_list", + "length": 8, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float16" + } + } + ] + } + }, + { + "arrow_type": "fixed_size_list", + "json": { + "type": "fixed_size_list", + "length": 1, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "uint8" + } + } + ] + } + }, + { + "arrow_type": "list>", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + } + ] + } + }, + { + "arrow_type": "fixed_size_list, 2>", + "json": { + "type": "fixed_size_list", + "length": 2, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "int32" + } + } + ] + } + } + ] + } + }, + { + "arrow_type": "list>", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "fixed_size_list", + "length": 3, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + } + ] + } + } + ], + "server_only": [ + { + "arrow_type": "null", + "json": { + "type": "null" + } + } + ], + "invalid": [ + "", + "list<>", + "list[3]", + "fixed_size_list", + "fixed_size_list", + "fixed_size_list", + "map", + "decimal128(10, 2)", + "timestamp[us]", + "struct" + ] +} \ No newline at end of file diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json new file mode 100644 index 000000000..7d92f1454 --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json @@ -0,0 +1,79 @@ +{ + "new_columns": [ + { + "name": "embedding", + "all_null": true + } + ], + "function": { + "application": { + "function": { + "name": "embed", + "version": "fv_01K3EXACT" + }, + "inputs": [ + { + "parameter": "text", + "kind": "column", + "value": { + "path": "description" + } + } + ], + "output": { + "kind": "scalar", + "arrow_type": "fixed_size_list", + "nullable": false + }, + "group_id": "fg_fixed" + }, + "binding_metadata_version": 1, + "input_bindings": [ + { + "parameter": "text", + "field_path": "description", + "arrow_type": "utf8", + "nullable": true + } + ], + "input_schema": { + "fields": [ + { + "name": "text", + "nullable": true, + "type": { + "type": "utf8" + } + } + ] + }, + "output_schema": { + "fields": [ + { + "name": "embedding", + "nullable": true, + "type": { + "type": "fixed_size_list", + "length": 3, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + } + ] + }, + "outputs": [ + { + "result_field": "$value", + "output_name": "embedding", + "output_ordinal": 0 + } + ] + } +} \ No newline at end of file From 2fea7cd48d86ef149ab7eee93f7751efeb410233 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Tue, 25 Aug 2026 06:26:01 +0000 Subject: [PATCH 45/58] =?UTF-8?q?Bump=20version:=200.38.0-beta.7=20?= =?UTF-8?q?=E2=86=92=200.38.0-beta.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index bcf714002..86a5064f7 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.7" +current_version = "0.38.0-beta.8" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index a60a5ddf1..8d0058569 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5402,7 +5402,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.7" +version = "0.38.0-beta.8" dependencies = [ "ahash", "anyhow", @@ -5490,7 +5490,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.7" +version = "0.38.0-beta.8" dependencies = [ "arrow-array", "arrow-buffer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.7" +version = "0.38.0-beta.8" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 59ee8a7d6..28de11c30 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.7 + 0.38.0-beta.8 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index b24ad61ab..6f4b0744b 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.7 + 0.38.0-beta.8 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 0ee59bcb7..ac6196a45 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.7 + 0.38.0-beta.8 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 8732d1965..e85da66ec 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.7" +version = "0.38.0-beta.8" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 56480c0cb..f920c9e91 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index a1102ee29..ad6926039 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 695862baf..ccd39f18e 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 9099ee4de..50446b597 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index f04c2a884..d4da82dc7 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index f9b35af32..dddc3a470 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index cc5340105..23ac122f7 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 6604da6cd..21deb8dd8 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index 66777d60c..96f4bafd5 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index d2ccf3064..bffac2ef1 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.7" +version = "0.38.0-beta.8" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 071c876d9..7290694f1 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.7" +version = "0.38.0-beta.8" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From c988e4848dade1ebca7ff73a669190761c3e34ac Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 25 Aug 2026 18:30:09 +0800 Subject: [PATCH 46/58] refactor: simplify Function binding identity (#4046) Function applications and bindings currently encode `group_id` and binding `revision` even though `binding_id` already owns the complete immutable binding lifecycle and `outputs` already defines the atomic multi-output set. Make `binding_id` the sole binding identity, remove the redundant fields from the Rust and Python client contracts, and describe multi-output declarations directly. This intentionally replaces the removed wire fields without a compatibility path. --- python/python/lancedb/functions.py | 11 ++--- python/python/lancedb/table.py | 6 +-- .../tests/test_first_class_function_slice1.py | 16 +++---- rust/lancedb/src/function.rs | 21 ++------- rust/lancedb/src/query.rs | 9 ++-- rust/lancedb/src/remote/table.rs | 11 ++--- rust/lancedb/src/table.rs | 2 +- rust/lancedb/src/table/add_columns.rs | 2 +- rust/lancedb/src/table/computed_columns.rs | 45 +++++++------------ .../tests/first_class_function_slice1.rs | 1 - ...remote_fixed_size_declaration_request.json | 5 +-- ...remote_function_application.canonical.json | 2 +- .../v1/remote_function_application.json | 1 - .../v1/remote_function_application_float.json | 3 +- .../v1/remote_function_binding.canonical.json | 2 +- .../v1/remote_function_binding.json | 4 +- ...ote_multi_output_declaration_request.json} | 1 - .../v1/remote_refresh_job.json | 2 +- .../v1/remote_scalar_declaration_request.json | 3 +- 19 files changed, 49 insertions(+), 98 deletions(-) rename rust/lancedb/tests/fixtures/first_class_functions/v1/{remote_grouped_declaration_request.json => remote_multi_output_declaration_request.json} (97%) diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 33a67b413..dd8cecfa9 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -25,7 +25,6 @@ import re import sys import textwrap import types -import uuid from collections.abc import Mapping from datetime import date, datetime from typing import ( @@ -276,7 +275,7 @@ class FunctionVersion(_RemoteValue): Every input must be a direct [lancedb.col][lancedb.expr.col] reference. The returned application is immutable and retains a - named-struct output as one sibling group, so every row's sibling values + named-struct output as one binding, so every row's sibling values come from one logical Function evaluation. Map result fields to table columns with [FunctionApplication.rename][lancedb.functions.FunctionApplication.rename], @@ -326,7 +325,6 @@ class FunctionVersion(_RemoteValue): function=FunctionVersionRef(name=self.name, version=self.version), inputs=tuple(bindings), output=self.signature.output, - group_id=f"fg_{uuid.uuid4().hex}", ) @@ -370,7 +368,7 @@ class ApplicationInput(_OpenRemoteValue): class FunctionApplication(_OpenRemoteValue): """Immutable pre-declaration application of an exact Function version. - A named-struct output remains one grouped application through table + A named-struct output remains one application through table declaration and execution. [FunctionApplication.rename][lancedb.functions.FunctionApplication.rename] records the result-field to table-column mapping without splitting sibling @@ -380,7 +378,6 @@ class FunctionApplication(_OpenRemoteValue): function: FunctionVersionRef inputs: tuple[ApplicationInput, ...] output: FunctionOutput - group_id: str columns: Mapping[str, str] = Field(default_factory=dict) def _known_dict(self) -> dict[str, Any]: @@ -452,12 +449,10 @@ class OutputMapping(_RemoteValue): class FunctionBinding(_RemoteValue): - """Immutable grouped binding persisted by the Enterprise table service.""" + """Immutable Function binding persisted by the Enterprise table service.""" binding_id: str - revision: _UInt64 function: FunctionVersionRef - group_id: str inputs: tuple[InputBinding, ...] outputs: tuple[OutputMapping, ...] input_schema: Optional[Mapping[str, Any]] = None diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 75b7b1db8..fe25d5353 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -1972,7 +1972,7 @@ class Table(ABC): A mapping with one ``FunctionApplication`` value keeps its scalar or named-struct result in the named table column. A bare named-struct application expands its ordered result fields as one - atomic sibling group; aliases come from ``rename(columns=...)``. + atomic binding; aliases come from ``rename(columns=...)``. Function columns are supported only on LanceDB Cloud and Enterprise. computed: Dict[str, str], optional @@ -6038,7 +6038,7 @@ class AsyncTable: A mapping with one ``FunctionApplication`` value keeps its scalar or named-struct result in the named table column. A bare named-struct application expands its ordered result fields as one - atomic sibling group; aliases come from ``rename(columns=...)``. + atomic binding; aliases come from ``rename(columns=...)``. Function columns are supported only on LanceDB Cloud and Enterprise. computed: Dict[str, str], optional @@ -6075,7 +6075,7 @@ class AsyncTable: isinstance(value, FunctionApplication) for value in transforms.values() ): raise ValueError( - "one add_columns call declares exactly one Function sibling group" + "one add_columns call declares exactly one Function binding" ) function_output_name, function_application = next(iter(transforms.items())) diff --git a/python/python/tests/test_first_class_function_slice1.py b/python/python/tests/test_first_class_function_slice1.py index 1bb8feaf4..ca0b30ede 100644 --- a/python/python/tests/test_first_class_function_slice1.py +++ b/python/python/tests/test_first_class_function_slice1.py @@ -121,7 +121,7 @@ def test_function_version_identity_is_immutable_and_exact(): assert FunctionVersion(**changed) != version -def test_function_version_binds_named_columns_as_one_immutable_group(): +def test_function_version_binds_named_columns_as_one_immutable_application(): version = FunctionVersion.from_json( json.dumps(job_result("remote_function_job.json")) ) @@ -131,13 +131,10 @@ def test_function_version_binds_named_columns_as_one_immutable_group(): assert application.function.name == version.name assert application.function.version == version.version assert application.output is version.signature.output - assert application.group_id.startswith("fg_") assert [ (value.parameter, value.kind, value.value["path"]) for value in application.inputs ] == [("text", "column", "documents.body")] - with pytest.raises((TypeError, ValueError)): - application.group_id = "fg_changed" def test_function_version_binding_validates_names_and_direct_columns(): @@ -156,7 +153,7 @@ def test_function_version_binding_validates_names_and_direct_columns(): def test_function_version_keeps_named_struct_outputs_in_one_application(): value = job_result("remote_function_job.json") value["name"] = "text_features" - value["version"] = "fv_grouped" + value["version"] = "fv_multi_output" value["signature"] = { "inputs": [ {"name": "title", "arrow_type": "utf8", "nullable": True}, @@ -221,7 +218,6 @@ def test_function_application_uses_rename_columns_only(): assert application.columns["normalized_text"] == "search_text" assert renamed.columns["normalized_text"] == "body_normalized" assert renamed.function == application.function - assert renamed.group_id == application.group_id assert not hasattr(application, "rename_outputs") with pytest.raises(TypeError, match="immutable"): renamed.columns["normalized_text"] = "changed" @@ -242,7 +238,6 @@ def test_function_application_uses_rename_columns_only(): def test_binding_and_refresh_result_keep_stable_remote_fields(): binding = FunctionBinding.from_json(fixture("remote_function_binding.json")) - assert binding.revision == 3 assert binding.function.version == "fv_01K3TEXT" assert [output.output_ordinal for output in binding.outputs] == [0, 1] assert binding.input_schema is not None @@ -322,7 +317,7 @@ def known_application() -> FunctionApplication: @pytest.mark.asyncio -async def test_add_columns_routes_struct_as_one_and_grouped_expansion_atomically(): +async def test_add_columns_routes_struct_as_one_and_multi_output_binding_atomically(): inner = _FunctionDeclarationInner() table = AsyncTable(inner) application = known_application() @@ -343,12 +338,12 @@ async def test_add_columns_routes_struct_as_one_and_grouped_expansion_atomically @pytest.mark.asyncio -async def test_add_columns_rejects_mixed_groups_and_unknown_newer_application(): +async def test_add_columns_rejects_multiple_bindings_and_unknown_newer_application(): inner = _FunctionDeclarationInner() table = AsyncTable(inner) application = known_application() - with pytest.raises(ValueError, match="exactly one Function sibling group"): + with pytest.raises(ValueError, match="exactly one Function binding"): await table.add_columns({"a": application, "b": application}) future = json.loads(fixture("remote_function_application.json")) @@ -376,7 +371,6 @@ def test_rename_requires_named_struct_and_keeps_partial_mapping_immutable(): "arrow_type": "list", "nullable": False, }, - "group_id": "fg_scalar", } ) ) diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index f02158871..433a7e04c 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -446,7 +446,6 @@ pub struct FunctionApplication { function: FunctionVersionRef, inputs: Vec, output: FunctionOutput, - group_id: String, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] columns: BTreeMap, #[serde(default, flatten, skip_serializing)] @@ -468,10 +467,6 @@ impl FunctionApplication { &self.output } - pub fn group_id(&self) -> &str { - &self.group_id - } - pub fn columns(&self) -> &BTreeMap { &self.columns } @@ -513,7 +508,7 @@ pub struct InputBinding { pub nullable: bool, } -/// Ordered result-field to table-field mapping for a grouped binding. +/// Ordered result-field to table-field mapping for a Function binding. /// /// Assignment state is not part of the Slice 1 client contract. During the /// NULL transition there is no public Lance cell-flag identifier to persist. @@ -527,20 +522,18 @@ pub struct OutputMapping { pub nullable: bool, } -/// Immutable grouped binding persisted by the Enterprise table service. +/// Immutable Function binding persisted by the Enterprise table service. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FunctionBinding { binding_id: String, - revision: u64, function: FunctionVersionRef, - group_id: String, inputs: Vec, outputs: Vec, /// Exact Arrow schema presented to the Function, encoded with the Lance /// Namespace Arrow JSON representation. #[serde(default, skip_serializing_if = "Option::is_none")] input_schema: Option, - /// Exact physical Arrow schema of the grouped table outputs. + /// Exact physical Arrow schema of the binding's table outputs. #[serde(default, skip_serializing_if = "Option::is_none")] output_schema: Option, } @@ -550,18 +543,10 @@ impl FunctionBinding { &self.binding_id } - pub fn revision(&self) -> u64 { - self.revision - } - pub fn function(&self) -> &FunctionVersionRef { &self.function } - pub fn group_id(&self) -> &str { - &self.group_id - } - pub fn inputs(&self) -> &[InputBinding] { &self.inputs } diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index b2c5fefbe..654777adb 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -1774,11 +1774,14 @@ mod tests { .postfilter(); let result = query.execute().await; let mut stream = result.expect("should have result"); - // should only have one batch + let mut num_rows = 0; while let Some(batch) = stream.next().await { - // post filter should have removed some rows - assert!(batch.expect("should be Ok").num_rows() < 10); + let batch = batch.expect("should be Ok"); + let ids: &Int32Array = batch["id"].as_primitive(); + assert!(ids.iter().all(|id| id.unwrap() % 2 == 0)); + num_rows += batch.num_rows(); } + assert!(num_rows <= 10); let query = table .query() diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 35ac56970..742ab547d 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -6787,8 +6787,7 @@ mod tests { r#"{ "function":{"name":"embed","version":"fv_01K3EXACT"}, "inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}], - "output":{"kind":"scalar","arrow_type":"list","nullable":false}, - "group_id":"fg_scalar" + "output":{"kind":"scalar","arrow_type":"list","nullable":false} }"#, ) .unwrap(); @@ -6834,8 +6833,7 @@ mod tests { r#"{ "function":{"name":"embed","version":"fv_01K3EXACT"}, "inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}], - "output":{"kind":"scalar","arrow_type":"fixed_size_list","nullable":false}, - "group_id":"fg_fixed" + "output":{"kind":"scalar","arrow_type":"fixed_size_list","nullable":false} }"#, ) .unwrap(); @@ -6850,7 +6848,7 @@ mod tests { } #[tokio::test] - async fn test_add_named_struct_function_expands_one_atomic_sibling_group() { + async fn test_add_named_struct_function_expands_one_atomic_binding() { let table = Table::new_with_handler("my_table", |request| match request.url().path() { "/v1/table/my_table/describe/" => http::Response::builder() .status(200) @@ -6865,7 +6863,7 @@ mod tests { let actual: serde_json::Value = serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); let expected: serde_json::Value = serde_json::from_str(include_str!( - "../../tests/fixtures/first_class_functions/v1/remote_grouped_declaration_request.json" + "../../tests/fixtures/first_class_functions/v1/remote_multi_output_declaration_request.json" )) .unwrap(); assert_eq!(actual, expected); @@ -6887,7 +6885,6 @@ mod tests { {"name":"normalized_text","arrow_type":"utf8","nullable":false}, {"name":"token_count","arrow_type":"int64","nullable":false} ]}, - "group_id":"fg_01K3TEXT", "columns":{"normalized_text":"search_text"} }"#, ) diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 356c87bbe..ecd95f161 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -751,7 +751,7 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { message: "computed columns are not supported on this table type".into(), }) } - /// Declare one immutable registered-Function output group. + /// Declare one immutable registered-Function binding. async fn add_function_columns( &self, _application: &crate::function::FunctionApplication, diff --git a/rust/lancedb/src/table/add_columns.rs b/rust/lancedb/src/table/add_columns.rs index 3b91c30e4..1ac0c6b4f 100644 --- a/rust/lancedb/src/table/add_columns.rs +++ b/rust/lancedb/src/table/add_columns.rs @@ -88,7 +88,7 @@ impl AddColumnsBuilder { } /// Declare every field of a named-struct Function result as one atomic - /// sibling group. Result-field aliases come from + /// binding. Result-field aliases come from /// [`FunctionApplication::columns`](crate::function::FunctionApplication::columns). /// /// ``` diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index 133f37a55..2b95cb34c 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -13,7 +13,7 @@ //! self-describing -- both are derived from the expression, so a caller writes //! neither -- while a kind resolved through a registry cannot be typed without //! consulting it. Registered Functions use an exact remote version plus a -//! schema-level grouped binding; unknown newer kinds remain readable and fail +//! schema-level Function binding; unknown newer kinds remain readable and fail //! closed before mutation. //! //! [`computed_columns`] and [`computed_column_from_field`] read declarations @@ -46,16 +46,16 @@ pub const EXPRESSION_META_KEY: &str = "computed_column.expression"; /// Field metadata key holding the column's inputs, as a JSON array of names. pub const INPUTS_META_KEY: &str = "computed_column.inputs"; -/// Field metadata key holding the grouped Function binding identity. +/// Field metadata key holding the Function binding identity. pub const FUNCTION_BINDING_ID_META_KEY: &str = "computed_column.function.binding_id"; /// Field metadata key holding this sibling's ordered Function output ordinal. pub const FUNCTION_OUTPUT_ORDINAL_META_KEY: &str = "computed_column.function.output_ordinal"; -/// Schema metadata key holding all immutable grouped Function bindings. +/// Schema metadata key holding all immutable Function bindings. pub const FUNCTION_BINDINGS_META_KEY: &str = "lancedb::function_bindings"; -/// Version of the schema-level grouped Function binding envelope. +/// Version of the schema-level Function binding envelope. pub const FUNCTION_BINDINGS_VERSION: u32 = 1; /// Value of [`KIND_META_KEY`] for a column defined by a SQL expression. @@ -81,7 +81,7 @@ pub enum ComputedColumnKind { /// The expression. expression: String, }, - /// One physical output in an immutable grouped registered-Function + /// One physical output in an immutable registered-Function /// binding. The full binding lives in schema metadata. Function { /// Shared immutable binding identity. @@ -159,7 +159,7 @@ struct FunctionBindingEnvelope { bindings: Vec, } -/// Encode immutable grouped bindings for schema-level persistence. +/// Encode immutable Function bindings for schema-level persistence. pub fn function_bindings_metadata(bindings: &[FunctionBinding]) -> Result { let bindings = bindings .iter() @@ -177,7 +177,7 @@ pub fn function_bindings_metadata(bindings: &[FunctionBinding]) -> Result Result> { let Some(envelope) = function_binding_envelope(schema)? else { @@ -238,21 +238,15 @@ pub(crate) fn ensure_supported_function_metadata(schema: &ArrowSchema) -> Result message: format!("duplicate Function binding '{}'", binding.binding_id()), }); } - if binding.revision() == 0 || binding.outputs().is_empty() { + if binding.outputs().is_empty() { return Err(Error::InvalidInput { - message: format!( - "Function binding '{}' has no immutable revision or outputs", - binding.binding_id() - ), + message: format!("Function binding '{}' has no outputs", binding.binding_id()), }); } - if binding.function().name.is_empty() - || binding.function().version.is_empty() - || binding.group_id().is_empty() - { + if binding.function().name.is_empty() || binding.function().version.is_empty() { return Err(Error::InvalidInput { message: format!( - "Function binding '{}' has no exact version or group identity", + "Function binding '{}' has no exact version", binding.binding_id() ), }); @@ -493,9 +487,7 @@ fn ensure_known_binding_shape(value: &Value) -> Result<()> { value, &[ "binding_id", - "revision", "function", - "group_id", "inputs", "outputs", "input_schema", @@ -786,12 +778,9 @@ pub(crate) fn plan_function_application( message: "Function application contains fields from a newer contract".into(), }); } - if application.function().name.is_empty() - || application.function().version.is_empty() - || application.group_id().is_empty() - { + if application.function().name.is_empty() || application.function().version.is_empty() { return Err(invalid_function( - "Function application requires an exact version and group identity", + "Function application requires an exact version", )); } @@ -2261,7 +2250,6 @@ mod tests { {{"name":"normalized_text","arrow_type":"utf8","nullable":false}}, {{"name":"token_count","arrow_type":"int64","nullable":false}} ]}}, - "group_id":"fg_exact", "columns":{columns} }}"# )) @@ -2442,8 +2430,7 @@ mod tests { r#"{ "function":{"name":"f","version":"fv"}, "inputs":[{"parameter":"title","kind":"future_source","value":{"path":"title"}}], - "output":{"kind":"scalar","arrow_type":"int64","nullable":false}, - "group_id":"fg" + "output":{"kind":"scalar","arrow_type":"int64","nullable":false} }"#, ) .unwrap(); @@ -2456,7 +2443,6 @@ mod tests { "function":{"name":"f","version":"fv"}, "inputs":[], "output":{"kind":"scalar","arrow_type":"int64","nullable":false}, - "group_id":"fg", "future_declaration":{"mode":"managed"} }"#, ) @@ -2470,8 +2456,7 @@ mod tests { r#"{ "function":{"name":"f","version":"fv"}, "inputs":[], - "output":{"kind":"scalar","arrow_type":"int64","nullable":false,"assignment":"cell_flag"}, - "group_id":"fg" + "output":{"kind":"scalar","arrow_type":"int64","nullable":false,"assignment":"cell_flag"} }"#, ) .unwrap(); diff --git a/rust/lancedb/tests/first_class_function_slice1.rs b/rust/lancedb/tests/first_class_function_slice1.rs index dc565d309..aec264650 100644 --- a/rust/lancedb/tests/first_class_function_slice1.rs +++ b/rust/lancedb/tests/first_class_function_slice1.rs @@ -84,7 +84,6 @@ fn application_and_binding_match_shared_remote_goldens() { let binding = FunctionBinding::from_json(&fixture("remote_function_binding.json")) .expect("binding fixture"); - assert_eq!(binding.revision(), 3); assert_eq!(binding.function().version, "fv_01K3TEXT"); assert_eq!(binding.outputs()[0].output_ordinal, 0); assert_eq!(binding.outputs()[1].output_ordinal, 1); diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json index 7d92f1454..fd3944412 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json @@ -24,8 +24,7 @@ "kind": "scalar", "arrow_type": "fixed_size_list", "nullable": false - }, - "group_id": "fg_fixed" + } }, "binding_metadata_version": 1, "input_bindings": [ @@ -76,4 +75,4 @@ } ] } -} \ No newline at end of file +} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.canonical.json index 05da9fe39..b91dd1061 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.canonical.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.canonical.json @@ -1 +1 @@ -{"columns":{"normalized_text":"search_text","token_count":"search_token_count"},"function":{"name":"text_features","version":"fv_01K3TEXT"},"group_id":"fg_01K3TEXT","inputs":[{"kind":"column","parameter":"title","value":{"path":"title"}},{"kind":"column","parameter":"body","value":{"path":"body"}}],"output":{"fields":[{"arrow_type":"utf8","name":"normalized_text","nullable":false},{"arrow_type":"int64","name":"token_count","nullable":false}],"kind":"named_struct"}} +{"columns":{"normalized_text":"search_text","token_count":"search_token_count"},"function":{"name":"text_features","version":"fv_01K3TEXT"},"inputs":[{"kind":"column","parameter":"title","value":{"path":"title"}},{"kind":"column","parameter":"body","value":{"path":"body"}}],"output":{"fields":[{"arrow_type":"utf8","name":"normalized_text","nullable":false},{"arrow_type":"int64","name":"token_count","nullable":false}],"kind":"named_struct"}} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.json index 44aeff460..177821aaa 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.json @@ -11,7 +11,6 @@ {"name": "token_count", "arrow_type": "int64", "nullable": false} ] }, - "group_id": "fg_01K3TEXT", "columns": { "normalized_text": "search_text", "token_count": "search_token_count" diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application_float.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application_float.json index 47724eee0..c23c8f3ca 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application_float.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application_float.json @@ -3,6 +3,5 @@ "inputs": [ {"parameter": "threshold", "kind": "literal", "value": 1e-7} ], - "output": {"kind": "scalar", "arrow_type": "bool", "nullable": false}, - "group_id": "fg_01K3FLOAT" + "output": {"kind": "scalar", "arrow_type": "bool", "nullable": false} } diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.canonical.json index 7bf93b8a8..143190f78 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.canonical.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.canonical.json @@ -1 +1 @@ -{"binding_id":"fb_01K3TEXT","function":{"name":"text_features","version":"fv_01K3TEXT"},"group_id":"fg_01K3TEXT","input_schema":{"fields":[{"name":"title","nullable":true,"type":{"type":"utf8"}},{"name":"body","nullable":true,"type":{"type":"utf8"}}]},"inputs":[{"arrow_type":"utf8","field_id":11,"field_path":"title","nullable":true,"parameter":"title"},{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"body"}],"output_schema":{"fields":[{"name":"search_text","nullable":true,"type":{"type":"utf8"}},{"name":"search_token_count","nullable":true,"type":{"type":"int64"}}]},"outputs":[{"arrow_type":"utf8","nullable":false,"output_field_id":21,"output_name":"search_text","output_ordinal":0,"result_field":"normalized_text"},{"arrow_type":"int64","nullable":false,"output_field_id":22,"output_name":"search_token_count","output_ordinal":1,"result_field":"token_count"}],"revision":3} +{"binding_id":"fb_01K3TEXT","function":{"name":"text_features","version":"fv_01K3TEXT"},"input_schema":{"fields":[{"name":"title","nullable":true,"type":{"type":"utf8"}},{"name":"body","nullable":true,"type":{"type":"utf8"}}]},"inputs":[{"arrow_type":"utf8","field_id":11,"field_path":"title","nullable":true,"parameter":"title"},{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"body"}],"output_schema":{"fields":[{"name":"search_text","nullable":true,"type":{"type":"utf8"}},{"name":"search_token_count","nullable":true,"type":{"type":"int64"}}]},"outputs":[{"arrow_type":"utf8","nullable":false,"output_field_id":21,"output_name":"search_text","output_ordinal":0,"result_field":"normalized_text"},{"arrow_type":"int64","nullable":false,"output_field_id":22,"output_name":"search_token_count","output_ordinal":1,"result_field":"token_count"}]} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.json index 1a2053e42..dec3fa3f8 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.json @@ -1,8 +1,6 @@ { "binding_id": "fb_01K3TEXT", - "revision": 3, "function": {"name": "text_features", "version": "fv_01K3TEXT"}, - "group_id": "fg_01K3TEXT", "inputs": [ {"parameter": "title", "field_id": 11, "field_path": "title", "arrow_type": "utf8", "nullable": true}, {"parameter": "body", "field_id": 12, "field_path": "body", "arrow_type": "utf8", "nullable": true} @@ -23,5 +21,5 @@ {"name": "search_token_count", "nullable": true, "type": {"type": "int64"}} ] }, - "future_binding": {"metadata_revision": 1} + "future_binding": {"mode": "managed"} } diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_grouped_declaration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_multi_output_declaration_request.json similarity index 97% rename from rust/lancedb/tests/fixtures/first_class_functions/v1/remote_grouped_declaration_request.json rename to rust/lancedb/tests/fixtures/first_class_functions/v1/remote_multi_output_declaration_request.json index d0b42cc99..c4ef5009f 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_grouped_declaration_request.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_multi_output_declaration_request.json @@ -17,7 +17,6 @@ {"name": "token_count", "arrow_type": "int64", "nullable": false} ] }, - "group_id": "fg_01K3TEXT", "columns": {"normalized_text": "search_text"} }, "binding_metadata_version": 1, diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_job.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_job.json index bb2490bc8..c2d49827d 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_job.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_job.json @@ -3,7 +3,7 @@ "job_type": "refresh_function_columns", "job_state": "DONE", "creation_ms": 1787270400001, - "spec": {"table": "documents", "binding_revision": 3}, + "spec": {"table": "documents", "binding_id": "fb_01K3TEXT"}, "result": { "rows_assigned": 999998800, "rows_failed": 0, diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json index 0aaa0cf72..357834de0 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json @@ -8,8 +8,7 @@ "inputs": [ {"parameter": "text", "kind": "column", "value": {"path": "description"}} ], - "output": {"kind": "scalar", "arrow_type": "list", "nullable": false}, - "group_id": "fg_scalar" + "output": {"kind": "scalar", "arrow_type": "list", "nullable": false} }, "binding_metadata_version": 1, "input_bindings": [ From 81c3f108ce35f148123b831c399c335c0c79b46a Mon Sep 17 00:00:00 2001 From: Lance Release Date: Tue, 25 Aug 2026 10:31:36 +0000 Subject: [PATCH 47/58] =?UTF-8?q?Bump=20version:=200.38.0-beta.8=20?= =?UTF-8?q?=E2=86=92=200.38.0-beta.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 86a5064f7..fb1323d7e 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.8" +current_version = "0.38.0-beta.9" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 8d0058569..80db934d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5402,7 +5402,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.8" +version = "0.38.0-beta.9" dependencies = [ "ahash", "anyhow", @@ -5490,7 +5490,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.8" +version = "0.38.0-beta.9" dependencies = [ "arrow-array", "arrow-buffer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.8" +version = "0.38.0-beta.9" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 28de11c30..7aa12d0a0 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.8 + 0.38.0-beta.9 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 6f4b0744b..58d75d214 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.8 + 0.38.0-beta.9 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index ac6196a45..69c8f5ce1 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.8 + 0.38.0-beta.9 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index e85da66ec..99decbd9d 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.8" +version = "0.38.0-beta.9" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index f920c9e91..911d6495f 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.8", + "version": "0.38.0-beta.9", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index ad6926039..078b32dd1 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.8", + "version": "0.38.0-beta.9", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index ccd39f18e..0fc54cdee 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.8", + "version": "0.38.0-beta.9", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 50446b597..ecaf22680 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.8", + "version": "0.38.0-beta.9", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index d4da82dc7..081c4c8ba 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.8", + "version": "0.38.0-beta.9", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index dddc3a470..609f701b7 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.8", + "version": "0.38.0-beta.9", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 23ac122f7..59eb8e06e 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.8", + "version": "0.38.0-beta.9", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 21deb8dd8..d8b38ce0f 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.8", + "version": "0.38.0-beta.9", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.8", + "version": "0.38.0-beta.9", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index 96f4bafd5..2c0ff7d69 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.8", + "version": "0.38.0-beta.9", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index bffac2ef1..8c73b602d 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.8" +version = "0.38.0-beta.9" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 7290694f1..6f194556a 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.8" +version = "0.38.0-beta.9" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From d0bcc6c6fe71b057f53be07a75be4bdefd463fb8 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 25 Aug 2026 18:35:27 +0800 Subject: [PATCH 48/58] fix: remove unsupported Function secrets contract (#4047) ## Problem The unreleased First-Class Function authoring API exposed `secrets=[...]` and serialized `required_secrets`, promising runtime resolution and injection that Sophon does not implement. ## Behavior Remove the secrets dimension from the public Python decorator, Python and Rust registration/version models, and shared wire fixtures. Existing successfully registered Functions retain stable identity: the field could only be empty and empty values were already omitted from canonical serialization. A stable LanceDB release has not shipped this Function API, so this contracts the surface before it becomes a published compatibility commitment. ## Validation gap The Rust shared-golden Function suites and Python formatting/lint checks pass locally. Python pytest was not run because the local environment lacks its runtime dependencies and the frozen native editable build did not complete in practical time. --- python/python/lancedb/functions.py | 37 +++--------------- .../tests/test_first_class_function_slice1.py | 25 ------------ .../tests/test_first_class_function_slice2.py | 29 -------------- rust/lancedb/src/function.rs | 20 +--------- .../tests/first_class_function_slice1.rs | 38 ------------------- .../tests/first_class_function_slice2.rs | 26 ------------- .../v1/remote_function_job.json | 1 - ...nction_registration_request.canonical.json | 2 +- .../remote_function_registration_request.json | 5 +-- .../v1/remote_function_version.canonical.json | 2 +- 10 files changed, 10 insertions(+), 175 deletions(-) diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index dd8cecfa9..237ed73c2 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -4,7 +4,7 @@ """Canonical Function values exchanged with LanceDB Enterprise services. These immutable models contain client/wire state only. Catalog persistence, -environment bake, secret resolution, and execution are owned by Sophon. +environment bake, and execution are owned by Sophon. ``RefreshColumnResult`` is also the backend-neutral result of a local expression-backed refresh job. """ @@ -228,7 +228,7 @@ class PythonEnvironmentSpec(_RemoteValue): class PythonRuntimeSpec(_RemoteValue): - """Remote runtime definition with non-secret environment values. + """Remote runtime definition with environment values. V1 supports ``kind="python"``. Newer runtime kinds remain readable, while their unknown payload fields are intentionally not retained by the client. @@ -267,7 +267,6 @@ class FunctionVersion(_RemoteValue): runtime: PythonRuntimeSpec runtime_digest: str environment_digest: str - required_secrets: tuple[str, ...] = () created_at: str def __call__(self, **inputs: Any) -> FunctionApplication: @@ -329,17 +328,12 @@ class FunctionVersion(_RemoteValue): class FunctionRegistrationRequest(_RemoteValue): - """Stable remote registration envelope produced by :func:`udf`. - - Only secret names are represented. Secret values are resolved inside the - remote service and have no client request field. - """ + """Stable remote registration envelope produced by :func:`udf`.""" name: str artifact: FunctionArtifactRequest signature: FunctionSignature runtime: PythonRuntimeSpec - required_secrets: tuple[str, ...] = () class FunctionVersionRef(_OpenRemoteValue): @@ -484,7 +478,6 @@ class RefreshColumnResult(_RemoteValue): _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$") -_SECRET_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") _GRAMMAR_PRIMITIVES = ( @@ -915,7 +908,6 @@ class UdfDefinition: output_schema: Optional[pa.DataType | pa.Field | pa.Schema], pip: tuple[str, ...], env: Mapping[str, str], - secrets: tuple[str, ...], python_version: Optional[str], ): function_name = name or function.__name__ @@ -930,18 +922,6 @@ class UdfDefinition: for key, value in environment.items() ): raise TypeError("Function env keys and values must be strings") - required_secrets = tuple(sorted(set(secrets))) - invalid_secrets = [ - secret for secret in required_secrets if not _SECRET_NAME.fullmatch(secret) - ] - if invalid_secrets: - raise ValueError(f"invalid Function secret names: {invalid_secrets!r}") - overlap = set(environment) & set(required_secrets) - if overlap: - raise ValueError( - f"Function env and secret names must be disjoint: {sorted(overlap)!r}" - ) - signature = _infer_signature(function, input_schema, output_schema) source = _package_source(function) digest = f"sha256:{hashlib.sha256(source).hexdigest()}" @@ -970,7 +950,6 @@ class UdfDefinition: ), signature=signature, runtime=runtime, - required_secrets=required_secrets, ) functools.update_wrapper(self, function) @@ -996,7 +975,6 @@ def udf( output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None, pip: tuple[str, ...] | list[str] = (), env: Optional[Mapping[str, str]] = None, - secrets: tuple[str, ...] | list[str] = (), python_version: Optional[str] = None, ) -> Callable[[Callable[..., Any]], UdfDefinition]: ... @@ -1009,7 +987,6 @@ def udf( output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None, pip: tuple[str, ...] | list[str] = (), env: Optional[Mapping[str, str]] = None, - secrets: tuple[str, ...] | list[str] = (), python_version: Optional[str] = None, ): """Prepare a scalar Python callable for remote Function registration. @@ -1034,10 +1011,7 @@ def udf( pip : sequence of str, optional Pip requirements for the remote environment. env : mapping of str to str, optional - Non-secret environment variables. Use ``secrets`` for credentials. - secrets : sequence of str, optional - Names of secrets resolved by the remote service. Secret values are not - accepted by this API or included in the registration request. + Environment variables included in the Function definition. python_version : str, optional Remote Python major/minor version. Defaults to the client version. @@ -1059,7 +1033,7 @@ def udf( Examples -------- >>> from lancedb import udf - >>> @udf(pip=["numpy==2.2.0"], secrets=["MODEL_TOKEN"]) + >>> @udf(pip=["numpy==2.2.0"]) ... def score(value: float) -> float: ... return value * 2 >>> score(1.5) @@ -1074,7 +1048,6 @@ def udf( output_schema=output_schema, pip=tuple(pip), env={} if env is None else env, - secrets=tuple(secrets), python_version=python_version, ) diff --git a/python/python/tests/test_first_class_function_slice1.py b/python/python/tests/test_first_class_function_slice1.py index ca0b30ede..89172ba3f 100644 --- a/python/python/tests/test_first_class_function_slice1.py +++ b/python/python/tests/test_first_class_function_slice1.py @@ -37,21 +37,6 @@ def job_result(name: str) -> dict: return json.loads(fixture(name))["result"] -def assert_no_secret_values(value): - if isinstance(value, dict): - for key, child in value.items(): - assert key not in { - "secret_value", - "secret_values", - "resolved_secret", - "resolved_secrets", - } - assert_no_secret_values(child) - elif isinstance(value, list): - for child in value: - assert_no_secret_values(child) - - def test_public_function_values_are_in_api_reference(): docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md" rendered = docs.read_text() @@ -109,7 +94,6 @@ def test_function_version_identity_is_immutable_and_exact(): version = FunctionVersion.from_json(json.dumps(value)) assert version.name == "embed" assert version.version == "fv_01K3EXACT" - assert version.required_secrets == ("HF_TOKEN",) with pytest.raises((TypeError, ValueError)): version.version = "fv_changed" @@ -292,15 +276,6 @@ def test_refresh_result_rejects_non_u64_values(field): RefreshColumnResult.from_json(json.dumps(value)) -def test_canonical_client_values_contain_secret_names_only(): - version = FunctionVersion.from_json( - json.dumps(job_result("remote_function_job.json")) - ) - canonical = json.loads(version.to_canonical_json()) - assert canonical["required_secrets"] == ["HF_TOKEN"] - assert_no_secret_values(canonical) - - class _FunctionDeclarationInner: def __init__(self): self.calls = [] diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index c67e17520..55257d322 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -39,28 +39,12 @@ FIXTURES = ( @udf( pip=["numpy>=2"], env={"MODE": "test"}, - secrets=["API_TOKEN"], python_version="3.12", ) def normalize_score(value: float) -> float: return value / 100.0 -def _assert_no_secret_values(value): - if isinstance(value, dict): - for key, child in value.items(): - assert key not in { - "secret_value", - "secret_values", - "resolved_secret", - "resolved_secrets", - } - _assert_no_secret_values(child) - elif isinstance(value, list): - for child in value: - _assert_no_secret_values(child) - - def test_scalar_udf_matches_shared_registration_golden_and_remains_callable(): assert isinstance(normalize_score, UdfDefinition) assert normalize_score(25.0) == 0.25 @@ -75,8 +59,6 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable(): "kind": "scalar_to_arrow_batch", "version": 1, } - assert request["required_secrets"] == ["API_TOKEN"] - _assert_no_secret_values(request) def _run_packaged(definition, *args): @@ -370,7 +352,6 @@ def test_udf_recursion_versus_a_rebound_module_name(tmp_path): output_schema=None, pip=(), env={}, - secrets=(), python_version=None, ) with pytest.raises(ValueError, match="binds that name to another value"): @@ -525,14 +506,6 @@ def test_annotation_and_explicit_schema_validation_fail_closed(): return value -def test_environment_rejects_secret_value_overlap(): - with pytest.raises(ValueError, match="must be disjoint"): - - @udf(env={"TOKEN": "plaintext"}, secrets=["TOKEN"]) - def overlapping(value: int) -> int: - return value - - def test_local_function_catalog_operations_are_not_supported(tmp_path): db = lancedb.connect(tmp_path) message = "Function catalog operations are not supported by this database" @@ -569,7 +542,6 @@ def _mock_remote_function_catalog(): "runtime": body["runtime"], "runtime_digest": "sha256:runtime", "environment_digest": "sha256:environment", - "required_secrets": body.get("required_secrets", []), "created_at": "2026-08-21T00:00:00Z", } response = {"job_id": "job-register"} @@ -628,7 +600,6 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip(): assert create_request == json.loads( normalize_score.registration_request.to_canonical_json() ) - _assert_no_secret_values(create_request) def test_blocking_remote_registration_returns_function_version(): diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index 433a7e04c..52c70a4b1 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -5,7 +5,7 @@ //! backend-neutral terminal result of a computed-column refresh. //! //! This module contains client/wire values only. Catalog persistence, -//! environment bake, secret resolution, and execution are owned by Sophon. +//! environment bake, and execution are owned by Sophon. use std::collections::BTreeMap; @@ -195,9 +195,6 @@ pub struct PythonEnvironmentSpec { } /// Reproducible Python runtime definition understood by Sophon. -/// -/// `env` contains non-secret values. Secret values have no client model; -/// [`FunctionVersion::required_secrets`] contains names only. #[derive(Debug, Clone, PartialEq, Eq)] #[non_exhaustive] pub enum PythonRuntimeSpec { @@ -239,7 +236,7 @@ impl PythonRuntimeSpec { } } - /// Non-secret environment variables, or `None` for an unknown kind. + /// Environment variables, or `None` for an unknown kind. pub fn env(&self) -> Option<&BTreeMap> { match self { Self::Python { env, .. } => Some(env), @@ -324,8 +321,6 @@ pub struct FunctionVersion { runtime: PythonRuntimeSpec, runtime_digest: String, environment_digest: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - required_secrets: Vec, created_at: String, } @@ -358,11 +353,6 @@ impl FunctionVersion { &self.environment_digest } - /// Required secret names. Resolved values exist only inside Sophon. - pub fn required_secrets(&self) -> &[String] { - &self.required_secrets - } - pub fn created_at(&self) -> &str { &self.created_at } @@ -404,18 +394,12 @@ pub struct FunctionArtifactRequest { } /// Stable request envelope for remote immutable Function registration. -/// -/// Secret values deliberately have no field in this model. The only secret -/// material the client may send is the ordered set of names Sophon resolves -/// inside the remote runtime. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FunctionRegistrationRequest { pub name: String, pub artifact: FunctionArtifactRequest, pub signature: FunctionSignature, pub runtime: PythonRuntimeSpec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub required_secrets: Vec, } impl_json!(FunctionRegistrationRequest); diff --git a/rust/lancedb/tests/first_class_function_slice1.rs b/rust/lancedb/tests/first_class_function_slice1.rs index aec264650..ce020bd53 100644 --- a/rust/lancedb/tests/first_class_function_slice1.rs +++ b/rust/lancedb/tests/first_class_function_slice1.rs @@ -20,25 +20,6 @@ fn job_result(name: &str) -> Value { serde_json::from_str::(&fixture(name)).expect("remote Job fixture")["result"].clone() } -fn assert_no_secret_values(value: &Value) { - match value { - Value::Object(values) => { - for (key, value) in values { - assert!( - !matches!( - key.as_str(), - "secret_value" | "secret_values" | "resolved_secret" | "resolved_secrets" - ), - "client canonical value must not model resolved secret material" - ); - assert_no_secret_values(value); - } - } - Value::Array(values) => values.iter().for_each(assert_no_secret_values), - _ => {} - } -} - #[test] fn function_version_job_result_matches_shared_canonical_golden() { let result = job_result("remote_function_job.json"); @@ -47,7 +28,6 @@ fn function_version_job_result_matches_shared_canonical_golden() { assert_eq!(version.name(), "embed"); assert_eq!(version.version(), "fv_01K3EXACT"); assert_eq!(version.runtime_digest(), "sha256:runtime"); - assert_eq!(version.required_secrets(), &["HF_TOKEN"]); assert_eq!( version.to_canonical_json().expect("canonical JSON"), fixture("remote_function_version.canonical.json").trim() @@ -162,21 +142,3 @@ fn floating_point_application_literals_are_rejected_consistently() { .contains("floating-point Function literals") ); } - -#[test] -fn canonical_client_values_contain_secret_names_only() { - let result = job_result("remote_function_job.json"); - let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result"); - let canonical: Value = serde_json::from_str( - &version - .to_canonical_json() - .expect("canonical FunctionVersion"), - ) - .expect("canonical JSON"); - - assert_eq!( - canonical["required_secrets"], - serde_json::json!(["HF_TOKEN"]) - ); - assert_no_secret_values(&canonical); -} diff --git a/rust/lancedb/tests/first_class_function_slice2.rs b/rust/lancedb/tests/first_class_function_slice2.rs index 3bae57122..93252dde4 100644 --- a/rust/lancedb/tests/first_class_function_slice2.rs +++ b/rust/lancedb/tests/first_class_function_slice2.rs @@ -6,7 +6,6 @@ use std::path::PathBuf; use lancedb::Error; use lancedb::function::FunctionRegistrationRequest; -use serde_json::Value; fn fixture(name: &str) -> String { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -15,25 +14,6 @@ fn fixture(name: &str) -> String { fs::read_to_string(path).expect("fixture must be readable") } -fn assert_no_secret_values(value: &Value) { - match value { - Value::Object(values) => { - for (key, value) in values { - assert!( - !matches!( - key.as_str(), - "secret_value" | "secret_values" | "resolved_secret" | "resolved_secrets" - ), - "registration requests must not model resolved secret material" - ); - assert_no_secret_values(value); - } - } - Value::Array(values) => values.iter().for_each(assert_no_secret_values), - _ => {} - } -} - #[test] fn registration_request_matches_shared_canonical_golden() { let request = FunctionRegistrationRequest::from_json(&fixture( @@ -42,16 +22,10 @@ fn registration_request_matches_shared_canonical_golden() { .expect("registration request"); assert_eq!(request.name, "normalize_score"); assert_eq!(request.artifact.adapter.kind, "scalar_to_arrow_batch"); - assert_eq!(request.required_secrets, ["API_TOKEN"]); assert_eq!( request.to_canonical_json().expect("canonical request"), fixture("remote_function_registration_request.canonical.json").trim() ); - - let value: Value = - serde_json::from_str(&request.to_canonical_json().expect("canonical request")) - .expect("request JSON"); - assert_no_secret_values(&value); } #[tokio::test] diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json index 6ba4eb226..39a279692 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json @@ -24,7 +24,6 @@ }, "runtime_digest": "sha256:runtime", "environment_digest": "sha256:environment", - "required_secrets": ["HF_TOKEN"], "created_at": "2026-08-21T00:00:00Z" }, "future_job": {"trace_id": "trace-1"} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.canonical.json index 24fa2cf30..a2f2c4c21 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.canonical.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.canonical.json @@ -1 +1 @@ -{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIG5vcm1hbGl6ZV9zY29yZSh2YWx1ZTogZmxvYXQpIC0+IGZsb2F0OgogICAgcmV0dXJuIHZhbHVlIC8gMTAwLjAK","encoding":"base64"},"digest":"sha256:760784bdcef57b802f389b97804cc0b618aae39e86044733451bcd0b13089a7f","entrypoint":"normalize_score","kind":"python_callable"},"name":"normalize_score","required_secrets":["API_TOKEN"],"runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["numpy>=2"]},"kind":"python","python_version":"3.12"},"signature":{"inputs":[{"arrow_type":"float64","name":"value","nullable":false}],"output":{"arrow_type":"float64","kind":"scalar","nullable":false}}} +{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIG5vcm1hbGl6ZV9zY29yZSh2YWx1ZTogZmxvYXQpIC0+IGZsb2F0OgogICAgcmV0dXJuIHZhbHVlIC8gMTAwLjAK","encoding":"base64"},"digest":"sha256:760784bdcef57b802f389b97804cc0b618aae39e86044733451bcd0b13089a7f","entrypoint":"normalize_score","kind":"python_callable"},"name":"normalize_score","runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["numpy>=2"]},"kind":"python","python_version":"3.12"},"signature":{"inputs":[{"arrow_type":"float64","name":"value","nullable":false}],"output":{"arrow_type":"float64","kind":"scalar","nullable":false}}} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.json index bbfec3169..092d76dc2 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.json @@ -39,8 +39,5 @@ "env": { "MODE": "test" } - }, - "required_secrets": [ - "API_TOKEN" - ] + } } diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json index 7ab632a98..2670ad0b2 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json @@ -1 +1 @@ -{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","required_secrets":["HF_TOKEN"],"runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"} +{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"} From ec4ad54ba253e11c4cfd894cde6d47e788447886 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Tue, 25 Aug 2026 10:36:33 +0000 Subject: [PATCH 49/58] =?UTF-8?q?Bump=20version:=200.38.0-beta.9=20?= =?UTF-8?q?=E2=86=92=200.38.0-beta.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index fb1323d7e..1c4aea809 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.9" +current_version = "0.38.0-beta.10" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 80db934d2..33e43f9fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5402,7 +5402,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.9" +version = "0.38.0-beta.10" dependencies = [ "ahash", "anyhow", @@ -5490,7 +5490,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.9" +version = "0.38.0-beta.10" dependencies = [ "arrow-array", "arrow-buffer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.9" +version = "0.38.0-beta.10" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 7aa12d0a0..06dc267f3 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.9 + 0.38.0-beta.10 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 58d75d214..25e3b10e3 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.9 + 0.38.0-beta.10 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 69c8f5ce1..4d83153ab 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.9 + 0.38.0-beta.10 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 99decbd9d..fd08e7a5e 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.9" +version = "0.38.0-beta.10" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 911d6495f..ff6347c4d 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.9", + "version": "0.38.0-beta.10", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 078b32dd1..ed99fee05 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.9", + "version": "0.38.0-beta.10", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 0fc54cdee..5b215dcc0 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.9", + "version": "0.38.0-beta.10", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index ecaf22680..e0f5a9f26 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.9", + "version": "0.38.0-beta.10", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 081c4c8ba..d42541707 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.9", + "version": "0.38.0-beta.10", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 609f701b7..496a40720 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.9", + "version": "0.38.0-beta.10", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 59eb8e06e..734013343 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.9", + "version": "0.38.0-beta.10", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index d8b38ce0f..732a7c01c 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.9", + "version": "0.38.0-beta.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.9", + "version": "0.38.0-beta.10", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index 2c0ff7d69..262d4c4a7 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.9", + "version": "0.38.0-beta.10", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 8c73b602d..3a8a05522 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.9" +version = "0.38.0-beta.10" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 6f194556a..8276e5bb3 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.9" +version = "0.38.0-beta.10" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 1d880f11ff12edbf6b5c26505b6914bd1817e9e0 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 25 Aug 2026 19:51:40 +0800 Subject: [PATCH 50/58] fix(python): use dev profile for editable builds (#4049) ## Problem `uv run ... maturin develop` synchronizes the project as an editable package before running the command. Maturin's editable build otherwise uses the release profile, which enables the repository's fat LTO configuration during local bootstrap. ## Behavior Editable Python builds now explicitly use Cargo's dev profile. The minimum maturin version is raised to 1.10, where `editable-profile` support was introduced. --- python/pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index fad3b1001..22a41a8a9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -101,9 +101,12 @@ azure = ["adlfs>=2024.2.0"] [tool.maturin] python-source = "python" module-name = "lancedb._lancedb" +# uv installs the project as an editable package before `uv run`, so keep that +# bootstrap build consistent with `maturin develop`. +editable-profile = "dev" [build-system] -requires = ["maturin>=1.9.4"] +requires = ["maturin>=1.10"] build-backend = "maturin" [tool.ruff.lint] From a614400755b00c472eba6cf743802e4f0c734132 Mon Sep 17 00:00:00 2001 From: Drew Date: Tue, 25 Aug 2026 07:59:53 -0700 Subject: [PATCH 51/58] feat: accept blob URI writes (#3954) #3528 added blob declarations and binary coercion. String values were still rejected. They now coerce to the blob `uri` child. ```python table.add([{"id": 1, "image": "s3://bucket/media/cat.jpg"}]) payload = table.fetch_blobs("image", table.search().to_arrow()) ``` A URI under a registered base writes with no extra options. An unregistered URI fails. `allow_external_blob_outside_bases` is a local escape hatch that stores an absolute URI. It does not register a base. Remote `add` rejects that flag before making a request. String input still coerces and is sent as a `uri` struct. `add_bases` is a follow-up. `merge_insert` does not coerce string blob input. ### Testing - `cargo test -p lancedb --test blob_integration` - `cargo test -p lancedb blob_coerce` - `cargo test -p lancedb --features remote --lib add_rejects_external_blob_flag add_string_blob_becomes_uri_struct` - `cd python && uv run --extra tests pytest python/tests/test_blob.py -k uri -q` Co-authored-by: Xuanwo --- python/python/lancedb/_lancedb.pyi | 1 + python/python/lancedb/remote/table.py | 4 + python/python/lancedb/table.py | 15 ++ python/python/tests/test_blob.py | 68 ++++++ python/src/table.rs | 8 +- rust/lancedb/src/remote/table.rs | 96 +++++++- rust/lancedb/src/table.rs | 5 +- rust/lancedb/src/table/add_data.rs | 14 ++ .../src/table/datafusion/blob_coerce.rs | 153 +++++++++--- rust/lancedb/tests/blob_integration.rs | 230 +++++++++++++++++- 10 files changed, 552 insertions(+), 42 deletions(-) diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 1e314ede8..593bceffa 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -283,6 +283,7 @@ class Table: mode: Literal["append", "overwrite"], progress: Optional[Any] = None, write_parallelism: Optional[int] = None, + allow_external_blob_outside_bases: bool = False, ) -> AddResult: ... async def update( self, updates: Dict[str, str], where: Optional[str] diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 41394ed71..d0bf9f67a 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -610,6 +610,7 @@ class RemoteTable(Table): fill_value: float = 0.0, progress: Optional[Union[bool, Callable, Any]] = None, write_parallelism: Optional[int] = None, + allow_external_blob_outside_bases: bool = False, ) -> AddResult: """Add more data to the [Table][lancedb.table.Table]. @@ -642,6 +643,8 @@ class RemoteTable(Table): data in flight. Defaults to an estimate based on the data size, capped at the number of CPU cores. Lower this if bulk ingestion is using too much memory. + allow_external_blob_outside_bases: bool, default False + Not supported on LanceDB Cloud. Setting this raises. Returns ------- @@ -658,6 +661,7 @@ class RemoteTable(Table): fill_value=fill_value, progress=progress, write_parallelism=write_parallelism, + allow_external_blob_outside_bases=allow_external_blob_outside_bases, ) ) finally: diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index fe25d5353..b3cab006e 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -1269,6 +1269,7 @@ class Table(ABC): fill_value: float = 0.0, progress: Optional[Union[bool, Callable, Any]] = None, write_parallelism: Optional[int] = None, + allow_external_blob_outside_bases: bool = False, ) -> AddResult: """Add more data to the [Table][lancedb.table.Table]. @@ -1320,6 +1321,10 @@ class Table(ABC): data in flight. Defaults to an estimate based on the data size, capped at the number of CPU cores. Lower this if bulk ingestion is using too much memory. + allow_external_blob_outside_bases: bool, default False + Store blob URIs that sit outside registered blob bases. The row + keeps a reference, so the object has to stay readable. Local + tables only. Returns ------- @@ -3409,6 +3414,7 @@ class LanceTable(Table): fill_value: float = 0.0, progress: Optional[Union[bool, Callable, Any]] = None, write_parallelism: Optional[int] = None, + allow_external_blob_outside_bases: bool = False, ) -> AddResult: """Add data to the table. If vector columns are missing and the table @@ -3436,6 +3442,9 @@ class LanceTable(Table): data in flight. Defaults to an estimate based on the data size, capped at the number of CPU cores. Lower this if bulk ingestion is using too much memory. + allow_external_blob_outside_bases: bool, default False + Allow blob URIs outside registered bases. See :meth:`Table.add`. + Local tables only. Returns ------- @@ -3452,6 +3461,7 @@ class LanceTable(Table): fill_value=fill_value, progress=progress, write_parallelism=write_parallelism, + allow_external_blob_outside_bases=allow_external_blob_outside_bases, ) ) finally: @@ -5365,6 +5375,7 @@ class AsyncTable: fill_value: Optional[float] = None, progress: Optional[Union[bool, Callable, Any]] = None, write_parallelism: Optional[int] = None, + allow_external_blob_outside_bases: bool = False, ) -> AddResult: """Add more data to the [AsyncTable][lancedb.table.AsyncTable]. @@ -5395,6 +5406,9 @@ class AsyncTable: data in flight. Defaults to an estimate based on the data size, capped at the number of CPU cores. Lower this if bulk ingestion is using too much memory. + allow_external_blob_outside_bases: bool, default False + Allow blob URIs outside registered bases. See :meth:`Table.add`. + Local tables only. """ schema = await self.schema() @@ -5431,6 +5445,7 @@ class AsyncTable: mode or "append", progress=progress, write_parallelism=write_parallelism, + allow_external_blob_outside_bases=allow_external_blob_outside_bases, ) except RuntimeError as e: if "Cast error" in str(e): diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index b205c20a6..5d7682f24 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -617,3 +617,71 @@ def test_fetch_blobs_nested_path_survives_sort_after_query(): def _identifiable_payload(size: int) -> bytes: block = 256 return b"".join(bytes([i % 256]) * block for i in range(size // block)) + + +def _external_uri_blob_array(uris): + blob_type = lancedb.blob("image").type + storage_type = blob_type.storage_type + child_names = [field.name for field in storage_type] + assert "uri" in child_names, "blob layout no longer has a uri child" + children = [ + pa.array(uris if field.name == "uri" else [None] * len(uris), type=field.type) + for field in storage_type + ] + storage = pa.StructArray.from_arrays(children, fields=list(storage_type)) + return pa.ExtensionArray.from_storage(blob_type, storage) + + +def _external_uri_table_and_rows(name, uris): + db = lancedb.connect("memory:///") + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = db.create_table(name, schema=schema) + rows = pa.Table.from_arrays( + [ + pa.array(range(len(uris)), type=pa.int64()), + _external_uri_blob_array(uris), + ], + schema=schema, + ) + return table, rows + + +def test_add_external_uri_struct_round_trips_with_flag(tmp_path): + payload = b"external-uri-bytes" + blob_path = tmp_path / "payload.bin" + blob_path.write_bytes(payload) + + table, rows = _external_uri_table_and_rows("external_struct", [blob_path.as_uri()]) + table.add(rows, allow_external_blob_outside_bases=True) + + hits = table.search().to_arrow() + blobs = table.fetch_blobs("image", hits) + assert blobs[0].as_py() == payload + + +def test_add_external_uri_without_flag_raises(tmp_path): + blob_path = tmp_path / "payload.bin" + blob_path.write_bytes(b"unreachable") + + table, rows = _external_uri_table_and_rows("external_no_flag", [blob_path.as_uri()]) + with pytest.raises(ValueError, match="allow_external_blob_outside_bases"): + table.add(rows) + assert table.count_rows() == 0 + + +def test_add_external_uri_string_round_trips_with_flag(tmp_path): + payload = b"external-uri-bytes" + blob_path = tmp_path / "payload.bin" + blob_path.write_bytes(payload) + + db = lancedb.connect("memory:///") + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = db.create_table("external_string", schema=schema) + table.add( + [{"id": 1, "image": blob_path.as_uri()}], + allow_external_blob_outside_bases=True, + ) + + hits = table.search().to_arrow() + blobs = table.fetch_blobs("image", hits) + assert blobs[0].as_py() == payload diff --git a/python/src/table.rs b/python/src/table.rs index b225b191f..784d29136 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -780,15 +780,19 @@ impl Table { }) } - #[pyo3(signature = (data, mode, progress=None, write_parallelism=None))] + #[pyo3(signature = (data, mode, progress=None, write_parallelism=None, allow_external_blob_outside_bases=false))] pub fn add<'a>( self_: PyRef<'a, Self>, data: PyScannable, mode: String, progress: Option>, write_parallelism: Option, + allow_external_blob_outside_bases: bool, ) -> PyResult> { - let mut op = self_.inner_ref()?.add(data); + let mut op = self_ + .inner_ref()? + .add(data) + .allow_external_blob_outside_bases(allow_external_blob_outside_bases); if mode == "append" { op = op.mode(AddDataMode::Append); } else if mode == "overwrite" { diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 742ab547d..44c92310f 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2161,6 +2161,15 @@ impl BaseTable for RemoteTable { async fn add(&self, mut add: AddDataBuilder) -> Result { self.check_mutable().await?; + if add.allow_external_blob_outside_bases { + return Err(Error::NotSupported { + message: "allow_external_blob_outside_bases is only supported on local tables" + .to_string(), + }); + } + // String blob values still coerce to the uri child in into_plan. + // Remote and local share that input shape. + let table_schema = self.schema().await?; crate::table::computed_columns::ensure_supported_function_metadata(table_schema.as_ref())?; let table_def = TableDefinition::try_from_rich_schema(table_schema.clone())?; @@ -3269,7 +3278,10 @@ mod tests { use arrow::{array::AsArray, compute::concat_batches, datatypes::Int32Type}; use arrow_array::Array; use arrow_array::builder::LargeBinaryBuilder; - use arrow_array::{BinaryArray, Int32Array, RecordBatch, RecordBatchIterator, record_batch}; + use arrow_array::{ + BinaryArray, Int32Array, Int64Array, RecordBatch, RecordBatchIterator, StringArray, + StructArray, record_batch, + }; use arrow_schema::{DataType, Field, Schema}; use chrono::{DateTime, Utc}; use futures::{StreamExt, TryFutureExt, future::BoxFuture}; @@ -3621,6 +3633,88 @@ mod tests { assert_eq!(&body, &expected_body); } + #[tokio::test] + async fn add_rejects_external_blob_flag_before_any_request() { + let table = Table::new_with_handler::("my_table", |request| { + panic!("Unexpected request: {}", request.url().path()) + }); + let data = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), + vec![Arc::new(Int32Array::from(vec![1]))], + ) + .unwrap(); + + let err = table + .add(data) + .allow_external_blob_outside_bases(true) + .execute() + .await + .unwrap_err(); + + assert!(matches!(err, Error::NotSupported { .. }), "got {err:?}"); + assert!(err.to_string().contains("local tables")); + } + + #[tokio::test] + async fn add_string_blob_becomes_uri_struct_without_the_local_flag() { + let table_schema = Schema::new(vec![ + Field::new("id", DataType::Int64, false), + crate::blob("image", true), + ]); + let describe_body = describe_response(&table_schema); + let input = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("image", DataType::Utf8, true), + ])), + vec![ + Arc::new(Int64Array::from(vec![1])), + Arc::new(StringArray::from(vec![Some("s3://bucket/key")])), + ], + ) + .unwrap(); + + let (sender, receiver) = std::sync::mpsc::channel(); + let table = + Table::new_with_handler("my_table", move |mut request| match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(describe_body.clone()) + .unwrap(), + "/v1/table/my_table/insert/" => { + let mut body_out = reqwest::Body::from(Vec::new()); + std::mem::swap(request.body_mut().as_mut().unwrap(), &mut body_out); + sender.send(body_out).unwrap(); + http::Response::builder() + .status(200) + .body(r#"{"version": 2}"#.to_string()) + .unwrap() + } + path => panic!("Unexpected path: {path}"), + }); + + table.add(input).execute().await.unwrap(); + + let body = collect_body(receiver.recv().unwrap()).await; + let mut reader = + arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(body), None).unwrap(); + let batch = reader.next().unwrap().unwrap(); + let image = batch + .column_by_name("image") + .unwrap() + .as_any() + .downcast_ref::() + .expect("remote add should send the coerced blob struct"); + let uri: &StringArray = image + .column_by_name("uri") + .unwrap() + .as_any() + .downcast_ref() + .unwrap(); + assert_eq!(uri.value(0), "s3://bucket/key"); + assert!(image.column_by_name("data").unwrap().is_null(0)); + } + #[rstest] #[case(true)] #[case(false)] diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index ecd95f161..efc36d260 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -3208,7 +3208,7 @@ impl BaseTable for NativeTable { let output = add.into_plan(&table_schema, &table_def)?; - let lance_params = output + let mut lance_params = output .write_options .lance_write_params .unwrap_or(WriteParams { @@ -3218,6 +3218,9 @@ impl BaseTable for NativeTable { }, ..Default::default() }); + if output.allow_external_blob_outside_bases { + lance_params.allow_external_blob_outside_bases = true; + } // Repartition for write parallelism if beneficial. let plan = if num_partitions > 1 { diff --git a/rust/lancedb/src/table/add_data.rs b/rust/lancedb/src/table/add_data.rs index 11ba43dd6..15ccf9f66 100644 --- a/rust/lancedb/src/table/add_data.rs +++ b/rust/lancedb/src/table/add_data.rs @@ -60,6 +60,7 @@ pub struct AddDataBuilder { pub(crate) embedding_registry: Option>, pub(crate) progress_callback: Option, pub(crate) write_parallelism: Option, + pub(crate) allow_external_blob_outside_bases: bool, } impl std::fmt::Debug for AddDataBuilder { @@ -87,6 +88,7 @@ impl AddDataBuilder { embedding_registry, progress_callback: None, write_parallelism: None, + allow_external_blob_outside_bases: false, } } @@ -141,6 +143,16 @@ impl AddDataBuilder { self } + /// Store blob URIs that sit outside registered blob bases. + /// + /// The row keeps a reference, so the object has to stay readable. + /// [`crate::table::Table::fetch_blobs`] reads from that location. + /// Defaults to `false`. Local tables only. + pub fn allow_external_blob_outside_bases(mut self, allow: bool) -> Self { + self.allow_external_blob_outside_bases = allow; + self + } + pub async fn execute(self) -> Result { if self.write_parallelism.map(|p| p == 0).unwrap_or(false) { return Err(Error::InvalidInput { @@ -199,6 +211,7 @@ impl AddDataBuilder { write_options: self.write_options, mode: self.mode, tracker, + allow_external_blob_outside_bases: self.allow_external_blob_outside_bases, }) } } @@ -212,6 +225,7 @@ pub struct PreprocessingOutput { pub write_options: WriteOptions, pub mode: AddDataMode, pub tracker: Option>, + pub allow_external_blob_outside_bases: bool, } /// Check that the input schema is valid for insert. diff --git a/rust/lancedb/src/table/datafusion/blob_coerce.rs b/rust/lancedb/src/table/datafusion/blob_coerce.rs index b29b2423b..cb984f7f4 100644 --- a/rust/lancedb/src/table/datafusion/blob_coerce.rs +++ b/rust/lancedb/src/table/datafusion/blob_coerce.rs @@ -7,7 +7,7 @@ use std::sync::Arc; -use arrow_schema::{DataType, Field, FieldRef}; +use arrow_schema::{DataType, Field, FieldRef, Fields}; use datafusion::functions::core::{get_field, named_struct}; use datafusion_common::ScalarValue; use datafusion_common::config::ConfigOptions; @@ -35,8 +35,9 @@ pub(super) fn coerce_blob_expr( }); }; - let input_struct_children = match input_field.data_type() { - DataType::Binary | DataType::LargeBinary | DataType::BinaryView => None, + let input_shape = match input_field.data_type() { + DataType::Binary | DataType::LargeBinary | DataType::BinaryView => BlobInputShape::Bytes, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => BlobInputShape::String, DataType::Struct(children) => { if !children .iter() @@ -49,13 +50,15 @@ pub(super) fn coerce_blob_expr( ), }); } - Some(children) + BlobInputShape::Struct(children) } other => { return Err(Error::InvalidInput { message: format!( "cannot coerce column '{}' with type {} into a blob v2 struct. \ - expected Binary, LargeBinary, BinaryView, or a Struct with a 'data' or 'uri' child", + expected binary bytes (Binary, LargeBinary, BinaryView), \ + strings (Utf8, LargeUtf8, Utf8View), \ + or a Struct with a 'data' or 'uri' child", table_field.name(), other, ), @@ -69,9 +72,8 @@ pub(super) fn coerce_blob_expr( declared.name().as_str(), )))); - let value: Arc = match input_struct_children { - // Raw binary lands in `data` and everything else is a typed null. - None => { + let value: Arc = match &input_shape { + BlobInputShape::Bytes => { if declared.name() == "data" { Arc::new(CastExpr::new( input_expr.clone(), @@ -82,30 +84,43 @@ pub(super) fn coerce_blob_expr( typed_null(declared.data_type())? } } - Some(children) => match children.iter().find(|c| c.name() == declared.name()) { - Some(child) => { - let field_expr: Arc = Arc::new(ScalarFunctionExpr::new( - &format!("get_field({})", declared.name()), - get_field(), - vec![ - input_expr.clone(), - Arc::new(Literal::new(ScalarValue::from(declared.name().as_str()))), - ], - Arc::new(child.as_ref().clone()), - config.clone(), - )); - if child.data_type() == declared.data_type() { - field_expr - } else { - Arc::new(CastExpr::new( - field_expr, - declared.data_type().clone(), - None, - )) - } + BlobInputShape::String => { + if declared.name() == "uri" { + Arc::new(CastExpr::new( + input_expr.clone(), + declared.data_type().clone(), + None, + )) + } else { + typed_null(declared.data_type())? } - None => typed_null(declared.data_type())?, - }, + } + BlobInputShape::Struct(children) => { + match children.iter().find(|c| c.name() == declared.name()) { + Some(child) => { + let field_expr: Arc = Arc::new(ScalarFunctionExpr::new( + &format!("get_field({})", declared.name()), + get_field(), + vec![ + input_expr.clone(), + Arc::new(Literal::new(ScalarValue::from(declared.name().as_str()))), + ], + Arc::new(child.as_ref().clone()), + config.clone(), + )); + if child.data_type() == declared.data_type() { + field_expr + } else { + Arc::new(CastExpr::new( + field_expr, + declared.data_type().clone(), + None, + )) + } + } + None => typed_null(declared.data_type())?, + } + } }; ns_args.push(value); } @@ -120,6 +135,12 @@ pub(super) fn coerce_blob_expr( Ok((expr, table_field.clone())) } +enum BlobInputShape<'a> { + Bytes, + String, + Struct(&'a Fields), +} + fn typed_null(data_type: &DataType) -> Result> { let scalar = ScalarValue::try_from(data_type).map_err(|e| Error::InvalidInput { message: format!("cannot build null literal for blob child type {data_type}: {e}"), @@ -134,7 +155,7 @@ mod tests { use crate::blob::blob; use arrow_array::{ Array, ArrayRef, BinaryArray, BinaryViewArray, Int32Array, Int64Array, LargeBinaryArray, - RecordBatch, StringArray, StructArray, UInt8Array, UInt64Array, + RecordBatch, StringArray, StringViewArray, StructArray, UInt8Array, UInt64Array, }; use arrow_schema::Schema; use datafusion::prelude::SessionContext; @@ -436,14 +457,78 @@ mod tests { #[tokio::test] async fn unsupported_input_type_is_rejected_with_column_name() { let batch = batch_with_image( - Field::new("image", DataType::Utf8, true), - Arc::new(StringArray::from(vec!["not bytes"])), + Field::new("image", DataType::Int64, true), + Arc::new(Int64Array::from(vec![42])), ); let err = coerce_err(batch, &blob_table_schema()).await; assert!(matches!(err, Error::InvalidInput { .. }), "got {err:?}"); assert!(err.to_string().contains("image")); } + #[tokio::test] + async fn utf8_string_coerces_to_uri_child() { + let batch = batch_with_image( + Field::new("image", DataType::Utf8, true), + Arc::new(StringArray::from(vec![Some("s3://bucket/key"), None])), + ); + let coerced = coerce(batch, &blob_table_schema()).await; + let image = image_struct(&coerced); + let uri: &StringArray = image + .column_by_name("uri") + .unwrap() + .as_any() + .downcast_ref() + .unwrap(); + assert_eq!(uri.value(0), "s3://bucket/key"); + assert!(image.column_by_name("data").unwrap().is_null(0)); + assert!(uri.is_null(1)); + } + + #[tokio::test] + async fn large_utf8_string_coerces_into_four_child_blob_layout() { + use arrow_array::LargeStringArray; + + let table_schema = Schema::new(vec![ + Field::new("id", DataType::Int64, false), + wide_blob_field("image"), + ]); + let batch = batch_with_image( + Field::new("image", DataType::LargeUtf8, true), + Arc::new(LargeStringArray::from(vec!["file:///tmp/blob.bin"])), + ); + let coerced = coerce(batch, &table_schema).await; + let image = image_struct(&coerced); + assert_eq!(image.num_columns(), 4); + let uri: &StringArray = image + .column_by_name("uri") + .unwrap() + .as_any() + .downcast_ref() + .unwrap(); + assert_eq!(uri.value(0), "file:///tmp/blob.bin"); + assert!(image.column_by_name("data").unwrap().is_null(0)); + assert!(image.column_by_name("position").unwrap().is_null(0)); + assert!(image.column_by_name("size").unwrap().is_null(0)); + } + + #[tokio::test] + async fn utf8_view_string_coerces_to_uri_child() { + let batch = batch_with_image( + Field::new("image", DataType::Utf8View, true), + Arc::new(StringViewArray::from(vec![Some("s3://bucket/view-key")])), + ); + let coerced = coerce(batch, &blob_table_schema()).await; + let image = image_struct(&coerced); + let uri: &StringArray = image + .column_by_name("uri") + .unwrap() + .as_any() + .downcast_ref() + .unwrap(); + assert_eq!(uri.value(0), "s3://bucket/view-key"); + assert!(image.column_by_name("data").unwrap().is_null(0)); + } + #[tokio::test] async fn blob_metadata_survives_cast_of_sibling_column() { let batch = RecordBatch::try_new( diff --git a/rust/lancedb/tests/blob_integration.rs b/rust/lancedb/tests/blob_integration.rs index b92f961f4..7b709b645 100644 --- a/rust/lancedb/tests/blob_integration.rs +++ b/rust/lancedb/tests/blob_integration.rs @@ -5,12 +5,14 @@ use std::sync::Arc; use arrow_array::{ Array, ArrayRef, BinaryArray, Int64Array, LargeBinaryArray, RecordBatch, StringArray, - StructArray, UInt64Array, + StructArray, UInt64Array, new_null_array, }; use arrow_schema::{DataType, Field, Fields, Schema}; use futures::TryStreamExt; use lance::Dataset; +use lance::dataset::WriteParams; use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; +use lance_table::format::BasePath; use lancedb::{ Connection, Error, Result, Table, blob::{BlobRangeRequest, blob}, @@ -19,7 +21,7 @@ use lancedb::{ ListingDatabaseOptions, NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, }, query::{ExecutableQuery, QueryBase}, - table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats}, + table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats, WriteOptions}, }; use tempfile::tempdir; @@ -261,11 +263,11 @@ async fn add_rejects_uncoercible_blob_input() -> Result<()> { let batch = RecordBatch::try_new( Arc::new(Schema::new(vec![ Field::new("id", DataType::Int64, false), - Field::new("image", DataType::Utf8, true), + Field::new("image", DataType::Int64, true), ])), vec![ Arc::new(Int64Array::from(vec![1])), - Arc::new(StringArray::from(vec!["not bytes"])), + Arc::new(Int64Array::from(vec![42])), ], ) .unwrap(); @@ -1332,3 +1334,223 @@ async fn optimize_preserves_blob_v2_null_and_empty_distinction() -> Result<()> { ); Ok(()) } + +fn uri_struct_batch(id: i64, uri: &str) -> RecordBatch { + let image_field = blob("image", true); + let DataType::Struct(child_fields) = image_field.data_type().clone() else { + unreachable!("blob field is a struct"); + }; + let children: Vec = child_fields + .iter() + .map(|field| match field.name().as_str() { + "uri" => Arc::new(StringArray::from(vec![Some(uri)])) as ArrayRef, + _ => new_null_array(field.data_type(), 1), + }) + .collect(); + let image = StructArray::new(child_fields, children, None); + RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + image_field, + ])), + vec![Arc::new(Int64Array::from(vec![id])), Arc::new(image)], + ) + .unwrap() +} + +fn uri_string_batch(id: i64, uri: &str) -> RecordBatch { + RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("image", DataType::Utf8, true), + ])), + vec![ + Arc::new(Int64Array::from(vec![id])), + Arc::new(StringArray::from(vec![Some(uri)])), + ], + ) + .unwrap() +} + +fn write_payload_file_uri(dir: &std::path::Path, name: &str, payload: &[u8]) -> String { + let path = dir.join(name); + std::fs::write(&path, payload).unwrap(); + url::Url::from_file_path(&path).unwrap().to_string() +} + +#[tokio::test] +async fn external_uri_struct_round_trips_with_flag() -> Result<()> { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().join("db").to_str().unwrap()) + .execute() + .await?; + let payload: &[u8] = b"external-struct-payload"; + let uri = write_payload_file_uri(tmp.path(), "payload.bin", payload); + let table = db + .create_empty_table("t", blob_table_schema()) + .execute() + .await?; + + table + .add(uri_struct_batch(1, &uri)) + .allow_external_blob_outside_bases(true) + .execute() + .await?; + + let ids = collect_row_ids(&table).await?; + let bytes = table.fetch_blobs("image", &ids).await?; + assert_eq!(bytes.value(0), payload); + Ok(()) +} + +#[tokio::test] +async fn external_uri_add_requires_opt_in() -> Result<()> { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().join("db").to_str().unwrap()) + .execute() + .await?; + let uri = write_payload_file_uri(tmp.path(), "payload.bin", b"unreachable"); + let table = db + .create_empty_table("t", blob_table_schema()) + .execute() + .await?; + + let err = table + .add(uri_struct_batch(1, &uri)) + .execute() + .await + .unwrap_err(); + + assert!( + err.to_string() + .contains("allow_external_blob_outside_bases"), + "got: {err}" + ); + assert_eq!(table.count_rows(None).await?, 0); + Ok(()) +} + +#[tokio::test] +async fn string_uri_input_round_trips_as_external_reference() -> Result<()> { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().join("db").to_str().unwrap()) + .execute() + .await?; + let payload: &[u8] = b"external-string-payload"; + let uri = write_payload_file_uri(tmp.path(), "payload.bin", payload); + let table = db + .create_empty_table("t", blob_table_schema()) + .execute() + .await?; + + table + .add(uri_string_batch(1, &uri)) + .allow_external_blob_outside_bases(true) + .execute() + .await?; + + let ids = collect_row_ids(&table).await?; + let bytes = table.fetch_blobs("image", &ids).await?; + assert_eq!(bytes.value(0), payload); + + let files = table.fetch_blob_files("image", &ids).await?; + let file = files[0].as_ref().expect("missing blob file"); + assert_eq!(file.uri(), Some(uri.as_str())); + Ok(()) +} + +#[tokio::test] +async fn string_uri_inside_registered_base_does_not_need_the_flag() -> Result<()> { + let tmp = tempdir().unwrap(); + let db_path = tmp.path().join("db"); + let external_base = tmp.path().join("external_base"); + let object_dir = external_base.join("objects"); + std::fs::create_dir_all(&object_dir).unwrap(); + let payload: &[u8] = b"mapped-in-base"; + let object_path = object_dir.join("mapped.bin"); + std::fs::write(&object_path, payload).unwrap(); + let object_uri = url::Url::from_file_path(&object_path).unwrap().to_string(); + let base_uri = url::Url::from_file_path(&external_base) + .unwrap() + .to_string(); + + let db = connect(db_path.to_str().unwrap()).execute().await?; + let table = db + .create_empty_table("t", blob_table_schema()) + .write_options(WriteOptions { + lance_write_params: Some(WriteParams { + initial_bases: Some(vec![BasePath { + id: 1, + name: Some("external".to_string()), + path: base_uri, + is_dataset_root: false, + }]), + ..Default::default() + }), + }) + .execute() + .await?; + + table + .add(uri_string_batch(1, &object_uri)) + .execute() + .await?; + + let ids = collect_row_ids(&table).await?; + let bytes = table.fetch_blobs("image", &ids).await?; + assert_eq!(bytes.value(0), payload); + Ok(()) +} + +#[tokio::test] +async fn external_uri_rows_mix_with_inline_rows() -> Result<()> { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().join("db").to_str().unwrap()) + .execute() + .await?; + let external_payload: &[u8] = b"external-bytes"; + let uri = write_payload_file_uri(tmp.path(), "payload.bin", external_payload); + let table = + create_inline_blob_table(&db, "t", &[1], &[Some(b"inline-bytes".as_slice())]).await?; + + table + .add(uri_string_batch(2, &uri)) + .allow_external_blob_outside_bases(true) + .execute() + .await?; + + let pairs = collect_id_rowid(&table).await?; + let row_ids: Vec = pairs.iter().map(|(_, r)| *r).collect(); + let bytes = table.fetch_blobs("image", &row_ids).await?; + for (i, (id, _)) in pairs.iter().enumerate() { + match id { + 1 => assert_eq!(bytes.value(i), b"inline-bytes"), + 2 => assert_eq!(bytes.value(i), external_payload), + _ => unreachable!(), + } + } + Ok(()) +} + +#[tokio::test] +async fn malformed_string_uri_is_rejected_at_write() -> Result<()> { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().join("db").to_str().unwrap()) + .execute() + .await?; + let table = db + .create_empty_table("t", blob_table_schema()) + .execute() + .await?; + + let err = table + .add(uri_string_batch(1, "not a uri")) + .allow_external_blob_outside_bases(true) + .execute() + .await + .unwrap_err(); + + assert!(err.to_string().contains("not a uri"), "got: {err}"); + assert_eq!(table.count_rows(None).await?, 0); + Ok(()) +} From a57fb68891a081aac6e6466f772ff50b21445172 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 00:31:14 +0800 Subject: [PATCH 52/58] docs(python): fix Azure storage options examples (#3899) ## Summary - document that Azure Blob Storage credentials can be passed directly through `storage_options` - provide valid quoted `account_name` and `account_key` examples for both sync and async Python connections - execute the option dictionaries during doctests so the original unquoted-key mistake is caught ## Root cause The historical Python storage guide used `account_name` and `account_key` as bare identifiers in dictionary literals. Following that example either raised `NameError` or, when those names were predefined, produced incorrect option keys. The runtime already accepts direct Azure credentials, but the current Python API reference did not contain a corrected Azure example. ## Validation - `python/.venv/bin/ruff format --check python/python/lancedb/__init__.py` - `python/.venv/bin/ruff check .` - `cd python && uv run --no-sync pytest --doctest-modules python/lancedb/__init__.py -q` - `cd python && uv run --no-sync pytest python/tests/test_import.py -q` Fixes #2236 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo --- python/python/lancedb/__init__.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index df8950686..8cb85a3ed 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -179,6 +179,18 @@ def connect( ... }, ... ) + For Azure Blob Storage, credentials can be passed directly without setting + environment variables: + + >>> azure_storage_options = { + ... "account_name": "some-account", + ... "account_key": "some-key", + ... } + >>> db = lancedb.connect( # doctest: +SKIP + ... "az://my-container/my-database", + ... storage_options=azure_storage_options, + ... ) + For tests and temporary data, use an in-memory database: >>> db = lancedb.connect("memory://") @@ -465,6 +477,10 @@ async def connect_async( -------- >>> import lancedb + >>> azure_storage_options = { + ... "account_name": "some-account", + ... "account_key": "some-key", + ... } >>> async def doctest_example(): ... # For a local directory, provide a path to the database ... db = await lancedb.connect_async("~/.lancedb") @@ -472,6 +488,11 @@ async def connect_async( ... db = await lancedb.connect_async("s3://my-bucket/lancedb", ... storage_options={ ... "aws_access_key_id": "***"}) + ... # Azure credentials can also be passed directly + ... db = await lancedb.connect_async( + ... "az://my-container/my-database", + ... storage_options=azure_storage_options, + ... ) ... # For tests and temporary data, use an in-memory database ... db = await lancedb.connect_async("memory://") ... # Connect to LanceDB cloud From 35b5d015ac8a38313d8322c26f4b2f86ef6f4e23 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 04:17:32 +0800 Subject: [PATCH 53/58] fix(node): preserve embedding registration in server bundles (#3806) ## Summary - lazily initialize built-in OpenAI and Hugging Face providers when consumers call the public embedding registry API - choose automatic vector versus FTS search from embedding metadata on a fresh pinned table revision for every execution - expose automatic string searches as an `AutoQuery` with only operations common to both native query families - keep the registry shared and built-in registration safe across duplicated module graphs ## Root cause Nitro treats dependency modules as side-effect-free and removes the bare OpenAI provider import from its generated route. Registration therefore never runs, so `getRegistry().get("openai")` remains undefined even when the registry itself is shared globally. Bundlers may also duplicate the provider and registry module graphs. The public embedding entry point now initializes built-in providers only when `getRegistry()` is explicitly called, keeping initialization on a live path that Nitro retains. Each terminal automatic-search execution pins the exact table revision visible at dispatch, reads embedding metadata and computes an embedding from that snapshot, replays the builder operations, and constructs and executes the selected native query against the same snapshot. Pinned native snapshots execute locally when namespace pushdown cannot carry their revision, while remote snapshots are seeded directly from one version-and-schema response. The public `AutoQuery` builder exposes only the operations shared by FTS and vector search, so runtime class narrowing cannot expose invalid vector-only methods. Repeated built-in registration replaces stale constructors from duplicated module graphs while public `register()` retains its duplicate-alias error. ## Validation - `cargo fmt --all` - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` - `pnpm build` - `pnpm lint` - `pnpm run docs` - `pnpm test --runInBand` (783 passed, 5 skipped) - serial examples suite with a local OpenAI mock (11 passed), including `sentence-transformers.test.ts` - packaged Nitro 2.13.4 server route using the reported imports returned `{"registered":true}` - fresh-process FTS fixture initialized both public built-ins and confirmed automatic string search still returned the indexed row - schema-consistency regressions cover read-consistency refresh, checkout, checkoutLatest, restore, runtime class narrowing, concurrent overwrite during embedding computation, and reused automatic-search builders - focused regressions confirm pinned native snapshots bypass unversioned namespace pushdown and remote snapshots use one describe request Fixes #2429 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo --- docs/src/js/classes/AutoQuery.md | 518 ++++++++++++++++++ docs/src/js/classes/Table.md | 4 +- docs/src/js/globals.md | 1 + .../embedding/functions/getRegistry.md | 14 +- nodejs/__test__/embedding_registry.test.ts | 95 ++++ nodejs/__test__/fixtures/auto_fts_search.cjs | 33 ++ nodejs/__test__/table.test.ts | 191 +++++++ nodejs/lancedb/embedding/index.ts | 44 +- nodejs/lancedb/embedding/openai.ts | 5 +- nodejs/lancedb/embedding/registry.ts | 60 +- nodejs/lancedb/embedding/transformers.ts | 5 +- nodejs/lancedb/index.ts | 1 + nodejs/lancedb/query.ts | 139 +++-- nodejs/lancedb/table.ts | 44 +- nodejs/src/table.rs | 6 + rust/lancedb/src/remote/table.rs | 38 ++ rust/lancedb/src/table.rs | 32 ++ rust/lancedb/src/table/query.rs | 49 +- 18 files changed, 1211 insertions(+), 68 deletions(-) create mode 100644 docs/src/js/classes/AutoQuery.md create mode 100644 nodejs/__test__/embedding_registry.test.ts create mode 100644 nodejs/__test__/fixtures/auto_fts_search.cjs diff --git a/docs/src/js/classes/AutoQuery.md b/docs/src/js/classes/AutoQuery.md new file mode 100644 index 000000000..1d7ea6952 --- /dev/null +++ b/docs/src/js/classes/AutoQuery.md @@ -0,0 +1,518 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / AutoQuery + +# Class: AutoQuery + +A builder for automatic string searches. + +Automatic search determines whether to use full-text or vector search from +the table revision selected for each execution. This builder exposes the +common operations supported by both query families. + +## Extends + +- `StandardQueryBase`<`NativeQuery` \| `NativeVectorQuery`> + +## Properties + +### inner + +```ts +protected inner: Query | VectorQuery | Promise; +``` + +#### Inherited from + +`StandardQueryBase.inner` + +## Methods + +### analyzePlan() + +```ts +analyzePlan(distributedMetrics?): Promise +``` + +Executes the query and returns the physical query plan annotated with runtime metrics. + +This is useful for debugging and performance analysis, as it shows how the query was executed +and includes metrics such as elapsed time, rows processed, and I/O statistics. + +#### Parameters + +* **distributedMetrics?**: [`AnalyzePlanDistributedMetrics`](../type-aliases/AnalyzePlanDistributedMetrics.md) + How distributed worker metrics are displayed for remote query plans. + Defaults to `"aggregate"`. + +#### Returns + +`Promise`<`string`> + +A query execution plan with runtime metrics for each step. + +#### Example + +```ts +import * as lancedb from "@lancedb/lancedb" + +const db = await lancedb.connect("./.lancedb"); +const table = await db.createTable("my_table", [ + { vector: [1.1, 0.9], id: "1" }, +]); + +const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan(); + +Example output (with runtime metrics inlined): +AnalyzeExec verbose=true, metrics=[] + ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs] + Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1] + CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs] + GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns] + FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs] + SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1] + KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1] + LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2] +``` + +#### Inherited from + +`StandardQueryBase.analyzePlan` + +*** + +### execute() + +```ts +protected execute(options?): AsyncGenerator, void, unknown> +``` + +Execute the query and return the results as an + +#### Parameters + +* **options?**: `Partial`<[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)> + +#### Returns + +`AsyncGenerator`<`RecordBatch`<`any`>, `void`, `unknown`> + +#### See + + - AsyncIterator +of + - RecordBatch. + +By default, LanceDb will use many threads to calculate results and, when +the result set is large, multiple batches will be processed at one time. +This readahead is limited however and backpressure will be applied if this +stream is consumed slowly (this constrains the maximum memory used by a +single query) + +#### Inherited from + +`StandardQueryBase.execute` + +*** + +### explainPlan() + +```ts +explainPlan(verbose): Promise +``` + +Generates an explanation of the query execution plan. + +#### Parameters + +* **verbose**: `boolean` = `false` + If true, provides a more detailed explanation. Defaults to false. + +#### Returns + +`Promise`<`string`> + +A Promise that resolves to a string containing the query execution plan explanation. + +#### Example + +```ts +import * as lancedb from "@lancedb/lancedb" +const db = await lancedb.connect("./.lancedb"); +const table = await db.createTable("my_table", [ + { vector: [1.1, 0.9], id: "1" }, +]); +const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan(); +``` + +#### Inherited from + +`StandardQueryBase.explainPlan` + +*** + +### fastSearch() + +```ts +fastSearch(): this +``` + +Skip searching un-indexed data. This can make search faster, but will miss +any data that is not yet indexed. + +Use [Table#optimize](Table.md#optimize) to index all un-indexed data. + +#### Returns + +`this` + +#### Inherited from + +`StandardQueryBase.fastSearch` + +*** + +### ~~filter()~~ + +```ts +filter(predicate): this +``` + +A filter statement to be applied to this query. + +#### Parameters + +* **predicate**: `string` + +#### Returns + +`this` + +#### See + +where + +#### Deprecated + +Use `where` instead + +#### Inherited from + +`StandardQueryBase.filter` + +*** + +### fullTextSearch() + +```ts +fullTextSearch(query, options?): this +``` + +#### Parameters + +* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md) + +* **options?**: `Partial`<[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)> + +#### Returns + +`this` + +#### Inherited from + +`StandardQueryBase.fullTextSearch` + +*** + +### limit() + +```ts +limit(limit): this +``` + +Set the maximum number of results to return. + +By default, a plain search has no limit. If this method is not +called then every valid row from the table will be returned. + +#### Parameters + +* **limit**: `number` + +#### Returns + +`this` + +#### Inherited from + +`StandardQueryBase.limit` + +*** + +### offset() + +```ts +offset(offset): this +``` + +Set the number of rows to skip before returning results. + +This is useful for pagination. + +#### Parameters + +* **offset**: `number` + +#### Returns + +`this` + +#### Inherited from + +`StandardQueryBase.offset` + +*** + +### orderBy() + +```ts +orderBy(ordering): this +``` + +Sort the results by the specified column(s). + +#### Parameters + +* **ordering**: [`ColumnOrdering`](../interfaces/ColumnOrdering.md) \| [`ColumnOrdering`](../interfaces/ColumnOrdering.md)[] + +#### Returns + +`this` + +This query builder. + +#### Inherited from + +`StandardQueryBase.orderBy` + +*** + +### outputSchema() + +```ts +outputSchema(): Promise> +``` + +Returns the schema of the output that will be returned by this query. + +This can be used to inspect the types and names of the columns that will be +returned by the query before executing it. + +#### Returns + +`Promise`<`Schema`<`any`>> + +An Arrow Schema describing the output columns. + +#### Inherited from + +`StandardQueryBase.outputSchema` + +*** + +### select() + +```ts +select(columns): this +``` + +Return only the specified columns. + +By default a query will return all columns from the table. However, this can have +a very significant impact on latency. LanceDb stores data in a columnar fashion. This +means we can finely tune our I/O to select exactly the columns we need. + +As a best practice you should always limit queries to the columns that you need. If you +pass in an array of column names then only those columns will be returned. + +You can also use this method to create new "dynamic" columns based on your existing columns. +For example, you may not care about "a" or "b" but instead simply want "a + b". This is often +seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`). + +To create dynamic columns you can pass in a Map. A column will be returned +for each entry in the map. The key provides the name of the column. The value is +an SQL string used to specify how the column is calculated. + +For example, an SQL query might state `SELECT a + b AS combined, c`. The equivalent +input to this method would be: + +#### Parameters + +* **columns**: `string` \| `string`[] \| `Record`<`string`, `string`> \| `Map`<`string`, `string`> + +#### Returns + +`this` + +#### Example + +```ts +new Map([["combined", "a + b"], ["c", "c"]]) + +Columns will always be returned in the order given, even if that order is different than +the order used when adding the data. + +Note that you can pass in a `Record` (e.g. an object literal). This method +uses `Object.entries` which should preserve the insertion order of the object. However, +object insertion order is easy to get wrong and `Map` is more foolproof. +``` + +#### Inherited from + +`StandardQueryBase.select` + +*** + +### toArray() + +```ts +toArray(options?): Promise +``` + +Collect the results as an array of objects. + +#### Parameters + +* **options?**: `Partial`<[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)> + +#### Returns + +`Promise`<`any`[]> + +#### Inherited from + +`StandardQueryBase.toArray` + +*** + +### toArrow() + +```ts +toArrow(options?): Promise> +``` + +Collect the results as an Arrow + +#### Parameters + +* **options?**: `Partial`<[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)> + +#### Returns + +`Promise`<`Table`<`any`>> + +#### See + +ArrowTable. + +#### Inherited from + +`StandardQueryBase.toArrow` + +*** + +### useLsm() + +```ts +useLsm(enable): this +``` + +Control MemWAL read routing for this query. + +By default (unset), when the table carries a MemWAL write spec (see +[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so +they also return data written via the `mergeInsert` LSM path that has not yet +been compacted into the base table (the active/frozen in-memory memtables and +the flushed generations), deduplicated by primary key; a table without a spec +reads the base table. + +#### Parameters + +* **enable**: `boolean` + `true` forces the LSM scanner and errors if the table has no + MemWAL write spec. `false` bypasses the MemWAL and reads the base table only, + even when a spec is present. + Note: the LSM scanner does not support every query shape (e.g. reranking, + hybrid search, `orderBy`). On a MemWAL table those shapes error unless + `useLsm(false)` is set, because a base-only read would silently exclude + un-compacted MemWAL data. + +#### Returns + +`this` + +#### Inherited from + +`StandardQueryBase.useLsm` + +*** + +### where() + +```ts +where(predicate): this +``` + +A filter statement to be applied to this query. + +The filter should be supplied as an SQL query string. For example: + +#### Parameters + +* **predicate**: `string` + +#### Returns + +`this` + +#### Example + +```ts +x > 10 +y > 0 AND y < 100 +x > 5 OR y = 'test' + +Filtering performance can often be improved by creating a scalar index +on the filter column(s). + +Calling this multiple times combines the filters with a logical AND rather +than replacing the previous filter. +``` + +#### Inherited from + +`StandardQueryBase.where` + +*** + +### withRowId() + +```ts +withRowId(): this +``` + +Whether to return the row id in the results. + +This column can be used to match results between different queries. For +example, to match results from a full text search and a vector search in +order to perform hybrid search. + +#### Returns + +`this` + +#### Inherited from + +`StandardQueryBase.withRowId` diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 06dc8479e..9a85d0d96 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -942,7 +942,7 @@ Get the schema of the table. abstract search( query, queryType?, - ftsColumns?): Query | VectorQuery + ftsColumns?): Query | VectorQuery | AutoQuery ``` Create a search query to find the nearest neighbors @@ -964,7 +964,7 @@ of the given query #### Returns -[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md) +[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md) \| [`AutoQuery`](AutoQuery.md) *** diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index e0635ab65..beb9cbeff 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -18,6 +18,7 @@ ## Classes +- [AutoQuery](classes/AutoQuery.md) - [BooleanQuery](classes/BooleanQuery.md) - [BoostQuery](classes/BoostQuery.md) - [BranchContents](classes/BranchContents.md) diff --git a/docs/src/js/namespaces/embedding/functions/getRegistry.md b/docs/src/js/namespaces/embedding/functions/getRegistry.md index 331dbd60b..b149a4a88 100644 --- a/docs/src/js/namespaces/embedding/functions/getRegistry.md +++ b/docs/src/js/namespaces/embedding/functions/getRegistry.md @@ -10,16 +10,12 @@ function getRegistry(): EmbeddingFunctionRegistry ``` -Utility function to get the global instance of the registry +Get the global embedding function registry. + +LanceDB built-in providers are initialized when this public API is first +used, so importing the root package does not change automatic search +selection for tables without embedding metadata. ## Returns [`EmbeddingFunctionRegistry`](../classes/EmbeddingFunctionRegistry.md) - -`EmbeddingFunctionRegistry` The global instance of the registry - -## Example - -```ts -const registry = getRegistry(); -const openai = registry.get("openai").create(); diff --git a/nodejs/__test__/embedding_registry.test.ts b/nodejs/__test__/embedding_registry.test.ts new file mode 100644 index 000000000..83933399a --- /dev/null +++ b/nodejs/__test__/embedding_registry.test.ts @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { execFileSync } from "node:child_process"; +import { resolve } from "node:path"; + +import type { OpenAIEmbeddingFunction } from "../lancedb/embedding/openai"; +import type { EmbeddingFunctionRegistry } from "../lancedb/embedding/registry"; + +type EmbeddingModule = typeof import("../lancedb/embedding"); +type OpenAIModule = typeof import("../lancedb/embedding/openai"); +type RegistryModule = typeof import("../lancedb/embedding/registry"); + +describe("embedding function registry", () => { + const registries: EmbeddingFunctionRegistry[] = []; + + afterEach(() => { + for (const registry of registries) { + registry.reset(); + } + registries.length = 0; + }); + + it("defers built-in providers until the public registry API is used", () => { + jest.isolateModules(() => { + const embedding = require("../lancedb/embedding") as EmbeddingModule; + const { getRegistry: getInternalRegistry } = + require("../lancedb/embedding/registry") as RegistryModule; + const registry = getInternalRegistry(); + registries.push(registry); + + expect(registry.length()).toBe(0); + expect(embedding.getRegistry()).toBe(registry); + expect(registry.get("openai")).toBeDefined(); + expect(registry.get("huggingface")).toBeDefined(); + }); + }); + + it("preserves automatic FTS search in a fresh process", () => { + execFileSync( + process.execPath, + [resolve(__dirname, "fixtures", "auto_fts_search.cjs")], + { stdio: "pipe" }, + ); + }); + + it("shares registrations across duplicated provider module graphs", () => { + let registeringRegistry: EmbeddingFunctionRegistry | undefined; + let latestOpenAIConstructor: typeof OpenAIEmbeddingFunction | undefined; + + jest.isolateModules(() => { + require("../lancedb/embedding/openai"); + const { getRegistry } = + require("../lancedb/embedding/registry") as RegistryModule; + registeringRegistry = getRegistry(); + registries.push(registeringRegistry); + expect(registeringRegistry.get("openai")).toBeDefined(); + }); + + expect(() => { + jest.isolateModules(() => { + const { OpenAIEmbeddingFunction } = + require("../lancedb/embedding/openai") as OpenAIModule; + latestOpenAIConstructor = OpenAIEmbeddingFunction; + const { getRegistry } = + require("../lancedb/embedding/registry") as RegistryModule; + registries.push(getRegistry()); + }); + }).not.toThrow(); + + const previousApiKey = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = "test"; + try { + const latestOpenAI = registeringRegistry! + .get("openai")! + .create(); + expect(latestOpenAI).toBeInstanceOf(latestOpenAIConstructor!); + } finally { + if (previousApiKey === undefined) { + delete process.env.OPENAI_API_KEY; + } else { + process.env.OPENAI_API_KEY = previousApiKey; + } + } + + jest.isolateModules(() => { + const { getRegistry } = + require("../lancedb/embedding") as EmbeddingModule; + const publicRegistry = getRegistry(); + registries.push(publicRegistry); + expect(publicRegistry).toBe(registeringRegistry); + expect(publicRegistry.get("openai")).toBeDefined(); + }); + }); +}); diff --git a/nodejs/__test__/fixtures/auto_fts_search.cjs b/nodejs/__test__/fixtures/auto_fts_search.cjs new file mode 100644 index 000000000..b5ab060b3 --- /dev/null +++ b/nodejs/__test__/fixtures/auto_fts_search.cjs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +const assert = require("node:assert/strict"); +const tmp = require("tmp"); +const { connect, embedding, Index } = require("../../dist"); +const { getRegistry } = require("../../dist/embedding/registry"); + +async function main() { + assert.equal(typeof embedding.getRegistry, "function"); + assert.equal(getRegistry().length(), 0); + assert.equal(embedding.getRegistry(), getRegistry()); + assert.equal(getRegistry().length(), 2); + + const dir = tmp.dirSync({ unsafeCleanup: true }); + let db; + try { + db = await connect(dir.name); + const table = await db.createTable("docs", [{ text: "hello world" }]); + await table.createIndex("text", { config: Index.fts() }); + + const rows = await table.search("hello").toArray(); + assert.equal(rows[0].text, "hello world"); + } finally { + db?.close(); + dir.removeCallback(); + } +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 0f6ed3615..0aee5caf0 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -11,10 +11,13 @@ import * as arrow17 from "apache-arrow-17"; import * as arrow18 from "apache-arrow-18"; import { + AutoQuery, Connection, MatchQuery, PhraseQuery, + Query, Table, + VectorQuery, connect, tokenize, } from "../lancedb"; @@ -1777,6 +1780,194 @@ describe("Read consistency interval", () => { }); }); +describe("automatic search schema consistency", () => { + let tmpDir: tmp.DirResult; + + class SchemaRefreshEmbedding extends EmbeddingFunction { + ndims() { + return 2; + } + + embeddingDataType() { + return new Float32(); + } + + async computeSourceEmbeddings(data: string[]) { + return data.map((value) => [value.length, 1]); + } + + async computeQueryEmbeddings(value: string) { + return [value.length, 1]; + } + } + + function embeddingSchema() { + const func = new SchemaRefreshEmbedding(); + return LanceSchema({ + text: func.sourceField(new Utf8()), + vector: func.vectorField(), + }); + } + + beforeEach(() => { + getRegistry().reset(); + register("schema-refresh")(SchemaRefreshEmbedding); + tmpDir = tmp.dirSync({ unsafeCleanup: true }); + }); + + afterEach(() => { + getRegistry().reset(); + tmpDir.removeCallback(); + }); + + it("uses the schema refreshed from another connection", async () => { + const first = await connect(tmpDir.name, { readConsistencyInterval: 0 }); + const second = await connect(tmpDir.name, { readConsistencyInterval: 0 }); + + try { + const stale = await first.createTable("docs", [{ text: "before" }], { + schema: embeddingSchema(), + }); + const replacement = await second.createTable( + "docs", + [{ text: "after hello" }], + { mode: "overwrite" }, + ); + await replacement.createIndex("text", { config: Index.fts() }); + + const search = stale.search("hello"); + expect(search).toBeInstanceOf(AutoQuery); + expect(search).not.toBeInstanceOf(Query); + expect(search).not.toBeInstanceOf(VectorQuery); + expect("nprobes" in search).toBe(false); + + const rows = await search.toArray(); + expect(rows[0].text).toBe("after hello"); + expect((await stale.schema()).metadata.has("embedding_functions")).toBe( + false, + ); + } finally { + first.close(); + second.close(); + } + }); + + it("tracks embedding metadata across checkout and restore", async () => { + const first = await connect(tmpDir.name, { readConsistencyInterval: 0 }); + const second = await connect(tmpDir.name, { readConsistencyInterval: 0 }); + + try { + await first.createTable("docs", [{ text: "before" }], { + schema: embeddingSchema(), + }); + const table = await second.createTable( + "docs", + [{ text: "after hello" }], + { mode: "overwrite" }, + ); + await table.createIndex("text", { config: Index.fts() }); + + await table.checkout(1); + expect((await table.search("before").toArray())[0].text).toBe("before"); + + await table.checkoutLatest(); + expect((await table.search("hello").toArray())[0].text).toBe( + "after hello", + ); + + await table.checkout(1); + await table.restore(); + expect((await table.search("before").toArray())[0].text).toBe("before"); + } finally { + first.close(); + second.close(); + } + }); + + it("pins automatic search while computing an embedding", async () => { + let markStarted!: () => void; + let releaseEmbedding!: () => void; + const started = new Promise((resolve) => { + markStarted = resolve; + }); + const released = new Promise((resolve) => { + releaseEmbedding = resolve; + }); + + class BlockingEmbedding extends SchemaRefreshEmbedding { + async computeQueryEmbeddings(value: string) { + markStarted(); + await released; + return [value.length, 1]; + } + } + + register("schema-refresh-blocking")(BlockingEmbedding); + const func = new BlockingEmbedding(); + const schema = LanceSchema({ + text: func.sourceField(new Utf8()), + vector: func.vectorField(), + }); + const first = await connect(tmpDir.name, { readConsistencyInterval: 0 }); + const second = await connect(tmpDir.name, { readConsistencyInterval: 0 }); + + try { + const table = await first.createTable( + "docs", + [{ text: "hello before" }], + { schema }, + ); + const pending = table.search("hello").toArray(); + await started; + + const replacement = await second.createTable( + "docs", + [{ text: "hello after" }], + { mode: "overwrite" }, + ); + await replacement.createIndex("text", { config: Index.fts() }); + releaseEmbedding(); + + expect((await pending)[0].text).toBe("hello before"); + } finally { + releaseEmbedding(); + first.close(); + second.close(); + } + }); + + it("refreshes a reused automatic search for every execution", async () => { + const first = await connect(tmpDir.name, { readConsistencyInterval: 0 }); + const second = await connect(tmpDir.name, { readConsistencyInterval: 0 }); + + try { + const table = await first.createTable("docs", [ + { text: "hello before", marker: "before" }, + ]); + await table.createIndex("text", { config: Index.fts() }); + const search = table.search("hello").select(["text"]); + + const before = (await search.toArray())[0]; + expect(before.text).toBe("hello before"); + expect(before.marker).toBeUndefined(); + + const replacement = await second.createTable( + "docs", + [{ text: "hello after", marker: "after" }], + { mode: "overwrite" }, + ); + await replacement.createIndex("text", { config: Index.fts() }); + + const after = (await search.toArray())[0]; + expect(after.text).toBe("hello after"); + expect(after.marker).toBeUndefined(); + } finally { + first.close(); + second.close(); + } + }); +}); + describe("schema evolution", function () { let tmpDir: tmp.DirResult; beforeEach(() => { diff --git a/nodejs/lancedb/embedding/index.ts b/nodejs/lancedb/embedding/index.ts index d0ffaec0d..d748e245a 100644 --- a/nodejs/lancedb/embedding/index.ts +++ b/nodejs/lancedb/embedding/index.ts @@ -4,7 +4,15 @@ import { Field, Schema } from "../arrow"; import { sanitizeType } from "../sanitize"; import { EmbeddingFunction } from "./embedding_function"; -import { EmbeddingFunctionConfig, getRegistry } from "./registry"; +import { + EmbeddingFunctionConfig, + EmbeddingFunctionRegistry, + getRegistry as getGlobalRegistry, + registerBuiltIn, +} from "./registry"; + +type OpenAIModule = typeof import("./openai"); +type TransformersModule = typeof import("./transformers"); export { FieldOptions, @@ -14,7 +22,39 @@ export { EmbeddingFunctionConstructor, } from "./embedding_function"; -export * from "./registry"; +export { + EmbeddingFunctionRegistry, + parseEmbeddingMetadata, + register, +} from "./registry"; +export type { + CreateReturnType, + EmbeddingFunctionConfig, + EmbeddingFunctionCreate, + EmbeddingMetadataEntry, + ResolvedEmbeddingFunctionConfig, +} from "./registry"; + +function initializeBuiltInProviders() { + const { OpenAIEmbeddingFunction } = require("./openai") as OpenAIModule; + const { TransformersEmbeddingFunction } = + require("./transformers") as TransformersModule; + + registerBuiltIn("openai", OpenAIEmbeddingFunction); + registerBuiltIn("huggingface", TransformersEmbeddingFunction); +} + +/** + * Get the global embedding function registry. + * + * LanceDB built-in providers are initialized when this public API is first + * used, so importing the root package does not change automatic search + * selection for tables without embedding metadata. + */ +export function getRegistry(): EmbeddingFunctionRegistry { + initializeBuiltInProviders(); + return getGlobalRegistry(); +} /** * Create a schema with embedding functions. diff --git a/nodejs/lancedb/embedding/openai.ts b/nodejs/lancedb/embedding/openai.ts index 5771cfeb5..2218d44bc 100644 --- a/nodejs/lancedb/embedding/openai.ts +++ b/nodejs/lancedb/embedding/openai.ts @@ -5,14 +5,13 @@ import type OpenAI from "openai"; import type { EmbeddingCreateParams } from "openai/resources/index"; import { Float, Float32 } from "../arrow"; import { EmbeddingFunction } from "./embedding_function"; -import { register } from "./registry"; +import { registerBuiltIn } from "./registry"; export type OpenAIOptions = { apiKey: string; model: EmbeddingCreateParams["model"]; }; -@register("openai") export class OpenAIEmbeddingFunction extends EmbeddingFunction< string, Partial @@ -100,3 +99,5 @@ export class OpenAIEmbeddingFunction extends EmbeddingFunction< return response.data[0].embedding; } } + +registerBuiltIn("openai", OpenAIEmbeddingFunction); diff --git a/nodejs/lancedb/embedding/registry.ts b/nodejs/lancedb/embedding/registry.ts index 5f32f683c..c9ee9135e 100644 --- a/nodejs/lancedb/embedding/registry.ts +++ b/nodejs/lancedb/embedding/registry.ts @@ -7,6 +7,10 @@ import { } from "./embedding_function"; import "reflect-metadata"; +const builtInFunctionsKey = Symbol.for( + "@lancedb/lancedb::embedding-built-in-functions::v1", +); + export type CreateReturnType = T extends { init: () => Promise } ? Promise : T; @@ -59,6 +63,15 @@ export class EmbeddingFunctionRegistry { }; } + /** @ignore */ + setBuiltIn< + T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor, + >(name: string, ctor: T): T { + this.#functions.set(name, ctor); + Reflect.defineMetadata("lancedb::embedding::name", name, ctor); + return ctor; + } + get>( name: string, ): EmbeddingFunctionCreate | undefined; @@ -96,6 +109,7 @@ export class EmbeddingFunctionRegistry { */ reset(this: EmbeddingFunctionRegistry) { this.#functions.clear(); + getBuiltInFunctions(this).clear(); } /** @@ -183,12 +197,56 @@ export class EmbeddingFunctionRegistry { } } -const _REGISTRY = new EmbeddingFunctionRegistry(); +function getBuiltInFunctions(registry: EmbeddingFunctionRegistry): Set { + const registryWithBuiltIns = registry as EmbeddingFunctionRegistry & { + [key: symbol]: Set | undefined; + }; + let builtInFunctions = registryWithBuiltIns[builtInFunctionsKey]; + if (builtInFunctions === undefined) { + builtInFunctions = new Set(); + registryWithBuiltIns[builtInFunctionsKey] = builtInFunctions; + } + return builtInFunctions; +} + +// Server bundlers can load the side-effect embedding entry points and the public +// embedding API from separate module graphs. Keep their registry shared. +const registryKey = Symbol.for( + "@lancedb/lancedb::embedding-function-registry::v1", +); +const registryGlobal = globalThis as typeof globalThis & { + [key: symbol]: EmbeddingFunctionRegistry | undefined; +}; + +function getGlobalRegistry(): EmbeddingFunctionRegistry { + const existingRegistry = registryGlobal[registryKey]; + if (existingRegistry !== undefined) { + return existingRegistry; + } + const registry = new EmbeddingFunctionRegistry(); + registryGlobal[registryKey] = registry; + return registry; +} + +const _REGISTRY = getGlobalRegistry(); export function register(name?: string) { return _REGISTRY.register(name); } +/** @ignore */ +export function registerBuiltIn< + T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor, +>(name: string, ctor: T): T { + const builtInFunctions = getBuiltInFunctions(_REGISTRY); + if (builtInFunctions.has(name)) { + return _REGISTRY.setBuiltIn(name, ctor); + } + _REGISTRY.register(name)(ctor); + builtInFunctions.add(name); + return ctor; +} + /** * Utility function to get the global instance of the registry * @returns `EmbeddingFunctionRegistry` The global instance of the registry diff --git a/nodejs/lancedb/embedding/transformers.ts b/nodejs/lancedb/embedding/transformers.ts index 161575285..06043ea7c 100644 --- a/nodejs/lancedb/embedding/transformers.ts +++ b/nodejs/lancedb/embedding/transformers.ts @@ -3,7 +3,7 @@ import { Float, Float32 } from "../arrow"; import { EmbeddingFunction } from "./embedding_function"; -import { register } from "./registry"; +import { registerBuiltIn } from "./registry"; export type XenovaTransformerOptions = { /** The wasm compatible model to use */ @@ -31,7 +31,6 @@ export type XenovaTransformerOptions = { }; }; -@register("huggingface") export class TransformersEmbeddingFunction extends EmbeddingFunction< string, Partial @@ -158,6 +157,8 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction< } } +registerBuiltIn("huggingface", TransformersEmbeddingFunction); + const tensorDiv = ( src: import("@huggingface/transformers").Tensor, divBy: number, diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index ebc8cda8d..34d7ce4d9 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -103,6 +103,7 @@ export { } from "./native.js"; export { + AutoQuery, ExecutableQuery, Query, QueryBase, diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index 843a1276f..3b9b286a0 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -111,13 +111,15 @@ export class QueryBase< NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery, > implements AsyncIterable { + protected inner!: NativeQueryType | Promise; + /** * @hidden */ - protected constructor( - protected inner: NativeQueryType | Promise, - ) { - // intentionally empty + protected constructor(inner?: NativeQueryType | Promise) { + if (inner !== undefined) { + this.inner = inner; + } } // call a function on the inner (either a promise or the actual object) @@ -135,6 +137,15 @@ export class QueryBase< } } + /** + * Return the native query used by the next terminal operation. + * + * @hidden + */ + protected async getInner(): Promise { + return this.inner; + } + /** * Return only the specified columns. * @@ -207,16 +218,11 @@ export class QueryBase< /** * @hidden */ - protected nativeExecute( + protected async nativeExecute( options?: Partial, ): Promise { - if (this.inner instanceof Promise) { - return this.inner.then((inner) => - inner.execute(options?.maxBatchLength, options?.timeoutMs), - ); - } else { - return this.inner.execute(options?.maxBatchLength, options?.timeoutMs); - } + const inner = await this.getInner(); + return inner.execute(options?.maxBatchLength, options?.timeoutMs); } /** @@ -245,12 +251,7 @@ export class QueryBase< /** Collect the results as an Arrow @see {@link ArrowTable}. */ async toArrow(options?: Partial): Promise { const batches = []; - let inner; - if (this.inner instanceof Promise) { - inner = await this.inner; - } else { - inner = this.inner; - } + const inner = await this.getInner(); for await (const batch of new RecordBatchIterable(inner, options)) { batches.push(batch); } @@ -279,11 +280,8 @@ export class QueryBase< * @returns A Promise that resolves to a string containing the query execution plan explanation. */ async explainPlan(verbose = false): Promise { - if (this.inner instanceof Promise) { - return this.inner.then((inner) => inner.explainPlan(verbose)); - } else { - return this.inner.explainPlan(verbose); - } + const inner = await this.getInner(); + return inner.explainPlan(verbose); } /** @@ -321,13 +319,8 @@ export class QueryBase< distributedMetrics?: AnalyzePlanDistributedMetrics, ): Promise { const distributedMetricsMode = distributedMetrics ?? "aggregate"; - if (this.inner instanceof Promise) { - return this.inner.then((inner) => - inner.analyzePlan(distributedMetricsMode), - ); - } else { - return this.inner.analyzePlan(distributedMetricsMode); - } + const inner = await this.getInner(); + return inner.analyzePlan(distributedMetricsMode); } /** @@ -339,12 +332,8 @@ export class QueryBase< * @returns An Arrow Schema describing the output columns. */ async outputSchema(): Promise { - let schemaBuffer: Buffer; - if (this.inner instanceof Promise) { - schemaBuffer = await this.inner.then((inner) => inner.outputSchema()); - } else { - schemaBuffer = await this.inner.outputSchema(); - } + const inner = await this.getInner(); + const schemaBuffer = await inner.outputSchema(); const schema = tableFromIPC(schemaBuffer).schema; return schema; } @@ -356,7 +345,7 @@ export class StandardQueryBase< extends QueryBase implements ExecutableQuery { - constructor(inner: NativeQueryType | Promise) { + constructor(inner?: NativeQueryType | Promise) { super(inner); } @@ -788,6 +777,51 @@ export class TakeQuery extends QueryBase { } } +/** + * A builder for automatic string searches. + * + * Automatic search determines whether to use full-text or vector search from + * the table revision selected for each execution. This builder exposes the + * common operations supported by both query families. + * + * @hideconstructor + */ +export class AutoQuery extends StandardQueryBase< + NativeQuery | NativeVectorQuery +> { + private readonly calls: Array< + (inner: NativeQuery | NativeVectorQuery) => void + > = []; + + /** @hidden */ + constructor( + private readonly createInner: () => Promise< + NativeQuery | NativeVectorQuery + >, + ) { + super(); + } + + /** @hidden */ + protected override doCall( + fn: (inner: NativeQuery | NativeVectorQuery) => void, + ) { + this.calls.push(fn); + } + + /** @hidden */ + protected override async getInner(): Promise< + NativeQuery | NativeVectorQuery + > { + const calls = [...this.calls]; + const inner = await this.createInner(); + for (const call of calls) { + call(inner); + } + return inner; + } +} + /** A builder for LanceDB queries. * * @see {@link Table#query}, {@link Table#search} @@ -802,6 +836,37 @@ export class Query extends StandardQueryBase { super(tbl.query()); } + /** @hidden */ + static autoSearch( + tbl: () => Promise, + query: string, + vector: (tbl: NativeTable) => Promise | undefined>, + columns?: string[], + ): AutoQuery { + const nativeQuery = async () => { + const snapshot = await Promise.resolve(tbl()); + const resolved = await vector(snapshot); + const inner = snapshot.query(); + if (resolved === undefined) { + inner.fullTextSearch({ + query, + columns: columns ?? null, + }); + return inner; + } + + const raw = Array.isArray(resolved) + ? null + : extractVectorBuffer(resolved); + if (raw) { + return inner.nearestToRaw(raw.data, raw.dtype); + } + return inner.nearestTo(Float32Array.from(resolved as number[])); + }; + + return new AutoQuery(nativeQuery); + } + /** * Find the nearest vectors to the given query vector. * diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 28603cca9..a4fc76ef1 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -43,6 +43,7 @@ import { Table as _NativeTable, } from "./native"; import { + AutoQuery, FullTextQuery, Query, TakeQuery, @@ -523,7 +524,7 @@ export abstract class Table { query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[], - ): VectorQuery | Query; + ): VectorQuery | Query | AutoQuery; /** * Search the table with a given query vector. * @@ -975,10 +976,11 @@ export class LocalTable extends Table { return this.inner.display(); } - private async getEmbeddingFunctions(): Promise< - Map - > { - const schema = await this.schema(); + private async getEmbeddingFunctions( + inner: _NativeTable = this.inner, + ): Promise> { + const schemaBuf = await inner.schema(); + const schema = tableFromIPC(schemaBuf).schema; const registry = getRegistry(); return registry.parseFunctions(schema.metadata); } @@ -1160,7 +1162,7 @@ export class LocalTable extends Table { query: string | IntoVector | MultiVector | FullTextQuery, queryType: string = "auto", ftsColumns?: string | string[], - ): VectorQuery | Query { + ): VectorQuery | Query | AutoQuery { if (typeof query !== "string" && !instanceOfFullTextQuery(query)) { if (queryType === "fts") { throw new Error("Cannot perform full text search on a vector query"); @@ -1175,17 +1177,35 @@ export class LocalTable extends Table { }); } - // The query type is auto or vector - // fall back to full text search if no embedding functions are defined and the query is a string - if ( - queryType === "auto" && - (getRegistry().length() === 0 || instanceOfFullTextQuery(query)) - ) { + if (queryType === "auto" && typeof query !== "string") { return this.query().fullTextSearch(query, { columns: ftsColumns, }); } + if (queryType === "auto" && typeof query === "string") { + const vector = async (snapshot: _NativeTable) => { + const functions = await this.getEmbeddingFunctions(snapshot); + // TODO: Support multiple embedding functions + const embeddingFunc: EmbeddingFunctionConfig | undefined = functions + .values() + .next().value; + if (embeddingFunc === undefined) { + return undefined; + } + return await embeddingFunc.function.computeQueryEmbeddings(query); + }; + + const columns = + typeof ftsColumns === "string" ? [ftsColumns] : ftsColumns; + return Query.autoSearch( + () => this.inner.checkoutCurrent(), + query, + vector, + columns, + ); + } + const queryPromise = this.getEmbeddingFunctions().then( async (functions) => { // TODO: Support multiple embedding functions diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 9d60b2056..ff16ac042 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -554,6 +554,12 @@ impl Table { .default_error() } + #[napi(catch_unwind)] + pub async fn checkout_current(&self) -> napi::Result { + let table = self.inner_ref()?.checkout_current().await.default_error()?; + Ok(Self::new(table)) + } + #[napi(catch_unwind)] pub async fn checkout(&self, version: i64) -> napi::Result<()> { self.inner_ref()? diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 44c92310f..77d5983eb 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -1725,6 +1725,22 @@ impl BaseTable for RemoteTable { async fn version(&self) -> Result { self.describe().await.map(|desc| desc.version) } + + async fn checkout_current(&self) -> Result> { + let description = self.describe().await?; + let TableDescription { + version, + schema, + location, + } = description; + let schema = Arc::new(arrow_schema::Schema::try_from(schema)?); + let snapshot = self.with_branch(self.branch.clone()); + *snapshot.version.write().await = Some(version); + *snapshot.location.write().await = location; + snapshot.schema_cache.seed(schema); + Ok(Arc::new(snapshot)) + } + async fn checkout(&self, version: u64) -> Result<()> { // Validate the version exists. The describe is sent without freshness // headers so a stale `min_version` from a previous write doesn't ride @@ -8739,6 +8755,28 @@ mod tests { } } + /// A pinned snapshot should reuse the version and schema returned by its + /// initial describe instead of issuing two more describe requests. + #[tokio::test] + async fn test_checkout_current_seeds_schema_from_single_describe() { + let describe_calls = Arc::new(AtomicUsize::new(0)); + let calls = describe_calls.clone(); + let table = Table::new_with_handler("my_table", move |request| { + assert_eq!(request.url().path(), "/v1/table/my_table/describe/"); + calls.fetch_add(1, Ordering::SeqCst); + http::Response::builder() + .status(200) + .body( + r#"{"version":42,"schema":{"fields":[{"name":"a","type":{"type":"int32"},"nullable":false}]}}"#, + ) + .unwrap() + }); + + let snapshot = table.checkout_current().await.unwrap(); + assert_eq!(snapshot.schema().await.unwrap().fields().len(), 1); + assert_eq!(describe_calls.load(Ordering::SeqCst), 1); + } + /// Test that schema cache is invalidated after checkout #[tokio::test] async fn test_schema_cache_invalidation_on_checkout() { diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index efc36d260..5007a61d9 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -785,6 +785,12 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { async fn drop_columns(&self, columns: &[&str]) -> Result; /// Get the version of the table. async fn version(&self) -> Result; + /// Return a new table handle pinned to the exact revision currently visible. + async fn checkout_current(&self) -> Result> { + Err(Error::NotSupported { + message: "checkout_current is not supported on this table type".into(), + }) + } /// Checkout a specific version of the table. async fn checkout(&self, version: u64) -> Result<()>; /// Checkout a table version referenced by a tag. @@ -1944,6 +1950,20 @@ impl Table { self.inner.version().await } + /// Return a new table handle pinned to the exact revision currently visible. + /// + /// This is used when asynchronous preparation must remain consistent with + /// the revision used for a later read. + #[doc(hidden)] + pub async fn checkout_current(&self) -> Result { + let inner = self.inner.checkout_current().await?; + Ok(Self { + inner, + database: self.database.clone(), + embedding_registry: self.embedding_registry.clone(), + }) + } + /// Checks out a specific version of the Table /// /// Any read operation on the table will now access the data at the checked out version. @@ -3043,6 +3063,18 @@ impl BaseTable for NativeTable { Ok(self.dataset.get().await?.version().version) } + async fn checkout_current(&self) -> Result> { + let current = self.dataset.get().await?; + let dataset = dataset::DatasetConsistencyWrapper::new_time_travel( + current.as_ref().clone(), + self.read_consistency_interval, + ); + Ok(Arc::new(Self { + dataset, + ..self.clone() + })) + } + async fn checkout(&self, version: u64) -> Result<()> { self.dataset.as_time_travel(version).await } diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index 9b81786cd..d35286c84 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -697,6 +697,7 @@ mod tests { use super::*; use crate::query::{QueryExecutionOptions, QueryRequest}; + use crate::table::BaseTable; fn fixed_size_list_array(values: Vec, dimension: i32) -> FixedSizeListArray { FixedSizeListArray::try_new_from_values(Float32Array::from(values), dimension).unwrap() @@ -889,10 +890,56 @@ mod tests { async fn query_table(&self, _request: NsQueryTableRequest) -> lance::Result { self.query_table_calls.fetch_add(1, Ordering::SeqCst); - panic!("approx_mode queries must not be pushed down to namespace query_table"); + panic!("query must not be pushed down to namespace query_table"); } } + #[tokio::test] + async fn test_execute_query_pinned_snapshot_with_namespace_pushdown_runs_locally() { + use crate::connect; + use arrow_array::{Int32Array, RecordBatch}; + use arrow_schema::{DataType, Field, Schema}; + + let conn = connect("memory://").execute().await.unwrap(); + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))], + ) + .unwrap(); + let table = conn + .create_table("test_pinned_namespace_fallback", vec![batch]) + .execute() + .await + .unwrap(); + + let namespace_client = Arc::new(CountingNamespaceClient::default()); + let mut native_table = table.as_native().unwrap().clone(); + native_table.namespace_client = Some(namespace_client.clone()); + native_table + .pushdown_operations + .insert(NamespaceClientPushdownOperation::QueryTable); + + let snapshot = native_table.checkout_current().await.unwrap(); + let snapshot = snapshot.as_any().downcast_ref::().unwrap(); + assert!(snapshot.dataset.time_travel_version().is_some()); + + let query = AnyQuery::Query(QueryRequest { + filter: Some(QueryFilter::Sql("id > 3".to_string())), + ..Default::default() + }); + let stream = execute_query(snapshot, &query, QueryExecutionOptions::default()) + .await + .unwrap(); + let batches = stream.try_collect::>().await.unwrap(); + + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 2 + ); + assert_eq!(namespace_client.query_table_calls.load(Ordering::SeqCst), 0); + } + #[tokio::test] async fn test_execute_query_approx_mode_with_namespace_pushdown_runs_locally() { use crate::connect; From 302b21aa94317dc74cbe6601212242f54e592d8a Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 06:42:29 +0800 Subject: [PATCH 54/58] test(node): cover nested PDF metadata queries (#3827) ## Summary - add an end-to-end Node regression matching LangChain PDFLoader metadata - verify create/query round trips rich nested `loc` and `pdf.info` fields against the currently configured Apache Arrow peer ## Root cause LanceDB v0.14 delegated nested object inference to Apache Arrow. Nested strings were dictionary-encoded with colliding dictionary IDs, so serializing query results as an IPC file failed with a dictionary-replacement error. Current `main` recursively infers nested fields and avoids those invalid dictionaries, but the reported LangChain path had no end-to-end regression coverage. ## Validation - `pnpm build` - `pnpm lint` - `pnpm run docs` - `pnpm test --runInBand` (678 passed, 5 skipped) Fixes #1963 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo --- nodejs/__test__/table.test.ts | 50 +++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 0aee5caf0..aaa144f16 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -685,6 +685,56 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( }, ); +// https://github.com/lancedb/lancedb/issues/1963 +it("should query documents with LangChain PDF metadata", async () => { + const tmpDir = tmp.dirSync({ unsafeCleanup: true }); + try { + const db = await connect(tmpDir.name); + const documents = [ + { + text: "first page", + vector: [1, 0], + source: "first.pdf", + loc: { pageNumber: 1, lines: { from: 1, to: 12 } }, + pdf: { + version: "1.10.100", + info: { + format: "PDF 1.7", + producer: "pdf.js", + creator: "Writer", + }, + totalPages: 2, + }, + }, + { + text: "second page", + vector: [0, 1], + source: "second.pdf", + loc: { pageNumber: 2, lines: { from: 13, to: 24 } }, + pdf: { + version: "1.10.100", + info: { + format: "PDF 1.7", + producer: "pdf.js", + creator: "Writer", + }, + totalPages: 2, + }, + }, + ]; + const documentsTable = await db.createTable("documents", documents); + + const results = await documentsTable.query().toArray(); + + expect(results).toHaveLength(2); + expect(results[0].source).toBe("first.pdf"); + expect(results[0].pdf.info.producer).toBe("pdf.js"); + expect(results[1].loc.pageNumber).toBe(2); + } finally { + tmpDir.removeCallback(); + } +}); + describe("merge insert", () => { let tmpDir: tmp.DirResult; let table: Table; From 8083232dd57c556a8b45bd82e8fa0d85e5be26bb Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Tue, 25 Aug 2026 16:49:17 -0700 Subject: [PATCH 55/58] chore: update lance dependency to v12.0.0-beta.1 (#4055) Updates the Lance Rust workspace dependencies and Java lance-core dependency to [v12.0.0-beta.1](https://github.com/lance-format/lance/releases/tag/v12.0.0-beta.1). Includes compatibility updates for the renamed shard-manifest API and paginated object-store wrappers. --- Cargo.lock | 84 +++++++++---------- Cargo.toml | 28 +++---- java/pom.xml | 2 +- rust/lancedb/src/io/object_store.rs | 10 ++- .../src/io/object_store/io_tracking.rs | 10 ++- rust/lancedb/src/table.rs | 16 ++++ rust/lancedb/src/table/query/lsm.rs | 2 +- 7 files changed, 92 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 33e43f9fa..63ee13e78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arc-swap", "arrow", @@ -4888,8 +4888,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow-array", "arrow-buffer", @@ -4911,7 +4911,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow-array", "arrow-buffer", @@ -4925,7 +4925,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow-array", "arrow-schema", @@ -4934,8 +4934,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrayref", "crunchy", @@ -4945,8 +4945,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow-array", "arrow-buffer", @@ -4983,8 +4983,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow", "arrow-array", @@ -5013,8 +5013,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow", "arrow-array", @@ -5031,8 +5031,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "proc-macro2", "quote", @@ -5041,8 +5041,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow-arith", "arrow-array", @@ -5075,8 +5075,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow-arith", "arrow-array", @@ -5107,8 +5107,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arc-swap", "arrow", @@ -5172,8 +5172,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow-array", "arrow-schema", @@ -5195,8 +5195,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow", "arrow-array", @@ -5236,8 +5236,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow-array", "arrow-schema", @@ -5251,8 +5251,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow", "async-trait", @@ -5264,8 +5264,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow", "arrow-ipc", @@ -5318,8 +5318,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow-array", "arrow-buffer", @@ -5333,8 +5333,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow", "arrow-array", @@ -5374,8 +5374,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "arrow-array", "arrow-schema", @@ -5388,8 +5388,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "11.0.0-beta.22" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" +version = "12.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 716b14d7a..3f2248e4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=12.0.0-beta.1", default-features = false, "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=12.0.0-beta.1", default-features = false, "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=12.0.0-beta.1", default-features = false, "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 4d83153ab..78aa17bb7 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 11.0.0-beta.22 + 12.0.0-beta.1 false 2.30.0 1.7 diff --git a/rust/lancedb/src/io/object_store.rs b/rust/lancedb/src/io/object_store.rs index d594bd857..c4a9a4f7e 100644 --- a/rust/lancedb/src/io/object_store.rs +++ b/rust/lancedb/src/io/object_store.rs @@ -10,7 +10,7 @@ use lance::io::WrappingObjectStore; use object_store::{ CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, - UploadPart, path::Path, + UploadPart, list::PaginatedListStore, path::Path, }; use async_trait::async_trait; @@ -187,6 +187,14 @@ impl WrappingObjectStore for MirroringObjectStoreWrapper { secondary: self.secondary.clone(), }) } + + fn wrap_paginated( + &self, + _store_prefix: &str, + original: Arc, + ) -> Option> { + Some(original) + } } // windows pathing can't be simply concatenated diff --git a/rust/lancedb/src/io/object_store/io_tracking.rs b/rust/lancedb/src/io/object_store/io_tracking.rs index bd4f8f54a..7f9750216 100644 --- a/rust/lancedb/src/io/object_store/io_tracking.rs +++ b/rust/lancedb/src/io/object_store/io_tracking.rs @@ -12,7 +12,7 @@ use lance::io::WrappingObjectStore; use object_store::{ CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult, - UploadPart, path::Path, + UploadPart, list::PaginatedListStore, path::Path, }; #[derive(Debug, Default)] @@ -57,6 +57,14 @@ impl WrappingObjectStore for IoStatsHolder { stats: self.0.clone(), }) } + + fn wrap_paginated( + &self, + _store_prefix: &str, + original: Arc, + ) -> Option> { + Some(original) + } } impl IoTrackingStore { diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 5007a61d9..e544607f4 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -4118,6 +4118,14 @@ mod tests { parent_list_calls: self.parent_list_calls.clone(), }) } + + fn wrap_paginated( + &self, + _store_prefix: &str, + _original: Arc, + ) -> Option> { + None + } } #[tokio::test] @@ -4221,6 +4229,14 @@ mod tests { self.called.store(true, Ordering::Relaxed); original } + + fn wrap_paginated( + &self, + _store_prefix: &str, + original: Arc, + ) -> Option> { + Some(original) + } } #[tokio::test] diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 255d649b1..5c340cc35 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -298,7 +298,7 @@ async fn build_read_context( for shard_id in shard_ids { let manifest_store = ShardManifestStore::new(store.clone(), &base_path, shard_id, scan_batch_size); - if let Some(manifest) = manifest_store.read_latest().await? { + if let Some(manifest) = manifest_store.latest().await? { snapshots.push(snapshot_from_manifest(shard_id, &manifest, &exclude)); } } From 9b825c5f298f45001afd1837bd360682c7282ecc Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:08:48 +0800 Subject: [PATCH 56/58] fix(node): route auto search using table embeddings (#3832) ## Summary - Resolve automatic string-search routing from the active table schema whenever the query executes. - Defer embedding-provider construction while leaving explicit vector and FTS routes unchanged. - Cover unrelated global registrations and metadata transitions across repeated executions of one query builder. ## Root cause LocalTable.search used the number of globally registered embedding providers to choose between vector and full-text search. A provider registered for any other table therefore sent a plain FTS table down the vector path. A wrapper-lifetime metadata snapshot avoided that contamination but became stale after time travel or read-consistency refreshes. The query now records fluent builder operations and creates the appropriate native vector or FTS query from the active schema on each execution. ## Validation - pnpm build - pnpm tsc - pnpm lint - pnpm run docs - pnpm test --runInBand (681 passed, 5 skipped) Fixes #1557 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo --- nodejs/__test__/table.test.ts | 343 +++++++++++++++++++++++++++++- nodejs/lancedb/query.ts | 224 ++++++++++--------- nodejs/lancedb/table.ts | 39 ++-- nodejs/src/table.rs | 7 + rust/lancedb/src/remote/table.rs | 14 ++ rust/lancedb/src/table.rs | 28 +++ rust/lancedb/src/table/dataset.rs | 69 +++++- rust/lancedb/src/table/merge.rs | 38 ++++ rust/lancedb/src/table/query.rs | 31 +++ 9 files changed, 672 insertions(+), 121 deletions(-) diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index aaa144f16..dae640850 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2585,7 +2585,24 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( ); }); - test("full text search if no embedding function provided", async () => { + test("full text search if only an unrelated embedding function is registered", async () => { + register("unused")( + class extends EmbeddingFunction { + ndims() { + return 3; + } + embeddingDataType() { + return new Float32(); + } + async computeQueryEmbeddings(_data: string) { + return [1, 2, 3]; + } + async computeSourceEmbeddings(data: string[]) { + return data.map(() => [1, 2, 3]); + } + }, + ); + const db = await connect(tmpDir.name); const data = [ { text: "hello world", vector: [0.1, 0.2, 0.3] }, @@ -2607,6 +2624,306 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( expect(results2[0].text).toBe(data[1].text); }); + test("auto search stays consistent with the active revision", async () => { + let initCalls = 0; + let queryCalls = 0; + let markStarted!: () => void; + const started = new Promise((resolve) => { + markStarted = resolve; + }); + let releaseEmbedding!: () => void; + const embeddingReleased = new Promise((resolve) => { + releaseEmbedding = resolve; + }); + + @register("refresh-test") + class TestEmbedding extends EmbeddingFunction { + async init() { + initCalls += 1; + } + ndims() { + return 1; + } + embeddingDataType() { + return new arrow.Float32(); + } + async computeQueryEmbeddings(value: string) { + queryCalls += 1; + if (value === "blocked") { + markStarted(); + await embeddingReleased; + } + return value === "greetings" ? [0.1] : [0.2]; + } + async computeSourceEmbeddings(values: string[]) { + return values.map((value) => + value === "hello world" ? [0.1] : [0.2], + ); + } + } + + const writer = await connect(tmpDir.name); + await writer.createTable("test", [{ text: "plain", vector: [0.0] }]); + const reader = await connect(tmpDir.name, { + readConsistencyInterval: 0, + }); + const tracked = await reader.openTable("test"); + type SnapshotCountingNative = { + querySnapshot: () => Promise; + }; + const native = (tracked as unknown as { inner: SnapshotCountingNative }) + .inner; + const querySnapshot = native.querySnapshot.bind(native); + let snapshotCalls = 0; + native.querySnapshot = async () => { + snapshotCalls += 1; + return await querySnapshot(); + }; + const autoQuery = tracked.search("greetings").select(["text"]).limit(1); + + const func = new TestEmbedding(); + const schema = LanceSchema({ + text: func.sourceField(new arrow.Utf8()), + vector: func.vectorField(), + }); + const data = [{ text: "hello world" }, { text: "goodbye world" }]; + await writer.createTable("test", data, { mode: "overwrite", schema }); + const baselineInitCalls = initCalls; + + expect( + (await tracked.schema()).metadata.get("embedding_functions"), + ).toBeDefined(); + const results = await autoQuery.toArray(); + expect(results[0].text).toBe(data[0].text); + expect(initCalls).toBe(baselineInitCalls + 1); + expect(queryCalls).toBe(1); + expect(snapshotCalls).toBe(1); + + const repeatedResults = await autoQuery.toArray(); + expect(repeatedResults[0].text).toBe(data[0].text); + expect(initCalls).toBe(baselineInitCalls + 1); + expect(queryCalls).toBe(1); + expect(snapshotCalls).toBe(2); + + const pending = tracked + .search("blocked") + .select(["text"]) + .limit(1) + .toArray(); + await started; + + const ftsData = [ + { text: "greetings from full text", vector: [0.0] }, + { text: "blocked from full text", vector: [0.0] }, + ]; + const ftsTable = await writer.createTable("test", ftsData, { + mode: "overwrite", + }); + await ftsTable.createIndex("text", { config: Index.fts() }); + releaseEmbedding(); + + const pendingResults = await pending; + expect(pendingResults[0].text).toBe(data[1].text); + + expect( + (await tracked.schema()).metadata.get("embedding_functions"), + ).toBeUndefined(); + const ftsResults = await autoQuery.toArray(); + expect(ftsResults[0].text).toBe(ftsData[0].text); + }); + + test("auto search keeps newer preparation during a revision race", async () => { + let aCalls = 0; + let bCalls = 0; + let markAStarted!: () => void; + const aStarted = new Promise((resolve) => { + markAStarted = resolve; + }); + let releaseA!: () => void; + const aReleased = new Promise((resolve) => { + releaseA = resolve; + }); + let markBStarted!: () => void; + const bStarted = new Promise((resolve) => { + markBStarted = resolve; + }); + let releaseB!: () => void; + const bReleased = new Promise((resolve) => { + releaseB = resolve; + }); + + @register("race-a") + class EmbeddingA extends EmbeddingFunction { + ndims() { + return 1; + } + embeddingDataType() { + return new arrow.Float32(); + } + async computeQueryEmbeddings() { + aCalls += 1; + markAStarted(); + await aReleased; + return [0.1]; + } + async computeSourceEmbeddings(values: string[]) { + return values.map(() => [0.1]); + } + } + + @register("race-b") + class EmbeddingB extends EmbeddingFunction { + ndims() { + return 1; + } + embeddingDataType() { + return new arrow.Float32(); + } + async computeQueryEmbeddings() { + bCalls += 1; + markBStarted(); + await bReleased; + return [0.2]; + } + async computeSourceEmbeddings(values: string[]) { + return values.map(() => [0.2]); + } + } + + const writer = await connect(tmpDir.name); + const embeddingA = new EmbeddingA(); + const schemaA = LanceSchema({ + text: embeddingA.sourceField(new arrow.Utf8()), + vector: embeddingA.vectorField(), + }); + await writer.createTable("race", [{ text: "revision a" }], { + schema: schemaA, + }); + const reader = await connect(tmpDir.name, { + readConsistencyInterval: 0, + }); + const tracked = await reader.openTable("race"); + const query = tracked.search("query"); + + const first = query.toArray(); + await aStarted; + + const embeddingB = new EmbeddingB(); + const schemaB = LanceSchema({ + text: embeddingB.sourceField(new arrow.Utf8()), + vector: embeddingB.vectorField(), + }); + await writer.createTable("race", [{ text: "revision b" }], { + mode: "overwrite", + schema: schemaB, + }); + const second = query.toArray(); + await bStarted; + + releaseA(); + releaseB(); + await Promise.all([first, second]); + expect(aCalls).toBe(1); + expect(bCalls).toBe(1); + }); + + test("stale FTS routing keeps newer vector preparation", async () => { + let vectorCalls = 0; + let markVectorStarted!: () => void; + const vectorStarted = new Promise((resolve) => { + markVectorStarted = resolve; + }); + let releaseVector!: () => void; + const vectorReleased = new Promise((resolve) => { + releaseVector = resolve; + }); + + @register("stale-fts-race") + class RaceEmbedding extends EmbeddingFunction { + ndims() { + return 1; + } + embeddingDataType() { + return new arrow.Float32(); + } + async computeQueryEmbeddings() { + vectorCalls += 1; + markVectorStarted(); + await vectorReleased; + return [0.1]; + } + async computeSourceEmbeddings(values: string[]) { + return values.map(() => [0.1]); + } + } + + const writer = await connect(tmpDir.name); + const ftsTable = await writer.createTable("stale_fts", [ + { text: "hello", vector: [0.0] }, + ]); + await ftsTable.createIndex("text", { config: Index.fts() }); + + const reader = await connect(tmpDir.name, { + readConsistencyInterval: 0, + }); + const tracked = await reader.openTable("stale_fts"); + type Snapshot = { + schema: () => Promise; + }; + type NativeWithSnapshot = { + querySnapshot: () => Promise; + }; + const native = (tracked as unknown as { inner: NativeWithSnapshot }) + .inner; + const querySnapshot = native.querySnapshot.bind(native); + let snapshotCalls = 0; + let markStaleSchemaStarted!: () => void; + const staleSchemaStarted = new Promise((resolve) => { + markStaleSchemaStarted = resolve; + }); + let releaseStaleSchema!: () => void; + const staleSchemaReleased = new Promise((resolve) => { + releaseStaleSchema = resolve; + }); + native.querySnapshot = async () => { + const snapshot = await querySnapshot(); + snapshotCalls += 1; + if (snapshotCalls === 1) { + const schema = snapshot.schema.bind(snapshot); + snapshot.schema = async () => { + markStaleSchemaStarted(); + await staleSchemaReleased; + return await schema(); + }; + } + return snapshot; + }; + + const query = tracked.search("hello"); + const staleFtsExecution = query.toArray(); + await staleSchemaStarted; + + const embedding = new RaceEmbedding(); + const vectorSchema = LanceSchema({ + text: embedding.sourceField(new arrow.Utf8()), + vector: embedding.vectorField(), + }); + await writer.createTable("stale_fts", [{ text: "hello" }], { + mode: "overwrite", + schema: vectorSchema, + }); + + const vectorExecution = query.toArray(); + await vectorStarted; + releaseStaleSchema(); + await staleFtsExecution; + releaseVector(); + await vectorExecution; + + await query.toArray(); + expect(vectorCalls).toBe(1); + }); + test("tokenizes FTS queries by column or index name", async () => { const db = await connect(tmpDir.name); const data = [ @@ -3157,6 +3474,30 @@ describe("column name options", () => { expect(results[1].query_index).toBe(1); }); + test("observes promised additional vectors while the query is pending", async () => { + const initialVector = new Promise(() => undefined); + const query = table.query().nearestTo(initialVector); + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown) => unhandled.push(reason); + process.on("unhandledRejection", onUnhandled); + + try { + query.addQueryVector(Promise.reject(new Error("extra vector failed"))); + await new Promise((resolve) => setImmediate(resolve)); + expect(unhandled).toEqual([]); + + const rejectedQuery = table + .query() + .nearestTo([0.1, 0.2]) + .addQueryVector(Promise.reject(new Error("consumed vector failed"))); + await expect(rejectedQuery.toArray()).rejects.toThrow( + "consumed vector failed", + ); + } finally { + process.off("unhandledRejection", onUnhandled); + } + }); + test("index and search multivectors", async () => { const db = await connect(tmpDir.name); const data = []; diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index 3b9b286a0..f1d31eae1 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -100,6 +100,29 @@ export interface FullTextSearchOptions { columns?: string | string[]; } +function nearestToNative( + inner: NativeQuery, + vector: Awaited, +): NativeVectorQuery { + const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector); + if (raw) { + return inner.nearestToRaw(raw.data, raw.dtype); + } + return inner.nearestTo(Float32Array.from(vector as number[])); +} + +function addQueryVectorToNative( + inner: NativeVectorQuery, + vector: Awaited, +) { + const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector); + if (raw) { + inner.addQueryVectorRaw(raw.data, raw.dtype); + } else { + inner.addQueryVector(Float32Array.from(vector as number[])); + } +} + /** Common methods supported by all query types * * @see {@link Query} @@ -499,6 +522,13 @@ export class VectorQuery extends StandardQueryBase { super(inner); } + /** + * @hidden + */ + protected doVectorCall(fn: (inner: NativeVectorQuery) => void) { + super.doCall(fn); + } + /** * Set the number of partitions to search (probe) * @@ -526,7 +556,7 @@ export class VectorQuery extends StandardQueryBase { * the minimum and maximum to the same value. */ nprobes(nprobes: number): VectorQuery { - super.doCall((inner) => inner.nprobes(nprobes)); + this.doVectorCall((inner) => inner.nprobes(nprobes)); return this; } @@ -540,7 +570,7 @@ export class VectorQuery extends StandardQueryBase { * but will also increase latency. */ minimumNprobes(minimumNprobes: number): VectorQuery { - super.doCall((inner) => inner.minimumNprobes(minimumNprobes)); + this.doVectorCall((inner) => inner.minimumNprobes(minimumNprobes)); return this; } @@ -554,7 +584,7 @@ export class VectorQuery extends StandardQueryBase { * potential false negatives. */ maximumNprobes(maximumNprobes: number): VectorQuery { - super.doCall((inner) => inner.maximumNprobes(maximumNprobes)); + this.doVectorCall((inner) => inner.maximumNprobes(maximumNprobes)); return this; } @@ -567,7 +597,7 @@ export class VectorQuery extends StandardQueryBase { * `undefined` means no lower or upper bound. */ distanceRange(lowerBound?: number, upperBound?: number): VectorQuery { - super.doCall((inner) => inner.distanceRange(lowerBound, upperBound)); + this.doVectorCall((inner) => inner.distanceRange(lowerBound, upperBound)); return this; } @@ -581,7 +611,7 @@ export class VectorQuery extends StandardQueryBase { * also increase the latency of your query. The default value is 1.5*limit. */ ef(ef: number): VectorQuery { - super.doCall((inner) => inner.ef(ef)); + this.doVectorCall((inner) => inner.ef(ef)); return this; } @@ -595,7 +625,7 @@ export class VectorQuery extends StandardQueryBase { * whose data type is a fixed-size-list of floats. */ column(column: string): VectorQuery { - super.doCall((inner) => inner.column(column)); + this.doVectorCall((inner) => inner.column(column)); return this; } @@ -616,7 +646,7 @@ export class VectorQuery extends StandardQueryBase { distanceType( distanceType: Required["distanceType"], ): VectorQuery { - super.doCall((inner) => inner.distanceType(distanceType)); + this.doVectorCall((inner) => inner.distanceType(distanceType)); return this; } @@ -650,7 +680,7 @@ export class VectorQuery extends StandardQueryBase { * distance between the query vector and the actual uncompressed vector. */ refineFactor(refineFactor: number): VectorQuery { - super.doCall((inner) => inner.refineFactor(refineFactor)); + this.doVectorCall((inner) => inner.refineFactor(refineFactor)); return this; } @@ -675,7 +705,7 @@ export class VectorQuery extends StandardQueryBase { * factor can often help restore some of the results lost by post filtering. */ postfilter(): VectorQuery { - super.doCall((inner) => inner.postfilter()); + this.doVectorCall((inner) => inner.postfilter()); return this; } @@ -689,7 +719,7 @@ export class VectorQuery extends StandardQueryBase { * calculate your recall to select an appropriate value for nprobes. */ bypassVectorIndex(): VectorQuery { - super.doCall((inner) => inner.bypassVectorIndex()); + this.doVectorCall((inner) => inner.bypassVectorIndex()); return this; } @@ -705,35 +735,31 @@ export class VectorQuery extends StandardQueryBase { */ addQueryVector(vector: IntoVector): VectorQuery { if (vector instanceof Promise) { + // Observe the promise as soon as it is accepted. The existing native + // query may still be pending, and delaying observation until it resolves + // can otherwise surface a fast rejection as unhandled. + const settledVector = vector.then( + (value) => ({ status: "fulfilled" as const, value }), + (reason) => ({ status: "rejected" as const, reason }), + ); const res = (async () => { - try { - const v = await vector; - // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping - const value: any = this.addQueryVector(v); - const inner = value.inner as - | NativeVectorQuery - | Promise; - return inner; - } catch (e) { - return Promise.reject(e); + const inner = await this.getInner(); + const outcome = await settledVector; + if (outcome.status === "rejected") { + throw outcome.reason; } + addQueryVectorToNative(inner, outcome.value); + return inner; })(); return new VectorQuery(res); } else { - super.doCall((inner) => { - const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector); - if (raw) { - inner.addQueryVectorRaw(raw.data, raw.dtype); - } else { - inner.addQueryVector(Float32Array.from(vector as number[])); - } - }); + this.doVectorCall((inner) => addQueryVectorToNative(inner, vector)); return this; } } rerank(reranker: Reranker): VectorQuery { - super.doCall((inner) => + this.doVectorCall((inner) => inner.rerank(async (args) => { const vecResults = await fromBufferToRecordBatch(args.vecResults); const ftsResults = await fromBufferToRecordBatch(args.ftsResults); @@ -752,6 +778,71 @@ export class VectorQuery extends StandardQueryBase { } } +/** + * Create a string query whose vector/FTS routing is resolved against the active + * table schema when the query executes. + * + * @hidden + */ +export function createAutoQuery( + table: NativeTable, + query: string, + columns: string[] | null, + getVector: (metadata: string) => Promise>, +): AutoQuery { + type RouteSnapshot = { + table: NativeTable; + embeddingMetadata: string | undefined; + }; + type CachedPreparation = { + metadata: string; + vector: Promise>; + }; + + let cachedPreparation: CachedPreparation | undefined; + + const snapshotRoute = async (): Promise => { + const snapshot = await table.querySnapshot(); + const schema = tableFromIPC(await snapshot.schema()).schema; + return { + table: snapshot, + embeddingMetadata: schema.metadata.get("embedding_functions"), + }; + }; + + const createInner = async (): Promise => { + const route = await snapshotRoute(); + if (route.embeddingMetadata === undefined) { + const inner = route.table.query(); + inner.fullTextSearch({ query, columns }); + return inner; + } + + const metadata = route.embeddingMetadata; + if (cachedPreparation?.metadata !== metadata) { + cachedPreparation = { + metadata, + vector: Promise.resolve().then(() => getVector(metadata)), + }; + } + + const preparation = cachedPreparation; + let vector: Awaited; + try { + vector = await preparation.vector; + } catch (error) { + if (cachedPreparation === preparation) { + cachedPreparation = undefined; + } + throw error; + } + + return nearestToNative(route.table.query(), vector); + }; + + return new AutoQuery(createInner); +} + /** * A query that returns a subset of the rows in the table. * @@ -836,37 +927,6 @@ export class Query extends StandardQueryBase { super(tbl.query()); } - /** @hidden */ - static autoSearch( - tbl: () => Promise, - query: string, - vector: (tbl: NativeTable) => Promise | undefined>, - columns?: string[], - ): AutoQuery { - const nativeQuery = async () => { - const snapshot = await Promise.resolve(tbl()); - const resolved = await vector(snapshot); - const inner = snapshot.query(); - if (resolved === undefined) { - inner.fullTextSearch({ - query, - columns: columns ?? null, - }); - return inner; - } - - const raw = Array.isArray(resolved) - ? null - : extractVectorBuffer(resolved); - if (raw) { - return inner.nearestToRaw(raw.data, raw.dtype); - } - return inner.nearestTo(Float32Array.from(resolved as number[])); - }; - - return new AutoQuery(nativeQuery); - } - /** * Find the nearest vectors to the given query vector. * @@ -905,45 +965,19 @@ export class Query extends StandardQueryBase { * a default `limit` of 10 will be used. @see {@link Query#limit} */ nearestTo(vector: IntoVector): VectorQuery { - const callNearestTo = ( - inner: NativeQuery, - resolved: Float32Array | Float64Array | Uint8Array | number[], - ): NativeVectorQuery => { - const raw = Array.isArray(resolved) - ? null - : extractVectorBuffer(resolved); - if (raw) { - return inner.nearestToRaw(raw.data, raw.dtype); - } - return inner.nearestTo(Float32Array.from(resolved as number[])); - }; - - if (this.inner instanceof Promise) { - const nativeQuery = this.inner.then(async (inner) => { - const resolved = vector instanceof Promise ? await vector : vector; - return callNearestTo(inner, resolved); - }); + const inner = this.inner; + if (inner instanceof Promise) { + const nativeQuery = inner.then(async (resolvedInner) => + nearestToNative(resolvedInner, await vector), + ); return new VectorQuery(nativeQuery); } if (vector instanceof Promise) { - const res = (async () => { - try { - const v = await vector; - // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping - const value: any = this.nearestTo(v); - const inner = value.inner as - | NativeVectorQuery - | Promise; - return inner; - } catch (e) { - return Promise.reject(e); - } - })(); - return new VectorQuery(res); - } else { - const vectorQuery = callNearestTo(this.inner, vector); - return new VectorQuery(vectorQuery); + return new VectorQuery( + vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)), + ); } + return new VectorQuery(nearestToNative(inner, vector)); } nearestToText(query: string | FullTextQuery, columns?: string[]): Query { diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index a4fc76ef1..82f23e2f2 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -48,6 +48,7 @@ import { Query, TakeQuery, VectorQuery, + createAutoQuery, instanceOfFullTextQuery, } from "./query"; import { sanitizeType } from "./sanitize"; @@ -1177,33 +1178,29 @@ export class LocalTable extends Table { }); } - if (queryType === "auto" && typeof query !== "string") { - return this.query().fullTextSearch(query, { - columns: ftsColumns, - }); - } + if (queryType === "auto") { + if (instanceOfFullTextQuery(query)) { + return this.query().fullTextSearch(query, { + columns: ftsColumns, + }); + } - if (queryType === "auto" && typeof query === "string") { - const vector = async (snapshot: _NativeTable) => { - const functions = await this.getEmbeddingFunctions(snapshot); + const columns = + typeof ftsColumns === "string" ? [ftsColumns] : (ftsColumns ?? null); + return createAutoQuery(this.inner, query, columns, async (metadata) => { + const functions = await getRegistry().parseFunctions( + new Map([["embedding_functions", metadata]]), + ); // TODO: Support multiple embedding functions const embeddingFunc: EmbeddingFunctionConfig | undefined = functions .values() .next().value; - if (embeddingFunc === undefined) { - return undefined; - } + // The route only calls this callback when embedding metadata exists. + // parseFunctions either yields a provider or reports malformed metadata. + if (!embeddingFunc) + throw new Error("Invalid embedding function metadata"); return await embeddingFunc.function.computeQueryEmbeddings(query); - }; - - const columns = - typeof ftsColumns === "string" ? [ftsColumns] : ftsColumns; - return Query.autoSearch( - () => this.inner.checkoutCurrent(), - query, - vector, - columns, - ); + }); } const queryPromise = this.getEmbeddingFunctions().then( diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index ff16ac042..db74d38fa 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -278,6 +278,13 @@ impl Table { Ok(Query::new(self.inner_ref()?.query())) } + /// Return a read-only table handle pinned to the current query revision. + #[napi(catch_unwind)] + pub async fn query_snapshot(&self) -> napi::Result { + let snapshot = self.inner_ref()?.query_snapshot().await.default_error()?; + Ok(Self::new(snapshot)) + } + #[napi(catch_unwind)] pub fn take_offsets(&self, offsets: Vec) -> napi::Result { Ok(TakeQuery::new( diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 77d5983eb..b116083c8 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -1722,6 +1722,20 @@ impl BaseTable for RemoteTable { fn id(&self) -> &str { &self.identifier } + async fn query_snapshot(&self) -> Result> { + let description = self.describe().await?; + let TableDescription { + version, + schema, + location, + } = description; + let schema = Arc::new(arrow_schema::Schema::try_from(schema)?); + let snapshot = self.with_branch(self.branch.clone()); + *snapshot.version.write().await = Some(version); + *snapshot.location.write().await = location; + snapshot.schema_cache.seed(schema); + Ok(Arc::new(snapshot)) + } async fn version(&self) -> Result { self.describe().await.map(|desc| desc.version) } diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index e544607f4..6b5f687c7 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -560,6 +560,13 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { fn id(&self) -> &str; /// Get the arrow [Schema] of the table. async fn schema(&self) -> Result; + /// Create a read-only handle pinned to the table's current active revision. + /// + /// The returned handle is independent from later refreshes or checkouts on + /// this handle. This is used by bindings that must prepare client-side + /// query state from the same revision that the query will execute against. + #[doc(hidden)] + async fn query_snapshot(&self) -> Result>; /// Count the number of rows in this table. async fn count_rows(&self, filter: Option) -> Result; /// Create a physical plan for the query. @@ -1139,6 +1146,16 @@ impl Table { self.inner.schema().await } + /// Create a read-only handle pinned to the current active revision. + #[doc(hidden)] + pub async fn query_snapshot(&self) -> Result { + Ok(Self { + inner: self.inner.query_snapshot().await?, + database: self.database.clone(), + embedding_registry: self.embedding_registry.clone(), + }) + } + /// Count the number of rows in this dataset. /// /// # Arguments @@ -3059,6 +3076,17 @@ impl BaseTable for NativeTable { &self.id } + async fn query_snapshot(&self) -> Result> { + let snapshot = self.dataset.new_query_snapshot().await?; + let mut table = self.with_dataset(snapshot); + // QueryTable requests do not carry a revision. A pinned snapshot must + // execute locally until the namespace API can accept that revision. + table + .pushdown_operations + .remove(&NamespaceClientPushdownOperation::QueryTable); + Ok(Arc::new(table)) + } + async fn version(&self) -> Result { Ok(self.dataset.get().await?.version().version) } diff --git a/rust/lancedb/src/table/dataset.rs b/rust/lancedb/src/table/dataset.rs index e37c3fc9f..5e3733b85 100644 --- a/rust/lancedb/src/table/dataset.rs +++ b/rust/lancedb/src/table/dataset.rs @@ -32,6 +32,10 @@ struct DatasetState { /// `Some(version)` = pinned to a specific version (time travel), /// `None` = tracking latest. pinned_version: Option, + /// Whether the pin is an internal query snapshot rather than user-visible + /// time travel. Query snapshots remain read-only but preserve MemWAL read + /// semantics. + query_snapshot: bool, } #[derive(Debug, Clone)] @@ -70,6 +74,7 @@ impl DatasetConsistencyWrapper { state: Arc::new(Mutex::new(DatasetState { dataset, pinned_version: None, + query_snapshot: false, })), consistency, shard_writer: Arc::new(ShardWriterCache::default()), @@ -93,6 +98,36 @@ impl DatasetConsistencyWrapper { wrapper } + /// Create an independent read-only wrapper pinned to the current dataset + /// while retaining this wrapper's live MemWAL read context. + pub async fn new_query_snapshot(&self) -> Result { + // Apply the configured consistency policy before taking the snapshot. + // The returned dataset is intentionally discarded: a checkout may race + // after this await, so the dataset and its pin provenance must instead + // be cloned together from one authoritative state sample below. + self.get().await?; + + let (dataset, query_snapshot) = { + let state = self.state.lock()?; + // Preserve user time travel so the MemWAL safety guard still sees + // it. Latest and already-internal snapshots remain internal pins. + ( + state.dataset.clone(), + state.query_snapshot || state.pinned_version.is_none(), + ) + }; + let version = dataset.version().version; + Ok(Self { + state: Arc::new(Mutex::new(DatasetState { + dataset, + pinned_version: Some(version), + query_snapshot, + })), + consistency: ConsistencyMode::Lazy, + shard_writer: self.shard_writer.clone(), + }) + } + /// The MemWAL `ShardWriter` cache co-located with this dataset. pub(crate) fn shard_writer(&self) -> &Arc { &self.shard_writer @@ -169,6 +204,7 @@ impl DatasetConsistencyWrapper { let mut state = self.state.lock()?; state.dataset = Arc::new(new_dataset); state.pinned_version = None; + state.query_snapshot = false; drop(state); if let ConsistencyMode::Eventual(bg_cache) = &self.consistency { bg_cache.invalidate(); @@ -202,10 +238,10 @@ impl DatasetConsistencyWrapper { /// Returns the version, if in time travel mode, or None otherwise. pub fn time_travel_version(&self) -> Option { - self.state - .lock() - .unwrap_or_else(|e| e.into_inner()) - .pinned_version + let state = self.state.lock().unwrap_or_else(|e| e.into_inner()); + (!state.query_snapshot) + .then_some(state.pinned_version) + .flatten() } /// Convert into a wrapper in latest version mode. @@ -225,6 +261,7 @@ impl DatasetConsistencyWrapper { if state.pinned_version.is_some() { state.dataset = Arc::new(new_dataset); state.pinned_version = None; + state.query_snapshot = false; } drop(state); if let ConsistencyMode::Eventual(bg_cache) = &self.consistency { @@ -260,6 +297,7 @@ impl DatasetConsistencyWrapper { let mut state = self.state.lock()?; state.dataset = Arc::new(new_dataset); state.pinned_version = Some(version_value); + state.query_snapshot = false; Ok(()) } @@ -461,6 +499,29 @@ mod tests { assert_eq!(wrapper.time_travel_version(), Some(1)); } + #[tokio::test] + async fn test_query_snapshot_samples_dataset_and_pin_together() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let ds = create_test_dataset(uri).await; + + let wrapper = DatasetConsistencyWrapper::new_latest(ds, None); + wrapper.as_time_travel(1u64).await.unwrap(); + let stale_time_travel_dataset = wrapper.get().await.unwrap(); + + append_to_dataset(uri).await; + wrapper.as_latest().await.unwrap(); + + let snapshot = wrapper.new_query_snapshot().await.unwrap(); + let snapshot_dataset = snapshot.get().await.unwrap(); + assert_eq!(snapshot_dataset.version().version, 2); + assert_ne!( + snapshot_dataset.version().version, + stale_time_travel_dataset.version().version + ); + assert_eq!(snapshot.time_travel_version(), None); + } + #[tokio::test] async fn test_as_latest_from_time_travel() { let dir = tempfile::tempdir().unwrap(); diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index b9dd5732b..3227e3edf 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -1056,6 +1056,44 @@ mod lsm_tests { ); } + #[tokio::test] + async fn query_snapshot_preserves_lsm_read_semantics() { + let dir = tempdir().unwrap(); + let table = id_value_table(&dir).await; + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + lsm_upsert(&table, vec![4, 5]).await; + + let snapshot = table.query_snapshot().await.unwrap(); + let rows = collect_id_value(snapshot.query().execute().await.unwrap()).await; + assert_eq!( + rows.iter().map(|(id, _)| *id).collect::>(), + vec![1, 2, 3, 4, 5] + ); + } + + #[tokio::test] + async fn query_snapshot_preserves_time_travel_lsm_guard() { + let dir = tempdir().unwrap(); + let table = id_value_table(&dir).await; + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + lsm_upsert(&table, vec![4]).await; + + let version = table.version().await.unwrap(); + table.checkout(version).await.unwrap(); + let direct_error = table.query().execute().await.err().unwrap(); + assert!(matches!(direct_error, Error::NotSupported { .. })); + + let snapshot = table.query_snapshot().await.unwrap(); + let snapshot_error = snapshot.query().execute().await.err().unwrap(); + assert!(matches!(snapshot_error, Error::NotSupported { .. })); + } + #[tokio::test] async fn lsm_read_dedup_newest_wins() { let dir = tempdir().unwrap(); diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index d35286c84..629cb4e6f 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -1056,6 +1056,37 @@ mod tests { assert_eq!(namespace_client.query_table_calls.load(Ordering::SeqCst), 0); } + #[tokio::test] + async fn test_query_snapshot_disables_namespace_pushdown() { + use crate::connect; + use crate::table::BaseTable; + use arrow_array::{Int32Array, RecordBatch}; + use arrow_schema::{DataType, Field, Schema}; + + let conn = connect("memory://").execute().await.unwrap(); + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]).unwrap(); + let table = conn + .create_table("test_snapshot_namespace_fallback", vec![batch]) + .execute() + .await + .unwrap(); + let mut native_table = table.as_native().unwrap().clone(); + native_table.namespace_client = Some(Arc::new(CountingNamespaceClient::default())); + native_table + .pushdown_operations + .insert(NamespaceClientPushdownOperation::QueryTable); + + let snapshot = BaseTable::query_snapshot(&native_table).await.unwrap(); + let snapshot = snapshot.as_any().downcast_ref::().unwrap(); + assert!( + !can_execute_namespace_query(snapshot, &AnyQuery::Query(QueryRequest::default()),) + .await + .unwrap() + ); + } + #[tokio::test] async fn test_create_plan_multivector_structure() { use arrow_array::{Float32Array, RecordBatch}; From 21530432a06ddd3814ec8b0e617abe3eda041fc3 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Tue, 25 Aug 2026 19:56:26 -0700 Subject: [PATCH 57/58] chore: update lance dependency to v12.0.0-beta.2 (#4056) Updates the Rust workspace Lance crates and Java lance-core dependency to [v12.0.0-beta.2](https://github.com/lance-format/lance/releases/tag/v12.0.0-beta.2). No compatibility fixes were required; full-workspace Clippy passes with all features and warnings denied. --- Cargo.lock | 84 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 28 +++++++++--------- java/pom.xml | 2 +- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 63ee13e78..e27f9f271 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arc-swap", "arrow", @@ -4888,8 +4888,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -4911,7 +4911,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -4925,7 +4925,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow-array", "arrow-schema", @@ -4934,8 +4934,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrayref", "crunchy", @@ -4945,8 +4945,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -4983,8 +4983,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow", "arrow-array", @@ -5013,8 +5013,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow", "arrow-array", @@ -5031,8 +5031,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "proc-macro2", "quote", @@ -5041,8 +5041,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow-arith", "arrow-array", @@ -5075,8 +5075,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow-arith", "arrow-array", @@ -5107,8 +5107,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arc-swap", "arrow", @@ -5172,8 +5172,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow-array", "arrow-schema", @@ -5195,8 +5195,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow", "arrow-array", @@ -5236,8 +5236,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow-array", "arrow-schema", @@ -5251,8 +5251,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow", "async-trait", @@ -5264,8 +5264,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow", "arrow-ipc", @@ -5318,8 +5318,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -5333,8 +5333,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow", "arrow-array", @@ -5374,8 +5374,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "arrow-array", "arrow-schema", @@ -5388,8 +5388,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "12.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.1#e37cee4397abc6a71df3cdfa0e637c274f820aa6" +version = "12.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 3f2248e4b..a16f0412c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=12.0.0-beta.1", default-features = false, "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=12.0.0-beta.1", default-features = false, "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=12.0.0-beta.1", default-features = false, "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=12.0.0-beta.1", "tag" = "v12.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 78aa17bb7..a2cec19c0 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 12.0.0-beta.1 + 12.0.0-beta.2 false 2.30.0 1.7 From 391cac903438fdf67c7e3675757b4e4898ddb150 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Tue, 25 Aug 2026 21:54:36 -0700 Subject: [PATCH 58/58] fix(remote): centralize timeline consistency (#4053) Centralizes remote table freshness fencing and response-version tracking in the default transport path. Covers schema and blob bypass paths, keeps explicit time-travel and cross-timeline operations unfenced, and advances freshness after refresh and index job completion. --- rust/lancedb/src/job.rs | 4 + rust/lancedb/src/remote/table.rs | 1354 ++++++++++++++++++----- rust/lancedb/src/remote/table/blobs.rs | 77 +- rust/lancedb/src/remote/table/insert.rs | 83 +- 4 files changed, 1218 insertions(+), 300 deletions(-) diff --git a/rust/lancedb/src/job.rs b/rust/lancedb/src/job.rs index 94d1ba2b6..22f1a0450 100644 --- a/rust/lancedb/src/job.rs +++ b/rust/lancedb/src/job.rs @@ -47,6 +47,10 @@ impl TerminalResult { } } + pub(crate) fn value(&self) -> Option<&Value> { + self.value.as_ref() + } + fn decode(self) -> Result { let value = self.value.ok_or_else(|| match &self.request_id { Some(request_id) => Error::Http { diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index b116083c8..b1a02a260 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -86,22 +86,31 @@ const METRIC_TYPE_KEY: &str = "metric_type"; const INDEX_TYPE_KEY: &str = "index_type"; const SCHEMA_CACHE_TTL: Duration = Duration::from_secs(30); const SCHEMA_CACHE_REFRESH_WINDOW: Duration = Duration::from_secs(5); +const SCHEMA_SELECTOR_CHANGED: &str = "table selector changed while fetching schema"; /// Per-table state driving the freshness headers (`x-lancedb-min-version`, -/// `x-lancedb-min-timestamp`, and `x-lancedb-min-read-version`) sent on read +/// `x-lancedb-min-timestamp`, and `x-lancedb-min-read-version`) sent on table /// requests. #[derive(Debug, Default, Clone, Copy)] struct FreshnessState { + /// Identifies the handle timeline that produced this state. Explicit + /// checkout operations advance the generation so responses from older + /// in-flight requests cannot repopulate the new timeline's constraints. + generation: u64, + /// Exact-version, tag, and snapshot handles must not carry latest-timeline + /// constraints. Their request body already selects the precise version. + pinned: bool, /// Provides read-your-write within a single handle: writes that return a /// version update this, and reads send it as `x-lancedb-min-version`. min_version: Option, - /// Highest dataset version observed in a *read* response on this handle. - /// Reads send it as `x-lancedb-min-read-version` so a load-balanced query + /// Highest committed dataset version advertised by a successful table + /// response on this handle. Later requests send it as + /// `x-lancedb-min-read-version` so a load-balanced query /// node whose cache is behind this version must refresh before serving, /// giving monotonic reads across nodes regardless of which one the load - /// balancer routes to. Sourced only from reads (always committed dataset - /// versions), never from writes (which may return WAL entry ids), so it is - /// unaffected by the WAL/version mismatch that retired `min_version`. + /// balancer routes to. Unlike write result bodies, this is sourced only + /// from the server's committed dataset-version response header or other + /// typed dataset-version fields, so WAL entry ids cannot enter it. min_read_version: Option, /// Wall-clock time captured at the last [`BaseTable::checkout_latest`] /// call. Subsequent reads send @@ -118,14 +127,22 @@ struct FreshnessState { checkout_baseline: Option, } -/// Snapshot of the headers that should be attached to a single read request. +/// Snapshot of the headers that should be attached to a single table request. #[derive(Debug, Default, Clone, Copy)] struct FreshnessHeaders { + generation: u64, min_version: Option, min_timestamp: Option, min_read_version: Option, } +#[derive(Debug, Default, Clone, Copy)] +struct ReadSnapshot { + version: Option, + freshness_state: FreshnessState, + freshness: FreshnessHeaders, +} + impl FreshnessHeaders { fn apply(self, mut request: RequestBuilder) -> RequestBuilder { if let Some(v) = self.min_version { @@ -140,6 +157,61 @@ impl FreshnessHeaders { } request } + + fn observe_version(self, freshness: &Mutex, version: u64) { + track_read_version_for_generation(freshness, self.generation, version); + } + + fn observe_headers( + self, + freshness: &Mutex, + headers: &reqwest::header::HeaderMap, + ) { + if let Some(version) = headers + .get(&VERSION_HEADER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + { + self.observe_version(freshness, version); + } + } + + fn update_if_current( + self, + freshness: &Mutex, + update: impl FnOnce(&mut FreshnessState), + ) { + let mut state = freshness.lock().unwrap(); + if state.generation == self.generation { + update(&mut state); + } + } + + fn is_current(self, freshness: &Mutex) -> bool { + freshness.lock().unwrap().generation == self.generation + } +} + +fn track_read_version(freshness: &Mutex, version: u64) { + if version == 0 { + return; + } + let mut state = freshness.lock().unwrap(); + state.min_read_version = Some(state.min_read_version.map_or(version, |v| v.max(version))); +} + +fn track_read_version_for_generation( + freshness: &Mutex, + generation: u64, + version: u64, +) { + if version == 0 { + return; + } + let mut state = freshness.lock().unwrap(); + if state.generation == generation { + state.min_read_version = Some(state.min_read_version.map_or(version, |v| v.max(version))); + } } /// A backfill job whose successful wait establishes a read-freshness @@ -150,6 +222,8 @@ struct FreshnessJob { inner: RemoteJob, freshness: Arc>, version: Arc>>, + track_refresh_result: bool, + freshness_request: FreshnessHeaders, } #[async_trait] @@ -166,7 +240,31 @@ impl crate::job::JobHandle for FreshnessJob { let result = crate::job::JobHandle::wait(&self.inner).await?; let version = self.version.read().await; if version.is_none() { - self.freshness.lock().unwrap().checkout_baseline = Some(SystemTime::now()); + let result_version = self + .track_refresh_result + .then(|| result.value()) + .flatten() + .and_then(|value| { + serde_json::from_value::(value.clone()) + .ok() + }) + .map(|result| { + result + .published_version + .map_or(result.source_version, |version| { + version.max(result.source_version) + }) + }) + .filter(|version| *version != 0); + if let Some(version) = result_version { + self.freshness_request + .observe_version(&self.freshness, version); + } else { + self.freshness_request + .update_if_current(&self.freshness, |state| { + state.checkout_baseline = Some(SystemTime::now()); + }); + } } Ok(result) } @@ -193,6 +291,38 @@ fn compute_min_timestamp( } } +fn freshness_headers_snapshot( + freshness: &Mutex, + interval: Option, +) -> FreshnessHeaders { + freshness_state_snapshot(freshness, interval).1 +} + +fn freshness_state_snapshot( + freshness: &Mutex, + interval: Option, +) -> (FreshnessState, FreshnessHeaders) { + let state = *freshness.lock().unwrap(); + if state.pinned { + return ( + state, + FreshnessHeaders { + generation: state.generation, + ..FreshnessHeaders::default() + }, + ); + } + ( + state, + FreshnessHeaders { + generation: state.generation, + min_version: state.min_version, + min_timestamp: compute_min_timestamp(&state, interval, SystemTime::now()), + min_read_version: state.min_read_version, + }, + ) +} + /// Normalize a branch selector: trim whitespace and treat `""` or `"main"` as /// the (absent) main branch, matching the server's convention. fn normalize_branch(branch: Option) -> Option { @@ -210,8 +340,9 @@ impl Tags for RemoteTags<'_, S> { async fn list(&self) -> Result> { let request = self .inner - .post_read(&format!("/v1/table/{}/tags/list/", self.inner.identifier)); - let (request_id, response) = self.inner.send(request, true).await?; + .client + .post(&format!("/v1/table/{}/tags/list/", self.inner.identifier)); + let (request_id, response) = self.inner.send_unfenced(request, true).await?; let response = self .inner .check_table_response(&request_id, response) @@ -241,12 +372,12 @@ impl Tags for RemoteTags<'_, S> { } async fn get_version(&self, tag: &str) -> Result { - let request = self.inner.post_read(&format!( + let request = self.inner.client.post(&format!( "/v1/table/{}/tags/version/", self.inner.identifier )); self.inner - .resolve_tag_version_with_request(tag, request) + .resolve_tag_version_with_request(tag, request, false) .await } @@ -276,7 +407,7 @@ impl Tags for RemoteTags<'_, S> { .post(&format!("/v1/table/{}/tags/delete/", self.inner.identifier)) .json(&serde_json::json!({ "tag": tag })); - let (request_id, response) = self.inner.send(request, true).await?; + let (request_id, response) = self.inner.send_unfenced(request, true).await?; self.inner .check_table_response(&request_id, response) .await?; @@ -468,6 +599,7 @@ impl RemoteTable { let Ok(description) = serde_json::from_str::(describe_body) else { return; }; + self.track_read_version(description.version); if let Ok(schema) = arrow_schema::Schema::try_from(description.schema) { self.schema_cache.seed(Arc::new(schema)); } @@ -510,23 +642,50 @@ impl RemoteTable { } async fn describe(&self) -> Result { - let version = self.current_version().await; - self.describe_version(version).await + self.describe_read_snapshot(self.snapshot_read_state().await) + .await } - async fn describe_version(&self, version: Option) -> Result { - let request = self.post_read(&format!("/v1/table/{}/describe/", self.identifier)); - self.describe_with_request(request, version).await + async fn describe_read_snapshot( + &self, + read_snapshot: ReadSnapshot, + ) -> Result { + let request = self + .client + .post(&format!("/v1/table/{}/describe/", self.identifier)); + self.describe_with_request( + request, + read_snapshot.version, + Some(read_snapshot.freshness), + ) + .await + } + + async fn schema_read_snapshot(&self, read_snapshot: ReadSnapshot) -> Result { + if read_snapshot.freshness.is_current(&self.freshness) + && let Some(schema) = self.schema_cache.try_get() + && read_snapshot.freshness.is_current(&self.freshness) + { + return Ok(schema); + } + + let description = self.describe_read_snapshot(read_snapshot).await?; + Ok(Arc::new(description.schema.try_into()?)) } async fn resolve_tag_version_with_request( &self, tag: &str, request: RequestBuilder, + fenced: bool, ) -> Result { let request = request.json(&serde_json::json!({ "tag": tag })); - let (request_id, response) = self.send(request, true).await?; + let (request_id, response) = if fenced { + self.send(request, true).await? + } else { + self.send_unfenced(request, true).await? + }; let response = self.check_table_response(&request_id, response).await?; match response.text().await { @@ -565,7 +724,7 @@ impl RemoteTable { .client .post(&format!("/v1/table/{}/tags/version/", self.identifier)) .json(&serde_json::json!({ "tag": tag })); - let (request_id, response) = self.send(request, true).await?; + let (request_id, response) = self.send_unfenced(request, true).await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; let value: serde_json::Value = serde_json::from_str(&body).map_err(|e| Error::Http { @@ -592,21 +751,34 @@ impl RemoteTable { &self, request: RequestBuilder, version: Option, + freshness_request: Option, ) -> Result { let mut body = serde_json::json!({ "version": version }); self.apply_branch_body(&mut body); let request = request.json(&body); - let (request_id, response) = self.send(request, true).await?; + let (request_id, response) = if let Some(freshness_request) = freshness_request { + self.send_with_freshness(request, true, freshness_request) + .await? + } else { + self.send_unfenced(request, true).await? + }; let response = self.check_table_response(&request_id, response).await?; match response.text().await { - Ok(body) => serde_json::from_str(&body).map_err(|e| Error::Http { - source: format!("Failed to parse table description: {}", e).into(), - request_id, - status_code: None, - }), + Ok(body) => { + let description: TableDescription = + serde_json::from_str(&body).map_err(|e| Error::Http { + source: format!("Failed to parse table description: {}", e).into(), + request_id, + status_code: None, + })?; + if let Some(freshness_request) = freshness_request { + freshness_request.observe_version(&self.freshness, description.version); + } + Ok(description) + } Err(err) => { let status_code = err.status(); Err(Error::Http { @@ -619,14 +791,41 @@ impl RemoteTable { } async fn send(&self, req: RequestBuilder, with_retry: bool) -> Result<(String, Response)> { + let freshness_request = self.snapshot_freshness_headers(); + self.send_with_freshness(req, with_retry, freshness_request) + .await + } + + async fn send_with_freshness( + &self, + req: RequestBuilder, + with_retry: bool, + freshness_request: FreshnessHeaders, + ) -> Result<(String, Response)> { + let req = freshness_request.apply(req); let res = if with_retry { self.client.send_with_retry(req, None, true).await? } else { self.client.send(req).await? }; + if res.1.status().is_success() { + freshness_request.observe_headers(&self.freshness, res.1.headers()); + } Ok(res) } + async fn send_unfenced( + &self, + req: RequestBuilder, + with_retry: bool, + ) -> Result<(String, Response)> { + if with_retry { + self.client.send_with_retry(req, None, true).await + } else { + self.client.send(req).await + } + } + pub(super) async fn handle_table_not_found( table_name: &str, response: reqwest::Response, @@ -1003,24 +1202,32 @@ impl RemoteTable { } } - async fn current_version(&self) -> Option { - let read_guard = self.version.read().await; - *read_guard + async fn snapshot_read_state(&self) -> ReadSnapshot { + let version = self.version.read().await; + let (freshness_state, freshness) = + freshness_state_snapshot(&self.freshness, self.client.read_consistency_interval); + ReadSnapshot { + version: *version, + freshness_state, + freshness, + } } - /// Snapshot the freshness headers to attach to a single read request. + /// Snapshot the freshness headers to attach to a single table request. /// Computed at call time so that retries reuse the same snapshot. fn snapshot_freshness_headers(&self) -> FreshnessHeaders { - let state = *self.freshness.lock().unwrap(); - FreshnessHeaders { - min_version: state.min_version, - min_timestamp: compute_min_timestamp( - &state, - self.client.read_consistency_interval, - SystemTime::now(), - ), - min_read_version: state.min_read_version, - } + freshness_headers_snapshot(&self.freshness, self.client.read_consistency_interval) + } + + fn reset_freshness(&self, checkout_baseline: Option, pinned: bool) { + let mut state = self.freshness.lock().unwrap(); + let generation = state.generation.wrapping_add(1); + *state = FreshnessState { + generation, + pinned, + checkout_baseline, + ..FreshnessState::default() + }; } /// Send an LSM operator request with the transport retry layer **off**. @@ -1035,46 +1242,25 @@ impl RemoteTable { Ok((request_id, response)) } - /// Build a POST request and attach the read-freshness headers - /// (`x-lancedb-min-version`, `x-lancedb-min-timestamp`). - fn post_read(&self, uri: &str) -> RequestBuilder { - self.snapshot_freshness_headers() - .apply(self.client.post(uri)) - } - /// Record a version returned by a write so subsequent reads can request at /// least that version via `x-lancedb-min-version`. A returned `0` from a /// backward-compatible old server is ignored. - fn track_write_version(&self, version: u64) { + fn track_write_version(&self, freshness_request: FreshnessHeaders, version: u64) { if version == 0 { return; } - let mut state = self.freshness.lock().unwrap(); - state.min_version = Some(state.min_version.map_or(version, |v| v.max(version))); + freshness_request.update_if_current(&self.freshness, |state| { + state.min_version = Some(state.min_version.map_or(version, |v| v.max(version))); + }); } - /// Record a dataset version observed in a *read* response so subsequent - /// reads request at least this version via `x-lancedb-min-read-version`, + /// Record a committed dataset version observed in a table response so + /// subsequent requests ask for at least this version via + /// `x-lancedb-min-read-version`, /// giving monotonic reads across load-balanced query nodes. A returned `0` /// (or absent header from an old server) is ignored. fn track_read_version(&self, version: u64) { - if version == 0 { - return; - } - let mut state = self.freshness.lock().unwrap(); - state.min_read_version = Some(state.min_read_version.map_or(version, |v| v.max(version))); - } - - /// Parse the `x-lancedb-version` response header (the dataset version a read - /// reflects) and fold it into the read-version watermark. - fn track_read_version_from_headers(&self, headers: &reqwest::header::HeaderMap) { - if let Some(version) = headers - .get(&VERSION_HEADER) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.parse::().ok()) - { - self.track_read_version(version); - } + track_read_version(&self.freshness, version); } async fn execute_query( @@ -1082,7 +1268,9 @@ impl RemoteTable { query: &AnyQuery, options: &QueryExecutionOptions, ) -> Result>>> { - let mut request = self.post_read(&format!("/v1/table/{}/query/", self.identifier)); + let mut request = self + .client + .post(&format!("/v1/table/{}/query/", self.identifier)); if let Some(timeout) = options.timeout { // Also send to server, so it can abort the query if it takes too long. @@ -1092,15 +1280,17 @@ impl RemoteTable { } } - let query_bodies = self.prepare_query_bodies(query).await?; + let read_snapshot = self.snapshot_read_state().await; + let query_bodies = self.prepare_query_bodies(query, read_snapshot.version)?; let requests: Vec = query_bodies .into_iter() .map(|body| request.try_clone().unwrap().json(&body)) .collect(); let futures = requests.into_iter().map(|req| async move { - let (request_id, response) = self.send(req, true).await?; - self.track_read_version_from_headers(response.headers()); + let (request_id, response) = self + .send_with_freshness(req, true, read_snapshot.freshness) + .await?; self.read_arrow_response(&request_id, response).await }); let streams = futures::future::try_join_all(futures); @@ -1125,8 +1315,11 @@ impl RemoteTable { } } - async fn prepare_query_bodies(&self, query: &AnyQuery) -> Result> { - let version = self.current_version().await; + fn prepare_query_bodies( + &self, + query: &AnyQuery, + version: Option, + ) -> Result> { let mut base_body = serde_json::json!({ "version": version }); self.apply_branch_body(&mut base_body); @@ -1230,14 +1423,15 @@ async fn fetch_schema( client: &RestfulLanceDbClient, identifier: &str, table_name: &str, - version: Option, + read_snapshot: ReadSnapshot, branch: Option, - freshness_headers: FreshnessHeaders, + freshness: Arc>, ) -> Result { - let mut body = serde_json::json!({ "version": version }); + let mut body = serde_json::json!({ "version": read_snapshot.version }); if let Some(branch) = &branch { body["branch"] = serde_json::Value::String(branch.clone()); } + let freshness_headers = read_snapshot.freshness; let request = freshness_headers .apply(client.post(&format!("/v1/table/{}/describe/", identifier))) .json(&body); @@ -1257,6 +1451,7 @@ async fn fetch_schema( } let response = client.check_response(&request_id, response).await?; + freshness_headers.observe_headers(&freshness, response.headers()); let body = response.text().await.map_err(|e| { let status_code = e.status(); Error::Http { @@ -1271,6 +1466,12 @@ async fn fetch_schema( request_id, status_code: None, })?; + freshness_headers.observe_version(&freshness, description.version); + if !freshness_headers.is_current(&freshness) { + return Err(Error::Runtime { + message: SCHEMA_SELECTOR_CHANGED.to_string(), + }); + } let arrow_schema: arrow_schema::Schema = description.schema.try_into()?; Ok(Arc::new(arrow_schema)) @@ -1401,18 +1602,25 @@ impl RemoteTable { use crate::remote::retry::RetryCounter; let _guard = output.tracker.as_ref().map(|t| t.track_task()); + let freshness_request = self.snapshot_freshness_headers(); - let mut insert: Arc = Arc::new(RemoteWriteExec::new( - self.name.clone(), - self.identifier.clone(), - self.client.clone(), - output.plan, - WriteOp::Insert { - overwrite: output.overwrite, - }, - output.tracker.clone(), - self.branch.clone(), - )); + let mut insert: Arc = Arc::new( + RemoteWriteExec::new( + self.name.clone(), + self.identifier.clone(), + self.client.clone(), + output.plan, + WriteOp::Insert { + overwrite: output.overwrite, + }, + output.tracker.clone(), + self.branch.clone(), + ) + .with_freshness( + self.freshness.clone(), + self.client.read_consistency_interval, + ), + ); let mut retry_counter = RetryCounter::new(&self.client.retry_config, uuid::Uuid::new_v4().to_string()); @@ -1431,7 +1639,7 @@ impl RemoteTable { if output.overwrite { self.invalidate_schema_cache(); } - self.track_write_version(add_result.version); + self.track_write_version(freshness_request, add_result.version); return Ok(add_result); } @@ -1457,6 +1665,7 @@ impl RemoteTable { RetryCounter::new(&self.client.retry_config, uuid::Uuid::new_v4().to_string()); loop { + let freshness_request = self.snapshot_freshness_headers(); let upload_id = self.create_multipart_write().await?; let result = self @@ -1469,7 +1678,7 @@ impl RemoteTable { if output.overwrite { self.invalidate_schema_cache(); } - self.track_write_version(result.version); + self.track_write_version(freshness_request, result.version); return Ok(result); } Err(e) => { @@ -1525,18 +1734,24 @@ impl RemoteTable { )?, ) as Arc; - let insert = Arc::new(RemoteWriteExec::new_multipart( - self.name.clone(), - self.identifier.clone(), - self.client.clone(), - plan, - output.overwrite, - upload_id.to_string(), - output.tracker.clone(), - self.branch.clone(), - self.client.max_bytes_per_request(), - self.client.max_request_duration(), - )); + let insert = Arc::new( + RemoteWriteExec::new_multipart( + self.name.clone(), + self.identifier.clone(), + self.client.clone(), + plan, + output.overwrite, + upload_id.to_string(), + output.tracker.clone(), + self.branch.clone(), + self.client.max_bytes_per_request(), + self.client.max_request_duration(), + ) + .with_freshness( + self.freshness.clone(), + self.client.read_consistency_interval, + ), + ); let task_ctx = Arc::new(datafusion_execution::TaskContext::default()); let tracker = output.tracker.clone(); @@ -1599,6 +1814,38 @@ where } impl RemoteTable { + async fn index_stats_read_snapshot( + &self, + index_name: &str, + read_snapshot: ReadSnapshot, + ) -> Result> { + let encoded_name = urlencoding::encode(index_name); + let mut body = serde_json::json!({ "version": read_snapshot.version }); + self.apply_branch_body(&mut body); + let request = self + .client + .post(&format!( + "/v1/table/{}/index/{encoded_name}/stats/", + self.identifier + )) + .json(&body); + + let (request_id, response) = self + .send_with_freshness(request, true, read_snapshot.freshness) + .await?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + let response = self.check_table_response(&request_id, response).await?; + let body = response.text().await.err_to_http(request_id.clone())?; + let stats = serde_json::from_str(&body).map_err(|e| Error::Http { + source: format!("Failed to parse index statistics: {}", e).into(), + request_id, + status_code: None, + })?; + Ok(Some(stats)) + } + /// Parse the response from `/index/list/` into `IndexConfig` entries. /// /// When the server returns `index_type` inline, all enriched fields are @@ -1610,6 +1857,7 @@ impl RemoteTable { body: &str, request_id: &str, schema: &SchemaRef, + read_snapshot: ReadSnapshot, ) -> Result> { use crate::index::IndexType; @@ -1678,7 +1926,10 @@ impl RemoteTable { })) } else { // Legacy response: fetch index type via stats endpoint. - match self.index_stats(&entry.index_name).await { + match self + .index_stats_read_snapshot(&entry.index_name, read_snapshot) + .await + { Ok(Some(stats)) => Ok(Some(IndexConfig { name: entry.index_name, index_type: stats.index_type, @@ -1762,7 +2013,7 @@ impl BaseTable for RemoteTable { let request = self .client .post(&format!("/v1/table/{}/describe/", self.identifier)); - self.describe_with_request(request, Some(version)) + self.describe_with_request(request, Some(version), None) .await .map_err(|e| match e { // try to map the error to a more user-friendly error telling them @@ -1775,58 +2026,52 @@ impl BaseTable for RemoteTable { })?; let mut write_guard = self.version.write().await; + // Commit the selector and its freshness mode while holding the selector + // write lock, with no cancellation point between the two updates. + self.reset_freshness(None, true); *write_guard = Some(version); - drop(write_guard); - - // Explicit time-travel: drop any read-your-write / freshness - // constraints so the user sees exactly the requested version. - *self.freshness.lock().unwrap() = FreshnessState::default(); - - // Invalidate schema cache since we're switching versions self.invalidate_schema_cache(); + drop(write_guard); Ok(()) } async fn checkout_latest(&self) -> Result<()> { let mut write_guard = self.version.write().await; - *write_guard = None; - drop(write_guard); - // Drop any per-handle read/write tracking; subsequent reads use the // baseline timestamp captured now to guarantee freshness. - *self.freshness.lock().unwrap() = FreshnessState { - min_version: None, - checkout_baseline: Some(SystemTime::now()), - min_read_version: None, - }; - - // Invalidate schema cache since we're switching versions + self.reset_freshness(Some(SystemTime::now()), false); + *write_guard = None; self.invalidate_schema_cache(); + drop(write_guard); Ok(()) } async fn snapshot_at_current_version(&self) -> Result>> { // A checked-out handle already names its snapshot. Otherwise resolve // latest exactly once before creating the independent pinned handle. - let version = match self.current_version().await { + let read_snapshot = self.snapshot_read_state().await; + let version = match read_snapshot.version { Some(version) => version, - None => self.describe().await?.version, + None => self.describe_read_snapshot(read_snapshot).await?.version, }; let snapshot = self.with_branch(self.branch.clone()); *snapshot.version.write().await = Some(version); + snapshot.reset_freshness(None, true); Ok(Some(Arc::new(snapshot))) } async fn restore(&self) -> Result<()> { let mut request = self .client .post(&format!("/v1/table/{}/restore/", self.identifier)); - let version = self.current_version().await; - let mut body = serde_json::json!({ "version": version }); + let read_snapshot = self.snapshot_read_state().await; + let mut body = serde_json::json!({ "version": read_snapshot.version }); self.apply_branch_body(&mut body); request = request.json(&body); - let (request_id, response) = self.send(request, true).await?; + let (request_id, response) = self + .send_with_freshness(request, true, read_snapshot.freshness) + .await?; self.check_table_response(&request_id, response).await?; self.checkout_latest().await?; Ok(()) @@ -1834,7 +2079,8 @@ impl BaseTable for RemoteTable { async fn list_versions(&self) -> Result> { let request = self.apply_branch_query( - self.post_read(&format!("/v1/table/{}/version/list/", self.identifier)), + self.client + .post(&format!("/v1/table/{}/version/list/", self.identifier)), ); let (request_id, response) = self.send(request, true).await?; let response = self.check_table_response(&request_id, response).await?; @@ -1898,31 +2144,42 @@ impl BaseTable for RemoteTable { } async fn schema(&self) -> Result { - if let Some(schema) = self.schema_cache.try_get() { - return Ok(schema); - } + loop { + let read_snapshot = self.snapshot_read_state().await; + if let Some(schema) = self.schema_cache.try_get() { + return Ok(schema); + } - let version = self.current_version().await; - let client = self.client.clone(); - let identifier = self.identifier.clone(); - let table_name = self.name.clone(); - let branch = self.branch.clone(); - let freshness_headers = self.snapshot_freshness_headers(); + let client = self.client.clone(); + let identifier = self.identifier.clone(); + let table_name = self.name.clone(); + let branch = self.branch.clone(); + let freshness = self.freshness.clone(); - self.schema_cache - .get(move || async move { - fetch_schema( - &client, - &identifier, - &table_name, - version, - branch, - freshness_headers, - ) + match self + .schema_cache + .get(move || async move { + fetch_schema( + &client, + &identifier, + &table_name, + read_snapshot, + branch, + freshness, + ) + .await + }) .await - }) - .await - .map_err(unwrap_shared_error) + { + Ok(schema) => return Ok(schema), + Err(error) + if matches!( + &*error, + Error::Runtime { message } if message == SCHEMA_SELECTOR_CHANGED + ) => {} + Err(error) => return Err(unwrap_shared_error(error)), + } + } } async fn create_branch( @@ -1964,7 +2221,7 @@ impl BaseTable for RemoteTable { // Send without retry so the expected 409 (branch already exists) is // surfaced as a response we can map, rather than being retried. - let (request_id, response) = self.send(request, false).await?; + let (request_id, response) = self.send_unfenced(request, false).await?; match response.status() { StatusCode::CONFLICT => { return Err(Error::TableAlreadyExists { @@ -2025,8 +2282,10 @@ impl BaseTable for RemoteTable { async fn list_branches(&self) -> Result> { use lance::dataset::refs::BranchContents; - let request = self.post_read(&format!("/v1/table/{}/branches/list/", self.identifier)); - let (request_id, response) = self.send(request, true).await?; + let request = self + .client + .post(&format!("/v1/table/{}/branches/list/", self.identifier)); + let (request_id, response) = self.send_unfenced(request, true).await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; @@ -2080,7 +2339,7 @@ impl BaseTable for RemoteTable { .client .post(&format!("/v1/table/{}/branches/diff/", self.identifier)) .json(&serde_json::json!({ "from_branch": from_branch })); - let (request_id, response) = self.send(request, true).await?; + let (request_id, response) = self.send_unfenced(request, true).await?; if response.status() == StatusCode::NOT_FOUND { return Err(Error::TableNotFound { name: format!("{} (branch: {})", self.name, from_branch), @@ -2106,6 +2365,12 @@ impl BaseTable for RemoteTable { message: "Branch name cannot be empty.".into(), }); } + let read_snapshot = self.snapshot_read_state().await; + let target_freshness = if self.branch.is_none() && read_snapshot.version.is_none() { + Some(read_snapshot.freshness) + } else { + None + }; let request = self .client .post(&format!( @@ -2117,7 +2382,7 @@ impl BaseTable for RemoteTable { "dry_run": dry_run, })); // No retry. HTTP 409 is CherryPickStatus::Failed with a body, not a transport error. - let (request_id, response) = self.send(request, false).await?; + let (request_id, response) = self.send_unfenced(request, false).await?; let status = response.status(); if status == StatusCode::NOT_FOUND { return Err(Error::TableNotFound { @@ -2135,7 +2400,7 @@ impl BaseTable for RemoteTable { }); } let body = response.text().await.err_to_http(request_id.clone())?; - serde_json::from_str(&body).map_err(|err| Error::Http { + let result: CherryPickResult = serde_json::from_str(&body).map_err(|err| Error::Http { source: format!( "Failed to parse cherry_pick response: {}, body: {}", err, body @@ -2143,7 +2408,15 @@ impl BaseTable for RemoteTable { .into(), request_id, status_code: Some(status), - }) + })?; + if !dry_run + && status == StatusCode::OK + && result.status == crate::table::CherryPickStatus::CherryPicked + && let (Some(freshness), Some(version)) = (target_freshness, result.main_version_after) + { + freshness.observe_version(&self.freshness, version); + } + Ok(result) } fn current_branch(&self) -> Option { @@ -2151,23 +2424,28 @@ impl BaseTable for RemoteTable { } async fn count_rows(&self, filter: Option) -> Result { - let mut request = self.post_read(&format!("/v1/table/{}/count_rows/", self.identifier)); + let mut request = self + .client + .post(&format!("/v1/table/{}/count_rows/", self.identifier)); - let version = self.current_version().await; + let read_snapshot = self.snapshot_read_state().await; let mut body = if let Some(filter) = filter { let filter_sql = match filter { Filter::Sql(sql) => sql.clone(), Filter::Datafusion(expr) => expr_to_sql_string(&expr)?, }; - serde_json::json!({ "predicate": filter_sql, "version": version }) + serde_json::json!({ "predicate": filter_sql, "version": read_snapshot.version }) } else { - serde_json::json!({ "version": version }) + serde_json::json!({ "version": read_snapshot.version }) }; self.apply_branch_body(&mut body); request = request.json(&body); - let (request_id, response) = match self.send(request, true).await { + let (request_id, response) = match self + .send_with_freshness(request, true, read_snapshot.freshness) + .await + { Ok((id, resp)) => { // check_table_response now handles error-based invalidation let response = self.check_table_response(&id, resp).await?; @@ -2179,7 +2457,6 @@ impl BaseTable for RemoteTable { } }; - self.track_read_version_from_headers(response.headers()); let body = response.text().await.err_to_http(request_id.clone())?; serde_json::from_str(&body).map_err(|e| Error::Http { @@ -2312,9 +2589,12 @@ impl BaseTable for RemoteTable { } async fn explain_plan(&self, query: &AnyQuery, verbose: bool) -> Result { - let base_request = self.post_read(&format!("/v1/table/{}/explain_plan/", self.identifier)); + let base_request = self + .client + .post(&format!("/v1/table/{}/explain_plan/", self.identifier)); - let query_bodies = self.prepare_query_bodies(query).await?; + let read_snapshot = self.snapshot_read_state().await; + let query_bodies = self.prepare_query_bodies(query, read_snapshot.version)?; let requests: Vec = query_bodies .into_iter() .map(|query_body| { @@ -2328,7 +2608,9 @@ impl BaseTable for RemoteTable { .collect::>(); let futures = requests.into_iter().map(|req| async move { - let (request_id, response) = self.send(req, true).await?; + let (request_id, response) = self + .send_with_freshness(req, true, read_snapshot.freshness) + .await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; @@ -2359,7 +2641,9 @@ impl BaseTable for RemoteTable { query: &AnyQuery, options: QueryExecutionOptions, ) -> Result { - let mut request = self.post_read(&format!("/v1/table/{}/analyze_plan/", self.identifier)); + let mut request = self + .client + .post(&format!("/v1/table/{}/analyze_plan/", self.identifier)); if options.analyze_plan_distributed_metrics != AnalyzePlanDistributedMetrics::Aggregate { request = request.query(&[( @@ -2368,14 +2652,17 @@ impl BaseTable for RemoteTable { )]); } - let query_bodies = self.prepare_query_bodies(query).await?; + let read_snapshot = self.snapshot_read_state().await; + let query_bodies = self.prepare_query_bodies(query, read_snapshot.version)?; let requests: Vec = query_bodies .into_iter() .map(|body| request.try_clone().unwrap().json(&body)) .collect(); let futures = requests.into_iter().map(|req| async move { - let (request_id, response) = self.send(req, true).await?; + let (request_id, response) = self + .send_with_freshness(req, true, read_snapshot.freshness) + .await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; @@ -2419,7 +2706,10 @@ impl BaseTable for RemoteTable { self.apply_branch_body(&mut body); let request = request.json(&body); - let (request_id, response) = self.send(request, true).await?; + let freshness_request = self.snapshot_freshness_headers(); + let (request_id, response) = self + .send_with_freshness(request, true, freshness_request) + .await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; @@ -2438,7 +2728,7 @@ impl BaseTable for RemoteTable { status_code: None, })?; - self.track_write_version(update_response.version); + self.track_write_version(freshness_request, update_response.version); Ok(update_response) } @@ -2454,7 +2744,10 @@ impl BaseTable for RemoteTable { .client .post(&format!("/v1/table/{}/delete/", self.identifier)) .json(&body); - let (request_id, response) = self.send(request, true).await?; + let freshness_request = self.snapshot_freshness_headers(); + let (request_id, response) = self + .send_with_freshness(request, true, freshness_request) + .await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; if body.trim().is_empty() { @@ -2470,7 +2763,7 @@ impl BaseTable for RemoteTable { request_id, status_code: None, })?; - self.track_write_version(delete_response.version); + self.track_write_version(freshness_request, delete_response.version); Ok(delete_response) } @@ -2480,7 +2773,13 @@ impl BaseTable for RemoteTable { async fn create_index_async(&self, index: IndexBuilder) -> Result { Ok(match self.submit_create_index(index).await? { - Some(job_id) => Job::new(Box::new(RemoteJob::new(self.client.clone(), job_id))), + Some(job_id) => Job::new(Box::new(FreshnessJob { + inner: RemoteJob::new(self.client.clone(), job_id), + freshness: self.freshness.clone(), + version: self.version.clone(), + track_refresh_result: false, + freshness_request: self.snapshot_freshness_headers(), + })), None => Job::new_done(), }) } @@ -2520,16 +2819,23 @@ impl BaseTable for RemoteTable { let rescannable = source.rescannable(); let input: Arc = Arc::new(crate::table::datafusion::scannable_exec::ScannableExec::new(source, None)); + let freshness_request = self.snapshot_freshness_headers(); - let mut merge: Arc = Arc::new(RemoteWriteExec::new( - self.name.clone(), - self.identifier.clone(), - self.client.clone(), - input, - WriteOp::MergeInsert { query, timeout }, - None, - self.branch.clone(), - )); + let mut merge: Arc = Arc::new( + RemoteWriteExec::new( + self.name.clone(), + self.identifier.clone(), + self.client.clone(), + input, + WriteOp::MergeInsert { query, timeout }, + None, + self.branch.clone(), + ) + .with_freshness( + self.freshness.clone(), + self.client.read_consistency_interval, + ), + ); let mut retry_counter = crate::remote::retry::RetryCounter::new( &self.client.retry_config, @@ -2547,7 +2853,7 @@ impl BaseTable for RemoteTable { .and_then(|m| m.merge_result()) .unwrap_or_default(); - self.track_write_version(merge_result.version); + self.track_write_version(freshness_request, merge_result.version); return Ok(merge_result); } Err(err) if rescannable && self.is_retryable_write_error(&err) => { @@ -2586,7 +2892,8 @@ impl BaseTable for RemoteTable { async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result> { // Read-semantics POST, like `get_lsm_write_spec`. let request = self - .post_read(&format!("/v1/table/{}/get_lsm_stats/", self.identifier)) + .client + .post(&format!("/v1/table/{}/get_lsm_stats/", self.identifier)) .json(&serde_json::json!({ "include_generation_rows": include_generation_rows, })); @@ -2660,7 +2967,7 @@ impl BaseTable for RemoteTable { // re-encodes it into the same sophon-owned shape the set endpoint // accepts — no lance/lancedb types cross the wire. `lsm_write_spec` is // null when the LSM write path is not enabled for the table. - let request = self.post_read(&format!( + let request = self.client.post(&format!( "/v1/table/{}/get_lsm_write_spec/", self.identifier )); @@ -2726,18 +3033,17 @@ impl BaseTable for RemoteTable { let request = self .client .post(&format!("/v1/table/{}/tags/version/", self.identifier)); - let version = self.resolve_tag_version_with_request(tag, request).await?; + let version = self + .resolve_tag_version_with_request(tag, request, false) + .await?; let mut write_guard = self.version.write().await; + // Commit the selector and its freshness mode while holding the selector + // write lock, with no cancellation point between the two updates. + self.reset_freshness(None, true); *write_guard = Some(version); - drop(write_guard); - - // Explicit time-travel: drop any read-your-write / freshness - // constraints so the user sees exactly the tagged version. - *self.freshness.lock().unwrap() = FreshnessState::default(); - - // Invalidate schema cache since we're switching versions self.invalidate_schema_cache(); + drop(write_guard); Ok(()) } @@ -2774,7 +3080,10 @@ impl BaseTable for RemoteTable { .client .post(&format!("/v1/table/{}/add_columns/", self.identifier)) .json(&body); - let (request_id, response) = self.send(request, true).await?; + let freshness_request = self.snapshot_freshness_headers(); + let (request_id, response) = self + .send_with_freshness(request, true, freshness_request) + .await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; @@ -2791,7 +3100,7 @@ impl BaseTable for RemoteTable { })?; self.invalidate_schema_cache(); - self.track_write_version(result.version); + self.track_write_version(freshness_request, result.version); Ok(result) } @@ -2827,7 +3136,10 @@ impl BaseTable for RemoteTable { .client .post(&format!("/v1/table/{}/add_columns/", self.identifier)) .json(&body); - let (request_id, response) = self.send(request, true).await?; + let freshness_request = self.snapshot_freshness_headers(); + let (request_id, response) = self + .send_with_freshness(request, true, freshness_request) + .await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; @@ -2843,7 +3155,7 @@ impl BaseTable for RemoteTable { })?; self.invalidate_schema_cache(); - self.track_write_version(result.version); + self.track_write_version(freshness_request, result.version); Ok(result) } @@ -2886,7 +3198,10 @@ impl BaseTable for RemoteTable { .client .post(&format!("/v1/table/{}/add_columns/", self.identifier)) .json(&body); - let (request_id, response) = self.send(request, true).await?; + let freshness_request = self.snapshot_freshness_headers(); + let (request_id, response) = self + .send_with_freshness(request, true, freshness_request) + .await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; @@ -2901,7 +3216,7 @@ impl BaseTable for RemoteTable { })?; self.invalidate_schema_cache(); - self.track_write_version(result.version); + self.track_write_version(freshness_request, result.version); Ok(result) } @@ -2923,7 +3238,8 @@ impl BaseTable for RemoteTable { let mut body = serde_json::json!({ "column": column }); self.apply_branch_body(&mut body); let request = self - .post_read(&format!("/v1/table/{}/backfill_column", self.identifier)) + .client + .post(&format!("/v1/table/{}/backfill_column", self.identifier)) .json(&body); let (request_id, response) = self.send(request, true).await?; let response = self.check_table_response(&request_id, response).await?; @@ -2943,6 +3259,8 @@ impl BaseTable for RemoteTable { inner: RemoteJob::new(self.client.clone(), response.job_id), freshness: self.freshness.clone(), version: self.version.clone(), + track_refresh_result: true, + freshness_request: self.snapshot_freshness_headers(), }))) } @@ -2974,7 +3292,10 @@ impl BaseTable for RemoteTable { .client .post(&format!("/v1/table/{}/alter_columns/", self.identifier)) .json(&body); - let (request_id, response) = self.send(request, true).await?; + let freshness_request = self.snapshot_freshness_headers(); + let (request_id, response) = self + .send_with_freshness(request, true, freshness_request) + .await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; @@ -2990,7 +3311,7 @@ impl BaseTable for RemoteTable { })?; self.invalidate_schema_cache(); - self.track_write_version(result.version); + self.track_write_version(freshness_request, result.version); Ok(result) } @@ -3009,7 +3330,10 @@ impl BaseTable for RemoteTable { self.identifier )) .json(&body); - let (request_id, response) = self.send(request, true).await?; + let freshness_request = self.snapshot_freshness_headers(); + let (request_id, response) = self + .send_with_freshness(request, true, freshness_request) + .await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; @@ -3021,7 +3345,7 @@ impl BaseTable for RemoteTable { })?; self.invalidate_schema_cache(); - self.track_write_version(result.version); + self.track_write_version(freshness_request, result.version); Ok(result) } @@ -3033,7 +3357,10 @@ impl BaseTable for RemoteTable { .client .post(&format!("/v1/table/{}/drop_columns/", self.identifier)) .json(&body); - let (request_id, response) = self.send(request, true).await?; + let freshness_request = self.snapshot_freshness_headers(); + let (request_id, response) = self + .send_with_freshness(request, true, freshness_request) + .await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; @@ -3049,55 +3376,34 @@ impl BaseTable for RemoteTable { })?; self.invalidate_schema_cache(); - self.track_write_version(result.version); + self.track_write_version(freshness_request, result.version); Ok(result) } async fn list_indices(&self) -> Result> { - let mut request = self.post_read(&format!("/v1/table/{}/index/list/", self.identifier)); - let version = self.current_version().await; - let mut body = serde_json::json!({ "version": version }); + let mut request = self + .client + .post(&format!("/v1/table/{}/index/list/", self.identifier)); + let read_snapshot = self.snapshot_read_state().await; + let mut body = serde_json::json!({ "version": read_snapshot.version }); self.apply_branch_body(&mut body); request = request.json(&body); - let (request_id, response) = self.send(request, true).await?; + let (request_id, response) = self + .send_with_freshness(request, true, read_snapshot.freshness) + .await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; - let schema = self.schema().await?; + let schema = self.schema_read_snapshot(read_snapshot).await?; - self.parse_index_list_response(&body, &request_id, &schema) + self.parse_index_list_response(&body, &request_id, &schema, read_snapshot) .await } async fn index_stats(&self, index_name: &str) -> Result> { - let encoded_name = urlencoding::encode(index_name); - let mut request = self.post_read(&format!( - "/v1/table/{}/index/{encoded_name}/stats/", - self.identifier - )); - let version = self.current_version().await; - let mut body = serde_json::json!({ "version": version }); - self.apply_branch_body(&mut body); - request = request.json(&body); - - let (request_id, response) = self.send(request, true).await?; - - if response.status() == StatusCode::NOT_FOUND { - return Ok(None); - } - - let response = self.check_table_response(&request_id, response).await?; - - let body = response.text().await.err_to_http(request_id.clone())?; - - let stats = serde_json::from_str(&body).map_err(|e| Error::Http { - source: format!("Failed to parse index statistics: {}", e).into(), - request_id, - status_code: None, - })?; - - Ok(Some(stats)) + self.index_stats_read_snapshot(index_name, self.snapshot_read_state().await) + .await } async fn drop_index(&self, index_name: &str) -> Result<()> { @@ -3187,7 +3493,9 @@ impl BaseTable for RemoteTable { } async fn stats(&self) -> Result { - let mut request = self.post_read(&format!("/v1/table/{}/stats/", self.identifier)); + let mut request = self + .client + .post(&format!("/v1/table/{}/stats/", self.identifier)); if let Some(branch) = &self.branch { request = request.json(&serde_json::json!({ "branch": branch })); } @@ -3209,15 +3517,21 @@ impl BaseTable for RemoteTable { write_params: lance::dataset::WriteParams, ) -> Result> { let overwrite = matches!(write_params.mode, lance::dataset::WriteMode::Overwrite); - Ok(Arc::new(insert::RemoteWriteExec::new( - self.name.clone(), - self.identifier.clone(), - self.client.clone(), - input, - WriteOp::Insert { overwrite }, - None, - self.branch.clone(), - ))) + Ok(Arc::new( + insert::RemoteWriteExec::new( + self.name.clone(), + self.identifier.clone(), + self.client.clone(), + input, + WriteOp::Insert { overwrite }, + None, + self.branch.clone(), + ) + .with_freshness( + self.freshness.clone(), + self.client.read_consistency_interval, + ), + )) } } @@ -7113,12 +7427,12 @@ mod tests { } /// The gate's reproducer: after a successful wait, a same-handle read - /// must carry a freshness baseline so a stale server cache cannot serve - /// the pre-backfill snapshot. + /// must carry the exact published version so a stale server cache cannot + /// serve the pre-backfill snapshot. #[tokio::test] async fn test_backfill_wait_establishes_read_freshness() { - let saw_min_timestamp = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let saw = saw_min_timestamp.clone(); + let saw_published_version = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let saw = saw_published_version.clone(); let table = Table::new_with_handler("my_table", move |request| match request.url().path() { "/v1/table/my_table/backfill_column" => http::Response::builder() @@ -7131,7 +7445,11 @@ mod tests { .unwrap(), "/v1/table/my_table/count_rows/" => { saw.store( - request.headers().contains_key("x-lancedb-min-timestamp"), + request + .headers() + .get("x-lancedb-min-read-version") + .and_then(|value| value.to_str().ok()) + == Some("8"), std::sync::atomic::Ordering::SeqCst, ); http::Response::builder() @@ -7148,8 +7466,8 @@ mod tests { assert_eq!(result.published_version, Some(8)); table.count_rows(None).await.unwrap(); assert!( - saw_min_timestamp.load(std::sync::atomic::Ordering::SeqCst), - "read after wait carried no freshness baseline" + saw_published_version.load(std::sync::atomic::Ordering::SeqCst), + "read after wait did not carry the published version" ); } @@ -7324,12 +7642,11 @@ mod tests { } /// checkout_latest keeps the handle on latest, so a completed backfill - /// must still establish its post-fill baseline -- strictly later than the - /// checkout's own, or a pre-fill cache could still serve. + /// must retain the checkout timestamp and add its exact published version. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_checkout_latest_during_submission_keeps_the_fence() { - let seen_min_timestamp = Arc::new(std::sync::Mutex::new(None::)); - let saw = seen_min_timestamp.clone(); + let seen_headers = Arc::new(std::sync::Mutex::new(None::)); + let saw = seen_headers.clone(); let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); let release_rx = Arc::new(std::sync::Mutex::new(release_rx)); let (arrived_tx, arrived_rx) = std::sync::mpsc::channel::<()>(); @@ -7353,10 +7670,7 @@ mod tests { .body(refresh_done("j-11")) .unwrap(), "/v1/table/my_table/count_rows/" => { - *saw.lock().unwrap() = request - .headers() - .get("x-lancedb-min-timestamp") - .map(|v| v.to_str().unwrap().to_string()); + *saw.lock().unwrap() = Some(request.headers().clone()); http::Response::builder() .status(200) .body("1".to_string()) @@ -7377,25 +7691,18 @@ mod tests { .await .unwrap(); table.checkout_latest().await.unwrap(); - let after_checkout = SystemTime::now(); - // Real separation between the checkout baseline and completion. - tokio::time::sleep(std::time::Duration::from_millis(50)).await; release_tx.send(()).unwrap(); let job = submit.await.unwrap().unwrap(); job.wait().await.unwrap(); table.count_rows(None).await.unwrap(); - let header = seen_min_timestamp - .lock() - .unwrap() - .clone() - .expect("no baseline"); - let sent: SystemTime = chrono::DateTime::parse_from_rfc3339(&header) - .unwrap() - .into(); - assert!( - sent > after_checkout, - "baseline {header} did not advance past the checkout" + let headers = seen_headers.lock().unwrap().clone().expect("no request"); + assert!(headers.contains_key("x-lancedb-min-timestamp")); + assert_eq!( + headers + .get("x-lancedb-min-read-version") + .and_then(|value| value.to_str().ok()), + Some("8") ); } @@ -8853,6 +9160,50 @@ mod tests { assert_ne!(Arc::as_ptr(&schema3), Arc::as_ptr(&schema1)); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_schema_fetch_does_not_cross_checkout_generation() { + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let release_rx = Arc::new(std::sync::Mutex::new(release_rx)); + let (arrived_tx, arrived_rx) = std::sync::mpsc::channel::<()>(); + let arrived_tx = Arc::new(std::sync::Mutex::new(arrived_tx)); + let table = Table::new_with_handler("my_table", move |request| { + let body = request_body_json(&request); + let pinned = body["version"].as_u64() == Some(5); + if !pinned { + arrived_tx.lock().unwrap().send(()).unwrap(); + release_rx + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(10)) + .unwrap(); + } + let field = if pinned { "pinned" } else { "latest" }; + http::Response::builder() + .status(200) + .body(format!( + r#"{{"version":5,"schema":{{"fields":[{{"name":"{field}","type":{{"type":"int32"}},"nullable":false}}]}}}}"# + )) + .unwrap() + }); + + let schema_fetch = tokio::spawn({ + let table = table.clone(); + async move { table.schema().await } + }); + tokio::task::spawn_blocking(move || { + arrived_rx.recv_timeout(Duration::from_secs(10)).unwrap() + }) + .await + .unwrap(); + table.checkout(5).await.unwrap(); + release_tx.send(()).unwrap(); + + let schema = schema_fetch.await.unwrap().unwrap(); + assert!(schema.field_with_name("pinned").is_ok()); + let cached = table.schema().await.unwrap(); + assert!(cached.field_with_name("pinned").is_ok()); + } + /// Test that schema cache is invalidated after checkout_latest #[tokio::test] async fn test_schema_cache_invalidation_on_checkout_latest() { @@ -10087,6 +10438,7 @@ mod tests { min_version: None, checkout_baseline: Some(baseline), min_read_version: None, + ..FreshnessState::default() }; assert_eq!(compute_min_timestamp(&state, None, now), Some(baseline)); @@ -10112,6 +10464,7 @@ mod tests { min_version: None, checkout_baseline: Some(baseline), min_read_version: None, + ..FreshnessState::default() }; assert_eq!( compute_min_timestamp(&state, Some(Duration::from_secs(10)), now), @@ -10124,6 +10477,7 @@ mod tests { min_version: None, checkout_baseline: Some(recent_baseline), min_read_version: None, + ..FreshnessState::default() }; assert_eq!( compute_min_timestamp(&state, Some(Duration::from_secs(60)), now), @@ -10200,6 +10554,110 @@ mod tests { assert!(!headers.contains_key("x-lancedb-min-version")); } + #[tokio::test] + async fn test_checkout_disables_read_consistency_interval() { + let (handler, captured) = capturing_handler(|path| match path { + "/v1/table/my_table/describe/" => r#"{"version":5,"schema":{"fields":[]}}"#.to_string(), + "/v1/table/my_table/count_rows/" => "42".to_string(), + _ => panic!("unexpected path: {}", path), + }); + let table = + Table::new_with_handler_and_interval("my_table", handler, Some(Duration::from_secs(0))); + + table.checkout(5).await.unwrap(); + table.count_rows(None).await.unwrap(); + + let headers = captured.lock().unwrap().clone().unwrap(); + assert!(!headers.contains_key("x-lancedb-min-timestamp")); + assert!(!headers.contains_key("x-lancedb-min-version")); + assert!(!headers.contains_key("x-lancedb-min-read-version")); + } + + #[tokio::test] + async fn test_read_snapshot_keeps_selector_and_freshness_generation_bound() { + let table = RemoteTable::new_mock_with_consistency_interval( + "my_table".to_string(), + |_| { + http::Response::builder() + .status(200) + .body(r#"{"version":5,"schema":{"fields":[]}}"#.to_string()) + .unwrap() + }, + Some(Duration::ZERO), + ); + + let latest = table.snapshot_read_state().await; + table.checkout(5).await.unwrap(); + + let latest_request = latest + .freshness + .apply( + table + .client + .post("/v1/table/my_table/count_rows/") + .json(&serde_json::json!({ "version": latest.version })), + ) + .build() + .unwrap(); + assert!(request_body_json(&latest_request)["version"].is_null()); + assert!(latest_request.headers().contains_key(MIN_TIMESTAMP_HEADER)); + + let pinned = table.snapshot_read_state().await; + let pinned_request = pinned + .freshness + .apply( + table + .client + .post("/v1/table/my_table/count_rows/") + .json(&serde_json::json!({ "version": pinned.version })), + ) + .build() + .unwrap(); + assert_eq!(request_body_json(&pinned_request)["version"], 5); + assert!(!pinned_request.headers().contains_key(MIN_TIMESTAMP_HEADER)); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_cancelled_checkout_keeps_latest_freshness_enabled() { + let (described_tx, described_rx) = std::sync::mpsc::channel::<()>(); + let table = Arc::new(RemoteTable::new_mock_with_consistency_interval( + "my_table".to_string(), + move |_| { + described_tx.send(()).unwrap(); + http::Response::builder() + .status(200) + .body(r#"{"version":5,"schema":{"fields":[]}}"#.to_string()) + .unwrap() + }, + Some(Duration::from_secs(0)), + )); + + let version_guard = table.version.write().await; + let checkout = tokio::spawn({ + let table = table.clone(); + async move { table.checkout(5).await } + }); + tokio::task::spawn_blocking(move || { + described_rx.recv_timeout(Duration::from_secs(10)).unwrap() + }) + .await + .unwrap(); + for _ in 0..100 { + if table.freshness.lock().unwrap().pinned { + break; + } + tokio::task::yield_now().await; + } + assert!(!checkout.is_finished()); + + checkout.abort(); + assert!(checkout.await.unwrap_err().is_cancelled()); + drop(version_guard); + + assert_eq!(*table.version.read().await, None); + assert!(table.snapshot_freshness_headers().min_timestamp.is_some()); + } + #[tokio::test] async fn test_freshness_positive_interval_sends_now_minus_interval() { let (handler, captured) = capturing_handler(|_| "42".to_string()); @@ -10268,6 +10726,62 @@ mod tests { ); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_inflight_write_result_cannot_cross_checkout_generation() { + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let release_rx = Arc::new(std::sync::Mutex::new(release_rx)); + let (arrived_tx, arrived_rx) = std::sync::mpsc::channel::<()>(); + let arrived_tx = Arc::new(std::sync::Mutex::new(arrived_tx)); + let count_headers = Arc::new(std::sync::Mutex::new(None)); + let captured = count_headers.clone(); + let table = + Table::new_with_handler("my_table", move |request| match request.url().path() { + "/v1/table/my_table/update/" => { + arrived_tx.lock().unwrap().send(()).unwrap(); + release_rx + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(10)) + .unwrap(); + http::Response::builder() + .status(200) + .body(r#"{"rows_updated":1,"version":100}"#.to_string()) + .unwrap() + } + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(r#"{"version":5,"schema":{"fields":[]}}"#.to_string()) + .unwrap(), + "/v1/table/my_table/count_rows/" => { + *captured.lock().unwrap() = Some(request.headers().clone()); + http::Response::builder() + .status(200) + .body("1".to_string()) + .unwrap() + } + path => panic!("unexpected path: {path}"), + }); + + let update = tokio::spawn({ + let table = table.clone(); + async move { table.update().column("a", "a + 1").execute().await } + }); + tokio::task::spawn_blocking(move || { + arrived_rx.recv_timeout(Duration::from_secs(10)).unwrap() + }) + .await + .unwrap(); + table.checkout(5).await.unwrap(); + release_tx.send(()).unwrap(); + update.await.unwrap().unwrap(); + table.count_rows(None).await.unwrap(); + + let headers = count_headers.lock().unwrap(); + let headers = headers.as_ref().unwrap(); + assert!(!headers.contains_key("x-lancedb-min-version")); + assert!(!headers.contains_key("x-lancedb-min-read-version")); + } + /// A handler that records every request's headers and answers each read with /// an `x-lancedb-version` response header taken from `versions` (by call /// index, saturating at the last entry). An empty string means "no header". @@ -10314,6 +10828,164 @@ mod tests { ); } + #[tokio::test] + async fn test_schema_response_advances_read_watermark() { + let requests = Arc::new(std::sync::Mutex::new(Vec::new())); + let captured = requests.clone(); + let table = Table::new_with_handler("my_table", move |request| { + captured + .lock() + .unwrap() + .push((request.url().path().to_string(), request.headers().clone())); + match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .header("x-lancedb-version", "100") + .body(r#"{"version":100,"schema":{"fields":[]}}"#.to_string()) + .unwrap(), + "/v1/table/my_table/count_rows/" => http::Response::builder() + .status(200) + .body("42".to_string()) + .unwrap(), + path => panic!("unexpected path: {path}"), + } + }); + + table.schema().await.unwrap(); + assert_eq!(table.count_rows(None).await.unwrap(), 42); + + let requests = requests.lock().unwrap(); + let count_headers = &requests[1].1; + assert_eq!( + count_headers + .get("x-lancedb-min-read-version") + .and_then(|value| value.to_str().ok()), + Some("100") + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_inflight_schema_response_cannot_cross_checkout_generation() { + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let release_rx = Arc::new(std::sync::Mutex::new(release_rx)); + let (arrived_tx, arrived_rx) = std::sync::mpsc::channel::<()>(); + let arrived_tx = Arc::new(std::sync::Mutex::new(arrived_tx)); + let count_headers = Arc::new(std::sync::Mutex::new(None)); + let captured = count_headers.clone(); + let table = + Table::new_with_handler("my_table", move |request| match request.url().path() { + "/v1/table/my_table/describe/" => { + let body = request_body_json(&request); + if body["version"].is_null() { + arrived_tx.lock().unwrap().send(()).unwrap(); + release_rx + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(10)) + .unwrap(); + http::Response::builder() + .status(200) + .header("x-lancedb-version", "100") + .body(r#"{"version":100,"schema":{"fields":[]}}"#.to_string()) + .unwrap() + } else { + http::Response::builder() + .status(200) + .body(r#"{"version":5,"schema":{"fields":[]}}"#.to_string()) + .unwrap() + } + } + "/v1/table/my_table/count_rows/" => { + *captured.lock().unwrap() = Some(request.headers().clone()); + http::Response::builder() + .status(200) + .body("1".to_string()) + .unwrap() + } + path => panic!("unexpected path: {path}"), + }); + + let schema = tokio::spawn({ + let table = table.clone(); + async move { table.schema().await } + }); + tokio::task::spawn_blocking(move || { + arrived_rx.recv_timeout(Duration::from_secs(10)).unwrap() + }) + .await + .unwrap(); + table.checkout(5).await.unwrap(); + release_tx.send(()).unwrap(); + schema.await.unwrap().unwrap(); + table.count_rows(None).await.unwrap(); + + assert!( + !count_headers + .lock() + .unwrap() + .as_ref() + .unwrap() + .contains_key("x-lancedb-min-read-version") + ); + } + + #[tokio::test] + async fn test_streaming_write_uses_and_advances_read_watermark() { + let data = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), + vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], + ) + .unwrap(); + let describe_body = serde_json::to_string(&json!({ + "version": 7, + "schema": JsonSchema::try_from(data.schema().as_ref()).unwrap(), + })) + .unwrap(); + let requests = Arc::new(std::sync::Mutex::new(Vec::new())); + let captured = requests.clone(); + let table = Table::new_with_handler("my_table", move |request| { + captured + .lock() + .unwrap() + .push((request.url().path().to_string(), request.headers().clone())); + match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(describe_body.clone()) + .unwrap(), + "/v1/table/my_table/insert/" => http::Response::builder() + .status(200) + .header("x-lancedb-version", "8") + .body(r#"{"version":8}"#.to_string()) + .unwrap(), + "/v1/table/my_table/count_rows/" => http::Response::builder() + .status(200) + .body("3".to_string()) + .unwrap(), + path => panic!("unexpected path: {path}"), + } + }); + + assert_eq!(table.add(data).execute().await.unwrap().version, 8); + assert_eq!(table.count_rows(None).await.unwrap(), 3); + + let requests = requests.lock().unwrap(); + let insert_headers = &requests[1].1; + assert_eq!( + insert_headers + .get("x-lancedb-min-read-version") + .and_then(|value| value.to_str().ok()), + Some("7") + ); + let count_headers = &requests[2].1; + assert_eq!( + count_headers + .get("x-lancedb-min-read-version") + .and_then(|value| value.to_str().ok()), + Some("8") + ); + } + #[tokio::test] async fn test_read_version_watermark_keeps_max() { // Server reports 100 then a stale 50; the watermark must not regress. @@ -10729,6 +11401,86 @@ mod tests { ); } + #[tokio::test] + async fn test_main_only_metadata_is_unfenced_from_branch_timeline() { + let requests = Arc::new(std::sync::Mutex::new(HashMap::new())); + let captured = requests.clone(); + let saw_delete_response_floor = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let saw_high_floor = saw_delete_response_floor.clone(); + let table = RemoteTable::new_mock( + "my_table".to_string(), + move |request| { + let path = request.url().path().to_string(); + captured + .lock() + .unwrap() + .insert(path.clone(), request.headers().clone()); + match path.as_str() { + "/v1/table/my_table/count_rows/" => { + saw_high_floor.store( + request + .headers() + .get("x-lancedb-min-read-version") + .and_then(|value| value.to_str().ok()) + == Some("100"), + std::sync::atomic::Ordering::SeqCst, + ); + http::Response::builder() + .status(200) + .header("x-lancedb-version", "2") + .body("1".to_string()) + .unwrap() + } + "/v1/table/my_table/tags/list/" => http::Response::builder() + .status(200) + .body("{}".to_string()) + .unwrap(), + "/v1/table/my_table/tags/version/" => http::Response::builder() + .status(200) + .body(r#"{"version":1}"#.to_string()) + .unwrap(), + "/v1/table/my_table/tags/delete/" => http::Response::builder() + .status(200) + .header("x-lancedb-version", "100") + .body("{}".to_string()) + .unwrap(), + "/v1/table/my_table/branches/list/" => http::Response::builder() + .status(200) + .body(r#"{"branches":{}}"#.to_string()) + .unwrap(), + path => panic!("unexpected path: {path}"), + } + }, + None, + ); + let branch = table.with_branch(Some("exp".to_string())); + + branch.count_rows(None).await.unwrap(); + let mut tags = branch.tags().await.unwrap(); + tags.list().await.unwrap(); + tags.get_version("v1").await.unwrap(); + tags.delete("v1").await.unwrap(); + branch.list_branches().await.unwrap(); + branch.count_rows(None).await.unwrap(); + + let requests = requests.lock().unwrap(); + for path in [ + "/v1/table/my_table/tags/list/", + "/v1/table/my_table/tags/version/", + "/v1/table/my_table/tags/delete/", + "/v1/table/my_table/branches/list/", + ] { + assert!( + !requests[path].contains_key("x-lancedb-min-read-version"), + "{path} inherited the branch timeline" + ); + } + assert!( + !saw_delete_response_floor.load(std::sync::atomic::Ordering::SeqCst), + "tag deletion contaminated the branch timeline" + ); + } + #[tokio::test] async fn test_delete_branch() { let table = Table::new_with_handler("my_table", |request| { @@ -10820,6 +11572,50 @@ mod tests { assert!(result.main_version_after.is_none()); } + #[tokio::test] + async fn test_successful_cherry_pick_advances_main_read_watermark() { + let count_headers = Arc::new(std::sync::Mutex::new(None)); + let captured = count_headers.clone(); + let table = + Table::new_with_handler("my_table", move |request| match request.url().path() { + "/v1/table/my_table/branches/cherry_pick/" => { + let response = serde_json::json!({ + "status": "cherryPicked", + "diff": serde_json::from_str::(sample_branch_diff_json()) + .unwrap(), + "preview": { "promotedColumns": ["tag"] }, + "mainVersionAfter": 2 + }); + http::Response::builder() + .status(200) + .body(response.to_string()) + .unwrap() + } + "/v1/table/my_table/count_rows/" => { + *captured.lock().unwrap() = Some(request.headers().clone()); + http::Response::builder() + .status(200) + .body("1".to_string()) + .unwrap() + } + path => panic!("unexpected path: {path}"), + }); + + let result = table.cherry_pick("exp", false).await.unwrap(); + assert_eq!(result.status, crate::table::CherryPickStatus::CherryPicked); + table.count_rows(None).await.unwrap(); + assert_eq!( + count_headers + .lock() + .unwrap() + .as_ref() + .unwrap() + .get("x-lancedb-min-read-version") + .and_then(|value| value.to_str().ok()), + Some("2") + ); + } + #[tokio::test] async fn test_cherry_pick_failed_returns_ok_with_body() { let table = Table::new_with_handler("my_table", |request| { diff --git a/rust/lancedb/src/remote/table/blobs.rs b/rust/lancedb/src/remote/table/blobs.rs index 387d3c6dc..597b41782 100644 --- a/rust/lancedb/src/remote/table/blobs.rs +++ b/rust/lancedb/src/remote/table/blobs.rs @@ -6,6 +6,7 @@ use std::ops::Range; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; use arrow_array::{Array, LargeBinaryArray}; use arrow_schema::DataType; @@ -20,7 +21,7 @@ use crate::error::Result; use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient}; use crate::table::BaseTable; -use super::{FreshnessHeaders, RemoteTable}; +use super::{FreshnessHeaders, FreshnessState, RemoteTable, freshness_headers_snapshot}; #[derive(Debug, Clone, Copy)] enum RangeRequestMode { @@ -43,7 +44,10 @@ struct TableBlobRangeRequester { path: String, version: Option, branch: Option, - freshness: FreshnessHeaders, + freshness: Arc>, + parent_freshness: Arc>, + parent_freshness_request: FreshnessHeaders, + read_consistency_interval: Option, } #[async_trait::async_trait] @@ -53,8 +57,9 @@ impl BlobRangeRequester for TableBlobRangeRequester { range_header: &str, mode: RangeRequestMode, ) -> Result<(String, Response)> { - let mut request = self - .freshness + let freshness_request = + freshness_headers_snapshot(&self.freshness, self.read_consistency_interval); + let mut request = freshness_request .apply(self.client.get(&self.path)) .header(header::RANGE, range_header); if let Some(version) = self.version { @@ -71,6 +76,9 @@ impl BlobRangeRequester for TableBlobRangeRequester { return Ok((request_id, response)); } let response = self.client.check_response(&request_id, response).await?; + freshness_request.observe_headers(&self.freshness, response.headers()); + self.parent_freshness_request + .observe_headers(&self.parent_freshness, response.headers()); Ok((request_id, response)) } } @@ -361,18 +369,21 @@ impl RemoteTable { message: "fetch_blobs is not supported on this LanceDB Cloud server".into(), }); } - let version = self.current_version().await; + let read_snapshot = self.snapshot_read_state().await; let mut body = serde_json::json!({ - "version": version, + "version": read_snapshot.version, "column": column, "row_ids": row_ids, }); self.apply_branch_body(&mut body); let request = self - .post_read(&format!("/v1/table/{}/fetch_blobs/", self.identifier)) + .client + .post(&format!("/v1/table/{}/fetch_blobs/", self.identifier)) .json(&body); - let (request_id, response) = self.send(request, true).await?; + let (request_id, response) = self + .send_with_freshness(request, true, read_snapshot.freshness) + .await?; let mut stream = self.read_arrow_response(&request_id, response).await?; let mut blob_chunks: Vec> = Vec::new(); @@ -448,8 +459,7 @@ impl RemoteTable { }); } - let version = self.current_version().await; - let freshness = self.snapshot_freshness_headers(); + let read_snapshot = self.snapshot_read_state().await; let encoded_column = urlencoding::encode(column); let requesters = row_ids .iter() @@ -461,9 +471,12 @@ impl RemoteTable { let requester: Arc = Arc::new(TableBlobRangeRequester { client: self.client.clone(), path, - version, + version: read_snapshot.version, branch: self.branch.clone(), - freshness, + freshness: Arc::new(std::sync::Mutex::new(read_snapshot.freshness_state)), + parent_freshness: self.freshness.clone(), + parent_freshness_request: read_snapshot.freshness, + read_consistency_interval: self.client.read_consistency_interval, }); requester }) @@ -685,6 +698,46 @@ mod tests { assert!(requests.lock().unwrap().contains(&"bytes=5-11".to_string())); } + #[tokio::test] + async fn remote_blob_file_keeps_the_open_timeline_after_parent_checkout() { + let range_requests = Arc::new(StdMutex::new(Vec::new())); + let captured = range_requests.clone(); + let table = RemoteTable::new_mock( + "my_table".to_string(), + move |request| match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(r#"{"version":5,"schema":{"fields":[]}}"#.as_bytes().to_vec()) + .unwrap(), + "/v1/table/my_table/blob/image/10/bytes" => { + captured.lock().unwrap().push(( + request.url().query().unwrap_or_default().to_string(), + request.headers().clone(), + )); + range_response(&request, PAYLOAD) + } + path => panic!("unexpected path: {path}"), + }, + Some(Version::new(0, 5, 0)), + ); + + table.checkout(5).await.unwrap(); + let file = table + .fetch_blob_files_impl("image", &[10]) + .await + .unwrap() + .pop() + .flatten() + .unwrap(); + table.checkout_latest().await.unwrap(); + file.read_range(5..12).await.unwrap(); + + let requests = range_requests.lock().unwrap(); + let (query, headers) = requests.last().unwrap(); + assert!(query.contains("version=5")); + assert!(!headers.contains_key("x-lancedb-min-timestamp")); + } + #[tokio::test] async fn remote_blob_file_reuses_sequential_response_until_seek() { let requests = Arc::new(StdMutex::new(Vec::new())); diff --git a/rust/lancedb/src/remote/table/insert.rs b/rust/lancedb/src/remote/table/insert.rs index a4a28a9c6..4e0e0d666 100644 --- a/rust/lancedb/src/remote/table/insert.rs +++ b/rust/lancedb/src/remote/table/insert.rs @@ -24,7 +24,10 @@ use lance::io::exec::utils::InstrumentedRecordBatchStreamAdapter; use crate::Error; use crate::remote::ARROW_STREAM_CONTENT_TYPE; use crate::remote::client::{HttpSend, RestfulLanceDbClient, Sender}; -use crate::remote::table::{MergeInsertRequest, REQUEST_TIMEOUT_HEADER, RemoteTable}; +use crate::remote::table::{ + FreshnessHeaders, FreshnessState, MergeInsertRequest, REQUEST_TIMEOUT_HEADER, RemoteTable, + freshness_headers_snapshot, +}; use crate::table::datafusion::insert::COUNT_SCHEMA; use crate::table::write_progress::WriteProgressTracker; use crate::table::{AddResult, MergeResult}; @@ -54,6 +57,38 @@ pub enum WriteResult { Merge(MergeResult), } +#[derive(Debug, Clone, Default)] +struct WriteFreshness { + state: Option>>, + read_consistency_interval: Option, +} + +impl WriteFreshness { + fn prepare( + &self, + request: reqwest::RequestBuilder, + ) -> (reqwest::RequestBuilder, Option) { + match &self.state { + Some(state) => { + let freshness_request = + freshness_headers_snapshot(state, self.read_consistency_interval); + (freshness_request.apply(request), Some(freshness_request)) + } + None => (request, None), + } + } + + fn observe( + &self, + freshness_request: Option, + headers: &reqwest::header::HeaderMap, + ) { + if let (Some(state), Some(freshness_request)) = (&self.state, freshness_request) { + freshness_request.observe_headers(state, headers); + } + } +} + /// ExecutionPlan for streaming a write (add or merge_insert) to a remote /// LanceDB table. /// @@ -71,6 +106,7 @@ pub struct RemoteWriteExec { table_name: String, identifier: String, client: RestfulLanceDbClient, + freshness: WriteFreshness, input: Arc, op: WriteOp, properties: Arc, @@ -170,6 +206,7 @@ impl RemoteWriteExec { table_name, identifier, client, + freshness: WriteFreshness::default(), input, op, properties: Arc::new(properties), @@ -183,6 +220,18 @@ impl RemoteWriteExec { } } + pub(super) fn with_freshness( + mut self, + state: Arc>, + read_consistency_interval: Option, + ) -> Self { + self.freshness = WriteFreshness { + state: Some(state), + read_consistency_interval, + }; + self + } + /// Get the add result after execution, if this exec ran an insert. pub fn add_result(&self) -> Option { match self @@ -285,6 +334,7 @@ impl RemoteWriteExec { /// each threading the same handful of arguments. struct PartRequestCtx<'a, S: HttpSend> { client: &'a RestfulLanceDbClient, + freshness: &'a WriteFreshness, identifier: &'a str, table_name: &'a str, upload_id: &'a str, @@ -352,7 +402,11 @@ impl PartRequestCtx<'_, S> { } /// Build the `/insert` request for a single multipart part. - fn build_part_request(&self, part_id: &str, body: reqwest::Body) -> reqwest::RequestBuilder { + fn build_part_request( + &self, + part_id: &str, + body: reqwest::Body, + ) -> (reqwest::RequestBuilder, Option) { let mut request = self .client .post(&format!("/v1/table/{}/insert/", self.identifier)) @@ -368,12 +422,16 @@ impl PartRequestCtx<'_, S> { if let Some(b) = self.branch { request = request.query(&[("branch", b)]); } - request.body(body) + self.freshness.prepare(request.body(body)) } /// Send a single part's request and drain the response, mapping HTTP and /// table-not-found errors into `DataFusionError`. - async fn send_part_request(&self, request: reqwest::RequestBuilder) -> DataFusionResult<()> { + async fn send_part_request( + &self, + request: reqwest::RequestBuilder, + freshness_request: Option, + ) -> DataFusionResult<()> { let (request_id, response) = self .client .send(request) @@ -388,6 +446,8 @@ impl PartRequestCtx<'_, S> { .check_response(&request_id, response) .await .map_err(|e| DataFusionError::External(Box::new(e)))?; + self.freshness + .observe(freshness_request, response.headers()); response.bytes().await.map_err(|e| { DataFusionError::External(Box::new(Error::Http { source: Box::new(e), @@ -419,7 +479,7 @@ impl PartRequestCtx<'_, S> { let body = reqwest::Body::wrap_stream(chunk_rx); let part_id = uuid::Uuid::new_v4().to_string(); - let request = self.build_part_request(&part_id, body); + let (request, freshness_request) = self.build_part_request(&part_id, body); // Measured from just before the request is sent, matching the window the // client read timeout applies to the upload. @@ -495,7 +555,7 @@ impl PartRequestCtx<'_, S> { Ok::(input_ended) }; - let send = self.send_part_request(request); + let send = self.send_part_request(request, freshness_request); // `join!` rather than `tokio::spawn`: the producer borrows `input` (and // `schema`), so it cannot satisfy the `'static` bound a spawned task @@ -569,7 +629,7 @@ impl ExecutionPlan for RemoteWriteExec { // Building a fresh exec (with a new, empty `result`) is what makes the // outer rescannable retry loop work: `reset_state()` clears the captured // result so a re-execution starts clean. - Ok(Arc::new(Self::new_inner( + let mut exec = Self::new_inner( self.table_name.clone(), self.identifier.clone(), self.client.clone(), @@ -580,7 +640,9 @@ impl ExecutionPlan for RemoteWriteExec { self.branch.clone(), self.max_bytes_per_request, self.max_request_duration, - ))) + ); + exec.freshness = self.freshness.clone(); + Ok(Arc::new(exec)) } fn execute( @@ -613,6 +675,7 @@ impl ExecutionPlan for RemoteWriteExec { &self.metrics, )); let client = self.client.clone(); + let freshness = self.freshness.clone(); let identifier = self.identifier.clone(); let op = self.op.clone(); let result_slot = self.result.clone(); @@ -634,6 +697,7 @@ impl ExecutionPlan for RemoteWriteExec { let overwrite = matches!(op, WriteOp::Insert { overwrite: true }); let ctx = PartRequestCtx { client: &client, + freshness: &freshness, identifier: &identifier, table_name: &table_name, upload_id, @@ -688,7 +752,7 @@ impl ExecutionPlan for RemoteWriteExec { let (error_tx, mut error_rx) = tokio::sync::oneshot::channel(); let body = Self::stream_as_http_body(input_stream, error_tx, tracker)?; - let request = request.body(body); + let (request, freshness_request) = freshness.prepare(request.body(body)); let result: DataFusionResult<(String, _)> = async { let (request_id, response) = client @@ -708,6 +772,7 @@ impl ExecutionPlan for RemoteWriteExec { .check_response(&request_id, response) .await .map_err(|e| DataFusionError::External(Box::new(e)))?; + freshness.observe(freshness_request, response.headers()); Ok((request_id, response)) }