From a35f7044ee8273d3746e7e1256449d6831f4518d Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:43:04 -0700 Subject: [PATCH 01/31] test(rust): cover Azure table URI separators on Windows (#3810) ## Summary - add cross-platform regression coverage for Azure table URI construction - assert that az:// database paths always produce forward-slash blob keys ## Root cause ListingDatabase previously used the host filesystem Path join operation for object-store URIs, which inserted a backslash on Windows. The URI construction was corrected in #2575, but the original Azure report had no regression coverage and remained open. ## Validation - cargo fmt --all - cargo test --quiet --features remote -p lancedb test_table_uri_uses_forward_slashes_for_azure - cargo check --quiet --features remote --tests --examples - cargo clippy --quiet --features remote --tests --examples Fixes #2283 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/database/listing.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index 5ebbe6c5c..d8d687c7c 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -2569,6 +2569,21 @@ mod tests { } } + /// Regression test for https://github.com/lancedb/lancedb/issues/2283. + /// + /// Object-store URIs must use `/` on every platform. In particular, joining + /// with `std::path::Path` used to insert a `\\` into Azure blob keys on + /// Windows. + #[tokio::test] + async fn test_table_uri_uses_forward_slashes_for_azure() { + let (_tempdir, mut db) = setup_database().await; + db.uri = "az://test/db/test".to_string(); + + let uri = db.table_uri("test").unwrap(); + + assert_eq!(uri, "az://test/db/test/test.lance"); + } + /// Regression: connecting via a URL-style URI (which goes through /// `url::Url::parse` and the `query_pairs_mut()` path) must not /// append a trailing `?` to per-table URIs when the input URI has From c7cb0b9afa8f4807fb29c1af370348818416d231 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:13:49 -0700 Subject: [PATCH 02/31] docs(python): clarify threading on two-CPU containers (#3807) ## Summary - document that current LanceDB releases use one compute worker without warning on two-vCPU containers - distinguish compute-worker tuning from storage I/O concurrency - direct users of affected LanceDB 0.21.1 installations to upgrade and link the current threading guidance ## Root cause The Lance version bundled with LanceDB 0.21.1 warned whenever the detected CPU count was less than or equal to its default two-core I/O reservation. A two-vCPU deployment therefore emitted the warning on every query even though falling back to one compute worker was the intended behavior. Lance fixed that warning condition upstream in lance-format/lance#3710, and LanceDB current main already pins a version containing the runtime fix; the Python package documentation did not explain the corrected behavior or the distinct thread controls. ## Validation - `git diff --check` - verified the linked Lance threading-model documentation returns HTTP 200 Fixes #2326 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/python/README.md b/python/README.md index 550698500..3a81c486a 100644 --- a/python/README.md +++ b/python/README.md @@ -38,6 +38,25 @@ Stable releases are created about every 2 weeks. For the latest features and bug pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb ``` +### Threading in CPU-limited containers + +LanceDB uses separate pools for compute work and storage I/O. On a container with +two visible CPUs, current releases intentionally use one compute worker by default; +no manual configuration is needed. If every query logs an I/O core reservation +warning on a two-CPU container, upgrade from LanceDB 0.21.1 or earlier. + +The two commonly tuned environment variables control different resources: + +- `LANCE_CPU_THREADS` overrides the number of compute workers. One worker is the + appropriate setting for a two-CPU container when an explicit override is needed. +- `LANCE_IO_THREADS` controls concurrent storage operations, not reserved CPU + cores. Its default can be greater than the number of CPUs because I/O workers + spend much of their time waiting for storage. + +Keep the defaults unless measurements show that the workload benefits from an +override. See the [Lance threading model](https://lance.org/guide/performance/#threading-model) +for the current defaults and tuning guidance. + ## Usage ### Basic Example From 01679e37fdefcbf12ce42c7b2ac579ad6098d8fd Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 21 Aug 2026 16:17:03 -0700 Subject: [PATCH 03/31] feat: materialized view declarations on local tables (#3930) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A materialized view is a table whose contents are defined by a query over one source table and maintained by refresh rather than by writes. The declaration half: create_materialized_view(name, source) resolves a projected, filtered and limited definition against the source schema -- output types come from the DataFusion planner, never the caller -- and commits an empty table carrying it as kind-tagged JSON in schema metadata. The tag lets a kind added later read back as a view this version cannot refresh rather than as a plain table. Views open and list as ordinary tables. Sources must have stable row ids, checked here because the property cannot be enabled later: each view row records its source row in __source_row_id, and that provenance survives compactions, updates and deletes only when row ids are stable. A view inherits the metadata describing its columns and none governing how a table is written, so blob markers carry through while declarations its always-nullable fields would contradict are stripped. Embedding configuration is rewritten to the view's column names, and dropped where it does not project both ends of a function. Stack created with GitHub Stacks CLI • Give Feedback 💬 --- rust/lancedb/src/connection.rs | 2 +- rust/lancedb/src/database/listing.rs | 302 +++- rust/lancedb/src/database/namespace.rs | 202 ++- rust/lancedb/src/error.rs | 2 + rust/lancedb/src/lib.rs | 2 + rust/lancedb/src/materialized_view.rs | 1992 ++++++++++++++++++++++++ rust/lancedb/src/remote/table.rs | 14 + rust/lancedb/src/table.rs | 5 + rust/lancedb/src/table/refresh.rs | 2 +- 9 files changed, 2406 insertions(+), 117 deletions(-) create mode 100644 rust/lancedb/src/materialized_view.rs diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 8935855a8..187e2df0e 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -41,7 +41,7 @@ use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider}; mod create_table; -fn merge_storage_options( +pub(crate) fn merge_storage_options( store_params: &mut ObjectStoreParams, pairs: impl IntoIterator, ) { diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index d8d687c7c..c9e9bcb22 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -765,60 +765,13 @@ impl ListingDatabase { } } - /// Extract storage option overrides from the request - fn extract_storage_overrides( - &self, - request: &CreateTableRequest, - ) -> Result<(Option, Option, Option)> { - let storage_options = request - .write_options - .lance_write_params - .as_ref() - .and_then(|p| p.store_params.as_ref()) - .and_then(|sp| sp.storage_options()); - - let storage_version_override = storage_options - .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION)) - .map(|s| s.parse::()) - .transpose()?; - - let v2_manifest_override = storage_options - .and_then(|opts| opts.get(OPT_NEW_TABLE_V2_MANIFEST_PATHS)) - .map(|s| s.parse::()) - .transpose() - .map_err(|_| Error::InvalidInput { - message: "enable_v2_manifest_paths must be a boolean".to_string(), - })?; - - let stable_row_ids_override = storage_options - .and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS)) - .map(|s| s.parse::()) - .transpose() - .map_err(|_| Error::InvalidInput { - message: "enable_stable_row_ids must be a boolean".to_string(), - })?; - - Ok(( - storage_version_override, - v2_manifest_override, - stable_row_ids_override, - )) - } - /// Prepare write parameters for table creation fn prepare_write_params( &self, request: &CreateTableRequest, - storage_version_override: Option, - v2_manifest_override: Option, - stable_row_ids_override: Option, + mut write_params: lance::dataset::WriteParams, + overrides: NewTableConfig, ) -> lance::dataset::WriteParams { - let mut write_params = request - .write_options - .lance_write_params - .clone() - .unwrap_or_default(); - // Only modify the storage options if we actually have something to // inherit. There is a difference between storage_options=None and // storage_options=Some({}). Using storage_options=None will cause the @@ -842,18 +795,21 @@ impl ListingDatabase { store_params.storage_options_accessor = Some(Arc::new(accessor)); } - write_params.data_storage_version = storage_version_override + write_params.data_storage_version = overrides + .data_storage_version .or(write_params.data_storage_version) .or(self.new_table_config.data_storage_version); - if let Some(enable_v2_manifest_paths) = - v2_manifest_override.or(self.new_table_config.enable_v2_manifest_paths) + if let Some(enable_v2_manifest_paths) = overrides + .enable_v2_manifest_paths + .or(self.new_table_config.enable_v2_manifest_paths) { write_params.enable_v2_manifest_paths = enable_v2_manifest_paths; } let data_schema = request.data.arrow_schema(); - if let Some(enable_stable_row_ids) = stable_row_ids_override + if let Some(enable_stable_row_ids) = overrides + .enable_stable_row_ids .or(self.new_table_config.enable_stable_row_ids) .or(has_blob_columns(&data_schema).then_some(true)) { @@ -1048,15 +1004,13 @@ impl Database for ListingDatabase { .clone() .unwrap_or_else(|| self.table_uri(&request.name).unwrap()); - let (storage_version_override, v2_manifest_override, stable_row_ids_override) = - self.extract_storage_overrides(&request)?; - - let write_params = self.prepare_write_params( - &request, - storage_version_override, - v2_manifest_override, - stable_row_ids_override, - ); + let mut write_params = request + .write_options + .lance_write_params + .clone() + .unwrap_or_default(); + let overrides = take_request_creation_overrides(&mut write_params)?; + let write_params = self.prepare_write_params(&request, write_params, overrides); let data_schema = request.data.arrow_schema(); @@ -1288,8 +1242,232 @@ impl Database for ListingDatabase { } } +/// Parse the request-level `new_table_*` creation keys into overrides and +/// strip them from the store options in one step: every create path that +/// honors them must also keep them out of the object store. +pub(crate) fn take_request_creation_overrides( + params: &mut lance::dataset::WriteParams, +) -> Result { + let storage_options = params + .store_params + .as_ref() + .and_then(|sp| sp.storage_options()); + let overrides = NewTableConfig { + data_storage_version: storage_options + .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION)) + .map(|s| s.parse::()) + .transpose()?, + enable_v2_manifest_paths: storage_options + .and_then(|opts| opts.get(OPT_NEW_TABLE_V2_MANIFEST_PATHS)) + .map(|s| s.parse::()) + .transpose() + .map_err(|_| Error::InvalidInput { + message: "enable_v2_manifest_paths must be a boolean".to_string(), + })?, + enable_stable_row_ids: storage_options + .and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS)) + .map(|s| s.parse::()) + .transpose() + .map_err(|_| Error::InvalidInput { + message: "enable_stable_row_ids must be a boolean".to_string(), + })?, + }; + if let Some(store_params) = params.store_params.as_mut() { + strip_new_table_creation_keys(store_params); + } + Ok(overrides) +} + +/// Strip the `new_table_*` creation keys from request store options: they are +/// creation config, not credentials, and left in place they fork a fresh +/// store connection for the request. +fn strip_new_table_creation_keys(store_params: &mut ObjectStoreParams) { + let mut options = store_params.storage_options().cloned().unwrap_or_default(); + let mut removed = false; + for key in [ + OPT_NEW_TABLE_STORAGE_VERSION, + OPT_NEW_TABLE_V2_MANIFEST_PATHS, + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, + ] { + removed |= options.remove(key).is_some(); + } + if !removed { + return; + } + let provider = store_params + .storage_options_accessor + .as_ref() + .and_then(|accessor| accessor.provider().cloned()); + store_params.storage_options_accessor = match (options.is_empty(), provider) { + (true, None) => None, + (true, Some(provider)) => Some(Arc::new(StorageOptionsAccessor::with_provider(provider))), + (false, Some(provider)) => Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider(options, provider), + )), + (false, None) => Some(Arc::new(StorageOptionsAccessor::with_static_options( + options, + ))), + }; +} + #[cfg(test)] mod tests { + #[tokio::test] + async fn request_level_creation_keys_do_not_fork_the_store() { + use crate::query::ExecutableQuery; + use futures::TryStreamExt; + + let db = crate::connect("memory://").execute().await.unwrap(); + let batch = arrow_array::record_batch!(("x", Int32, [1, 2])).unwrap(); + let store_params = ObjectStoreParams { + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + HashMap::from([( + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(), + "true".to_string(), + )]), + ))), + ..Default::default() + }; + db.create_table("t", batch) + .write_options(crate::table::WriteOptions { + lance_write_params: Some(lance::dataset::WriteParams { + store_params: Some(store_params), + ..Default::default() + }), + }) + .execute() + .await + .unwrap(); + + let table = db.open_table("t").execute().await.unwrap(); + let rows: usize = table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap() + .iter() + .map(|b| b.num_rows()) + .sum(); + assert_eq!(rows, 2, "the table must live in the session's store"); + } + + mod strip_new_table_creation_keys { + use super::super::*; + + #[derive(Debug)] + struct EmptyProvider; + + #[async_trait::async_trait] + impl StorageOptionsProvider for EmptyProvider { + async fn fetch_storage_options( + &self, + ) -> lance_core::Result>> { + Ok(Some(HashMap::new())) + } + + fn provider_id(&self) -> String { + "empty-test-provider".into() + } + } + + fn params_with_static(options: &[(&str, &str)]) -> ObjectStoreParams { + ObjectStoreParams { + storage_options_accessor: Some(Arc::new( + StorageOptionsAccessor::with_static_options( + options + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ), + )), + ..Default::default() + } + } + + #[test] + fn creation_keys_are_removed_and_store_keys_kept() { + let mut params = params_with_static(&[ + ("region", "us-west-2"), + (OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true"), + ]); + strip_new_table_creation_keys(&mut params); + let options = params.storage_options().cloned().unwrap(); + assert_eq!(options.get("region").map(String::as_str), Some("us-west-2")); + assert!(!options.contains_key(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS)); + + // Creation keys alone: no accessor survives to fork a store. + let mut params = params_with_static(&[(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true")]); + strip_new_table_creation_keys(&mut params); + assert!(params.storage_options_accessor.is_none()); + } + + /// A provider must survive every shape of strip: untouched accessors + /// keep their identity, emptied ones still fetch, and residual + /// statics ride along. + #[test] + fn provider_accessors_survive_the_strip() { + let accessor = Arc::new(StorageOptionsAccessor::with_provider(Arc::new( + EmptyProvider, + ))); + let mut params = ObjectStoreParams { + storage_options_accessor: Some(accessor.clone()), + ..Default::default() + }; + strip_new_table_creation_keys(&mut params); + assert!(Arc::ptr_eq( + params.storage_options_accessor.as_ref().unwrap(), + &accessor + )); + + let mut params = ObjectStoreParams { + storage_options_accessor: Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider( + HashMap::from([ + ("region".to_string(), "us-west-2".to_string()), + ( + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(), + "true".to_string(), + ), + ]), + Arc::new(EmptyProvider), + ), + )), + ..Default::default() + }; + strip_new_table_creation_keys(&mut params); + let accessor = params.storage_options_accessor.unwrap(); + assert!(accessor.has_provider()); + assert_eq!( + accessor + .initial_storage_options() + .and_then(|o| o.get("region").cloned()) + .as_deref(), + Some("us-west-2") + ); + + // Emptied entirely: a first-fetch accessor, not one caching {}. + let mut params = ObjectStoreParams { + storage_options_accessor: Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider( + HashMap::from([( + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(), + "true".to_string(), + )]), + Arc::new(EmptyProvider), + ), + )), + ..Default::default() + }; + strip_new_table_creation_keys(&mut params); + let accessor = params.storage_options_accessor.unwrap(); + assert!(accessor.has_provider()); + assert!(accessor.initial_storage_options().is_none()); + } + } + use super::*; use crate::Table; use crate::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream}; diff --git a/rust/lancedb/src/database/namespace.rs b/rust/lancedb/src/database/namespace.rs index 740e11645..250d933f6 100644 --- a/rust/lancedb/src/database/namespace.rs +++ b/rust/lancedb/src/database/namespace.rs @@ -26,10 +26,7 @@ use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler; use crate::blob::{ensure_blob_storage_version, has_blob_columns}; use crate::connection::NamespaceClientPushdownOperation; use crate::database::ReadConsistency; -use crate::database::listing::{ - NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, OPT_NEW_TABLE_STORAGE_VERSION, - OPT_NEW_TABLE_V2_MANIFEST_PATHS, -}; +use crate::database::listing::{NewTableConfig, take_request_creation_overrides}; use crate::database::read_freshness::{ FreshnessBaselines, ReadFreshnessContextProvider, TableFreshness, }; @@ -197,69 +194,28 @@ impl LanceNamespaceDatabase { TableFreshness::new(self.freshness_baselines.clone(), key) } - fn extract_storage_overrides( - &self, - request: &DbCreateTableRequest, - ) -> Result<( - Option, - Option, - Option, - )> { - let storage_options = request - .write_options - .lance_write_params - .as_ref() - .and_then(|p| p.store_params.as_ref()) - .and_then(|sp| sp.storage_options()); - - let storage_version_override = storage_options - .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION)) - .map(|s| s.parse::()) - .transpose()?; - - let v2_manifest_override = storage_options - .and_then(|opts| opts.get(OPT_NEW_TABLE_V2_MANIFEST_PATHS)) - .map(|s| s.parse::()) - .transpose() - .map_err(|_| Error::InvalidInput { - message: "enable_v2_manifest_paths must be a boolean".to_string(), - })?; - - let stable_row_ids_override = storage_options - .and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS)) - .map(|s| s.parse::()) - .transpose() - .map_err(|_| Error::InvalidInput { - message: "enable_stable_row_ids must be a boolean".to_string(), - })?; - - Ok(( - storage_version_override, - v2_manifest_override, - stable_row_ids_override, - )) - } - fn apply_new_table_config( &self, params: &mut lance::dataset::WriteParams, request: &DbCreateTableRequest, ) -> Result<()> { - let (storage_version_override, v2_manifest_override, stable_row_ids_override) = - self.extract_storage_overrides(request)?; + let overrides = take_request_creation_overrides(params)?; - params.data_storage_version = storage_version_override + params.data_storage_version = overrides + .data_storage_version .or(params.data_storage_version) .or(self.new_table_config.data_storage_version); - if let Some(enable_v2_manifest_paths) = - v2_manifest_override.or(self.new_table_config.enable_v2_manifest_paths) + if let Some(enable_v2_manifest_paths) = overrides + .enable_v2_manifest_paths + .or(self.new_table_config.enable_v2_manifest_paths) { params.enable_v2_manifest_paths = enable_v2_manifest_paths; } let data_schema = request.data.schema(); - if let Some(enable_stable_row_ids) = stable_row_ids_override + if let Some(enable_stable_row_ids) = overrides + .enable_stable_row_ids .or(self.new_table_config.enable_stable_row_ids) .or(has_blob_columns(data_schema.as_ref()).then_some(true)) { @@ -644,6 +600,146 @@ mod tests { RecordBatch::try_new(schema, vec![Arc::new(id_array), Arc::new(name_array)]).unwrap() } + /// The shared parse-and-sanitize boundary is wired into this path: the + /// request-level creation key must act as an override (the strip itself + /// is covered by the listing tests). + #[tokio::test] + async fn request_level_creation_keys_are_taken_as_overrides() { + use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS; + + let tmp_dir = tempdir().unwrap(); + let mut properties = HashMap::new(); + properties.insert( + "root".to_string(), + tmp_dir.path().to_str().unwrap().to_string(), + ); + let db = connect_namespace("dir", properties) + .execute() + .await + .unwrap(); + + let store_params = ObjectStoreParams { + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + HashMap::from([( + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(), + "true".to_string(), + )]), + ))), + ..Default::default() + }; + let table = db + .create_table("t", create_test_data()) + .write_options(crate::table::WriteOptions { + lance_write_params: Some(lance::dataset::WriteParams { + store_params: Some(store_params), + ..Default::default() + }), + }) + .execute() + .await + .unwrap(); + let native = table.as_native().unwrap(); + assert!( + native + .dataset + .get() + .await + .unwrap() + .manifest + .uses_stable_row_ids(), + "the creation key must be honored as an override" + ); + + let table = db.open_table("t").execute().await.unwrap(); + let rows: usize = table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap() + .iter() + .map(|b| b.num_rows()) + .sum(); + assert_eq!(rows, 5); + } + + /// Sanitation on this path: apply must strip the creation keys from the + /// store options while genuine options and the provider survive. + #[tokio::test] + async fn apply_new_table_config_sanitizes_request_store_options() { + use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS; + use lance_io::object_store::StorageOptionsProvider; + + #[derive(Debug)] + struct EmptyProvider; + + #[async_trait::async_trait] + impl StorageOptionsProvider for EmptyProvider { + async fn fetch_storage_options( + &self, + ) -> lance_core::Result>> { + Ok(Some(HashMap::new())) + } + + fn provider_id(&self) -> String { + "empty-test-provider".into() + } + } + + let tmp_dir = tempdir().unwrap(); + let mut properties = HashMap::new(); + properties.insert( + "root".to_string(), + tmp_dir.path().to_str().unwrap().to_string(), + ); + let db = LanceNamespaceDatabase::connect_with_new_table_config( + "dir", + properties, + HashMap::new(), + None, + None, + HashSet::new(), + NewTableConfig::default(), + ) + .await + .unwrap(); + + let request = DbCreateTableRequest::new("t".to_string(), Box::new(create_test_data())); + let mut params = lance::dataset::WriteParams { + store_params: Some(ObjectStoreParams { + storage_options_accessor: Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider( + HashMap::from([ + ("region".to_string(), "us-west-2".to_string()), + ( + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(), + "true".to_string(), + ), + ]), + Arc::new(EmptyProvider), + ), + )), + ..Default::default() + }), + ..Default::default() + }; + db.apply_new_table_config(&mut params, &request).unwrap(); + + assert!(params.enable_stable_row_ids); + let store_params = params.store_params.unwrap(); + let options = store_params.storage_options().cloned().unwrap(); + assert!(!options.contains_key(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS)); + assert_eq!(options.get("region").map(String::as_str), Some("us-west-2")); + assert!( + store_params + .storage_options_accessor + .unwrap() + .has_provider() + ); + } + #[tokio::test] async fn test_namespace_connection_simple() { // Test that namespace connections work with simple connect_namespace(impl_type, properties) diff --git a/rust/lancedb/src/error.rs b/rust/lancedb/src/error.rs index 6bd1ffa2b..be4641388 100644 --- a/rust/lancedb/src/error.rs +++ b/rust/lancedb/src/error.rs @@ -77,6 +77,8 @@ pub enum Error { ColumnAlreadyExists { name: String }, #[snafu(display("Column '{name}' is not a computed column"))] NotAComputedColumn { name: String }, + #[snafu(display("Table '{name}' is not a materialized view"))] + NotAMaterializedView { name: String }, #[snafu(display("Invalid expression for column '{column}': {message}"))] InvalidExpression { column: String, message: String }, diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index 291dcaf65..3a937db5b 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -186,6 +186,7 @@ pub mod index; pub mod io; pub mod ipc; pub mod job; +pub mod materialized_view; #[cfg(feature = "metrics-otel")] pub mod metrics_otel; #[cfg(feature = "polars")] @@ -210,6 +211,7 @@ pub use function::FunctionVersion; pub use job::Job; use lance_index::vector::ApproxMode as LanceApproxMode; use lance_linalg::distance::DistanceType as LanceDistanceType; +pub use materialized_view::{MaterializedView, MaterializedViewDefinition}; /// Re-export of the [`metrics`](https://docs.rs/metrics) crate facade. Enable /// the `metrics` feature to publish LanceDB's internal metrics; install any /// `metrics`-compatible recorder to collect them. See also [`metrics_otel`] for diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs new file mode 100644 index 000000000..f1eddf9da --- /dev/null +++ b/rust/lancedb/src/materialized_view.rs @@ -0,0 +1,1992 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Materialized views. +//! +//! A materialized view is a table whose contents are defined by a query over +//! one source table and maintained by refresh rather than by writes. Creation +//! commits an empty table carrying the kind-tagged definition in schema +//! metadata; a kind added later reads back as unrefreshable, not as a plain +//! table. Queries, indexes and search work on the view unchanged. + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_schema::{DataType, Field as ArrowField, FieldRef, Schema as ArrowSchema, SchemaRef}; +use datafusion_common::ScalarValue; +use lance_core::ROW_ID; +use lance_datafusion::planner::Planner; +use serde::{Deserialize, Serialize}; + +use crate::connection::Connection; +use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS; +use crate::database::{CreateTableRequest, Database, OpenTableRequest}; +use crate::embeddings::EmbeddingDefinition; +use crate::table::Table; +use crate::table::refresh::quote_identifier; +use crate::table::{ColumnDefinition, ColumnKind}; +use crate::{Error, Result}; + +/// Schema metadata key holding the view definition, as kind-tagged JSON. +pub const DEFINITION_META_KEY: &str = "mv.definition"; + +/// Schema metadata key holding the source table version the view was last +/// refreshed to. Absent until the first refresh. +pub const SOURCE_VERSION_META_KEY: &str = "mv.source_version"; + +/// Schema metadata key holding the wall-clock time of the last refresh, +/// in milliseconds since the epoch. +pub const REFRESHED_AT_MS_META_KEY: &str = "mv.refreshed_at_ms"; + +/// Column recording which source row produced each view row: the source's +/// stable `_rowid` at refresh time, which is why sources must keep stable +/// row ids. +pub const SOURCE_ROW_ID_COLUMN: &str = "__source_row_id"; + +/// Field metadata namespace for declarations about schema structure, such as +/// an unenforced primary key. +const SCHEMA_DECLARATION_META_PREFIX: &str = "lance-schema:"; + +/// A field's identity in its own schema, which is not the view's. +const LANCE_FIELD_ID_KEY: &str = "lance:field_id"; + +/// Schema metadata key holding embedding-function configuration. It describes +/// columns rather than storage, so a view carries it through. +const EMBEDDING_FUNCTIONS_META_KEY: &str = "embedding_functions"; + +/// Schema metadata key holding lancedb's own column definitions, one per +/// field in schema order. It marks which columns an embedding function +/// produces, which is what lets a query embed its own text. +const COLUMN_DEFINITIONS_META_KEY: &str = "lancedb::column_definitions"; + +/// Value of the definition's `kind` tag for the projected `select` form. +pub const SELECT_KIND: &str = "select"; + +/// Which view outputs each source column is projected to directly. A column +/// may be projected more than once, so each carries every name the view gives +/// it, in projection order. +type Lineage = HashMap>; + +/// One projected output column of a view. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ViewProjection { + /// Name of the column in the view. + pub output: String, + /// SQL expression over the source table that computes it. + pub expression: String, +} + +/// The query that defines a materialized view. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MaterializedViewDefinition { + /// Name of the source table, in the same database as the view. + pub source_table: String, + /// The projected output columns, in view schema order. + pub projections: Vec, + /// SQL predicate selecting the source rows the view holds. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub filter: Option, + /// Cap on the number of rows the view holds, in materialization order. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, + /// Source columns the projections and filter read, derived at creation. + #[serde(default)] + pub inputs: Vec, +} + +/// A view definition as read back from schema metadata. Non-exhaustive so a +/// kind added later is additive. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum MaterializedViewKind { + /// The projected `select` form. + Select(MaterializedViewDefinition), + /// A kind written by a newer version, reported so a caller can tell an + /// unrefreshable view apart from a plain table. Nothing produces this. + Unrecognized { + /// The kind as it was found in the metadata. + kind: String, + }, +} + +/// Serialize `definition` into the kind-tagged form stored under +/// [`DEFINITION_META_KEY`]. +pub(crate) fn definition_to_metadata(definition: &MaterializedViewDefinition) -> Result { + let mut value = serde_json::to_value(definition).map_err(|e| Error::Runtime { + message: format!("failed to serialize view definition: {e}"), + })?; + value["kind"] = serde_json::Value::String(SELECT_KIND.to_string()); + Ok(value.to_string()) +} + +/// Read a view declaration off a schema metadata map, if it carries one. +/// `Ok(None)` for a plain table; a declaration that does not parse is an +/// error, because treating a view as plain would let it be rewritten. +pub fn materialized_view_kind( + metadata: &HashMap, +) -> Result> { + let Some(raw) = metadata.get(DEFINITION_META_KEY) else { + return Ok(None); + }; + let unreadable = |e: &dyn std::fmt::Display| Error::Runtime { + message: format!("unreadable materialized view definition: {e}"), + }; + let value: serde_json::Value = serde_json::from_str(raw).map_err(|e| unreadable(&e))?; + let kind = value + .get("kind") + .and_then(|k| k.as_str()) + .ok_or_else(|| unreadable(&"missing kind tag"))?; + if kind != SELECT_KIND { + return Ok(Some(MaterializedViewKind::Unrecognized { + kind: kind.to_string(), + })); + } + let definition = serde_json::from_value(value).map_err(|e| unreadable(&e))?; + Ok(Some(MaterializedViewKind::Select(definition))) +} + +/// Resolve a definition against the source schema into the view's projected +/// fields, with `inputs` filled in. Everything statically checkable is +/// checked here rather than at refresh time. Empty `projections` selects +/// every source column as the schema stands now. +pub(crate) fn plan( + source_schema: SchemaRef, + source_table: &str, + projections: &[(String, String)], + filter: Option<&str>, + limit: Option, +) -> Result<(MaterializedViewDefinition, Vec, Lineage)> { + let projections: Vec<(String, String)> = if projections.is_empty() { + source_schema + .fields() + .iter() + // A source that is itself a view carries its own provenance + // column; the new view records its own, not a copy. + .filter(|f| f.name() != SOURCE_ROW_ID_COLUMN) + .map(|f| (f.name().clone(), quote_identifier(f.name()))) + .collect() + } else { + projections.to_vec() + }; + + // A scan takes the cap as i64. Rejecting it here keeps creation and + // refresh from disagreeing about whether a view is valid. + if let Some(limit) = limit + && i64::try_from(limit).is_err() + { + return Err(Error::InvalidInput { + message: format!("view limit {limit} exceeds the maximum of {}", i64::MAX), + }); + } + + let planner = Planner::new(source_schema.clone()); + let mut fields = Vec::with_capacity(projections.len()); + let mut inputs = Vec::new(); + let mut declared: Vec<&str> = Vec::with_capacity(projections.len()); + let mut lineage: Lineage = HashMap::new(); + + for (output, expression) in &projections { + if declared.contains(&output.as_str()) { + return Err(Error::ColumnAlreadyExists { + name: output.clone(), + }); + } + if output == SOURCE_ROW_ID_COLUMN || output == ROW_ID { + return Err(Error::InvalidInput { + message: format!("view column name '{output}' is reserved"), + }); + } + + let parsed = planner + .parse_expr(expression) + .map_err(|e| Error::InvalidExpression { + column: output.clone(), + message: e.to_string(), + })?; + // Before optimization: the simplifier folds a stable-but-not-immutable + // call like now() into a literal, hiding it from the check while the + // stored definition keeps the call. + ensure_immutable(&parsed, |message| Error::InvalidExpression { + column: output.clone(), + message, + })?; + let expr = planner + .optimize_expr(parsed) + .map_err(|e| Error::InvalidExpression { + column: output.clone(), + message: e.to_string(), + })?; + let expr_inputs = + resolve_inputs(&source_schema, &expr, |message| Error::InvalidExpression { + column: output.clone(), + message, + })?; + + // Physical expressions address columns by position, so the planner + // that types the expression is built on the projected schema. + let read_schema = project_schema(&source_schema, &expr_inputs); + let physical = Planner::new(read_schema.clone()) + .create_physical_expr(&expr) + .map_err(|e| Error::InvalidExpression { + column: output.clone(), + message: e.to_string(), + })?; + let data_type = + physical + .data_type(read_schema.as_ref()) + .map_err(|e| Error::InvalidExpression { + column: output.clone(), + message: e.to_string(), + })?; + + // Always nullable: what a refresh appends must fit the declared field + // whatever nullability the evaluator reports for a given batch. + let mut field = ArrowField::new(output, data_type, true); + // Identity projections keep descriptive field metadata (blob markers); + // computed values carry none. Structural declarations never come along. + if let Some(source_field) = projected_field(&expr, &source_schema) { + field = field.with_metadata(source_field.metadata().clone()); + } + if let Some(path) = projected_path(&expr) + && let [column] = path.as_slice() + { + lineage + .entry(column.clone()) + .or_default() + .push(output.clone()); + } + fields.push(without_declarations(&field)); + inputs.extend(expr_inputs); + declared.push(output); + } + + if let Some(filter) = filter { + let expr = planner + .parse_filter(filter) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })?; + ensure_immutable(&expr, |message| Error::InvalidInput { + message: format!("invalid view filter: {message}"), + })?; + let filter_inputs = resolve_inputs(&source_schema, &expr, |message| Error::InvalidInput { + message: format!("invalid view filter: {message}"), + })?; + // A committed filter has to be usable as a predicate. + let read_schema = project_schema(&source_schema, &filter_inputs); + let data_type = Planner::new(read_schema.clone()) + .create_physical_expr(&expr) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })? + .data_type(read_schema.as_ref()) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })?; + if data_type != DataType::Boolean { + return Err(Error::InvalidInput { + message: format!("view filter must be a boolean predicate, not {data_type}"), + }); + } + inputs.extend(filter_inputs); + } + + inputs.sort(); + inputs.dedup(); + + let definition = MaterializedViewDefinition { + source_table: source_table.to_string(), + projections: projections + .into_iter() + .map(|(output, expression)| ViewProjection { output, expression }) + .collect(), + filter: filter.map(String::from), + limit, + inputs, + }; + Ok((definition, fields, lineage)) +} + +/// Reject any function that is not immutable: a view definition has to +/// evaluate identically across refreshes, or incremental maintenance would +/// mix rows from different evaluations of the same definition. +fn ensure_immutable(expr: &datafusion_expr::Expr, error: impl Fn(String) -> Error) -> Result<()> { + use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; + use datafusion_expr::Volatility; + + // Labeled immutable but not determined by row values alone: version() + // depends on the build, the arrow_* introspectors on schema state. + const NOT_VALUE_DETERMINED: &[&str] = + &["version", "arrow_typeof", "arrow_field", "arrow_metadata"]; + + let mut offending: Option = None; + expr.apply(|node| { + if let datafusion_expr::Expr::ScalarFunction(function) = node { + let name = function.func.name(); + if function.func.signature().volatility != Volatility::Immutable + || NOT_VALUE_DETERMINED.contains(&name) + { + offending = Some(name.to_string()); + return Ok(TreeNodeRecursion::Stop); + } + } + Ok(TreeNodeRecursion::Continue) + }) + .map_err(|e| error(e.to_string()))?; + match offending { + Some(name) => Err(error(format!( + "function '{name}' is not immutable and would evaluate differently \ + across refreshes" + ))), + None => Ok(()), + } +} + +/// The root of a possibly-dotted column path: `metadata.age` -> `metadata`. +fn root(path: &str) -> &str { + path.split('.').next().unwrap_or(path) +} + +/// The columns `expr` reads, kept as the planner reports them (a nested +/// reference stays a dotted path) but resolved by root field. +/// Embedding configuration rewritten for the view: entries whose columns the +/// view projects directly are kept under the view's names; the rest describe +/// a table that does not exist and are dropped. +fn embedding_config_for_view(raw: &str, lineage: &Lineage) -> Option { + // Every representation the writers use: the Python bindings name the + // destination `vector_column`, the Rust definition `dest_column`, and the + // Node bindings spell both halves in camelCase. + const SOURCE_KEYS: [&str; 2] = ["source_column", "sourceColumn"]; + const DEST_KEYS: [&str; 4] = ["vector_column", "dest_column", "vectorColumn", "destColumn"]; + + let entries: Vec = serde_json::from_str(raw).ok()?; + let mut kept = Vec::new(); + for entry in &entries { + let Some(object) = entry.as_object() else { + continue; + }; + let named = |keys: &[&str]| { + let key = keys.iter().find(|key| object.contains_key(**key))?; + let outputs = lineage.get(object.get(*key)?.as_str()?)?; + Some(((*key).to_string(), outputs)) + }; + let (Some((source_key, sources)), Some((dest_key, dests))) = + (named(&SOURCE_KEYS), named(&DEST_KEYS)) + else { + continue; + }; + // A projection may give one source column several names, and every + // pairing of the two is a real relationship in the view. + for source in sources { + for dest in dests { + let mut object = object.clone(); + object.insert(source_key.clone(), source.clone().into()); + object.insert(dest_key.clone(), dest.clone().into()); + kept.push(serde_json::Value::Object(object)); + } + } + } + (!kept.is_empty()).then(|| serde_json::Value::Array(kept).to_string()) +} + +/// Lancedb's column definitions rewritten for the view: positional, one per +/// view field. Directly projected embedding columns keep their definition +/// under the view's names; everything else is physical. `None` = no key. +fn column_definitions_for_view( + raw: &str, + source_schema: &ArrowSchema, + view_fields: &[ArrowField], + lineage: &Lineage, +) -> Option { + let source_definitions: Vec = serde_json::from_str(raw).ok()?; + // The definition sits on the column the function writes, so the source + // schema's field name at that position is the embedding's destination. + let embeddings: HashMap<&str, &EmbeddingDefinition> = source_schema + .fields() + .iter() + .zip(&source_definitions) + .filter_map(|(field, definition)| match &definition.kind { + ColumnKind::Embedding(embedding) => Some((field.name().as_str(), embedding)), + ColumnKind::Physical => None, + }) + .collect(); + let sources: HashMap<&str, &str> = lineage + .iter() + .flat_map(|(source, outputs)| outputs.iter().map(move |o| (o.as_str(), source.as_str()))) + .collect(); + + let mut kept = false; + let definitions: Vec = view_fields + .iter() + .map(|field| { + let kind = embedding_for_output(field.name(), &embeddings, &sources, lineage) + .map(|embedding| { + kept = true; + ColumnKind::Embedding(embedding) + }) + .unwrap_or(ColumnKind::Physical); + ColumnDefinition { kind } + }) + .collect(); + kept.then(|| serde_json::to_string(&definitions).ok())? +} + +/// The embedding `output` inherits, renamed to the view's columns. `None` +/// unless the view projects both the function's input and its output +/// directly: anything else advertises a column the view cannot recompute. +fn embedding_for_output( + output: &str, + embeddings: &HashMap<&str, &EmbeddingDefinition>, + sources: &HashMap<&str, &str>, + lineage: &Lineage, +) -> Option { + let embedding = embeddings.get(sources.get(output)?)?; + // The input may be projected several times; the first name the view gives + // it is the one this column is defined against. + let input = lineage.get(&embedding.source_column)?.first()?; + Some(EmbeddingDefinition { + source_column: input.clone(), + dest_column: Some(output.to_string()), + embedding_name: embedding.embedding_name.clone(), + }) +} + +/// The source field a projection reads directly, if it reads one: a bare +/// column, or a path of struct field accesses over one. Anything computed +/// produces a new value and has no source field. +fn projected_field<'a>( + expr: &datafusion_expr::Expr, + schema: &'a ArrowSchema, +) -> Option<&'a ArrowField> { + let path = projected_path(expr)?; + let mut segments = path.iter(); + let mut field = schema.field_with_name(segments.next()?).ok()?; + for segment in segments { + let DataType::Struct(children) = field.data_type() else { + return None; + }; + field = children.iter().find(|c| c.name() == segment)?; + } + Some(field) +} + +/// The dotted path a projection reads directly, root first. +fn projected_path(expr: &datafusion_expr::Expr) -> Option> { + let mut path = Vec::new(); + let mut node = expr; + loop { + match node { + datafusion_expr::Expr::Column(column) => { + path.push(column.name.clone()); + break; + } + // `a.b` parses to get_field(a, "b"), nested for deeper paths. + datafusion_expr::Expr::ScalarFunction(call) if call.func.name() == "get_field" => { + let [ + inner, + datafusion_expr::Expr::Literal(ScalarValue::Utf8(Some(name)), _), + ] = call.args.as_slice() + else { + return None; + }; + path.push(name.clone()); + node = inner; + } + _ => return None, + } + } + + path.reverse(); + Some(path) +} + +/// `field` without the metadata that declares how a column is written, at +/// every depth; descriptive metadata (blob markers) stays. A view is written +/// by refresh alone, and its always-nullable fields contradict declarations. +fn is_declaration(key: &str) -> bool { + key.starts_with(SCHEMA_DECLARATION_META_PREFIX) + || key == LANCE_FIELD_ID_KEY + || crate::table::computed_columns::is_declaration_key(key) +} + +fn without_declarations(field: &ArrowField) -> ArrowField { + let metadata: HashMap = field + .metadata() + .iter() + .filter(|(key, _)| !is_declaration(key)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + let strip = |child: &FieldRef| Arc::new(without_declarations(child)); + // Every Arrow variant that carries a field carries that field's metadata + // with it, so all of them are descended. + let data_type = match field.data_type() { + DataType::Struct(children) => DataType::Struct(children.iter().map(strip).collect()), + DataType::List(child) => DataType::List(strip(child)), + DataType::ListView(child) => DataType::ListView(strip(child)), + DataType::LargeList(child) => DataType::LargeList(strip(child)), + DataType::LargeListView(child) => DataType::LargeListView(strip(child)), + DataType::Map(entries, sorted) => DataType::Map(strip(entries), *sorted), + DataType::FixedSizeList(child, len) => DataType::FixedSizeList(strip(child), *len), + DataType::Union(variants, mode) => DataType::Union( + variants + .iter() + .map(|(id, child)| (id, strip(child))) + .collect(), + *mode, + ), + DataType::RunEndEncoded(run_ends, values) => { + DataType::RunEndEncoded(strip(run_ends), strip(values)) + } + other => other.clone(), + }; + ArrowField::new(field.name(), data_type, field.is_nullable()).with_metadata(metadata) +} + +fn resolve_inputs( + schema: &ArrowSchema, + expr: &datafusion_expr::Expr, + error: impl Fn(String) -> Error, +) -> Result> { + let mut inputs = Planner::column_names_in_expr(expr); + inputs.sort(); + inputs.dedup(); + for input in &inputs { + if schema.field_with_name(root(input)).is_err() { + return Err(error(format!("unknown column '{input}'"))); + } + } + Ok(inputs) +} + +/// Project the root fields of `columns`, deduplicated, in schema order. +fn project_schema(schema: &ArrowSchema, columns: &[String]) -> SchemaRef { + let roots: std::collections::HashSet<&str> = columns.iter().map(|c| root(c)).collect(); + let fields: Vec = schema + .fields() + .iter() + .filter(|f| roots.contains(f.name().as_str())) + .map(|f| f.as_ref().clone()) + .collect(); + Arc::new(ArrowSchema::new(fields)) +} + +/// A validated view declaration, ready to become a table: the projected +/// fields plus [`SOURCE_ROW_ID_COLUMN`], definition stamped in metadata. +/// Produced only by [`prepare_declaration`]. +#[derive(Clone)] +pub struct PreparedDeclaration { + schema: SchemaRef, + definition: MaterializedViewDefinition, + /// The source's own database: the only place + /// [`PreparedDeclaration::create`] will put the view, because refresh + /// resolves the recorded source name through the view's database. + database: Arc, +} + +impl std::fmt::Debug for PreparedDeclaration { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PreparedDeclaration") + .field("definition", &self.definition) + .finish_non_exhaustive() + } +} + +impl PreparedDeclaration { + /// The query the declaration records. + pub fn definition(&self) -> &MaterializedViewDefinition { + &self.definition + } + + /// Create the view table and verify it, consuming the declaration. + /// + /// The view goes in the source's own database, where refresh resolves the + /// recorded source name. Stable row ids are requested at both levels and + /// verified rather than trusted; nothing is rolled back on failure. + pub async fn create(self, name: &str) -> Result { + let empty: Vec> = + vec![]; + let reader: Box = + Box::new(arrow_array::RecordBatchIterator::new(empty, self.schema)); + let mut request = CreateTableRequest::new(name.to_string(), Box::new(reader)); + let write_params = request + .write_options + .lance_write_params + .get_or_insert_with(Default::default); + write_params.enable_stable_row_ids = true; + let store_params = write_params + .store_params + .get_or_insert_with(Default::default); + crate::connection::merge_storage_options( + store_params, + [( + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(), + "true".to_string(), + )], + ); + let table = self.database.clone().create_table(request).await?; + let table = Table::new(table, self.database); + let stable = match table.as_native() { + Some(native) => native.dataset.get().await?.manifest.uses_stable_row_ids(), + None => false, + }; + if !stable { + return Err(Error::Runtime { + message: format!( + "view '{name}' was created without stable row ids: the database \ + ignored the creation option; the table remains and is not \ + usable as a materialized view" + ), + }); + } + Ok(MaterializedView { + table, + definition: self.definition, + }) + } +} + +/// Validate a view declaration against its live source and hold what its +/// creation needs. The declaration is canonicalized through the coordinate a +/// refresh will resolve, so a handle that does not resolve back to itself is +/// rejected, as is a namespaced source. Same creation-time checks as +/// [`Connection::create_materialized_view`]. +/// +/// ```no_run +/// # #![recursion_limit = "256"] +/// # use lancedb::materialized_view::prepare_declaration; +/// # async fn declare(source: &lancedb::Table) -> Result<(), Box> { +/// let prepared = prepare_declaration( +/// source, +/// &[("id".into(), "id".into()), ("double".into(), "value * 2".into())], +/// Some("value > 0"), +/// None, +/// ) +/// .await?; +/// let view = prepared.create("doubles").await?; +/// # Ok(()) +/// # } +/// ``` +pub async fn prepare_declaration( + source: &Table, + projections: &[(String, String)], + filter: Option<&str>, + limit: Option, +) -> Result { + let Some(caller_native) = source.as_native() else { + return Err(Error::NotSupported { + message: "materialized views are supported only on local databases".into(), + }); + }; + // The definition records the source by bare name; any other source + // form would be recorded as a name its refresh cannot resolve. + if !source.namespace().is_empty() { + return Err(Error::NotSupported { + message: format!( + "a namespaced source cannot be recorded in a view definition; \ + '{}' must be a root-namespace table", + source.name() + ), + }); + } + let database = source + .database_opt() + .ok_or_else(|| Error::InvalidInput { + message: "the source was not opened through a database connection".into(), + })? + .clone(); + + // Canonicalize: resolve the recorded coordinate exactly the way a + // refresh will, and plan from what it reaches. A handle that does not + // resolve back to itself must not be declared under this name. + let resolved = database + .open_table(OpenTableRequest { + name: source.name().to_string(), + namespace_path: vec![], + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await?; + let resolved = Table::new(resolved, database.clone()); + let Some(native) = resolved.as_native() else { + return Err(Error::NotSupported { + message: "materialized views are supported only on local databases".into(), + }); + }; + let caller_uri = caller_native.dataset.get().await?.uri().to_string(); + let resolved_uri = native.dataset.get().await?.uri().to_string(); + if caller_uri != resolved_uri { + return Err(Error::InvalidInput { + message: format!( + "the source handle does not resolve to itself through its \ + database: '{}' resolves to '{resolved_uri}', but the handle \ + reads '{caller_uri}'", + source.name() + ), + }); + } + if !native.dataset.get().await?.manifest.uses_stable_row_ids() { + return Err(Error::InvalidInput { + message: format!( + "materialized views require stable row ids on the source table; \ + create '{}' with storage option new_table_enable_stable_row_ids=true", + source.name() + ), + }); + } + let source_schema = resolved.schema().await?; + let source_metadata = source_schema.metadata().clone(); + let (definition, mut fields, lineage) = plan( + source_schema.clone(), + resolved.name(), + projections, + filter, + limit, + )?; + fields.push(ArrowField::new( + SOURCE_ROW_ID_COLUMN, + DataType::UInt64, + false, + )); + // Only column-describing metadata comes along: structural declarations + // describe how a table is written, and a view is written by refresh alone. + let mut metadata: HashMap = HashMap::new(); + if let Some(raw) = source_metadata.get(EMBEDDING_FUNCTIONS_META_KEY) + && let Some(rewritten) = embedding_config_for_view(raw, &lineage) + { + metadata.insert(EMBEDDING_FUNCTIONS_META_KEY.to_string(), rewritten); + } + if let Some(raw) = source_metadata.get(COLUMN_DEFINITIONS_META_KEY) + && let Some(rewritten) = column_definitions_for_view(raw, &source_schema, &fields, &lineage) + { + metadata.insert(COLUMN_DEFINITIONS_META_KEY.to_string(), rewritten); + } + metadata.insert( + DEFINITION_META_KEY.to_string(), + definition_to_metadata(&definition)?, + ); + Ok(PreparedDeclaration { + schema: Arc::new(ArrowSchema::new_with_metadata(fields, metadata)), + definition, + database, + }) +} + +/// One row of [`Connection::list_materialized_views`]: a view's name and its +/// definition kind, which may be one this version cannot refresh. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MaterializedViewEntry { + /// Name of the view's table. + pub name: String, + /// The view's definition as stored. + pub kind: MaterializedViewKind, +} + +/// Materialized views are local-only; refuse a remote connection before any +/// request is made. +fn ensure_local(connection: &Connection) -> Result<()> { + if connection.uri().starts_with("db://") { + return Err(Error::NotSupported { + message: "materialized views are supported only on local databases".into(), + }); + } + Ok(()) +} + +/// Builds a materialized view. Created by +/// [`Connection::create_materialized_view`]. +pub struct CreateMaterializedViewBuilder { + connection: Connection, + name: String, + source: String, + projections: Vec<(String, String)>, + filter: Option, + limit: Option, +} + +impl CreateMaterializedViewBuilder { + pub(crate) fn new(connection: Connection, name: String, source: String) -> Self { + Self { + connection, + name, + source, + projections: Vec::new(), + filter: None, + limit: None, + } + } + + /// The view's columns, as `(name, SQL expression)` pairs. Not calling + /// this selects every source column, expanded at creation time. + pub fn select( + mut self, + columns: impl IntoIterator, impl Into)>, + ) -> Self { + self.projections = columns + .into_iter() + .map(|(output, expression)| (output.into(), expression.into())) + .collect(); + self + } + + /// Only source rows matching the SQL predicate appear in the view. + pub fn only_if(mut self, filter: impl Into) -> Self { + self.filter = Some(filter.into()); + self + } + + /// Cap the view at `limit` rows, in materialization order. + pub fn limit(mut self, limit: u64) -> Self { + self.limit = Some(limit); + self + } + + /// Create the view: an empty table carrying the definition; refresh + /// computes the rows. The source must keep stable row ids -- they hold + /// provenance across compaction, and cannot be enabled later. + pub async fn execute(self) -> Result { + ensure_local(&self.connection)?; + let source = self.connection.open_table(&self.source).execute().await?; + let prepared = prepare_declaration( + &source, + &self.projections, + self.filter.as_deref(), + self.limit, + ) + .await?; + prepared.create(&self.name).await + } +} + +/// A handle on a materialized view: the view table plus its parsed definition. +#[derive(Debug, Clone)] +pub struct MaterializedView { + table: Table, + definition: MaterializedViewDefinition, +} + +impl MaterializedView { + /// Interpret `table` as a materialized view: [`Error::NotAMaterializedView`] + /// for a plain table, [`Error::NotSupported`] for a kind this version + /// cannot refresh. + pub async fn from_table(table: Table) -> Result { + // Same local-only boundary the connection-level entry points hold, + // applied before the schema read so a remote table costs no request. + if table.as_native().is_none() { + return Err(Error::NotSupported { + message: "materialized views are supported only on local databases".into(), + }); + } + let schema = table.schema().await?; + match materialized_view_kind(schema.metadata())? { + Some(MaterializedViewKind::Select(definition)) => Ok(Self { table, definition }), + Some(MaterializedViewKind::Unrecognized { kind }) => Err(Error::NotSupported { + message: format!( + "materialized view '{}' is defined by '{kind}', which this version of \ + lancedb cannot refresh", + table.name() + ), + }), + None => Err(Error::NotAMaterializedView { + name: table.name().to_string(), + }), + } + } + + /// The view, as the table it is. Queries, indexes and search all apply. + pub fn table(&self) -> &Table { + &self.table + } + + /// The view's table name. + pub fn name(&self) -> &str { + self.table.name() + } + + /// The query that defines the view. + pub fn definition(&self) -> &MaterializedViewDefinition { + &self.definition + } +} + +impl Connection { + /// Define a materialized view named `name` over `source`. + /// + /// The view is created empty, with the definition recorded in its schema + /// metadata; refresh computes the rows. Local databases only. + /// + /// ```no_run + /// # use lancedb::Connection; + /// # async fn create(conn: &Connection) -> Result<(), Box> { + /// let view = conn + /// .create_materialized_view("loud_adults", "people") + /// .select([("name", "upper(name)"), ("age", "age")]) + /// .only_if("age >= 18") + /// .execute() + /// .await?; + /// println!("{}", view.definition().source_table); + /// # Ok(()) + /// # } + /// ``` + pub fn create_materialized_view( + &self, + name: impl Into, + source: impl Into, + ) -> CreateMaterializedViewBuilder { + CreateMaterializedViewBuilder::new(self.clone(), name.into(), source.into()) + } + + /// Open the materialized view named `name`. + pub async fn open_materialized_view( + &self, + name: impl Into, + ) -> Result { + ensure_local(self)?; + let table = self.open_table(name).execute().await?; + MaterializedView::from_table(table).await + } + + /// The materialized views in this database, unrefreshable kinds included. + /// Costs a table open per table; one that cannot be opened is skipped + /// rather than failing the listing. + pub async fn list_materialized_views(&self) -> Result> { + ensure_local(self)?; + let names = self.table_names().execute().await?; + let mut views = Vec::new(); + for name in names { + let Ok(table) = self.open_table(&name).execute().await else { + continue; + }; + let schema = table.schema().await?; + if let Some(kind) = materialized_view_kind(schema.metadata())? { + views.push(MaterializedViewEntry { name, kind }); + } + } + Ok(views) + } +} + +#[cfg(test)] +mod tests { + use arrow_array::record_batch; + + use super::*; + use crate::connect; + use crate::table::WriteOptions; + + async fn people_db() -> Connection { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!( + ("name", Utf8, ["ada", "grace", "alan"]), + ("age", Int32, [36, 85, 41]) + ) + .unwrap(); + conn.create_table("people", batch) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + conn + } + + /// Sources must keep stable row ids; see the create-time gate. + pub(super) fn stable_row_ids() -> WriteOptions { + WriteOptions { + lance_write_params: Some(lance::dataset::WriteParams { + enable_stable_row_ids: true, + ..Default::default() + }), + } + } + + /// The error a doomed declaration against `people` produces. + async fn declare_err( + cfg: impl FnOnce(CreateMaterializedViewBuilder) -> CreateMaterializedViewBuilder, + ) -> Error { + let conn = people_db().await; + cfg(conn.create_materialized_view("bad", "people")) + .execute() + .await + .unwrap_err() + } + + #[tokio::test] + async fn test_create_records_the_definition() { + let conn = people_db().await; + let view = conn + .create_materialized_view("adults", "people") + .select([("name", "name"), ("shout", "upper(name)")]) + .only_if("age >= 18") + .limit(10) + .execute() + .await + .unwrap(); + + assert_eq!(view.name(), "adults"); + assert_eq!( + view.definition(), + &MaterializedViewDefinition { + source_table: "people".into(), + projections: vec![ + ViewProjection { + output: "name".into(), + expression: "name".into() + }, + ViewProjection { + output: "shout".into(), + expression: "upper(name)".into() + }, + ], + filter: Some("age >= 18".into()), + limit: Some(10), + inputs: vec!["age".into(), "name".into()], + } + ); + + // The definition round-trips off the stored schema, not the handle. + let reopened = conn.open_materialized_view("adults").await.unwrap(); + assert_eq!(reopened.definition(), view.definition()); + } + + #[tokio::test] + async fn test_view_schema_is_derived_from_the_query() { + let conn = people_db().await; + let view = conn + .create_materialized_view("shapes", "people") + .select([("shout", "upper(name)"), ("next_age", "age + 1")]) + .execute() + .await + .unwrap(); + + let schema = view.table().schema().await.unwrap(); + assert_eq!( + schema.field_with_name("shout").unwrap().data_type(), + &DataType::Utf8 + ); + assert_eq!( + schema.field_with_name("next_age").unwrap().data_type(), + &DataType::Int32 + ); + assert_eq!( + schema + .field_with_name(SOURCE_ROW_ID_COLUMN) + .unwrap() + .data_type(), + &DataType::UInt64 + ); + assert_eq!(view.table().count_rows(None).await.unwrap(), 0); + } + + /// No projection selects every source column, expanded now: the schema + /// captured at creation is the definition. + #[tokio::test] + async fn test_default_projection_captures_the_source_schema() { + let conn = people_db().await; + let view = conn + .create_materialized_view("copy", "people") + .execute() + .await + .unwrap(); + assert_eq!( + view.definition() + .projections + .iter() + .map(|p| p.output.as_str()) + .collect::>(), + vec!["name", "age"] + ); + assert_eq!(view.definition().inputs, vec!["age", "name"]); + } + + #[tokio::test] + async fn test_unknown_column_fails_at_create_time() { + let conn = people_db().await; + let err = conn + .create_materialized_view("bad", "people") + .select([("x", "missing + 1")]) + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidExpression { column, .. } if column == "x")); + let names = conn.table_names().execute().await.unwrap(); + assert!(!names.contains(&"bad".to_string())); + } + + #[tokio::test] + async fn test_unknown_filter_column_fails_at_create_time() { + let err = declare_err(|b| b.only_if("missing > 1")).await; + assert!(matches!(err, Error::InvalidInput { message } if message.contains("missing"))); + } + + #[tokio::test] + async fn test_duplicate_output_is_rejected() { + let err = declare_err(|b| b.select([("dup", "age"), ("dup", "age + 1")])).await; + assert!(matches!(err, Error::ColumnAlreadyExists { name } if name == "dup")); + } + + #[tokio::test] + async fn test_reserved_output_name_is_rejected() { + let err = declare_err(|b| b.select([(SOURCE_ROW_ID_COLUMN, "age")])).await; + assert!(matches!(err, Error::InvalidInput { message } if message.contains("reserved"))); + } + + #[tokio::test] + async fn test_missing_source_fails() { + let conn = connect("memory://").execute().await.unwrap(); + let err = conn + .create_materialized_view("v", "nope") + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::TableNotFound { .. })); + } + + /// Provenance has to survive source compactions and updates, and stable + /// row ids cannot be enabled after a table exists -- so the requirement + /// is checked at the last moment the caller can still act on it. + #[tokio::test] + async fn test_source_without_stable_row_ids_is_refused() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("x", Int32, [1, 2])).unwrap(); + conn.create_table("plain", batch).execute().await.unwrap(); + + let err = conn + .create_materialized_view("v", "plain") + .execute() + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("stable row ids")) + ); + assert!( + !conn + .table_names() + .execute() + .await + .unwrap() + .contains(&"v".to_string()) + ); + } + + #[tokio::test] + async fn test_name_collision_fails() { + let conn = people_db().await; + let err = conn + .create_materialized_view("people", "people") + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::TableAlreadyExists { .. })); + } + + #[tokio::test] + async fn test_a_plain_table_is_not_a_view() { + let conn = people_db().await; + let table = conn.open_table("people").execute().await.unwrap(); + let err = MaterializedView::from_table(table).await.unwrap_err(); + assert!(matches!(err, Error::NotAMaterializedView { name } if name == "people")); + + let err = conn.open_materialized_view("people").await.unwrap_err(); + assert!(matches!(err, Error::NotAMaterializedView { .. })); + } + + /// The reason the kind is tagged: a definition written by a newer version + /// reads back as a view this one cannot refresh, not as a plain table. + #[tokio::test] + async fn test_unrecognized_kind_is_refused_by_name() { + let conn = people_db().await; + conn.create_materialized_view("v", "people") + .execute() + .await + .unwrap(); + let table = conn.open_table("v").execute().await.unwrap(); + table + .as_native() + .unwrap() + .replace_schema_metadata(HashMap::from([( + DEFINITION_META_KEY.to_string(), + r#"{"kind": "join"}"#.to_string(), + )])) + .await + .unwrap(); + + let err = conn.open_materialized_view("v").await.unwrap_err(); + assert!(matches!(err, Error::NotSupported { message } if message.contains("join"))); + } + + #[tokio::test] + async fn test_list_reports_views_and_only_views() { + let conn = people_db().await; + conn.create_materialized_view("adults", "people") + .only_if("age >= 18") + .execute() + .await + .unwrap(); + + let views = conn.list_materialized_views().await.unwrap(); + assert_eq!( + views.iter().map(|v| v.name.as_str()).collect::>(), + vec!["adults"] + ); + let MaterializedViewKind::Select(definition) = &views[0].kind else { + panic!("expected a select view"); + }; + assert_eq!(definition.filter.as_deref(), Some("age >= 18")); + } + + /// The creation option outranks a connection configured to create + /// unstable tables: the view still gets stable row ids, on the same + /// store (no fork -- the table must be reachable through the + /// connection afterwards). + #[tokio::test] + async fn test_view_is_stable_despite_connection_override() { + let conn = connect("memory://") + .storage_options([("new_table_enable_stable_row_ids", "false")]) + .execute() + .await + .unwrap(); + let batch = record_batch!(("x", Int32, [1])).unwrap(); + conn.create_table("src", batch) + .storage_option("new_table_enable_stable_row_ids", "true") + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap(); + let stable = view + .table() + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .manifest + .uses_stable_row_ids(); + assert!(stable); + conn.open_materialized_view("v").await.unwrap(); + } + + /// A committed filter has to be usable as a predicate. + #[tokio::test] + async fn test_non_boolean_filter_is_rejected() { + let err = declare_err(|b| b.only_if("age + 1")).await; + assert!(matches!(err, Error::InvalidInput { message } if message.contains("boolean"))); + } + + /// Nested references stay dotted paths; resolution is by root field. + #[tokio::test] + async fn test_struct_columns_can_be_declared() { + use arrow_array::{ArrayRef, Int32Array, StructArray}; + + let conn = connect("memory://").execute().await.unwrap(); + let ages = StructArray::from(vec![( + Arc::new(ArrowField::new("age", DataType::Int32, false)), + Arc::new(Int32Array::from(vec![36, 17])) as ArrayRef, + )]); + let batch = + arrow_array::RecordBatch::try_from_iter(vec![("metadata", Arc::new(ages) as ArrayRef)]) + .unwrap(); + conn.create_table("people", batch) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("ages", "people") + .select([("age", "metadata.age")]) + .only_if("metadata.age >= 18") + .execute() + .await + .unwrap(); + assert_eq!(view.definition().inputs, vec!["metadata.age"]); + let schema = view.table().schema().await.unwrap(); + assert_eq!( + schema.field_with_name("age").unwrap().data_type(), + &DataType::Int32 + ); + } + + /// A newer-kind view must not disappear from the listing. + #[tokio::test] + async fn test_unrecognized_kind_is_listed_with_its_kind() { + let conn = people_db().await; + conn.create_materialized_view("v", "people") + .execute() + .await + .unwrap(); + let table = conn.open_table("v").execute().await.unwrap(); + table + .as_native() + .unwrap() + .replace_schema_metadata(HashMap::from([( + DEFINITION_META_KEY.to_string(), + r#"{"kind": "join"}"#.to_string(), + )])) + .await + .unwrap(); + + let views = conn.list_materialized_views().await.unwrap(); + assert_eq!(views.len(), 1); + assert_eq!(views[0].name, "v"); + assert_eq!( + views[0].kind, + MaterializedViewKind::Unrecognized { + kind: "join".into() + } + ); + } + + /// Remote connections are refused before any request is made. + #[cfg(feature = "remote")] + #[tokio::test] + async fn test_remote_connection_is_refused_up_front() { + let conn = connect("db://nowhere") + .api_key("sk_test") + .region("us-east-1") + .execute() + .await + .unwrap(); + let err = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. })); + let err = conn.open_materialized_view("v").await.unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. })); + let err = conn.list_materialized_views().await.unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. })); + } + + /// A definition must evaluate identically across refreshes; anything + /// less makes incremental maintenance a mix of evaluations. + #[tokio::test] + async fn test_volatile_and_unstable_expressions_are_rejected() { + let conn = people_db().await; + for expression in [ + "random()", + "now()", + "version()", + "arrow_typeof(age)", + "arrow_metadata(age, 'k')", + ] { + let err = conn + .create_materialized_view("bad", "people") + .select([("x", expression)]) + .execute() + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidExpression { message, .. } + if message.contains("not immutable")), + "{expression} was not rejected" + ); + } + for filter in ["age > random() * 100", "age >= 0 and now() is not null"] { + let err = conn + .create_materialized_view("bad", "people") + .only_if(filter) + .execute() + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("not immutable")), + "{filter} was not rejected" + ); + } + } + + /// A column projected as itself stays the column it was: blob discovery + /// and the blob APIs key off field metadata, which a bare rebuild of the + /// field would drop. + #[tokio::test] + async fn test_identity_projection_keeps_field_metadata() { + let conn = connect("memory://").execute().await.unwrap(); + let schema = Arc::new(ArrowSchema::new_with_metadata( + vec![ + ArrowField::new("id", DataType::Int32, true), + crate::blob("payload", true), + ], + HashMap::new(), + )); + conn.create_empty_table("src", schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap(); + let view_schema = view.table().schema().await.unwrap(); + + let payload = view_schema.field_with_name("payload").unwrap(); + assert!( + crate::blob::is_blob(payload), + "default projection dropped the blob marker: {:?}", + payload.metadata() + ); + assert_eq!( + view.table().blob_columns().await.unwrap(), + vec!["payload".to_string()], + "blob discovery no longer finds the projected column" + ); + assert!(view_schema.metadata().contains_key(DEFINITION_META_KEY)); + // Structural declarations describe how a table is written; a view is + // written by refresh, and its fields are always nullable. + assert!(!view_schema.metadata().contains_key("lance:primary_key")); + + // A computed column is a new value and carries no source metadata. + let computed = conn + .create_materialized_view("c", "src") + .select([("payload", "payload"), ("n", "id + 1")]) + .execute() + .await + .unwrap(); + let computed_schema = computed.table().schema().await.unwrap(); + assert!(crate::blob::is_blob( + computed_schema.field_with_name("payload").unwrap() + )); + assert!( + computed_schema + .field_with_name("n") + .unwrap() + .metadata() + .is_empty() + ); + } + + /// A nested column projected straight through is still that column, and a + /// declaration buried in a struct child binds as hard as one on top. + #[tokio::test] + async fn test_nested_projection_metadata_and_declarations() { + let conn = connect("memory://").execute().await.unwrap(); + let payload = crate::blob("payload", true).with_metadata(HashMap::from([ + ("lance-encoding:blob".to_string(), "true".to_string()), + ( + "lance-schema:unenforced-primary-key".to_string(), + "0".to_string(), + ), + ])); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("meta", DataType::Struct(vec![payload].into()), true), + ])); + conn.create_empty_table("src", schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + // A nested path is a direct projection: the leaf's metadata comes with + // it, so the blob stays a blob rather than a plain struct. + let lifted = conn + .create_materialized_view("lifted", "src") + .select([("payload", "meta.payload")]) + .execute() + .await + .unwrap(); + let field = lifted.table().schema().await.unwrap(); + let field = field.field_with_name("payload").unwrap().clone(); + assert_eq!( + field.metadata().get("lance-encoding:blob"), + Some(&"true".to_string()), + "nested projection lost the leaf's metadata" + ); + assert!( + !field + .metadata() + .contains_key("lance-schema:unenforced-primary-key"), + "a structural declaration rode along" + ); + + // Projecting the struct whole must not carry the child's declaration + // out to a view whose fields are nullable. + let whole = conn + .create_materialized_view("whole", "src") + .select([("meta", "meta")]) + .execute() + .await + .unwrap(); + let schema = whole.table().schema().await.unwrap(); + let DataType::Struct(children) = schema.field_with_name("meta").unwrap().data_type() else { + panic!("meta is not a struct"); + }; + let child = children.iter().find(|c| c.name() == "payload").unwrap(); + assert!( + !child + .metadata() + .contains_key("lance-schema:unenforced-primary-key"), + "a nested declaration survived: {:?}", + child.metadata() + ); + assert_eq!( + child.metadata().get("lance-encoding:blob"), + Some(&"true".to_string()) + ); + } + + /// A map's entries are fields like any other, and a declaration on one + /// binds the view's writes just as hard as one on top. + #[tokio::test] + async fn test_map_declarations_are_stripped() { + let conn = connect("memory://").execute().await.unwrap(); + let value = + ArrowField::new("value", DataType::Utf8, false).with_metadata(HashMap::from([( + "lance-schema:unenforced-clustering-key:position".to_string(), + "1".to_string(), + )])); + let entries = ArrowField::new( + "entries", + DataType::Struct(vec![ArrowField::new("key", DataType::Utf8, false), value].into()), + false, + ); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "props", + DataType::Map(Arc::new(entries), false), + true, + )])); + conn.create_empty_table("src", schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("view", "src") + .execute() + .await + .unwrap(); + let schema = view.table().schema().await.unwrap(); + let DataType::Map(entries, _) = schema.field_with_name("props").unwrap().data_type() else { + panic!("props is not a map"); + }; + let DataType::Struct(children) = entries.data_type() else { + panic!("map entries are not a struct"); + }; + let value = children.iter().find(|c| c.name() == "value").unwrap(); + assert!( + !value + .metadata() + .contains_key("lance-schema:unenforced-clustering-key:position"), + "a declaration survived inside a map: {:?}", + value.metadata() + ); + } + + /// A computed column is declared by field metadata. Projecting one -- + /// as itself or under an alias -- must carry its description without its + /// declaration, which the target table would reject as foreign. + #[tokio::test] + async fn test_view_over_a_computed_column() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("id", Int32, [1, 2])).unwrap(); + let source = conn + .create_table("src", batch) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + source + .add_columns() + .computed("doubled", "id * 2") + .execute() + .await + .unwrap(); + + // Default projection reaches the computed column too. + let whole = conn + .create_materialized_view("whole", "src") + .execute() + .await + .unwrap(); + let schema = whole.table().schema().await.unwrap(); + let field = schema.field_with_name("doubled").unwrap(); + assert!( + !field + .metadata() + .keys() + .any(|k| k.starts_with("computed_column")), + "a computed-column declaration rode along: {:?}", + field.metadata() + ); + + // And under an alias. + conn.create_materialized_view("aliased", "src") + .select([("twice", "doubled")]) + .execute() + .await + .unwrap(); + } + + /// Embedding configuration names columns. It comes along only for the + /// columns a view actually projects, under the names the view gives them. + #[tokio::test] + async fn test_embedding_config_follows_the_projection() { + let config = r#"[{"name":"f","model":{},"source_column":"text","vector_column":"vec"}]"#; + let conn = connect("memory://").execute().await.unwrap(); + let schema = Arc::new(ArrowSchema::new_with_metadata( + vec![ + ArrowField::new("text", DataType::Utf8, true), + ArrowField::new("vec", DataType::Float32, true), + ], + HashMap::from([("embedding_functions".to_string(), config.to_string())]), + )); + conn.create_empty_table("src", schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let carried = |view: &MaterializedView| { + let view = view.table().clone(); + async move { + view.schema() + .await + .unwrap() + .metadata() + .get("embedding_functions") + .cloned() + } + }; + + // Both columns projected as themselves: kept as it stands. + let whole = conn + .create_materialized_view("whole", "src") + .execute() + .await + .unwrap(); + let kept = carried(&whole).await.expect("config dropped"); + assert!(kept.contains(r#""source_column":"text""#), "{kept}"); + assert!(kept.contains(r#""vector_column":"vec""#), "{kept}"); + + // Only the source column: the configuration names a vector column the + // view does not have, so it describes nothing and goes. + let partial = conn + .create_materialized_view("partial", "src") + .select([("text", "text")]) + .execute() + .await + .unwrap(); + assert_eq!(carried(&partial).await, None); + + // Renamed: the configuration follows the names the view uses. + let renamed = conn + .create_materialized_view("renamed", "src") + .select([("body", "text"), ("embedding", "vec")]) + .execute() + .await + .unwrap(); + let remapped = carried(&renamed).await.expect("config dropped"); + assert!(remapped.contains(r#""source_column":"body""#), "{remapped}"); + assert!( + remapped.contains(r#""vector_column":"embedding""#), + "{remapped}" + ); + + // The Node bindings spell the same configuration in camelCase, and + // the Rust definition names the destination `dest_column`. + for (config, source_key, dest_key) in [ + ( + r#"[{"name":"f","model":{},"sourceColumn":"text","vectorColumn":"vec"}]"#, + "sourceColumn", + "vectorColumn", + ), + ( + r#"[{"name":"f","model":{},"source_column":"text","dest_column":"vec"}]"#, + "source_column", + "dest_column", + ), + ] { + let schema = Arc::new(ArrowSchema::new_with_metadata( + vec![ + ArrowField::new("text", DataType::Utf8, true), + ArrowField::new("vec", DataType::Float32, true), + ], + HashMap::from([("embedding_functions".to_string(), config.to_string())]), + )); + let name = format!("src_{source_key}"); + conn.create_empty_table(&name, schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + let view = conn + .create_materialized_view(format!("v_{source_key}"), &name) + .select([("body", "text"), ("embedding", "vec")]) + .execute() + .await + .unwrap(); + let carried = carried(&view).await.expect("config dropped"); + assert!( + carried.contains(&format!(r#""{source_key}":"body""#)), + "{carried}" + ); + assert!( + carried.contains(&format!(r#""{dest_key}":"embedding""#)), + "{carried}" + ); + } + + // A computed column is not the source column under another name. + let computed = conn + .create_materialized_view("computed", "src") + .select([("body", "upper(text)"), ("embedding", "vec")]) + .execute() + .await + .unwrap(); + assert_eq!(carried(&computed).await, None); + + // One column projected twice is two columns in the view, and the + // configuration has to describe both rather than whichever came last. + let twice = conn + .create_materialized_view("twice", "src") + .select([("body", "text"), ("a", "vec"), ("b", "vec")]) + .execute() + .await + .unwrap(); + let carried = carried(&twice).await.expect("config dropped"); + let entries: Vec = serde_json::from_str(&carried).unwrap(); + let mut vectors: Vec<&str> = entries + .iter() + .filter_map(|e| e["vector_column"].as_str()) + .collect(); + vectors.sort_unstable(); + assert_eq!(vectors, ["a", "b"], "{carried}"); + } + + /// The native Rust producer records embeddings as column definitions + /// rather than as `embedding_functions`, and a query embeds its own text + /// through them. They are positional, so the view's list covers every one + /// of its fields. + #[tokio::test] + async fn test_native_column_definitions_follow_the_projection() { + let conn = connect("memory://").execute().await.unwrap(); + let rich = crate::table::TableDefinition::new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("text", DataType::Utf8, true), + ArrowField::new("vector", DataType::Float32, true), + ])), + vec![ + ColumnDefinition { + kind: ColumnKind::Physical, + }, + ColumnDefinition { + kind: ColumnKind::Embedding(EmbeddingDefinition::new( + "text", + "model", + Some("vector"), + )), + }, + ], + ) + .into_rich_schema(); + conn.create_empty_table("src", rich) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("view", "src") + .select([("body", "text"), ("embedding", "vector")]) + .execute() + .await + .unwrap(); + let schema = view.table().schema().await.unwrap(); + let raw = schema + .metadata() + .get(COLUMN_DEFINITIONS_META_KEY) + .expect("the view dropped the native column definitions"); + let definitions: Vec = serde_json::from_str(raw).unwrap(); + assert_eq!( + definitions.len(), + schema.fields().len(), + "column definitions are positional" + ); + let ColumnKind::Embedding(embedding) = &definitions[1].kind else { + panic!("the embedding column came back physical: {raw}"); + }; + assert_eq!(embedding.source_column, "body"); + assert_eq!(embedding.dest_column.as_deref(), Some("embedding")); + assert_eq!(embedding.embedding_name, "model"); + assert!(matches!(definitions[0].kind, ColumnKind::Physical)); + assert!(matches!(definitions[2].kind, ColumnKind::Physical)); + + // Without the column the function reads, the view cannot recompute + // the embedding, so it carries no definition for it. + let partial = conn + .create_materialized_view("partial", "src") + .select([("embedding", "vector")]) + .execute() + .await + .unwrap(); + assert_eq!( + partial + .table() + .schema() + .await + .unwrap() + .metadata() + .get(COLUMN_DEFINITIONS_META_KEY), + None + ); + } + + /// A scan takes the cap as i64, so a larger one is refused where it is + /// declared rather than at the refresh that cannot run it. What a cap of + /// zero means is a refresh question, tested there. + #[tokio::test] + async fn test_limit_above_i64_max_is_refused_at_creation() { + let conn = people_db().await; + let err = conn + .create_materialized_view("too_big", "people") + .limit(i64::MAX as u64 + 1) + .execute() + .await + .unwrap_err(); + assert!( + matches!(&err, Error::InvalidInput { message } if message.contains("exceeds the maximum")), + "got {err:?}" + ); + + // The boundary itself is accepted. + conn.create_materialized_view("at_max", "people") + .limit(i64::MAX as u64) + .execute() + .await + .unwrap(); + } + + #[tokio::test] + async fn test_drop_is_drop_table() { + let conn = people_db().await; + conn.create_materialized_view("v", "people") + .execute() + .await + .unwrap(); + conn.drop_table("v", &[]).await.unwrap(); + assert!(conn.list_materialized_views().await.unwrap().is_empty()); + } + + /// The public declaration contract: prepare validates the source and + /// create consumes the declaration into a verified view table; a + /// source that cannot anchor refresh and a target outside the + /// source's database are both refused. + #[tokio::test] + async fn prepare_and_create_bind_the_declaration_lifecycle() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("id", Int32, [1, 2]), ("value", Int32, [3, 4])).unwrap(); + let source = conn + .create_table("src", batch.clone()) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let projections = [ + ("id".to_string(), "id".to_string()), + ("double".to_string(), "value * 2".to_string()), + ]; + let prepared = prepare_declaration(&source, &projections, Some("value > 0"), None) + .await + .unwrap(); + assert_eq!(prepared.definition().source_table, "src"); + + let view = prepared.create("v").await.unwrap(); + let schema = view.table().schema().await.unwrap(); + let names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert_eq!(names, ["id", "double", SOURCE_ROW_ID_COLUMN]); + assert!(schema.metadata().contains_key(DEFINITION_META_KEY)); + + // The same call rejects a source without stable row ids, so an + // external creation path cannot skip the check. + conn.create_table("plain", batch).execute().await.unwrap(); + let plain = conn.open_table("plain").execute().await.unwrap(); + let err = prepare_declaration(&plain, &[], None, None) + .await + .unwrap_err(); + assert!(err.to_string().contains("stable row ids"), "{err}"); + + // A handle whose location does not resolve back through its name is + // refused: the definition would record a name reaching other data. + let plain_uri = plain + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .uri() + .to_string(); + let masquerade = conn + .open_table("src") + .location(plain_uri) + .execute() + .await + .unwrap(); + let err = prepare_declaration(&masquerade, &[], None, None) + .await + .unwrap_err(); + assert!( + err.to_string().contains("does not resolve to itself"), + "{err}" + ); + + // A table created at a custom location is refused outright: its + // recorded name reaches nothing at the database root, so the + // canonical reopen fails before any URI comparison. + let custom = conn + .create_table( + "custom_loc", + record_batch!(("id", Int32, [1, 2]), ("value", Int32, [3, 4])).unwrap(), + ) + .location("memory://elsewhere/custom_loc") + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + let err = prepare_declaration(&custom, &[], None, None) + .await + .unwrap_err(); + assert!(err.to_string().contains("custom_loc"), "{err}"); + + // A namespaced source cannot be recorded in the definition: the + // bare name refresh resolves would reach a different table or none. + let namespaced = crate::table::NativeTable::create( + "memory://ns_src", + "ns_src", + vec!["ns".to_string()], + Box::new(arrow_array::RecordBatchIterator::new( + vec![], + std::sync::Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new( + "id", + arrow_schema::DataType::Int32, + true, + )])), + )) as Box, + None, + None, + None, + None, + std::collections::HashSet::new(), + ) + .await + .unwrap(); + let namespaced = Table::new(std::sync::Arc::new(namespaced), conn.database().clone()); + let err = prepare_declaration(&namespaced, &[], None, None) + .await + .unwrap_err(); + assert!(err.to_string().contains("namespaced source"), "{err}"); + } +} diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index f98d4c8dd..6c446907d 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -10410,6 +10410,20 @@ mod tests { ); } + #[tokio::test] + async fn test_materialized_view_refused_without_a_request() { + // Materialized views are local-only. The table-level entry the + // bindings use must refuse a remote table before reading its schema, + // so the panicking handler is the assertion. + let table = Table::new_with_handler("my_table", |request| -> http::Response { + panic!("unexpected request: {}", request.url().path()) + }); + let err = crate::MaterializedView::from_table(table) + .await + .unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. }), "got {err:?}"); + } + #[tokio::test] async fn test_create_branch_empty_name_rejected_client_side() { use lance::dataset::refs::Ref; diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 096d60345..ca9ed8f26 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -1066,6 +1066,11 @@ impl Table { self.database.as_ref().unwrap() } + /// The database this handle was opened through, when it was. + pub fn database_opt(&self) -> Option<&Arc> { + self.database.as_ref() + } + pub fn embedding_registry(&self) -> &Arc { &self.embedding_registry } diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index a3f3da92f..2715a163c 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -193,7 +193,7 @@ fn declared_expression(dataset: &Dataset, column: &str) -> Result { /// /// Lance's dialect delimits with backticks, so a double-quoted name would /// parse as a string literal rather than a column. -fn quote_identifier(name: &str) -> String { +pub(crate) fn quote_identifier(name: &str) -> String { format!("`{}`", name.replace('`', "``")) } From 9e8f1c1a6dffeafcdebf218d4f9e7212f90917f6 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:31:34 -0700 Subject: [PATCH 04/31] fix(python): expose FTS build memory limits (#3796) ## Summary - expose `memory_limit` and `num_workers` on the Python FTS configuration for local builds - forward both build-only settings to the Lance inverted-index builder - add an end-to-end regression proving the configured memory budget reaches the native build ## Root cause LanceDB 0.26.1 pinned Lance 1.0.1. That Lance version used an FTS partition-merge path whose retained data made memory grow with merge progress on very large indexes. Upstream Lance [#5754](https://github.com/lance-format/lance/pull/5754) changed partition merging to stream its inputs, reducing peak memory by about 25%. Lance [#6174](https://github.com/lance-format/lance/pull/6174) then removed the old merge phase, compressed posting lists during construction, reduced indexing memory by about 60%, and introduced a total build `memory_limit` for bounded workers. Current `main` pins Lance 11.0.0-beta.3, which contains those architectural fixes. This PR does not duplicate or claim the upstream leak fix; it addresses the remaining Python API gap. ## This repair LanceDB Python did not expose the native FTS builder resource controls. `memory_limit` now sets the total local-build budget in MiB, divided among effective workers, and `num_workers` controls build parallelism. Both are build-only settings and do not affect remote builds or persisted index configuration. ## Validation - `cargo check --quiet --features remote --tests --examples` - `cargo fmt --all` - `uv run --project python --extra tests --extra dev ruff check .` - `uv run --project python --extra tests --extra dev ruff format --check python/python/lancedb/index.py python/python/tests/test_fts.py` - `uv run --project python --extra tests pytest python/tests/test_fts.py -q` (51 passed) Fixes #2923 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/Cargo.toml | 9 +++-- python/pyproject.toml | 2 +- python/python/lancedb/index.py | 11 +++++++ python/python/tests/test_fts.py | 8 +++++ python/src/index.rs | 58 ++++++++++++++++++++++++++++++++- 5 files changed, 81 insertions(+), 7 deletions(-) diff --git a/python/Cargo.toml b/python/Cargo.toml index 6181f704f..ddc13b69f 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -26,7 +26,9 @@ lance-namespace-impls.workspace = true lance-io.workspace = true env_logger.workspace = true log.workspace = true -pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "chrono"] } +# Maturin enables extension-module mode for Python builds. Keeping it out of +# Cargo features lets Rust unit tests link against libpython. +pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] } chrono.workspace = true pyo3-async-runtimes = { version = "0.28", features = [ "attributes", @@ -41,10 +43,7 @@ tokio.workspace = true libc = "0.2" [build-dependencies] -pyo3-build-config = { version = "0.28", features = [ - "extension-module", - "abi3-py310", -] } +pyo3-build-config = { version = "0.28", features = ["abi3-py310"] } [features] default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"] diff --git a/python/pyproject.toml b/python/pyproject.toml index ae42172c0..fad3b1001 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -103,7 +103,7 @@ python-source = "python" module-name = "lancedb._lancedb" [build-system] -requires = ["maturin>=1.4"] +requires = ["maturin>=1.9.4"] build-backend = "maturin" [tool.ruff.lint] diff --git a/python/python/lancedb/index.py b/python/python/lancedb/index.py index aa7846892..d2b63baf6 100644 --- a/python/python/lancedb/index.py +++ b/python/python/lancedb/index.py @@ -163,6 +163,15 @@ class FTS: The number of documents per compressed posting block. Supported values are 128 and 256. A value of 256 uses the experimental FTS V3 format and may introduce breaking changes. + memory_limit : int, optional + The total memory limit in MiB for the local FTS build stage. The limit + is divided evenly among indexing workers. This build-only setting is + not persisted with the index and does not apply to remote tables. + num_workers : int, optional + The number of workers for a local FTS build. By default Lance uses + roughly half of the available CPU cores. The effective value is + limited by the available compute capacity. This build-only setting is + not persisted with the index and does not apply to remote tables. Notes ----- @@ -185,6 +194,8 @@ class FTS: prefix_only: bool = False block_size: int = 128 custom_stop_words: Optional[List[str]] = None + memory_limit: Optional[int] = None + num_workers: Optional[int] = None @dataclass diff --git a/python/python/tests/test_fts.py b/python/python/tests/test_fts.py index f791f9886..625198d92 100644 --- a/python/python/tests/test_fts.py +++ b/python/python/tests/test_fts.py @@ -245,6 +245,14 @@ def test_create_inverted_index_rejects_invalid_block_size(table): table.create_index("text", config=FTS(block_size=129)) +def test_create_inverted_index_respects_build_memory_limit(table): + with pytest.raises(ValueError, match="exceeds worker memory limit"): + table.create_index( + "text", + config=FTS(memory_limit=0, num_workers=1), + ) + + def test_custom_stop_words_list(table): table.create_index( "text", diff --git a/python/src/index.rs b/python/src/index.rs index dd362373e..a5ca63c68 100644 --- a/python/src/index.rs +++ b/python/src/index.rs @@ -42,7 +42,7 @@ pub fn extract_index_params(source: &Option>) -> PyResult Ok(LanceDbIndex::Fm(FmIndexBuilder::default())), "FTS" => { let params = source.extract::()?; - let inner_opts = FtsIndexBuilder::default() + let mut inner_opts = FtsIndexBuilder::default() .base_tokenizer(params.base_tokenizer) .language(¶ms.language) .map_err(|_| { @@ -61,6 +61,12 @@ pub fn extract_index_params(source: &Option>) -> PyResult, + num_workers: Option, } #[derive(FromPyObject)] @@ -444,3 +452,51 @@ impl IndexConfig { } } } + +#[cfg(test)] +mod tests { + use super::*; + use pyo3::types::{PyDict, PyDictMethods}; + use serde_json::json; + + #[test] + fn fts_build_controls_are_forwarded() { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c"class FTS: + with_position = True + base_tokenizer = 'simple' + language = 'English' + max_token_length = None + lower_case = True + stem = False + remove_stop_words = False + custom_stop_words = None + ascii_folding = False + ngram_min_length = 3 + ngram_max_length = 3 + prefix_only = False + block_size = 128 + memory_limit = 2048 + num_workers = 7 + +config = FTS()", + None, + Some(&locals), + ) + .unwrap(); + + let config = locals.get_item("config").unwrap().unwrap(); + let index = extract_index_params(&Some(config)).unwrap(); + let LanceDbIndex::FTS(params) = index else { + panic!("expected FTS index parameters"); + }; + let training_json = params.to_training_json().unwrap(); + + assert_eq!(training_json.get("memory_limit"), Some(&json!(2048))); + assert_eq!(training_json.get("num_workers"), Some(&json!(7))); + }); + } +} From c0df2c63b6f14c29bef699f93b33153c39dfbc96 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:39:13 -0700 Subject: [PATCH 05/31] test(rust): cover fixed-size-list merge overflow (#3907) ## Summary - add a merge-insert regression test whose fixed-size-list child count crosses `u32::MAX` - verify delete-by-source updates the matching row, deletes every other row, and completes without an Arrow panic - use a null child array so the boundary case avoids allocating a real vector payload ## Root cause and fix The affected Lance merge fallback carried the target payload through a full outer hash join. Arrow's fixed-size-list take kernel uses `u32` child indices, so taking a target row whose child offset crossed `u32::MAX` wrapped the offset and produced child data shorter than the parent array, triggering the reported `ArrayData::slice` assertion. The projection-aware merge path in the Lance version now used by `main` avoids materializing the target fixed-size-list payload in that join. This regression test locks in that production behavior at the exact child-index boundary. ## Validation - `cargo fmt --all` - `cargo test --quiet --features remote -p lancedb test_merge_insert_fixed_size_list_above_u32_child_count` - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` Fixes #2874 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/table/merge.rs | 71 ++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index ea68e99df..b9dd5732b 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -321,7 +321,8 @@ pub(crate) async fn execute_merge_insert( mod tests { use arrow_array::builder::FixedSizeBinaryBuilder; use arrow_array::{ - Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, UInt64Array, + FixedSizeListArray, Int32Array, NullArray, RecordBatch, RecordBatchIterator, + RecordBatchReader, StringArray, UInt32Array, UInt64Array, }; use arrow_schema::{DataType, Field, Schema}; use std::sync::Arc; @@ -529,6 +530,74 @@ mod tests { assert_eq!(result.num_deleted_rows, 5); assert_eq!(table.count_rows(None).await.unwrap(), 5); } + + #[tokio::test] + async fn test_merge_insert_fixed_size_list_above_u32_child_count() { + // Arrow's FixedSizeList take kernel uses u32 child indices. Previously, + // delete-by-source materialized the target payload in a full outer join, + // causing the final list below to overflow those indices and panic. + // A Null child keeps this boundary test small in memory. + const LIST_SIZE: i32 = 65_536; + const ROW_COUNT: usize = (u32::MAX as usize / LIST_SIZE as usize) + 1; + const BATCH_SIZE: usize = 8_192; + + let item = Arc::new(Field::new("item", DataType::Null, true)); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false), + Field::new( + "vector", + DataType::FixedSizeList(item.clone(), LIST_SIZE), + false, + ), + ])); + let batch = |start: usize, len: usize| { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values( + start as u32..(start + len) as u32, + )), + Arc::new(FixedSizeListArray::new( + item.clone(), + LIST_SIZE, + Arc::new(NullArray::new(len * LIST_SIZE as usize)), + None, + )), + ], + ) + .unwrap() + }; + + let target_batches = (0..ROW_COUNT) + .step_by(BATCH_SIZE) + .map(|start| { + let len = (ROW_COUNT - start).min(BATCH_SIZE); + Ok(batch(start, len)) + }) + .collect::>(); + let target_data: Box = + Box::new(RecordBatchIterator::new(target_batches, schema.clone())); + let conn = connect("memory://").execute().await.unwrap(); + let table = conn + .create_table("fixed_size_list_overflow", target_data) + .execute() + .await + .unwrap(); + + let source = batch(ROW_COUNT - 1, 1); + let mut merge = table.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_by_source_delete(None); + let result = merge + .execute(Box::new(RecordBatchIterator::new([Ok(source)], schema))) + .await + .unwrap(); + + assert_eq!(result.num_updated_rows, 1); + assert_eq!(result.num_deleted_rows, (ROW_COUNT - 1) as u64); + assert_eq!(table.count_rows(None).await.unwrap(), 1); + } } #[cfg(test)] From 5468f3d490229ab0dc18e4dd3e6639779f377ad3 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:44:21 -0700 Subject: [PATCH 06/31] fix(rust): reject bitmap indexes on JSON fields (#3895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - reject whole-document `lance.json` fields during native BITMAP index preparation - preserve BITMAP support for raw `LargeBinary` fields - return guidance to use a JSON-path scalar index or FTS instead - add regression coverage for the logical JSON type while retaining the existing raw binary coverage ## Root cause Native scalar-index validation resolved the complete Arrow field but checked BITMAP compatibility only against its physical data type. Because `lance.json` is stored as `LargeBinary`, it was incorrectly accepted under the raw binary compatibility rule. The fix reuses Lance’s `lance_arrow::json::is_json_field` helper before physical type validation. Remote serialization is unchanged, so remote clients continue to send the requested BITMAP type for server-side validation. ## Validation - `cargo fmt --all -- --check` - `cargo test --quiet --features remote -p lancedb test_create_bitmap_index -- --nocapture` - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` - `cargo test --quiet --features remote --tests` Fixes #3889 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/table/create_index.rs | 38 ++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 144c6dbfb..e373522bc 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -12,6 +12,7 @@ use arrow_schema::{DataType, Field}; use lance::index::DatasetIndexExt; use lance::index::vector::VectorIndexParams; use lance::index::vector::utils::infer_vector_dim; +use lance_arrow::json::is_json_field; use lance_index::IndexType; use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; use lance_index::vector::bq::RQBuildParams; @@ -219,6 +220,14 @@ impl NativeTable { ))) } Index::Bitmap(_) => { + if is_json_field(field) { + return Err(Error::Schema { + message: format!( + "A BITMAP index cannot be created on the whole-document lance.json field `{}`. Create a JSON-path scalar index for structured equality or range predicates, or use FTS for document search", + field.name() + ), + }); + } Self::validate_index_type(field, "Bitmap", supported_bitmap_data_type)?; Ok(Box::new(ScalarIndexParams::for_builtin( BuiltinIndexType::Bitmap, @@ -1465,6 +1474,35 @@ mod tests { assert_eq!(stats.distance_type, None); } + #[tokio::test] + async fn test_create_bitmap_index_rejects_lance_json() { + let conn = connect("memory://").execute().await.unwrap(); + let schema = Arc::new(Schema::new(vec![lance_arrow::json::json_field( + "metadata", true, + )])); + let table = conn + .create_empty_table("json_bitmap", schema) + .execute() + .await + .unwrap(); + + let err = table + .create_index(&["metadata"], Index::Bitmap(Default::default())) + .execute() + .await + .expect_err("a whole-document lance.json field must not support a bitmap index"); + let message = err.to_string(); + assert!( + message.contains("lance.json"), + "unexpected error: {message}" + ); + assert!( + message.contains("JSON-path scalar index"), + "unexpected error: {message}" + ); + assert!(message.contains("FTS"), "unexpected error: {message}"); + } + #[tokio::test] async fn test_create_label_list_index() { let conn = connect("memory://").execute().await.unwrap(); From 7801e2746aae9438b7c87e814fe52d5631d0fd9c Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Fri, 21 Aug 2026 21:43:45 -0700 Subject: [PATCH 07/31] chore: update lance dependency to v11.0.0-beta.19 (#4025) Updates the Lance dependencies and Java lance-core dependency to v11.0.0-beta.19. No compatibility fixes were required; workspace clippy with all features passes. Triggering tag: https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.19 --- Cargo.lock | 84 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 28 +++++++++--------- java/pom.xml | 2 +- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73f5923fe..5947187cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arc-swap", "arrow", @@ -4888,8 +4888,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-buffer", @@ -4911,7 +4911,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-buffer", @@ -4925,7 +4925,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-schema", @@ -4934,8 +4934,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrayref", "crunchy", @@ -4945,8 +4945,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-buffer", @@ -4983,8 +4983,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow", "arrow-array", @@ -5013,8 +5013,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow", "arrow-array", @@ -5031,8 +5031,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "proc-macro2", "quote", @@ -5041,8 +5041,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-arith", "arrow-array", @@ -5075,8 +5075,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-arith", "arrow-array", @@ -5107,8 +5107,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arc-swap", "arrow", @@ -5172,8 +5172,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-schema", @@ -5195,8 +5195,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow", "arrow-array", @@ -5232,8 +5232,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-schema", @@ -5247,8 +5247,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow", "async-trait", @@ -5260,8 +5260,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow", "arrow-ipc", @@ -5314,8 +5314,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-buffer", @@ -5329,8 +5329,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow", "arrow-array", @@ -5370,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "arrow-array", "arrow-schema", @@ -5384,8 +5384,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 9aeccd76f..c147b06db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 978f7c7c7..36ec01d9e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 11.0.0-beta.18 + 11.0.0-beta.19 false 2.30.0 1.7 From a578e9ff7f63f0ef534b088c2f9fc9c4deaa0cb6 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 21 Aug 2026 22:39:47 -0700 Subject: [PATCH 08/31] feat: refresh materialized views (#4010) A declared view holds no rows; refresh computes them. It pins one source version, brings the view to exactly the definition's result at that version, and records the version as a watermark in the view's schema metadata. It is incremental when it can reconcile what changed: appended rows are computed and appended, and rows the source deleted or updated are found by the lance delta and evicted by their __source_row_id provenance, the updated ones recomputed in the same commit. Compaction rearranges rows without changing them, so its outputs cost nothing -- which is what keeps routine background compaction from rebuilding the view. A vacuumed watermark, a delta the transaction-log walk cannot classify, a Legacy-storage source, or more staged ids than a fixed cap all fall back to a rebuild; rebuilding an indexed view swaps every fragment in one Update, so readers never see it unindexed or empty. Concurrent refreshes serialize at commit -- each carries the same sentinel row id in its inserted-rows filter, so the loser lands nothing. On the append path the watermark moves in a follow-up commit, so a crash between the two re-appends those rows. Bumps lance to v11.0.0-beta.19 for the delta reader. --- rust/lancedb/src/lib.rs | 4 +- rust/lancedb/src/materialized_view.rs | 60 +- rust/lancedb/src/materialized_view/refresh.rs | 2719 +++++++++++++++++ rust/lancedb/src/table/merge/lsm.rs | 7 + rust/lancedb/src/table/refresh.rs | 2 +- 5 files changed, 2789 insertions(+), 3 deletions(-) create mode 100644 rust/lancedb/src/materialized_view/refresh.rs diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index 3a937db5b..9c3c199ff 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -211,7 +211,9 @@ pub use function::FunctionVersion; pub use job::Job; use lance_index::vector::ApproxMode as LanceApproxMode; use lance_linalg::distance::DistanceType as LanceDistanceType; -pub use materialized_view::{MaterializedView, MaterializedViewDefinition}; +pub use materialized_view::{ + MaterializedView, MaterializedViewDefinition, RefreshMaterializedViewResult, RefreshMode, +}; /// Re-export of the [`metrics`](https://docs.rs/metrics) crate facade. Enable /// the `metrics` feature to publish LanceDB's internal metrics; install any /// `metrics`-compatible recorder to collect them. See also [`metrics_otel`] for diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index f1eddf9da..8b553165b 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -9,6 +9,7 @@ //! metadata; a kind added later reads back as unrefreshable, not as a plain //! table. Queries, indexes and search work on the view unchanged. +pub mod refresh; use std::collections::HashMap; use std::sync::Arc; @@ -27,6 +28,8 @@ use crate::table::refresh::quote_identifier; use crate::table::{ColumnDefinition, ColumnKind}; use crate::{Error, Result}; +pub use refresh::{RefreshMaterializedViewResult, RefreshMode}; + /// Schema metadata key holding the view definition, as kind-tagged JSON. pub const DEFINITION_META_KEY: &str = "mv.definition"; @@ -736,6 +739,12 @@ pub async fn prepare_declaration( ), }); } + refresh::ensure_no_mem_wal( + native.dataset.get().await?.as_ref(), + "source table", + resolved.name(), + ) + .await?; let source_schema = resolved.schema().await?; let source_metadata = source_schema.metadata().clone(); let (definition, mut fields, lineage) = plan( @@ -909,6 +918,54 @@ impl MaterializedView { pub fn definition(&self) -> &MaterializedViewDefinition { &self.definition } + + /// Recompute the view from its source. + /// + /// By default the refresh is incremental when the source's changes can be + /// reconciled into the view, and otherwise rebuilds; see + /// [`RefreshMaterializedViewBuilder`]. + /// + /// ```no_run + /// # #![recursion_limit = "256"] + /// # use lancedb::materialized_view::MaterializedView; + /// # async fn refresh(view: &MaterializedView) -> Result<(), Box> { + /// let result = view.refresh().execute().await?; + /// println!("{:?}: {} rows", result.mode, result.rows_written); + /// # Ok(()) + /// # } + /// ``` + pub fn refresh(&self) -> RefreshMaterializedViewBuilder { + RefreshMaterializedViewBuilder { + view: self.clone(), + full: false, + source_version: None, + } + } +} + +/// Builds a refresh. Created by [`MaterializedView::refresh`]. +pub struct RefreshMaterializedViewBuilder { + view: MaterializedView, + full: bool, + source_version: Option, +} + +impl RefreshMaterializedViewBuilder { + /// Rebuild the view even where an incremental refresh would do. + pub fn full(mut self, full: bool) -> Self { + self.full = full; + self + } + + /// Refresh to this source table version instead of the latest. + pub fn source_version(mut self, version: u64) -> Self { + self.source_version = Some(version); + self + } + + pub async fn execute(self) -> Result { + refresh::execute_refresh(&self.view.table, self.full, self.source_version).await + } } impl Connection { @@ -918,6 +975,7 @@ impl Connection { /// metadata; refresh computes the rows. Local databases only. /// /// ```no_run + /// # #![recursion_limit = "256"] /// # use lancedb::Connection; /// # async fn create(conn: &Connection) -> Result<(), Box> { /// let view = conn @@ -926,7 +984,7 @@ impl Connection { /// .only_if("age >= 18") /// .execute() /// .await?; - /// println!("{}", view.definition().source_table); + /// view.refresh().execute().await?; /// # Ok(()) /// # } /// ``` diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs new file mode 100644 index 000000000..7ac39a9f5 --- /dev/null +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -0,0 +1,2719 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Refreshing materialized views. +//! +//! A refresh pins one source version and brings the view to exactly the +//! definition's result at that version: added rows are computed and appended, +//! removed or changed rows are evicted by provenance id and recomputed in the +//! same pass. Compaction outputs cost nothing, which is only sound while +//! [`SOURCE_ROW_ID_COLUMN`] stays valid across the rewrite. Anything the +//! classifier cannot prove intact rebuilds; an indexed rebuild swaps all +//! fragments in one commit that retains index definitions. +//! +//! The watermark ([`SOURCE_VERSION_META_KEY`]) lands in a follow-up commit; a +//! crash or race between the two leaves the view visibly unstamped and the +//! next refresh rebuilds. In-process refreshes serialize on a per-view lock; +//! across processes the commit's inserted-rows filter carries a shared token, +//! so two refreshes of one view conflict and only one lands. + +use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex as StdMutex, OnceLock}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use arrow_array::cast::AsArray; +use arrow_array::types::UInt64Type; +use arrow_array::{RecordBatch, UInt64Array}; +use arrow_schema::{Schema as ArrowSchema, SchemaRef}; +use datafusion::common::ScalarValue; +use datafusion::error::DataFusionError; +use datafusion::physical_plan::SendableRecordBatchStream; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::prelude::{col, lit}; +use futures::{StreamExt, TryStreamExt}; +use lance::Dataset; +use lance::dataset::mem_wal::DatasetMemWalExt; +use lance::dataset::transaction::{Operation, Transaction}; +use lance::dataset::write::delete::DeleteBuilder; +use lance::dataset::write::merge_insert::inserted_rows::{ + KeyExistenceFilter, KeyExistenceFilterBuilder, KeyValue, +}; +use lance::dataset::{CommitBuilder, InsertBuilder, WriteDestination, WriteMode, WriteParams}; +use lance_core::{ROW_CREATED_AT_VERSION, ROW_ID, ROW_LAST_UPDATED_AT_VERSION}; +use lance_file::version::ConcreteFileVersion; +use lance_table::format::Fragment; +use serde::{Deserialize, Serialize}; + +use super::{ + MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY, SOURCE_ROW_ID_COLUMN, + SOURCE_VERSION_META_KEY, +}; +use crate::database::OpenTableRequest; +use crate::table::{NativeTable, NativeTableExt, Table}; +use crate::{Error, Result}; + +/// How a refresh brought the view up to date. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RefreshMode { + /// The view was recomputed from scratch. + Rebuild, + /// Rows from source fragments added since the last refresh were appended. + Incremental, + /// The view was already at the requested source version. + NoOp, +} + +/// The result of refreshing a materialized view. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RefreshMaterializedViewResult { + /// How the view was brought up to date. + pub mode: RefreshMode, + /// Rows written to the view: everything on a rebuild, and on an + /// incremental refresh both the rows added and the rows recomputed in + /// place of ones the source changed. + pub rows_written: u64, + /// The source table version the view now reflects. + pub source_version: u64, + /// The view table version after the refresh. + pub version: u64, +} + +/// Schema metadata key holding the view table version a successful refresh +/// left behind. Any other commit on the view is drift, and refresh rebuilds. +pub const VIEW_VERSION_META_KEY: &str = "mv.view_version"; + +/// Schema metadata key holding the commit timestamp of the watermark's source +/// manifest. A dropped and recreated source reuses version numbers but never +/// their timestamps, so a mismatch means the watermark describes a different +/// incarnation and refresh rebuilds. +pub const SOURCE_VERSION_TS_META_KEY: &str = "mv.source_version_ts"; + +/// One refresh per view at a time within this process. +fn refresh_lock(uri: &str) -> Arc> { + static LOCKS: OnceLock>>>> = + OnceLock::new(); + LOCKS + .get_or_init(Default::default) + .lock() + .expect("refresh lock registry poisoned") + .entry(uri.to_string()) + .or_default() + .clone() +} + +/// Internal implementation of the refresh logic. +pub(crate) async fn execute_refresh( + view: &Table, + full: bool, + pinned: Option, +) -> Result { + let view_native = view.as_native().ok_or_else(|| Error::NotSupported { + message: "materialized views are supported only on local tables".into(), + })?; + view_native.dataset.ensure_mutable()?; + let lock = refresh_lock(view_native.dataset.get().await?.uri()); + let _guard = lock.lock().await; + // Force-load the latest view state under the lock: each handle caches + // lazily, and a second handle would otherwise plan from a snapshot taken + // before another handle's commit -- appending the same rows again or + // reporting NoOp over a mutated view. + view_native.dataset.reload().await?; + let view_ds = view_native.dataset.get().await?.as_ref().clone(); + + // The definition a handle cached at open may since have been replaced; + // what refresh executes and what it stamps must be one generation. + let definition = match super::materialized_view_kind(&view_ds.schema().metadata)? { + Some(super::MaterializedViewKind::Select(definition)) => definition, + Some(super::MaterializedViewKind::Unrecognized { kind }) => { + return Err(Error::NotSupported { + message: format!( + "materialized view '{}' is defined by '{kind}', which this \ + version of lancedb cannot refresh", + view.name() + ), + }); + } + None => { + return Err(Error::NotAMaterializedView { + name: view.name().to_string(), + }); + } + }; + let definition = &definition; + ensure_no_mem_wal(&view_ds, "materialized view", view.name()).await?; + + let source_ds = open_source(view, definition).await?; + let source_ds = match pinned { + Some(version) => source_ds.checkout_version(version).await?, + None => source_ds, + }; + ensure_no_mem_wal(&source_ds, "source table", &definition.source_table).await?; + let source_version = source_ds.version().version; + let source_ts = source_ds.manifest.timestamp_nanos; + + // Re-plan the persisted definition against the current source schema and + // require its planned output to be exactly the view's physical schema: a + // definition the stored table cannot represent must not be certified. + let source_schema = Arc::new(ArrowSchema::from(source_ds.schema())); + let projections: Vec<(String, String)> = definition + .projections + .iter() + .map(|p| (p.output.clone(), p.expression.clone())) + .collect(); + validate_inputs(&source_ds, definition)?; + let (replanned, mut planned_fields, _renames) = super::plan( + source_schema, + &definition.source_table, + &projections, + definition.filter.as_deref(), + definition.limit, + )?; + planned_fields.push(arrow_schema::Field::new( + SOURCE_ROW_ID_COLUMN, + arrow_schema::DataType::UInt64, + false, + )); + let physical = ArrowSchema::from(view_ds.schema()); + let planned_shape: Vec<_> = planned_fields + .iter() + .map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable())) + .collect(); + let physical_shape: Vec<_> = physical + .fields() + .iter() + .map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable())) + .collect(); + if planned_shape != physical_shape { + return Err(Error::Schema { + message: format!( + "the stored definition of view '{}' does not produce this \ + view's schema; recreate the view", + view.name() + ), + }); + } + let definition = &replanned; + + let metadata = &view_ds.schema().metadata; + let watermark: Option = metadata + .get(SOURCE_VERSION_META_KEY) + .and_then(|raw| raw.parse().ok()); + let recorded_ts: Option = metadata + .get(SOURCE_VERSION_TS_META_KEY) + .and_then(|raw| raw.parse().ok()); + // The watermark speaks only for the view state its refresh left behind; + // any other commit on the view since then is drift. + let view_intact = metadata + .get(VIEW_VERSION_META_KEY) + .and_then(|raw| raw.parse::().ok()) + == Some(view_ds.version().version); + + if !full && watermark == Some(source_version) && view_intact && recorded_ts == Some(source_ts) { + return Ok(RefreshMaterializedViewResult { + mode: RefreshMode::NoOp, + rows_written: 0, + source_version, + version: view_ds.version().version, + }); + } + + let watermark = watermark.filter(|_| view_intact); + match plan_increment( + &source_ds, + source_version, + watermark, + recorded_ts, + full, + definition, + ) + .await + { + Some(increment) => { + let reconciled = incremental( + view_native, + &view_ds, + &source_ds, + source_version, + source_ts, + increment, + definition, + watermark, + ) + .await?; + match reconciled { + Some(result) => Ok(result), + // The delta was too large to reconcile in bounded memory. + None => { + rebuild( + view_native, + &view_ds, + &source_ds, + source_version, + source_ts, + definition, + ) + .await + } + } + } + None => { + rebuild( + view_native, + &view_ds, + &source_ds, + source_version, + source_ts, + definition, + ) + .await + } + } +} + +/// The source fragments whose rows are new since the watermark, or `None` +/// where the view has to rebuild. Two tiers: the transaction walk is exact +/// where it applies; the fragment-signature check is the fallback for deltas +/// the walk cannot read, and under it any fragment churn rebuilds. +async fn plan_increment( + source_ds: &Dataset, + source_version: u64, + watermark: Option, + recorded_ts: Option, + full: bool, + definition: &MaterializedViewDefinition, +) -> Option { + if full { + return None; + } + let watermark = watermark?; + if watermark > source_version { + return None; + } + let old = source_ds.checkout_version(watermark).await.ok()?; + // A recreated source reuses version numbers, never their timestamps: a + // mismatch means the watermark describes a different incarnation. + if recorded_ts != Some(old.manifest.timestamp_nanos) { + return None; + } + let old_ids: HashSet = old.get_fragments().iter().map(|f| f.id() as u64).collect(); + let live: Vec = source_ds + .get_fragments() + .iter() + .map(|f| f.metadata().clone()) + .collect(); + + if let Some(delta) = appends_and_rewrites(source_ds, watermark, source_version).await { + // A rewrite that consumed a fragment neither present at the watermark + // nor produced by an earlier rewrite swallowed a mid-delta append; + // its rows cannot be told apart from already-materialized ones. + let folded = delta + .rewritten + .iter() + .any(|id| !old_ids.contains(id) && !delta.produced.contains(id)); + if folded { + return None; + } + // An update rewrites a whole fragment. If it touched one the watermark + // never saw, that fragment holds rows appended since -- new rows the + // update did not change, which the recompute does not cover and which + // this fragment's exclusion from the append set would drop. + if delta + .updated_in_place + .iter() + .any(|id| !old_ids.contains(id)) + { + return None; + } + // Rows past the cap left every delta when the watermark advanced, so + // a capped view cannot reconcile a removal incrementally. The rebuild + // is cheap for the same reason it is capped: the scan stops there. + if definition.limit.is_some() && (delta.deleted_rows || delta.updated_rows) { + return None; + } + // Legacy storage cannot serve the row-version columns update + // discovery scans; deletes and appends need none of them. + if delta.updated_rows + && source_ds.manifest.data_storage_format.lance_file_format() == ConcreteFileVersion::V1 + { + return None; + } + // Every other fragment new at head is an append: appends and rewrites + // are the only operations in the delta that add fragments, and the + // rewrite outputs are already-materialized rows rearranged. + return Some(Increment { + appended: live + .into_iter() + .filter(|f| !old_ids.contains(&f.id) && !delta.produced.contains(&f.id)) + .collect(), + evict_deleted: delta.deleted_rows, + replace_updated: delta.updated_rows, + }); + } + + is_pure_append(&old, source_ds, &relevant_field_ids(source_ds, definition)).then(|| Increment { + appended: live + .into_iter() + .filter(|f| !old_ids.contains(&f.id)) + .collect(), + evict_deleted: false, + replace_updated: false, + }) +} + +/// Version gap beyond which per-version transaction reads stop being cheaper +/// than one fragment scan. +const MAX_TRANSACTION_WALK: u64 = 512; + +/// Fragment ids moved by the `Rewrite` operations of the delta. Only rewrite +/// ids are real in transaction files (an Append's are placeholders assigned +/// at commit), so appends are derived as new-at-head minus rewrite outputs. +/// What an incremental refresh must do to bring the view up to date. +struct Increment { + /// Source fragments whose rows are not in the view yet. + appended: Vec, + /// Source rows left the range, so the view holds rows to evict. + evict_deleted: bool, + /// Source rows changed in the range, so the view holds rows to replace. + replace_updated: bool, +} + +struct TxnDelta { + /// Fragments consumed by `Rewrite` operations. + rewritten: HashSet, + /// Fragments produced by `Rewrite` operations. + produced: HashSet, + /// The delta removed source rows, so the view holds rows to evict. + deleted_rows: bool, + /// The delta changed source rows in place, so the view holds rows to + /// recompute. + updated_rows: bool, + /// Fragments an update modified in place, as opposed to produced. + updated_in_place: HashSet, +} + +/// Read the delta from the transaction log, `None` where it holds anything +/// but appends and rewrites or cannot be read; `None` only sends the caller +/// to a slower check. `ReserveFragments` moves no rows and rides along. +async fn appends_and_rewrites(cur: &Dataset, from: u64, to: u64) -> Option { + if to <= from || to - from > MAX_TRANSACTION_WALK { + return None; + } + let mut delta = TxnDelta { + rewritten: HashSet::new(), + produced: HashSet::new(), + deleted_rows: false, + updated_rows: false, + updated_in_place: HashSet::new(), + }; + for version in (from + 1)..=to { + let Ok(Some(txn)) = cur.read_transaction_by_version(version).await else { + return None; + }; + match txn.operation { + Operation::Append { .. } | Operation::ReserveFragments { .. } => {} + // A delete removes source rows without changing the ones that + // remain, so the view's other rows stay valid: the refresh + // evicts exactly the ids that left. + Operation::Delete { .. } => delta.deleted_rows = true, + // Update outputs carry no new rows, so they are excluded from + // the append set like rewrite outputs; the changed rows are + // replaced individually below. + Operation::Update { + removed_fragment_ids, + new_fragments, + updated_fragments, + .. + } => { + delta.updated_rows = true; + // merge_insert reaches here too, and its by-source arm deletes + // rows rather than changing them. + delta.deleted_rows = true; + delta.rewritten.extend(removed_fragment_ids.iter().copied()); + // Only pre-existing fragment ids are real here; created ones + // are placeholders. Rewritten rows are excluded by creation + // version below, not by fragment identity. + delta + .produced + .extend(updated_fragments.iter().map(|f| f.id)); + delta + .updated_in_place + .extend(updated_fragments.iter().map(|f| f.id)); + let _ = new_fragments; + } + Operation::Rewrite { groups, .. } => { + for group in groups { + delta + .rewritten + .extend(group.old_fragments.iter().map(|f| f.id)); + delta + .produced + .extend(group.new_fragments.iter().map(|f| f.id)); + } + } + _ => return None, + } + } + Some(delta) +} + +/// Fallback pure-append check: every old fragment still present with an +/// identical signature over the columns the view reads. Compaction, deletes +/// and updates each break it and force a rebuild; a change to a column the +/// view does not read leaves it alone, which is what lets this tier pass +/// deltas the transaction walk cannot. +fn is_pure_append(old: &Dataset, cur: &Dataset, relevant: &HashSet) -> bool { + let signature = |fragment: &lance::dataset::fragment::FileFragment| { + fragment_signature(fragment.metadata(), relevant) + }; + let current: HashSet<(u64, String)> = cur.get_fragments().iter().map(signature).collect(); + old.get_fragments() + .iter() + .all(|fragment| current.contains(&signature(fragment))) +} + +/// A fragment's identity as the view observes it: data files and overlays +/// touching the columns it reads, plus the deletion file. Overlays change no +/// file path, so they must be part of the signature. +fn fragment_signature(metadata: &Fragment, relevant: &HashSet) -> (u64, String) { + let touches_relevant = + |fields: &[i32]| relevant.is_empty() || fields.iter().any(|id| relevant.contains(id)); + let mut files: Vec<&str> = metadata + .files + .iter() + .filter(|file| touches_relevant(&file.fields)) + .map(|file| file.path.as_str()) + .collect(); + files.sort_unstable(); + let mut overlays: Vec = metadata + .overlays + .iter() + .filter(|overlay| touches_relevant(&overlay.data_file.fields)) + .map(|overlay| format!("{}@{}", overlay.data_file.path, overlay.committed_version)) + .collect(); + overlays.sort_unstable(); + ( + metadata.id, + format!( + "{}|{}|{:?}", + files.join(","), + overlays.join(","), + metadata.deletion_file + ), + ) +} + +/// Field ids (with struct descendants) of the source columns the view reads. +fn relevant_field_ids(source: &Dataset, definition: &MaterializedViewDefinition) -> HashSet { + fn collect(field: &lance_core::datatypes::Field, ids: &mut HashSet) { + ids.insert(field.id); + for child in &field.children { + collect(child, ids); + } + } + let mut ids = HashSet::new(); + for input in &definition.inputs { + if let Some(field) = source.schema().field(input) { + collect(field, &mut ids); + } + } + ids +} + +/// Error if a column the view reads no longer exists in the source. +fn validate_inputs(source: &Dataset, definition: &MaterializedViewDefinition) -> Result<()> { + for input in &definition.inputs { + if source.schema().field(input).is_none() { + return Err(Error::Schema { + message: format!( + "source column '{input}' read by the view no longer exists \ + (dropped or renamed in '{}')", + definition.source_table + ), + }); + } + } + Ok(()) +} + +/// Reject MemWAL/LSM state on a refresh participant: un-compacted tiers are +/// invisible to the fragment-planned refresh scan. An active write spec and +/// retained rows both disqualify; shard directories on storage are the +/// durable evidence of the latter. +pub(crate) async fn ensure_no_mem_wal(dataset: &Dataset, role: &str, name: &str) -> Result<()> { + let retained = !dataset.list_mem_wal_latest_shard_ids().await?.is_empty(); + if retained || dataset.mem_wal_index_details().await?.is_some() { + return Err(Error::NotSupported { + message: format!( + "{role} '{name}' has an LSM write spec or retained un-compacted \ + rows: rows in un-compacted tiers are invisible to refresh" + ), + }); + } + Ok(()) +} + +async fn open_source(view: &Table, definition: &MaterializedViewDefinition) -> Result { + let database = view.database_opt().ok_or_else(|| Error::InvalidInput { + message: "the view was not opened through a database connection".into(), + })?; + let source = database + .open_table(OpenTableRequest { + name: definition.source_table.clone(), + namespace_path: Vec::new(), + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await?; + let native = source.as_native().ok_or_else(|| Error::NotSupported { + message: "materialized views are supported only on local tables".into(), + })?; + let dataset = native.dataset.get().await?.as_ref().clone(); + if !dataset.manifest.uses_stable_row_ids() { + return Err(Error::InvalidInput { + message: format!( + "source table '{}' does not have stable row ids; it is not the \ + table this view was declared over", + definition.source_table + ), + }); + } + Ok(dataset) +} + +#[allow(clippy::too_many_arguments)] +async fn incremental( + view_native: &NativeTable, + view_ds: &Dataset, + source_ds: &Dataset, + source_version: u64, + source_ts: u128, + increment: Increment, + definition: &MaterializedViewDefinition, + watermark: Option, +) -> Result> { + let new_fragments = increment.appended; + let watermark_version = watermark.unwrap_or(0); + // Provenance ids this refresh removes: dropped rows, plus changed rows + // recomputed in the same commit. One view row per source row makes the + // eviction exact. Staged, not committed: removals ride with the rows + // that replace them, so a reader never sees the view without either. + let mut eviction = Eviction::new(view_ds, EVICTION_CHUNK); + let mut updated_rows = false; + if (increment.evict_deleted || increment.replace_updated) + && let Some(watermark) = watermark + { + let delta = source_ds + .delta() + .with_begin_version(watermark) + .with_end_version(source_version) + .build()?; + // Reconciling holds the delta's provenance ids in staged deletion + // vectors; past the cap, the streamed rebuild is the bounded path. + let cap = eviction_rebuild_cap(); + let mut evicted = 0usize; + if increment.evict_deleted { + let mut stream = delta.get_deleted_row_ids().await?; + while let Some(batch) = stream.try_next().await? { + let ids = row_ids_of(&batch)?; + evicted += ids.len(); + if evicted > cap { + return Ok(None); + } + eviction.push(ids).await?; + } + } + if increment.replace_updated { + // Ids only: `get_updated_rows` carries every column of every + // updated row, and discovery needs none of them. + let mut scanner = source_ds.scan(); + scanner.with_row_id().project(&[ROW_CREATED_AT_VERSION])?; + // A fixed bound: the configured default could make one discovery + // batch arbitrarily large before the fallback cap is consulted. + scanner.batch_size(8192); + scanner.filter(&format!( + "{ROW_CREATED_AT_VERSION} <= {watermark} + AND {ROW_LAST_UPDATED_AT_VERSION} > {watermark} + AND {ROW_LAST_UPDATED_AT_VERSION} <= {source_version}" + ))?; + let mut stream = scanner.try_into_stream().await?; + while let Some(batch) = stream.try_next().await? { + let ids = row_ids_of(&batch)?; + evicted += ids.len(); + if evicted > cap { + return Ok(None); + } + updated_rows |= !ids.is_empty(); + eviction.push(ids).await?; + } + } + } + let eviction = eviction.finish().await?; + + // The cap counts rows already materialized, in first-materialized order. + let remaining = match definition.limit { + Some(limit) => { + let held = view_ds.count_rows(None).await? as u64; + Some(limit.saturating_sub(held)) + } + None => None, + }; + + let mut result = RefreshMaterializedViewResult { + mode: RefreshMode::Incremental, + rows_written: 0, + source_version, + version: view_ds.version().version, + }; + let nothing_to_add = (new_fragments.is_empty() && !updated_rows) || remaining == Some(0); + if nothing_to_add && eviction.is_none() { + result.version = + stamp_watermark(view_native, view_ds.clone(), source_version, source_ts).await?; + return Ok(Some(result)); + } + // Rows left but none arrive: the removals still have to be published. + if nothing_to_add { + let filter = refresh_filter(&empty_keys(view_ds)?)?; + let published = publish(view_ds, eviction, Vec::new(), Some(filter)).await?; + result.version = stamp_watermark(view_native, published, source_version, source_ts).await?; + return Ok(Some(result)); + } + + // Appends carry the view's schema as it stands; the watermark moves in a + // follow-up commit (see the module docs for the crash window). + let schema = Arc::new(ArrowSchema::from(view_ds.schema())); + let rows_written = Arc::new(AtomicU64::new(0)); + // compute_stream counts what it produces; the truncation below can drop + // some of that, so the written count comes from the tee instead. + let computed = Arc::new(AtomicU64::new(0)); + let mut stream = compute_stream( + source_ds, + definition, + RowScope { + fragments: Some(new_fragments), + // An update rewrites whole fragments, so a fragment new at head + // can hold rows the view already has. Their creation version + // does not change, so it -- not fragment identity -- says which + // rows are new. + created_after: increment.replace_updated.then_some(watermark_version), + limit: remaining, + ..Default::default() + }, + schema.clone(), + computed.clone(), + ) + .await?; + + // The updated rows' current values, computed the same way and appended + // in the same commit as the new fragments' rows. + if updated_rows { + let recomputed = compute_stream( + source_ds, + definition, + RowScope { + updated_between: Some((watermark_version, source_version)), + ..Default::default() + }, + schema.clone(), + computed.clone(), + ) + .await?; + stream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + recomputed.chain(stream), + )); + } + + // Nothing survived the filter: the watermark still has to advance or the + // same fragments would be rescanned forever, but any removals still do. + let Some(first) = stream.try_next().await? else { + let published = if eviction.is_some() { + publish( + view_ds, + eviction, + Vec::new(), + Some(refresh_filter(&empty_keys(view_ds)?)?), + ) + .await? + } else { + view_ds.clone() + }; + result.version = stamp_watermark(view_native, published, source_version, source_ts).await?; + return Ok(Some(result)); + }; + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::iter([Ok(first)]).chain(stream), + )); + + // Any two refreshes of one view must not both commit: the filter's + // shared token makes lance reject the loser on key overlap however + // their planned rows relate. + let keys = Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new(vec![ + source_row_id_field_id(view_ds)?, + ]))); + let stream = collect_source_row_ids(stream, keys.clone(), rows_written.clone()); + + let ds = Arc::new(view_ds.clone()); + let write_txn = InsertBuilder::new(WriteDestination::Dataset(ds.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await?; + let Operation::Append { + fragments: new_fragments, + } = write_txn.operation + else { + return Err(Error::Runtime { + message: "expected an append when staging the view's new rows".into(), + }); + }; + let filter = refresh_filter(&keys)?; + let appended = publish(view_ds, eviction, new_fragments, Some(filter)).await?; + result.rows_written = rows_written.load(Ordering::Relaxed); + result.version = stamp_watermark(view_native, appended, source_version, source_ts).await?; + Ok(Some(result)) +} + +async fn rebuild( + view_native: &NativeTable, + view_ds: &Dataset, + source_ds: &Dataset, + source_version: u64, + source_ts: u128, + definition: &MaterializedViewDefinition, +) -> Result { + let rows_written = Arc::new(AtomicU64::new(0)); + let schema = Arc::new(ArrowSchema::from(view_ds.schema())); + let stream = compute_stream( + source_ds, + definition, + RowScope { + limit: definition.limit, + ..Default::default() + }, + schema, + rows_written.clone(), + ) + .await?; + let keys = Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new(vec![ + source_row_id_field_id(view_ds)?, + ]))); + let stream = collect_source_row_ids(stream, keys.clone(), Arc::new(AtomicU64::new(0))); + // Every rebuild is one fragment swap, indexed or not: an Update commit + // carries no schema metadata, so it cannot erase a definition update + // that raced in the way an overwrite (which adopts its stream's schema) + // durably would -- and it must land on the planned generation or abort. + let replaced = replace_retaining_indices(view_ds.clone(), stream, keys).await?; + let version = stamp_watermark(view_native, replaced, source_version, source_ts).await?; + Ok(RefreshMaterializedViewResult { + mode: RefreshMode::Rebuild, + rows_written: rows_written.load(Ordering::Relaxed), + source_version, + version, + }) +} + +/// Replace all of the view's data in one commit that retains its index +/// definitions: new fragments staged uncommitted, one `Update` removing every +/// old fragment. `Update` prunes index bitmaps only for modified fields and +/// none are modified here, so readers never see the view unindexed or empty. +async fn replace_retaining_indices( + view_ds: Dataset, + stream: SendableRecordBatchStream, + keys: Arc>, +) -> Result { + let ds = Arc::new(view_ds); + let read_version = ds.version().version; + #[cfg(test)] + tests::hold_before_publish(ds.uri()).await; + let removed_fragment_ids: Vec = ds.get_fragments().iter().map(|f| f.id() as u64).collect(); + + let write_txn = InsertBuilder::new(WriteDestination::Dataset(ds.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await?; + let Operation::Append { + fragments: new_fragments, + } = write_txn.operation + else { + return Err(Error::Runtime { + message: "expected an append when staging the view's replacement rows".into(), + }); + }; + + // Built only now: the tee fills as the staging drains the stream. + let filter = refresh_filter(&keys)?; + let transaction = Transaction::new( + read_version, + Operation::Update { + removed_fragment_ids, + updated_fragments: Vec::new(), + new_fragments, + fields_modified: Vec::new(), + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: Vec::new(), + update_mode: None, + // Two refreshes that materialized the same source rows must not + // both land -- a raced first rebuild would double the view. + inserted_rows_filter: Some(filter), + updated_fragment_offsets: None, + }, + None, + ); + let committed = CommitBuilder::new(WriteDestination::Dataset(ds)) + .execute(transaction) + .await?; + if committed.version().version != read_version + 1 { + return Err(Error::Runtime { + message: format!( + "a concurrent commit raced this refresh (view version {}); the + refresh is unrecorded and the next one will rebuild", + committed.version().version + ), + }); + } + Ok(committed) +} + +/// Record that the view now reflects `source_version`, including the view +/// version this very commit produces. The version is predicted and then +/// verified; on a mismatch another commit raced in between, and the stamp +/// ABORTS rather than certify that commit as the refresh's own generation. +/// The view is left visibly unstamped, so the next refresh rebuilds. +async fn stamp_watermark( + view_native: &NativeTable, + mut dataset: Dataset, + source_version: u64, + source_ts: u128, +) -> Result { + let predicted = dataset.version().version + 1; + dataset + .update_schema_metadata([ + ( + SOURCE_VERSION_META_KEY.to_string(), + Some(source_version.to_string()), + ), + ( + SOURCE_VERSION_TS_META_KEY.to_string(), + Some(source_ts.to_string()), + ), + ( + REFRESHED_AT_MS_META_KEY.to_string(), + Some(now_ms().to_string()), + ), + ( + VIEW_VERSION_META_KEY.to_string(), + Some(predicted.to_string()), + ), + ]) + .await?; + let actual = dataset.version().version; + if actual != predicted { + return Err(Error::Runtime { + message: format!( + "a concurrent commit raced this refresh (view version {actual}, \ + expected {predicted}); the refresh is unrecorded and the next \ + one will rebuild" + ), + }); + } + view_native.dataset.update(dataset); + Ok(predicted) +} + +fn now_ms() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} + +/// Evaluate the definition over `source`, restricted to `fragments` when +/// given, as batches in the view's schema. The filter is pushed into the +/// scan; projections are `(name, expression)` pairs, never spliced into SQL; +/// [`SOURCE_ROW_ID_COLUMN`] is filled by the scan's row id. +/// Which source rows a compute pass reads. +#[derive(Default)] +struct RowScope { + /// Read only these fragments. + fragments: Option>, + /// Read only rows created after this version. + created_after: Option, + /// Read only rows changed in place after the first version and no later + /// than the second. + updated_between: Option<(u64, u64)>, + /// Stop after this many rows. + limit: Option, +} + +async fn compute_stream( + source: &Dataset, + definition: &MaterializedViewDefinition, + scope: RowScope, + schema: SchemaRef, + rows_written: Arc, +) -> Result { + let RowScope { + fragments, + created_after, + updated_between, + limit, + } = scope; + let mut scanner = source.scan(); + if let Some(fragments) = fragments { + scanner.with_fragments(fragments); + } + scanner.with_row_id(); + // Narrowing keeps the definition's filter, so a row updated out of the + // view simply does not come back. Changed rows are named by the predicate + // `DatasetDelta::get_updated_rows` uses, not its streamed ids: an id list + // grows with the delta, this does not. + let updated_filter = updated_between.map(|(from, to)| { + format!( + "{ROW_CREATED_AT_VERSION} <= {from} \ + AND {ROW_LAST_UPDATED_AT_VERSION} > {from} \ + AND {ROW_LAST_UPDATED_AT_VERSION} <= {to}" + ) + }); + let created_filter = + created_after.map(|version| format!("{ROW_CREATED_AT_VERSION} > {version}")); + let clauses: Vec = definition + .filter + .clone() + .map(|f| format!("({f})")) + .into_iter() + .chain(updated_filter) + .chain(created_filter) + .collect(); + if !clauses.is_empty() { + scanner.filter(&clauses.join(" AND "))?; + } + let transforms: Vec<(&str, &str)> = definition + .projections + .iter() + .map(|p| (p.output.as_str(), p.expression.as_str())) + .collect(); + scanner.project_with_transform(&transforms)?; + // A scan reads a limit of zero as no limit at all, so a view capped at + // nothing is answered without one. + if limit == Some(0) { + return Ok(Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::empty(), + ))); + } + if let Some(limit) = limit { + let limit = i64::try_from(limit).map_err(|_| Error::InvalidInput { + message: format!("view limit {limit} exceeds the maximum of {}", i64::MAX), + })?; + scanner.limit(Some(limit), None)?; + } + + let out_schema = schema.clone(); + let mapped = scanner.try_into_stream().await?.map(move |batch| { + let batch = batch.map_err(|e| DataFusionError::External(Box::new(e)))?; + let mut columns = Vec::with_capacity(out_schema.fields().len()); + for field in out_schema.fields() { + let name = if field.name() == SOURCE_ROW_ID_COLUMN { + ROW_ID + } else { + field.name() + }; + let column = batch.column_by_name(name).ok_or_else(|| { + DataFusionError::Internal(format!( + "view column '{}' is not produced by the view's definition", + field.name() + )) + })?; + columns.push(column.clone()); + } + rows_written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + Ok(RecordBatch::try_new(out_schema.clone(), columns)?) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new(schema, mapped))) +} + +/// Commit the view's removals and additions as one change, on the exact +/// generation the refresh planned from. Lance rejects an overlapping +/// provenance key, but an unrelated write to the view is not a key conflict, +/// so the generation is checked here too. +async fn publish( + view_ds: &Dataset, + eviction: Option<(Vec, Vec)>, + new_fragments: Vec, + keys: Option, +) -> Result { + let planned = view_ds.version().version; + #[cfg(test)] + tests::hold_before_publish(view_ds.uri()).await; + let (updated_fragments, removed_fragment_ids) = eviction.unwrap_or_default(); + let committed = CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .execute(Transaction::new( + planned, + Operation::Update { + removed_fragment_ids, + updated_fragments, + new_fragments, + fields_modified: Vec::new(), + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: Vec::new(), + update_mode: None, + inserted_rows_filter: keys, + updated_fragment_offsets: None, + }, + None, + )) + .await?; + if committed.version().version != planned + 1 { + return Err(Error::Runtime { + message: format!( + "a concurrent commit raced this refresh (view version {}); + the refresh is unrecorded and the next one will rebuild", + committed.version().version + ), + }); + } + Ok(committed) +} + +/// A delta batch's row id column, borrowed: a delta batch is as large as one +/// fragment's deletions, so copying it out would be the unbounded step the +/// chunking exists to avoid. +fn row_ids_of(batch: &RecordBatch) -> Result<&UInt64Array> { + let column = batch.column_by_name(ROW_ID).ok_or_else(|| Error::Runtime { + message: format!("'{ROW_ID}' is missing from a delta batch"), + })?; + column + .as_primitive_opt::() + .ok_or_else(|| Error::Runtime { + message: "row ids are not UInt64".into(), + }) +} + +/// A provenance id no source row can hold, carried in every refresh's +/// inserted-rows filter: any two refreshes of one view overlap on it, so +/// the loser of a race conflicts at commit whatever rows each planned. +const REFRESH_TOKEN_ID: u64 = u64::MAX; + +/// A builder with no collected keys, for publishes that only remove rows. +fn empty_keys(view_ds: &Dataset) -> Result>> { + Ok(Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new( + vec![source_row_id_field_id(view_ds)?], + )))) +} + +/// An inserted-rows filter holding the refresh token plus whatever the +/// tee collected. +fn refresh_filter(keys: &Arc>) -> Result { + let mut keys = keys.lock().map_err(|_| Error::Runtime { + message: "the provenance key filter was poisoned mid-refresh".into(), + })?; + keys.insert(KeyValue::UInt64(REFRESH_TOKEN_ID)) + .map_err(|e| Error::Runtime { + message: format!("failed to mark the refresh's filter: {e}"), + })?; + Ok(keys.build()) +} + +/// Reconciled ids past this fall back to the streamed rebuild, whose memory +/// does not grow with the delta. +fn eviction_rebuild_cap() -> usize { + #[cfg(test)] + if let Some(cap) = tests::eviction_cap_override() { + return cap; + } + 4 * 1024 * 1024 +} + +/// Provenance ids per staged eviction: the delete predicate carries one +/// literal per id, so a whole delta at once is unbounded. Each chunk costs a +/// pass over the view's provenance column, which is why it is not smaller. +const EVICTION_CHUNK: usize = 64 * 1024; + +/// Accumulates the view's removals from bounded chunks of provenance ids into +/// the one set of fragment changes the refresh publishes. +struct Eviction { + /// The view as the chunks staged so far leave it. A chunk's delete has to + /// see the deletion vectors the earlier ones wrote, or it stages a + /// fragment that drops them. + snapshot: Dataset, + chunk: usize, + updated: HashMap, + removed: Vec, + pending: Vec, + staged: bool, + /// Largest the buffer ever got, which is the bound under test. + #[cfg(test)] + peak: usize, +} + +impl Eviction { + fn new(view_ds: &Dataset, chunk: usize) -> Self { + Self { + snapshot: view_ds.clone(), + chunk, + updated: HashMap::new(), + removed: Vec::new(), + pending: Vec::with_capacity(chunk), + staged: false, + #[cfg(test)] + peak: 0, + } + } + + /// Fill a chunk at a time. Taking the batch whole and splitting it would + /// hold a fragment's worth of ids and recopy the tail per chunk. + async fn push(&mut self, ids: &UInt64Array) -> Result<()> { + for id in ids.values() { + self.pending.push(*id); + #[cfg(test)] + { + self.peak = self.peak.max(self.pending.len()); + } + if self.pending.len() == self.chunk { + self.flush().await?; + } + } + Ok(()) + } + + /// Stage what is buffered, keeping the buffer's allocation. + async fn flush(&mut self) -> Result<()> { + let mut chunk = std::mem::take(&mut self.pending); + let staged = self.stage(&chunk).await; + chunk.clear(); + self.pending = chunk; + staged + } + + /// The staged fragment changes, or `None` where nothing was evicted. + async fn finish(mut self) -> Result, Vec)>> { + if !self.pending.is_empty() { + self.flush().await?; + } + if !self.staged { + return Ok(None); + } + let mut updated: Vec = self.updated.into_values().collect(); + updated.sort_unstable_by_key(|f| f.id); + Ok(Some((updated, self.removed))) + } + + async fn stage(&mut self, ids: &[u64]) -> Result<()> { + let (updated, removed) = stage_eviction(&self.snapshot, ids).await?; + self.snapshot = advance(&self.snapshot, &updated); + for fragment in updated { + self.updated.insert(fragment.id, fragment); + } + self.removed.extend(removed); + self.staged = true; + Ok(()) + } +} + +/// The view as staged removals leave it, without committing them. Fragments +/// are replaced in place so the fragment bitmap stays in step; an emptied one +/// is left alone, since the delta names each provenance id only once. +fn advance(view_ds: &Dataset, updated: &[Fragment]) -> Dataset { + if updated.is_empty() { + return view_ds.clone(); + } + let by_id: HashMap = updated.iter().map(|f| (f.id, f)).collect(); + let fragments = view_ds + .manifest + .fragments + .iter() + .map(|f| by_id.get(&f.id).map_or_else(|| f.clone(), |u| (*u).clone())) + .collect(); + let mut manifest = view_ds.manifest.as_ref().clone(); + manifest.fragments = Arc::new(fragments); + let mut snapshot = view_ds.clone(); + snapshot.manifest = Arc::new(manifest); + snapshot +} + +/// The fragment changes that remove the view's rows for `ids`, staged rather +/// than committed so they can ride in the refresh's single data commit. +async fn stage_eviction(view_ds: &Dataset, ids: &[u64]) -> Result<(Vec, Vec)> { + // An expression rather than SQL text: the id list is a value here, not a + // predicate string that grows with the delta and has to be parsed. + let predicate = col(SOURCE_ROW_ID_COLUMN).in_list( + ids.iter() + .map(|id| lit(ScalarValue::UInt64(Some(*id)))) + .collect(), + false, + ); + let staged = DeleteBuilder::from_expr(Arc::new(view_ds.clone()), predicate) + .execute_uncommitted() + .await?; + let Operation::Delete { + updated_fragments, + deleted_fragment_ids, + .. + } = staged.transaction.operation + else { + return Err(Error::Runtime { + message: "expected a delete when staging the view's evictions".into(), + }); + }; + Ok((updated_fragments, deleted_fragment_ids)) +} + +fn source_row_id_field_id(view_ds: &Dataset) -> Result { + view_ds + .schema() + .field(SOURCE_ROW_ID_COLUMN) + .map(|f| f.id) + .ok_or_else(|| Error::Runtime { + message: format!("the view has no '{SOURCE_ROW_ID_COLUMN}' column"), + }) +} + +/// Tee the provenance ids of everything written into `keys`. +fn collect_source_row_ids( + stream: SendableRecordBatchStream, + keys: Arc>, + written: Arc, +) -> SendableRecordBatchStream { + let schema = stream.schema(); + let mapped = stream.map(move |batch| { + let batch = batch?; + let column = batch + .column_by_name(SOURCE_ROW_ID_COLUMN) + .ok_or_else(|| { + DataFusionError::Internal(format!( + "'{SOURCE_ROW_ID_COLUMN}' is missing from the rows being written" + )) + })? + .as_primitive_opt::() + .ok_or_else(|| { + DataFusionError::Internal(format!("'{SOURCE_ROW_ID_COLUMN}' is not a uint64")) + })?; + let mut keys = keys + .lock() + .map_err(|_| DataFusionError::Internal("provenance key filter poisoned".into()))?; + for id in column.values() { + keys.insert(KeyValue::UInt64(*id)) + .map_err(|e| DataFusionError::Internal(e.to_string()))?; + } + written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + Ok(batch) + }); + Box::pin(RecordBatchStreamAdapter::new(schema, mapped)) +} + +#[cfg(test)] +mod tests { + + /// Park a refresh between planning and publication so a test can move + /// the view underneath it. Inert unless [`DRIFT_TARGET`] names this view. + pub(super) async fn hold_before_publish(uri: &str) { + { + let mut target = DRIFT_TARGET.lock().unwrap(); + if target.as_deref() != Some(uri) { + return; + } + // Take it: memory:// uris are relative and repeat across tests, so + // leaving it armed would park an unrelated refresh forever. + *target = None; + } + DRIFT_PLANNED.notify_one(); + DRIFT_RELEASED.notified().await; + } + + /// The rendezvous below is one global pair, so the cases that use it run + /// one at a time rather than trading each other's signals. + pub(super) static EVICTION_CAP: StdMutex> = StdMutex::new(None); + pub(super) fn eviction_cap_override() -> Option { + *EVICTION_CAP.lock().unwrap() + } + + pub(super) static DRIFT_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + pub(super) static DRIFT_TARGET: StdMutex> = StdMutex::new(None); + pub(super) static DRIFT_PLANNED: tokio::sync::Notify = tokio::sync::Notify::const_new(); + pub(super) static DRIFT_RELEASED: tokio::sync::Notify = tokio::sync::Notify::const_new(); + use arrow_array::{Int32Array, record_batch}; + use futures::TryStreamExt; + use lance::dataset::NewColumnTransform; + use lance_file::version::LanceFileVersion; + + use super::*; + use crate::connect; + use crate::connection::Connection; + use crate::index::Index; + use crate::index::scalar::BTreeIndexBuilder; + use crate::materialized_view::MaterializedView; + use crate::query::{ExecutableQuery, QueryBase, Select}; + use crate::table::{CompactionOptions, OptimizeAction}; + + async fn db_with_source(values: Vec) -> (Connection, Table) { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("x", Int32, values)).unwrap(); + let table = conn + .create_table("src", batch) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + (conn, table) + } + + async fn doubled_view(conn: &Connection) -> MaterializedView { + conn.create_materialized_view("doubled", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap() + } + + /// A refreshed doubled view over a source holding `values`. + async fn refreshed_doubled(values: Vec) -> (Connection, Table, MaterializedView) { + let (conn, source) = db_with_source(values).await; + let view = doubled_view(&conn).await; + view.refresh().execute().await.unwrap(); + (conn, source, view) + } + + async fn read(table: &Table, column: &str) -> Vec { + let batches = table + .query() + .select(Select::columns(&[column])) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut values: Vec = batches + .iter() + .flat_map(|batch| { + batch[column] + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .flatten() + .collect::>() + }) + .collect(); + values.sort(); + values + } + + async fn append(table: &Table, values: Vec) { + let batch = record_batch!(("x", Int32, values)).unwrap(); + table.add(batch).execute().await.unwrap(); + } + + #[tokio::test] + async fn test_first_refresh_materializes_the_view() { + let (conn, _) = db_with_source(vec![1, 2, 3]).await; + let view = doubled_view(&conn).await; + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(result.rows_written, 3); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]); + + // The watermark survives on the stored schema, not just the handle. + let reopened = conn.open_materialized_view("doubled").await.unwrap(); + let again = reopened.refresh().execute().await.unwrap(); + assert_eq!(again.mode, RefreshMode::NoOp); + assert_eq!(again.rows_written, 0); + } + + #[tokio::test] + async fn test_filter_selects_the_source_rows() { + let (conn, _) = db_with_source(vec![1, 20, 3, 40]).await; + let view = conn + .create_materialized_view("big", "src") + .select([("x", "x")]) + .only_if("x > 10") + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 2); + assert_eq!(read(view.table(), "x").await, vec![20, 40]); + } + + #[tokio::test] + async fn test_append_refreshes_incrementally() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await; + + append(&source, vec![5]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 10]); + } + + #[tokio::test] + async fn test_incremental_applies_the_filter() { + let (conn, source) = db_with_source(vec![1, 20]).await; + let view = conn + .create_materialized_view("big", "src") + .select([("x", "x")]) + .only_if("x > 10") + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + append(&source, vec![3, 30]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1); + assert_eq!(read(view.table(), "x").await, vec![20, 30]); + } + + /// The watermark has to advance even when no appended row matches, or the + /// same fragments would be rescanned by every later refresh. + #[tokio::test] + async fn test_incremental_with_nothing_matching_advances_the_watermark() { + let (conn, source) = db_with_source(vec![20]).await; + let view = conn + .create_materialized_view("big", "src") + .select([("x", "x")]) + .only_if("x > 10") + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + append(&source, vec![1, 2]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + + let again = view.refresh().execute().await.unwrap(); + assert_eq!(again.mode, RefreshMode::NoOp); + } + + /// Unlike a computed column, a view reflects source mutation: an update + /// rebuilds rather than going stale. + #[tokio::test] + async fn test_update_replaces_the_rows_it_changed() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + + // An update changes rows in place, so the view replaces exactly + // those rows and leaves the rest of what it holds alone. + source + .update() + .column("x", "20") + .only_if("x = 2") + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1, "only the changed row is recomputed"); + assert_eq!(read(view.table(), "twice").await, vec![2, 6, 40]); + } + + /// Legacy storage cannot serve the row-version columns update discovery + /// scans; appends stay incremental, updates rebuild rather than fail. + #[tokio::test] + async fn test_a_legacy_storage_source_is_reconciled() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("x", Int32, [1, 2, 3])).unwrap(); + let source = conn + .create_table("legacy_src", batch) + .write_options(crate::table::WriteOptions { + lance_write_params: Some(lance::dataset::WriteParams { + enable_stable_row_ids: true, + data_storage_version: Some(LanceFileVersion::Legacy), + ..Default::default() + }), + }) + .execute() + .await + .unwrap(); + let view = conn + .create_materialized_view("legacy_doubled", "legacy_src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + source + .add(record_batch!(("x", Int32, [4])).unwrap()) + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6, 8]); + + source + .update() + .column("x", "20") + .only_if("x = 2") + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![2, 6, 8, 40]); + } + + #[tokio::test] + async fn test_delete_evicts_the_view_rows_it_removed() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + + // A delete removes source rows without changing the ones that + // remain, so the view evicts exactly those rows and keeps the rest + // rather than recomputing every row it already held. + source.delete("x = 2").await.unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![2, 6]); + + // A delete and an append in one span: both are applied. + source.delete("x = 1").await.unwrap(); + source + .add(record_batch!(("x", Int32, vec![4])).unwrap()) + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![6, 8]); + } + + /// merge_insert commits an `Update`, and its by-source arm removes rows + /// rather than changing them, so the classifier must treat that + /// transaction form as a source of deletions. + #[tokio::test] + async fn test_merge_insert_by_source_delete_evicts_the_view_rows() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + + let batch = record_batch!(("x", Int32, vec![1, 3])).unwrap(); + let reader = arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut merge = source.merge_insert(&["x"]); + merge.when_not_matched_by_source_delete(None); + merge.execute(Box::new(reader)).await.unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![2, 6]); + } + + /// A refresh may only certify the generation it planned from. A write + /// that lands between planning and publication is drift the refresh did + /// not account for, so it aborts rather than stamp it as materialized. + #[tokio::test(flavor = "multi_thread")] + async fn test_refresh_aborts_rather_than_certify_view_drift() { + let _serial = DRIFT_LOCK.lock().await; + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("drifting_view", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + let certified = source_version_of(view.table()).await; + + // The source loses a row, so the next refresh plans an eviction. + source.delete("x = 2").await.unwrap(); + + let uri = view + .table() + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .uri() + .to_string(); + *DRIFT_TARGET.lock().unwrap() = Some(uri); + let refreshing = tokio::spawn(async move { view.refresh().execute().await }); + + // Move the view once the refresh has planned against it. + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("the refresh never reached the publication boundary"); + let drifted = conn.open_table("drifting_view").execute().await.unwrap(); + drifted.delete("twice = 6").await.unwrap(); + DRIFT_RELEASED.notify_one(); + + // Publishing removals and additions as one change makes the drift a + // conflict lance itself rejects; a pure append, which touches no + // existing fragment, still relies on the generation check. + let err = refreshing.await.unwrap().unwrap_err(); + let message = err.to_string(); + assert!( + message.contains("raced this refresh") || message.contains("preempted by concurrent"), + "got {err:?}" + ); + // The watermark still names the generation that was actually proven. + assert_eq!(source_version_of(&drifted).await, certified); + } + + async fn source_version_of(table: &Table) -> Option { + table + .schema() + .await + .unwrap() + .metadata() + .get(SOURCE_VERSION_META_KEY) + .cloned() + } + + /// A transaction file carries placeholder ids for the fragments it + /// creates. Into an empty source those collide with the ids the commit + /// assigns, so treating them as already-materialized drops the very + /// first rows while the watermark still advances past them. + #[tokio::test] + async fn test_merge_into_an_empty_source_is_materialized() { + let (_conn, source, view) = refreshed_doubled(vec![]).await; + assert_eq!(read(view.table(), "twice").await, Vec::::new()); + + let batch = record_batch!(("x", Int32, vec![1, 2])).unwrap(); + let reader = arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut merge = source.merge_insert(&["x"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all(); + merge.execute(Box::new(reader)).await.unwrap(); + + view.refresh().execute().await.unwrap(); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + // A second refresh must not double them either. + view.refresh().execute().await.unwrap(); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// A refresh publishes what it removes and what it adds as one change, + /// so an update never exposes the view without the rows it is replacing. + #[tokio::test(flavor = "multi_thread")] + async fn test_an_update_is_never_visible_as_a_gap() { + let _serial = DRIFT_LOCK.lock().await; + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("atomic_view", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + source + .update() + .column("x", "x + 10") + .execute() + .await + .unwrap(); + + let uri = view + .table() + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .uri() + .to_string(); + *DRIFT_TARGET.lock().unwrap() = Some(uri); + let refreshing = tokio::spawn(async move { view.refresh().execute().await }); + + // Read the view while the refresh is staged but not yet published. + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("the refresh never reached the publication boundary"); + let midway = conn.open_table("atomic_view").execute().await.unwrap(); + assert_eq!( + read(&midway, "twice").await, + vec![2, 4, 6], + "the pre-refresh rows must still be there in full" + ); + DRIFT_RELEASED.notify_one(); + + refreshing.await.unwrap().unwrap(); + let after = conn.open_table("atomic_view").execute().await.unwrap(); + assert_eq!(read(&after, "twice").await, vec![22, 24, 26]); + } + + async fn provenance_by_x(table: &Table) -> HashMap { + let batches = table + .query() + .select(Select::columns(&["x", SOURCE_ROW_ID_COLUMN])) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + batches + .iter() + .flat_map(|batch| { + let xs = batch["x"].as_any().downcast_ref::().unwrap(); + let ids = batch[SOURCE_ROW_ID_COLUMN].as_primitive::(); + (0..batch.num_rows()) + .map(|i| (xs.value(i), ids.value(i))) + .collect::>() + }) + .collect() + } + + /// One delta batch is as large as a fragment's deletions, so it arrives + /// well past a chunk: it has to be drained a chunk at a time without ever + /// holding the batch, and the passes' deletion vectors have to accumulate + /// rather than replace each other. + #[tokio::test] + async fn test_a_chunked_eviction_removes_every_row_it_names() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2, 3, 4, 5, 6]).await; + + let native = view.table().as_native().unwrap(); + let view_ds = native.dataset.get().await.unwrap().as_ref().clone(); + let provenance = provenance_by_x(view.table()).await; + + let mut eviction = Eviction::new(&view_ds, 2); + let batch = UInt64Array::from_iter_values([1, 2, 3, 4].map(|x| provenance[&x])); + eviction.push(&batch).await.unwrap(); + assert_eq!( + eviction.peak, 2, + "a batch past the chunk was buffered whole" + ); + let staged = eviction.finish().await.unwrap(); + assert!(staged.is_some(), "four ids over a chunk of two stage twice"); + publish(&view_ds, staged, Vec::new(), None).await.unwrap(); + native.dataset.reload().await.unwrap(); + + assert_eq!(read(view.table(), "x").await, vec![5, 6]); + } + + /// Two first rebuilds materialize the same source rows; the loser must + /// key-conflict at commit and land nothing, or the view doubles. + #[tokio::test] + async fn test_a_raced_first_rebuild_lands_nothing() { + let _guard = DRIFT_LOCK.lock().await; + let (conn, _) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("raced_rebuild", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + let native = view.table().as_native().unwrap(); + let view_ds = native.dataset.get().await.unwrap().as_ref().clone(); + let uri = view_ds.uri().to_string(); + *DRIFT_TARGET.lock().unwrap() = Some(uri); + let racing = tokio::spawn(async move { view.refresh().execute().await }); + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("the rebuild never reached the publication boundary"); + + // The other process's first rebuild, reduced to its commit. Its + // source rows are DISJOINT from ours -- the sentinel alone must make + // the two rebuilds conflict. + let batch = record_batch!( + ("x", Int32, [8, 9]), + ("twice", Int32, [16, 18]), + ("__source_row_id", UInt64, [7u64, 8]) + ) + .unwrap(); + let schema = Arc::new(ArrowSchema::from(view_ds.schema())); + let batch = RecordBatch::try_new( + schema.clone(), + schema + .fields() + .iter() + .map(|f| batch.column_by_name(f.name()).unwrap().clone()) + .collect(), + ) + .unwrap(); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter([Ok(batch)]), + )); + let keys = Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new(vec![ + source_row_id_field_id(&view_ds).unwrap(), + ]))); + let stream = collect_source_row_ids(stream, keys.clone(), Arc::new(AtomicU64::new(0))); + let write_txn = InsertBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await + .unwrap(); + let Operation::Append { fragments } = write_txn.operation else { + panic!("expected an append"); + }; + let filter = { + let mut keys = keys.lock().unwrap(); + keys.insert(KeyValue::UInt64(super::REFRESH_TOKEN_ID)) + .unwrap(); + keys.build() + }; + CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .execute(Transaction::new( + view_ds.version().version, + Operation::Update { + removed_fragment_ids: Vec::new(), + updated_fragments: Vec::new(), + new_fragments: fragments, + fields_modified: Vec::new(), + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: Vec::new(), + update_mode: None, + inserted_rows_filter: Some(filter), + updated_fragment_offsets: None, + }, + None, + )) + .await + .unwrap(); + DRIFT_RELEASED.notify_one(); + + let err = racing.await.unwrap().unwrap_err(); + // The loser lands nothing: only the winner's rows remain. + let raced = conn.open_table("raced_rebuild").execute().await.unwrap(); + assert_eq!( + read(&raced, "twice").await, + vec![16, 18], + "the losing rebuild must not union with the winner ({err})" + ); + } + + /// An incremental refresh racing any other refresh must land nothing, + /// even when their planned rows are disjoint: the shared token makes the + /// commits conflict. + #[tokio::test] + async fn test_a_raced_incremental_lands_nothing() { + let _guard = DRIFT_LOCK.lock().await; + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("raced_incremental", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + append(&source, vec![4]).await; + + let native = view.table().as_native().unwrap(); + native.dataset.reload().await.unwrap(); + let view_ds = native.dataset.get().await.unwrap().as_ref().clone(); + *DRIFT_TARGET.lock().unwrap() = Some(view_ds.uri().to_string()); + let racing = tokio::spawn(async move { view.refresh().execute().await }); + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("the refresh never reached the publication boundary"); + + // The concurrent refresh, reduced to its commit: disjoint rows, the + // shared token alone must collide. + let batch = record_batch!( + ("x", Int32, [9]), + ("twice", Int32, [18]), + ("__source_row_id", UInt64, [8u64]) + ) + .unwrap(); + let schema = Arc::new(ArrowSchema::from(view_ds.schema())); + let batch = RecordBatch::try_new( + schema.clone(), + schema + .fields() + .iter() + .map(|f| batch.column_by_name(f.name()).unwrap().clone()) + .collect(), + ) + .unwrap(); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter([Ok(batch)]), + )); + let keys = empty_keys(&view_ds).unwrap(); + let stream = collect_source_row_ids(stream, keys.clone(), Arc::new(AtomicU64::new(0))); + let write_txn = InsertBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await + .unwrap(); + let Operation::Append { fragments } = write_txn.operation else { + panic!("expected an append"); + }; + let filter = refresh_filter(&keys).unwrap(); + CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .execute(Transaction::new( + view_ds.version().version, + Operation::Update { + removed_fragment_ids: Vec::new(), + updated_fragments: Vec::new(), + new_fragments: fragments, + fields_modified: Vec::new(), + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: Vec::new(), + update_mode: None, + inserted_rows_filter: Some(filter), + updated_fragment_offsets: None, + }, + None, + )) + .await + .unwrap(); + DRIFT_RELEASED.notify_one(); + + let err = racing.await.unwrap().unwrap_err(); + let raced = conn + .open_table("raced_incremental") + .execute() + .await + .unwrap(); + assert_eq!( + read(&raced, "twice").await, + vec![2, 4, 6, 18], + "the losing increment must not land its rows ({err})" + ); + } + + /// Past the eviction cap the refresh falls back to the streamed rebuild, + /// and the result is identical either way. + #[tokio::test] + async fn test_oversized_delta_falls_back_to_rebuild() { + let (conn, source) = db_with_source((1..=20).collect()).await; + let view = doubled_view(&conn).await; + view.refresh().execute().await.unwrap(); + + source.delete("x <= 10").await.unwrap(); + *tests::EVICTION_CAP.lock().unwrap() = Some(4); + let result = view.refresh().execute().await; + *tests::EVICTION_CAP.lock().unwrap() = None; + let result = result.unwrap(); + assert_eq!( + result.mode, + RefreshMode::Rebuild, + "ten evictions, cap of four" + ); + assert_eq!(read(view.table(), "x").await, (11..=20).collect::>()); + + // Under the cap the same shape stays incremental. + source.delete("x = 11").await.unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "x").await, (12..=20).collect::>()); + } + + /// A cap of zero is a view that holds nothing, not a view without a cap. + #[tokio::test] + async fn test_zero_limit_holds_no_rows() { + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("empty", "src") + .select([("x", "x")]) + .limit(0) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 0); + assert_eq!(view.table().count_rows(None).await.unwrap(), 0); + + // Still nothing after the source grows, and the watermark advances. + append(&source, vec![4]).await; + view.refresh().execute().await.unwrap(); + assert_eq!(view.table().count_rows(None).await.unwrap(), 0); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// An update rewrites a whole fragment. When it lands on one appended + /// since the watermark, that fragment also holds rows the update never + /// touched -- rows the recompute does not cover and the append set no + /// longer reaches. + #[tokio::test] + async fn test_update_touching_a_new_fragment_keeps_its_untouched_rows() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await; + + // One fragment, appended after the watermark, holding both a row the + // update will change and a row it will not. + append(&source, vec![3, 40]).await; + source + .update() + .column("x", "99") + .only_if("x = 3") + .execute() + .await + .unwrap(); + + view.refresh().execute().await.unwrap(); + assert_eq!( + read(view.table(), "twice").await, + vec![2, 4, 80, 198], + "a row appended into the updated fragment went missing" + ); + } + + async fn compact(source: &Table) { + source + .optimize(OptimizeAction::Compact { + options: CompactionOptions::default(), + remap_options: None, + }) + .await + .unwrap(); + } + + /// A compaction rearranges rows without changing them, so it costs the + /// view nothing: the watermark advances and no row is recomputed. + #[tokio::test] + async fn test_compaction_alone_refreshes_incrementally() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await; + append(&source, vec![3]).await; + view.refresh().execute().await.unwrap(); + + compact(&source).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// Rows appended after a compaction are separable through the transaction + /// log: only the appended fragments are computed. + #[tokio::test] + async fn test_append_after_compaction_stays_incremental() { + let (_conn, source, view) = refreshed_doubled(vec![1]).await; + append(&source, vec![2]).await; + view.refresh().execute().await.unwrap(); + + compact(&source).await; + append(&source, vec![3]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]); + } + + /// An append swallowed by a later compaction cannot be told apart from + /// the rows the view already holds, so the refresh rebuilds -- once -- + /// rather than duplicate or drop. + #[tokio::test] + async fn test_append_folded_into_compaction_rebuilds() { + let (_conn, source, view) = refreshed_doubled(vec![1]).await; + + append(&source, vec![2]).await; + compact(&source).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// The create-time gate holds across a drop-and-recreate of the source + /// under the same name. + #[tokio::test] + async fn test_refresh_refuses_a_recreated_source_without_stable_row_ids() { + let (conn, _, view) = refreshed_doubled(vec![1]).await; + + conn.drop_table("src", &[]).await.unwrap(); + let batch = record_batch!(("x", Int32, [9])).unwrap(); + conn.create_table("src", batch).execute().await.unwrap(); + + let err = view.refresh().execute().await.unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("stable row ids")) + ); + } + + /// A change to a column the view does not read is not a reason to + /// rebuild: the exact signature check is scoped to the view's inputs. + #[tokio::test] + async fn test_unrelated_column_change_does_not_rebuild() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await; + + source + .add_columns() + .transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new( + vec![arrow_schema::Field::new( + "unrelated", + arrow_schema::DataType::Int32, + true, + )], + )))) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + #[tokio::test] + async fn test_full_forces_a_rebuild() { + let (_conn, source, view) = refreshed_doubled(vec![1]).await; + + append(&source, vec![2]).await; + let result = view.refresh().full(true).execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(result.rows_written, 2); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// A cap and incremental reconciliation do not compose: rows skipped at + /// the cap fall behind the watermark, so room freed later cannot be + /// refilled from any delta. A capped view rebuilds instead, which its + /// own cap keeps cheap. + #[tokio::test] + async fn test_limited_view_rebuilds_rather_than_reconcile() { + let (conn, source) = db_with_source(vec![1, 2]).await; + let view = conn + .create_materialized_view("capped", "src") + .select([("x", "x")]) + .limit(2) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + assert_eq!(read(view.table(), "x").await, vec![1, 2]); + + append(&source, vec![3, 4]).await; + source + .update() + .column("x", "11") + .only_if("x = 1") + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + let held = read(view.table(), "x").await; + assert_eq!(held.len(), 2, "the cap holds: {held:?}"); + let selectable = read(&source, "x").await; + assert!( + held.iter().all(|x| selectable.contains(x)), + "{held:?} is not a subset of {selectable:?}" + ); + } + + #[tokio::test] + async fn test_limit_caps_the_view() { + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("capped", "src") + .select([("x", "x")]) + .limit(4) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 3); + + // The cap counts already-held rows, so only one appended row lands. + append(&source, vec![4, 5, 6]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1); + assert_eq!(view.table().count_rows(None).await.unwrap(), 4); + + // At the cap, later appends only move the watermark. + append(&source, vec![7]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 0); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// The whole point of the fragment-swap commit: a rebuild must never + /// leave the view without its index definitions. + #[tokio::test] + async fn test_rebuild_retains_indexes() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + + view.table() + .create_index(&["twice"], Index::BTree(BTreeIndexBuilder::default())) + .execute() + .await + .unwrap(); + assert_eq!(view.table().list_indices().await.unwrap().len(), 1); + + source + .update() + .column("x", "x + 10") + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(view.table().list_indices().await.unwrap().len(), 1); + assert_eq!(read(view.table(), "twice").await, vec![22, 24, 26]); + + // The swapped-in rows are reachable through an indexed query. + let batches = view + .table() + .query() + .only_if("twice = 24") + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 1); + } + + #[tokio::test] + async fn test_rebuild_of_an_empty_result_is_an_empty_view() { + let (conn, _) = db_with_source(vec![1, 2]).await; + let view = conn + .create_materialized_view("none", "src") + .select([("x", "x")]) + .only_if("x > 100") + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(result.rows_written, 0); + assert_eq!(view.table().count_rows(None).await.unwrap(), 0); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// Provenance: every view row records the source row that produced it. + #[tokio::test] + async fn test_source_row_ids_are_recorded() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2, 3]).await; + + let batches = view + .table() + .query() + .select(Select::columns(&[SOURCE_ROW_ID_COLUMN])) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let total: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total, 3); + for batch in &batches { + assert_eq!(batch[SOURCE_ROW_ID_COLUMN].null_count(), 0); + } + } + + #[tokio::test] + async fn test_dropping_a_source_input_fails_the_refresh() { + let (conn, source) = db_with_source(vec![1]).await; + let view = conn + .create_materialized_view("v", "src") + .select([("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + append(&source, vec![2]).await; + source + .add_columns() + .transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new( + vec![arrow_schema::Field::new( + "y", + arrow_schema::DataType::Int32, + true, + )], + )))) + .execute() + .await + .unwrap(); + source.drop_columns(&["x"]).await.unwrap(); + + let err = view.refresh().execute().await.unwrap_err(); + assert!(matches!(err, Error::Schema { message } if message.contains("'x'"))); + } + + /// A pinned refresh materializes the source as of `version`; catching up + /// to the appends beyond it stays incremental. + #[tokio::test] + async fn test_pinned_refresh_and_catch_up() { + let (conn, source) = db_with_source(vec![1]).await; + let view = doubled_view(&conn).await; + let pinned = source.version().await.unwrap(); + + append(&source, vec![2]).await; + let result = view + .refresh() + .source_version(pinned) + .execute() + .await + .unwrap(); + assert_eq!(result.source_version, pinned); + assert_eq!(read(view.table(), "twice").await, vec![2]); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// Views chain: a view's stable row ids and provenance column make it a + /// source like any other, and the default projection takes its declared + /// columns without copying its provenance. + #[tokio::test] + async fn test_a_view_can_source_another_view() { + let (conn, source) = db_with_source(vec![1, 2, 30]).await; + let first = doubled_view(&conn).await; + first.refresh().execute().await.unwrap(); + + let second = conn + .create_materialized_view("second", "doubled") + .only_if("twice > 10") + .execute() + .await + .unwrap(); + assert!( + second + .definition() + .projections + .iter() + .all(|p| p.output != SOURCE_ROW_ID_COLUMN) + ); + let result = second.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 1); + assert_eq!(read(second.table(), "twice").await, vec![60]); + + append(&source, vec![50]).await; + first.refresh().execute().await.unwrap(); + let result = second.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(second.table(), "twice").await, vec![60, 100]); + } + + /// The watermark speaks only for the state a refresh left behind: a + /// direct write to the view is drift, and the next refresh rebuilds + /// rather than preserving it as current. + #[tokio::test] + async fn test_direct_view_mutation_forces_a_rebuild() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; + + view.table().delete("x = 1").await.unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// A dropped and recreated source reuses version numbers but never their + /// timestamps; the watermark must not vouch for the replacement's rows. + #[tokio::test] + async fn test_source_recreation_forces_a_rebuild() { + let (conn, _, view) = refreshed_doubled(vec![1]).await; + + conn.drop_table("src", &[]).await.unwrap(); + let batch = record_batch!(("x", Int32, [7])).unwrap(); + conn.create_table("src", batch) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![14]); + } + + /// In-process refreshes of one view serialize: the loser of the race + /// observes the winner's watermark instead of appending the same rows. + #[tokio::test(flavor = "multi_thread")] + async fn test_concurrent_refreshes_do_not_duplicate() { + let (_conn, source, view) = refreshed_doubled(vec![1]).await; + + append(&source, vec![2, 3]).await; + let (a, b) = tokio::join!(view.refresh().execute(), view.refresh().execute()); + let (a, b) = (a.unwrap(), b.unwrap()); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]); + let modes = [a.mode, b.mode]; + assert!(modes.contains(&RefreshMode::Incremental)); + assert!(modes.contains(&RefreshMode::NoOp)); + } + + /// A second handle's lazy cache must not defeat the lock: after another + /// handle's refresh commits, the stale handle plans from the reloaded + /// state and no-ops instead of appending the same fragments again. + #[tokio::test] + async fn test_a_second_handle_does_not_double_append() { + let (conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + let stale = conn.open_materialized_view("doubled").await.unwrap(); + + append(&source, vec![4]).await; + view.refresh().execute().await.unwrap(); + + let result = stale.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::NoOp); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6, 8]); + } + + /// A commit racing between a refresh's data commit and its stamp must + /// not be certified as the refresh's generation: the stamp aborts, and + /// the next refresh rebuilds from the drifted state. + #[tokio::test] + async fn test_stamp_aborts_on_a_racing_commit() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; + + let view_native = view.table().as_native().unwrap(); + let stale = view_native.dataset.get().await.unwrap().as_ref().clone(); + view.table().delete("x = 1").await.unwrap(); + + let err = stamp_watermark(view_native, stale, 99, 99).await; + assert!(err.is_err()); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// What refresh executes and what it stamps must be one generation: a + /// replaced definition wins over whatever a stale handle cached. + #[tokio::test] + async fn test_refresh_uses_the_latest_persisted_definition() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; + + let replacement = crate::materialized_view::MaterializedViewDefinition { + source_table: "src".into(), + projections: vec![ + crate::materialized_view::ViewProjection { + output: "x".into(), + expression: "x".into(), + }, + crate::materialized_view::ViewProjection { + output: "twice".into(), + expression: "x * 3".into(), + }, + ], + filter: None, + limit: None, + inputs: vec!["x".into()], + }; + let mut metadata = HashMap::new(); + metadata.insert( + crate::materialized_view::DEFINITION_META_KEY.to_string(), + crate::materialized_view::definition_to_metadata(&replacement).unwrap(), + ); + view.table() + .as_native() + .unwrap() + .replace_schema_metadata(metadata) + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![3, 6]); + } + + /// A persisted definition that does not produce this view's schema must + /// not refresh at all, let alone be certified. + #[tokio::test] + async fn test_definition_view_schema_mismatch_is_refused() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; + + let narrower = crate::materialized_view::MaterializedViewDefinition { + source_table: "src".into(), + projections: vec![crate::materialized_view::ViewProjection { + output: "x".into(), + expression: "x".into(), + }], + filter: None, + limit: None, + inputs: vec!["x".into()], + }; + let mut metadata = HashMap::new(); + metadata.insert( + crate::materialized_view::DEFINITION_META_KEY.to_string(), + crate::materialized_view::definition_to_metadata(&narrower).unwrap(), + ); + view.table() + .as_native() + .unwrap() + .replace_schema_metadata(metadata) + .await + .unwrap(); + + let err = view.refresh().execute().await.unwrap_err(); + assert!(matches!(err, Error::Schema { message } if message.contains("does not produce")),); + } + + /// An overlay replaces cell values without changing any file path; the + /// signature must see it, scoped to the columns the view reads like + /// data files are. + #[test] + fn test_fragment_signature_sees_overlays() { + use lance_file::version::ConcreteFileVersion; + use lance_table::format::DataFile; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + + let base = Fragment::new(7); + let mut file = DataFile::new_unstarted("f0.lance", ConcreteFileVersion::V2_1); + file.fields = vec![0, 1].into(); + let mut with_file = base.clone(); + with_file.files.push(file.clone()); + + let overlay = |field: i32| { + let mut data_file = DataFile::new_unstarted("o0.lance", ConcreteFileVersion::V2_1); + data_file.fields = vec![field].into(); + DataOverlayFile { + data_file, + coverage: OverlayCoverage::PerField(Vec::new()), + committed_version: 9, + } + }; + let relevant: HashSet = [0].into_iter().collect(); + + let mut overlaid_relevant = with_file.clone(); + overlaid_relevant.overlays.push(overlay(0)); + assert_ne!( + fragment_signature(&with_file, &relevant), + fragment_signature(&overlaid_relevant, &relevant), + ); + + let mut overlaid_unrelated = with_file.clone(); + overlaid_unrelated.overlays.push(overlay(5)); + assert_eq!( + fragment_signature(&with_file, &relevant), + fragment_signature(&overlaid_unrelated, &relevant), + ); + } + + /// An output whose name needs quoting flows through as a projection + /// alias, never spliced into SQL text. + #[tokio::test] + async fn test_output_names_needing_quotes() { + let (conn, _) = db_with_source(vec![1, 2]).await; + let view = conn + .create_materialized_view("v", "src") + .select([("double value", "x * 2")]) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 2); + assert_eq!(read(view.table(), "double value").await, vec![2, 4]); + } + + /// MemWAL tiers are visible to reads but not to the refresh scan, so LSM + /// state disqualifies every participant: the source at create, either + /// side at refresh, and the view can never accept a spec. Retained + /// un-compacted rows (the catch-up flag outlives unset) count as state. + #[tokio::test] + async fn lsm_state_disqualifies_source_and_view() { + use crate::table::LsmWriteSpec; + use arrow_array::RecordBatchIterator; + + let tmp_dir = tempfile::tempdir().unwrap(); + let conn = connect(tmp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + // Hand-rolled: the LSM primary key must be non-nullable, which + // record_batch! cannot express. + let schema = Arc::new(ArrowSchema::new(vec![ + arrow_schema::Field::new("id", arrow_schema::DataType::Int64, false), + arrow_schema::Field::new("x", arrow_schema::DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(arrow_array::Int64Array::from(vec![1, 2])) as _, + Arc::new(Int32Array::from(vec![1, 2])) as _, + ], + ) + .unwrap(); + let table = conn + .create_table("src", batch.clone()) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + + // An active-LSM source is refused at create. + let err = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap_err(); + assert!(err.to_string().contains("un-compacted"), "{err}"); + + // Unset with nothing written clears the state: the view creates, + // and a spec can never be installed over it. + table.unset_lsm_write_spec().await.unwrap(); + let view = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap(); + let err = view + .table() + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap_err(); + assert!(err.to_string().contains("materialized view"), "{err}"); + + // A source that acquires a spec after create fails refresh. + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + let err = view.refresh().execute().await.unwrap_err(); + assert!(err.to_string().contains("source table 'src'"), "{err}"); + + // Retained rows outlive unset: write through the WAL, unset, and + // refresh still refuses. + let mut merge = table.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all() + .use_lsm(true); + merge + .execute(Box::new(RecordBatchIterator::new( + vec![Ok(batch.clone())], + batch.schema(), + ))) + .await + .unwrap(); + table.unset_lsm_write_spec().await.unwrap(); + let err = view.refresh().execute().await.unwrap_err(); + assert!(err.to_string().contains("source table 'src'"), "{err}"); + } +} diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index 44a2874de..a06507ba0 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -104,6 +104,13 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) .into(), }); } + if crate::materialized_view::materialized_view_kind(&dataset.schema().metadata)?.is_some() { + return Err(Error::NotSupported { + message: "an LSM write spec cannot be installed on a materialized view: \ + rows in un-compacted tiers are invisible to refresh" + .into(), + }); + } let mut builder = dataset.initialize_mem_wal(); let writer_config_defaults = match spec { LsmWriteSpec::Bucket { diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 2715a163c..7d74bbae7 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -597,7 +597,7 @@ mod tests { /// A fragment spanning several scan batches exercises the streamed fill: /// the probe buffers only until the first gained value and the rest flows - /// through write_column a batch at a time. + /// through write_columns a batch at a time. #[tokio::test] async fn test_refresh_streams_a_multi_batch_fragment() { let values: Vec = (0..20_000).collect(); From d04ac7ed202181049cb871c2d6b97e5e5d9d54f0 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 21 Aug 2026 23:04:04 -0700 Subject: [PATCH 09/31] test: differential refresh harness for materialized views (#3932) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Example tests pin behaviors; the refresh contract is a property: after any sequence of source mutations, a view maintained by default refreshes equals the definition evaluated against the source directly, and so does a forced rebuild. This drives every mutation sequence up to length three -- appends, deletes, updates crossing the filter, compactions, unrelated column adds -- over an identity and a filtered view shape, checking against an oracle that shares nothing with the refresh path: a plain column scan with the filter applied in Rust. The oracle runs after every step because a later rebuild-forcing mutation silently heals an incremental error; end-state checks miss exactly the transient bugs that matter. A length-four sweep runs behind ignore. Named regressions additionally assert the refresh mode, which value comparison cannot: a wrongly rebuilding classifier still matches the oracle, so the append, unrelated-column and compaction cases pin that the incremental path actually ran. Stack created with GitHub Stacks CLI • Give Feedback 💬 --- rust/lancedb/src/materialized_view.rs | 4 + .../src/materialized_view/differential.rs | 730 ++++++++++++++++++ rust/lancedb/src/materialized_view/refresh.rs | 38 + 3 files changed, 772 insertions(+) create mode 100644 rust/lancedb/src/materialized_view/differential.rs diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 8b553165b..1d7cbb5e7 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -10,6 +10,10 @@ //! table. Queries, indexes and search work on the view unchanged. pub mod refresh; + +#[cfg(test)] +mod differential; + use std::collections::HashMap; use std::sync::Arc; diff --git a/rust/lancedb/src/materialized_view/differential.rs b/rust/lancedb/src/materialized_view/differential.rs new file mode 100644 index 000000000..396b4e65b --- /dev/null +++ b/rust/lancedb/src/materialized_view/differential.rs @@ -0,0 +1,730 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Differential refresh testing. +//! +//! The refresh contract is a property: after any sequence of source +//! mutations, a view maintained by default (incremental-where-possible) +//! refreshes equals the definition evaluated against the source directly, +//! and so does a forced rebuild. The oracle is an independent read of the +//! source -- plain column scan, filter applied in Rust -- so it shares +//! nothing with the refresh path it checks. +//! +//! The oracle runs after every step, not just at the end: a later mutation +//! that forces a rebuild would silently heal an incremental error, and those +//! transient errors are exactly the bugs this exists to catch. + +use arrow_array::{Float32Array, Int32Array, RecordBatch}; +use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; +use futures::{StreamExt, TryStreamExt}; +use lance::dataset::NewColumnTransform; +use std::sync::Arc; + +use super::MaterializedView; +use super::refresh::RefreshMode; +use crate::connect; +use crate::connection::Connection; +use crate::query::{ExecutableQuery, QueryBase, Select}; +use crate::table::{CompactionOptions, OptimizeAction, Table}; + +/// One source mutation, one per correctness-relevant class. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SrcOp { + /// Fresh non-colliding ids of both parities, so every other op has + /// view-resident rows to act on: the only op that should refresh + /// incrementally. + AppendNew, + /// Deletion in surviving fragments must break the pure-append check. + DeleteEven, + /// An in-place update; on the filtered shape it crosses the predicate, + /// so rows must leave the view. + UpdateOddScore, + /// Fragment rewrite/renumber must break the pure-append check. + Compact, + /// A column the view does not read must NOT force a rebuild. + AddColumn, + /// merge_insert commits an Update whose by-source arm deletes rows, so a + /// classifier that reads Update as "changed only" loses those deletions. + MergeDropLargest, + /// merge_insert that both changes existing rows and inserts new ones in + /// one transaction. + MergeUpsert, +} + +const ALL_OPS: [SrcOp; 7] = [ + SrcOp::AppendNew, + SrcOp::DeleteEven, + SrcOp::UpdateOddScore, + SrcOp::Compact, + SrcOp::AddColumn, + SrcOp::MergeDropLargest, + SrcOp::MergeUpsert, +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Shape { + /// SELECT id, score. + Identity, + /// SELECT id, score WHERE score > 50: additionally sensitive to rows + /// crossing the predicate. + Filtered, + /// SELECT id, score LIMIT 4. Which rows are held depends on the order + /// they were first materialized, so the oracle checks containment and + /// the cap rather than equality. + Limited, +} + +impl Shape { + fn filter(&self) -> Option<&'static str> { + match self { + Self::Identity | Self::Limited => None, + Self::Filtered => Some("score > 50"), + } + } + + fn matches(&self, score: f32) -> bool { + match self { + Self::Identity | Self::Limited => true, + Self::Filtered => score > 50.0, + } + } + + fn limit(&self) -> Option { + match self { + Self::Limited => Some(4), + _ => None, + } + } +} + +struct Case { + conn: Connection, + source: Table, + view: MaterializedView, + shape: Shape, + next_id: i32, + added_columns: u32, +} + +fn rows_batch(ids: &[i32]) -> RecordBatch { + let scores: Vec = ids.iter().map(|id| (*id * 10) as f32).collect(); + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("score", DataType::Float32, true), + ])), + vec![ + Arc::new(Int32Array::from(ids.to_vec())), + Arc::new(Float32Array::from(scores)), + ], + ) + .unwrap() +} + +fn merge_batch(ids: &[i32]) -> RecordBatch { + let scores: Vec = ids.iter().map(|id| (*id * 10 + 5) as f32).collect(); + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("score", DataType::Float32, true), + ])), + vec![ + Arc::new(Int32Array::from(ids.to_vec())), + Arc::new(Float32Array::from(scores)), + ], + ) + .unwrap() +} + +impl Case { + async fn new(shape: Shape) -> Self { + let conn = connect("memory://").execute().await.unwrap(); + let source = conn + .create_table("src", rows_batch(&[1, 2, 3, 4])) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + let mut builder = conn + .create_materialized_view("view", "src") + .select([("id", "id"), ("score", "score")]); + if let Some(filter) = shape.filter() { + builder = builder.only_if(filter); + } + if let Some(limit) = shape.limit() { + builder = builder.limit(limit as u64); + } + let view = builder.execute().await.unwrap(); + Self { + conn, + source, + view, + shape, + next_id: 100, + added_columns: 0, + } + } + + async fn apply(&mut self, op: SrcOp) { + match op { + SrcOp::AppendNew => { + // Mixed parity: the middle id is odd, so UpdateOddScore always + // has a filter-matching appended row to evict. + let ids = vec![self.next_id, self.next_id + 101, self.next_id + 202]; + self.next_id += 303; + self.source.add(rows_batch(&ids)).execute().await.unwrap(); + } + SrcOp::DeleteEven => { + self.source.delete("id % 2 = 0").await.unwrap(); + } + SrcOp::UpdateOddScore => { + self.source + .update() + .column("score", "-1.0") + .only_if("id % 2 = 1") + .execute() + .await + .unwrap(); + } + SrcOp::Compact => { + self.source + .optimize(OptimizeAction::Compact { + options: CompactionOptions::default(), + remap_options: None, + }) + .await + .unwrap(); + } + SrcOp::MergeDropLargest => { + let mut ids = self.source_ids().await; + ids.sort_unstable(); + ids.pop(); + if ids.is_empty() { + return; + } + let batch = rows_batch(&ids); + let reader = + arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut merge = self.source.merge_insert(&["id"]); + merge.when_not_matched_by_source_delete(None); + merge.execute(Box::new(reader)).await.unwrap(); + } + SrcOp::MergeUpsert => { + let mut ids = self.source_ids().await; + ids.sort_unstable(); + // One row that exists (updated in place) and one that does not. + let existing = ids.first().copied().unwrap_or(self.next_id); + let fresh = self.next_id; + self.next_id += 1; + let batch = merge_batch(&[existing, fresh]); + let reader = + arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut merge = self.source.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all(); + merge.execute(Box::new(reader)).await.unwrap(); + } + SrcOp::AddColumn => { + self.added_columns += 1; + let field = ArrowField::new( + format!("extra_{}", self.added_columns), + DataType::Int32, + true, + ); + self.source + .add_columns() + .transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new( + vec![field], + )))) + .execute() + .await + .unwrap(); + } + } + } + + async fn source_ids(&self) -> Vec { + read_rows( + self.source + .query() + .select(Select::columns(&["id", "score"])), + ) + .await + .into_iter() + .map(|(id, _)| id) + .collect() + } + + /// The definition's result, read independently of the refresh path: + /// plain column scan, filter applied here, sorted. + async fn oracle(&self) -> Vec<(i32, i32)> { + let mut rows = read_rows( + self.source + .query() + .select(Select::columns(&["id", "score"])), + ) + .await + .into_iter() + .filter(|(_, score)| self.shape.matches(*score as f32)) + .collect::>(); + rows.sort_unstable(); + rows + } + + async fn view_rows(&self) -> Vec<(i32, i32)> { + let mut rows = read_rows( + self.view + .table() + .query() + .select(Select::columns(&["id", "score"])), + ) + .await; + rows.sort_unstable(); + rows + } + + async fn check(&self, label: &str) -> Result<(), String> { + let expected = self.oracle().await; + let actual = self.view_rows().await; + let Some(cap) = self.shape.limit() else { + if expected != actual { + return Err(format!( + "{label}: view diverged from oracle\n expected: {expected:?}\n actual: {actual:?}" + )); + } + return Ok(()); + }; + // A capped view holds some subset of the definition's result, never + // more than the cap, and never the same row twice. + if actual.len() > cap { + return Err(format!( + "{label}: view holds {} rows, over its cap of {cap}: {actual:?}", + actual.len() + )); + } + let mut unique = actual.clone(); + unique.dedup(); + if unique.len() != actual.len() { + return Err(format!("{label}: view holds a row twice: {actual:?}")); + } + if let Some(stray) = actual.iter().find(|row| !expected.contains(row)) { + return Err(format!( + "{label}: view holds {stray:?}, which the definition does not select: {expected:?}" + )); + } + // Below the cap the view must be complete, or a row was lost. + if actual.len() < cap.min(expected.len()) { + return Err(format!( + "{label}: view holds {} of {} selectable rows under a cap of {cap}: {actual:?}", + actual.len(), + expected.len() + )); + } + Ok(()) + } +} + +async fn read_rows(query: impl ExecutableQuery) -> Vec<(i32, i32)> { + let batches = query + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + batches + .iter() + .flat_map(|batch| { + let ids = batch["id"].as_any().downcast_ref::().unwrap(); + let scores = batch["score"] + .as_any() + .downcast_ref::() + .unwrap(); + // Scores are integer-valued by construction; compare exactly. + (0..batch.num_rows()) + .map(|i| (ids.value(i), scores.value(i) as i32)) + .collect::>() + }) + .collect() +} + +/// Drive one mutation sequence: refresh + oracle-check after every step, +/// then a forced rebuild checked against the same oracle. +async fn run_sequence(ops: &[SrcOp], shape: Shape) -> Result<(), String> { + let label = format!("{shape:?} {ops:?}"); + let mut case = Case::new(shape).await; + case.view + .refresh() + .execute() + .await + .map_err(|e| format!("{label}: initial refresh failed: {e}"))?; + case.check(&format!("{label} (initial)")).await?; + + for (step, op) in ops.iter().enumerate() { + case.apply(*op).await; + case.view + .refresh() + .execute() + .await + .map_err(|e| format!("{label}: refresh at step {step} failed: {e}"))?; + case.check(&format!("{label} (step {step}, {op:?})")) + .await?; + } + + case.view + .refresh() + .full(true) + .execute() + .await + .map_err(|e| format!("{label}: final full refresh failed: {e}"))?; + case.check(&format!("{label} (final rebuild)")).await?; + // Silence the unused-connection lint without dropping it mid-case. + let _ = &case.conn; + Ok(()) +} + +/// Every op sequence up to `max_len`. +fn all_sequences(max_len: u32) -> Vec> { + let mut sequences = Vec::new(); + for len in 1..=max_len { + for mut index in 0..ALL_OPS.len().pow(len) { + let mut ops = Vec::with_capacity(len as usize); + for _ in 0..len { + ops.push(ALL_OPS[index % ALL_OPS.len()]); + index /= ALL_OPS.len(); + } + sequences.push(ops); + } + } + sequences +} + +async fn run_exhaustive(max_len: u32) { + let mut cases = Vec::new(); + for shape in [Shape::Identity, Shape::Filtered, Shape::Limited] { + for ops in all_sequences(max_len) { + cases.push((ops, shape)); + } + } + let failures: Vec = futures::stream::iter(cases) + .map(|(ops, shape)| async move { run_sequence(&ops, shape).await.err() }) + .buffer_unordered(8) + .filter_map(|failure| async move { failure }) + .collect() + .await; + assert!( + failures.is_empty(), + "{} sequences diverged; first: {}", + failures.len(), + failures[0] + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn differential_exhaustive() { + run_exhaustive(3).await; +} + +#[tokio::test(flavor = "multi_thread")] +#[ignore = "longer sweep; run manually"] +async fn differential_exhaustive_deep() { + run_exhaustive(4).await; +} + +/// Named interleavings that double as repro handles. The mode assertions pin +/// the classifier, which value comparison alone cannot: a wrongly rebuilt +/// view still matches the oracle. +#[tokio::test(flavor = "multi_thread")] +async fn differential_named_regressions() { + // An append is the one op that must stay incremental. + let mut case = Case::new(Shape::Identity).await; + case.view.refresh().execute().await.unwrap(); + case.apply(SrcOp::AppendNew).await; + let result = case.view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + case.check("append stays incremental").await.unwrap(); + + // A column the view does not read must not force a rebuild. + let mut case = Case::new(Shape::Identity).await; + case.view.refresh().execute().await.unwrap(); + case.apply(SrcOp::AddColumn).await; + let result = case.view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + + // Compaction rearranges rows without changing them: the watermark + // advances and nothing rebuilds. + let mut case = Case::new(Shape::Identity).await; + case.view.refresh().execute().await.unwrap(); + case.apply(SrcOp::AppendNew).await; + case.view.refresh().execute().await.unwrap(); + case.apply(SrcOp::Compact).await; + let result = case.view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + case.check("compaction alone").await.unwrap(); + + // Fragment bookkeeping stays coherent across the compaction: the next + // append is separable and computed alone. + case.apply(SrcOp::AppendNew).await; + let result = case.view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 3); + case.check("compact then append").await.unwrap(); + + // A row updated to no longer match the filter must leave the view -- + // and the fixture must prove the eviction happened, not merely that the + // end state matches: an update that never touched a view-resident row + // would also "match". + let mut case = Case::new(Shape::Filtered).await; + case.apply(SrcOp::AppendNew).await; + case.view.refresh().execute().await.unwrap(); + let before = case.view_rows().await.len(); + case.apply(SrcOp::UpdateOddScore).await; + case.view.refresh().execute().await.unwrap(); + let after = case.view_rows().await.len(); + assert!( + after < before, + "no view-resident row was evicted ({before} -> {after}); the fixture \ + no longer exercises the filtered-update transition" + ); + case.check("update crosses the filter").await.unwrap(); +} + +// --------------------------------------------------------------------------- +// Concurrency +// --------------------------------------------------------------------------- +// +// The sequential cases above cannot observe a cross-process race: the +// per-view refresh lock is process-local, so a second refresh in this +// process queues behind the first. What is missing is not more op +// sequences but a second process. These cases add one, and assert the same +// property the harness always asserts -- the view holds each row once. + +/// Rows the definition selects from the source: every id but the first, +/// read straight from the source, sharing nothing with the refresh path. +async fn concurrency_oracle(conn: &Connection) -> Vec { + let batches: Vec = conn + .open_table("src") + .execute() + .await + .unwrap() + .query() + .select(Select::columns(&["id"])) + .execute() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let mut ids = Vec::new(); + for batch in &batches { + let column = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + if column.value(i) > 1 { + ids.push(column.value(i)); + } + } + } + ids.sort_unstable(); + ids +} + +/// The view's ids, sorted. +async fn concurrency_view_ids(conn: &Connection) -> Vec { + let batches: Vec = conn + .open_table("mv") + .execute() + .await + .unwrap() + .query() + .select(Select::columns(&["id"])) + .execute() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let mut ids = Vec::new(); + for batch in &batches { + let column = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + ids.push(column.value(i)); + } + } + ids.sort_unstable(); + ids +} + +/// One refresh of the view at `MV_RACE_DIR`, in its own process. +/// +/// Setup happens before the start barrier so warm-up does not stagger the +/// two processes. What makes the race certain rather than likely is the +/// second barrier inside `refresh()` itself, which holds every participant +/// between staging and commit. +#[tokio::test] +#[ignore = "spawned as a child process by the concurrency cases"] +async fn cross_process_refresh_child() { + let Ok(dir) = std::env::var("MV_RACE_DIR") else { + return; + }; + let dir = std::path::PathBuf::from(dir); + let tag = std::env::var("MV_RACE_TAG").unwrap(); + + let conn = connect(dir.to_str().unwrap()).execute().await.unwrap(); + let table = conn.open_table("mv").execute().await.unwrap(); + let _ = table.schema().await.unwrap(); + let _ = table.count_rows(None).await.unwrap(); + let source = conn.open_table("src").execute().await.unwrap(); + let _ = source.count_rows(None).await.unwrap(); + let view = MaterializedView::from_table(table).await.unwrap(); + + std::fs::write(dir.join(format!("ready-{tag}")), b"1").unwrap(); + while !dir.join("START").exists() { + std::thread::sleep(std::time::Duration::from_millis(2)); + } + + let outcome = match view.refresh().execute().await { + Ok(result) => format!("committed rows={}", result.rows_written), + Err(err) if is_commit_conflict(&err) => "conflicted".to_string(), + Err(err) => format!("failed {err}"), + }; + std::fs::write(dir.join(format!("outcome-{tag}")), outcome).unwrap(); +} + +/// Whether a refresh lost its commit to a concurrent one, as opposed to +/// failing for any other reason. +fn is_commit_conflict(err: &crate::Error) -> bool { + let text = err.to_string(); + text.contains("Retryable commit conflict") || text.contains("preempted by concurrent") +} + +/// Two processes refreshing one view concurrently must leave the view +/// equal to the oracle: each selected row present exactly once. +/// +/// Both plan the same incremental delta from one watermark. A refresh is +/// meant to land on the generation it planned or leave nothing behind, so +/// at most one of them may write. +#[tokio::test] +async fn concurrent_refreshes_hold_each_row_once() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().to_str().unwrap().to_string(); + let conn = connect(&path).execute().await.unwrap(); + conn.create_table("src", rows_batch(&[1, 2, 3, 4])) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + let view = conn + .create_materialized_view("mv", "src") + .select([("id", "id"), ("score", "score")]) + .only_if("id > 1") + .execute() + .await + .unwrap(); + // Seed the watermark so the racing refreshes are both incremental. + view.refresh().execute().await.unwrap(); + + // Large enough that a refresh is real work rather than a formality. + let ids: Vec = (100..200_100).collect(); + conn.open_table("src") + .execute() + .await + .unwrap() + .add(rows_batch(&ids)) + .execute() + .await + .unwrap(); + + let tags = ["a", "b"]; + let exe = std::env::current_exe().unwrap(); + let children: Vec = tags + .iter() + .map(|tag| { + std::process::Command::new(&exe) + .args([ + "--exact", + "materialized_view::differential::cross_process_refresh_child", + "--ignored", + "--nocapture", + ]) + .env("MV_RACE_DIR", dir.path()) + .env("MV_RACE_SYNC", dir.path()) + .env("MV_RACE_PEERS", "2") + .env("MV_RACE_TAG", tag) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap() + }) + .collect(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(180); + while tags + .iter() + .any(|tag| !dir.path().join(format!("ready-{tag}")).exists()) + { + assert!( + std::time::Instant::now() < deadline, + "children never became ready" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + std::fs::write(dir.path().join("START"), b"1").unwrap(); + for (tag, mut child) in tags.iter().zip(children) { + let status = loop { + match child.try_wait().unwrap() { + Some(status) => break status, + None if std::time::Instant::now() >= deadline => { + child.kill().unwrap(); + panic!("child {tag} never finished"); + } + None => std::thread::sleep(std::time::Duration::from_millis(10)), + } + }; + assert!(status.success(), "child {tag} exited {status}"); + } + + // Both refreshes reached the commit boundary before either committed -- + // the in-refresh barrier guarantees it -- so exactly one may win. + let outcomes: Vec = tags + .iter() + .map(|tag| { + std::fs::read_to_string(dir.path().join(format!("outcome-{tag}"))) + .unwrap_or_else(|_| panic!("child {tag} recorded no outcome")) + }) + .collect(); + for tag in tags { + assert!( + dir.path().join(format!("planned-{tag}")).exists(), + "child {tag} never reached the commit boundary, so nothing was synchronized" + ); + } + let committed = outcomes.iter().filter(|o| o.contains("committed")).count(); + let conflicted = outcomes.iter().filter(|o| o.contains("conflicted")).count(); + assert_eq!( + (committed, conflicted), + (1, 1), + "exactly one refresh may win the generation both planned: {outcomes:?}" + ); + + let expected = concurrency_oracle(&conn).await; + let actual = concurrency_view_ids(&conn).await; + assert_eq!( + actual.len(), + expected.len(), + "the view holds {} rows, the oracle {}: a losing refresh left rows behind", + actual.len(), + expected.len() + ); + assert_eq!(actual, expected, "the view does not match the oracle"); +} diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index 7ac39a9f5..24d828e01 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -1056,6 +1056,8 @@ async fn publish( let planned = view_ds.version().version; #[cfg(test)] tests::hold_before_publish(view_ds.uri()).await; + #[cfg(test)] + tests::hold_until_peers_planned(); let (updated_fragments, removed_fragment_ids) = eviction.unwrap_or_default(); let committed = CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) .execute(Transaction::new( @@ -1342,6 +1344,42 @@ mod tests { pub(super) static DRIFT_TARGET: StdMutex> = StdMutex::new(None); pub(super) static DRIFT_PLANNED: tokio::sync::Notify = tokio::sync::Notify::const_new(); pub(super) static DRIFT_RELEASED: tokio::sync::Notify = tokio::sync::Notify::const_new(); + + /// Block until every participant in a cross-process race has planned and + /// staged its write, so the commits they then attempt genuinely contend + /// rather than depending on the scheduler to overlap them. Inert unless + /// `MV_RACE_SYNC` names a directory shared by the participants. + pub(super) fn hold_until_peers_planned() { + let (Ok(dir), Ok(tag), Ok(peers)) = ( + std::env::var("MV_RACE_SYNC"), + std::env::var("MV_RACE_TAG"), + std::env::var("MV_RACE_PEERS"), + ) else { + return; + }; + let dir = std::path::PathBuf::from(dir); + let peers: usize = peers.parse().unwrap(); + std::fs::write(dir.join(format!("planned-{tag}")), b"1").unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120); + while planned_count(&dir) < peers { + assert!( + std::time::Instant::now() < deadline, + "peers never reached the commit boundary" + ); + std::thread::sleep(std::time::Duration::from_millis(2)); + } + } + + fn planned_count(dir: &std::path::Path) -> usize { + std::fs::read_dir(dir) + .map(|entries| { + entries + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().starts_with("planned-")) + .count() + }) + .unwrap_or(0) + } use arrow_array::{Int32Array, record_batch}; use futures::TryStreamExt; use lance::dataset::NewColumnTransform; From 851fa16b47ecdde2b9aa116ad4100adfaf68279c Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 21 Aug 2026 23:08:41 -0700 Subject: [PATCH 10/31] feat(python): materialized view bindings (#3933) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposes materialized views to Python in both the async and sync clients: create_materialized_view / open_materialized_view / list_materialized_views on the connections, and MaterializedView / AsyncMaterializedView handles carrying the parsed definition and refresh(full=, source_version=), which returns the typed refresh result. select accepts column names, (alias, expression) pairs, or a dict of the same; the definition reads back off the stored schema, so a reopened handle needs no side channel. Remote connections raise NotImplementedError up front rather than failing deep in a request, matching the computed-column convention. Stack created with GitHub Stacks CLI • Give Feedback 💬 --- docs/src/python/python.md | 10 + python/python/lancedb/__init__.py | 8 + python/python/lancedb/_lancedb.pyi | 18 ++ python/python/lancedb/db.py | 166 +++++++++++ python/python/lancedb/materialized_view.py | 178 ++++++++++++ python/python/lancedb/namespace.py | 68 +++++ python/python/lancedb/remote/db.py | 27 ++ .../python/tests/test_materialized_views.py | 268 ++++++++++++++++++ python/src/connection.rs | 34 +++ python/src/lib.rs | 5 +- python/src/table.rs | 55 ++++ 11 files changed, 835 insertions(+), 2 deletions(-) create mode 100644 python/python/lancedb/materialized_view.py create mode 100644 python/python/tests/test_materialized_views.py diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 70b2a7207..8a24ea199 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -102,6 +102,12 @@ listing a storage directory. ::: lancedb.job.AsyncJob +## Materialized Views (Synchronous) + +::: lancedb.materialized_view.MaterializedView + +::: lancedb.materialized_view.MaterializedViewDefinition + ## Expressions Type-safe expression builder for filters and projections. Use these instead @@ -295,6 +301,10 @@ Table hold your actual data as a collection of records / rows. ::: lancedb.table.AsyncBranches +## Materialized Views (Asynchronous) + +::: lancedb.materialized_view.AsyncMaterializedView + ## Indices (Asynchronous) Indices can be created on a table to speed up queries. This section diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 0ceda4558..aa473c5f8 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -32,6 +32,11 @@ from .functions import ( UdfDefinition as UdfDefinition, udf as udf, ) +from .materialized_view import ( + AsyncMaterializedView, + MaterializedView, + MaterializedViewDefinition, +) from .table import AsyncTable, Table from .types import BaseTokenizerType from ._lancedb import Session @@ -506,6 +511,9 @@ async def connect_async( __all__ = [ + "AsyncMaterializedView", + "MaterializedView", + "MaterializedViewDefinition", "connect", "connect_async", "tokenize", diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 59537f45a..b23d79c85 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -197,6 +197,15 @@ class Connection(object): cur_namespace_path: Optional[List[str]] = None, new_namespace_path: Optional[List[str]] = None, ) -> None: ... + async def create_materialized_view( + self, + name: str, + source: str, + projections: Optional[List[Tuple[str, str]]] = None, + filter: Optional[str] = None, + limit: Optional[int] = None, + ) -> Table: ... + async def list_materialized_views(self) -> List[str]: ... async def drop_table( self, name: str, namespace_path: Optional[List[str]] = None ) -> None: ... @@ -355,6 +364,9 @@ class Table: ) -> AddColumnsResult: ... async def refresh_column(self, column: str) -> RefreshColumnResult: ... async def refresh_column_async(self, column: str) -> Job: ... + async def refresh_materialized_view( + self, full: bool = False, source_version: Optional[int] = None + ) -> RefreshMaterializedViewResult: ... async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ... async def alter_columns( self, columns: list[dict[str, Any]] @@ -704,6 +716,12 @@ class RefreshColumnResult: rows_filled: int version: int +class RefreshMaterializedViewResult: + mode: str + rows_written: int + source_version: int + version: int + class AlterColumnsResult: version: int diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index af18b6944..7ad749920 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -47,6 +47,12 @@ from . import __version__ from ._lancedb import connect as lancedb_connect # type: ignore from .functions import FunctionVersion, UdfDefinition from .job import AsyncJob, Job, _function_job +from .materialized_view import ( + AsyncMaterializedView, + MaterializedView, + SelectArg, + normalize_select, +) from .table import ( AsyncTable, LanceTable, @@ -510,6 +516,70 @@ class DBConnection(EnforceOverrides): """ raise NotImplementedError + def create_materialized_view( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> MaterializedView: + """Define a materialized view named ``name`` over the table ``source``. + + The view is created empty, with the query recorded in its schema + metadata; ``view.refresh()`` computes the rows. The view is a normal + table: it can be queried, indexed and searched, and it appears in + ``table_names``. Local databases only. + + The source table must have stable row ids (create it with the + ``new_table_enable_stable_row_ids`` storage option): they keep the + view's provenance valid across source compactions, and cannot be + enabled after a table exists. + + Parameters + ---------- + name: str + The name of the view. + source: str + The name of the source table, in this database. + select: list or dict, optional + The view's columns: column names, ``(alias, SQL expression)`` + pairs, or a dict of the same. Omitting it selects every source + column, expanded against the source schema at creation time. + where: str, optional + SQL predicate; only matching source rows appear in the view. + limit: int, optional + Cap the view at this many rows, in materialization order. + + Returns + ------- + MaterializedView + """ + raise NotImplementedError( + "materialized views are not supported on this connection type" + ) + + def open_materialized_view(self, name: str) -> MaterializedView: + """Open the materialized view named ``name``. + + Raises ``ValueError`` if the table exists but is not a materialized + view. + """ + raise NotImplementedError( + "materialized views are not supported on this connection type" + ) + + def list_materialized_views(self) -> List[str]: + """The names of the materialized views in this database. + + Found by reading every table's schema, so this costs an open per + table. + """ + raise NotImplementedError( + "materialized views are not supported on this connection type" + ) + def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): """Drop a table from the database. @@ -1136,6 +1206,58 @@ class LanceDBConnection(DBConnection): tbl.checkout(version) return tbl + @override + def create_materialized_view( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> MaterializedView: + """Define a materialized view named ``name`` over the table ``source``. + See + [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. + + Examples + -------- + >>> import lancedb + >>> db = lancedb.connect( + ... "./.lancedb", + ... storage_options={"new_table_enable_stable_row_ids": "true"}, + ... ) + >>> data = [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}] + >>> table = db.create_table("people", data) + >>> view = db.create_materialized_view( + ... "adults", + ... "people", + ... select=["name", ("shout", "upper(name)")], + ... where="age >= 18", + ... ) + >>> result = view.refresh() + >>> result.rows_written + 1 + """ + LOOP.run( + self._conn.create_materialized_view( + name, source, select=select, where=where, limit=limit + ) + ) + return MaterializedView(self.open_table(name)) + + @override + def open_materialized_view(self, name: str) -> MaterializedView: + """Open the materialized view named ``name``.""" + view = MaterializedView(self.open_table(name)) + view.definition + return view + + @override + def list_materialized_views(self) -> List[str]: + """The names of the materialized views in this database.""" + return LOOP.run(self._conn.list_materialized_views()) + def clone_table( self, target_table_name: str, @@ -1906,6 +2028,50 @@ class AsyncConnection(object): await tbl.checkout(version) return tbl + async def create_materialized_view( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> AsyncMaterializedView: + """Define a materialized view named ``name`` over the table ``source``. + See + [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. + """ + inner = await self._inner.create_materialized_view( + name, + source, + projections=normalize_select(select), + filter=where, + limit=limit, + ) + return AsyncMaterializedView(AsyncTable(inner)) + + async def open_materialized_view(self, name: str) -> AsyncMaterializedView: + """Open the materialized view named ``name``. + + Raises ``ValueError`` if the table exists but is not a materialized + view. + """ + if self.uri.startswith("db://"): + raise NotImplementedError( + "materialized views are supported only on local databases" + ) + view = AsyncMaterializedView(await self.open_table(name)) + await view.definition() + return view + + async def list_materialized_views(self) -> List[str]: + """The names of the materialized views in this database. + + Found by reading every table's schema, so this costs an open per + table. + """ + return await self._inner.list_materialized_views() + async def clone_table( self, target_table_name: str, diff --git a/python/python/lancedb/materialized_view.py b/python/python/lancedb/materialized_view.py new file mode 100644 index 000000000..5abb44dc0 --- /dev/null +++ b/python/python/lancedb/materialized_view.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Materialized views: tables defined by a query over a source table and +maintained by refresh. See ``DBConnection.create_materialized_view``.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union + +from .background_loop import LOOP + +if TYPE_CHECKING: + import pyarrow as pa + + from ._lancedb import RefreshMaterializedViewResult + from .table import AsyncTable, LanceTable + +DEFINITION_META_KEY = b"mv.definition" + +SelectArg = Union[ + str, + Sequence[Union[str, Tuple[str, str]]], + Dict[str, str], + None, +] + + +@dataclass +class MaterializedViewDefinition: + """The query that defines a materialized view.""" + + source_table: str + """Name of the source table, in the same database as the view.""" + projections: List[Tuple[str, str]] + """``(output column, SQL expression)`` pairs, in view schema order.""" + filter: Optional[str] = None + """SQL predicate selecting the source rows the view holds.""" + limit: Optional[int] = None + """Cap on the number of rows the view holds.""" + inputs: List[str] = field(default_factory=list) + """Source columns the projections and filter read.""" + + +def _definition_from_schema( + schema: "pa.Schema", name: str +) -> MaterializedViewDefinition: + metadata = schema.metadata or {} + raw = metadata.get(DEFINITION_META_KEY) + if raw is None: + raise ValueError(f"Table '{name}' is not a materialized view") + value = json.loads(raw) + kind = value.get("kind") + if kind != "select": + raise NotImplementedError( + f"materialized view '{name}' is defined by '{kind}', which this " + "version of lancedb cannot refresh" + ) + return MaterializedViewDefinition( + source_table=value["source_table"], + projections=[ + (p["output"], p["expression"]) for p in value.get("projections", []) + ], + filter=value.get("filter"), + limit=value.get("limit"), + inputs=value.get("inputs", []), + ) + + +def _quote_identifier(name: str) -> str: + """Quote a column name as a Lance SQL identifier (backticks).""" + escaped = name.replace("`", "``") + return f"`{escaped}`" + + +def normalize_select(select: SelectArg) -> Optional[List[Tuple[str, str]]]: + """``select`` items may be a column name, an ``(alias, expression)`` pair, + or a dict of the same. A bare name projects itself and is quoted, so any + valid column name works; dict and pair entries are kept verbatim because + their right side is an expression. + + A lone string is one column, not a sequence of its characters.""" + if select is None: + return None + if isinstance(select, str): + select = [select] + if isinstance(select, dict): + return list(select.items()) + normalized = [] + for item in select: + if isinstance(item, str): + normalized.append((item, _quote_identifier(item))) + else: + alias, expression = item + normalized.append((alias, expression)) + return normalized + + +class AsyncMaterializedView: + """A handle on a materialized view: its table plus its definition. + + Obtained from ``AsyncConnection.create_materialized_view`` or + ``AsyncConnection.open_materialized_view``. + """ + + def __init__(self, table: "AsyncTable"): + self._table = table + + def __repr__(self) -> str: + return f"AsyncMaterializedView(name={self.name!r})" + + @property + def name(self) -> str: + return self._table.name + + @property + def table(self) -> "AsyncTable": + """The view, as the table it is. Queries, indexes and search all + apply; writes are not blocked, but a rebuild replaces them.""" + return self._table + + async def definition(self) -> MaterializedViewDefinition: + """The query that defines the view, read from its stored schema.""" + return _definition_from_schema(await self._table.schema(), self.name) + + async def refresh( + self, *, full: bool = False, source_version: Optional[int] = None + ) -> "RefreshMaterializedViewResult": + """Recompute the view from its source. + + The refresh is incremental when the source's changes can be + reconciled into the view -- rows added, changed or removed since the + last one -- and otherwise rebuilds. ``full=True`` forces a rebuild; + ``source_version`` refreshes to that source version instead of the + latest. + + Concurrent refreshes of one view do not duplicate its rows. Two that + plan the same source rows conflict on commit, and the loser raises + rather than writing them a second time. + """ + return await self._table._inner.refresh_materialized_view( + full=full, source_version=source_version + ) + + +class MaterializedView: + """Synchronous variant of + [AsyncMaterializedView][lancedb.materialized_view.AsyncMaterializedView].""" + + def __init__(self, table: "LanceTable"): + self._table = table + self._async = AsyncMaterializedView(table._table) + + def __repr__(self) -> str: + return f"MaterializedView(name={self.name!r})" + + @property + def name(self) -> str: + return self._table.name + + @property + def table(self) -> "LanceTable": + """The view, as the table it is.""" + return self._table + + @property + def definition(self) -> MaterializedViewDefinition: + """The query that defines the view, read from its stored schema.""" + return _definition_from_schema(self._table.schema, self.name) + + def refresh( + self, *, full: bool = False, source_version: Optional[int] = None + ) -> "RefreshMaterializedViewResult": + """Recompute the view from its source. See + [AsyncMaterializedView.refresh][lancedb.materialized_view.AsyncMaterializedView.refresh].""" + return LOOP.run(self._async.refresh(full=full, source_version=source_version)) diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index 0e60bd218..f2e553321 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -61,6 +61,11 @@ from lance_namespace import ( NamespaceExistsRequest, TableExistsRequest, ) +from lancedb.materialized_view import ( + AsyncMaterializedView, + MaterializedView, + SelectArg, +) from lancedb.table import AsyncTable, LanceTable, Table from lancedb.util import validate_table_name from lancedb.common import DATA @@ -619,6 +624,42 @@ class LanceNamespaceDBConnection(DBConnection): tbl.checkout(version) return tbl + @override + def create_materialized_view( + self, + name: str, + source: str, + *, + select: "SelectArg" = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> "MaterializedView": + """Define a materialized view over a table in the root namespace. + See + [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. + """ + return MaterializedView( + self.open_table( + LOOP.run( + self._inner.create_materialized_view( + name, source, select=select, where=where, limit=limit + ) + ).name + ) + ) + + @override + def open_materialized_view(self, name: str) -> "MaterializedView": + """Open the materialized view named ``name``.""" + view = MaterializedView(self.open_table(name)) + view.definition + return view + + @override + def list_materialized_views(self) -> List[str]: + """The names of the materialized views in the root namespace.""" + return LOOP.run(self._inner.list_materialized_views()) + @override def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): if namespace_path is None: @@ -1141,6 +1182,33 @@ class AsyncLanceNamespaceDBConnection: route_pushdown_to_rust=self._route_pushdown_to_rust, ) + async def create_materialized_view( + self, + name: str, + source: str, + *, + select: "SelectArg" = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> "AsyncMaterializedView": + """Define a materialized view over a table in the root namespace.""" + view = await self._inner.create_materialized_view( + name, source, select=select, where=where, limit=limit + ) + # Reopen through the namespace so the view's table carries the + # namespace client and pushdown configuration a bare inner table lacks. + return AsyncMaterializedView(await self.open_table(view.name)) + + async def open_materialized_view(self, name: str) -> "AsyncMaterializedView": + """Open the materialized view named ``name``.""" + view = AsyncMaterializedView(await self.open_table(name)) + await view.definition() + return view + + async def list_materialized_views(self) -> List[str]: + """The names of the materialized views in the root namespace.""" + return await self._inner.list_materialized_views() + async def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): """Drop a table from the namespace.""" if namespace_path is None: diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index 822756a34..b228cfb5b 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -25,6 +25,7 @@ from ..common import DATA from ..db import DBConnection, LOOP from ..functions import FunctionVersion, UdfDefinition from ..job import AsyncJob, Job +from ..materialized_view import MaterializedView, SelectArg if TYPE_CHECKING: from .._lancedb import JobDescription, JobInfo @@ -648,6 +649,32 @@ class RemoteDBConnection(DBConnection): namespace_path=namespace_path, ) + @override + def create_materialized_view( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> MaterializedView: + raise NotImplementedError( + "materialized views are supported only on local databases" + ) + + @override + def open_materialized_view(self, name: str) -> MaterializedView: + raise NotImplementedError( + "materialized views are supported only on local databases" + ) + + @override + def list_materialized_views(self) -> List[str]: + raise NotImplementedError( + "materialized views are supported only on local databases" + ) + @override def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): """Drop a table from the database. diff --git a/python/python/tests/test_materialized_views.py b/python/python/tests/test_materialized_views.py new file mode 100644 index 000000000..5fa3aa4fb --- /dev/null +++ b/python/python/tests/test_materialized_views.py @@ -0,0 +1,268 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import lancedb +import pytest +from lancedb.materialized_view import MaterializedViewDefinition + + +STABLE_ROW_IDS = {"new_table_enable_stable_row_ids": "true"} + + +def make_db(tmp_path): + db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS) + db.create_table( + "people", + [ + {"name": "ada", "age": 36}, + {"name": "kid", "age": 7}, + {"name": "grace", "age": 85}, + ], + ) + return db + + +def test_create_refresh_and_query(tmp_path): + db = make_db(tmp_path) + view = db.create_materialized_view( + "adults", + "people", + select=["name", ("shout", "upper(name)")], + where="age >= 18", + ) + assert view.name == "adults" + assert view.table.count_rows() == 0 + + result = view.refresh() + assert result.mode == "rebuild" + assert result.rows_written == 2 + + rows = view.table.search().to_list() + assert sorted(row["shout"] for row in rows) == ["ADA", "GRACE"] + + +def test_definition_round_trips(tmp_path): + db = make_db(tmp_path) + db.create_materialized_view("adults", "people", where="age >= 18") + + view = db.open_materialized_view("adults") + assert view.definition == MaterializedViewDefinition( + source_table="people", + projections=[("name", "`name`"), ("age", "`age`")], + filter="age >= 18", + inputs=["age", "name"], + ) + + +def test_incremental_refresh_after_append(tmp_path): + db = make_db(tmp_path) + view = db.create_materialized_view("copy", "people") + view.refresh() + + db.open_table("people").add([{"name": "alan", "age": 41}]) + result = view.refresh() + assert result.mode == "incremental" + assert result.rows_written == 1 + assert view.table.count_rows() == 4 + + assert view.refresh().mode == "no_op" + + +def test_incremental_refresh_after_update(tmp_path): + db = make_db(tmp_path) + view = db.create_materialized_view("copy", "people") + view.refresh() + + db.open_table("people").update(where="name = 'kid'", values={"age": 8}) + result = view.refresh() + assert result.mode == "incremental" + assert result.rows_written == 1 + rows = view.table.search().to_list() + assert sorted(row["age"] for row in rows) == [8, 36, 85] + + +def test_legacy_storage_source_update_rebuilds(tmp_path): + db = lancedb.connect( + tmp_path, + storage_options={**STABLE_ROW_IDS, "new_table_data_storage_version": "legacy"}, + ) + db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]) + view = db.create_materialized_view("copy", "people") + view.refresh() + + db.open_table("people").update(where="name = 'kid'", values={"age": 8}) + result = view.refresh() + assert result.mode == "rebuild" + rows = view.table.search().to_list() + assert sorted(row["age"] for row in rows) == [8, 36] + + +def test_list_and_not_a_view(tmp_path): + db = make_db(tmp_path) + db.create_materialized_view("adults", "people", where="age >= 18") + + assert db.list_materialized_views() == ["adults"] + with pytest.raises(ValueError, match="not a materialized view"): + db.open_materialized_view("people") + + +def test_invalid_expression_fails_at_create(tmp_path): + db = make_db(tmp_path) + with pytest.raises(Exception, match="missing"): + db.create_materialized_view("bad", "people", select=[("x", "missing + 1")]) + assert "bad" not in db.list_tables().tables + + +@pytest.mark.asyncio +async def test_async_create_refresh_and_open(tmp_path): + db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) + await db.create_table("people", [{"name": "ada", "age": 36}]) + + view = await db.create_materialized_view( + "shouts", "people", select=[("shout", "upper(name)")] + ) + result = await view.refresh() + assert result.mode == "rebuild" + assert result.rows_written == 1 + + reopened = await db.open_materialized_view("shouts") + definition = await reopened.definition() + assert definition.projections == [("shout", "upper(name)")] + assert await db.list_materialized_views() == ["shouts"] + + +@pytest.mark.asyncio +async def test_async_incremental(tmp_path): + db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) + await db.create_table("people", [{"name": "ada", "age": 36}]) + view = await db.create_materialized_view("copy", "people") + await view.refresh() + + table = await db.open_table("people") + await table.add([{"name": "alan", "age": 41}]) + result = await view.refresh() + assert result.mode == "incremental" + assert result.rows_written == 1 + + +def test_source_requires_stable_row_ids(tmp_path): + db = lancedb.connect(tmp_path) + db.create_table("plain", [{"x": 1}]) + with pytest.raises(Exception, match="stable row ids"): + db.create_materialized_view("v", "plain") + + +def test_bare_select_names_are_quoted(tmp_path): + db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS) + db.create_table("odd_names", [{"order item": "widget", "select": 2}]) + + view = db.create_materialized_view( + "quoted", "odd_names", select=["order item", "select"] + ) + result = view.refresh() + assert result.rows_written == 1 + rows = view.table.search().to_list() + assert rows[0]["order item"] == "widget" + assert rows[0]["select"] == 2 + + +@pytest.mark.asyncio +async def test_async_remote_is_refused_without_network(): + db = await lancedb.connect_async( + "db://nowhere", api_key="sk_test", region="us-east-1" + ) + with pytest.raises(NotImplementedError, match="local"): + await db.create_materialized_view("v", "src") + with pytest.raises(NotImplementedError, match="local"): + await db.open_materialized_view("v") + with pytest.raises(NotImplementedError, match="local"): + await db.list_materialized_views() + + +def test_scalar_select_is_one_column(tmp_path): + db = make_db(tmp_path) + view = db.create_materialized_view("just_name", "people", select="name") + view.refresh() + rows = view.table.search().to_list() + assert set(rows[0]) - {"__source_row_id"} == {"name"} + assert sorted(row["name"] for row in rows) == ["ada", "grace", "kid"] + + +@pytest.mark.asyncio +async def test_async_scalar_select_is_one_column(tmp_path): + db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) + await db.create_table("people", [{"name": "ada", "age": 36}]) + view = await db.create_materialized_view("just_name", "people", select="name") + await view.refresh() + rows = await view.table.query().to_list() + assert set(rows[0]) - {"__source_row_id"} == {"name"} + + +def test_limit_above_i64_max_is_refused(tmp_path): + db = make_db(tmp_path) + with pytest.raises(ValueError, match="exceeds the maximum"): + db.create_materialized_view("too_big", "people", limit=2**63) + # The boundary is fine, and zero still means an empty view. + db.create_materialized_view("at_max", "people", limit=2**63 - 1) + empty = db.create_materialized_view("none", "people", limit=0) + empty.refresh() + assert empty.table.count_rows() == 0 + + +def _namespace_db(tmp_path): + return lancedb.connect_namespace( + "dir", + {"root": str(tmp_path)}, + storage_options=STABLE_ROW_IDS, + ) + + +def test_namespace_connection_materialized_views(tmp_path): + db = _namespace_db(tmp_path) + db.create_table( + "people", + [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}], + storage_options=STABLE_ROW_IDS, + ) + + view = db.create_materialized_view("adults", "people", where="age >= 18") + view.refresh() + assert view.table.count_rows() == 1 + assert db.list_materialized_views() == ["adults"] + + reopened = db.open_materialized_view("adults") + assert reopened.definition.source_table == "people" + with pytest.raises(ValueError, match="not a materialized view"): + db.open_materialized_view("people") + + +@pytest.mark.asyncio +async def test_async_namespace_connection_materialized_views(tmp_path): + db = lancedb.connect_namespace_async( + "dir", + {"root": str(tmp_path)}, + storage_options=STABLE_ROW_IDS, + ) + await db.create_table( + "people", + [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}], + storage_options=STABLE_ROW_IDS, + ) + + view = await db.create_materialized_view("adults", "people", where="age >= 18") + await view.refresh() + assert await view.table.count_rows() == 1 + assert await db.list_materialized_views() == ["adults"] + + reopened = await db.open_materialized_view("adults") + assert (await reopened.definition()).source_table == "people" + + # The view's table came through the namespace, not straight from the + # inner connection: a bare inner table carries no namespace context, so + # its pushdown routing differs from a table the namespace opened. + through_namespace = await db.open_table("adults") + for handle in (view.table, reopened.table): + assert ( + handle._route_pushdown_to_rust == through_namespace._route_pushdown_to_rust + ) + assert handle._namespace_path == through_namespace._namespace_path diff --git a/python/src/connection.rs b/python/src/connection.rs index 87870b800..4143ac9c9 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -333,6 +333,40 @@ impl Connection { }) } + #[pyo3(signature = (name, source, projections=None, filter=None, limit=None))] + pub fn create_materialized_view( + self_: PyRef<'_, Self>, + name: String, + source: String, + projections: Option>, + filter: Option, + limit: Option, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + future_into_py(self_.py(), async move { + let mut builder = inner.create_materialized_view(name, source); + if let Some(projections) = projections { + builder = builder.select(projections); + } + if let Some(filter) = filter { + builder = builder.only_if(filter); + } + if let Some(limit) = limit { + builder = builder.limit(limit); + } + let view = builder.execute().await.infer_error()?; + Ok(Table::new(view.table().clone())) + }) + } + + pub fn list_materialized_views(self_: PyRef<'_, Self>) -> PyResult> { + let inner = self_.get_inner()?.clone(); + future_into_py(self_.py(), async move { + let views = inner.list_materialized_views().await.infer_error()?; + Ok(views.into_iter().map(|view| view.name).collect::>()) + }) + } + #[pyo3(signature = (name, namespace_path=None))] pub fn drop_table( self_: PyRef<'_, Self>, diff --git a/python/src/lib.rs b/python/src/lib.rs index 756b3557f..c1dfbc02b 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -16,8 +16,8 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery}; use session::Session; use table::{ AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken, - LsmWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, Table, UpdateFieldMetadataResult, - UpdateResult, + LsmWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult, + Table, UpdateFieldMetadataResult, UpdateResult, }; pub mod arrow; @@ -60,6 +60,7 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/src/table.rs b/python/src/table.rs index cb4752cce..a3a7c9d68 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -441,6 +441,41 @@ impl From for RefreshColumnResult { } } +#[pyclass(get_all, from_py_object)] +#[derive(Clone, Debug)] +pub struct RefreshMaterializedViewResult { + pub mode: String, + pub rows_written: u64, + pub source_version: u64, + pub version: u64, +} + +#[pymethods] +impl RefreshMaterializedViewResult { + pub fn __repr__(&self) -> String { + format!( + "RefreshMaterializedViewResult(mode={}, rows_written={}, source_version={}, version={})", + self.mode, self.rows_written, self.source_version, self.version + ) + } +} + +impl From for RefreshMaterializedViewResult { + fn from(result: lancedb::RefreshMaterializedViewResult) -> Self { + let mode = match result.mode { + lancedb::RefreshMode::Rebuild => "rebuild", + lancedb::RefreshMode::Incremental => "incremental", + lancedb::RefreshMode::NoOp => "no_op", + }; + Self { + mode: mode.to_string(), + rows_written: result.rows_written, + source_version: result.source_version, + version: result.version, + } + } +} + #[pymethods] impl AddColumnsResult { pub fn __repr__(&self) -> String { @@ -1588,6 +1623,26 @@ impl Table { }) } + #[pyo3(signature = (full=false, source_version=None))] + pub fn refresh_materialized_view( + self_: PyRef<'_, Self>, + full: bool, + source_version: Option, + ) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + future_into_py(self_.py(), async move { + let view = lancedb::MaterializedView::from_table(inner) + .await + .infer_error()?; + let mut builder = view.refresh().full(full); + if let Some(version) = source_version { + builder = builder.source_version(version); + } + let result = builder.execute().await.infer_error()?; + Ok(RefreshMaterializedViewResult::from(result)) + }) + } + pub fn add_columns_with_schema( self_: PyRef<'_, Self>, schema: PyArrowType, From e98d8ac6857fa058fa07e005e0796c2a9a940d12 Mon Sep 17 00:00:00 2001 From: Drew Gallardo Date: Fri, 21 Aug 2026 23:37:12 -0700 Subject: [PATCH 11/31] feat!: rename branch merge to cherry_pick (#3986) This PR is a **breaking** rename of #3686. merge reads like git merge w/ three-way, replay history, combine two lines of work. That is not this API. This call takes one additive change on a branch and lands it on main. New column, including a blob column. Main's existing columns are not rewritten. If it cannot land, you get `status="failed"` and `diff.errors`, not a merge conflict to resolve. Cherry-pick is terminology that aligns more with that. ```python table = db.open_table("images") table.branches.create("exp") exp = table.branches.checkout("exp") exp.add_columns({"tag": "cast('draft' as string)"}) diff = table.branches.diff("exp") preview = table.branches.cherry_pick("exp", dry_run=True) result = table.branches.cherry_pick("exp") if result["status"] == "cherryPicked": print("landed at", result["mainVersionAfter"]) elif result["status"] == "failed": print(result["diff"]["errors"]) ``` ### Behavior - Remote / Enterprise only. Local still NotSupported. - HTTP 409 is not an exception. It is Ok with status="failed" and diff.errors (CherryPickError). - Unknown error / status codes still parse as Unknown. - Requests are not retried. 409 is final and carries the body. - Endpoint is POST /v1/table/{id}/branches/cherry_pick/. - merge_insert and Table.merge are unchanged. ### Testing - `cargo test -p lancedb --features remote diff_branch` - `cargo test -p lancedb --features remote cherry_pick` - `pytest python/python/tests/test_remote_db.py -k cherry_pick` - node `remote.test.ts` diffs / cherry-picks path --- docs/src/js/classes/Branches.md | 50 ++++++------ docs/src/js/globals.md | 6 +- docs/src/js/interfaces/BranchDiff.md | 24 ++---- .../{MergeBlocker.md => CherryPickError.md} | 6 +- docs/src/js/interfaces/CherryPickPreview.md | 17 +++++ ...rgeBranchResult.md => CherryPickResult.md} | 12 +-- docs/src/js/interfaces/MergePreview.md | 17 ----- nodejs/__test__/remote.test.ts | 26 +++---- nodejs/lancedb/index.ts | 6 +- nodejs/lancedb/table.ts | 37 +++++---- nodejs/src/table.rs | 6 +- python/python/lancedb/_lancedb.pyi | 2 +- python/python/lancedb/table.py | 22 +++--- python/python/tests/test_remote_db.py | 17 ++--- python/src/table.rs | 4 +- rust/lancedb/src/remote/table.rs | 76 ++++++++++--------- rust/lancedb/src/table.rs | 30 ++++---- .../table/{branch_merge.rs => cherry_pick.rs} | 27 ++++--- 18 files changed, 188 insertions(+), 197 deletions(-) rename docs/src/js/interfaces/{MergeBlocker.md => CherryPickError.md} (54%) create mode 100644 docs/src/js/interfaces/CherryPickPreview.md rename docs/src/js/interfaces/{MergeBranchResult.md => CherryPickResult.md} (60%) delete mode 100644 docs/src/js/interfaces/MergePreview.md rename rust/lancedb/src/table/{branch_merge.rs => cherry_pick.rs} (86%) diff --git a/docs/src/js/classes/Branches.md b/docs/src/js/classes/Branches.md index 5296d0d08..6680fbfee 100644 --- a/docs/src/js/classes/Branches.md +++ b/docs/src/js/classes/Branches.md @@ -37,6 +37,31 @@ latest and stays writable. *** +### cherryPick() + +```ts +cherryPick(fromBranch, dryRun): Promise +``` + +Cherry-pick a branch onto main. + +Set `dryRun` to `true` to preview. A failed cherry-pick resolves +with `status: "failed"` instead of throwing. + +#### Parameters + +* **fromBranch**: `string` + Branch to cherry-pick from. + +* **dryRun**: `boolean` = `false` + When true, only preview. Defaults to false. + +#### Returns + +`Promise`<[`CherryPickResult`](../interfaces/CherryPickResult.md)> + +*** + ### create() ```ts @@ -112,28 +137,3 @@ List all branches, mapping name to branch metadata. #### Returns `Promise`<`Record`<`string`, [`BranchContents`](BranchContents.md)>> - -*** - -### merge() - -```ts -merge(fromBranch, dryRun): Promise -``` - -Merge a branch into main. - -Set `dryRun` to `true` to preview the merge. A rejected merge resolves -with `status: "rejected"` instead of throwing. - -#### Parameters - -* **fromBranch**: `string` - Branch to merge from. - -* **dryRun**: `boolean` = `false` - When true, only preview the merge. Defaults to false. - -#### Returns - -`Promise`<[`MergeBranchResult`](../interfaces/MergeBranchResult.md)> diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 462907cfd..4deeed1bc 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -59,6 +59,9 @@ - [BranchIndexSummary](interfaces/BranchIndexSummary.md) - [BranchRowCountSummary](interfaces/BranchRowCountSummary.md) - [BucketStats](interfaces/BucketStats.md) +- [CherryPickError](interfaces/CherryPickError.md) +- [CherryPickPreview](interfaces/CherryPickPreview.md) +- [CherryPickResult](interfaces/CherryPickResult.md) - [ClientConfig](interfaces/ClientConfig.md) - [ColumnAlteration](interfaces/ColumnAlteration.md) - [ColumnOrdering](interfaces/ColumnOrdering.md) @@ -99,9 +102,6 @@ - [LsmStats](interfaces/LsmStats.md) - [LsmWriteSpec](interfaces/LsmWriteSpec.md) - [MemtableStats](interfaces/MemtableStats.md) -- [MergeBlocker](interfaces/MergeBlocker.md) -- [MergeBranchResult](interfaces/MergeBranchResult.md) -- [MergePreview](interfaces/MergePreview.md) - [MergeResult](interfaces/MergeResult.md) - [NativeOAuthConfig](interfaces/NativeOAuthConfig.md) - [OAuthConfig](interfaces/OAuthConfig.md) diff --git a/docs/src/js/interfaces/BranchDiff.md b/docs/src/js/interfaces/BranchDiff.md index 224be1991..b408ebe11 100644 --- a/docs/src/js/interfaces/BranchDiff.md +++ b/docs/src/js/interfaces/BranchDiff.md @@ -50,6 +50,14 @@ changedColumns: BranchColumnChange[]; *** +### errors + +```ts +errors: CherryPickError[]; +``` + +*** + ### fromBranch ```ts @@ -66,22 +74,6 @@ mainVersion: number; *** -### mergeBlockers - -```ts -mergeBlockers: MergeBlocker[]; -``` - -*** - -### mergeable - -```ts -mergeable: boolean; -``` - -*** - ### parentVersion ```ts diff --git a/docs/src/js/interfaces/MergeBlocker.md b/docs/src/js/interfaces/CherryPickError.md similarity index 54% rename from docs/src/js/interfaces/MergeBlocker.md rename to docs/src/js/interfaces/CherryPickError.md index 6c8f84b37..84f8fe012 100644 --- a/docs/src/js/interfaces/MergeBlocker.md +++ b/docs/src/js/interfaces/CherryPickError.md @@ -2,11 +2,11 @@ *** -[@lancedb/lancedb](../globals.md) / MergeBlocker +[@lancedb/lancedb](../globals.md) / CherryPickError -# Interface: MergeBlocker +# Interface: CherryPickError -A reason why a branch cannot currently be merged. +A reason why a cherry-pick cannot currently land. ## Properties diff --git a/docs/src/js/interfaces/CherryPickPreview.md b/docs/src/js/interfaces/CherryPickPreview.md new file mode 100644 index 000000000..9620068a2 --- /dev/null +++ b/docs/src/js/interfaces/CherryPickPreview.md @@ -0,0 +1,17 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / CherryPickPreview + +# Interface: CherryPickPreview + +Changes that would be, or were, promoted by a cherry-pick. + +## Properties + +### promotedColumns + +```ts +promotedColumns: string[]; +``` diff --git a/docs/src/js/interfaces/MergeBranchResult.md b/docs/src/js/interfaces/CherryPickResult.md similarity index 60% rename from docs/src/js/interfaces/MergeBranchResult.md rename to docs/src/js/interfaces/CherryPickResult.md index 61c2d0d33..c837ab143 100644 --- a/docs/src/js/interfaces/MergeBranchResult.md +++ b/docs/src/js/interfaces/CherryPickResult.md @@ -2,11 +2,11 @@ *** -[@lancedb/lancedb](../globals.md) / MergeBranchResult +[@lancedb/lancedb](../globals.md) / CherryPickResult -# Interface: MergeBranchResult +# Interface: CherryPickResult -Result of previewing or attempting a branch merge. +Result of previewing or attempting a cherry-pick. ## Properties @@ -29,7 +29,7 @@ optional mainVersionAfter: number; ### preview ```ts -preview: MergePreview; +preview: CherryPickPreview; ``` *** @@ -38,9 +38,9 @@ preview: MergePreview; ```ts status: + | "failed" | "unknown" - | "rejected" | "ready" | "notImplemented" - | "merged"; + | "cherryPicked"; ``` diff --git a/docs/src/js/interfaces/MergePreview.md b/docs/src/js/interfaces/MergePreview.md deleted file mode 100644 index 0d9717289..000000000 --- a/docs/src/js/interfaces/MergePreview.md +++ /dev/null @@ -1,17 +0,0 @@ -[**@lancedb/lancedb**](../README.md) • **Docs** - -*** - -[@lancedb/lancedb](../globals.md) / MergePreview - -# Interface: MergePreview - -Changes that would be, or were, promoted by a branch merge. - -## Properties - -### promotedColumns - -```ts -promotedColumns: string[]; -``` diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index e766b3d2a..01320d2b0 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -311,7 +311,7 @@ describe("remote connection", () => { expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]); }); - it("diffs and merges remote branches", async () => { + it("diffs and cherry-picks remote branches", async () => { const sampleDiff = { fromBranch: "exp", parentVersion: 1, @@ -333,10 +333,9 @@ describe("remote connection", () => { changedColumns: [], addedIndexes: [], removedIndexes: [], - mergeable: true, - mergeBlockers: [], + errors: [], }; - const mergeBodies: Record[] = []; + const cherryPickBodies: Record[] = []; await withMockDatabase( (req, res) => { @@ -366,17 +365,16 @@ describe("remote connection", () => { .end(JSON.stringify(sampleDiff)); return; } - if (path.endsWith("/branches/merge/")) { - mergeBodies.push(body); + if (path.endsWith("/branches/cherry_pick/")) { + cherryPickBodies.push(body); const dryRun = body["dry_run"] === true; const response = { - status: dryRun ? "ready" : "rejected", + status: dryRun ? "ready" : "failed", diff: dryRun ? sampleDiff : { ...sampleDiff, - mergeable: false, - mergeBlockers: [ + errors: [ { code: "baseMoved", message: "main has advanced" }, ], }, @@ -398,19 +396,19 @@ describe("remote connection", () => { await expect(branches.diff("exp")).resolves.toEqual(sampleDiff); - const rejected = await branches.merge("exp"); - expect(rejected.status).toBe("rejected"); - expect(rejected.diff.mergeBlockers).toEqual([ + const failed = await branches.cherryPick("exp"); + expect(failed.status).toBe("failed"); + expect(failed.diff.errors).toEqual([ { code: "baseMoved", message: "main has advanced" }, ]); - const preview = await branches.merge("exp", true); + const preview = await branches.cherryPick("exp", true); expect(preview.status).toBe("ready"); expect(preview.preview.promotedColumns).toEqual(["tag"]); }, ); - expect(mergeBodies).toEqual([ + expect(cherryPickBodies).toEqual([ // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format { from_branch: "exp", dry_run: false }, // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 6a5bfe3b4..aa9c7ef13 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -135,10 +135,10 @@ export { BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, - MergeBlocker, + CherryPickError, BranchDiff, - MergePreview, - MergeBranchResult, + CherryPickPreview, + CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 964c2cea3..4e323ec64 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -1557,8 +1557,8 @@ export interface BranchRowCountSummary { deltaAvailable: boolean; } -/** A reason why a branch cannot currently be merged. */ -export interface MergeBlocker { +/** A reason why a cherry-pick cannot currently land. */ +export interface CherryPickError { code: string; message: string; } @@ -1578,20 +1578,19 @@ export interface BranchDiff { changedColumns: BranchColumnChange[]; addedIndexes: BranchIndexSummary[]; removedIndexes: BranchIndexSummary[]; - mergeable: boolean; - mergeBlockers: MergeBlocker[]; + errors: CherryPickError[]; } -/** Changes that would be, or were, promoted by a branch merge. */ -export interface MergePreview { +/** Changes that would be, or were, promoted by a cherry-pick. */ +export interface CherryPickPreview { promotedColumns: string[]; } -/** Result of previewing or attempting a branch merge. */ -export interface MergeBranchResult { - status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown"; +/** Result of previewing or attempting a cherry-pick. */ +export interface CherryPickResult { + status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown"; diff: BranchDiff; - preview: MergePreview; + preview: CherryPickPreview; mainVersionAfter?: number; } @@ -1654,21 +1653,21 @@ export class Branches { } /** - * Merge a branch into main. + * Cherry-pick a branch onto main. * - * Set `dryRun` to `true` to preview the merge. A rejected merge resolves - * with `status: "rejected"` instead of throwing. + * Set `dryRun` to `true` to preview. A failed cherry-pick resolves + * with `status: "failed"` instead of throwing. * - * @param fromBranch Branch to merge from. - * @param dryRun When true, only preview the merge. Defaults to false. + * @param fromBranch Branch to cherry-pick from. + * @param dryRun When true, only preview. Defaults to false. */ - async merge( + async cherryPick( fromBranch: string, dryRun: boolean = false, - ): Promise { - return (await this.#inner.merge( + ): Promise { + return (await this.#inner.cherryPick( fromBranch, dryRun, - )) as unknown as MergeBranchResult; + )) as unknown as CherryPickResult; } } diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index b15491202..694b6a704 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -1605,18 +1605,18 @@ impl Branches { } #[napi(ts_return_type = "Promise>")] - pub async fn merge( + pub async fn cherry_pick( &self, from_branch: String, dry_run: Option, ) -> napi::Result { let result = self .inner - .merge_branch(&from_branch, dry_run.unwrap_or(false)) + .cherry_pick(&from_branch, dry_run.unwrap_or(false)) .await .default_error()?; serde_json::to_value(result).map_err(|err| { - napi::Error::from_reason(format!("failed to serialize branch merge result: {err}")) + napi::Error::from_reason(format!("failed to serialize cherry-pick result: {err}")) }) } } diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index b23d79c85..648469579 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -432,7 +432,7 @@ class Branches: async def checkout(self, name: str, version: Optional[int] = None) -> Table: ... async def delete(self, name: str) -> None: ... async def diff(self, from_branch: str) -> Dict[str, Any]: ... - async def merge( + async def cherry_pick( self, from_branch: str, dry_run: bool = False ) -> Dict[str, Any]: ... diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 913ab5289..0c84b4036 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -6801,21 +6801,21 @@ class Branches: """Diff a branch against main.""" return LOOP.run(self._table.branches.diff(from_branch)) - def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]: - """Merge a branch into main, or dry-run. + def cherry_pick(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]: + """Cherry-pick a branch onto main, or dry-run. Parameters ---------- from_branch: str - Branch to merge from. + Branch to cherry-pick from. dry_run: bool, default False - When True, only preview. When False, attempt the merge. + When True, only preview. When False, attempt the cherry-pick. Notes ----- - A rejected merge returns ``status="rejected"`` instead of raising. + A failed cherry-pick returns ``status="failed"`` instead of raising. """ - return LOOP.run(self._table.branches.merge(from_branch, dry_run)) + return LOOP.run(self._table.branches.cherry_pick(from_branch, dry_run)) def _wrap( self, async_table: "AsyncTable", version: Optional[int] = None @@ -6951,9 +6951,11 @@ class AsyncBranches: """Diff a branch against main.""" return await self._table.branches.diff(from_branch) - async def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]: - """Merge a branch into main, or dry-run. + async def cherry_pick( + self, from_branch: str, dry_run: bool = False + ) -> Dict[str, Any]: + """Cherry-pick a branch onto main, or dry-run. - A rejected merge returns ``status="rejected"`` instead of raising. + A failed cherry-pick returns ``status="failed"`` instead of raising. """ - return await self._table.branches.merge(from_branch, dry_run) + return await self._table.branches.cherry_pick(from_branch, dry_run) diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 13ffc4415..08f5550eb 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -242,8 +242,8 @@ def test_remote_table_branches_sync(): table.branches.delete("exp") -def test_remote_table_branch_merge_defaults_to_execute(): - merge_bodies = [] +def test_remote_table_cherry_pick_defaults_to_execute(): + cherry_pick_bodies = [] diff = { "fromBranch": "exp", "parentVersion": 1, @@ -265,8 +265,7 @@ def test_remote_table_branch_merge_defaults_to_execute(): "changedColumns": [], "addedIndexes": [], "removedIndexes": [], - "mergeable": True, - "mergeBlockers": [], + "errors": [], } def handler(request): @@ -276,11 +275,11 @@ def test_remote_table_branch_merge_defaults_to_execute(): else: content_len = int(request.headers.get("Content-Length")) request_body = json.loads(request.rfile.read(content_len)) - merge_bodies.append(request_body) + cherry_pick_bodies.append(request_body) dry_run = request_body["dry_run"] status = 200 if dry_run else 409 body = { - "status": "ready" if dry_run else "rejected", + "status": "ready" if dry_run else "failed", "diff": diff, "preview": {"promotedColumns": []}, } @@ -292,10 +291,10 @@ def test_remote_table_branch_merge_defaults_to_execute(): with mock_lancedb_connection(handler) as db: branches = db.open_table("test").branches - assert branches.merge("exp")["status"] == "rejected" - assert branches.merge("exp", dry_run=True)["status"] == "ready" + assert branches.cherry_pick("exp")["status"] == "failed" + assert branches.cherry_pick("exp", dry_run=True)["status"] == "ready" - assert merge_bodies == [ + assert cherry_pick_bodies == [ {"from_branch": "exp", "dry_run": False}, {"from_branch": "exp", "dry_run": True}, ] diff --git a/python/src/table.rs b/python/src/table.rs index a3a7c9d68..5cdcc3653 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -1940,7 +1940,7 @@ impl Branches { } #[pyo3(signature = (from_branch, dry_run=false))] - pub fn merge( + pub fn cherry_pick( self_: PyRef<'_, Self>, from_branch: String, dry_run: bool, @@ -1948,7 +1948,7 @@ impl Branches { let inner = self_.inner.clone(); future_into_py(self_.py(), async move { let result = inner - .merge_branch(&from_branch, dry_run) + .cherry_pick(&from_branch, dry_run) .await .infer_error()?; Python::attach(|py| struct_to_wire_py(py, &result)) diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 6c446907d..fb7278db9 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -21,11 +21,11 @@ use crate::remote::job::RemoteJob; use crate::table::AddColumnsResult; use crate::table::AddResult; use crate::table::BranchDiff; +use crate::table::CherryPickResult; use crate::table::DeleteResult; use crate::table::DropColumnsResult; use crate::table::LsmStats; use crate::table::LsmWriteSpec; -use crate::table::MergeBranchResult; use crate::table::MergeResult; use crate::table::Tags; use crate::table::UpdateResult; @@ -2031,7 +2031,7 @@ impl BaseTable for RemoteTable { async fn diff_branch(&self, from_branch: &str) -> Result { if from_branch.trim().is_empty() { return Err(Error::InvalidInput { - message: "from_branch must be a non-empty string".into(), + message: "Branch name cannot be empty.".into(), }); } let request = self @@ -2058,20 +2058,23 @@ impl BaseTable for RemoteTable { }) } - async fn merge_branch(&self, from_branch: &str, dry_run: bool) -> Result { + async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result { if from_branch.trim().is_empty() { return Err(Error::InvalidInput { - message: "from_branch must be a non-empty string".into(), + message: "Branch name cannot be empty.".into(), }); } let request = self .client - .post(&format!("/v1/table/{}/branches/merge/", self.identifier)) + .post(&format!( + "/v1/table/{}/branches/cherry_pick/", + self.identifier + )) .json(&serde_json::json!({ "from_branch": from_branch, "dry_run": dry_run, })); - // No retry. 409 rejected merge is final and carries a body. + // No retry. HTTP 409 is CherryPickStatus::Failed with a body, not a transport error. let (request_id, response) = self.send(request, false).await?; let status = response.status(); if status == StatusCode::NOT_FOUND { @@ -2080,11 +2083,11 @@ impl BaseTable for RemoteTable { source: format!("branch '{}' does not exist", from_branch).into(), }); } - // 200 and 409 both carry MergeBranchResult. + // 200 and 409 both carry CherryPickResult. if status != StatusCode::OK && status != StatusCode::CONFLICT { let body = response.text().await.unwrap_or_default(); return Err(Error::Http { - source: format!("unexpected status {status} from merge_branch: {body}").into(), + source: format!("unexpected status {status} from cherry_pick: {body}").into(), request_id, status_code: Some(status), }); @@ -2092,7 +2095,7 @@ impl BaseTable for RemoteTable { let body = response.text().await.err_to_http(request_id.clone())?; serde_json::from_str(&body).map_err(|err| Error::Http { source: format!( - "Failed to parse merge_branch response: {}, body: {}", + "Failed to parse cherry_pick response: {}, body: {}", err, body ) .into(), @@ -10513,8 +10516,7 @@ mod tests { "changedColumns":[], "addedIndexes":[], "removedIndexes":[], - "mergeable":true, - "mergeBlockers":[] + "errors":[] }"# } @@ -10532,15 +10534,18 @@ mod tests { }); let diff = table.diff_branch("exp").await.unwrap(); assert_eq!(diff.from_branch, "exp"); - assert!(diff.mergeable); + assert!(diff.errors.is_empty()); assert_eq!(diff.added_columns.len(), 1); assert_eq!(diff.added_columns[0].name, "tag"); } #[tokio::test] - async fn test_merge_branch_dry_run() { + async fn test_cherry_pick_dry_run() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/"); + assert_eq!( + request.url().path(), + "/v1/table/my_table/branches/cherry_pick/" + ); let body = request_body_json(&request); assert_eq!(body["from_branch"], "exp"); assert_eq!(body["dry_run"], true); @@ -10550,27 +10555,29 @@ mod tests { ); http::Response::builder().status(200).body(resp).unwrap() }); - let result = table.merge_branch("exp", true).await.unwrap(); - assert_eq!(result.status, crate::table::MergeBranchStatus::Ready); + let result = table.cherry_pick("exp", true).await.unwrap(); + assert_eq!(result.status, crate::table::CherryPickStatus::Ready); assert_eq!(result.preview.promoted_columns, vec!["tag".to_string()]); assert!(result.main_version_after.is_none()); } #[tokio::test] - async fn test_merge_branch_rejected_returns_ok_with_body() { + async fn test_cherry_pick_failed_returns_ok_with_body() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/"); + assert_eq!( + request.url().path(), + "/v1/table/my_table/branches/cherry_pick/" + ); let body = request_body_json(&request); assert_eq!(body["dry_run"], false); let mut diff: serde_json::Value = serde_json::from_str(sample_branch_diff_json()).unwrap(); - diff["mergeable"] = serde_json::json!(false); - diff["mergeBlockers"] = serde_json::json!([{ + diff["errors"] = serde_json::json!([{ "code": "baseMoved", "message": "main has advanced" }]); let resp = serde_json::json!({ - "status": "rejected", + "status": "failed", "diff": diff, "preview": { "promotedColumns": [] } }); @@ -10579,24 +10586,23 @@ mod tests { .body(resp.to_string()) .unwrap() }); - let result = table.merge_branch("exp", false).await.unwrap(); - assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected); - assert!(!result.diff.mergeable); - assert_eq!(result.diff.merge_blockers.len(), 1); + let result = table.cherry_pick("exp", false).await.unwrap(); + assert_eq!(result.status, crate::table::CherryPickStatus::Failed); + assert!(!result.diff.errors.is_empty()); + assert_eq!(result.diff.errors.len(), 1); } #[tokio::test] - async fn test_merge_branch_unknown_blocker_code_parses() { + async fn test_cherry_pick_unknown_error_code_parses() { let table = Table::new_with_handler("my_table", |_| { let mut diff: serde_json::Value = serde_json::from_str(sample_branch_diff_json()).unwrap(); - diff["mergeable"] = serde_json::json!(false); - diff["mergeBlockers"] = serde_json::json!([{ + diff["errors"] = serde_json::json!([{ "code": "multipleCommits", "message": "branch has more than one data commit" }]); let resp = serde_json::json!({ - "status": "rejected", + "status": "failed", "diff": diff, "preview": { "operation": "append", "rowsAdded": 2 } }); @@ -10605,24 +10611,24 @@ mod tests { .body(resp.to_string()) .unwrap() }); - let result = table.merge_branch("exp", false).await.unwrap(); - assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected); + let result = table.cherry_pick("exp", false).await.unwrap(); + assert_eq!(result.status, crate::table::CherryPickStatus::Failed); assert_eq!( - result.diff.merge_blockers[0].code, - crate::table::MergeBlockerCode::Unknown + result.diff.errors[0].code, + crate::table::CherryPickErrorCode::Unknown ); assert!(result.preview.promoted_columns.is_empty()); } #[tokio::test] - async fn test_merge_branch_unexpected_2xx_is_error() { + async fn test_cherry_pick_unexpected_2xx_is_error() { let table = Table::new_with_handler("my_table", |_| { http::Response::builder() .status(204) .body(String::new()) .unwrap() }); - let err = table.merge_branch("exp", false).await.unwrap_err(); + let err = table.cherry_pick("exp", false).await.unwrap_err(); match err { Error::Http { status_code: Some(code), diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index ca9ed8f26..c2c12fff5 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -66,8 +66,8 @@ use self::merge::MergeInsertBuilder; pub mod add_columns; mod add_data; -pub mod branch_merge; pub mod checkpoint; +pub mod cherry_pick; pub mod computed_columns; mod create_index; pub mod datafusion; @@ -87,9 +87,9 @@ pub use add_columns::AddColumnsBuilder; #[cfg(feature = "remote")] pub(crate) use add_data::PreprocessingOutput; pub use add_data::{AddDataBuilder, AddDataMode, AddResult, NaNVectorBehavior}; -pub use branch_merge::{ - BranchDiff, ColumnChange, ColumnSummary, IndexSummary, MergeBlocker, MergeBlockerCode, - MergeBranchResult, MergeBranchStatus, MergePreview, RowCountSummary, +pub use cherry_pick::{ + BranchDiff, CherryPickError, CherryPickErrorCode, CherryPickPreview, CherryPickResult, + CherryPickStatus, ColumnChange, ColumnSummary, IndexSummary, RowCountSummary, }; pub use chrono::Duration; pub use computed_columns::{ @@ -832,14 +832,14 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { /// Diff a branch against main. Remote only. async fn diff_branch(&self, _from_branch: &str) -> Result { Err(Error::NotSupported { - message: "diff_branch is only supported on remote tables".into(), + message: "Branch diffs are only supported on Enterprise tables.".into(), }) } - /// Merge a branch into main, or dry-run. Remote only. - /// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`]. - async fn merge_branch(&self, _from_branch: &str, _dry_run: bool) -> Result { + /// Cherry-pick a branch onto main, or dry-run. Remote only. + /// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`]. + async fn cherry_pick(&self, _from_branch: &str, _dry_run: bool) -> Result { Err(Error::NotSupported { - message: "merge_branch is only supported on remote tables".into(), + message: "Cherry-picking branches is only supported on Enterprise tables.".into(), }) } /// The branch this handle is scoped to, or `None` for `main`. @@ -2263,14 +2263,10 @@ impl Table { self.inner.diff_branch(from_branch).await } - /// Merge a branch into main, or dry-run. Remote only. - /// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`]. - pub async fn merge_branch( - &self, - from_branch: &str, - dry_run: bool, - ) -> Result { - self.inner.merge_branch(from_branch, dry_run).await + /// Cherry-pick a branch onto main, or dry-run. Remote only. + /// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`]. + pub async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result { + self.inner.cherry_pick(from_branch, dry_run).await } /// The branch this handle is scoped to, or `None` for `main`. diff --git a/rust/lancedb/src/table/branch_merge.rs b/rust/lancedb/src/table/cherry_pick.rs similarity index 86% rename from rust/lancedb/src/table/branch_merge.rs rename to rust/lancedb/src/table/cherry_pick.rs index ad81ab64c..93bc76637 100644 --- a/rust/lancedb/src/table/branch_merge.rs +++ b/rust/lancedb/src/table/cherry_pick.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Types for remote branch diff / merge against main. +//! Types for remote branch diff / cherry-pick onto main. use serde::{Deserialize, Serialize}; @@ -44,13 +44,13 @@ pub struct RowCountSummary { #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub enum MergeBlockerCode { +pub enum CherryPickErrorCode { BaseMoved, RowCountMismatch, RowsChanged, ColumnRemoved, ColumnChanged, - NoMergeableChanges, + NothingToApply, NoColumnChanges, InputColumnDependency, ParentNotMain, @@ -60,8 +60,8 @@ pub enum MergeBlockerCode { #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct MergeBlocker { - pub code: MergeBlockerCode, +pub struct CherryPickError { + pub code: CherryPickErrorCode, pub message: String, } @@ -81,34 +81,33 @@ pub struct BranchDiff { pub changed_columns: Vec, pub added_indexes: Vec, pub removed_indexes: Vec, - pub mergeable: bool, - pub merge_blockers: Vec, + pub errors: Vec, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct MergePreview { +pub struct CherryPickPreview { #[serde(default)] pub promoted_columns: Vec, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub enum MergeBranchStatus { +pub enum CherryPickStatus { Ready, - Rejected, + Failed, NotImplemented, - Merged, + CherryPicked, #[serde(other)] Unknown, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct MergeBranchResult { - pub status: MergeBranchStatus, +pub struct CherryPickResult { + pub status: CherryPickStatus, pub diff: BranchDiff, - pub preview: MergePreview, + pub preview: CherryPickPreview, #[serde(default, skip_serializing_if = "Option::is_none")] pub main_version_after: Option, } From 68749ecfa38f966bec6e650bc41c3f6b8e498818 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 21 Aug 2026 23:48:43 -0700 Subject: [PATCH 12/31] feat(nodejs): materialized view bindings (#3935) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposes materialized views to TypeScript: createMaterializedView, openMaterializedView and listMaterializedViews on Connection, and a MaterializedView handle carrying the parsed definition and refresh({full, sourceVersion}), which returns the typed refresh result. select accepts column names, [alias, expression] pairs, or a record of the same; the definition reads back off the stored schema, so a reopened handle needs no side channel. Remote connections surface the core's not-supported error up front. The napi crate needed the same recursion-limit raise as the core crate: the refresh future's type graph overflows the default trait-recursion depth. Stack created with GitHub Stacks CLI • Give Feedback 💬 --- docs/src/js/classes/Connection.md | 80 ++++++++- docs/src/js/classes/MaterializedView.md | 101 +++++++++++ docs/src/js/globals.md | 4 + .../interfaces/MaterializedViewDefinition.md | 59 +++++++ .../RefreshMaterializedViewResult.md | 41 +++++ .../js/type-aliases/MaterializedViewSelect.md | 14 ++ nodejs/__test__/embedding.test.ts | 48 ++++++ nodejs/__test__/materialized_view.test.ts | 147 ++++++++++++++++ nodejs/__test__/remote.test.ts | 19 +++ nodejs/__test__/table.test.ts | 2 +- nodejs/lancedb/connection.ts | 70 ++++++++ nodejs/lancedb/index.ts | 6 + nodejs/lancedb/materialized_view.ts | 161 ++++++++++++++++++ nodejs/lancedb/table.ts | 20 +++ nodejs/src/connection.rs | 52 ++++++ nodejs/src/lib.rs | 4 + nodejs/src/table.rs | 45 +++++ 17 files changed, 867 insertions(+), 6 deletions(-) create mode 100644 docs/src/js/classes/MaterializedView.md create mode 100644 docs/src/js/interfaces/MaterializedViewDefinition.md create mode 100644 docs/src/js/interfaces/RefreshMaterializedViewResult.md create mode 100644 docs/src/js/type-aliases/MaterializedViewSelect.md create mode 100644 nodejs/__test__/materialized_view.test.ts create mode 100644 nodejs/lancedb/materialized_view.ts diff --git a/docs/src/js/classes/Connection.md b/docs/src/js/classes/Connection.md index e4cbc1e96..92cfd2568 100644 --- a/docs/src/js/classes/Connection.md +++ b/docs/src/js/classes/Connection.md @@ -169,6 +169,45 @@ Creates a new empty Table *** +### createMaterializedView() + +```ts +abstract createMaterializedView( + name, + source, + options?): Promise +``` + +Define a materialized view named `name` over the table `source`. + +The view is created empty, with the query recorded in its schema +metadata; `view.refresh()` computes the rows. The view is a normal +table: it can be queried, indexed and searched, and it appears in +`tableNames`. The source table must have stable row ids (create it with +the `newTableEnableStableRowIds` storage option); they keep the view's +provenance valid across source compactions and cannot be enabled after +a table exists. Local databases only. + +#### Parameters + +* **name**: `string` + +* **source**: `string` + +* **options?** + +* **options.limit?**: `number` + +* **options.select?**: [`MaterializedViewSelect`](../type-aliases/MaterializedViewSelect.md) + +* **options.where?**: `string` + +#### Returns + +`Promise`<[`MaterializedView`](MaterializedView.md)> + +*** + ### createNamespace() ```ts @@ -499,6 +538,22 @@ List server-side jobs across the database's tables. *** +### listMaterializedViews() + +```ts +abstract listMaterializedViews(): Promise +``` + +The names of the materialized views in this database. + +Found by reading every table's schema, so this costs an open per table. + +#### Returns + +`Promise`<`string`[]> + +*** + ### listNamespaces() ```ts @@ -529,6 +584,26 @@ Child namespace names and *** +### openMaterializedView() + +```ts +abstract openMaterializedView(name): Promise +``` + +Open the materialized view named `name`. + +Rejects a table that exists but is not a materialized view. + +#### Parameters + +* **name**: `string` + +#### Returns + +`Promise`<[`MaterializedView`](MaterializedView.md)> + +*** + ### openTable() ```ts @@ -538,18 +613,13 @@ abstract openTable( options?): Promise ``` -Open a table in the database. - #### Parameters * **name**: `string` - The name of the table * **namespacePath?**: `string`[] - The namespace path of the table (defaults to root namespace) * **options?**: `Partial`<[`OpenTableOptions`](../interfaces/OpenTableOptions.md)> - Additional options #### Returns diff --git a/docs/src/js/classes/MaterializedView.md b/docs/src/js/classes/MaterializedView.md new file mode 100644 index 000000000..e6ff66142 --- /dev/null +++ b/docs/src/js/classes/MaterializedView.md @@ -0,0 +1,101 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MaterializedView + +# Class: MaterializedView + +A handle on a materialized view: its table plus its definition. + +Obtained from [Connection#createMaterializedView](Connection.md#creatematerializedview) or +[Connection#openMaterializedView](Connection.md#openmaterializedview). The view is a normal table -- +queries, indexes and search all apply through [MaterializedView#table](MaterializedView.md#table) +-- whose contents are maintained by [MaterializedView#refresh](MaterializedView.md#refresh). + +## Constructors + +### new MaterializedView() + +```ts +new MaterializedView(table): MaterializedView +``` + +#### Parameters + +* **table**: [`Table`](Table.md) + +#### Returns + +[`MaterializedView`](MaterializedView.md) + +## Accessors + +### name + +```ts +get name(): string +``` + +#### Returns + +`string` + +## Methods + +### definition() + +```ts +definition(): Promise +``` + +The query that defines the view, read from its stored schema. + +#### Returns + +`Promise`<[`MaterializedViewDefinition`](../interfaces/MaterializedViewDefinition.md)> + +*** + +### refresh() + +```ts +refresh(options?): Promise +``` + +Recompute the view from its source. + +The refresh is incremental when the source's changes can be reconciled +into the view -- rows added, changed or removed since the last one -- +and otherwise rebuilds. `full` forces a rebuild; `sourceVersion` +refreshes to that source version instead of the latest. + +Concurrent refreshes of one view do not duplicate its rows. Two that +plan the same source rows conflict on commit, and the loser throws +rather than writing them a second time. + +#### Parameters + +* **options?** + +* **options.full?**: `boolean` + +* **options.sourceVersion?**: `number` + +#### Returns + +`Promise`<[`RefreshMaterializedViewResult`](../interfaces/RefreshMaterializedViewResult.md)> + +*** + +### table() + +```ts +table(): Table +``` + +The view, as the table it is. + +#### Returns + +[`Table`](Table.md) diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 4deeed1bc..6a8f644eb 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -28,6 +28,7 @@ - [Job](classes/Job.md) - [MakeArrowTableOptions](classes/MakeArrowTableOptions.md) - [MatchQuery](classes/MatchQuery.md) +- [MaterializedView](classes/MaterializedView.md) - [MergeInsertBuilder](classes/MergeInsertBuilder.md) - [MultiMatchQuery](classes/MultiMatchQuery.md) - [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md) @@ -101,6 +102,7 @@ - [ListNamespacesResponse](interfaces/ListNamespacesResponse.md) - [LsmStats](interfaces/LsmStats.md) - [LsmWriteSpec](interfaces/LsmWriteSpec.md) +- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md) - [MemtableStats](interfaces/MemtableStats.md) - [MergeResult](interfaces/MergeResult.md) - [NativeOAuthConfig](interfaces/NativeOAuthConfig.md) @@ -110,6 +112,7 @@ - [OptimizeStats](interfaces/OptimizeStats.md) - [QueryExecutionOptions](interfaces/QueryExecutionOptions.md) - [RefreshColumnResult](interfaces/RefreshColumnResult.md) +- [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md) - [RemovalStats](interfaces/RemovalStats.md) - [RenameTableOptions](interfaces/RenameTableOptions.md) - [RestNamespaceConfig](interfaces/RestNamespaceConfig.md) @@ -142,6 +145,7 @@ - [FieldLike](type-aliases/FieldLike.md) - [IntoSql](type-aliases/IntoSql.md) - [IntoVector](type-aliases/IntoVector.md) +- [MaterializedViewSelect](type-aliases/MaterializedViewSelect.md) - [MultiVector](type-aliases/MultiVector.md) - [RecordBatchLike](type-aliases/RecordBatchLike.md) - [SchemaLike](type-aliases/SchemaLike.md) diff --git a/docs/src/js/interfaces/MaterializedViewDefinition.md b/docs/src/js/interfaces/MaterializedViewDefinition.md new file mode 100644 index 000000000..741bbba31 --- /dev/null +++ b/docs/src/js/interfaces/MaterializedViewDefinition.md @@ -0,0 +1,59 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MaterializedViewDefinition + +# Interface: MaterializedViewDefinition + +The query that defines a materialized view. + +## Properties + +### filter? + +```ts +optional filter: string; +``` + +SQL predicate selecting the source rows the view holds. + +*** + +### inputs + +```ts +inputs: string[]; +``` + +Source columns the projections and filter read. + +*** + +### limit? + +```ts +optional limit: number; +``` + +Cap on the number of rows the view holds. + +*** + +### projections + +```ts +projections: [string, string][]; +``` + +`[output column, SQL expression]` pairs, in view schema order. + +*** + +### sourceTable + +```ts +sourceTable: string; +``` + +Name of the source table, in the same database as the view. diff --git a/docs/src/js/interfaces/RefreshMaterializedViewResult.md b/docs/src/js/interfaces/RefreshMaterializedViewResult.md new file mode 100644 index 000000000..cb7100cd8 --- /dev/null +++ b/docs/src/js/interfaces/RefreshMaterializedViewResult.md @@ -0,0 +1,41 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / RefreshMaterializedViewResult + +# Interface: RefreshMaterializedViewResult + +## Properties + +### mode + +```ts +mode: string; +``` + +How the view was brought up to date: "rebuild", "incremental" or "no_op". + +*** + +### rowsWritten + +```ts +rowsWritten: number; +``` + +*** + +### sourceVersion + +```ts +sourceVersion: number; +``` + +*** + +### version + +```ts +version: number; +``` diff --git a/docs/src/js/type-aliases/MaterializedViewSelect.md b/docs/src/js/type-aliases/MaterializedViewSelect.md new file mode 100644 index 000000000..7b246e945 --- /dev/null +++ b/docs/src/js/type-aliases/MaterializedViewSelect.md @@ -0,0 +1,14 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MaterializedViewSelect + +# Type Alias: MaterializedViewSelect + +```ts +type MaterializedViewSelect: (string | [string, string])[] | Record; +``` + +The view's columns: column names, `[alias, SQL expression]` pairs, or a +record of the same. A bare name projects itself. diff --git a/nodejs/__test__/embedding.test.ts b/nodejs/__test__/embedding.test.ts index 06184751e..2a8494e0f 100644 --- a/nodejs/__test__/embedding.test.ts +++ b/nodejs/__test__/embedding.test.ts @@ -487,4 +487,52 @@ describe("embedding functions", () => { expect(stringSchema3).toEqual(stringExpectedSchema); }, ); + test("parses one function writing several vector columns", async () => { + class MockEmbeddingFunction extends EmbeddingFunction { + ndims() { + return 3; + } + embeddingDataType(): Float { + return new Float32(); + } + async computeQueryEmbeddings(_data: string) { + return [1, 2, 3]; + } + async computeSourceEmbeddings(data: string[]) { + return Array.from({ length: data.length }).fill([ + 1, 2, 3, + ]) as number[][]; + } + } + const registry = getRegistry(); + registry.register("multi_output_mock")(MockEmbeddingFunction); + + // A materialized view can project one source vector column under two + // names, so a table's configuration names the same function twice. + const parsed = await registry.parseFunctions( + new Map([ + [ + "embedding_functions", + JSON.stringify([ + { + name: "multi_output_mock", + sourceColumn: "text", + vectorColumn: "vector_a", + model: {}, + }, + { + name: "multi_output_mock", + sourceColumn: "text", + vectorColumn: "vector_b", + model: {}, + }, + ]), + ], + ]), + ); + + expect( + [...parsed.values()].map(({ vectorColumn }) => vectorColumn).sort(), + ).toEqual(["vector_a", "vector_b"]); + }); }); diff --git a/nodejs/__test__/materialized_view.test.ts b/nodejs/__test__/materialized_view.test.ts new file mode 100644 index 000000000..2e7b2ec4d --- /dev/null +++ b/nodejs/__test__/materialized_view.test.ts @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import * as tmp from "tmp"; + +import { Connection, connect } from "../lancedb"; +import { + DEFINITION_META_KEY, + definitionFromMetadata, +} from "../lancedb/materialized_view"; + +describe("materialized views", () => { + let tmpDir: tmp.DirResult; + let db: Connection; + + beforeEach(async () => { + tmpDir = tmp.dirSync({ unsafeCleanup: true }); + db = await connect(tmpDir.name); + await db.createTable( + "people", + [ + { name: "ada", age: 36 }, + { name: "kid", age: 7 }, + { name: "grace", age: 85 }, + ], + { storageOptions: { newTableEnableStableRowIds: "true" } }, + ); + }); + afterEach(() => tmpDir.removeCallback()); + + it("rejects a stored limit a number cannot carry", () => { + const big = new Map([ + [ + DEFINITION_META_KEY, + '{"kind":"select","source_table":"people","limit":9007199254740993}', + ], + ]); + expect(() => definitionFromMetadata(big, "v")).toThrow( + /too large to represent exactly/, + ); + + const safe = new Map([ + [ + DEFINITION_META_KEY, + '{"kind":"select","source_table":"people","limit":42}', + ], + ]); + expect(definitionFromMetadata(safe, "v").limit).toBe(42); + }); + + it("creates, refreshes and queries a view", async () => { + const view = await db.createMaterializedView("adults", "people", { + select: ["name", ["shout", "upper(name)"]], + where: "age >= 18", + }); + expect(view.name).toBe("adults"); + expect(await view.table().countRows()).toBe(0); + + const result = await view.refresh(); + expect(result.mode).toBe("rebuild"); + expect(Number(result.rowsWritten)).toBe(2); + + const rows = await view.table().query().toArray(); + expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]); + }); + + it("round-trips the definition", async () => { + await db.createMaterializedView("adults", "people", { + where: "age >= 18", + }); + const view = await db.openMaterializedView("adults"); + const definition = await view.definition(); + expect(definition.sourceTable).toBe("people"); + expect(definition.filter).toBe("age >= 18"); + expect(definition.projections).toEqual([ + ["name", "`name`"], + ["age", "`age`"], + ]); + expect(definition.inputs).toEqual(["age", "name"]); + }); + + it("refreshes incrementally after an append", async () => { + const view = await db.createMaterializedView("copy", "people"); + await view.refresh(); + + const people = await db.openTable("people"); + await people.add([{ name: "alan", age: 41 }]); + const result = await view.refresh(); + expect(result.mode).toBe("incremental"); + expect(Number(result.rowsWritten)).toBe(1); + expect(await view.table().countRows()).toBe(4); + + expect((await view.refresh()).mode).toBe("no_op"); + }); + + it("lists views and rejects non-views", async () => { + await db.createMaterializedView("adults", "people", { + where: "age >= 18", + }); + expect(await db.listMaterializedViews()).toEqual(["adults"]); + await expect(db.openMaterializedView("people")).rejects.toThrow( + "not a materialized view", + ); + }); + + it("rejects an invalid expression at create time", async () => { + await expect( + db.createMaterializedView("bad", "people", { + select: [["x", "missing + 1"]], + }), + ).rejects.toThrow("missing"); + }); + + it("rejects invalid numeric options before creating anything", async () => { + for (const limit of [-5, 1.5, Infinity, NaN]) { + await expect( + db.createMaterializedView("bad", "people", { limit }), + ).rejects.toThrow("non-negative integer"); + } + expect(await db.listMaterializedViews()).toEqual([]); + + const view = await db.createMaterializedView("copy", "people"); + for (const sourceVersion of [-1, 1.5, Infinity, NaN]) { + await expect(view.refresh({ sourceVersion })).rejects.toThrow( + "non-negative integer", + ); + } + }); + + it("quotes bare select names", async () => { + await db.createTable("odd_names", [{ "order item": "widget" }], { + storageOptions: { newTableEnableStableRowIds: "true" }, + }); + const view = await db.createMaterializedView("quoted", "odd_names", { + select: ["order item"], + }); + const result = await view.refresh(); + expect(Number(result.rowsWritten)).toBe(1); + }); + + it("requires stable row ids on the source", async () => { + await db.createTable("plain", [{ x: 1 }]); + await expect(db.createMaterializedView("v", "plain")).rejects.toThrow( + "stable row ids", + ); + }); +}); diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 01320d2b0..c51cbbbb7 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -75,6 +75,25 @@ async function withMockDatabase( } describe("remote connection", () => { + it("refuses materialized views before issuing any request", async () => { + const paths: string[] = []; + await withMockDatabase( + (req, res) => { + paths.push(req.url ?? ""); + res.writeHead(404).end(); + }, + async (db) => { + await expect(db.openMaterializedView("secret_table")).rejects.toThrow( + /only on local databases/, + ); + await expect(db.listMaterializedViews()).rejects.toThrow( + /only on local databases/, + ); + expect(paths).toEqual([]); + }, + ); + }); + it("should accept partial connection options", async () => { await connect("db://test", { apiKey: "fake", diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 80c50f1ac..0f6ed3615 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2953,7 +2953,7 @@ describe("column name options", () => { .limit(10) .toArray(); expect(results2.length).toBe(10); - }); + }, 30_000); }); describe("when creating an empty table", () => { diff --git a/nodejs/lancedb/connection.ts b/nodejs/lancedb/connection.ts index a81dc0442..e5528f8c6 100644 --- a/nodejs/lancedb/connection.ts +++ b/nodejs/lancedb/connection.ts @@ -16,6 +16,12 @@ import { makeEmptyTable, } from "./arrow"; import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry"; +import { + MaterializedView, + MaterializedViewSelect, + normalizeSelect, + validateNonNegativeInteger, +} from "./materialized_view"; import { Connection as LanceDbConnection } from "./native"; import type { CreateNamespaceResponse, @@ -247,6 +253,41 @@ export abstract class Connection { * @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace) * @param {Partial} options - Additional options */ + /** + * Define a materialized view named `name` over the table `source`. + * + * The view is created empty, with the query recorded in its schema + * metadata; `view.refresh()` computes the rows. The view is a normal + * table: it can be queried, indexed and searched, and it appears in + * `tableNames`. The source table must have stable row ids (create it with + * the `newTableEnableStableRowIds` storage option); they keep the view's + * provenance valid across source compactions and cannot be enabled after + * a table exists. Local databases only. + */ + abstract createMaterializedView( + name: string, + source: string, + options?: { + select?: MaterializedViewSelect; + where?: string; + limit?: number; + }, + ): Promise; + + /** + * Open the materialized view named `name`. + * + * Rejects a table that exists but is not a materialized view. + */ + abstract openMaterializedView(name: string): Promise; + + /** + * The names of the materialized views in this database. + * + * Found by reading every table's schema, so this costs an open per table. + */ + abstract listMaterializedViews(): Promise; + abstract openTable( name: string, namespacePath?: string[], @@ -531,6 +572,35 @@ export class LocalConnection extends Connection { ); } + async createMaterializedView( + name: string, + source: string, + options?: { + select?: MaterializedViewSelect; + where?: string; + limit?: number; + }, + ): Promise { + validateNonNegativeInteger(options?.limit, "limit"); + const innerTable = await this.inner.createMaterializedView( + name, + source, + normalizeSelect(options?.select), + options?.where, + options?.limit, + ); + return new MaterializedView(new LocalTable(innerTable)); + } + + async openMaterializedView(name: string): Promise { + const innerTable = await this.inner.openMaterializedView(name); + return new MaterializedView(new LocalTable(innerTable)); + } + + async listMaterializedViews(): Promise { + return await this.inner.listMaterializedViews(); + } + async openTable( name: string, namespacePath?: string[], diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index aa9c7ef13..da13b434f 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -21,6 +21,11 @@ import type { BaseTokenizer } from "./indices"; import type { FtsToken } from "./table"; // Re-export native header provider for use with connectWithHeaderProvider +export { + MaterializedView, + MaterializedViewDefinition, + MaterializedViewSelect, +} from "./materialized_view"; export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js"; // OpenTelemetry metrics bridge. Only the high-level entry point is public; the @@ -51,6 +56,7 @@ export { AddResult, AddColumnsResult, RefreshColumnResult, + RefreshMaterializedViewResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, diff --git a/nodejs/lancedb/materialized_view.ts b/nodejs/lancedb/materialized_view.ts new file mode 100644 index 000000000..b26dee59e --- /dev/null +++ b/nodejs/lancedb/materialized_view.ts @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { RefreshMaterializedViewResult } from "./native"; +import { Table } from "./table"; + +/** Schema metadata key holding a materialized view's definition. */ +export const DEFINITION_META_KEY = "mv.definition"; + +/** The query that defines a materialized view. */ +export interface MaterializedViewDefinition { + /** Name of the source table, in the same database as the view. */ + sourceTable: string; + /** `[output column, SQL expression]` pairs, in view schema order. */ + projections: [string, string][]; + /** SQL predicate selecting the source rows the view holds. */ + filter?: string; + /** Cap on the number of rows the view holds. */ + limit?: number; + /** Source columns the projections and filter read. */ + inputs: string[]; +} + +/** + * The view's columns: column names, `[alias, SQL expression]` pairs, or a + * record of the same. A bare name projects itself. + */ +export type MaterializedViewSelect = + | (string | [string, string])[] + | Record; + +/** + * @internal Reject a numeric option N-API would otherwise silently coerce: + * `Infinity` reaches Rust as 0, `1.5` as 1. + */ +export function validateNonNegativeInteger( + value: number | undefined, + name: string, +): void { + if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) { + throw new Error(`${name} must be a non-negative integer`); + } +} + +/** @internal Quote a column name as a Lance SQL identifier (backticks). */ +function quoteIdentifier(name: string): string { + return "`" + name.replace(/`/g, "``") + "`"; +} + +/** + * @internal Normalize a select argument into `[alias, expression]` pairs. + * A bare name projects itself and is quoted, so any valid column name works; + * pair and record entries are kept verbatim because their right side is an + * expression. + */ +export function normalizeSelect( + select?: MaterializedViewSelect, +): [string, string][] | undefined { + if (select === undefined) { + return undefined; + } + if (Array.isArray(select)) { + return select.map((item) => + typeof item === "string" ? [item, quoteIdentifier(item)] : item, + ); + } + return Object.entries(select); +} + +/** @internal Parse a definition off a table's stored schema metadata. */ +export function definitionFromMetadata( + metadata: Map, + name: string, +): MaterializedViewDefinition { + const raw = metadata.get(DEFINITION_META_KEY); + if (raw === undefined) { + throw new Error(`Table '${name}' is not a materialized view`); + } + // biome-ignore lint/suspicious/noExplicitAny: raw JSON + const value: any = JSON.parse(raw); + if (value.kind !== "select") { + throw new Error( + `materialized view '${name}' is defined by '${value.kind}', which this ` + + "version of lancedb cannot refresh", + ); + } + const limit = value.limit ?? undefined; + // JSON.parse rounds integers past 2^53; every exact u64 parses to a safe + // integer and every rounded one does not, so this rejects precisely the + // values a number cannot carry. + if (limit !== undefined && !Number.isSafeInteger(limit)) { + throw new Error( + `materialized view '${name}' has a stored limit too large to represent exactly`, + ); + } + return { + sourceTable: value.source_table, + // biome-ignore lint/suspicious/noExplicitAny: raw JSON + projections: (value.projections ?? []).map((p: any) => [ + p.output, + p.expression, + ]), + filter: value.filter ?? undefined, + limit, + inputs: value.inputs ?? [], + }; +} + +/** + * A handle on a materialized view: its table plus its definition. + * + * Obtained from {@link Connection#createMaterializedView} or + * {@link Connection#openMaterializedView}. The view is a normal table -- + * queries, indexes and search all apply through {@link MaterializedView#table} + * -- whose contents are maintained by {@link MaterializedView#refresh}. + */ +export class MaterializedView { + private readonly inner: Table; + + constructor(table: Table) { + this.inner = table; + } + + get name(): string { + return this.inner.name; + } + + /** The view, as the table it is. */ + table(): Table { + return this.inner; + } + + /** The query that defines the view, read from its stored schema. */ + async definition(): Promise { + const schema = await this.inner.schema(); + return definitionFromMetadata(schema.metadata, this.name); + } + + /** + * Recompute the view from its source. + * + * The refresh is incremental when the source's changes can be reconciled + * into the view -- rows added, changed or removed since the last one -- + * and otherwise rebuilds. `full` forces a rebuild; `sourceVersion` + * refreshes to that source version instead of the latest. + * + * Concurrent refreshes of one view do not duplicate its rows. Two that + * plan the same source rows conflict on commit, and the loser throws + * rather than writing them a second time. + */ + async refresh(options?: { + full?: boolean; + sourceVersion?: number; + }): Promise { + validateNonNegativeInteger(options?.sourceVersion, "sourceVersion"); + return await this.inner.refreshMaterializedView( + options?.full, + options?.sourceVersion, + ); + } +} diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 4e323ec64..28603cca9 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -35,6 +35,7 @@ import { Branches as NativeBranches, OptimizeStats, RefreshColumnResult, + RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, @@ -602,6 +603,18 @@ export abstract class Table { */ abstract refreshColumnAsync(column: string): Promise; + /** + * Recompute this table's contents from its materialized-view definition. + * + * Plumbing for {@link MaterializedView.refresh}, which is the way to call + * it: rejects tables that carry no view definition. Local tables only. + * @ignore + */ + abstract refreshMaterializedView( + full?: boolean, + sourceVersion?: number, + ): Promise; + /** * Alter the name or nullability of columns. * @param {ColumnAlteration[]} columnAlterations One or more alterations to @@ -1264,6 +1277,13 @@ export class LocalTable extends Table { return await this.inner.refreshColumnAsync(column); } + async refreshMaterializedView( + full?: boolean, + sourceVersion?: number, + ): Promise { + return await this.inner.refreshMaterializedView(full, sourceVersion); + } + async alterColumns( columnAlterations: ColumnAlteration[], ): Promise { diff --git a/nodejs/src/connection.rs b/nodejs/src/connection.rs index c9f5e10ea..44eb68f32 100644 --- a/nodejs/src/connection.rs +++ b/nodejs/src/connection.rs @@ -266,6 +266,58 @@ impl Connection { Ok(Table::new(tbl)) } + #[napi(catch_unwind)] + pub async fn create_materialized_view( + &self, + name: String, + source: String, + projections: Option>>, + filter: Option, + limit: Option, + ) -> napi::Result
{ + let mut builder = self.get_inner()?.create_materialized_view(name, source); + if let Some(projections) = projections { + let mut pairs = Vec::with_capacity(projections.len()); + for pair in projections { + let [output, expression]: [String; 2] = pair.try_into().map_err(|_| { + napi::Error::from_reason("each projection must be an [output, expression] pair") + })?; + pairs.push((output, expression)); + } + builder = builder.select(pairs); + } + if let Some(filter) = filter { + builder = builder.only_if(filter); + } + if let Some(limit) = limit { + let limit = u64::try_from(limit) + .map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?; + builder = builder.limit(limit); + } + let view = builder.execute().await.default_error()?; + Ok(Table::new(view.table().clone())) + } + + #[napi(catch_unwind)] + pub async fn open_materialized_view(&self, name: String) -> napi::Result
{ + let view = self + .get_inner()? + .open_materialized_view(&name) + .await + .default_error()?; + Ok(Table::new(view.table().clone())) + } + + #[napi(catch_unwind)] + pub async fn list_materialized_views(&self) -> napi::Result> { + let views = self + .get_inner()? + .list_materialized_views() + .await + .default_error()?; + Ok(views.into_iter().map(|v| v.name).collect()) + } + #[napi(catch_unwind)] pub async fn open_table( &self, diff --git a/nodejs/src/lib.rs b/nodejs/src/lib.rs index 312b675bd..1110f6203 100644 --- a/nodejs/src/lib.rs +++ b/nodejs/src/lib.rs @@ -1,6 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors +// The materialized-view refresh future deepens the type graph past the +// default trait-recursion depth; same raise as the core crate applies. +#![recursion_limit = "256"] + use std::collections::HashMap; use env_logger::Env; diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 694b6a704..9d60b2056 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -381,6 +381,26 @@ impl Table { Ok(crate::job::Job::new(job)) } + #[napi(catch_unwind)] + pub async fn refresh_materialized_view( + &self, + full: Option, + source_version: Option, + ) -> napi::Result { + let view = lancedb::MaterializedView::from_table(self.inner_ref()?.clone()) + .await + .default_error()?; + let mut builder = view.refresh().full(full.unwrap_or(false)); + if let Some(version) = source_version { + let version = u64::try_from(version).map_err(|_| { + napi::Error::from_reason("sourceVersion must be a non-negative integer") + })?; + builder = builder.source_version(version); + } + let result = builder.execute().await.default_error()?; + Ok(result.into()) + } + #[napi(catch_unwind)] pub async fn add_columns_with_schema( &self, @@ -1387,6 +1407,31 @@ pub struct RefreshColumnResult { pub version: i64, } +#[napi(object)] +pub struct RefreshMaterializedViewResult { + /// How the view was brought up to date: "rebuild", "incremental" or "no_op". + pub mode: String, + pub rows_written: i64, + pub source_version: i64, + pub version: i64, +} + +impl From for RefreshMaterializedViewResult { + fn from(value: lancedb::RefreshMaterializedViewResult) -> Self { + let mode = match value.mode { + lancedb::RefreshMode::Rebuild => "rebuild", + lancedb::RefreshMode::Incremental => "incremental", + lancedb::RefreshMode::NoOp => "no_op", + }; + Self { + mode: mode.to_string(), + rows_written: value.rows_written as i64, + source_version: value.source_version as i64, + version: value.version as i64, + } + } +} + impl From for RefreshColumnResult { fn from(value: lancedb::table::RefreshColumnResult) -> Self { Self { From 45cd05347846bc2280103680e497e8fc0cdf480e Mon Sep 17 00:00:00 2001 From: Lance Release Date: Sat, 22 Aug 2026 16:38:08 +0000 Subject: [PATCH 13/31] =?UTF-8?q?Bump=20version:=200.38.0-beta.3=20?= =?UTF-8?q?=E2=86=92=200.38.0-beta.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index b1d2bf1dc..bba238f96 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.3" +current_version = "0.38.0-beta.4" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 5947187cc..87fec3378 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5398,7 +5398,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.3" +version = "0.38.0-beta.4" dependencies = [ "ahash", "anyhow", @@ -5486,7 +5486,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.3" +version = "0.38.0-beta.4" dependencies = [ "arrow-array", "arrow-buffer", @@ -5511,7 +5511,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.3" +version = "0.38.0-beta.4" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index a8869d319..cff4cf2f5 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.3 + 0.38.0-beta.4 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 9de14d1f9..98c10ca38 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.3 + 0.38.0-beta.4 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 36ec01d9e..56366bb15 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.3 + 0.38.0-beta.4 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index cba7012d1..99aed2e71 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.3" +version = "0.38.0-beta.4" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 3c76481fc..c84b83c45 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 7a1a3af03..52f2b7901 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 7175233af..afa7fac32 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index add5a1da2..36cfa0343 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 72a82bf6d..23ea473ef 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index c69beeb8a..880b65540 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index d21fc1eb3..8cc40de7b 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 1c368dab5..dba3ede7c 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index 7f72eb8b0..c2f5be2c8 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.4", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index ddc13b69f..dace5aa7c 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.3" +version = "0.38.0-beta.4" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index f785e6743..2920498b1 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.3" +version = "0.38.0-beta.4" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 1f1d03f306c322c42df93a42eb064225b0206f20 Mon Sep 17 00:00:00 2001 From: Weston Pace Date: Sun, 23 Aug 2026 00:28:07 -0700 Subject: [PATCH 14/31] feat(python): add backpressure to StreamingDataset post-transform queue (#3897) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename prefetch_batches → io_queue_depth and introduce transform_queue_depth as a symmetric pair: both express "number of batches to buffer per split at this pipeline stage." The old names are still accepted as keyword arguments but log a deprecation warning redirecting callers to the new names. transform_queue_depth caps how many transform-result batches can accumulate per split in the post-transform queue. Without this limit a slow consumer (e.g. a GPU training step) causes cooked rows to pile up unboundedly. The backpressure check in _try_submit_tx counts both already-cooked rows and rows expected from in-flight transforms; it skips proactive transform submission when the combined total reaches the limit. The reactive _ensure_cooked path bypasses the check so the consumer never stalls. --------- Co-authored-by: Claude Sonnet 4.6 --- python/python/lancedb/streaming.py | 65 +++++- .../python/tests/test_elastic_dataloader.py | 186 +++++++++++++++++- 2 files changed, 239 insertions(+), 12 deletions(-) diff --git a/python/python/lancedb/streaming.py b/python/python/lancedb/streaming.py index b27e606a4..6951fa0bc 100644 --- a/python/python/lancedb/streaming.py +++ b/python/python/lancedb/streaming.py @@ -61,7 +61,7 @@ class StreamingDataset(IterableDataset): Internally ``__iter__`` runs a two-stage pipeline: - - **Stage 1 (I/O)**: one thread pool with ``num_splits * prefetch_batches`` + - **Stage 1 (I/O)**: one thread pool with ``num_splits * io_queue_depth`` workers fetches raw ``RecordBatch`` objects from LanceDB in parallel across all splits and places them in a per-split raw-batch queue. - **Stage 2 (transform)**: a second thread pool with @@ -104,11 +104,11 @@ class StreamingDataset(IterableDataset): call. Larger values amortise per-request overhead (critical on object storage) at the cost of higher memory usage per split buffer. Defaults to ``DEFAULT_READ_BATCH_SIZE`` (64). - prefetch_batches: + io_queue_depth: Number of I/O batches to keep in flight per split. Higher values overlap storage latency with transform and training compute at the cost - of more memory and threads. Defaults to ``DEFAULT_PREFETCH_BATCHES`` - (4). + of more memory and threads. Must be greater than zero. Defaults to + ``DEFAULT_PREFETCH_BATCHES`` (4). columns: Optional list of column names to read. When set, only those columns are fetched from storage; all others are omitted. ``None`` (the @@ -175,6 +175,16 @@ class StreamingDataset(IterableDataset): Prefer the ``filter`` parameter when bad rows can be expressed as a SQL predicate (e.g. ``"col IS NOT NULL"``) — filtering happens before splits are built, so every guarantee is fully preserved. + transform_queue_depth: + Number of transform-result batches to buffer per split in the + post-transform queue before backpressure is applied to the transform + stage. When the combined count of in-flight transform futures and + already-buffered rows for a split reaches + ``transform_queue_depth * read_batch_size``, no new transforms are + submitted for that split until the consumer catches up. Useful for + capping peak memory when the consumer (e.g. a GPU training step) is + slower than the transform stage. Must be greater than zero. + ``None`` (the default) imposes no limit. worker_info_override: If set, used in place of ``torch.utils.data.get_worker_info()`` to determine the DataLoader worker assignment. Intended for unit tests @@ -194,17 +204,26 @@ class StreamingDataset(IterableDataset): rank: int = 0, world_size: int = 1, read_batch_size: int = DEFAULT_READ_BATCH_SIZE, - prefetch_batches: int = DEFAULT_PREFETCH_BATCHES, + io_queue_depth: int = DEFAULT_PREFETCH_BATCHES, columns: Optional[list[str]] = None, shuffle_clump_size: Optional[int] = None, filter: Optional[str] = None, transform: Optional[Callable] = None, transform_parallelism: Optional[int] = None, on_transform_error: Union[str, Callable[[Exception], bool]] = "raise", + transform_queue_depth: Optional[int] = None, connection_factory: Optional[Callable[[str], Any]] = None, worker_info_override=None, + # Deprecated; use io_queue_depth instead. + prefetch_batches: Optional[int] = None, ): super().__init__() + if prefetch_batches is not None: + logger.warning( + "prefetch_batches is deprecated and will be removed in a future " + "version; use io_queue_depth instead" + ) + io_queue_depth = prefetch_batches if num_splits is None: num_splits = world_size if shuffle_seed is None: @@ -214,6 +233,8 @@ class StreamingDataset(IterableDataset): f"num_splits ({num_splits}) must be divisible by " f"world_size ({world_size})" ) + if io_queue_depth <= 0: + raise ValueError("io_queue_depth must be greater than 0") if transform_parallelism is not None and transform_parallelism <= 0: raise ValueError("transform_parallelism must be greater than 0") if on_transform_error not in ("raise", "skip", "warn") and not callable( @@ -223,6 +244,8 @@ class StreamingDataset(IterableDataset): "on_transform_error must be 'raise', 'skip', 'warn', or a " f"callable, got {on_transform_error!r}" ) + if transform_queue_depth is not None and transform_queue_depth <= 0: + raise ValueError("transform_queue_depth must be greater than 0") self._table = table self._num_splits = num_splits @@ -232,13 +255,14 @@ class StreamingDataset(IterableDataset): self._rank = rank self._world_size = world_size self._read_batch_size = read_batch_size - self._prefetch_batches = prefetch_batches + self._io_queue_depth = io_queue_depth self._columns = columns self._shuffle_clump_size = shuffle_clump_size self._filter = filter self._transform = transform self._transform_parallelism = transform_parallelism self._on_transform_error = on_transform_error + self._transform_queue_depth = transform_queue_depth self._connection_factory = connection_factory self._worker_info_override = worker_info_override @@ -365,7 +389,7 @@ class StreamingDataset(IterableDataset): pos_consumed = list(initial_positions) batch_size = self._read_batch_size - max_prefetch = self._prefetch_batches + io_queue_depth = self._io_queue_depth transform_workers = ( self._transform_parallelism if self._transform_parallelism is not None @@ -374,6 +398,13 @@ class StreamingDataset(IterableDataset): final_transform = ( self._transform if self._transform is not None else Transforms.arrow2python ) + # None means no limit; otherwise cap rows per split to + # transform_queue_depth batches worth (including in-flight transforms). + max_cooked_rows = ( + self._transform_queue_depth * batch_size + if self._transform_queue_depth is not None + else None + ) # Per-split pipeline state. Batches are paired with the absolute # permutation position of their first row so that skipped rows can be @@ -409,7 +440,9 @@ class StreamingDataset(IterableDataset): io_pending[i].append((abs_start, io_pool.submit(_io_call, perm_i, indices))) def _fill_io(i: int) -> None: - while len(io_pending[i]) < max_prefetch and fetch_head[i] < split_sizes[i]: + while ( + len(io_pending[i]) < io_queue_depth and fetch_head[i] < split_sizes[i] + ): _submit_io(i) def _drain_io(i: int) -> None: @@ -487,7 +520,19 @@ class StreamingDataset(IterableDataset): def _try_submit_tx(i: int) -> None: """Submit transforms for raw_batches[i] up to available capacity.""" - while raw_batches[i] and tx_semaphore.acquire(blocking=False): + while raw_batches[i]: + # Backpressure: only submit a new transform when there is room + # for a full batch in the post-transform queue. Checking for + # a full batch prevents submitting a transform that would + # overflow the limit mid-batch (e.g. 990 rows queued with a + # capacity of 1000 and a batch_size of 128 must wait until + # 128 rows have been consumed, not just 1). + if max_cooked_rows is not None: + in_pipeline = len(cooked[i]) + len(tx_pending[i]) * batch_size + if in_pipeline + batch_size > max_cooked_rows: + break + if not tx_semaphore.acquire(blocking=False): + break abs_start, batch = raw_batches[i].popleft() tx_pending[i].append(tx_pool.submit(_tx_call_guarded, abs_start, batch)) @@ -531,7 +576,7 @@ class StreamingDataset(IterableDataset): # ── Main loop ───────────────────────────────────────────────────────── - with ThreadPoolExecutor(max_workers=n * max_prefetch) as io_pool: + with ThreadPoolExecutor(max_workers=n * io_queue_depth) as io_pool: with ThreadPoolExecutor(max_workers=transform_workers) as tx_pool: self._raw_batches_ref = raw_batches self._cooked_ref = cooked diff --git a/python/python/tests/test_elastic_dataloader.py b/python/python/tests/test_elastic_dataloader.py index 0c1a70765..4d860a5de 100644 --- a/python/python/tests/test_elastic_dataloader.py +++ b/python/python/tests/test_elastic_dataloader.py @@ -1374,6 +1374,188 @@ def test_transform_parallelism_must_be_positive(lance_table, transform_paralleli ) +# --------------------------------------------------------------------------- +# Backpressure / transform_queue_depth tests +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("transform_queue_depth", [0, -1]) +def test_transform_queue_depth_must_be_positive(lance_table, transform_queue_depth): + """transform_queue_depth=0 or negative must raise ValueError.""" + with pytest.raises( + ValueError, match="transform_queue_depth must be greater than 0" + ): + StreamingDataset( + lance_table, + num_splits=NUM_SPLITS, + transform_queue_depth=transform_queue_depth, + ) + + +@pytest.mark.parametrize("transform_queue_depth", [1, 2, 4]) +def test_transform_queue_depth_correctness(lance_table, transform_queue_depth): + """With backpressure enabled, every row is still yielded exactly once.""" + ds = StreamingDataset( + lance_table, + num_splits=NUM_SPLITS, + shuffle_seed=SHUFFLE_SEED, + transform_queue_depth=transform_queue_depth, + read_batch_size=8, + ) + items = list(ds) + assert sorted(item["id"] for item in items) == list(range(NUM_ROWS)) + + +def test_transform_queue_depth_matches_no_backpressure(lance_table): + """With backpressure enabled the same samples are produced as without it.""" + ds_unlimited = StreamingDataset( + lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED + ) + ds_limited = StreamingDataset( + lance_table, + num_splits=NUM_SPLITS, + shuffle_seed=SHUFFLE_SEED, + transform_queue_depth=1, + ) + assert [item["id"] for item in ds_unlimited] == [ + item["id"] for item in ds_limited + ], "transform_queue_depth must not affect the sample ordering or set" + + +def test_transform_queue_depth_bounds_cooked_rows(lance_table): + """prefetch_queue_depth stays within transform_queue_depth * read_batch_size + per split when observed from the main thread during iteration.""" + n_splits = 4 + batch_size = 8 + cooked_depth = 2 + # max cooked rows across all 4 splits: 4 * 2 * 8 = 64 + max_allowed = n_splits * cooked_depth * batch_size + + ds = StreamingDataset( + lance_table, + num_splits=n_splits, + shuffle_seed=SHUFFLE_SEED, + transform_queue_depth=cooked_depth, + read_batch_size=batch_size, + transform_parallelism=1, + world_size=1, + ) + + peak = 0 + for _ in ds: + depth = ds.prefetch_queue_depth + if depth > peak: + peak = depth + + # The main thread observes depth *after* popping a row, so the peak is at + # most max_allowed (one row already popped from the split just served). + assert peak <= max_allowed, ( + f"prefetch_queue_depth peaked at {peak}, expected <= {max_allowed}" + ) + + +def test_transform_queue_depth_does_not_admit_at_capacity_minus_one(tmp_path): + """Admission requires a full read_batch_size of free space, not just one slot. + + The test intercepts ThreadPoolExecutor.submit to make I/O calls execute + synchronously on the main thread. This ensures all raw batches land in + raw_batches (via _drain_io) before _try_submit_tx evaluates the admission + predicate for the first time. Without this, the I/O future for batch N+1 + might still be in io_pending at the capacity-minus-one transition, leaving + raw_batches empty and causing _try_submit_tx to skip the admission check + entirely — so both the correct and the broken predicate produce depth=0 + observations and the test cannot distinguish them. + + With all raw batches pre-loaded in raw_batches the 4→3 cooked transition + (consuming one row from a full cooked queue) always triggers _try_submit_tx + against a non-empty raw_batches. + + With transform_queue_depth=1 and batch_size=4, max_cooked_rows=4. + A transform may only be submitted when in_pipeline + batch_size <= 4, i.e. + when in_pipeline == 0 (cooked is completely empty). Under the old broken + predicate (in_pipeline >= max_cooked_rows) the second transform would be + admitted with cooked containing batch_size-1 rows still unconsumed. + """ + import concurrent.futures as cf + from concurrent.futures import ThreadPoolExecutor + from unittest.mock import patch + + db = lancedb.connect(tmp_path) + batch_size = 4 + # Four full batches → four transform submissions to observe. + table = db.create_table("t", pa.table({"id": list(range(batch_size * 4))})) + + cooked_at_submit: list[int] = [] + + original_submit = ThreadPoolExecutor.submit + + def tracking_submit(self, fn, *args, **kwargs): + name = getattr(fn, "__name__", "") + if name == "_io_call": + # Run I/O synchronously on the calling (main) thread and return an + # already-completed Future. _drain_io checks fut.done(), so a + # completed Future is moved to raw_batches immediately on the next + # _advance call — making raw-batch readiness deterministic at the + # capacity-minus-one transition instead of depending on I/O thread + # scheduling. + fut = cf.Future() + try: + fut.set_result(fn(*args, **kwargs)) + except Exception as exc: + fut.set_exception(exc) + return fut + if name == "_tx_call_guarded": + # Capture cooked depth synchronously on the main thread before the + # transform worker can drain the queue. + ref = ds._cooked_ref + cooked_at_submit.append(len(ref[0]) if ref is not None else -1) + return original_submit(self, fn, *args, **kwargs) + + with patch.object(ThreadPoolExecutor, "submit", tracking_submit): + ds = StreamingDataset( + table, + num_splits=1, + shuffle_seed=42, + read_batch_size=batch_size, + transform_queue_depth=1, + transform_parallelism=1, + ) + list(ds) + + assert len(cooked_at_submit) == 4, ( + f"Expected 4 transform submissions (one per batch), got {len(cooked_at_submit)}" + ) + # With full-batch backpressure each transform is only admitted when the + # cooked queue is completely empty (depth == 0). The old broken predicate + # would admit at depth == batch_size - 1 == 3. + assert all(depth == 0 for depth in cooked_at_submit), ( + "Transform admitted with non-empty cooked queue; full-batch backpressure " + "requires in_pipeline + batch_size <= max_cooked_rows before admission. " + f"Cooked depths at each submission: {cooked_at_submit}" + ) + + +# --------------------------------------------------------------------------- +# Deprecated parameter name tests +# --------------------------------------------------------------------------- + + +def test_prefetch_batches_deprecated_warns(lance_table, caplog): + """prefetch_batches logs a deprecation warning and behaves like io_queue_depth.""" + with caplog.at_level(logging.WARNING, logger="lancedb.streaming"): + ds = StreamingDataset( + lance_table, + num_splits=NUM_SPLITS, + shuffle_seed=SHUFFLE_SEED, + prefetch_batches=2, + ) + messages = [r.message for r in caplog.records if r.levelno >= logging.WARNING] + assert any("deprecated" in m.lower() and "io_queue_depth" in m for m in messages), ( + f"Expected deprecation warning mentioning io_queue_depth; got: {messages}" + ) + assert sorted(item["id"] for item in ds) == list(range(NUM_ROWS)) + + def test_filter_limits_rows(tmp_path): """A filter expression is applied to the permutation so only matching rows are yielded. IDs 0..59 pass ``id < 60``; the other 60 are excluded.""" @@ -1954,7 +2136,7 @@ def test_doc_example_basic(tmp_path): def test_doc_example_prefetch_params(tmp_path): - """doc: Prefetching — read_batch_size and prefetch_batches still cover all rows.""" + """doc: Prefetching — read_batch_size and io_queue_depth still cover all rows.""" db = lancedb.connect(tmp_path) table = db.create_table("t", pa.table({"id": list(range(NUM_ROWS))})) @@ -1963,7 +2145,7 @@ def test_doc_example_prefetch_params(tmp_path): num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED, read_batch_size=8, - prefetch_batches=2, + io_queue_depth=2, ) assert sorted(s["id"] for s in ds) == list(range(NUM_ROWS)) From 6cc77b573c0013e4d36205d1fee42f03972ae29a Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Sun, 23 Aug 2026 00:53:14 -0700 Subject: [PATCH 15/31] chore: update lance dependency to v11.0.0-beta.21 (#4029) Updates the Rust workspace and Java `lance-core` dependency to Lance v11.0.0-beta.21. No compatibility fixes were required; workspace Clippy passes with warnings denied. Triggering tag: https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.21 --- Cargo.lock | 84 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 28 +++++++++--------- java/pom.xml | 2 +- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87fec3378..804b5c8d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arc-swap", "arrow", @@ -4888,8 +4888,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-buffer", @@ -4911,7 +4911,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-buffer", @@ -4925,7 +4925,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-schema", @@ -4934,8 +4934,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrayref", "crunchy", @@ -4945,8 +4945,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-buffer", @@ -4983,8 +4983,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow", "arrow-array", @@ -5013,8 +5013,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow", "arrow-array", @@ -5031,8 +5031,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "proc-macro2", "quote", @@ -5041,8 +5041,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-arith", "arrow-array", @@ -5075,8 +5075,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-arith", "arrow-array", @@ -5107,8 +5107,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arc-swap", "arrow", @@ -5172,8 +5172,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-schema", @@ -5195,8 +5195,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow", "arrow-array", @@ -5232,8 +5232,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-schema", @@ -5247,8 +5247,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow", "async-trait", @@ -5260,8 +5260,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow", "arrow-ipc", @@ -5314,8 +5314,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-buffer", @@ -5329,8 +5329,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow", "arrow-array", @@ -5370,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "arrow-array", "arrow-schema", @@ -5384,8 +5384,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "11.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511" +version = "11.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index c147b06db..d45fe9a68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=11.0.0-beta.21", default-features = false, "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=11.0.0-beta.21", default-features = false, "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=11.0.0-beta.21", default-features = false, "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 56366bb15..e53ad869f 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 11.0.0-beta.19 + 11.0.0-beta.21 false 2.30.0 1.7 From 1b950188c3dc73383707fbab1ce85d4679787e07 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Sun, 23 Aug 2026 08:07:09 +0000 Subject: [PATCH 16/31] =?UTF-8?q?Bump=20version:=200.38.0-beta.4=20?= =?UTF-8?q?=E2=86=92=200.38.0-beta.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index bba238f96..daaec8171 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.4" +current_version = "0.38.0-beta.5" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 804b5c8d6..f83e81dc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5398,7 +5398,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.4" +version = "0.38.0-beta.5" dependencies = [ "ahash", "anyhow", @@ -5486,7 +5486,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.4" +version = "0.38.0-beta.5" dependencies = [ "arrow-array", "arrow-buffer", @@ -5511,7 +5511,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.4" +version = "0.38.0-beta.5" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index cff4cf2f5..b39c2dafb 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.4 + 0.38.0-beta.5 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 98c10ca38..7ebcb5743 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.4 + 0.38.0-beta.5 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index e53ad869f..10dc4c78c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.4 + 0.38.0-beta.5 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 99aed2e71..527715d8c 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.4" +version = "0.38.0-beta.5" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index c84b83c45..42e5bd09c 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 52f2b7901..d01e72d4e 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index afa7fac32..6c57484cf 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 36cfa0343..50a02b0de 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 23ea473ef..c8257452f 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 880b65540..cd98d84d4 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 8cc40de7b..1fda855df 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index dba3ede7c..429845ba8 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index c2f5be2c8..a82757bff 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.4", + "version": "0.38.0-beta.5", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index dace5aa7c..d88a25257 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.4" +version = "0.38.0-beta.5" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 2920498b1..439c9613b 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.4" +version = "0.38.0-beta.5" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 000e3b506b58ba71e4f0d43d1f5812006c696257 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Sun, 23 Aug 2026 10:31:32 -0700 Subject: [PATCH 17/31] chore: update lance dependency to v11.0.0-beta.22 (#4036) Updates the Rust workspace Lance dependencies and Java lance-core dependency to v11.0.0-beta.22, including the refreshed Cargo lockfile. No compatibility fixes were required; see the [Lance tag](https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.22). --- Cargo.lock | 88 +++++++++++++++++++++++++++------------------------- Cargo.toml | 28 ++++++++--------- java/pom.xml | 2 +- 3 files changed, 61 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f83e81dc3..89e4a1e30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arc-swap", "arrow", @@ -4888,8 +4888,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-buffer", @@ -4911,7 +4911,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-buffer", @@ -4925,7 +4925,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-schema", @@ -4934,8 +4934,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrayref", "crunchy", @@ -4945,8 +4945,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-buffer", @@ -4983,8 +4983,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-array", @@ -5013,8 +5013,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-array", @@ -5031,8 +5031,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "proc-macro2", "quote", @@ -5041,8 +5041,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-arith", "arrow-array", @@ -5075,8 +5075,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-arith", "arrow-array", @@ -5107,8 +5107,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arc-swap", "arrow", @@ -5172,8 +5172,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-schema", @@ -5195,8 +5195,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-array", @@ -5222,7 +5222,11 @@ dependencies = [ "pin-project", "prost", "rand 0.9.5", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", "serde", + "serde_json", "tempfile", "tokio", "tracing", @@ -5232,8 +5236,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-schema", @@ -5247,8 +5251,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "async-trait", @@ -5260,8 +5264,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-ipc", @@ -5314,8 +5318,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-buffer", @@ -5329,8 +5333,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-array", @@ -5370,8 +5374,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-schema", @@ -5384,8 +5388,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "11.0.0-beta.21" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.21#dd08336cf61b117701a7f5bbf76a7f7080f7e210" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index d45fe9a68..716b14d7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0-beta.21", default-features = false, "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=11.0.0-beta.21", default-features = false, "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=11.0.0-beta.21", default-features = false, "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=11.0.0-beta.21", "tag" = "v11.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 10dc4c78c..a6b1ced24 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 11.0.0-beta.21 + 11.0.0-beta.22 false 2.30.0 1.7 From 40d4d012e760ba0cc0bb9033ac9bf67fd1b83c43 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Sun, 23 Aug 2026 17:32:40 +0000 Subject: [PATCH 18/31] =?UTF-8?q?Bump=20version:=200.38.0-beta.5=20?= =?UTF-8?q?=E2=86=92=200.38.0-beta.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index daaec8171..f5f981b2c 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.5" +current_version = "0.38.0-beta.6" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 89e4a1e30..6c6dd4aea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5402,7 +5402,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.5" +version = "0.38.0-beta.6" dependencies = [ "ahash", "anyhow", @@ -5490,7 +5490,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.5" +version = "0.38.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.5" +version = "0.38.0-beta.6" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index b39c2dafb..d6c285fc3 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.5 + 0.38.0-beta.6 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 7ebcb5743..23d58124d 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.5 + 0.38.0-beta.6 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index a6b1ced24..a07414ec0 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.5 + 0.38.0-beta.6 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 527715d8c..ead873855 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.5" +version = "0.38.0-beta.6" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 42e5bd09c..d89eb5ad1 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index d01e72d4e..1721c970e 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 6c57484cf..177804dce 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 50a02b0de..9af443583 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index c8257452f..c98aa3812 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index cd98d84d4..05b1086f5 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 1fda855df..50432bdea 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 429845ba8..219025534 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index a82757bff..63c51a799 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.5", + "version": "0.38.0-beta.6", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index d88a25257..2e4f8996f 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.5" +version = "0.38.0-beta.6" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 439c9613b..d32d88ecd 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.5" +version = "0.38.0-beta.6" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 242ade8017e6935485f37f18a1bd5050434fec06 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Mon, 24 Aug 2026 13:26:36 +0530 Subject: [PATCH 19/31] feat(python): support sequence packing in streaming dataset (#3920) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## How packing works Consider four tokenized documents: [1] [2] [10, 11, 12, 13, 14, 15, 16, 17] [20] With: ``` StreamingDataset( table, shuffle=False, columns=["tokens"], num_splits=2, pack_sequences=5, eos_id=9, pad_id=0, blocks_per_epoch=6, ) ``` the documents are assigned to two fixed logical splits. Each split maintains an independent token buffer, appends eos_id after every document, and emits blocks of five tokens. Because blocks_per_epoch=6, each split emits exactly three blocks: Cycl/e 1: Split 0: [1, 9, 2, 9, 0] # 9 is eos, 0 is padding Split 1: [10, 11, 12, 13, 14] Cycle 2: Split 0: [0, 0, 0, 0, 0] Split 1: [15, 16, 17, 9, 20] Cycle 3: Split 0: [0, 0, 0, 0, 0] Split 1: [9, 0, 0, 0, 0] If a split runs out of tokens early, it emits padded blocks through the fixed budget. This prevents one rank from finishing before another. Logical splits are independent of rank and worker ownership. A checkpoint records each split’s consumed-document count, emitted-block count, remaining tokens, and document boundaries. Merging those per-split states allows the same packed stream to resume after the topology changes. doc_ids identifies document segments, including continuations across block boundaries. It is not a padding mask: padding retains the preceding document ID, so callers must mask padding using a reserved pad_id. blocks_per_epoch="auto" is also available. It estimates the budget from a deterministic bounded sample and warns that the result is approximate. WIP pre-training tests: ``` ┌────────────────────────────────────┬────────────────────────┬─────────────────────────────────────┐ │ │ GPT-2 124M │ GPT-2 medium 354M │ ├────────────────────────────────────┼────────────────────────┼─────────────────────────────────────┤ │ Corpus │ 2.4M docs / 12GB table │ 9.67M docs / 45GB table │ │ Tokens (Chinchilla) │ 2.43B │ 7.0B │ ├────────────────────────────────────┼────────────────────────┼─────────────────────────────────────┤ │ Data prep (ingest→curate→tokenize) │ ~12 min │ ~51 min │ ├────────────────────────────────────┼────────────────────────┼─────────────────────────────────────┤ │ Training wall time │ ~50 min │ 3h 06m │ ├────────────────────────────────────┼────────────────────────┼─────────────────────────────────────┤ │ Throughput / MFU │ 1.60M tok/s / 35% │ 684k tok/s / 42.0%, │ ├────────────────────────────────────┼────────────────────────┼─────────────────────────────────────┤ │ Final val loss │ 3.230 │ 2.840 │ └────────────────────────────────────┴────────────────────────┴─────────────────────────────────────┘ ``` --------- Co-authored-by: OpenAI Codex --- python/python/lancedb/streaming.py | 491 +++++++++++++++--- .../python/tests/test_elastic_dataloader.py | 208 ++++++++ 2 files changed, 638 insertions(+), 61 deletions(-) diff --git a/python/python/lancedb/streaming.py b/python/python/lancedb/streaming.py index 6951fa0bc..c54940ab7 100644 --- a/python/python/lancedb/streaming.py +++ b/python/python/lancedb/streaming.py @@ -24,11 +24,16 @@ import os import random import threading import time +import warnings from collections import deque from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy from multiprocessing import RawArray -from typing import Any, Callable, Iterator, Optional, Union +from typing import Any, Callable, cast, Iterator, Literal, Optional, Union +import pyarrow as pa +import pyarrow.compute as pc +import torch from torch.utils.data import IterableDataset, get_worker_info from .permutation import ( @@ -132,6 +137,39 @@ class StreamingDataset(IterableDataset): Maximum number of transforms to run concurrently. Must be greater than zero. When ``None`` (the default), uses ``os.cpu_count()`` or 1 when the CPU count is unavailable. + pack_sequences: + Sequence-packing mode: token lists from consecutive documents are + joined with ``eos_id`` and sliced into blocks of this many tokens. + Each item is then a dict of two ``(pack_sequences,)`` LongTensors — + ``input_ids`` and ``doc_ids`` (per-position document index within + the block, for block-diagonal masks or position-id resets). + * Packing happens independently per owned split and preserves per-split + resume state. + * When a split cannot fill a real block for a cycle but an owned sibling + still can, or when only a short tail remains at epoch end, the short buffer + is padded to ``pack_sequences`` with ``pad_id`` so every local cycle emits + one block per owned split. + * ``eos_id``, ``pad_id``, and ``columns`` naming a single integer-list + column are required; incompatible with ``transform``. + eos_id: + Separator token id between packed documents. Required with + ``pack_sequences``, ignored otherwise. + pad_id: + Padding token id used to complete blocks when a split runs out of + real tokens mid-cycle or at epoch end. Required with + ``pack_sequences``, ignored otherwise. It must be reserved for padding: + padding positions retain the preceding document's ``doc_id`` (or zero + in an all-padding block), so callers must mask them separately using + ``input_ids == pad_id``. + blocks_per_epoch: + Total number of packed blocks emitted globally per epoch. Required with + ``pack_sequences``. An integer must be divisible by ``num_splits``. + Every logical split emits exactly ``blocks_per_epoch / num_splits`` + blocks: exhausted splits emit padding, while tokens beyond the budget + are left out of the epoch. This fixed per-split budget keeps packed + iteration and checkpoints independent of rank topology. + Pass ``"auto"`` to estimate a corpus-level budget from a bounded sample + of token lists. The estimate may be inaccurate. on_transform_error: What to do when the transform raises an exception: @@ -210,6 +248,10 @@ class StreamingDataset(IterableDataset): filter: Optional[str] = None, transform: Optional[Callable] = None, transform_parallelism: Optional[int] = None, + pack_sequences: Optional[int] = None, + eos_id: Optional[int] = None, + pad_id: Optional[int] = None, + blocks_per_epoch: Optional[Union[int, Literal["auto"]]] = None, on_transform_error: Union[str, Callable[[Exception], bool]] = "raise", transform_queue_depth: Optional[int] = None, connection_factory: Optional[Callable[[str], Any]] = None, @@ -237,6 +279,55 @@ class StreamingDataset(IterableDataset): raise ValueError("io_queue_depth must be greater than 0") if transform_parallelism is not None and transform_parallelism <= 0: raise ValueError("transform_parallelism must be greater than 0") + if pack_sequences is not None: + if pack_sequences <= 0: + raise ValueError("pack_sequences must be greater than 0") + if eos_id is None: + raise ValueError("eos_id is required when pack_sequences is set") + if pad_id is None: + raise ValueError("pad_id is required when pack_sequences is set") + if blocks_per_epoch is None: + raise ValueError( + "blocks_per_epoch is required when pack_sequences is set" + ) + if blocks_per_epoch != "auto": + if not isinstance(blocks_per_epoch, int) or isinstance( + blocks_per_epoch, bool + ): + raise ValueError( + "blocks_per_epoch must be a positive integer or 'auto'" + ) + if blocks_per_epoch <= 0: + raise ValueError("blocks_per_epoch must be greater than 0") + if blocks_per_epoch % num_splits != 0: + raise ValueError( + f"blocks_per_epoch ({blocks_per_epoch}) must be divisible by " + f"num_splits ({num_splits})" + ) + if transform is not None: + raise ValueError("transform cannot be combined with pack_sequences") + if columns is None or len(columns) != 1: + raise ValueError( + "pack_sequences requires columns to name exactly one " + "list-typed column of token ids" + ) + field = table.schema.field(columns[0]) + if not ( + pa.types.is_list(field.type) + or pa.types.is_large_list(field.type) + or pa.types.is_fixed_size_list(field.type) + ): + raise ValueError( + f"pack_sequences requires a list-typed token column; " + f"{columns[0]} has type {field.type}" + ) + if not pa.types.is_integer(field.type.value_type): + raise ValueError( + "pack_sequences requires a token column with integer values; " + f"{columns[0]} has value type {field.type.value_type}" + ) + elif blocks_per_epoch is not None: + raise ValueError("blocks_per_epoch requires pack_sequences") if on_transform_error not in ("raise", "skip", "warn") and not callable( on_transform_error ): @@ -261,11 +352,20 @@ class StreamingDataset(IterableDataset): self._filter = filter self._transform = transform self._transform_parallelism = transform_parallelism + self._pack_sequences = pack_sequences + self._eos_id = eos_id + self._pad_id = pad_id + self._blocks_per_epoch = blocks_per_epoch self._on_transform_error = on_transform_error self._transform_queue_depth = transform_queue_depth self._connection_factory = connection_factory self._worker_info_override = worker_info_override + # Packing resume state: permutation positions and partial-block buffers. + self._pack_consumed: list[int] = [0] * num_splits + self._pack_buffers: dict[int, dict[str, list[int]]] = {} + self._pack_blocks_emitted: list[int] = [0] * num_splits + # Live references to pipeline state, set only while __iter__ is running # in the same process. Used by the observability properties when the # DataLoader runs with num_workers=0. @@ -315,6 +415,9 @@ class StreamingDataset(IterableDataset): else: self._perm_table = builder.split_sequential(fixed=num_splits).execute() + if self._blocks_per_epoch == "auto": + self._blocks_per_epoch = self._estimate_blocks_per_epoch() + # Contiguous block of global split indices assigned to this rank. splits_per_rank = num_splits // world_size rank_start = rank * splits_per_rank @@ -322,6 +425,71 @@ class StreamingDataset(IterableDataset): range(rank_start, rank_start + splits_per_rank) ) + def _estimate_blocks_per_epoch(self) -> int: + """Estimate a fixed packed-block budget from a bounded token sample.""" + # TODO: Replace this fallback with Lance's dedicated exact token-count + # estimation API once it is available. + if self._pack_sequences is None or not self._columns: + raise RuntimeError( + "packing must be configured before estimating its budget" + ) + + pack_len = self._pack_sequences + token_column = self._columns[0] + sample_cap_per_split = max(1, 100_000 // self._num_splits) + sampled_tokens = 0 + total_sampled = 0 + total_rows = 0 + rng = random.Random(self._shuffle_seed) + + warnings.warn( + "blocks_per_epoch='auto' uses an approximate token-count sample; " + "pass an explicit value for exact epoch sizing", + ) + + for split in range(self._num_splits): + permutation = Permutation.from_tables( + self._table, self._perm_table, split=split + ) + permutation = permutation.select_columns([token_column]) + permutation = permutation.with_transform(Transforms.arrow2arrow) + split_rows = permutation.num_rows + if split_rows == 0: + raise ValueError( + "blocks_per_epoch='auto' cannot estimate an empty dataset" + ) + + # Sample roughly 1% from each logical split, with at least one row + # per split and a global target cap of 100,000 rows. + sample_rows = min( + split_rows, + max(1, min((split_rows + 99) // 100, sample_cap_per_split)), + ) + sample_offsets = sorted(rng.sample(range(split_rows), sample_rows)) + sample_batch_size = max(1, self._read_batch_size) + for start in range(0, sample_rows, sample_batch_size): + batch = permutation.__getitems__( + sample_offsets[start : start + sample_batch_size] + ) + lengths = pc.list_value_length(batch.column(0)) + if lengths.null_count: + raise ValueError("pack_sequences does not support null token lists") + sampled_tokens += int(pc.sum(lengths).as_py()) + + total_sampled += sample_rows + total_rows += split_rows + + # Pool the samples into one global average. Each document contributes + # one EOS token. + estimated_tokens = ( + (sampled_tokens + total_sampled) * total_rows // total_sampled + ) + blocks = estimated_tokens // pack_len + return max( + self._num_splits, + blocks - blocks % self._num_splits, + ) + def _resolve_my_splits(self) -> list[int]: """Return the split indices this instance should read in __iter__.""" torch_worker_info = get_worker_info() @@ -372,8 +540,14 @@ class StreamingDataset(IterableDataset): ) if self._columns is not None: perm = perm.select_columns(self._columns) - perm = perm.with_transform(lambda batch: batch) - start_pos = self._resume_positions.get(split_idx, self._resume_offset) + perm = perm.with_transform(Transforms.arrow2arrow) + # Both modes resume from absolute permutation positions. Packing + # stores them separately because it also checkpoints partial blocks. + start_pos = ( + self._pack_consumed[split_idx] + if self._pack_sequences is not None + else self._resume_positions.get(split_idx, self._resume_offset) + ) if start_pos > 0: perm = perm.with_skip(start_pos) initial_positions.append(start_pos) @@ -395,9 +569,24 @@ class StreamingDataset(IterableDataset): if self._transform_parallelism is not None else (os.cpu_count() or 1) ) - final_transform = ( - self._transform if self._transform is not None else Transforms.arrow2python - ) + final_transform: Callable[[pa.RecordBatch], Any] + if self._pack_sequences is not None: + # Packing consumes raw token lists, one per document. + def arrow_tokens(batch: pa.RecordBatch) -> list[list[int]]: + token_column = batch.column(0) + if token_column.null_count or token_column.flatten().null_count: + raise ValueError( + "pack_sequences does not support null token lists or values" + ) + return cast(list[list[int]], token_column.to_pylist()) + + final_transform = arrow_tokens + else: + final_transform = ( + self._transform + if self._transform is not None + else Transforms.arrow2python + ) # None means no limit; otherwise cap rows per split to # transform_queue_depth batches worth (including in-flight transforms). max_cooked_rows = ( @@ -574,6 +763,82 @@ class StreamingDataset(IterableDataset): else: break # split exhausted + def _update_stats(*, idle: bool = False) -> None: + """Refresh pipeline statistics visible to the parent process.""" + ws = self._worker_stats + ws[0] = sum(split_sizes[j] - fetch_head[j] for j in range(n)) + ws[1] = ( + 0 + if idle + else sum(batch.num_rows for q in raw_batches for _, batch in q) + ) + ws[2] = 0 if idle else sum(len(q) for q in cooked) + ws[3] = sum(local_consumed) + ws[4] = self._bytes_loaded + ws[5] = int(self._fetch_time * 1_000_000) + ws[6] = int(self._transform_time * 1_000_000) + ws[7] = self._rows_skipped + + # Sequence-packing helpers + pack_len = cast(int, self._pack_sequences) + eos_id = cast(int, self._eos_id) + pad_id = cast(int, self._pad_id) + blocks_per_split = ( + cast(int, self._blocks_per_epoch) // self._num_splits + if self._pack_sequences is not None + else 0 + ) + pack_consumed = list(self._pack_consumed) + pack_buffers = deepcopy(self._pack_buffers) + pack_blocks_emitted = list(self._pack_blocks_emitted) + + def _pack_buffer(i: int) -> dict[str, list[int]]: + return pack_buffers.setdefault(my_splits[i], {"tokens": [], "starts": []}) + + def _fill_block(i: int) -> None: + """Fill split i's buffer to one block or exhaust the split.""" + buf = _pack_buffer(i) + while len(buf["tokens"]) < pack_len: + _ensure_cooked(i) + if not cooked[i]: + return + buf["starts"].append(len(buf["tokens"])) + pos, tokens = cooked[i].popleft() + buf["tokens"].extend(tokens) + buf["tokens"].append(eos_id) + pack_consumed[my_splits[i]] = pos + 1 + local_consumed[i] += 1 + _advance(i) + + def _emit_block(i: int) -> dict[str, Any]: + buf = _pack_buffer(i) + tokens, starts = buf["tokens"], buf["starts"] + # doc_ids label document segments within the block; 0 also covers + # the continuation of a document begun in a prior block. + doc_ids = torch.zeros(pack_len, dtype=torch.int64) + doc_starts = [s for s in starts if 0 < s < pack_len] + doc_ids[doc_starts] = 1 + doc_ids.cumsum_(dim=0) # cumulative sum marks document boundaries + block = { + "input_ids": torch.tensor(tokens[:pack_len], dtype=torch.int64), + "doc_ids": doc_ids, + } + del tokens[:pack_len] + # Shift start boundaries for the next call. + buf["starts"] = [s - pack_len for s in starts if s >= pack_len] + return block + + def _commit_pack_state() -> None: + self._pack_consumed = list(pack_consumed) + self._pack_buffers = { + split: { + "tokens": list(buffer["tokens"]), + "starts": list(buffer["starts"]), + } + for split, buffer in pack_buffers.items() + } + self._pack_blocks_emitted = list(pack_blocks_emitted) + # ── Main loop ───────────────────────────────────────────────────────── with ThreadPoolExecutor(max_workers=n * io_queue_depth) as io_pool: @@ -583,10 +848,42 @@ class StreamingDataset(IterableDataset): self._fetch_head_ref = fetch_head self._split_sizes_ref = split_sizes self._local_consumed_ref = local_consumed + try: for i in range(n): _fill_io(i) + if self._pack_sequences is not None: + first_count = pack_blocks_emitted[my_splits[0]] + if any( + pack_blocks_emitted[split] != first_count + for split in my_splits[1:] + ): + raise ValueError( + "Packed checkpoint is not aligned across the splits " + "owned by this iterator; merge every rank " + "state with merge_state_dicts before resuming on a " + "different topology" + ) + + while pack_blocks_emitted[my_splits[0]] < blocks_per_split: + # Each logical split gets one block per cycle. Exhausted + # splits are padded through the fixed global budget. + for i in range(n): + _fill_block(i) + + for i in range(n): + tokens = _pack_buffer(i)["tokens"] + if len(tokens) < pack_len: + tokens.extend([pad_id] * (pack_len - len(tokens))) + block = _emit_block(i) + pack_blocks_emitted[my_splits[i]] += 1 + if i == n - 1: + _commit_pack_state() + _update_stats() + yield block + return + while True: # A cycle only runs if every split can still produce a # row. Without skips all splits exhaust simultaneously @@ -620,21 +917,7 @@ class StreamingDataset(IterableDataset): self._resume_offset = initial_offset + local_consumed[i] for j, split_idx in enumerate(my_splits): self._resume_positions[split_idx] = pos_consumed[j] - ws = self._worker_stats - ws[0] = sum( - split_sizes[j] - fetch_head[j] for j in range(n) - ) - ws[1] = sum( - batch.num_rows - for q in raw_batches - for _, batch in q - ) - ws[2] = sum(len(q) for q in cooked) - ws[3] = sum(local_consumed) - ws[4] = self._bytes_loaded - ws[5] = int(self._fetch_time * 1_000_000) - ws[6] = int(self._transform_time * 1_000_000) - ws[7] = self._rows_skipped + _update_stats() yield row finally: @@ -642,15 +925,7 @@ class StreamingDataset(IterableDataset): # when iteration ends mid-cycle (e.g. a split whose rows # were all skipped before completing a single cycle), so # counters like rows_skipped would otherwise be stale. - ws = self._worker_stats - ws[0] = sum(split_sizes[j] - fetch_head[j] for j in range(n)) - ws[1] = 0 # queue-depth properties document 0 when idle - ws[2] = 0 - ws[3] = sum(local_consumed) - ws[4] = self._bytes_loaded - ws[5] = int(self._fetch_time * 1_000_000) - ws[6] = int(self._transform_time * 1_000_000) - ws[7] = self._rows_skipped + _update_stats(idle=True) self._raw_batches_ref = None self._cooked_ref = None self._fetch_head_ref = None @@ -808,21 +1083,31 @@ class StreamingDataset(IterableDataset): def state_dict(self) -> dict: """Snapshot the dataset's consumption state. - The returned dict is topology-independent: at global step boundaries - every split has been consumed the same number of times (by the - round-robin design), so the per-split count is a single uniform value - that is identical across all ranks and DataLoader workers. - - ``positions_consumed_per_split`` records how far into each split's - permutation iteration has advanced. It only differs from - ``samples_consumed_per_split`` when ``on_transform_error`` skipped - rows, in which case entries are exact for the splits this instance - iterated and a lower bound (the sample count) for splits owned by - other ranks or workers. Combine the state dicts from all ranks with + In row mode, the returned dict is topology-independent at global step + boundaries. ``positions_consumed_per_split`` records how far each + split's permutation has advanced, which can differ from the sample + count when ``on_transform_error`` skips rows. Combine state dicts from + every rank with [merge_state_dicts][lancedb.streaming.StreamingDataset.merge_state_dicts] - to recover the exact value for every split before resuming on a - different topology. + before resuming on a different topology. + + Packed state includes partial token buffers and emitted block counts + for every logical split. When packing is sharded, merge every rank + state with ``merge_state_dicts`` before loading it. """ + if self._pack_sequences is not None: + return { + "shuffle_seed": self._shuffle_seed, + "num_splits": self._num_splits, + "epoch": self._epoch, + "pack_sequences": self._pack_sequences, + "eos_id": self._eos_id, + "pad_id": self._pad_id, + "blocks_per_epoch": self._blocks_per_epoch, + "samples_consumed_per_split": list(self._pack_consumed), + "blocks_emitted_per_split": list(self._pack_blocks_emitted), + "pack_buffers": deepcopy(self._pack_buffers), + } positions = [ self._resume_positions.get(split, self._resume_offset) for split in range(self._num_splits) @@ -840,7 +1125,9 @@ class StreamingDataset(IterableDataset): Raises ``ValueError`` if ``num_splits`` or ``shuffle_seed`` differ from the checkpoint, since a different split structure or shuffle order - makes mid-epoch resumption meaningless. + makes mid-epoch resumption meaningless. Packed checkpoints + pin ``pack_sequences``, ``eos_id``, ``pad_id``, + ``blocks_per_epoch``, and ``epoch``. """ if state["num_splits"] != self._num_splits: raise ValueError( @@ -852,6 +1139,31 @@ class StreamingDataset(IterableDataset): f"shuffle_seed mismatch: checkpoint has {state['shuffle_seed']}, " f"current dataset has {self._shuffle_seed}" ) + + if "pack_buffers" in state or self._pack_sequences is not None: + for key in ( + "pack_sequences", + "eos_id", + "pad_id", + "blocks_per_epoch", + "epoch", + ): + ours = getattr(self, f"_{key}") + if state.get(key) != ours: + raise ValueError( + f"{key} mismatch: checkpoint has {state.get(key)}, " + f"current dataset has {ours}" + ) + self._pack_consumed = [int(c) for c in state["samples_consumed_per_split"]] + self._pack_blocks_emitted = [ + int(c) for c in state["blocks_emitted_per_split"] + ] + self._pack_buffers = { + int(g): {"tokens": list(b["tokens"]), "starts": list(b["starts"])} + for g, b in state["pack_buffers"].items() + } + return + consumed = state["samples_consumed_per_split"] # All entries are equal at step boundaries; use the first. if isinstance(consumed, list): @@ -873,25 +1185,22 @@ class StreamingDataset(IterableDataset): def merge_state_dicts(states: list[dict]) -> dict: """Merge state dicts saved by different ranks into one exact state. - Only needed when ``on_transform_error`` skips rows in multi-rank - training: each rank then knows the exact permutation position only for - its own splits, and records a lower bound for the rest. Because - exactly one rank owns each split, the elementwise maximum across all - ranks' ``positions_consumed_per_split`` recovers the exact position of - every split. Without skipped rows every rank's state is already - identical and merging is a no-op. + For row mode, the elementwise maximum of permutation positions recovers + splits advanced by different ranks after transform failures. For packed + mode, the state that emitted the most blocks for each logical split + supplies that split's permutation position and partial token buffer. Packed + states must cover every rank at the same global step. - Raises ``ValueError`` if the states are empty or were not produced by - the same run (mismatched seed, split count, epoch, or sample counts). + Raises ``ValueError`` if the states are empty, were not produced by + the same run, or do not represent the same global step. The merge is always all-to-all and topology-agnostic: collect the - ``state_dict()`` from every rank of the *previous* run into one list, - merge that whole list, and hand the identical merged result to every - rank of the *next* run — regardless of whether the rank count grew, - shrank, or stayed the same. There is no pairwise or subset merging - step, because each split's exact position is only known to whichever - rank owned that split, and the elementwise maximum needs every rank's - contribution to be correct. + ``state_dict()`` from every rank of the *previous* run into + one list, merge that whole list, and hand the identical merged result + to every rank of the *next* run — regardless of whether the + topology grew, shrank, or stayed the same. There is no pairwise or + subset merging step, because each split's exact state is only known to + whichever iterator owned that split. For example, checkpointing 8 ranks and resuming on 4 (the same pattern applies when growing, e.g. 4 ranks resuming on 8):: @@ -924,13 +1233,73 @@ class StreamingDataset(IterableDataset): if not states: raise ValueError("merge_state_dicts requires at least one state dict") first = states[0] + packed = "pack_buffers" in first + config_keys = ["shuffle_seed", "num_splits", "epoch"] + if packed: + config_keys.extend( + ["pack_sequences", "eos_id", "pad_id", "blocks_per_epoch"] + ) + for state in states[1:]: - for key in ("shuffle_seed", "num_splits", "epoch"): + if ("pack_buffers" in state) != packed: + raise ValueError("cannot merge packed and unpacked state dicts") + for key in config_keys: if state[key] != first[key]: raise ValueError( f"{key} mismatch across state dicts: " f"{state[key]} != {first[key]}" ) + + if packed: + num_splits = first["num_splits"] + for state in states: + for key in ( + "samples_consumed_per_split", + "blocks_emitted_per_split", + ): + if len(state[key]) != num_splits: + raise ValueError( + f"{key} must contain one entry per logical split" + ) + + merged_consumed = [] + merged_emitted = [] + merged_buffers = {} + for split in range(num_splits): + owner = states[0] + owner_progress = ( + owner["blocks_emitted_per_split"][split], + owner["samples_consumed_per_split"][split], + ) + for state in states[1:]: + progress = ( + state["blocks_emitted_per_split"][split], + state["samples_consumed_per_split"][split], + ) + if progress > owner_progress: + owner = state + owner_progress = progress + merged_consumed.append(owner["samples_consumed_per_split"][split]) + merged_emitted.append(owner["blocks_emitted_per_split"][split]) + buffer = owner["pack_buffers"].get( + split, owner["pack_buffers"].get(str(split)) + ) + if buffer is not None: + merged_buffers[split] = deepcopy(buffer) + + if len(set(merged_emitted)) > 1: + raise ValueError( + "packed state dicts were not captured at the same global " + "step or do not cover every rank" + ) + + merged = dict(first) + merged["samples_consumed_per_split"] = merged_consumed + merged["blocks_emitted_per_split"] = merged_emitted + merged["pack_buffers"] = merged_buffers + return merged + + for state in states[1:]: if ( state["samples_consumed_per_split"] != first["samples_consumed_per_split"] diff --git a/python/python/tests/test_elastic_dataloader.py b/python/python/tests/test_elastic_dataloader.py index 4d860a5de..ff65d100c 100644 --- a/python/python/tests/test_elastic_dataloader.py +++ b/python/python/tests/test_elastic_dataloader.py @@ -2113,6 +2113,214 @@ def test_shuffle_seed_none_generates_stable_seed(lance_table): assert first == second, "Same resolved seed must produce the same ordering" +# Sequence packing tests + + +def _create_token_table(tmp_path, documents): + db = lancedb.connect(tmp_path) + tokens = pa.array(documents, type=pa.list_(pa.int64())) + return db.create_table("tokens", pa.table({"tokens": tokens})) + + +def _packed_dataset(table, pack_sequences, *, blocks_per_epoch, pad_id=0, **kwargs): + return StreamingDataset( + table, + shuffle=False, + columns=["tokens"], + pack_sequences=pack_sequences, + eos_id=9, + pad_id=pad_id, + blocks_per_epoch=blocks_per_epoch, + **kwargs, + ) + + +def test_pack_sequences_emits_blocks_and_pads_final_tail(tmp_path): + table = _create_token_table(tmp_path, [[1, 2], [3, 4], [5]]) + dataset = _packed_dataset(table, 6, blocks_per_epoch=2) + + blocks = list(dataset) + + assert len(blocks) == 2 + assert blocks[0]["input_ids"].tolist() == [1, 2, 9, 3, 4, 9] + assert blocks[0]["doc_ids"].tolist() == [0, 0, 0, 1, 1, 1] + assert blocks[1]["input_ids"].tolist() == [5, 9, 0, 0, 0, 0] + assert blocks[1]["doc_ids"].tolist() == [0, 0, 0, 0, 0, 0] + assert blocks[0]["input_ids"].dtype == torch.int64 + assert blocks[0]["doc_ids"].dtype == torch.int64 + + +def test_pack_sequences_pads_lagging_splits(tmp_path): + table = _create_token_table( + tmp_path, + [[1], [2], [10, 11, 12, 13, 14, 15, 16, 17], [20]], + ) + dataset = _packed_dataset(table, 5, blocks_per_epoch=6, num_splits=2) + input_ids = [block["input_ids"].tolist() for block in dataset] + # Split 0 has four real tokens including EOS markers, while split 1 has + # eleven. Packing must emit three complete two-split cycles. + assert input_ids == [ + [1, 9, 2, 9, 0], + [10, 11, 12, 13, 14], + [0, 0, 0, 0, 0], + [15, 16, 17, 9, 20], + [0, 0, 0, 0, 0], + [9, 0, 0, 0, 0], + ] + + per_rank = [] + for rank in range(2): + rank_dataset = _packed_dataset( + table, + 5, + blocks_per_epoch=6, + num_splits=2, + world_size=2, + rank=rank, + ) + per_rank.append([block["input_ids"].tolist() for block in rank_dataset]) + + assert [len(blocks) for blocks in per_rank] == [3, 3] + sharded = [block for cycle in zip(*per_rank) for block in cycle] + assert sharded == input_ids + + +def test_pack_sequences_auto_estimates_filtered_token_column(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table( + "tokens", + pa.table( + { + "tokens": pa.array([[1] * 4, [2] * 9], type=pa.list_(pa.int64())), + "keep": [True, False], + } + ), + ) + table.add( + pa.table( + { + "tokens": pa.array([[3] * 4, [4] * 9], type=pa.list_(pa.int64())), + "keep": [True, False], + } + ) + ) + + with pytest.warns(UserWarning, match="approximate token-count sample"): + dataset = _packed_dataset( + table, + 5, + blocks_per_epoch="auto", + num_splits=2, + filter="keep", + ) + + # Two kept documents contain 8 tokens plus 2 EOS tokens: two blocks. + assert dataset.state_dict()["blocks_per_epoch"] == 2 + + +def test_pack_sequences_checkpoint_resumes_on_new_topology(tmp_path): + table = _create_token_table( + tmp_path, + [[1], [2], [10, 11, 12, 13, 14, 15, 16, 17], [20]], + ) + kwargs = dict(pack_sequences=5, blocks_per_epoch=6, num_splits=2) + reference = list(_packed_dataset(table, **kwargs)) + + datasets = [ + _packed_dataset(table, world_size=2, rank=rank, **kwargs) for rank in range(2) + ] + iterators = [iter(dataset) for dataset in datasets] + first_cycle = [next(iterator) for iterator in iterators] + checkpoint = StreamingDataset.merge_state_dicts( + [dataset.state_dict() for dataset in datasets] + ) + for iterator in iterators: + iterator.close() + + resumed = _packed_dataset(table, **kwargs) + resumed.load_state_dict(checkpoint) + actual_remaining = list(resumed) + + assert [block["input_ids"].tolist() for block in first_cycle] == [ + [1, 9, 2, 9, 0], + [10, 11, 12, 13, 14], + ] + assert checkpoint["blocks_emitted_per_split"] == [1, 1] + assert [block["input_ids"].tolist() for block in actual_remaining] == [ + block["input_ids"].tolist() for block in reference[2:] + ] + assert [block["doc_ids"].tolist() for block in actual_remaining] == [ + block["doc_ids"].tolist() for block in reference[2:] + ] + + +def test_pack_sequences_validates_configuration_and_tokens(tmp_path): + table = _create_token_table(tmp_path, [[1, 2]]) + + with pytest.raises(ValueError, match="pad_id is required"): + StreamingDataset( + table, + shuffle=False, + columns=["tokens"], + pack_sequences=4, + eos_id=9, + ) + + with pytest.raises(ValueError, match="blocks_per_epoch is required"): + StreamingDataset( + table, + shuffle=False, + columns=["tokens"], + pack_sequences=4, + eos_id=9, + pad_id=0, + ) + + with pytest.raises(ValueError, match="must be divisible"): + _packed_dataset(table, 4, blocks_per_epoch=3, num_splits=2) + + with pytest.raises(ValueError, match="positive integer or 'auto'"): + _packed_dataset(table, 4, blocks_per_epoch="estimate") + + checkpoint = _packed_dataset(table, 4, blocks_per_epoch=1).state_dict() + resumed = _packed_dataset(table, 4, blocks_per_epoch=1, pad_id=8) + with pytest.raises(ValueError, match="pad_id mismatch"): + resumed.load_state_dict(checkpoint) + + float_db = lancedb.connect(tmp_path / "float") + float_table = float_db.create_table( + "tokens", + pa.table({"tokens": pa.array([[1.5, 2.5]], type=pa.list_(pa.float64()))}), + ) + with pytest.raises(ValueError, match="token column with integer values"): + _packed_dataset(float_table, 4, blocks_per_epoch=1) + + null_db = lancedb.connect(tmp_path / "null") + null_table = null_db.create_table( + "tokens", + pa.table({"tokens": pa.array([None], type=pa.list_(pa.int64()))}), + ) + with pytest.raises(ValueError, match="does not support null token lists"): + list(_packed_dataset(null_table, 4, blocks_per_epoch=1)) + + null_value_db = lancedb.connect(tmp_path / "null_value") + null_value_table = null_value_db.create_table( + "tokens", + pa.table( + {"tokens": pa.array([[1], [2, None], [3]], type=pa.list_(pa.int64()))} + ), + ) + blocks = list( + _packed_dataset( + null_value_table, + 2, + blocks_per_epoch=2, + on_transform_error="skip", + ) + ) + assert [block["input_ids"].tolist() for block in blocks] == [[1, 9], [3, 9]] + + # --------------------------------------------------------------------------- # Doc examples — each test mirrors the code snippet in index.mdx so that # broken doc examples are caught before they ship. From b0dae5eb0b468a89b2f353a548e49011ce4881d6 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 25 Aug 2026 00:01:32 +0800 Subject: [PATCH 20/31] feat: return typed refresh job results (#4013) ## Problem `refresh_column_async` returned a unit-result job even though durable refresh jobs carry a canonical terminal result. Python callers could not obtain row counts or source and published versions through the public `Job` API, and local and remote refresh jobs exposed different result semantics. ## Behavior `refresh_column_async` now returns `Job[RefreshColumnResult]` for local and remote tables. The general typed-job bridge binds each endpoint to its public result model while preserving unit-result jobs and existing status, wait, cancel, and timeout behavior. A local no-op refresh reports no published version. The Node.js API continues to resolve `wait()` as `void`; its binding erases the Rust result type internally to preserve the existing public contract. ## Ownership and integration boundary LanceDB owns the language-neutral `Job` contract and language-binding decode. Sophon owns production and durable persistence of terminal payloads. Sophon #7348 and #7378 now publish the canonical refresh result for Function-backed and expression-backed refresh jobs, respectively. The remote client fixture matches the merged server schema; live deployment and end-to-end demo acceptance remain separate rollout checks. --- nodejs/src/job.rs | 7 +- python/python/lancedb/__init__.py | 1 + python/python/lancedb/_lancedb.pyi | 11 +- python/python/lancedb/db.py | 4 +- python/python/lancedb/functions.py | 10 +- python/python/lancedb/job.py | 58 ++++--- python/python/lancedb/remote/table.py | 4 +- python/python/lancedb/table.py | 39 ++++- python/python/tests/test_db.py | 4 +- .../tests/test_first_class_function_slice2.py | 2 +- python/python/tests/test_index.py | 2 +- python/python/tests/test_remote_db.py | 76 ++++++++- python/python/tests/test_table.py | 21 ++- python/src/connection.rs | 2 +- python/src/job.rs | 73 +++------ python/src/lib.rs | 1 - python/src/table.rs | 2 +- rust/lancedb/src/function.rs | 14 +- rust/lancedb/src/job.rs | 152 +++++++++++++----- rust/lancedb/src/remote/db.rs | 4 +- rust/lancedb/src/remote/table.rs | 36 ++++- rust/lancedb/src/table.rs | 22 ++- rust/lancedb/src/table/refresh.rs | 71 ++++++-- 23 files changed, 429 insertions(+), 187 deletions(-) diff --git a/nodejs/src/job.rs b/nodejs/src/job.rs index 6aaeee174..14013fd27 100644 --- a/nodejs/src/job.rs +++ b/nodejs/src/job.rs @@ -14,9 +14,12 @@ pub struct Job { } impl Job { - pub(crate) fn new(inner: lancedb::Job) -> Self { + pub(crate) fn new(inner: lancedb::Job) -> Self + where + T: Clone + Send + Sync + 'static, + { Self { - inner: Arc::new(inner), + inner: Arc::new(inner.map(|_| ())), } } } diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index aa473c5f8..df8950686 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -29,6 +29,7 @@ from .functions import ( FunctionRegistrationRequest as FunctionRegistrationRequest, FunctionVersion as FunctionVersion, PythonRuntimeSpec as PythonRuntimeSpec, + RefreshColumnResult as RefreshColumnResult, UdfDefinition as UdfDefinition, udf as udf, ) diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 648469579..1e314ede8 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -147,7 +147,7 @@ class Connection(object): limit: Optional[int], ) -> list[str]: ... # Deprecated: Use list_tables instead def job(self, job_id: str) -> Job: ... - async def create_function_async(self, request_json: str) -> FunctionJob: ... + async def create_function_async(self, request_json: str) -> Job: ... async def get_function(self, name: str, version: str) -> str: ... async def list_jobs(self) -> List[JobInfo]: ... async def get_job(self, job_id: str) -> Optional[JobDescription]: ... @@ -234,14 +234,7 @@ class Job: @property def id(self) -> Optional[str]: ... async def status(self) -> str: ... - async def wait(self) -> None: ... - async def cancel(self) -> None: ... - -class FunctionJob: - @property - def id(self) -> Optional[str]: ... - async def status(self) -> str: ... - async def wait(self) -> str: ... + async def wait(self) -> Optional[str]: ... async def cancel(self) -> None: ... class JobInfo: diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 7ad749920..51b8d9993 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -46,7 +46,7 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError from . import __version__ from ._lancedb import connect as lancedb_connect # type: ignore from .functions import FunctionVersion, UdfDefinition -from .job import AsyncJob, Job, _function_job +from .job import AsyncJob, Job, _typed_job from .materialized_view import ( AsyncMaterializedView, MaterializedView, @@ -2237,7 +2237,7 @@ class AsyncConnection(object): inner = await self._inner.create_function_async( definition.registration_request.to_canonical_json() ) - return _function_job(inner) + return _typed_job(inner, FunctionVersion.from_json) async def get_function(self, name: str, *, version: str) -> FunctionVersion: """Open one exact immutable Function version from the remote catalog.""" diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 9532ab8b9..c4ff9a9b3 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -1,10 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors -"""Canonical values exchanged with LanceDB Enterprise Function services. +"""Canonical Function values exchanged with LanceDB Enterprise services. These immutable models contain client/wire state only. Catalog persistence, environment bake, secret resolution, and execution are owned by Sophon. +``RefreshColumnResult`` is also the backend-neutral result of a local +expression-backed refresh job. """ from __future__ import annotations @@ -460,7 +462,11 @@ class FunctionBinding(_RemoteValue): class RefreshColumnResult(_RemoteValue): - """Terminal result of a remote Function-column refresh Job.""" + """Terminal result of an expression-backed or Function-backed refresh Job. + + Local jobs produce this value in process. LanceDB Cloud and Enterprise + decode the same value from the durable server-job terminal payload. + """ rows_assigned: _UInt64 rows_failed: _UInt64 diff --git a/python/python/lancedb/job.py b/python/python/lancedb/job.py index 7bd600a74..f688768cb 100644 --- a/python/python/lancedb/job.py +++ b/python/python/lancedb/job.py @@ -5,12 +5,11 @@ import asyncio from datetime import timedelta -from typing import Any, Generic, Optional, TypeVar, cast +from typing import Any, Callable, Generic, Optional, TypeVar, cast from lancedb.background_loop import LOOP from . import _lancedb -from .functions import FunctionVersion T = TypeVar("T") @@ -18,11 +17,18 @@ T = TypeVar("T") class AsyncJob(Generic[T]): """A handle to an operation that may still be running. - The operation may already be complete when the handle is created. + The operation may already be complete when the handle is created. ``T`` + is the endpoint's terminal result type; unit-result jobs resolve to + ``None``. """ - def __init__(self, inner: Optional[Any]): + def __init__( + self, + inner: Optional[Any], + result_decoder: Optional[Callable[[Any], T]] = None, + ): self._inner = inner + self._result_decoder = result_decoder @property def id(self) -> Optional[str]: @@ -50,17 +56,21 @@ class AsyncJob(Generic[T]): async def wait(self, timeout: Optional[timedelta] = None) -> T: """Wait until the operation reaches a terminal state. + Returns the endpoint's typed result, or ``None`` for a unit-result + job. + Raises `JobFailedError` if the operation failed, `JobCancelledError` if it was cancelled, and `TimeoutError` if `timeout` elapses first. """ if self._inner is None: return cast(T, None) if timeout is None: - return cast(T, await self._inner.wait()) - return cast( - T, - await asyncio.wait_for(self._inner.wait(), timeout.total_seconds()), - ) + result = await self._inner.wait() + else: + result = await asyncio.wait_for(self._inner.wait(), timeout.total_seconds()) + if self._result_decoder is not None: + return self._result_decoder(result) + return cast(T, result) async def cancel(self): """Request cancellation. Cancelling a finished operation is a no-op.""" @@ -70,7 +80,7 @@ class AsyncJob(Generic[T]): class Job(Generic[T]): - """Synchronous counterpart of `AsyncJob`.""" + """Synchronous counterpart of `AsyncJob` with the same result type.""" def __init__(self, inner: Optional[AsyncJob[T]]): self._inner = inner @@ -96,6 +106,9 @@ class Job(Generic[T]): def wait(self, timeout: Optional[timedelta] = None) -> T: """Block until the operation reaches a terminal state. + Returns the endpoint's typed result, or ``None`` for a unit-result + job. + Raises `JobFailedError` if the operation failed, `JobCancelledError` if it was cancelled, and `TimeoutError` if `timeout` elapses first. """ @@ -110,23 +123,8 @@ class Job(Generic[T]): LOOP.run(self._inner.cancel()) -class _FunctionJobAdapter: - def __init__(self, inner: "_lancedb.FunctionJob"): - self._inner = inner - - @property - def id(self) -> Optional[str]: - return self._inner.id - - async def status(self) -> str: - return await self._inner.status() - - async def wait(self) -> FunctionVersion: - return FunctionVersion.from_json(await self._inner.wait()) - - async def cancel(self): - await self._inner.cancel() - - -def _function_job(inner: "_lancedb.FunctionJob") -> AsyncJob[FunctionVersion]: - return AsyncJob(_FunctionJobAdapter(inner)) +def _typed_job( + inner: "_lancedb.Job", result_decoder: Callable[[str], T] +) -> AsyncJob[T]: + """Bind an internal JSON-producing job to its public result model.""" + return AsyncJob(inner, result_decoder) diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index b97f8f194..41394ed71 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -49,7 +49,7 @@ from lancedb.index import ( LabelList, ) from lancedb.job import Job -from lancedb.functions import FunctionApplication +from lancedb.functions import FunctionApplication, RefreshColumnResult from lancedb.remote.db import LOOP from lancedb.table import IndexConfigType, KNOWN_METRICS import pyarrow as pa @@ -972,7 +972,7 @@ class RemoteTable(Table): def refresh_column(self, column: str): return LOOP.run(self._table.refresh_column(column)) - def refresh_column_async(self, column: str) -> Job: + def refresh_column_async(self, column: str) -> Job[RefreshColumnResult]: return Job(LOOP.run(self._table.refresh_column_async(column))) def alter_columns( diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 0c84b4036..75b7b1db8 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -40,7 +40,7 @@ from ._blob import ( from .types import BlobMode from lancedb.arrow import peek_reader from lancedb.background_loop import LOOP, embedding_executor -from lancedb.job import AsyncJob, Job +from lancedb.job import AsyncJob, Job, _typed_job from .dependencies import ( _check_for_hugging_face, _check_for_lance, @@ -72,7 +72,10 @@ from .index import ( FTS, ) from .expr import Expr -from .functions import FunctionApplication +from .functions import ( + FunctionApplication, + RefreshColumnResult as RefreshColumnJobResult, +) from .merge import LanceMergeInsertBuilder from .pydantic import LanceModel, model_to_dict from .query import ( @@ -2039,7 +2042,7 @@ class Table(ABC): """ @abstractmethod - def refresh_column_async(self, column: str) -> Job: + def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]: """ Like :meth:`refresh_column`, but returns a handle to the refresh job instead of blocking until it completes. @@ -2050,6 +2053,12 @@ class Table(ABC): than failing the job. On local tables the job runs in-process; on LanceDB Cloud and Enterprise it is the server's backfill job. + Returns + ------- + Job[RefreshColumnResult] + A job whose successful ``wait`` returns row counts plus the source + and published table versions. + Examples -------- >>> import lancedb @@ -2058,7 +2067,9 @@ class Table(ABC): >>> table.add_columns(computed={"doubled": "x * 2"}) AddColumnsResult(version=2) >>> job = table.refresh_column_async("doubled") - >>> job.wait() + >>> result = job.wait() + >>> result.rows_assigned + 2 >>> job.status() 'finished' """ @@ -4082,7 +4093,7 @@ class LanceTable(Table): [`AsyncTable.refresh_column`][lancedb.AsyncTable.refresh_column].""" return LOOP.run(self._table.refresh_column(column)) - def refresh_column_async(self, column: str) -> Job: + def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]: """Fill a computed column's unfilled rows, returning a handle to the refresh job. See [`Table.refresh_column_async`][lancedb.table.Table.refresh_column_async]. @@ -6122,7 +6133,9 @@ class AsyncTable: """ return await self._inner.refresh_column(column) - async def refresh_column_async(self, column: str) -> AsyncJob: + async def refresh_column_async( + self, column: str + ) -> AsyncJob[RefreshColumnJobResult]: """ Like :meth:`refresh_column`, but returns a handle to the refresh job instead of blocking until it completes. @@ -6134,6 +6147,12 @@ class AsyncTable: in-process; on LanceDB Cloud and Enterprise it is the server's backfill job. + Returns + ------- + AsyncJob[RefreshColumnResult] + A job whose successful ``wait`` returns row counts plus the source + and published table versions. + Examples -------- >>> import asyncio @@ -6143,12 +6162,16 @@ class AsyncTable: ... table = await db.create_table("computed_job_async_demo", [{"x": 1}]) ... await table.add_columns(computed={"doubled": "x * 2"}) ... job = await table.refresh_column_async("doubled") - ... await job.wait() + ... result = await job.wait() + ... assert result.rows_assigned == 1 ... return await job.status() >>> asyncio.run(refresh_in_background()) 'finished' """ - return AsyncJob(await self._inner.refresh_column_async(column)) + return _typed_job( + await self._inner.refresh_column_async(column), + RefreshColumnJobResult.from_json, + ) async def alter_columns( self, *alterations: Iterable[dict[str, Any]] diff --git a/python/python/tests/test_db.py b/python/python/tests/test_db.py index 38bbb53fb..aeb9feeb8 100644 --- a/python/python/tests/test_db.py +++ b/python/python/tests/test_db.py @@ -774,7 +774,7 @@ def test_drop_table_async(tmp_db: lancedb.DBConnection): job = tmp_db.drop_table_async("test") assert job.id is None assert job.status() == "finished" - job.wait() + assert job.wait() is None assert tmp_db.table_names() == [] tmp_db.create_table("test", data=data) @@ -790,7 +790,7 @@ async def test_drop_table_async_connection(tmp_db_async: lancedb.AsyncConnection job = await tmp_db_async.drop_table_async("test") assert job.id is None assert await job.status() == "finished" - await job.wait() + assert await job.wait() is None assert await tmp_db_async.table_names() == [] diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 99c68876c..a20347771 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -187,7 +187,7 @@ def _mock_remote_function_catalog(): "job_state": "DONE", "result": state["version"], } - elif self.path == "/v1/functions/get": + elif self.path == "/v1/functions/describe": assert body == { "name": "normalize_score", "version": "fv_exact", diff --git a/python/python/tests/test_index.py b/python/python/tests/test_index.py index 94268a53e..fe6ebe87a 100644 --- a/python/python/tests/test_index.py +++ b/python/python/tests/test_index.py @@ -88,7 +88,7 @@ async def binary_table(db_async): async def test_create_index_async_returns_done_job(some_table: AsyncTable): job = await some_table.create_index_async("id", config=BTree()) assert job.id is None - await job.wait() + assert await job.wait() is None assert len(await some_table.list_indices()) == 1 await job.cancel() diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 08f5550eb..2952f00a4 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -875,11 +875,85 @@ def test_remote_create_index_async_returns_job(): table = db.create_table("test", [{"id": 1}]) job = table.create_index_async("id", config=BTree()) assert job.id == "job-1" - job.wait(timeout=timedelta(seconds=30)) + assert job.wait(timeout=timedelta(seconds=30)) is None assert len(describe_calls) == 2 job.cancel() +def test_remote_refresh_async_returns_typed_terminal_result(): + terminal_result = { + "rows_assigned": 12, + "rows_failed": 0, + "rows_remaining": 0, + "source_version": 7, + "published_version": 8, + } + + def handler(request): + content_len = int(request.headers.get("Content-Length", 0)) + body = request.rfile.read(content_len) if content_len > 0 else b"" + if request.path == "/v1/table/test/backfill_column": + assert json.loads(body)["column"] == "derived" + request.send_response(202) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b'{"job_id": "refresh-1"}') + elif request.path == "/v1/jobs/describe": + assert json.loads(body)["job_id"] == "refresh-1" + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write( + json.dumps( + { + "job_id": "refresh-1", + "job_type": "function_refresh", + "job_state": "DONE", + "result": terminal_result, + } + ).encode() + ) + elif request.path == "/v1/table/test/create/?mode=create": + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b"{}") + elif request.path == "/v1/table/test/describe/": + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write( + json.dumps( + { + "version": 1, + "schema": { + "fields": [ + { + "name": "id", + "type": {"type": "int64"}, + "nullable": False, + } + ] + }, + } + ).encode() + ) + else: + request.send_response(404) + request.end_headers() + + with mock_lancedb_connection(handler) as db: + table = db.create_table("test", [{"id": 1}]) + job = table.refresh_column_async("derived") + assert job.id == "refresh-1" + result = job.wait(timeout=timedelta(seconds=30)) + + assert isinstance(result, lancedb.RefreshColumnResult) + assert result.model_dump() == terminal_result + assert result.rows_filled == 12 + assert result.version == 8 + + def test_remote_job_wait_raises_on_failure(): from lancedb.exceptions import JobFailedError from lancedb.index import BTree diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 0f98219bf..56e0eacfd 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -1467,7 +1467,7 @@ def test_create_index_async_returns_done_job(mem_db: DBConnection): table = mem_db.create_table("job_test", [{"id": i} for i in range(10)]) job = table.create_index_async("id", config=BTree()) assert job.id is None - job.wait() + assert job.wait() is None assert len(table.list_indices()) == 1 job.cancel() @@ -3947,10 +3947,21 @@ def test_refresh_column_async_returns_job(tmp_path): job = table.refresh_column_async("doubled") assert job.id is None # in-process jobs have no server id - assert job.wait() is None + result = job.wait() + assert isinstance(result, lancedb.RefreshColumnResult) + assert result.rows_assigned == 2 + assert result.rows_failed == 0 + assert result.rows_remaining == 0 + assert result.source_version == 2 + assert result.published_version == 3 assert job.status() == "finished" assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4] + no_op = table.refresh_column_async("doubled").wait() + assert no_op.rows_assigned == 0 + assert no_op.source_version == 3 + assert no_op.published_version is None + # Bad input raises at the call, not through the job. with pytest.raises(Exception, match="not a computed column"): table.refresh_column_async("x") @@ -3963,6 +3974,10 @@ async def test_refresh_column_async_job_async_table(tmp_path): await table.add_columns(computed={"tripled": "x * 3"}) job = await table.refresh_column_async("tripled") - assert await job.wait() is None + result = await job.wait() + assert isinstance(result, lancedb.RefreshColumnResult) + assert result.rows_assigned == 1 + assert result.source_version == 2 + assert result.published_version == 3 assert await job.status() == "finished" assert (await table.to_arrow())["tripled"].to_pylist() == [9] diff --git a/python/src/connection.rs b/python/src/connection.rs index 4143ac9c9..902489f4f 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -609,7 +609,7 @@ impl Connection { .create_function_async(request) .await .infer_error() - .map(crate::job::FunctionJob::new) + .map(crate::job::Job::new_typed) }) } diff --git a/python/src/job.rs b/python/src/job.rs index a08b958a6..688cba7f9 100644 --- a/python/src/job.rs +++ b/python/src/job.rs @@ -5,72 +5,33 @@ use std::sync::Arc; use crate::runtime::future_into_py; use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods}; +use serde::Serialize; use crate::error::PythonErrorExt; #[pyclass] pub struct Job { - inner: Arc, -} - -/// Python bridge for a typed remote Function registration job. -/// -/// The public Python layer decodes the canonical JSON returned by `wait` -/// into its immutable `FunctionVersion` model. -#[pyclass] -pub struct FunctionJob { - inner: Arc>, -} - -impl FunctionJob { - pub(crate) fn new(inner: lancedb::Job) -> Self { - Self { - inner: Arc::new(inner), - } - } + inner: Arc, String>>>, } impl Job { pub(crate) fn new(inner: lancedb::Job) -> Self { Self { - inner: Arc::new(inner), + inner: Arc::new(inner.map(|()| Ok(None))), } } -} -#[pymethods] -impl FunctionJob { - #[getter] - pub fn id(&self) -> Option { - self.inner.id().map(str::to_string) - } - - pub fn status(self_: PyRef<'_, Self>) -> PyResult> { - let inner = self_.inner.clone(); - future_into_py( - self_.py(), - async move { inner.status().await.infer_error() }, - ) - } - - pub fn wait(self_: PyRef<'_, Self>) -> PyResult> { - let inner = self_.inner.clone(); - future_into_py(self_.py(), async move { - inner - .wait() - .await - .infer_error()? - .to_canonical_json() - .infer_error() - }) - } - - pub fn cancel(self_: PyRef<'_, Self>) -> PyResult> { - let inner = self_.inner.clone(); - future_into_py(self_.py(), async move { - inner.cancel().await.infer_error()?; - Ok(()) - }) + pub(crate) fn new_typed(inner: lancedb::Job) -> Self + where + T: Clone + Serialize + Send + Sync + 'static, + { + Self { + inner: Arc::new(inner.map(|result| { + serde_json::to_string(&result) + .map(Some) + .map_err(|error| format!("failed to serialize typed job result: {error}")) + })), + } } } @@ -92,8 +53,10 @@ impl Job { pub fn wait(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner.clone(); future_into_py(self_.py(), async move { - inner.wait().await.infer_error()?; - Ok(None::<()>) + let result = inner.wait().await.infer_error()?; + result + .map_err(|message| lancedb::Error::Runtime { message }) + .infer_error() }) } diff --git a/python/src/lib.rs b/python/src/lib.rs index c1dfbc02b..8d3eab787 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -47,7 +47,6 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::
()?; m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/src/table.rs b/python/src/table.rs index 5cdcc3653..b225b191f 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -1619,7 +1619,7 @@ impl Table { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { let job = inner.refresh_column_async(column).await.infer_error()?; - Ok(crate::job::Job::new(job)) + Ok(crate::job::Job::new_typed(job)) }) } diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index 835fca9a2..f02158871 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Canonical values exchanged with the Enterprise Function service. +//! Canonical Function values exchanged with the Enterprise service, plus the +//! backend-neutral terminal result of a computed-column refresh. //! //! This module contains client/wire values only. Catalog persistence, //! environment bake, secret resolution, and execution are owned by Sophon. @@ -580,13 +581,22 @@ impl FunctionBinding { impl_json!(FunctionBinding); -/// Stable terminal result of a remote Function-column refresh Job. +/// Stable terminal result of an expression-backed or Function-backed column +/// refresh [`crate::Job`]. +/// +/// Local refresh jobs produce this value in process. LanceDB Cloud and +/// Enterprise decode the same value from the durable job's terminal payload. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct RefreshColumnResult { + /// Rows assigned a value by this refresh. pub rows_assigned: u64, + /// Rows whose computation failed. pub rows_failed: u64, + /// Rows that still need a value when the job completes. pub rows_remaining: u64, + /// Exact table version the refresh read. pub source_version: u64, + /// Table version made visible by the refresh, when one was published. #[serde(default, skip_serializing_if = "Option::is_none")] pub published_version: Option, } diff --git a/rust/lancedb/src/job.rs b/rust/lancedb/src/job.rs index 1a76c7683..94d1ba2b6 100644 --- a/rust/lancedb/src/job.rs +++ b/rust/lancedb/src/job.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use async_trait::async_trait; -use serde::de::DeserializeOwned; +use serde::{Serialize, de::DeserializeOwned}; use serde_json::Value; use tokio::sync::watch; use tokio::task::{AbortHandle, JoinHandle}; @@ -26,20 +26,16 @@ pub(crate) trait JobHandle: Send + Sync { } /// A backend-neutral successful terminal result. -/// -/// Local operations do not carry a value. Remote operations may carry JSON -/// that the public [`Job`] decodes according to its result type. +#[derive(Clone)] pub(crate) struct TerminalResult { - #[allow(dead_code)] // Typed remote submit endpoints consume this after Slice 1. value: Option, - #[allow(dead_code)] // Preserved so typed decode errors retain request correlation. request_id: Option, } impl TerminalResult { - pub(crate) fn local() -> Self { + fn local(value: Value) -> Self { Self { - value: None, + value: Some(value), request_id: None, } } @@ -51,23 +47,31 @@ impl TerminalResult { } } - #[allow(dead_code)] // Exercised by the remote typed-result fixtures in Slice 1. fn decode(self) -> Result { - let request_id = self.request_id.unwrap_or_default(); - let value = self.value.ok_or_else(|| Error::Http { - source: "successful typed job response did not contain a result".into(), - request_id: request_id.clone(), - status_code: None, + let value = self.value.ok_or_else(|| match &self.request_id { + Some(request_id) => Error::Http { + source: "successful typed job response did not contain a result".into(), + request_id: request_id.clone(), + status_code: None, + }, + None => Error::Runtime { + message: "successful typed job did not contain a result".to_string(), + }, })?; - serde_json::from_value(value).map_err(|error| Error::Http { - source: format!("failed to parse typed job result: {error}").into(), - request_id, - status_code: None, + serde_json::from_value(value).map_err(|error| match self.request_id { + Some(request_id) => Error::Http { + source: format!("failed to parse typed job result: {error}").into(), + request_id, + status_code: None, + }, + None => Error::Runtime { + message: format!("failed to parse typed job result: {error}"), + }, }) } } -type ResultDecoder = fn(TerminalResult) -> Result; +type ResultDecoder = Arc Result + Send + Sync>; enum JobInner { Handle { @@ -79,7 +83,9 @@ enum JobInner { /// A handle to an operation that may still be running. /// -/// The operation may already be complete when the handle is created. +/// The operation may already be complete when the handle is created. `T` is +/// the endpoint's successful terminal result; unit-result operations use the +/// default `Job<()>`. pub struct Job where T: Clone + Send + Sync + 'static, @@ -111,15 +117,10 @@ impl Job<()> { Self { inner: JobInner::Handle { handle, - decode: |_| Ok(()), + decode: Arc::new(|_| Ok(())), }, } } - - /// A unit-result job running as a task in this process. - pub(crate) fn spawned(task: JoinHandle>) -> Self { - Self::new(Box::new(SpawnedJob::new(task))) - } } impl Job @@ -131,12 +132,22 @@ where Self { inner: JobInner::Handle { handle, - decode: TerminalResult::decode::, + decode: Arc::new(TerminalResult::decode::), }, } } } +impl Job +where + T: Clone + Serialize + DeserializeOwned + Send + Sync + 'static, +{ + /// A typed job running as a task in this process. + pub(crate) fn spawned(task: JoinHandle>) -> Self { + Self::new_typed(Box::new(SpawnedJob::new(task))) + } +} + impl Job where T: Clone + Send + Sync + 'static, @@ -169,11 +180,13 @@ where /// Waits until the operation reaches a terminal state. /// + /// Returns the endpoint's typed result. Unit-result jobs return `()`. + /// /// Returns [`crate::Error::JobFailed`] if the operation failed and /// [`crate::Error::JobCancelled`] if it was cancelled. pub async fn wait(&self) -> Result { match &self.inner { - JobInner::Handle { handle, decode } => decode(handle.wait().await?), + JobInner::Handle { handle, decode } => (decode)(handle.wait().await?), JobInner::Completed(result) => Ok(result.clone()), } } @@ -187,21 +200,53 @@ where JobInner::Completed(_) => Ok(()), } } + + /// Maps a successful terminal result without changing the job lifecycle. + /// The mapping may run once for each call to [`Job::wait`], so it should + /// be deterministic and free of externally visible side effects. + /// + /// ``` + /// use lancedb::{Job, function::RefreshColumnResult}; + /// + /// # async fn rows_assigned( + /// # job: Job, + /// # ) -> lancedb::Result { + /// let job = job.map(|result| result.rows_assigned); + /// job.wait().await + /// # } + /// ``` + pub fn map(self, map: F) -> Job + where + U: Clone + Send + Sync + 'static, + F: Fn(T) -> U + Send + Sync + 'static, + { + match self.inner { + JobInner::Handle { handle, decode } => Job { + inner: JobInner::Handle { + handle, + decode: Arc::new(move |result| Ok(map((decode)(result)?))), + }, + }, + JobInner::Completed(result) => Job { + inner: JobInner::Completed(map(result)), + }, + } + } } /// How an in-process operation ended. Cloneable so every waiter can be given /// the outcome; [`Error`] is not, so failures share one behind an [`Arc`]. #[derive(Clone)] enum Outcome { - Succeeded, + Succeeded(TerminalResult), Failed(Arc), Cancelled, } impl Outcome { - fn into_result(self) -> Result<()> { + fn into_result(self) -> Result { match self { - Self::Succeeded => Ok(()), + Self::Succeeded(result) => Ok(result), Self::Failed(source) => Err(Error::JobFailed { job_id: None, failure: JobFailure::from_source(source), @@ -220,12 +265,20 @@ struct SpawnedJob { } impl SpawnedJob { - fn new(task: JoinHandle>) -> Self { + fn new(task: JoinHandle>) -> Self + where + T: Serialize + Send + 'static, + { let abort = task.abort_handle(); let (tx, outcome) = watch::channel(None); tokio::spawn(async move { let outcome = match task.await { - Ok(Ok(())) => Outcome::Succeeded, + Ok(Ok(result)) => match serde_json::to_value(result) { + Ok(value) => Outcome::Succeeded(TerminalResult::local(value)), + Err(err) => Outcome::Failed(Arc::new(Error::Runtime { + message: format!("failed to serialize job result: {err}"), + })), + }, Ok(Err(err)) => Outcome::Failed(Arc::new(err)), Err(err) if err.is_cancelled() => Outcome::Cancelled, Err(err) => Outcome::Failed(Arc::new(Error::Runtime { @@ -243,7 +296,7 @@ impl JobHandle for SpawnedJob { async fn status(&self) -> Result { let label = match &*self.outcome.borrow() { None => "running", - Some(Outcome::Succeeded) => "finished", + Some(Outcome::Succeeded(_)) => "finished", Some(Outcome::Failed(_)) => "failed", Some(Outcome::Cancelled) => "cancelled", }; @@ -256,12 +309,11 @@ impl JobHandle for SpawnedJob { .wait_for(|outcome| outcome.is_some()) .await .map_err(|_| Error::Runtime { - message: "index job outcome was dropped before it completed".to_string(), + message: "job outcome was dropped before it completed".to_string(), })? .clone() .expect("wait_for returns once an outcome is set"); - settled.into_result()?; - Ok(TerminalResult::local()) + settled.into_result() } async fn cancel(&self) -> Result<()> { @@ -269,3 +321,29 @@ impl JobHandle for SpawnedJob { Ok(()) } } + +#[cfg(test)] +mod tests { + use std::future::pending; + + use super::*; + + #[tokio::test] + async fn mapped_spawned_job_reuses_outcome() { + let job = Job::spawned(tokio::spawn(async { Ok(41_u64) })).map(|value| value + 1); + + assert_eq!(job.wait().await.unwrap(), 42); + assert_eq!(job.wait().await.unwrap(), 42); + assert_eq!(job.status().await.unwrap(), "finished"); + } + + #[tokio::test] + async fn mapped_spawned_job_preserves_cancellation() { + let job = Job::spawned(tokio::spawn(async { pending::>().await })) + .map(|value| value.to_string()); + + job.cancel().await.unwrap(); + assert!(matches!(job.wait().await, Err(Error::JobCancelled { .. }))); + assert_eq!(job.status().await.unwrap(), "cancelled"); + } +} diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 169d0fda5..8265d22c0 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -513,7 +513,7 @@ impl Database for RemoteDatabase { async fn get_function(&self, name: &str, version: &str) -> Result { let req = self .client - .post("/v1/functions/get") + .post("/v1/functions/describe") .json(&serde_json::json!({ "name": name, "version": version, @@ -2520,7 +2520,7 @@ mod tests { ); let conn = Connection::new_with_handler(|request| { assert_eq!(request.method(), &reqwest::Method::POST); - assert_eq!(request.url().path(), "/v1/functions/get"); + assert_eq!(request.url().path(), "/v1/functions/describe"); let body: serde_json::Value = serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); assert_eq!( diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index fb7278db9..c6b078282 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2864,7 +2864,10 @@ impl BaseTable for RemoteTable { }) } - async fn refresh_column_async(&self, column: &str) -> Result { + async fn refresh_column_async( + &self, + column: &str, + ) -> Result> { self.check_mutable().await?; let mut body = serde_json::json!({ "column": column }); self.apply_branch_body(&mut body); @@ -2885,7 +2888,7 @@ impl BaseTable for RemoteTable { status_code: None, })?; - Ok(Job::new(Box::new(FreshnessJob { + Ok(Job::new_typed(Box::new(FreshnessJob { inner: RemoteJob::new(self.client.clone(), response.job_id), freshness: self.freshness.clone(), version: self.version.clone(), @@ -3280,6 +3283,21 @@ mod tests { }, }; + fn refresh_done(job_id: &str) -> String { + json!({ + "job_id": job_id, + "job_state": "DONE", + "result": { + "rows_assigned": 12, + "rows_failed": 0, + "rows_remaining": 0, + "source_version": 7, + "published_version": 8, + } + }) + .to_string() + } + #[tokio::test] async fn test_not_found() { let table = Table::new_with_handler("my_table", |_| { @@ -6892,7 +6910,7 @@ mod tests { .unwrap(), "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-7", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-7")) .unwrap(), "/v1/table/my_table/count_rows/" => { saw.store( @@ -6908,7 +6926,9 @@ mod tests { }); let job = table.refresh_column_async("doubled").await.unwrap(); - job.wait().await.unwrap(); + let result = job.wait().await.unwrap(); + assert_eq!(result.rows_assigned, 12); + assert_eq!(result.published_version, Some(8)); table.count_rows(None).await.unwrap(); assert!( saw_min_timestamp.load(std::sync::atomic::Ordering::SeqCst), @@ -6930,7 +6950,7 @@ mod tests { .unwrap(), "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-8", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-8")) .unwrap(), "/v1/table/my_table/describe/" => { let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]); @@ -6976,7 +6996,7 @@ mod tests { .unwrap(), "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-9", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-9")) .unwrap(), "/v1/table/my_table/tags/version/" => http::Response::builder() .status(200) @@ -7040,7 +7060,7 @@ mod tests { } "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-10", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-10")) .unwrap(), "/v1/table/my_table/describe/" => { let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]); @@ -7113,7 +7133,7 @@ mod tests { } "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-11", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-11")) .unwrap(), "/v1/table/my_table/count_rows/" => { *saw.lock().unwrap() = request diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index c2c12fff5..41551bd37 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -771,7 +771,10 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { } /// Fill a computed column's unfilled rows, returning a [`Job`] tracking /// the operation. - async fn refresh_column_async(&self, _column: &str) -> Result { + async fn refresh_column_async( + &self, + _column: &str, + ) -> Result> { Err(Error::NotSupported { message: "computed columns are supported only on local tables".into(), }) @@ -1708,7 +1711,9 @@ impl Table { /// operation instead of blocking until it completes. /// /// The job may already be complete when returned, and callers must not - /// assume the column is filled until [`Job::wait`] returns. Invalid input + /// assume the column is filled until [`Job::wait`] returns. A successful + /// wait returns the durable [`crate::function::RefreshColumnResult`] for + /// both expression-backed and Function-backed columns. Invalid input /// -- an unknown column, or one that is not computed -- is reported by /// this call rather than by the job. On local tables the job runs as an /// in-process task; on LanceDB Cloud and Enterprise it is the server's @@ -1719,11 +1724,15 @@ impl Table { /// # async fn refresh_in_background(table: &Table) -> Result<(), Box> { /// let job = table.refresh_column_async("doubled").await?; /// println!("refresh running: {:?}", job.status().await?); - /// job.wait().await?; + /// let result = job.wait().await?; + /// println!("assigned {} rows", result.rows_assigned); /// # Ok(()) /// # } /// ``` - pub async fn refresh_column_async(&self, column: impl AsRef) -> Result { + pub async fn refresh_column_async( + &self, + column: impl AsRef, + ) -> Result> { self.inner.refresh_column_async(column.as_ref()).await } @@ -3425,7 +3434,10 @@ impl BaseTable for NativeTable { Ok(result) } - async fn refresh_column_async(&self, column: &str) -> Result { + async fn refresh_column_async( + &self, + column: &str, + ) -> Result> { refresh::execute_refresh_column_async(self, column).await } diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 7d74bbae7..35f883411 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -49,11 +49,25 @@ pub struct RefreshColumnResult { pub version: u64, } +struct RefreshExecution { + result: RefreshColumnResult, + source_version: u64, +} + /// Internal implementation of the refresh logic. pub(crate) async fn execute_refresh_column( table: &NativeTable, column: &str, ) -> Result { + Ok(execute_refresh_column_with_source(table, column) + .await? + .result) +} + +async fn execute_refresh_column_with_source( + table: &NativeTable, + column: &str, +) -> Result { table.dataset.ensure_mutable()?; ensure_no_lsm_write_spec(table).await?; let dataset = table.dataset.get().await?; @@ -87,9 +101,13 @@ pub(crate) async fn execute_refresh_column( } if replacements.is_empty() { - return Ok(RefreshColumnResult { - rows_filled: 0, - version: dataset.version().version, + let source_version = dataset.version().version; + return Ok(RefreshExecution { + result: RefreshColumnResult { + rows_filled: 0, + version: source_version, + }, + source_version, }); } @@ -110,14 +128,20 @@ pub(crate) async fn execute_refresh_column( let version = new_dataset.version().version; table.dataset.update(new_dataset); - Ok(RefreshColumnResult { - rows_filled, - version, + Ok(RefreshExecution { + result: RefreshColumnResult { + rows_filled, + version, + }, + source_version: read_version, }) } /// Run the refresh as a [`Job`] in this process. -pub(crate) async fn execute_refresh_column_async(table: &NativeTable, column: &str) -> Result { +pub(crate) async fn execute_refresh_column_async( + table: &NativeTable, + column: &str, +) -> Result> { // Validate before spawning so bad input is reported by this call rather // than only by the job. table.dataset.ensure_mutable()?; @@ -129,9 +153,16 @@ pub(crate) async fn execute_refresh_column_async(table: &NativeTable, column: &s let table = table.clone(); let column = column.to_string(); Ok(Job::spawned(tokio::spawn(async move { - execute_refresh_column(&table, &column).await?; + let execution = execute_refresh_column_with_source(&table, &column).await?; table.bump_freshness(); - Ok(()) + Ok(crate::function::RefreshColumnResult { + rows_assigned: execution.result.rows_filled, + rows_failed: 0, + rows_remaining: 0, + source_version: execution.source_version, + published_version: (execution.result.rows_filled > 0) + .then_some(execution.result.version), + }) }))) } @@ -396,6 +427,17 @@ mod tests { read(&table, "doubled").await, vec![Some(2), Some(4), Some(6)] ); + + let no_op = table + .refresh_column_async("doubled") + .await + .unwrap() + .wait() + .await + .unwrap(); + assert_eq!(no_op.rows_assigned, 0); + assert_eq!(no_op.source_version, 3); + assert_eq!(no_op.published_version, None); } /// Values written after the last refresh must be reachable by another one. @@ -646,7 +688,12 @@ mod tests { let job = table.refresh_column_async("doubled").await.unwrap(); assert!(job.id().is_none(), "in-process jobs have no server id"); - job.wait().await.unwrap(); + let result = job.wait().await.unwrap(); + assert_eq!(result.rows_assigned, 3); + assert_eq!(result.rows_failed, 0); + assert_eq!(result.rows_remaining, 0); + assert_eq!(result.source_version, 2); + assert_eq!(result.published_version, Some(3)); assert_eq!(job.status().await.unwrap(), "finished"); assert_eq!( read(&table, "doubled").await, @@ -672,9 +719,9 @@ mod tests { declare_doubled(&table).await.unwrap(); let job = table.refresh_column_async("doubled").await.unwrap(); - job.wait().await.unwrap(); + let first = job.wait().await.unwrap(); // A second wait after completion observes the same outcome. - job.wait().await.unwrap(); + assert_eq!(job.wait().await.unwrap(), first); assert_eq!(job.status().await.unwrap(), "finished"); } From 94d484f539048ada0b92deb289251d6e5ed7211e Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 24 Aug 2026 12:00:48 -0700 Subject: [PATCH 21/31] fix(listing): don't drop a table at a page boundary (#4040) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Listing tables a page at a time against a local database silently skipped one table at every page boundary. `ListingDatabase::list_tables` returned the first name of the *next* page as that page's token, but resuming from a token drops every name at or before it — so the table the token named was never handed to the caller. Walking `[a, b, c, d, e]` with a limit of 2 returned `[a, b, d, e]`. This PR returns the last name of the page as the token instead, which is what resuming after the token expects. This is reachable from Python today through `db.list_tables(page_token=...)` on a local connection; it also affects `len(db)` and `name in db`, which walk the pages. Remote and namespace-backed connections page on the server and were never affected. ## Example ```python db = lancedb.connect(tmp_path) for name in ["a", "b", "c", "d", "e"]: db.create_table(name, [{"id": 1}]) names, token = [], None while True: page = db.list_tables(page_token=token, limit=2) names += page.tables token = page.page_token if not token: break # before: ['a', 'b', 'd', 'e'] # after: ['a', 'b', 'c', 'd', 'e'] ``` Co-authored-by: Claude Opus 5 (1M context) --- rust/lancedb/src/connection.rs | 44 ++++++++++++++++++++++++++++ rust/lancedb/src/database/listing.rs | 16 +++++----- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 187e2df0e..5f66d9dee 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -1679,6 +1679,50 @@ mod tests { assert_eq!(tables, names[..7]); } + #[tokio::test] + async fn test_list_tables_walks_page_boundaries() { + let tc = new_test_connection().await.unwrap(); + if tc.is_remote { + // What resumes a page is the server's to decide, and asserting it here would be + // asserting the server's contract rather than this one. + return; + } + let db = tc.connection; + let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)])); + let mut names = Vec::with_capacity(5); + for _ in 0..5 { + let name = uuid::Uuid::new_v4().to_string(); + names.push(name.clone()); + db.create_empty_table(name, schema.clone()) + .execute() + .await + .unwrap(); + } + names.sort(); + + // Walking in pages has to reach every table exactly once, with nothing lost at a + // page boundary. + let mut seen = Vec::with_capacity(names.len()); + let mut page_token = None; + loop { + let page = db + .list_tables(ListTablesRequest { + id: Some(Vec::new()), + limit: Some(2), + page_token, + ..Default::default() + }) + .await + .unwrap(); + seen.extend(page.tables); + page_token = page.page_token.filter(|token| !token.is_empty()); + if page_token.is_none() { + break; + } + } + assert_eq!(seen, names); + } + #[tokio::test] async fn test_open_table() { let tc = new_test_connection().await.unwrap(); diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index c9e9bcb22..17dc82756 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -974,17 +974,15 @@ impl Database for ListingDatabase { f.drain(0..index); } - // Determine if there's a next page - let next_page_token = if let Some(limit) = request.limit { - if f.len() > limit as usize { - let token = f[limit as usize].clone(); + // Determine if there's a next page. The token is the last name of this page, + // not the first of the next one: the next page resumes strictly after the + // token, so naming the next page's first entry would skip it. + let next_page_token = match request.limit { + Some(limit) if f.len() > limit as usize => { f.truncate(limit as usize); - Some(token) - } else { - None + f.last().cloned() } - } else { - None + _ => None, }; Ok(ListTablesResponse { From 105fd73bc6da822e40f2c7206c8612bd009023bd Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 04:22:22 +0800 Subject: [PATCH 22/31] fix(python): commit streaming worker checkpoints on consumption (#4023) ## Summary - add `StreamingDataLoader`, which transports worker snapshots with prefetched batches and commits them to the parent dataset only when the trainer receives each batch - preserve exact non-uniform per-split progress and resume lagging splits without replaying already-consumed rows - reject stale parent checkpoints after a standard multi-process `DataLoader` has started, with guidance to use the consumer-aware loader - document the new public loader and merge non-uniform state across ranks ## Root cause PyTorch runs `StreamingDataset.__iter__` in private worker-process copies, while callers invoke `state_dict()` on the parent dataset. Sharing producer counters would still be incorrect because DataLoader prefetch can advance workers beyond batches returned to the trainer. ## Validation - `uv run --extra tests pytest python/tests/test_elastic_dataloader.py -q` (154 passed) - focused non-uniform merge regression (1 passed) - `uv run --project python --extra tests --extra dev ruff format .` - `uv run --project python --extra tests --extra dev ruff check .` - `cd docs && PYTHONPATH=. ../python/.venv/bin/mkdocs build` Fixes #3967 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo --- docs/src/python/python.md | 2 + python/python/lancedb/streaming.py | 594 +++++++++++++++- .../python/tests/test_elastic_dataloader.py | 658 ++++++++++++++++++ 3 files changed, 1221 insertions(+), 33 deletions(-) diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 8a24ea199..3cbeee6f0 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -261,6 +261,8 @@ instead of being materialized with the rest of the row. ::: lancedb.streaming.StreamingDataset +::: lancedb.streaming.StreamingDataLoader + ::: lancedb.permutation.permutation_builder ::: lancedb.permutation.PermutationBuilder diff --git a/python/python/lancedb/streaming.py b/python/python/lancedb/streaming.py index c54940ab7..2c0e2d5c1 100644 --- a/python/python/lancedb/streaming.py +++ b/python/python/lancedb/streaming.py @@ -19,6 +19,7 @@ above. """ import ctypes +import heapq import logging import os import random @@ -29,12 +30,12 @@ from collections import deque from concurrent.futures import ThreadPoolExecutor from copy import deepcopy from multiprocessing import RawArray -from typing import Any, Callable, cast, Iterator, Literal, Optional, Union +from typing import Any, Callable, cast, Iterator, Literal, NamedTuple, Optional, Union import pyarrow as pa import pyarrow.compute as pc import torch -from torch.utils.data import IterableDataset, get_worker_info +from torch.utils.data import DataLoader, IterableDataset, get_worker_info from .permutation import ( Permutation, @@ -55,6 +56,155 @@ DEFAULT_READ_BATCH_SIZE = 64 DEFAULT_PREFETCH_BATCHES = 4 +class _WorkerSample(NamedTuple): + data: Any + dataset: "StreamingDataset" + + +class _WorkerBatch(NamedTuple): + data: Any + state: dict + + +class _ConsumerIteratorLease(NamedTuple): + owner_token: int + owner_thread: int + + +class _CheckpointCollate: + """Attach the worker's post-fetch state to a collated batch.""" + + def __init__(self, collate_fn: Callable): + self._collate_fn = collate_fn + + def __call__(self, samples): + try: + if isinstance(samples, list): + if not samples: + return _WorkerBatch(self._collate_fn(samples), {}) + worker_samples = samples + data = self._collate_fn([sample.data for sample in worker_samples]) + dataset = worker_samples[-1].dataset + else: + data = self._collate_fn(samples.data) + dataset = samples.dataset + except StopIteration as exc: + raise RuntimeError( + "collate_fn raised StopIteration before returning a batch" + ) from exc + return _WorkerBatch(data, dataset._checkpoint_snapshot()) + + +class _StreamingDatasetAdapter(IterableDataset): + """Yield private sample wrappers for :class:`StreamingDataLoader`.""" + + def __init__(self, dataset: "StreamingDataset"): + super().__init__() + self.dataset = dataset + + def __iter__(self): + for sample in self.dataset._iter(consumer_checkpoint_transport=True): + yield _WorkerSample(sample, self.dataset) + + def __getattr__(self, name): + dataset = self.__dict__.get("dataset") + if dataset is None: + raise AttributeError(name) + return getattr(dataset, name) + + +class _ConsumerCommitIterator: + def __init__( + self, + iterator, + dataset: "StreamingDataset", + *, + owner_token: int, + require_uniform: bool, + ): + self._iterator = iterator + self._dataset = dataset + self._owner_token = owner_token + self._require_uniform = require_uniform + self._released = False + self._terminal = False + + def __iter__(self): + return self + + def __next__(self): + if self._terminal: + raise StopIteration + try: + batch = next(self._iterator) + except StopIteration: + self._terminal = True + self._release() + raise + except BaseException as exc: + self._dataset._invalidate_checkpoint( + f"a DataLoader batch failed before it was returned: {exc}" + ) + raise + try: + if not isinstance(batch, _WorkerBatch): + raise RuntimeError( + "StreamingDataLoader did not receive worker checkpoint metadata" + ) + self._dataset._commit_worker_state( + batch.state, require_uniform=self._require_uniform + ) + return batch.data + except BaseException as exc: + self._dataset._invalidate_checkpoint( + f"a DataLoader batch failed before it was returned: {exc}" + ) + raise + + def _release(self) -> None: + if self.__dict__.get("_released", True): + return + self._released = True + dataset = self.__dict__.get("_dataset") + if dataset is not None: + dataset._release_consumer_iterator(self._owner_token) + + def _shutdown_workers(self): + if self.__dict__.get("_released", True): + return None + self._terminal = True + iterator = self.__dict__.get("_iterator") + shutdown = getattr(iterator, "_shutdown_workers", None) + try: + if shutdown is not None: + shutdown() + else: + fetcher = getattr(iterator, "_dataset_fetcher", None) + dataset_iterator = getattr(fetcher, "dataset_iter", None) + close = getattr(dataset_iterator, "close", None) + if close is None: + raise RuntimeError( + "StreamingDataLoader could not close its inner iterator" + ) + close() + except BaseException as exc: + self._dataset._invalidate_checkpoint( + f"a DataLoader iterator could not be shut down safely: {exc}" + ) + raise + else: + self._release() + + def __del__(self): + try: + self._shutdown_workers() + except BaseException: + pass + + def __getattr__(self, name): + return getattr(self._iterator, name) + + class StreamingDataset(IterableDataset): """An elastic, resumable PyTorch IterableDataset backed by a LanceDB table. @@ -384,6 +534,22 @@ class StreamingDataset(IterableDataset): # rows_skipped] self._worker_stats: RawArray = RawArray(ctypes.c_int64, 8) + # A standard multi-process DataLoader cannot report which prefetched + # batches were actually returned to its consumer. Workers set this + # shared flag so state_dict() can reject a stale parent checkpoint + # unless StreamingDataLoader installed the consumer-commit transport. + self._untracked_worker_iteration: RawArray = RawArray(ctypes.c_int64, 1) + + # Parent-side checkpoint lifecycle. A failed DataLoader task creates + # a permanent hole in that iterator's delivery stream, while a + # multi-worker checkpoint is safe to restore only after all splits + # reach the same logical step boundary. + self._checkpoint_invalid_reason: Optional[str] = None + self._consumer_checkpoint_requires_uniform = False + self._consumer_iterator_lock = threading.Lock() + self._consumer_iterator_generation = 0 + self._consumer_iterator_lease: Optional[_ConsumerIteratorLease] = None + # Cumulative bytes of Arrow buffer data fetched across all iterations. self._bytes_loaded: int = 0 # Cumulative seconds spent in LanceDB I/O and in transform functions. @@ -396,6 +562,10 @@ class StreamingDataset(IterableDataset): # step boundaries all splits have consumed this many samples, so a # single scalar captures the topology-independent checkpoint state. self._resume_offset: int = 0 + # Exact yielded-sample counts for splits this process has advanced. + # Missing entries use _resume_offset, which remains the lower-bound + # checkpoint inherited from an earlier uniform/global state. + self._resume_samples: dict[int, int] = {} # Permutation position each split has consumed through, keyed by # global split index. Equal to _resume_offset for every split unless # on_transform_error skipped rows, in which case skipped positions @@ -521,11 +691,45 @@ class StreamingDataset(IterableDataset): return self._rank_splits[start : start + splits_per_worker] def __iter__(self) -> Iterator[dict[str, Any]]: + return self._iter() + + def _iter( + self, *, consumer_checkpoint_transport: bool = False + ) -> Iterator[dict[str, Any]]: + owner_token = None + previous_lease = self._consumer_iterator_lease + if consumer_checkpoint_transport: + if not self._consumer_iterator_active: + raise RuntimeError( + "StreamingDataLoader worker transport requires an active " + "parent iterator reservation" + ) + else: + try: + owner_token = self._acquire_consumer_iterator() + except BaseException: + self._release_consumer_iterator_after_failed_acquire(previous_lease) + raise + try: + yield from self._iter_owned( + consumer_checkpoint_transport=consumer_checkpoint_transport + ) + finally: + if owner_token is not None: + self._release_consumer_iterator(owner_token) + + def _iter_owned( + self, *, consumer_checkpoint_transport: bool + ) -> Iterator[dict[str, Any]]: if self._raw_batches_ref is not None: raise RuntimeError( "StreamingDataset does not support concurrent iteration. " "Only one active iterator per dataset instance is allowed." ) + real_worker = get_worker_info() is not None + if real_worker and not consumer_checkpoint_transport: + self._untracked_worker_iteration[0] = 1 + my_splits = self._resolve_my_splits() if not my_splits: return @@ -533,6 +737,7 @@ class StreamingDataset(IterableDataset): # Set identity transform on each Permutation so __getitems__ returns # the raw RecordBatch. Stage 2 applies the real transform. permutations: list[Permutation] = [] + initial_samples: list[int] = [] initial_positions: list[int] = [] for split_idx in my_splits: perm = Permutation.from_tables( @@ -541,21 +746,22 @@ class StreamingDataset(IterableDataset): if self._columns is not None: perm = perm.select_columns(self._columns) perm = perm.with_transform(Transforms.arrow2arrow) + sample_count = self._resume_samples.get(split_idx, self._resume_offset) # Both modes resume from absolute permutation positions. Packing # stores them separately because it also checkpoints partial blocks. start_pos = ( self._pack_consumed[split_idx] if self._pack_sequences is not None - else self._resume_positions.get(split_idx, self._resume_offset) + else self._resume_positions.get(split_idx, sample_count) ) if start_pos > 0: perm = perm.with_skip(start_pos) + initial_samples.append(sample_count) initial_positions.append(start_pos) permutations.append(perm) n = len(permutations) split_sizes = [perm.num_rows for perm in permutations] - initial_offset = self._resume_offset local_consumed = [0] * n # Permutation position each split has consumed through (absolute, # i.e. counted from the start of the unskipped split). Runs ahead of @@ -853,6 +1059,27 @@ class StreamingDataset(IterableDataset): for i in range(n): _fill_io(i) + def _yield_row(i: int): + pos, row = cooked[i].popleft() + # Surface any completed prefetched failure before the + # current row becomes durable checkpoint progress. + _advance(i) + local_consumed[i] += 1 + pos_consumed[i] = pos + 1 + split_idx = my_splits[i] + self._resume_samples[split_idx] = ( + initial_samples[i] + local_consumed[i] + ) + self._resume_positions[split_idx] = pos_consumed[i] + return row + + def _update_progress_stats() -> None: + if not real_worker: + self._resume_offset = min( + initial_samples[j] + local_consumed[j] for j in range(n) + ) + _update_stats() + if self._pack_sequences is not None: first_count = pack_blocks_emitted[my_splits[0]] if any( @@ -878,12 +1105,38 @@ class StreamingDataset(IterableDataset): tokens.extend([pad_id] * (pack_len - len(tokens))) block = _emit_block(i) pack_blocks_emitted[my_splits[i]] += 1 + # Checkpoint state must advance before yielding so + # StreamingDataLoader can attach the exact state to + # the batch it transports to the parent process. + _commit_pack_state() if i == n - 1: - _commit_pack_state() _update_stats() yield block return + # A checkpoint taken between round-robin split turns has + # non-uniform counts. Resume lagging splits first so the + # exact canonical sequence continues without replaying + # already-consumed rows. + if len(set(initial_samples)) > 1: + catch_up_to = max(initial_samples) + pending = [ + (initial_samples[i], my_splits[i], i) + for i in range(n) + if initial_samples[i] < catch_up_to + ] + heapq.heapify(pending) + while pending: + consumed, _, i = heapq.heappop(pending) + _ensure_cooked(i) + if not cooked[i]: + return + row = _yield_row(i) + if consumed + 1 < catch_up_to: + heapq.heappush(pending, (consumed + 1, my_splits[i], i)) + _update_progress_stats() + yield row + while True: # A cycle only runs if every split can still produce a # row. Without skips all splits exhaust simultaneously @@ -904,20 +1157,14 @@ class StreamingDataset(IterableDataset): break for i in range(n): - pos, row = cooked[i].popleft() - local_consumed[i] += 1 - pos_consumed[i] = pos + 1 - _advance(i) + row = _yield_row(i) # After the last split in each cycle: update the # global offset and refresh the shared-memory stats # so the main process can observe pipeline depth # even when __iter__ runs in a worker process. if i == n - 1: - self._resume_offset = initial_offset + local_consumed[i] - for j, split_idx in enumerate(my_splits): - self._resume_positions[split_idx] = pos_consumed[j] - _update_stats() + _update_progress_stats() yield row finally: @@ -1064,6 +1311,7 @@ class StreamingDataset(IterableDataset): "_local_consumed_ref", ): state[key] = None + state["_consumer_iterator_lock"] = None return state def __setstate__(self, state): @@ -1074,6 +1322,7 @@ class StreamingDataset(IterableDataset): table_state = state.pop("_table") perm_name, perm_data = state.pop("_perm_table") self.__dict__.update(state) + self._consumer_iterator_lock = threading.Lock() if self._connection_factory is not None: self._table = self._connection_factory(table_name) else: @@ -1083,10 +1332,18 @@ class StreamingDataset(IterableDataset): def state_dict(self) -> dict: """Snapshot the dataset's consumption state. + When using DataLoader workers, construct a + [StreamingDataLoader][lancedb.streaming.StreamingDataLoader]. It + commits worker state only when a prefetched batch is returned to the + trainer. A standard multi-process ``DataLoader`` cannot expose that + boundary, so calling this method after one has started raises + ``RuntimeError`` instead of returning stale producer state. + In row mode, the returned dict is topology-independent at global step boundaries. ``positions_consumed_per_split`` records how far each split's permutation has advanced, which can differ from the sample - count when ``on_transform_error`` skips rows. Combine state dicts from + count when ``on_transform_error`` skips rows. ``StreamingDataLoader`` + combines worker state in its parent process. Combine state dicts from every rank with [merge_state_dicts][lancedb.streaming.StreamingDataset.merge_state_dicts] before resuming on a different topology. @@ -1095,6 +1352,43 @@ class StreamingDataset(IterableDataset): for every logical split. When packing is sharded, merge every rank state with ``merge_state_dicts`` before loading it. """ + if self._untracked_worker_iteration[0] and get_worker_info() is None: + raise RuntimeError( + "StreamingDataset cannot checkpoint a standard DataLoader with " + "num_workers > 0 because prefetched worker progress is not " + "consumer-committed. Use StreamingDataLoader instead." + ) + if self._checkpoint_invalid_reason is not None: + raise RuntimeError( + "StreamingDataset checkpointing is invalid because " + f"{self._checkpoint_invalid_reason}. Load the last valid " + "checkpoint into a fresh dataset before continuing." + ) + state = self._checkpoint_snapshot() + if self._pack_sequences is not None: + rank_blocks = [ + state["blocks_emitted_per_split"][split] for split in self._rank_splits + ] + if len(set(rank_blocks)) > 1: + raise RuntimeError( + "Packed StreamingDataset checkpointing is only safe at a " + "complete logical step boundary, when every split assigned " + "to this rank has emitted the same block count. Consume more " + "batches before calling state_dict()." + ) + elif self._consumer_checkpoint_requires_uniform: + samples = state["samples_consumed_per_split"] + rank_samples = [samples[split] for split in self._rank_splits] + if len(set(rank_samples)) > 1: + raise RuntimeError( + "StreamingDataLoader checkpointing with multiple workers is " + "only safe at a complete logical step boundary, when every " + "split assigned to this rank has the same consumed-sample " + "count. Consume more batches before calling state_dict()." + ) + return state + + def _checkpoint_snapshot(self) -> dict: if self._pack_sequences is not None: return { "shuffle_seed": self._shuffle_seed, @@ -1108,18 +1402,141 @@ class StreamingDataset(IterableDataset): "blocks_emitted_per_split": list(self._pack_blocks_emitted), "pack_buffers": deepcopy(self._pack_buffers), } + samples = [ + self._resume_samples.get(split, self._resume_offset) + for split in range(self._num_splits) + ] positions = [ - self._resume_positions.get(split, self._resume_offset) + self._resume_positions.get(split, samples[split]) for split in range(self._num_splits) ] return { "shuffle_seed": self._shuffle_seed, "num_splits": self._num_splits, "epoch": self._epoch, - "samples_consumed_per_split": [self._resume_offset] * self._num_splits, + "samples_consumed_per_split": samples, "positions_consumed_per_split": positions, } + def _invalidate_checkpoint(self, reason: str) -> None: + if self._checkpoint_invalid_reason is None: + self._checkpoint_invalid_reason = reason + + @property + def _consumer_iterator_active(self) -> bool: + return self._consumer_iterator_lease is not None + + @property + def _consumer_iterator_owner(self) -> Optional[int]: + lease = self._consumer_iterator_lease + return lease.owner_token if lease is not None else None + + @property + def _consumer_iterator_owner_thread(self) -> Optional[int]: + lease = self._consumer_iterator_lease + return lease.owner_thread if lease is not None else None + + def _acquire_consumer_iterator(self) -> int: + """Reserve this parent dataset for one checkpoint-aware iterator.""" + with self._consumer_iterator_lock: + if self._consumer_iterator_active or self._raw_batches_ref is not None: + raise RuntimeError( + "StreamingDataset does not support concurrent iteration. " + "Only one active iterator per dataset instance is allowed." + ) + owner_thread = threading.get_ident() + owner_token = self._consumer_iterator_generation + 1 + lease = _ConsumerIteratorLease(owner_token, owner_thread) + self._consumer_iterator_generation = owner_token + self._consumer_iterator_lease = lease + return owner_token + + def _release_consumer_iterator(self, owner_token: int) -> None: + with self._consumer_iterator_lock: + lease = self._consumer_iterator_lease + if lease is not None and lease.owner_token == owner_token: + self._consumer_iterator_lease = None + + def _release_consumer_iterator_after_failed_acquire( + self, previous_lease: Optional[_ConsumerIteratorLease] + ) -> None: + """Clean up when an interrupted acquire set a lease but did not return it.""" + owner_thread = threading.current_thread().ident + with self._consumer_iterator_lock: + lease = self._consumer_iterator_lease + if ( + lease is not None + and lease is not previous_lease + and lease.owner_thread == owner_thread + ): + self._consumer_iterator_lease = None + + def _commit_worker_state(self, state: dict, *, require_uniform: bool) -> None: + """Merge one trainer-consumed worker batch into parent state.""" + for key, expected in ( + ("shuffle_seed", self._shuffle_seed), + ("num_splits", self._num_splits), + ("epoch", self._epoch), + ): + if state.get(key) != expected: + raise ValueError( + f"{key} mismatch in worker checkpoint: " + f"{state.get(key)} != {expected}" + ) + packed = "pack_buffers" in state + if packed != (self._pack_sequences is not None): + raise ValueError("worker checkpoint mode does not match the dataset") + if packed: + for key in ("pack_sequences", "eos_id", "pad_id", "blocks_per_epoch"): + expected = getattr(self, f"_{key}") + if state.get(key) != expected: + raise ValueError( + f"{key} mismatch in worker checkpoint: " + f"{state.get(key)} != {expected}" + ) + samples = state["samples_consumed_per_split"] + emitted = state["blocks_emitted_per_split"] + if len(samples) != self._num_splits or len(emitted) != self._num_splits: + raise ValueError( + "packed worker checkpoint must contain one entry per split" + ) + buffers = state["pack_buffers"] + for split, (count, blocks) in enumerate(zip(samples, emitted)): + incoming = (int(blocks), int(count)) + current = ( + self._pack_blocks_emitted[split], + self._pack_consumed[split], + ) + if incoming > current: + self._pack_blocks_emitted[split] = incoming[0] + self._pack_consumed[split] = incoming[1] + buffer = buffers.get(split, buffers.get(str(split))) + if buffer is None: + self._pack_buffers.pop(split, None) + else: + self._pack_buffers[split] = { + "tokens": list(buffer["tokens"]), + "starts": list(buffer["starts"]), + } + self._consumer_checkpoint_requires_uniform |= require_uniform + return + + samples = state["samples_consumed_per_split"] + positions = state.get("positions_consumed_per_split", samples) + for split, count in enumerate(samples): + current = self._resume_samples.get(split, self._resume_offset) + self._resume_samples[split] = max(current, int(count)) + for split, position in enumerate(positions): + current = self._resume_positions.get( + split, self._resume_samples.get(split, self._resume_offset) + ) + self._resume_positions[split] = max(current, int(position)) + self._resume_offset = min( + self._resume_samples.get(split, self._resume_offset) + for split in range(self._num_splits) + ) + self._consumer_checkpoint_requires_uniform |= require_uniform + def load_state_dict(self, state: dict) -> None: """Resume from a previously snapshotted state. @@ -1139,6 +1556,7 @@ class StreamingDataset(IterableDataset): f"shuffle_seed mismatch: checkpoint has {state['shuffle_seed']}, " f"current dataset has {self._shuffle_seed}" ) + self._consumer_checkpoint_requires_uniform = False if "pack_buffers" in state or self._pack_sequences is not None: for key in ( @@ -1165,14 +1583,17 @@ class StreamingDataset(IterableDataset): return consumed = state["samples_consumed_per_split"] - # All entries are equal at step boundaries; use the first. if isinstance(consumed, list): - self._resume_offset = consumed[0] if consumed else 0 + self._resume_offset = min(consumed) if consumed else 0 + self._resume_samples = { + split: int(count) for split, count in enumerate(consumed) + } else: self._resume_offset = int(consumed) + self._resume_samples = {} # Older checkpoints predate positions_consumed_per_split; without # skipped rows positions equal sample counts, so falling back to - # _resume_offset (the .get default in __iter__) is exact. + # the per-split sample count (the .get default in __iter__) is exact. positions = state.get("positions_consumed_per_split") if positions is None: self._resume_positions = {} @@ -1185,10 +1606,11 @@ class StreamingDataset(IterableDataset): def merge_state_dicts(states: list[dict]) -> dict: """Merge state dicts saved by different ranks into one exact state. - For row mode, the elementwise maximum of permutation positions recovers - splits advanced by different ranks after transform failures. For packed - mode, the state that emitted the most blocks for each logical split - supplies that split's permutation position and partial token buffer. Packed + In row mode, each rank records exact consumer-committed progress for + its own splits and lower bounds for the rest, so elementwise maxima + recover both sample counts and permutation positions. In packed mode, + the state that emitted the most blocks for each logical split supplies + that split's permutation position and partial token buffer. Packed states must cover every rank at the same global step. Raises ``ValueError`` if the states are empty, were not produced by @@ -1299,17 +1721,13 @@ class StreamingDataset(IterableDataset): merged["pack_buffers"] = merged_buffers return merged - for state in states[1:]: - if ( - state["samples_consumed_per_split"] - != first["samples_consumed_per_split"] - ): - raise ValueError( - "samples_consumed_per_split mismatch across state dicts; " - "state_dict() must be called at the same global step " - "boundary on every rank" - ) merged = dict(first) + merged["samples_consumed_per_split"] = [ + max(per_split) + for per_split in zip( + *(state["samples_consumed_per_split"] for state in states) + ) + ] all_positions = [ state.get( "positions_consumed_per_split", state["samples_consumed_per_split"] @@ -1320,3 +1738,113 @@ class StreamingDataset(IterableDataset): max(per_split) for per_split in zip(*all_positions) ] return merged + + +class StreamingDataLoader(DataLoader): + """A PyTorch DataLoader with consumer-committed dataset checkpoints. + + PyTorch workers prefetch batches ahead of the trainer, so worker-local + producer progress is not a safe checkpoint. This loader carries a state + snapshot alongside every internal batch and applies it to the parent + [StreamingDataset][lancedb.streaming.StreamingDataset] only when that batch + is returned by ``next()``. + The trainer receives the same collated batch it would receive from a + standard ``torch.utils.data.DataLoader``. + + With more than one worker, row-mode ``state_dict()`` is available only at + complete logical step boundaries, when every split assigned to the rank has + the same consumed-sample count. Packed checkpoints require equal emitted-block + counts across the rank's splits for any worker count. ``persistent_workers=True`` + is not supported because prefetched worker copies cannot be restored from + parent-committed state. If batch collation raises, checkpointing remains + invalid for that dataset instance; restore the last valid checkpoint into a + fresh dataset before continuing. + Only one active iterator may own a dataset at a time, including when worker + processes are used. Exhausting or explicitly shutting down the iterator + releases that ownership. ``drop_last=True`` is not supported because worker + replicas discard incomplete tails independently, which cannot produce a + topology-independent checkpoint. + + Parameters are the same as ``torch.utils.data.DataLoader`` except that + ``dataset`` must be a + [StreamingDataset][lancedb.streaming.StreamingDataset]. + Subclasses that override ``StreamingDataset.__iter__`` are not supported + because the custom iterator cannot provide the exact per-yield checkpoint + snapshots required by this loader. + + Examples + -------- + >>> # dataset = StreamingDataset(table, num_splits=2) + >>> # loader = StreamingDataLoader(dataset, batch_size=8, num_workers=2) + >>> # batch = next(iter(loader)) + >>> # checkpoint = dataset.state_dict() + """ + + def __init__(self, dataset: StreamingDataset, *args, **kwargs): + if not isinstance(dataset, StreamingDataset): + raise TypeError("StreamingDataLoader requires a StreamingDataset") + if type(dataset).__iter__ is not StreamingDataset.__iter__: + raise TypeError( + "StreamingDataLoader does not support StreamingDataset subclasses " + "that override __iter__ because they cannot provide exact " + "per-yield checkpoint state" + ) + if kwargs.get("in_order", True) is False: + raise ValueError( + "StreamingDataLoader requires in_order=True for deterministic " + "consumer checkpoints" + ) + if kwargs.get("persistent_workers", False): + raise ValueError( + "StreamingDataLoader does not support persistent_workers=True " + "because worker prefetch state cannot be reset from a checkpoint" + ) + self._streaming_dataset = dataset + super().__init__(_StreamingDatasetAdapter(dataset), *args, **kwargs) + if self.drop_last: + raise ValueError( + "StreamingDataLoader does not support drop_last=True because " + "discarded worker tails cannot be checkpointed " + "topology-independently" + ) + self.collate_fn = _CheckpointCollate(self.collate_fn) + + def __iter__(self): + dataset = self._streaming_dataset + previous_lease = dataset._consumer_iterator_lease + owner_token = None + try: + owner_token = dataset._acquire_consumer_iterator() + state = dataset._checkpoint_snapshot() + packed = dataset._pack_sequences is not None + if packed: + blocks = state["blocks_emitted_per_split"] + rank_blocks = [blocks[split] for split in dataset._rank_splits] + if len(set(rank_blocks)) > 1: + raise RuntimeError( + "StreamingDataLoader cannot start from a partial packed " + "logical step; resume from a checkpoint whose splits " + "assigned to this rank have equal emitted-block counts" + ) + elif self.num_workers > 1: + samples = state["samples_consumed_per_split"] + rank_samples = [samples[split] for split in dataset._rank_splits] + if len(set(rank_samples)) > 1: + raise RuntimeError( + "StreamingDataLoader cannot start multiple workers from a " + "partial logical step; resume from a checkpoint whose " + "splits assigned to this rank have equal consumed-sample " + "counts" + ) + return _ConsumerCommitIterator( + super().__iter__(), + dataset, + owner_token=owner_token, + require_uniform=self.num_workers > 1 or packed, + ) + except BaseException: + if owner_token is not None: + dataset._release_consumer_iterator(owner_token) + else: + dataset._release_consumer_iterator_after_failed_acquire(previous_lease) + raise diff --git a/python/python/tests/test_elastic_dataloader.py b/python/python/tests/test_elastic_dataloader.py index ff65d100c..da27e5bfc 100644 --- a/python/python/tests/test_elastic_dataloader.py +++ b/python/python/tests/test_elastic_dataloader.py @@ -32,6 +32,7 @@ Parameters used throughout: import dataclasses import logging +import threading from unittest.mock import patch import lancedb @@ -46,6 +47,7 @@ from utils import ( torch = pytest.importorskip("torch") streaming = pytest.importorskip("lancedb.streaming") StreamingDataset = streaming.StreamingDataset +StreamingDataLoader = streaming.StreamingDataLoader # --------------------------------------------------------------------------- # Dataset parameters @@ -92,6 +94,27 @@ class FakeWorkerInfo: num_workers: int +def _collate_with_first_batch_error(samples): + ids = [sample["id"] for sample in samples] + if ids == [0, 1]: + raise ValueError("first batch fails") + return ids + + +def _collate_with_first_batch_stop(samples): + ids = [sample["id"] for sample in samples] + if ids == [0, 1]: + raise StopIteration("first batch stopped") + return ids + + +def _collate_with_first_batch_interrupt(samples): + ids = [sample["id"] for sample in samples] + if ids == [0, 1]: + raise KeyboardInterrupt("first batch interrupted") + return ids + + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -1008,6 +1031,565 @@ def test_multi_worker_elastic_det_across_worker_counts(lance_table): # ── Resumability with num_workers ───────────────────────────────────────────── +def test_streaming_dataloader_commits_only_consumed_worker_batches(tmp_path): + """Prefetched worker state is committed only as the trainer receives it.""" + db = lancedb.connect(tmp_path) + table = db.create_table( + "worker_commit", pa.table({"id": [1, 2, 3, 4, 10, 20, 30, 40]}) + ) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=2, + multiprocessing_context="spawn", + prefetch_factor=4, + ) + iterator = iter(loader) + try: + first = next(iterator)["id"].tolist() + + assert first == [1, 2] + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2, 0] + with pytest.raises(RuntimeError, match="complete logical step boundary"): + dataset.state_dict() + + second = next(iterator)["id"].tolist() + assert second == [10, 20] + checkpoint = dataset.state_dict() + assert checkpoint["samples_consumed_per_split"] == [2, 2] + uninterrupted = [batch["id"].tolist() for batch in iterator] + finally: + iterator._shutdown_workers() + + resumed = StreamingDataset(table, num_splits=2, shuffle=False) + resumed.load_state_dict(checkpoint) + resumed_loader = StreamingDataLoader( + resumed, + batch_size=2, + num_workers=2, + multiprocessing_context="spawn", + prefetch_factor=4, + ) + resumed_iterator = iter(resumed_loader) + try: + remaining = [batch["id"].tolist() for batch in resumed_iterator] + finally: + resumed_iterator._shutdown_workers() + assert remaining == uninterrupted == [[3, 4], [30, 40]] + + +def test_distributed_checkpoint_uses_rank_local_worker_boundary(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("rank_boundary", pa.table({"id": list(range(8))})) + dataset = StreamingDataset( + table, + num_splits=4, + shuffle=False, + rank=0, + world_size=2, + ) + loader = StreamingDataLoader( + dataset, + batch_size=1, + num_workers=2, + multiprocessing_context="spawn", + ) + iterator = iter(loader) + try: + assert next(iterator)["id"].tolist() == [0] + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [ + 1, + 0, + 0, + 0, + ] + with pytest.raises(RuntimeError, match="complete logical step boundary"): + dataset.state_dict() + + assert next(iterator)["id"].tolist() == [2] + checkpoint = dataset.state_dict() + remaining = [batch["id"].tolist() for batch in iterator] + finally: + iterator._shutdown_workers() + + assert checkpoint["samples_consumed_per_split"] == [1, 1, 0, 0] + assert remaining == [[1], [3]] + + +def test_standard_dataloader_rejects_stale_parent_checkpoint(tmp_path): + """A standard DataLoader must not expose prefetched producer progress.""" + db = lancedb.connect(tmp_path) + table = db.create_table("untracked_workers", pa.table({"id": [1, 2, 10, 20]})) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + # Merely constructing the checkpoint-aware loader must not authorize a + # later plain DataLoader's worker progress. + StreamingDataLoader(dataset, batch_size=2, num_workers=0) + loader = torch.utils.data.DataLoader( + dataset, + batch_size=2, + num_workers=2, + multiprocessing_context="spawn", + ) + iterator = iter(loader) + try: + assert next(iterator)["id"].tolist() == [1, 2] + with pytest.raises(RuntimeError, match="Use StreamingDataLoader"): + dataset.state_dict() + list(iterator) + finally: + iterator._shutdown_workers() + + +def test_streaming_dataloader_rejects_persistent_workers(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("persistent_workers", pa.table({"id": [1, 2]})) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + + with pytest.raises(ValueError, match="persistent_workers=True"): + StreamingDataLoader( + dataset, + batch_size=1, + num_workers=2, + persistent_workers=True, + ) + + +def test_collate_failure_invalidates_consumer_checkpoint(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table( + "collate_failure", pa.table({"id": [0, 1, 2, 3, 100, 101, 102, 103]}) + ) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=2, + multiprocessing_context="spawn", + collate_fn=_collate_with_first_batch_error, + prefetch_factor=2, + ) + iterator = iter(loader) + try: + with pytest.raises(ValueError, match="first batch fails"): + next(iterator) + assert next(iterator) == [100, 101] + assert next(iterator) == [2, 3] + with pytest.raises(RuntimeError, match="failed before it was returned"): + dataset.state_dict() + list(iterator) + finally: + iterator._shutdown_workers() + + +def test_collate_stop_iteration_invalidates_consumer_checkpoint(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("collate_stop", pa.table({"id": list(range(6))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=0, + collate_fn=_collate_with_first_batch_stop, + ) + iterator = iter(loader) + + with pytest.raises(RuntimeError, match="collate_fn raised StopIteration"): + next(iterator) + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2] + with pytest.raises(RuntimeError, match="failed before it was returned"): + dataset.state_dict() + assert list(iterator) == [[2, 3], [4, 5]] + + +def test_batch_base_exception_invalidates_consumer_checkpoint(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("collate_interrupt", pa.table({"id": list(range(6))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=0, + collate_fn=_collate_with_first_batch_interrupt, + ) + iterator = iter(loader) + + with pytest.raises(KeyboardInterrupt, match="first batch interrupted"): + next(iterator) + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2] + with pytest.raises(RuntimeError, match="failed before it was returned"): + dataset.state_dict() + assert list(iterator) == [[2, 3], [4, 5]] + + +def test_parent_commit_base_exception_invalidates_consumer_checkpoint(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("commit_interrupt", pa.table({"id": list(range(4))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0) + iterator = iter(loader) + real_commit = dataset._commit_worker_state + + def interrupt_after_commit(state, *, require_uniform): + real_commit(state, require_uniform=require_uniform) + raise KeyboardInterrupt("after parent commit") + + with patch.object( + dataset, "_commit_worker_state", side_effect=interrupt_after_commit + ): + with pytest.raises(KeyboardInterrupt, match="after parent commit"): + next(iterator) + + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2] + with pytest.raises(RuntimeError, match="failed before it was returned"): + dataset.state_dict() + + +def test_direct_iteration_surfaces_prefetch_failure_before_committing_row( + tmp_path, monkeypatch +): + db = lancedb.connect(tmp_path) + table = db.create_table("prefetch_failure", pa.table({"id": list(range(4))})) + release = threading.Event() + failed = threading.Event() + real_getitems = streaming.Permutation.__getitems__ + + def controlled_getitems(permutation, indices): + if indices and indices[0] >= 2: + assert release.wait(timeout=5) + failed.set() + raise RuntimeError("later prefetched I/O failed") + return real_getitems(permutation, indices) + + class SignalDict(dict): + def __setitem__(self, key, value): + super().__setitem__(key, value) + release.set() + assert failed.wait(timeout=5) + + monkeypatch.setattr(streaming.Permutation, "__getitems__", controlled_getitems) + dataset = StreamingDataset( + table, + num_splits=1, + shuffle=False, + read_batch_size=2, + io_queue_depth=2, + ) + dataset._resume_positions = SignalDict() + iterator = iter(dataset) + + assert next(iterator)["id"] == 0 + with pytest.raises(RuntimeError, match="later prefetched I/O failed"): + next(iterator) + + checkpoint = dataset.state_dict() + assert checkpoint["samples_consumed_per_split"] == [1] + assert checkpoint["positions_consumed_per_split"] == [1] + + +@pytest.mark.parametrize("workers", [0, 1, 2]) +def test_streaming_dataloader_rejects_drop_last(tmp_path, workers): + db = lancedb.connect(tmp_path) + table = db.create_table("drop_last", pa.table({"id": [0, 1, 2]})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + worker_options = {"multiprocessing_context": "spawn"} if workers else {} + + with pytest.raises(ValueError, match="drop_last=True"): + StreamingDataLoader( + dataset, + batch_size=2, + num_workers=workers, + drop_last=True, + **worker_options, + ) + + +def test_streaming_dataloader_owns_one_iterator_until_teardown(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("iterator_owner", pa.table({"id": list(range(4))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=1, + multiprocessing_context="spawn", + ) + + first = iter(loader) + try: + assert next(first)["id"].tolist() == [0, 1] + with pytest.raises(RuntimeError, match="concurrent iteration"): + iter(loader) + finally: + first._shutdown_workers() + + second = iter(loader) + try: + assert [batch["id"].tolist() for batch in second] == [[2, 3]] + except BaseException: + second._shutdown_workers() + raise + + # Natural exhaustion releases ownership too. + third = iter(loader) + try: + assert list(third) == [] + finally: + third._shutdown_workers() + + +def test_zero_worker_shutdown_closes_inner_iterator_before_release(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("zero_worker_shutdown", pa.table({"id": list(range(6))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0) + + first = iter(loader) + assert next(first)["id"].tolist() == [0, 1] + first._shutdown_workers() + + assert dataset._consumer_iterator_active is False + assert dataset._raw_batches_ref is None + second = iter(loader) + try: + with pytest.raises(StopIteration): + next(first) + assert next(second)["id"].tolist() == [2, 3] + finally: + second._shutdown_workers() + + +def test_direct_and_loader_admission_share_one_atomic_lease(tmp_path, monkeypatch): + db = lancedb.connect(tmp_path) + table = db.create_table("direct_loader_lease", pa.table({"id": list(range(4))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0) + entered = threading.Event() + release = threading.Event() + direct_result = [] + direct_error = [] + contender = [] + real_resolve = dataset._resolve_my_splits + + def controlled_resolve(): + if threading.current_thread().name == "direct-start": + entered.set() + assert release.wait(timeout=5) + return real_resolve() + + def advance_direct(iterator): + try: + direct_result.append(next(iterator)["id"]) + except BaseException as exc: + direct_error.append(exc) + + monkeypatch.setattr(dataset, "_resolve_my_splits", controlled_resolve) + direct = iter(dataset) + thread = threading.Thread( + target=advance_direct, args=(direct,), name="direct-start" + ) + thread.start() + assert entered.wait(timeout=5) + try: + with pytest.raises(RuntimeError, match="concurrent iteration"): + contender.append(iter(loader)) + finally: + release.set() + thread.join(timeout=5) + if contender: + contender[0]._shutdown_workers() + direct.close() + + assert not thread.is_alive() + assert direct_error == [] + assert direct_result == [0] + + +def test_loader_acquires_before_snapshot_and_cleans_interrupted_acquire( + tmp_path, monkeypatch +): + db = lancedb.connect(tmp_path) + table = db.create_table("lease_snapshot", pa.table({"id": list(range(4))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0) + first = iter(loader) + assert next(first)["id"].tolist() == [0, 1] + + entered = threading.Event() + release = threading.Event() + pending = [] + pending_errors = [] + observed_snapshots = [] + real_acquire = dataset._acquire_consumer_iterator + real_snapshot = dataset._checkpoint_snapshot + + def controlled_acquire(): + if threading.current_thread().name == "stale-start": + entered.set() + assert release.wait(timeout=5) + return real_acquire() + + def recording_snapshot(): + state = real_snapshot() + if threading.current_thread().name == "stale-start": + observed_snapshots.append(state["samples_consumed_per_split"]) + return state + + def create_pending_iterator(): + try: + pending.append(iter(loader)) + except BaseException as exc: + pending_errors.append(exc) + + monkeypatch.setattr(dataset, "_acquire_consumer_iterator", controlled_acquire) + monkeypatch.setattr(dataset, "_checkpoint_snapshot", recording_snapshot) + thread = threading.Thread(target=create_pending_iterator, name="stale-start") + thread.start() + assert entered.wait(timeout=5) + assert next(first)["id"].tolist() == [2, 3] + with pytest.raises(StopIteration): + next(first) + release.set() + thread.join(timeout=5) + + assert not thread.is_alive() + assert pending_errors == [] + assert observed_snapshots == [[4]] + assert len(pending) == 1 + assert list(pending[0]) == [] + assert dataset.state_dict()["samples_consumed_per_split"] == [4] + + def interrupted_acquire(): + real_acquire() + raise KeyboardInterrupt("after acquire") + + monkeypatch.setattr(dataset, "_acquire_consumer_iterator", interrupted_acquire) + with pytest.raises(KeyboardInterrupt, match="after acquire"): + iter(loader) + assert dataset._consumer_iterator_active is False + + +def test_consumer_iterator_lease_publication_is_atomic(tmp_path, monkeypatch): + db = lancedb.connect(tmp_path) + table = db.create_table("atomic_lease", pa.table({"id": [0, 1]})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=1, num_workers=0) + real_get_ident = streaming.threading.get_ident + calls = 0 + + def interrupt_during_publication(): + nonlocal calls + calls += 1 + if calls == 1: + raise KeyboardInterrupt("during lease mutation") + return real_get_ident() + + monkeypatch.setattr(streaming.threading, "get_ident", interrupt_during_publication) + with pytest.raises(KeyboardInterrupt, match="during lease mutation"): + iter(loader) + monkeypatch.setattr(streaming.threading, "get_ident", real_get_ident) + + assert dataset._consumer_iterator_active is False + iterator = iter(loader) + try: + assert next(iterator)["id"].tolist() == [0] + finally: + iterator._shutdown_workers() + + +def test_streaming_dataloader_rejects_dataset_iter_override(tmp_path): + class CustomizedDataset(StreamingDataset): + def __iter__(self): + return iter([1000, 1001]) + + db = lancedb.connect(tmp_path) + table = db.create_table("custom_iteration", pa.table({"id": [0, 1, 2]})) + dataset = CustomizedDataset(table, num_splits=1, shuffle=False) + + assert list(dataset) == [1000, 1001] + with pytest.raises(TypeError, match="override __iter__"): + StreamingDataLoader( + dataset, + batch_size=2, + num_workers=0, + collate_fn=list, + ) + + +def test_interleaved_adapters_do_not_authorize_plain_iteration(tmp_path): + db = lancedb.connect(tmp_path) + table_a = db.create_table("adapter_a", pa.table({"id": [0, 1]})) + table_b = db.create_table("adapter_b", pa.table({"id": [10, 11]})) + dataset_a = StreamingDataset(table_a, num_splits=1, shuffle=False) + dataset_b = StreamingDataset(table_b, num_splits=1, shuffle=False) + initial_state = dataset_a.state_dict() + + owner_a = dataset_a._acquire_consumer_iterator() + owner_b = dataset_b._acquire_consumer_iterator() + try: + iterator_a = iter(streaming._StreamingDatasetAdapter(dataset_a)) + iterator_b = iter(streaming._StreamingDatasetAdapter(dataset_b)) + assert next(iterator_a).data["id"] == 0 + assert next(iterator_b).data["id"] == 10 + assert [sample.data["id"] for sample in iterator_a] == [1] + assert [sample.data["id"] for sample in iterator_b] == [11] + finally: + dataset_a._release_consumer_iterator(owner_a) + dataset_b._release_consumer_iterator(owner_b) + + dataset_a.load_state_dict(initial_state) + with patch( + "lancedb.streaming.get_worker_info", + return_value=FakeWorkerInfo(id=0, num_workers=1), + ): + plain_iterator = iter(dataset_a) + assert next(plain_iterator)["id"] == 0 + plain_iterator.close() + + assert dataset_a._untracked_worker_iteration[0] == 1 + with pytest.raises(RuntimeError, match="Use StreamingDataLoader"): + dataset_a.state_dict() + + +def test_resume_from_partial_split_cycle_preserves_remaining_order(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("partial_cycle", pa.table({"id": [1, 2, 10, 20]})) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + iterator = iter(dataset) + + assert next(iterator)["id"] == 1 + checkpoint = dataset.state_dict() + iterator.close() + assert checkpoint["samples_consumed_per_split"] == [1, 0] + + resumed = StreamingDataset(table, num_splits=2, shuffle=False) + resumed.load_state_dict(checkpoint) + assert [row["id"] for row in resumed] == [10, 2, 20] + + +def test_partial_cycle_resume_preserves_skip_truncation(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table( + "partial_skip", pa.table({"id": [0, 1, 2, 3, 100, 101, 102, 103]}) + ) + kwargs = dict( + num_splits=2, + shuffle=False, + transform=_failing_transform({1, 2, 3}), + on_transform_error="skip", + ) + dataset = StreamingDataset(table, **kwargs) + iterator = iter(dataset) + + assert next(iterator)["id"] == 0 + checkpoint = dataset.state_dict() + uninterrupted = [row["id"] for row in iterator] + + resumed = StreamingDataset(table, **kwargs) + resumed.load_state_dict(checkpoint) + assert [row["id"] for row in resumed] == uninterrupted == [100] + + def test_multi_worker_resumability_same_topology(lance_table): """Checkpoint with num_workers=2, resume with num_workers=2: exact continuation.""" world_size = 1 @@ -2018,6 +2600,23 @@ def test_merge_state_dicts_validates_consistency(lance_table): StreamingDataset.merge_state_dicts([]) +def test_merge_state_dicts_combines_nonuniform_consumer_progress(lance_table): + dataset = StreamingDataset( + lance_table, num_splits=2, shuffle=False, shuffle_seed=SHUFFLE_SEED + ) + rank0 = dataset.state_dict() + rank0["samples_consumed_per_split"] = [2, 0] + rank0["positions_consumed_per_split"] = [2, 0] + rank1 = dataset.state_dict() + rank1["samples_consumed_per_split"] = [0, 2] + rank1["positions_consumed_per_split"] = [0, 2] + + merged = StreamingDataset.merge_state_dicts([rank0, rank1]) + + assert merged["samples_consumed_per_split"] == [2, 2] + assert merged["positions_consumed_per_split"] == [2, 2] + + def test_load_state_dict_without_positions_key(lance_table): """Checkpoints from before positions_consumed_per_split existed still resume exactly (positions equal sample counts when nothing is skipped).""" @@ -2254,6 +2853,65 @@ def test_pack_sequences_checkpoint_resumes_on_new_topology(tmp_path): ] +def test_packed_checkpoint_requires_complete_split_cycle(tmp_path): + table = _create_token_table(tmp_path, [[1], [2], [10], [20]]) + dataset = _packed_dataset(table, pack_sequences=3, blocks_per_epoch=4, num_splits=2) + iterator = iter(dataset) + + next(iterator) + with pytest.raises(RuntimeError, match="complete logical step boundary"): + dataset.state_dict() + + next(iterator) + assert dataset.state_dict()["blocks_emitted_per_split"] == [1, 1] + iterator.close() + + +def test_streaming_dataloader_commits_consumed_packed_batches(tmp_path): + table = _create_token_table( + tmp_path, + [[1], [2], [3], [4], [10], [20], [30], [40]], + ) + kwargs = dict(pack_sequences=4, blocks_per_epoch=4, num_splits=2) + dataset = _packed_dataset(table, **kwargs) + loader = StreamingDataLoader( + dataset, + batch_size=1, + num_workers=2, + multiprocessing_context="spawn", + prefetch_factor=2, + ) + iterator = iter(loader) + try: + next(iterator) + with pytest.raises(RuntimeError, match="complete logical step boundary"): + dataset.state_dict() + + next(iterator) + checkpoint = dataset.state_dict() + uninterrupted = [batch["input_ids"].tolist() for batch in iterator] + finally: + iterator._shutdown_workers() + + resumed = _packed_dataset(table, **kwargs) + resumed.load_state_dict(checkpoint) + resumed_loader = StreamingDataLoader( + resumed, + batch_size=1, + num_workers=2, + multiprocessing_context="spawn", + prefetch_factor=2, + ) + resumed_iterator = iter(resumed_loader) + try: + remaining = [batch["input_ids"].tolist() for batch in resumed_iterator] + finally: + resumed_iterator._shutdown_workers() + + assert checkpoint["blocks_emitted_per_split"] == [1, 1] + assert remaining == uninterrupted + + def test_pack_sequences_validates_configuration_and_tokens(tmp_path): table = _create_token_table(tmp_path, [[1, 2]]) From 93f47b8aab4f888ab6cd4da75403e2d41be80da4 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 24 Aug 2026 13:35:31 -0700 Subject: [PATCH 23/31] fix(remote): stop table_names inventing a page token for a namespace (#4039) `table_names` paginates using a `start_after` table name. This works for the `/v1/table` endpoint, which guarantees table-order. But the `/v1/namespace/{id}/table/list` does not. We change that caller to instead collect all table names, sort, and apply the pagination locally. We are deprecating this API, so this is just an interim fix. For good performance, users should move to the `list_tables` API instead, which uses opaque tokens that don't rely on lexical sorting. --------- Co-authored-by: Claude Opus 5 (1M context) --- rust/lancedb/src/remote/db.rs | 191 ++++++++++++++++++++++++++++++---- 1 file changed, 171 insertions(+), 20 deletions(-) diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 8265d22c0..3f4216bc8 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -344,6 +344,62 @@ impl RemoteDatabase { self.table_cache.remove(&cache_key).await; Ok((request_id, resp)) } + + /// Collect the tables of a namespace in name order, for `table_names`. + /// + /// `table_names` promises name order and resumes after a table name, but the namespace + /// route's `page_token` is opaque -- it belongs to the store the listing walks, and a + /// token this client invented would resume from the wrong place. So the whole namespace is + /// walked by handing each response's token straight back, and the name semantics are + /// applied here. Constructing no token is what makes this work against a server on either + /// side of the change: it only ever repeats what the server said. + /// + /// This is the cost `table_names` already paid -- the server used to enumerate and sort the + /// namespace on every request -- and it is why `list_tables` replaces it. + async fn table_names_in_namespace( + &self, + request: &TableNamesRequest, + ) -> Result<(Vec, ServerVersion)> { + let namespace_id = + build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter); + let path = format!("/v1/namespace/{}/table/list", namespace_id); + + let mut names = Vec::new(); + // Every page reports the same server, so keep the first page's version. + let mut version: Option = None; + let mut page_token: Option = None; + loop { + let mut req = self.client.get(&path); + if let Some(ref token) = page_token { + req = req.query(&[("page_token", token)]); + } + let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?; + let rsp = self.client.check_response(&request_id, rsp).await?; + if version.is_none() { + version = Some(parse_server_version(&request_id, &rsp)?); + } + let response: ListTablesResponse = rsp.json().await.err_to_http(request_id)?; + names.extend(response.tables); + // An empty token is the end of the listing, not a token to send back: a server + // that reads an empty token as "start from the beginning" would hand back the + // first page again. + match response.page_token.filter(|token| !token.is_empty()) { + // A server that repeated a token would never finish; treat that as the end + // rather than looping on it. + Some(token) if Some(&token) != page_token.as_ref() => page_token = Some(token), + _ => break, + } + } + + names.sort(); + if let Some(ref start_after) = request.start_after { + names.retain(|name| name > start_after); + } + if let Some(limit) = request.limit { + names.truncate(limit as usize); + } + Ok((names, version.unwrap_or_default())) + } } #[cfg(all(test, feature = "remote"))] @@ -621,29 +677,29 @@ impl Database for RemoteDatabase { } async fn table_names(&self, request: TableNamesRequest) -> Result> { - let mut req = if !request.namespace_path.is_empty() { - let namespace_id = - build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter); - self.client - .get(&format!("/v1/namespace/{}/table/list", namespace_id)) + let (tables, version) = if request.namespace_path.is_empty() { + // The flat route resumes after a table name and orders by name, which is exactly + // what `start_after` means, so the server does the paging. + let mut req = self.client.get("/v1/table/"); + if let Some(limit) = request.limit { + req = req.query(&[("limit", limit)]); + } + if let Some(ref start_after) = request.start_after { + req = req.query(&[("page_token", start_after)]); + } + let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?; + let rsp = self.client.check_response(&request_id, rsp).await?; + let version = parse_server_version(&request_id, &rsp)?; + let tables = rsp + .json::() + .await + .err_to_http(request_id)? + .tables; + (tables, version) } else { - self.client.get("/v1/table/") + self.table_names_in_namespace(&request).await? }; - if let Some(limit) = request.limit { - req = req.query(&[("limit", limit)]); - } - if let Some(start_after) = request.start_after { - req = req.query(&[("page_token", start_after)]); - } - let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?; - let rsp = self.client.check_response(&request_id, rsp).await?; - let version = parse_server_version(&request_id, &rsp)?; - let tables = rsp - .json::() - .await - .err_to_http(request_id)? - .tables; for table in &tables { let table_identifier = build_table_identifier(table, &request.namespace_path, &self.client.id_delimiter); @@ -1227,6 +1283,101 @@ mod tests { assert_eq!(names, vec!["table1", "table2"]); } + #[tokio::test] + async fn test_table_names_in_a_namespace_never_invents_a_page_token() { + // The namespace route's token belongs to the store, so `table_names` cannot build one + // from `start_after`. It walks the namespace on the server's own tokens and applies the + // name semantics itself, which is what keeps it working either side of the change. + let page = Arc::new(AtomicUsize::new(0)); + let conn = Connection::new_with_handler(move |request| { + assert_eq!(request.url().path(), "/v1/namespace/ns/table/list"); + let query = request.url().query().unwrap_or(""); + assert!( + !query.contains("page_token=users"), + "a table name must never be sent as a page token: {query}" + ); + match page.fetch_add(1, Ordering::SeqCst) { + 0 => { + assert!( + !query.contains("page_token"), + "the walk starts with no token" + ); + http::Response::builder() + .status(200) + .body(r#"{"tables": ["users", "orders"], "page_token": "opaque-1"}"#) + .unwrap() + } + _ => { + assert!(query.contains("page_token=opaque-1")); + http::Response::builder() + .status(200) + .body(r#"{"tables": ["widgets"]}"#) + .unwrap() + } + } + }); + + let names = conn + .table_names() + .namespace(vec!["ns".to_string()]) + .start_after("users") + .execute() + .await + .unwrap(); + // Name order, resumed after "users": "orders" sorts before it and is dropped. + assert_eq!(names, vec!["widgets"]); + } + + #[tokio::test] + async fn test_table_names_in_a_namespace_stops_on_a_repeated_token() { + // A server that handed back the token it was given would never finish the walk. + let conn = Connection::new_with_handler(|_request| { + http::Response::builder() + .status(200) + .body(r#"{"tables": ["a"], "page_token": "same"}"#) + .unwrap() + }); + + let names = conn + .table_names() + .namespace(vec!["ns".to_string()]) + .execute() + .await + .unwrap(); + // The guard bounds the walk instead of letting it run forever. The repeat is the + // server breaking the token contract and is not papered over here. + assert_eq!(names, vec!["a", "a"]); + } + + #[tokio::test] + async fn test_table_names_in_a_namespace_stops_on_an_empty_token() { + // An empty token ends the listing. Sending it back would ask a server that reads it + // as "start from the beginning" for the first page a second time, and every name on + // that page would be collected twice. + let requests = Arc::new(AtomicUsize::new(0)); + let seen = requests.clone(); + let conn = Connection::new_with_handler(move |request| { + seen.fetch_add(1, Ordering::SeqCst); + assert!( + !request.url().query().unwrap_or("").contains("page_token"), + "an empty token must never be sent back" + ); + http::Response::builder() + .status(200) + .body(r#"{"tables": ["a"], "page_token": ""}"#) + .unwrap() + }); + + let names = conn + .table_names() + .namespace(vec!["ns".to_string()]) + .execute() + .await + .unwrap(); + assert_eq!(names, vec!["a"]); + assert_eq!(requests.load(Ordering::SeqCst), 1); + } + #[tokio::test] async fn test_table_names_pagination() { let conn = Connection::new_with_handler(|request| { From c72f5b2960d08a1d6caee703191c360ecfe6f3ff Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Mon, 24 Aug 2026 14:06:40 -0700 Subject: [PATCH 24/31] feat: bind a materialized view refresh to the view incarnation (#4043) A caller that queues a refresh and executes it later can only tell the view it captured from a drop-and-recreate by comparing the definition and version. A recreated view with the same definition and an equal or higher version passes that check, and the check runs before the refresh reloads the view, so it never sees the state it commits against. This mints an `mv.incarnation` token in the schema metadata at each physical creation of a view table. `RefreshMaterializedViewBuilder::expect_incarnation` carries the captured token into the refresh, which compares it against the latest stored manifest before planning and again immediately before each commit (publish, fragment swap, watermark stamp), refusing to land in a different incarnation. A view with no token -- declared before tokens existed, or its metadata replaced wholesale -- is refused under a bound refresh with its own wording and is minted one by its next unbound refresh. The token is exposed through `MaterializedView::incarnation`; refreshes without an expectation are unchanged. This is best effort: the token is not part of lance's commit condition, so a recreation landing between the final pre-commit read and the commit itself is not caught. Closing that window needs a base-manifest precondition in lance's commit path. --- rust/lancedb/src/materialized_view.rs | 58 +++- rust/lancedb/src/materialized_view/refresh.rs | 265 +++++++++++++++++- 2 files changed, 307 insertions(+), 16 deletions(-) diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 1d7cbb5e7..b28d52931 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -37,6 +37,13 @@ pub use refresh::{RefreshMaterializedViewResult, RefreshMode}; /// Schema metadata key holding the view definition, as kind-tagged JSON. pub const DEFINITION_META_KEY: &str = "mv.definition"; +/// Schema metadata key holding the view's incarnation: a token minted at each +/// physical creation of a view table, so a view dropped and recreated under +/// the same name and definition is still told apart from the one a caller +/// captured. A view whose metadata was replaced wholesale, or one declared +/// before tokens existed, carries none until its next refresh mints one. +pub const INCARNATION_META_KEY: &str = "mv.incarnation"; + /// Schema metadata key holding the source table version the view was last /// refreshed to. Absent until the first refresh. pub const SOURCE_VERSION_META_KEY: &str = "mv.source_version"; @@ -612,8 +619,17 @@ impl PreparedDeclaration { pub async fn create(self, name: &str) -> Result { let empty: Vec> = vec![]; + // Minted here, not at preparation: a declaration can be cloned and + // create more than one physical table, and each needs its own token. + let incarnation = uuid::Uuid::new_v4().to_string(); + let mut metadata = self.schema.metadata().clone(); + metadata.insert(INCARNATION_META_KEY.to_string(), incarnation.clone()); + let schema = Arc::new(ArrowSchema::new_with_metadata( + self.schema.fields().clone(), + metadata, + )); let reader: Box = - Box::new(arrow_array::RecordBatchIterator::new(empty, self.schema)); + Box::new(arrow_array::RecordBatchIterator::new(empty, schema)); let mut request = CreateTableRequest::new(name.to_string(), Box::new(reader)); let write_params = request .write_options @@ -648,6 +664,7 @@ impl PreparedDeclaration { Ok(MaterializedView { table, definition: self.definition, + incarnation: Some(incarnation), }) } } @@ -878,6 +895,7 @@ impl CreateMaterializedViewBuilder { pub struct MaterializedView { table: Table, definition: MaterializedViewDefinition, + incarnation: Option, } impl MaterializedView { @@ -893,8 +911,13 @@ impl MaterializedView { }); } let schema = table.schema().await?; + let incarnation = schema.metadata().get(INCARNATION_META_KEY).cloned(); match materialized_view_kind(schema.metadata())? { - Some(MaterializedViewKind::Select(definition)) => Ok(Self { table, definition }), + Some(MaterializedViewKind::Select(definition)) => Ok(Self { + table, + definition, + incarnation, + }), Some(MaterializedViewKind::Unrecognized { kind }) => Err(Error::NotSupported { message: format!( "materialized view '{}' is defined by '{kind}', which this version of \ @@ -923,6 +946,13 @@ impl MaterializedView { &self.definition } + /// The view's incarnation token as of when this handle was opened; see + /// [`RefreshMaterializedViewBuilder::expect_incarnation`]. `None` for a + /// view that has none yet (see [`INCARNATION_META_KEY`]). + pub fn incarnation(&self) -> Option<&str> { + self.incarnation.as_deref() + } + /// Recompute the view from its source. /// /// By default the refresh is incremental when the source's changes can be @@ -943,6 +973,7 @@ impl MaterializedView { view: self.clone(), full: false, source_version: None, + expected_incarnation: None, } } } @@ -952,6 +983,7 @@ pub struct RefreshMaterializedViewBuilder { view: MaterializedView, full: bool, source_version: Option, + expected_incarnation: Option, } impl RefreshMaterializedViewBuilder { @@ -967,8 +999,28 @@ impl RefreshMaterializedViewBuilder { self } + /// Refresh only if the view is still the incarnation that minted `token` + /// (see [`MaterializedView::incarnation`]): a refresh requested against + /// one declaration must not land in a view dropped and recreated since, + /// even under the same name and definition. + /// + /// Best effort. The token is read from the latest stored manifest before + /// planning and again immediately before every commit, but it is not part + /// of the commit's own condition, so a recreation that lands between that + /// final read and the commit is not caught. + pub fn expect_incarnation(mut self, token: impl Into) -> Self { + self.expected_incarnation = Some(token.into()); + self + } + pub async fn execute(self) -> Result { - refresh::execute_refresh(&self.view.table, self.full, self.source_version).await + refresh::execute_refresh( + &self.view.table, + self.full, + self.source_version, + self.expected_incarnation.as_deref(), + ) + .await } } diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index 24d828e01..735751c27 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -46,8 +46,8 @@ use lance_table::format::Fragment; use serde::{Deserialize, Serialize}; use super::{ - MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY, SOURCE_ROW_ID_COLUMN, - SOURCE_VERSION_META_KEY, + INCARNATION_META_KEY, MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY, + SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY, }; use crate::database::OpenTableRequest; use crate::table::{NativeTable, NativeTableExt, Table}; @@ -108,6 +108,7 @@ pub(crate) async fn execute_refresh( view: &Table, full: bool, pinned: Option, + expected_incarnation: Option<&str>, ) -> Result { let view_native = view.as_native().ok_or_else(|| Error::NotSupported { message: "materialized views are supported only on local tables".into(), @@ -122,6 +123,8 @@ pub(crate) async fn execute_refresh( view_native.dataset.reload().await?; let view_ds = view_native.dataset.get().await?.as_ref().clone(); + ensure_incarnation(&view_ds, expected_incarnation, view.name()).await?; + // The definition a handle cached at open may since have been replaced; // what refresh executes and what it stamps must be one generation. let definition = match super::materialized_view_kind(&view_ds.schema().metadata)? { @@ -240,6 +243,7 @@ pub(crate) async fn execute_refresh( increment, definition, watermark, + expected_incarnation, ) .await?; match reconciled { @@ -253,6 +257,7 @@ pub(crate) async fn execute_refresh( source_version, source_ts, definition, + expected_incarnation, ) .await } @@ -266,6 +271,7 @@ pub(crate) async fn execute_refresh( source_version, source_ts, definition, + expected_incarnation, ) .await } @@ -595,6 +601,7 @@ async fn incremental( increment: Increment, definition: &MaterializedViewDefinition, watermark: Option, + expected_incarnation: Option<&str>, ) -> Result> { let new_fragments = increment.appended; let watermark_version = watermark.unwrap_or(0); @@ -671,15 +678,35 @@ async fn incremental( }; let nothing_to_add = (new_fragments.is_empty() && !updated_rows) || remaining == Some(0); if nothing_to_add && eviction.is_none() { - result.version = - stamp_watermark(view_native, view_ds.clone(), source_version, source_ts).await?; + result.version = stamp_watermark( + view_native, + view_ds.clone(), + source_version, + source_ts, + expected_incarnation, + ) + .await?; return Ok(Some(result)); } // Rows left but none arrive: the removals still have to be published. if nothing_to_add { let filter = refresh_filter(&empty_keys(view_ds)?)?; - let published = publish(view_ds, eviction, Vec::new(), Some(filter)).await?; - result.version = stamp_watermark(view_native, published, source_version, source_ts).await?; + let published = publish( + view_ds, + eviction, + Vec::new(), + Some(filter), + expected_incarnation, + ) + .await?; + result.version = stamp_watermark( + view_native, + published, + source_version, + source_ts, + expected_incarnation, + ) + .await?; return Ok(Some(result)); } @@ -737,12 +764,20 @@ async fn incremental( eviction, Vec::new(), Some(refresh_filter(&empty_keys(view_ds)?)?), + expected_incarnation, ) .await? } else { view_ds.clone() }; - result.version = stamp_watermark(view_native, published, source_version, source_ts).await?; + result.version = stamp_watermark( + view_native, + published, + source_version, + source_ts, + expected_incarnation, + ) + .await?; return Ok(Some(result)); }; let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( @@ -775,9 +810,23 @@ async fn incremental( }); }; let filter = refresh_filter(&keys)?; - let appended = publish(view_ds, eviction, new_fragments, Some(filter)).await?; + let appended = publish( + view_ds, + eviction, + new_fragments, + Some(filter), + expected_incarnation, + ) + .await?; result.rows_written = rows_written.load(Ordering::Relaxed); - result.version = stamp_watermark(view_native, appended, source_version, source_ts).await?; + result.version = stamp_watermark( + view_native, + appended, + source_version, + source_ts, + expected_incarnation, + ) + .await?; Ok(Some(result)) } @@ -788,6 +837,7 @@ async fn rebuild( source_version: u64, source_ts: u128, definition: &MaterializedViewDefinition, + expected_incarnation: Option<&str>, ) -> Result { let rows_written = Arc::new(AtomicU64::new(0)); let schema = Arc::new(ArrowSchema::from(view_ds.schema())); @@ -810,8 +860,16 @@ async fn rebuild( // carries no schema metadata, so it cannot erase a definition update // that raced in the way an overwrite (which adopts its stream's schema) // durably would -- and it must land on the planned generation or abort. - let replaced = replace_retaining_indices(view_ds.clone(), stream, keys).await?; - let version = stamp_watermark(view_native, replaced, source_version, source_ts).await?; + let replaced = + replace_retaining_indices(view_ds.clone(), stream, keys, expected_incarnation).await?; + let version = stamp_watermark( + view_native, + replaced, + source_version, + source_ts, + expected_incarnation, + ) + .await?; Ok(RefreshMaterializedViewResult { mode: RefreshMode::Rebuild, rows_written: rows_written.load(Ordering::Relaxed), @@ -828,11 +886,13 @@ async fn replace_retaining_indices( view_ds: Dataset, stream: SendableRecordBatchStream, keys: Arc>, + expected_incarnation: Option<&str>, ) -> Result { let ds = Arc::new(view_ds); let read_version = ds.version().version; #[cfg(test)] tests::hold_before_publish(ds.uri()).await; + ensure_incarnation(&ds, expected_incarnation, ds.uri()).await?; let removed_fragment_ids: Vec = ds.get_fragments().iter().map(|f| f.id() as u64).collect(); let write_txn = InsertBuilder::new(WriteDestination::Dataset(ds.clone())) @@ -886,6 +946,32 @@ async fn replace_retaining_indices( } /// Record that the view now reflects `source_version`, including the view +/// Refuse to act on a view that is not `expected`'s incarnation, judged from +/// the latest stored manifest. Not a commit condition; see +/// `RefreshMaterializedViewBuilder::expect_incarnation`. +async fn ensure_incarnation(view_ds: &Dataset, expected: Option<&str>, what: &str) -> Result<()> { + let Some(expected) = expected else { + return Ok(()); + }; + let mut latest = view_ds.clone(); + latest.checkout_latest().await?; + match latest.schema().metadata.get(INCARNATION_META_KEY) { + Some(actual) if actual == expected => Ok(()), + Some(_) => Err(Error::Runtime { + message: format!( + "materialized view '{what}' is not the incarnation this refresh was \ + requested for: it was dropped and recreated" + ), + }), + None => Err(Error::Runtime { + message: format!( + "materialized view '{what}' carries no incarnation token: its schema \ + metadata was replaced since the token was captured" + ), + }), + } +} + /// version this very commit produces. The version is predicted and then /// verified; on a mismatch another commit raced in between, and the stamp /// ABORTS rather than certify that commit as the refresh's own generation. @@ -895,10 +981,21 @@ async fn stamp_watermark( mut dataset: Dataset, source_version: u64, source_ts: u128, + expected_incarnation: Option<&str>, ) -> Result { + ensure_incarnation(&dataset, expected_incarnation, dataset.uri()).await?; let predicted = dataset.version().version + 1; + // A view with no token (declared before tokens existed, or its metadata + // replaced wholesale) starts a new incarnation here. + let incarnation = dataset + .schema() + .metadata + .get(INCARNATION_META_KEY) + .cloned() + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); dataset .update_schema_metadata([ + (INCARNATION_META_KEY.to_string(), Some(incarnation)), ( SOURCE_VERSION_META_KEY.to_string(), Some(source_version.to_string()), @@ -1052,12 +1149,14 @@ async fn publish( eviction: Option<(Vec, Vec)>, new_fragments: Vec, keys: Option, + expected_incarnation: Option<&str>, ) -> Result { let planned = view_ds.version().version; #[cfg(test)] tests::hold_before_publish(view_ds.uri()).await; #[cfg(test)] tests::hold_until_peers_planned(); + ensure_incarnation(view_ds, expected_incarnation, view_ds.uri()).await?; let (updated_fragments, removed_fragment_ids) = eviction.unwrap_or_default(); let committed = CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) .execute(Transaction::new( @@ -1830,7 +1929,9 @@ mod tests { ); let staged = eviction.finish().await.unwrap(); assert!(staged.is_some(), "four ids over a chunk of two stage twice"); - publish(&view_ds, staged, Vec::new(), None).await.unwrap(); + publish(&view_ds, staged, Vec::new(), None, None) + .await + .unwrap(); native.dataset.reload().await.unwrap(); assert_eq!(read(view.table(), "x").await, vec![5, 6]); @@ -2486,6 +2587,144 @@ mod tests { assert_eq!(read(view.table(), "twice").await, vec![14]); } + /// A refresh bound to an incarnation refuses a view dropped and recreated + /// since, even under the same name and definition; the recreated view's + /// own token is accepted, and the token survives a refresh's stamp. + #[tokio::test] + async fn test_refresh_refuses_a_recreated_view_incarnation() { + let (conn, _, view) = refreshed_doubled(vec![1]).await; + let token = view.incarnation().unwrap().to_string(); + view.refresh() + .expect_incarnation(&token) + .execute() + .await + .unwrap(); + let reopened = conn.open_materialized_view("doubled").await.unwrap(); + assert_eq!(reopened.incarnation(), Some(token.as_str())); + + conn.drop_table("doubled", &[]).await.unwrap(); + let recreated = doubled_view(&conn).await; + assert_ne!(recreated.incarnation(), Some(token.as_str())); + + let err = recreated + .refresh() + .expect_incarnation(&token) + .execute() + .await + .unwrap_err(); + assert!(err.to_string().contains("dropped and recreated"), "{err}"); + assert_eq!(read(recreated.table(), "twice").await, Vec::::new()); + + recreated + .refresh() + .expect_incarnation(recreated.incarnation().unwrap()) + .execute() + .await + .unwrap(); + assert_eq!(read(recreated.table(), "twice").await, vec![2]); + } + + /// A cloned declaration creates two physical tables; each gets its own + /// token. + #[tokio::test] + async fn test_cloned_declaration_mints_a_fresh_incarnation_per_create() { + let (conn, source) = db_with_source(vec![1]).await; + let prepared = crate::materialized_view::prepare_declaration( + &source, + &[("x".into(), "x".into()), ("twice".into(), "x * 2".into())], + None, + None, + ) + .await + .unwrap(); + let replacement = prepared.clone(); + let first = prepared.create("cloned").await.unwrap(); + let first_token = first.incarnation().unwrap().to_string(); + + conn.drop_table("cloned", &[]).await.unwrap(); + let second = replacement.create("cloned").await.unwrap(); + assert_ne!(second.incarnation(), Some(first_token.as_str())); + } + + /// A recreation that lands after planning but before publication is + /// caught by the pre-commit read: the stale refresh fails and the + /// replacement stays empty under its own token. + #[tokio::test(flavor = "multi_thread")] + async fn test_bound_refresh_cannot_publish_into_a_raced_recreation() { + let _serial = DRIFT_LOCK.lock().await; + let (conn, _) = db_with_source(vec![1]).await; + let view = doubled_view(&conn).await; + let token = view.incarnation().unwrap().to_string(); + let uri = view + .table() + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .uri() + .to_string(); + + *DRIFT_TARGET.lock().unwrap() = Some(uri); + let refreshing = + tokio::spawn(async move { view.refresh().expect_incarnation(token).execute().await }); + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("refresh never reached publication"); + + conn.drop_table("doubled", &[]).await.unwrap(); + let replacement = doubled_view(&conn).await; + let replacement_token = replacement.incarnation().unwrap().to_string(); + DRIFT_RELEASED.notify_one(); + + let result = refreshing.await.unwrap(); + assert!(result.is_err(), "the stale refresh unexpectedly succeeded"); + let reopened = conn.open_materialized_view("doubled").await.unwrap(); + assert_eq!(reopened.incarnation(), Some(replacement_token.as_str())); + assert_eq!(read(reopened.table(), "twice").await, Vec::::new()); + } + + /// Replacing the schema metadata wholesale drops the token. A refresh + /// bound to the old token is refused for that reason, not as a + /// recreation; an unbound refresh mints the view a fresh one. + #[tokio::test] + async fn test_a_view_whose_metadata_was_replaced_starts_a_new_incarnation() { + let (conn, _, view) = refreshed_doubled(vec![1]).await; + let token = view.incarnation().unwrap().to_string(); + let mut metadata = HashMap::new(); + metadata.insert( + crate::materialized_view::DEFINITION_META_KEY.to_string(), + crate::materialized_view::definition_to_metadata(view.definition()).unwrap(), + ); + view.table() + .as_native() + .unwrap() + .replace_schema_metadata(metadata) + .await + .unwrap(); + assert_eq!( + conn.open_materialized_view("doubled") + .await + .unwrap() + .incarnation(), + None + ); + + let err = view + .refresh() + .expect_incarnation(&token) + .execute() + .await + .unwrap_err(); + assert!(err.to_string().contains("no incarnation token"), "{err}"); + + view.refresh().execute().await.unwrap(); + let reopened = conn.open_materialized_view("doubled").await.unwrap(); + assert!(reopened.incarnation().is_some()); + assert_ne!(reopened.incarnation(), Some(token.as_str())); + } + /// In-process refreshes of one view serialize: the loser of the race /// observes the winner's watermark instead of appending the same rows. #[tokio::test(flavor = "multi_thread")] @@ -2528,7 +2767,7 @@ mod tests { let stale = view_native.dataset.get().await.unwrap().as_ref().clone(); view.table().delete("x = 1").await.unwrap(); - let err = stamp_watermark(view_native, stale, 99, 99).await; + let err = stamp_watermark(view_native, stale, 99, 99, None).await; assert!(err.is_err()); let result = view.refresh().execute().await.unwrap(); From 71f85a8d9fab89c238648c9963e7c89e1452e74d Mon Sep 17 00:00:00 2001 From: Lance Release Date: Mon, 24 Aug 2026 21:08:39 +0000 Subject: [PATCH 25/31] =?UTF-8?q?Bump=20version:=200.38.0-beta.6=20?= =?UTF-8?q?=E2=86=92=200.38.0-beta.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index f5f981b2c..bcf714002 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.6" +current_version = "0.38.0-beta.7" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 6c6dd4aea..a60a5ddf1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5402,7 +5402,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.6" +version = "0.38.0-beta.7" dependencies = [ "ahash", "anyhow", @@ -5490,7 +5490,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.6" +version = "0.38.0-beta.7" dependencies = [ "arrow-array", "arrow-buffer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.6" +version = "0.38.0-beta.7" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index d6c285fc3..59ee8a7d6 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.6 + 0.38.0-beta.7 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 23d58124d..b24ad61ab 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.6 + 0.38.0-beta.7 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index a07414ec0..0ee59bcb7 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.6 + 0.38.0-beta.7 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index ead873855..8732d1965 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.6" +version = "0.38.0-beta.7" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index d89eb5ad1..56480c0cb 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 1721c970e..a1102ee29 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 177804dce..695862baf 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 9af443583..9099ee4de 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index c98aa3812..f04c2a884 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 05b1086f5..f9b35af32 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 50432bdea..cc5340105 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 219025534..6604da6cd 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index 63c51a799..66777d60c 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.6", + "version": "0.38.0-beta.7", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 2e4f8996f..d2ccf3064 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.6" +version = "0.38.0-beta.7" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index d32d88ecd..071c876d9 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.6" +version = "0.38.0-beta.7" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 5013c176dd9a664813888c8bd8cf99161c047ec7 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:14:33 +0800 Subject: [PATCH 26/31] fix: pin remote snapshots during permutation construction (#4022) Fixes #4015 ## Root cause PermutationBuilder issued count_rows and the projected row-ID scan through an unpinned RemoteTable handle. Each request could independently resolve latest, so a concurrent table update could make the count and scanned rows come from different snapshots. ## Fix - add a backend hook for obtaining an independent handle pinned to the currently selected version - resolve latest once for remote tables while preserving an explicit checkout and leaving the caller handle unchanged - build the count, filtered projection, and scan from that pinned handle while retaining native-table behavior - add a remote mock regression that advances latest between count and scan and covers explicit checkout preservation ## Validation - cargo test --quiet --features remote -p lancedb --lib test_remote_permutation_builder_pins_snapshot - cargo test --quiet --features remote -p lancedb --lib dataloader::permutation::builder::tests - cargo check --quiet --features remote --tests --examples - cargo clippy --quiet --features remote --tests --examples - cargo fmt --all -- --check Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo --- .../src/dataloader/permutation/builder.rs | 128 +++++++++++++++++- rust/lancedb/src/remote/table.rs | 12 ++ rust/lancedb/src/table.rs | 8 ++ 3 files changed, 146 insertions(+), 2 deletions(-) diff --git a/rust/lancedb/src/dataloader/permutation/builder.rs b/rust/lancedb/src/dataloader/permutation/builder.rs index a3700d0ef..641c191d5 100644 --- a/rust/lancedb/src/dataloader/permutation/builder.rs +++ b/rust/lancedb/src/dataloader/permutation/builder.rs @@ -239,7 +239,20 @@ impl PermutationBuilder { } /// Builds the permutation table and stores it in the given database. - pub async fn build(self) -> Result
{ + pub async fn build(mut self) -> Result
{ + // Remote tables resolve latest independently for each request. Use a + // separate pinned handle so count, projection, and scan all refer to one + // snapshot without changing the caller's table checkout state. Native + // tables return `None` here and retain their existing behavior. + if let Some(snapshot) = self + .base_table + .base_table() + .snapshot_at_current_version() + .await? + { + self.base_table = Table::from(snapshot); + } + // Unflushed rows have no row id, so a permutation cannot address them. match self.base_table.base_table().get_lsm_write_spec().await { Ok(Some(_)) => { @@ -258,7 +271,6 @@ impl PermutationBuilder { // First pass, apply filter and load row ids. `Shuffler` permutes positions, so // every rank must scan the rows in the same order to build the same permutation. - // TODO: pin the version resolved here; remote does not implement Lazy. let mut rows = self.base_table.query().select(Select::columns(&[ROW_ID])); if let Some(filter) = &self.config.filter { @@ -409,6 +421,118 @@ mod tests { assert!(table.base_table().scan_order_is_deterministic()); } + #[cfg(feature = "remote")] + #[tokio::test] + async fn test_remote_permutation_builder_pins_snapshot() { + use std::sync::{ + Mutex, + atomic::{AtomicU64, Ordering}, + }; + + use arrow_array::{RecordBatch, UInt64Array}; + use arrow_schema::{DataType, Field, Schema}; + + let row_ids = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + ROW_ID, + DataType::UInt64, + false, + )])), + vec![Arc::new(UInt64Array::from(vec![100]))], + ) + .unwrap(); + let mut query_body = Vec::new(); + { + let mut writer = + arrow_ipc::writer::FileWriter::try_new(&mut query_body, &row_ids.schema()).unwrap(); + writer.write(&row_ids).unwrap(); + writer.finish().unwrap(); + } + + let latest = Arc::new(AtomicU64::new(7)); + let expected_snapshot = Arc::new(AtomicU64::new(7)); + let planning_versions = Arc::new(Mutex::new(Vec::new())); + let latest_ref = latest.clone(); + let expected_snapshot_ref = expected_snapshot.clone(); + let planning_versions_ref = planning_versions.clone(); + let table = Table::new_with_handler("remote_base", move |request| { + let path = request.url().path(); + let body = request + .body() + .and_then(|body| body.as_bytes()) + .map(|body| serde_json::from_slice::(body).unwrap()); + + match path { + "/v1/table/remote_base/describe/" => { + let requested = body.as_ref().and_then(|body| body["version"].as_u64()); + let version = requested.unwrap_or_else(|| latest_ref.load(Ordering::SeqCst)); + http::Response::builder() + .status(200) + .body( + format!(r#"{{"version":{version},"schema":{{"fields":[]}}}}"#) + .into_bytes(), + ) + .unwrap() + } + "/v1/table/remote_base/get_lsm_write_spec/" => http::Response::builder() + .status(200) + .body(br#"{"lsm_write_spec":null}"#.to_vec()) + .unwrap(), + "/v1/table/remote_base/count_rows/" => { + let body = body.unwrap(); + let version = body["version"].as_u64().unwrap(); + assert_eq!(version, expected_snapshot_ref.load(Ordering::SeqCst)); + assert_eq!(body["predicate"], "value > 0"); + planning_versions_ref.lock().unwrap().push(version); + + // Simulate a concurrent append after count_rows. An unpinned + // scan would now resolve version 8 and include different rows. + latest_ref.store(8, Ordering::SeqCst); + http::Response::builder() + .status(200) + .body(b"1".to_vec()) + .unwrap() + } + "/v1/table/remote_base/query/" => { + let body = body.unwrap(); + let version = body["version"].as_u64().unwrap(); + assert_eq!(version, expected_snapshot_ref.load(Ordering::SeqCst)); + assert_eq!(body["filter"], "value > 0"); + assert_eq!(body["columns"], serde_json::json!([ROW_ID])); + planning_versions_ref.lock().unwrap().push(version); + http::Response::builder() + .status(200) + .header("content-type", "application/vnd.apache.arrow.file") + .body(query_body.clone()) + .unwrap() + } + _ => panic!("unexpected request: {path}"), + } + }); + + let permutation = PermutationBuilder::new(table.clone()) + .with_filter("value > 0".to_string()) + .build() + .await + .unwrap(); + assert_eq!(permutation.count_rows(None).await.unwrap(), 1); + + // Building uses a separate handle and must not pin the caller's table. + assert_eq!(table.version().await.unwrap(), 8); + + // An explicit checkout is copied as-is and remains checked out afterward. + expected_snapshot.store(6, Ordering::SeqCst); + table.checkout(6).await.unwrap(); + let permutation = PermutationBuilder::new(table.clone()) + .with_filter("value > 0".to_string()) + .build() + .await + .unwrap(); + assert_eq!(permutation.count_rows(None).await.unwrap(), 1); + assert_eq!(table.version().await.unwrap(), 6); + assert_eq!(*planning_versions.lock().unwrap(), vec![7, 7, 6, 6]); + } + #[tokio::test] async fn test_permutation_builder() { let temp_dir = tempfile::tempdir().unwrap(); diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index c6b078282..633d0ffce 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -1775,6 +1775,18 @@ impl BaseTable for RemoteTable { Ok(()) } + async fn snapshot_at_current_version(&self) -> Result>> { + // A checked-out handle already names its snapshot. Otherwise resolve + // latest exactly once before creating the independent pinned handle. + let version = match self.current_version().await { + Some(version) => version, + None => self.describe().await?.version, + }; + + let snapshot = self.with_branch(self.branch.clone()); + *snapshot.version.write().await = Some(version); + Ok(Some(Arc::new(snapshot))) + } async fn restore(&self) -> Result<()> { let mut request = self .client diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 41551bd37..356c87bbe 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -792,6 +792,14 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { async fn checkout_tag(&self, tag: &str) -> Result<()>; /// Checkout the latest version of the table. async fn checkout_latest(&self) -> Result<()>; + /// Return an independent handle pinned to the version currently selected. + /// + /// Backends that can advance between requests should override this for + /// multi-request operations that need snapshot consistency. Backends whose + /// existing handles already provide the desired behavior return `None`. + async fn snapshot_at_current_version(&self) -> Result>> { + Ok(None) + } /// Whether repeated identical scans return rows in the same order. /// /// Callers that assign meaning to a row's position must order the results From fce45ba9fc5254d3e7fff8fc90c89b99706f120c Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 24 Aug 2026 16:25:14 -0700 Subject: [PATCH 27/31] feat(nodejs): add listTables, deprecate tableNames (#4041) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `table_names` is being replaced by `list_tables` across the SDKs, but TypeScript only had `tableNames`. This PR adds `listTables`, which returns a page of table names together with the token that resumes after it, and marks `tableNames` and `TableNamesOptions` deprecated in favor of it. It binds the `Connection::list_tables` that already exists, so nothing in the Rust API changes and nothing existing breaks. `pageToken` is documented as opaque rather than as a table name, since what resumes a listing is the database's to decide — that keeps callers off a detail that is going to change. Stacked on #4040, which fixes a table being dropped at every page boundary. The page-walking test here needs that fix to pass. Review the last commit only until #4040 lands. ## Example ```ts const names = []; let pageToken = undefined; do { const page = await conn.listTables({ pageToken, limit: 100 }); names.push(...page.tables); pageToken = page.pageToken; } while (pageToken); ``` A namespace can be listed by passing its path first, mirroring `tableNames`: ```ts const page = await conn.listTables(["analytics"], { limit: 100 }); ``` Co-authored-by: Claude Opus 5 (1M context) --- docs/src/js/classes/Connection.md | 74 ++++++++++++++++- docs/src/js/globals.md | 2 + docs/src/js/interfaces/ListTablesOptions.md | 34 ++++++++ docs/src/js/interfaces/ListTablesResponse.md | 23 ++++++ docs/src/js/interfaces/TableNamesOptions.md | 11 ++- nodejs/__test__/connection.test.ts | 69 +++++++++++++++- nodejs/lancedb/connection.ts | 85 ++++++++++++++++++++ nodejs/lancedb/index.ts | 2 + nodejs/src/connection.rs | 34 ++++++++ 9 files changed, 329 insertions(+), 5 deletions(-) create mode 100644 docs/src/js/interfaces/ListTablesOptions.md create mode 100644 docs/src/js/interfaces/ListTablesResponse.md diff --git a/docs/src/js/classes/Connection.md b/docs/src/js/classes/Connection.md index 92cfd2568..1c5abd89f 100644 --- a/docs/src/js/classes/Connection.md +++ b/docs/src/js/classes/Connection.md @@ -584,6 +584,70 @@ Child namespace names and *** +### listTables() + +#### listTables(options) + +```ts +abstract listTables(options?): Promise +``` + +List a page of the tables in this database. + +To retrieve the tables after the page, pass the `pageToken` the response +carries back in. A page can be shorter than `limit` without being the last +one, so walk until a response carries no page token: + +```ts +const names = []; +let pageToken = undefined; +do { + const page = await conn.listTables({ pageToken, limit: 100 }); + names.push(...page.tables); + pageToken = page.pageToken; +} while (pageToken); +``` + +##### Parameters + +* **options?**: `Partial`<[`ListTablesOptions`](../interfaces/ListTablesOptions.md)> + Pagination options + (`pageToken`, `limit`). + +##### Returns + +`Promise`<[`ListTablesResponse`](../interfaces/ListTablesResponse.md)> + +A page of table names and an + optional token for the tables after it. + +#### listTables(namespacePath, options) + +```ts +abstract listTables(namespacePath?, options?): Promise +``` + +List a page of the tables in this database. + +##### Parameters + +* **namespacePath?**: `string`[] + The namespace path to list tables from + (defaults to root namespace) + +* **options?**: `Partial`<[`ListTablesOptions`](../interfaces/ListTablesOptions.md)> + Pagination options + (`pageToken`, `limit`). + +##### Returns + +`Promise`<[`ListTablesResponse`](../interfaces/ListTablesResponse.md)> + +A page of table names and an + optional token for the tables after it. + +*** + ### openMaterializedView() ```ts @@ -660,7 +724,7 @@ a "not supported" error. *** -### tableNames() +### ~~tableNames()~~ #### tableNames(options) @@ -682,6 +746,10 @@ Tables will be returned in lexicographical order. `Promise`<`string`[]> +##### Deprecated + +Use [Connection.listTables](Connection.md#listtables) instead. + #### tableNames(namespacePath, options) ```ts @@ -704,3 +772,7 @@ Tables will be returned in lexicographical order. ##### Returns `Promise`<`string`[]> + +##### Deprecated + +Use [Connection.listTables](Connection.md#listtables) instead. diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 6a8f644eb..e0635ab65 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -100,6 +100,8 @@ - [JobInfo](interfaces/JobInfo.md) - [ListNamespacesOptions](interfaces/ListNamespacesOptions.md) - [ListNamespacesResponse](interfaces/ListNamespacesResponse.md) +- [ListTablesOptions](interfaces/ListTablesOptions.md) +- [ListTablesResponse](interfaces/ListTablesResponse.md) - [LsmStats](interfaces/LsmStats.md) - [LsmWriteSpec](interfaces/LsmWriteSpec.md) - [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md) diff --git a/docs/src/js/interfaces/ListTablesOptions.md b/docs/src/js/interfaces/ListTablesOptions.md new file mode 100644 index 000000000..52ace47e5 --- /dev/null +++ b/docs/src/js/interfaces/ListTablesOptions.md @@ -0,0 +1,34 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / ListTablesOptions + +# Interface: ListTablesOptions + +## Properties + +### limit? + +```ts +optional limit: number; +``` + +An upper bound on how many tables to return. + +A page may hold fewer than this and still not be the last one, so keep +going while the response carries a page token rather than while pages are +full. + +*** + +### pageToken? + +```ts +optional pageToken: string; +``` + +Token from a previous response, to resume listing where it left off. + +The token is opaque: it carries whatever the database needs to resume, and +callers should not construct or interpret one. diff --git a/docs/src/js/interfaces/ListTablesResponse.md b/docs/src/js/interfaces/ListTablesResponse.md new file mode 100644 index 000000000..76cac2b23 --- /dev/null +++ b/docs/src/js/interfaces/ListTablesResponse.md @@ -0,0 +1,23 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / ListTablesResponse + +# Interface: ListTablesResponse + +## Properties + +### pageToken? + +```ts +optional pageToken: string; +``` + +*** + +### tables + +```ts +tables: string[]; +``` diff --git a/docs/src/js/interfaces/TableNamesOptions.md b/docs/src/js/interfaces/TableNamesOptions.md index 45fa3d1b0..9254e9fa7 100644 --- a/docs/src/js/interfaces/TableNamesOptions.md +++ b/docs/src/js/interfaces/TableNamesOptions.md @@ -4,11 +4,16 @@ [@lancedb/lancedb](../globals.md) / TableNamesOptions -# Interface: TableNamesOptions +# Interface: ~~TableNamesOptions~~ + +## Deprecated + +Use [ListTablesOptions](ListTablesOptions.md) with [Connection.listTables](../classes/Connection.md#listtables) +instead. ## Properties -### limit? +### ~~limit?~~ ```ts optional limit: number; @@ -18,7 +23,7 @@ An optional limit to the number of results to return. *** -### startAfter? +### ~~startAfter?~~ ```ts optional startAfter: string; diff --git a/nodejs/__test__/connection.test.ts b/nodejs/__test__/connection.test.ts index af471b478..816f8c3de 100644 --- a/nodejs/__test__/connection.test.ts +++ b/nodejs/__test__/connection.test.ts @@ -4,7 +4,13 @@ import { readdirSync } from "fs"; import { Field, Float64, Schema } from "apache-arrow"; import * as tmp from "tmp"; -import { Connection, Table, connect, connectNamespace } from "../lancedb"; +import { + Connection, + ListTablesResponse, + Table, + connect, + connectNamespace, +} from "../lancedb"; import { LocalTable } from "../lancedb/table"; describe("when connecting", () => { @@ -47,6 +53,7 @@ describe("given a connection", () => { await db.close(); expect(db.isOpen()).toBe(false); await expect(db.tableNames()).rejects.toThrow("Connection is closed"); + await expect(db.listTables()).rejects.toThrow("Connection is closed"); await expect(db.renameTable("a", "b")).rejects.toThrow( "Connection is closed", ); @@ -129,6 +136,66 @@ describe("given a connection", () => { expect(tables).toEqual(["b", "c"]); }); + it("should respect limit and page token when listing tables", async () => { + const db = await connect(tmpDir.name); + + await db.createTable("b", [{ id: 1 }]); + await db.createTable("a", [{ id: 1 }]); + await db.createTable("c", [{ id: 1 }]); + + const all = await db.listTables(); + expect(all.tables).toEqual(["a", "b", "c"]); + expect(all.pageToken).toBeUndefined(); + + const first = await db.listTables({ limit: 1 }); + expect(first.tables).toEqual(["a"]); + expect(first.pageToken).toBeDefined(); + + const second = await db.listTables({ + limit: 1, + pageToken: first.pageToken, + }); + expect(second.tables).toEqual(["b"]); + }); + + it("should visit every table exactly once when walking pages", async () => { + const db = await connect(tmpDir.name); + + const created = ["a", "b", "c", "d", "e"]; + for (const name of created) { + await db.createTable(name, [{ id: 1 }]); + } + + const seen: string[] = []; + let pageToken: string | undefined = undefined; + do { + const page: ListTablesResponse = await db.listTables({ + limit: 2, + pageToken, + }); + seen.push(...page.tables); + pageToken = page.pageToken; + } while (pageToken); + + expect(seen).toEqual(created); + }); + + it("should list tables in a namespace", async () => { + const db = await connect(tmpDir.name, { + // biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust + namespaceClientProperties: { manifest_enabled: "true" }, + }); + await db.createNamespace(["child"]); + await db.createTable("nested", [{ id: 1 }], ["child"]); + + await expect(db.listTables(["child"])).resolves.toEqual( + expect.objectContaining({ tables: ["nested"] }), + ); + await expect(db.listTables()).resolves.toEqual( + expect.objectContaining({ tables: [] }), + ); + }); + it("should create tables in v2 mode", async () => { const db = await connect(tmpDir.name); const data = [...Array(10000).keys()].map((i) => ({ id: i })); diff --git a/nodejs/lancedb/connection.ts b/nodejs/lancedb/connection.ts index e5528f8c6..263a338ab 100644 --- a/nodejs/lancedb/connection.ts +++ b/nodejs/lancedb/connection.ts @@ -31,12 +31,14 @@ import type { JobDescription, JobInfo, ListNamespacesResponse, + ListTablesResponse, } from "./native"; export type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, ListNamespacesResponse, + ListTablesResponse, }; import { sanitizeTable } from "./sanitize"; import { LocalTable, Table } from "./table"; @@ -134,6 +136,10 @@ export interface OpenTableOptions { indexCacheSize?: number; } +/** + * @deprecated Use {@link ListTablesOptions} with {@link Connection.listTables} + * instead. + */ export interface TableNamesOptions { /** * If present, only return names that come lexicographically after the @@ -147,6 +153,24 @@ export interface TableNamesOptions { limit?: number; } +export interface ListTablesOptions { + /** + * Token from a previous response, to resume listing where it left off. + * + * The token is opaque: it carries whatever the database needs to resume, and + * callers should not construct or interpret one. + */ + pageToken?: string; + /** + * An upper bound on how many tables to return. + * + * A page may hold fewer than this and still not be the last one, so keep + * going while the response carries a page token rather than while pages are + * full. + */ + limit?: number; +} + export interface ListNamespacesOptions { /** Token from a previous response for pagination. */ pageToken?: string; @@ -231,6 +255,7 @@ export abstract class Connection { * @param {Partial} options - options to control the * paging / start point (backwards compatibility) * + * @deprecated Use {@link Connection.listTables} instead. */ abstract tableNames(options?: Partial): Promise; /** @@ -241,12 +266,53 @@ export abstract class Connection { * @param {Partial} options - options to control the * paging / start point * + * @deprecated Use {@link Connection.listTables} instead. */ abstract tableNames( namespacePath?: string[], options?: Partial, ): Promise; + /** + * List a page of the tables in this database. + * + * To retrieve the tables after the page, pass the `pageToken` the response + * carries back in. A page can be shorter than `limit` without being the last + * one, so walk until a response carries no page token: + * + * ```ts + * const names = []; + * let pageToken = undefined; + * do { + * const page = await conn.listTables({ pageToken, limit: 100 }); + * names.push(...page.tables); + * pageToken = page.pageToken; + * } while (pageToken); + * ``` + * + * @param {Partial} options - Pagination options + * (`pageToken`, `limit`). + * @returns {Promise} A page of table names and an + * optional token for the tables after it. + */ + abstract listTables( + options?: Partial, + ): Promise; + /** + * List a page of the tables in this database. + * + * @param {string[]} namespacePath - The namespace path to list tables from + * (defaults to root namespace) + * @param {Partial} options - Pagination options + * (`pageToken`, `limit`). + * @returns {Promise} A page of table names and an + * optional token for the tables after it. + */ + abstract listTables( + namespacePath?: string[], + options?: Partial, + ): Promise; + /** * Open a table in the database. * @param {string} name - The name of the table @@ -601,6 +667,25 @@ export class LocalConnection extends Connection { return await this.inner.listMaterializedViews(); } + async listTables( + namespacePathOrOptions?: string[] | Partial, + options?: Partial, + ): Promise { + // Detect if first argument is namespacePath array or options object + const namespacePath = Array.isArray(namespacePathOrOptions) + ? namespacePathOrOptions + : undefined; + const listTablesOptions = Array.isArray(namespacePathOrOptions) + ? options + : namespacePathOrOptions; + + return this.inner.listTables( + namespacePath ?? [], + listTablesOptions?.pageToken, + listTablesOptions?.limit, + ); + } + async openTable( name: string, namespacePath?: string[], diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index da13b434f..ebc8cda8d 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -81,11 +81,13 @@ export { Connection, CreateTableOptions, TableNamesOptions, + ListTablesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, + ListTablesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, diff --git a/nodejs/src/connection.rs b/nodejs/src/connection.rs index 44eb68f32..5cf676256 100644 --- a/nodejs/src/connection.rs +++ b/nodejs/src/connection.rs @@ -17,6 +17,7 @@ use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, conne use lance_namespace::models::{ CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest, + ListTablesRequest, }; use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema}; @@ -36,6 +37,12 @@ pub struct ListNamespacesResponse { pub page_token: Option, } +#[napi(object)] +pub struct ListTablesResponse { + pub tables: Vec, + pub page_token: Option, +} + #[napi(object)] pub struct CreateNamespaceResponse { pub properties: Option>, @@ -206,6 +213,33 @@ impl Connection { op.execute().await.default_error() } + /// List a page of tables in the database. + #[napi(catch_unwind)] + pub async fn list_tables( + &self, + namespace_path: Option>, + page_token: Option, + limit: Option, + ) -> napi::Result { + let request = ListTablesRequest { + // The root namespace is an empty path, not an absent one: a namespace-backed + // database rejects a request that names no namespace. + id: Some(namespace_path.unwrap_or_default()), + page_token, + limit: limit.map(|limit| i32::try_from(limit).unwrap_or(i32::MAX)), + ..Default::default() + }; + let response = self + .get_inner()? + .list_tables(request) + .await + .default_error()?; + Ok(ListTablesResponse { + tables: response.tables, + page_token: response.page_token, + }) + } + /// Create table from a Apache Arrow IPC (file) buffer. /// /// Parameters: From c1a8c3f089fbab95883bd58d9a568d4c30d41074 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:41:05 -0700 Subject: [PATCH 28/31] fix(node): validate inferred types across records (#3786) ## Summary - compare inferred Arrow types by their semantic representation across records - throw the schema inference error when a later record has an incompatible type - cover compatible and incompatible multi-record inference across supported Arrow versions ## Root cause Schema inference compared newly allocated Arrow DataType objects by identity, so equivalent inferred types did not compare equal. The mismatch path also constructed an Error without throwing it, which silently accepted incompatible values. ## Validation - pnpm test __test__/arrow.test.ts --runInBand (176 tests passed) - pnpm lint - pnpm build - pnpm run docs Fixes #3781 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/arrow.test.ts | 131 ++++++++ nodejs/lancedb/arrow.ts | 340 ++------------------ nodejs/lancedb/arrow_type.ts | 40 +++ nodejs/lancedb/schema.ts | 566 ++++++++++++++++++++++++++++++++++ 4 files changed, 770 insertions(+), 307 deletions(-) create mode 100644 nodejs/lancedb/arrow_type.ts create mode 100644 nodejs/lancedb/schema.ts diff --git a/nodejs/__test__/arrow.test.ts b/nodejs/__test__/arrow.test.ts index 160f4d5ef..c5bbbf169 100644 --- a/nodejs/__test__/arrow.test.ts +++ b/nodejs/__test__/arrow.test.ts @@ -515,6 +515,137 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( ); }); + it("will allow matching inferred types across records", function () { + expect(() => + makeArrowTable([{ value: 1 }, { value: 2 }]), + ).not.toThrow(); + }); + + it("will reject mismatched inferred types across records", function () { + expect(() => makeArrowTable([{ value: 1 }, { value: "two" }])).toThrow( + "Failed to infer schema for data. Previously inferred type Float64 but found Utf8 for field value at row 1. Consider providing an explicit schema.", + ); + }); + + it("will ignore generated dictionary IDs when comparing inferred types", function () { + const table = makeArrowTable([{ str: "a" }, { str: "b" }], { + dictionaryEncodeStrings: true, + }); + + expect(table.getChild("str")?.toJSON()).toEqual(["a", "b"]); + }); + + it("will preserve null values without treating them as type mismatches", function () { + for (const records of [ + [{ vector: [1, 2, 3] }, { vector: null }], + [{ vector: null }, { vector: [1, 2, 3] }], + ]) { + const table = makeArrowTable(records); + + expect(table.numRows).toBe(2); + expect(table.getChild("vector")?.nullCount).toBe(1); + } + }); + + it("will preserve empty variable-size lists", function () { + for (const records of [ + [{ items: [1] }, { items: [] }], + [{ items: [] }, { items: [1] }], + ]) { + const table = makeArrowTable(records); + expect( + table + .getChild("items") + ?.toJSON() + .map((value) => value.toJSON()), + ).toEqual(records.map((record) => record.items)); + } + }); + + it("will propagate deferred evidence through nested lists", function () { + for (const records of [ + [{ items: [1] }, { items: [null] }], + [{ items: [null] }, { items: [1] }], + [{ items: [null, 1] }, { items: [2, null] }], + ]) { + const table = makeArrowTable(records); + expect( + table + .getChild("items") + ?.toJSON() + .map((value) => value.toJSON()), + ).toEqual(records.map((record) => record.items)); + } + + const nestedRecords = [{ items: [[1]] }, { items: [[null]] }]; + const nestedTable = makeArrowTable(nestedRecords); + expect( + nestedTable + .getChild("items") + ?.toJSON() + .map((value) => + value + .toJSON() + .map((nestedValue: { toJSON: () => unknown[] }) => + nestedValue.toJSON(), + ), + ), + ).toEqual(nestedRecords.map((record) => record.items)); + }); + + it("will reject incompatible deferred evidence within a list", function () { + for (const items of [ + [[], 1], + [1, []], + [[null], 1], + [1, [null]], + ]) { + expect(() => makeArrowTable([{ items }])).toThrow( + "Failed to infer data type for field items at row 0.", + ); + } + }); + + it("will reject empty fixed-size lists", function () { + expect(() => + makeArrowTable([{ vector: [1, 2, 3] }, { vector: [] }]), + ).toThrow( + "Failed to infer schema for data. Previously inferred type FixedSizeList[3] but found List[0] for field vector at row 1.", + ); + }); + + it("will reject inferred leaf and branch shape changes", function () { + expect(() => + makeArrowTable([{ value: 1 }, { value: { nested: 2 } }]), + ).toThrow( + "Failed to infer schema for data. Previously inferred type Float64 but found Struct for field value at row 1.", + ); + expect(() => + makeArrowTable([{ value: { nested: 1 } }, { value: 2 }]), + ).toThrow( + "Failed to infer schema for data. Previously inferred type Struct but found Float64 for field value at row 1.", + ); + }); + + it("will allow null values around inferred struct values", function () { + for (const { records, nullIndex } of [ + { + records: [{ value: null }, { value: { nested: 2 } }], + nullIndex: 0, + }, + { + records: [{ value: { nested: 1 } }, { value: null }], + nullIndex: 1, + }, + ]) { + const table = makeArrowTable(records); + const values = table.getChild("value"); + + expect(values?.nullCount).toBe(1); + expect(values?.get(nullIndex)).toBeNull(); + } + }); + it("will allow a schema to be provided", async function () { await checkTableCreation( async (records, _, schema) => diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts index 8b388d593..b52ab50ef 100644 --- a/nodejs/lancedb/arrow.ts +++ b/nodejs/lancedb/arrow.ts @@ -5,7 +5,6 @@ import { Data as ArrowData, Table as ArrowTable, Binary, - Bool, BufferType, DataType, DateUnit, @@ -18,12 +17,7 @@ import { FixedSizeList, Float, Float32, - Float64, Int, - Int8, - Int16, - Int32, - Int64, LargeBinary, List, Null, @@ -36,17 +30,16 @@ import { Struct, Timestamp, Type, - Uint8, - Uint16, - Uint32, Utf8, Vector, makeVector as arrowMakeVector, + util as arrowUtil, vectorFromArray as badVectorFromArray, makeBuilder, makeData, } from "apache-arrow"; import { Buffers } from "apache-arrow/data"; +import { typedArrayToArrowType } from "./arrow_type"; import { type EmbeddingFunction } from "./embedding/embedding_function"; import { EmbeddingFunctionConfig, @@ -59,14 +52,7 @@ import { sanitizeTable, sanitizeType, } from "./sanitize"; - -/** - * Check if a field name indicates a vector column. - */ -function nameSuggestsVectorColumn(fieldName: string): boolean { - const nameLower = fieldName.toLowerCase(); - return nameLower.includes("vector") || nameLower.includes("embedding"); -} +import { inferSchema } from "./schema"; export * from "apache-arrow"; export type SchemaLike = @@ -459,110 +445,6 @@ export function makeArrowTable( return new ArrowTable(inferredSchema, finalColumns); } -function inferSchema( - data: Array>, - schema: Schema | undefined, - opts: MakeArrowTableOptions, -): Schema { - // We will collect all fields we see in the data. - const pathTree = new PathTree(); - - for (const [rowI, row] of data.entries()) { - for (const [path, value] of rowPathsAndValues(row)) { - if (!pathTree.has(path)) { - // First time seeing this field. - if (schema !== undefined) { - const field = getFieldForPath(schema, path); - if (field === undefined) { - throw new Error( - `Found field not in schema: ${path.join(".")} at row ${rowI}`, - ); - } else { - pathTree.set(path, field.type); - } - } else { - const inferredType = inferType(value, path, opts); - if (inferredType === undefined) { - throw new Error(`Failed to infer data type for field ${path.join( - ".", - )} at row ${rowI}. \ - Consider providing an explicit schema.`); - } - pathTree.set(path, inferredType); - } - } else if (schema === undefined) { - const currentType = pathTree.get(path); - const newType = inferType(value, path, opts); - if (currentType !== newType) { - new Error(`Failed to infer schema for data. Previously inferred type \ - ${currentType} but found ${newType} at row ${rowI}. Consider \ - providing an explicit schema.`); - } - } - } - } - - if (schema === undefined) { - function fieldsFromPathTree(pathTree: PathTree): Field[] { - const fields = []; - for (const [name, value] of pathTree.map.entries()) { - if (value instanceof PathTree) { - const children = fieldsFromPathTree(value); - fields.push(new Field(name, new Struct(children), true)); - } else { - fields.push(new Field(name, value, true)); - } - } - return fields; - } - const fields = fieldsFromPathTree(pathTree); - return new Schema(fields); - } else { - function takeMatchingFields( - fields: Field[], - pathTree: PathTree, - ): Field[] { - const outFields = []; - for (const field of fields) { - if (pathTree.map.has(field.name)) { - const value = pathTree.get([field.name]); - if (value instanceof PathTree) { - const struct = field.type as Struct; - const children = takeMatchingFields(struct.children, value); - outFields.push( - new Field(field.name, new Struct(children), field.nullable), - ); - } else { - outFields.push( - new Field(field.name, value as DataType, field.nullable), - ); - } - } - } - return outFields; - } - const fields = takeMatchingFields(schema.fields, pathTree); - return new Schema(fields); - } -} - -function* rowPathsAndValues( - row: Record, - basePath: string[] = [], -): Generator<[string[], unknown]> { - for (const [key, value] of Object.entries(row)) { - if (isObject(value)) { - yield* rowPathsAndValues(value, [...basePath, key]); - } else { - // Skip undefined values - they should be treated the same as missing fields - // for embedding function purposes - if (value !== undefined) { - yield [[...basePath, key], value]; - } - } - } -} - function isObject(value: unknown): value is Record { return ( typeof value === "object" && @@ -577,146 +459,19 @@ function isObject(value: unknown): value is Record { ); } -function getFieldForPath(schema: Schema, path: string[]): Field | undefined { - let current: Field | Schema = schema; +function valueAtPath(datum: Record, path: string[]): unknown { + let current: unknown = datum; for (const key of path) { - if (current instanceof Schema) { - const field: Field | undefined = current.fields.find( - (f) => f.name === key, - ); - if (field === undefined) { - return undefined; - } - current = field; - } else if (current instanceof Field && DataType.isStruct(current.type)) { - const struct: Struct = current.type; - const field = struct.children.find((f) => f.name === key); - if (field === undefined) { - return undefined; - } - current = field; + if (current == null) { + return null; + } + if (isObject(current) && (Object.hasOwn(current, key) || key in current)) { + current = current[key]; } else { return undefined; } } - if (current instanceof Field) { - return current; - } else { - return undefined; - } -} - -/** - * Try to infer which Arrow type to use for a given value. - * - * May return undefined if the type cannot be inferred. - */ -function inferType( - value: unknown, - path: string[], - opts: MakeArrowTableOptions, -): DataType | undefined { - if (typeof value === "bigint") { - return new Int64(); - } else if (typeof value === "number") { - // Even if it's an integer, it's safer to assume Float64. Users can - // always provide an explicit schema or use BigInt if they mean integer. - return new Float64(); - } else if (typeof value === "string") { - if (opts.dictionaryEncodeStrings) { - return new Dictionary(new Utf8(), new Int32()); - } else { - return new Utf8(); - } - } else if (typeof value === "boolean") { - return new Bool(); - } else if (value instanceof Buffer) { - return new Binary(); - } else if (ArrayBuffer.isView(value) && !(value instanceof DataView)) { - const info = typedArrayToArrowType(value); - if (info !== undefined) { - const child = new Field("item", info.elementType, true); - return new FixedSizeList(info.length, child); - } - return undefined; - } else if (Array.isArray(value)) { - if (value.length === 0) { - return undefined; // Without any values we can't infer the type - } - if (path.length === 1 && Object.hasOwn(opts.vectorColumns, path[0])) { - const floatType = sanitizeType(opts.vectorColumns[path[0]].type); - return new FixedSizeList( - value.length, - new Field("item", floatType, true), - ); - } - const valueType = inferType(value[0], path, opts); - if (valueType === undefined) { - return undefined; - } - // Try to automatically detect embedding columns. - if (nameSuggestsVectorColumn(path[path.length - 1])) { - // Check if value is a Uint8Array for integer vector type determination - if (value instanceof Uint8Array) { - // For integer vectors, we default to Uint8 (matching Python implementation) - const child = new Field("item", new Uint8(), true); - return new FixedSizeList(value.length, child); - } else { - // For float vectors, we default to Float32 - const child = new Field("item", new Float32(), true); - return new FixedSizeList(value.length, child); - } - } else { - const child = new Field("item", valueType, true); - return new List(child); - } - } else { - // TODO: timestamp - return undefined; - } -} - -class PathTree { - map: Map>; - - constructor(entries?: [string[], V][]) { - this.map = new Map(); - if (entries !== undefined) { - for (const [path, value] of entries) { - this.set(path, value); - } - } - } - has(path: string[]): boolean { - let ref: PathTree = this; - for (const part of path) { - if (!(ref instanceof PathTree) || !ref.map.has(part)) { - return false; - } - ref = ref.map.get(part) as PathTree; - } - return true; - } - get(path: string[]): V | undefined { - let ref: PathTree = this; - for (const part of path) { - if (!(ref instanceof PathTree) || !ref.map.has(part)) { - return undefined; - } - ref = ref.map.get(part) as PathTree; - } - return ref as V; - } - set(path: string[], value: V): void { - let ref: PathTree = this; - for (const part of path.slice(0, path.length - 1)) { - if (!ref.map.has(part)) { - ref.map.set(part, new PathTree()); - } - ref = ref.map.get(part) as PathTree; - } - ref.map.set(path[path.length - 1], value); - } + return current; } function transposeData( @@ -724,37 +479,26 @@ function transposeData( field: Field, path: string[] = [], ): Vector { + const valuesPath = [...path, field.name]; + const values = data.map((datum) => valueAtPath(datum, valuesPath)); if (field.type instanceof Struct) { const childFields = field.type.children; - const fullPath = [...path, field.name]; const childVectors = childFields.map((child) => { - return transposeData(data, child, fullPath); + return transposeData(data, child, valuesPath); }); + const nullCount = values.filter((value) => value === null).length; const structData = makeData({ type: field.type, + length: values.length, + nullCount, + nullBitmap: + nullCount > 0 + ? arrowUtil.packBools(values.map((value) => value !== null)) + : undefined, children: childVectors as unknown as ArrowData[], }); return arrowMakeVector(structData); } else { - const valuesPath = [...path, field.name]; - const values = data.map((datum) => { - let current: unknown = datum; - for (const key of valuesPath) { - if (current == null) { - return null; - } - - if ( - isObject(current) && - (Object.hasOwn(current, key) || key in current) - ) { - current = current[key]; - } else { - return null; - } - } - return current; - }); return makeVector(values, field.type, undefined, field.nullable); } } @@ -797,32 +541,6 @@ function makeListVector(lists: unknown[][]): Vector { return listBuilder.finish().toVector(); } -/** - * Map a JS TypedArray instance to the corresponding Arrow element DataType - * and its length. Returns undefined if the value is not a recognized TypedArray. - */ -function typedArrayToArrowType( - value: ArrayBufferView, -): { elementType: DataType; length: number } | undefined { - if (value instanceof Float32Array) - return { elementType: new Float32(), length: value.length }; - if (value instanceof Float64Array) - return { elementType: new Float64(), length: value.length }; - if (value instanceof Uint8Array) - return { elementType: new Uint8(), length: value.length }; - if (value instanceof Uint16Array) - return { elementType: new Uint16(), length: value.length }; - if (value instanceof Uint32Array) - return { elementType: new Uint32(), length: value.length }; - if (value instanceof Int8Array) - return { elementType: new Int8(), length: value.length }; - if (value instanceof Int16Array) - return { elementType: new Int16(), length: value.length }; - if (value instanceof Int32Array) - return { elementType: new Int32(), length: value.length }; - return undefined; -} - /** Helper function to convert an Array of JS values to an Arrow Vector */ function makeVector( values: unknown[], @@ -1462,8 +1180,12 @@ export function ensureNestedFieldsExist( completeRow[field.name] = row[field.name]; } } else { - // Field is missing from the data - set to null - completeRow[field.name] = null; + // Keep a missing struct valid while filling each of its children with + // null. This is distinct from an explicitly null struct value. + completeRow[field.name] = + field.type.constructor.name === "Struct" + ? ensureStructFieldsExist({}, field.type as Struct) + : null; } } @@ -1498,8 +1220,12 @@ function ensureStructFieldsExist( completeStruct[childField.name] = data[childField.name]; } } else { - // Field is missing - set to null - completeStruct[childField.name] = null; + // Keep a missing struct valid while filling each of its children with + // null. This is distinct from an explicitly null struct value. + completeStruct[childField.name] = + childField.type.constructor.name === "Struct" + ? ensureStructFieldsExist({}, childField.type as Struct) + : null; } } diff --git a/nodejs/lancedb/arrow_type.ts b/nodejs/lancedb/arrow_type.ts new file mode 100644 index 000000000..35da346ef --- /dev/null +++ b/nodejs/lancedb/arrow_type.ts @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { + type DataType, + Float32, + Float64, + Int8, + Int16, + Int32, + Uint8, + Uint16, + Uint32, +} from "apache-arrow"; + +/** + * Map a JS TypedArray instance to the corresponding Arrow element type and + * length. Returns undefined when the view is not a supported TypedArray. + */ +export function typedArrayToArrowType( + value: ArrayBufferView, +): { elementType: DataType; length: number } | undefined { + if (value instanceof Float32Array) + return { elementType: new Float32(), length: value.length }; + if (value instanceof Float64Array) + return { elementType: new Float64(), length: value.length }; + if (value instanceof Uint8Array) + return { elementType: new Uint8(), length: value.length }; + if (value instanceof Uint16Array) + return { elementType: new Uint16(), length: value.length }; + if (value instanceof Uint32Array) + return { elementType: new Uint32(), length: value.length }; + if (value instanceof Int8Array) + return { elementType: new Int8(), length: value.length }; + if (value instanceof Int16Array) + return { elementType: new Int16(), length: value.length }; + if (value instanceof Int32Array) + return { elementType: new Int32(), length: value.length }; + return undefined; +} diff --git a/nodejs/lancedb/schema.ts b/nodejs/lancedb/schema.ts new file mode 100644 index 000000000..e4749ef37 --- /dev/null +++ b/nodejs/lancedb/schema.ts @@ -0,0 +1,566 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { + Binary, + Bool, + DataType, + Dictionary, + Field, + FixedSizeList, + Float32, + Float64, + Int32, + Int64, + List, + Schema, + Struct, + Utf8, + util as arrowUtil, +} from "apache-arrow"; +import { typedArrayToArrowType } from "./arrow_type"; +import { sanitizeType } from "./sanitize"; + +type InferenceOptions = { + dictionaryEncodeStrings: boolean; + vectorColumns: Record; +}; + +/** + * Infer the Arrow schema represented by a set of records. + * + * This is the intentionally small interface to schema inference. The stateful + * details of combining partial type evidence are encapsulated below so callers + * only need to provide records, an optional schema, and inference options. + */ +export function inferSchema( + data: Array>, + schema: Schema | undefined, + options: InferenceOptions, +): Schema { + return new SchemaInferrer(schema, options).infer(data); +} + +class SchemaInferrer { + private readonly fields = new FieldTree(); + + constructor( + private readonly providedSchema: Schema | undefined, + private readonly options: InferenceOptions, + ) {} + + infer(data: Array>): Schema { + for (const [row, record] of data.entries()) { + for (const [path, value] of recordPathsAndValues(record)) { + this.observe(path, value, row); + } + } + + return this.providedSchema === undefined + ? new Schema(fieldsFromTree(this.fields)) + : new Schema(matchingFields(this.providedSchema.fields, this.fields)); + } + + private observe(path: string[], value: unknown, row: number): void { + const current = this.fields.get(path); + if (current === undefined) { + this.addField(path, value, row); + } else if (this.providedSchema === undefined) { + this.updateInferredField(path, value, row, current); + } + } + + private addField(path: string[], value: unknown, row: number): void { + if (this.providedSchema !== undefined) { + this.addSchemaField(this.providedSchema, path, row); + return; + } + + const evidence = + this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row); + if (evidence === undefined) { + throw typeInferenceError(path, row); + } + + const conflict = this.fields.set( + path, + evidence, + (existing) => + existing instanceof DeferredTypeEvidence && existing.isOnlyNulls(), + ); + if (conflict !== undefined) { + throw branchConflictError(conflict, row, "Struct"); + } + } + + private addSchemaField(schema: Schema, path: string[], row: number): void { + const field = fieldAtPath(schema, path); + if (field === undefined) { + throw new Error( + `Found field not in schema: ${path.join(".")} at row ${row}`, + ); + } + + const conflict = this.fields.set(path, field.type); + if (conflict !== undefined) { + throw branchConflictError(conflict, row, "Struct"); + } + } + + private updateInferredField( + path: string[], + value: unknown, + row: number, + current: FieldNode, + ): void { + const newType = this.inferType(value, path); + const deferred = DeferredTypeEvidence.from(value, row); + + if (current instanceof FieldTree) { + if (deferred?.isOnlyNulls()) { + return; + } + throw schemaInferenceError( + path, + row, + "Struct", + describeEvidence(newType ?? deferred), + ); + } + + if (current instanceof DeferredTypeEvidence) { + this.resolveDeferredField(path, row, current, newType, deferred); + return; + } + + if (newType !== undefined) { + if (!inferredTypesEqual(current, newType)) { + throw schemaInferenceError( + path, + row, + describeEvidence(current), + describeEvidence(newType), + ); + } + return; + } + + if (deferred === undefined || !deferred.matches(current)) { + throw schemaInferenceError( + path, + row, + describeEvidence(current), + describeEvidence(deferred), + ); + } + } + + private resolveDeferredField( + path: string[], + row: number, + current: DeferredTypeEvidence, + newType: DataType | undefined, + deferred: DeferredTypeEvidence | undefined, + ): void { + if (newType !== undefined) { + if (!current.matches(newType)) { + throw schemaInferenceError( + path, + row, + current.describe(), + describeEvidence(newType), + ); + } + this.fields.set(path, newType); + return; + } + + if (deferred !== undefined) { + this.fields.set(path, current.merge(deferred)); + return; + } + + throw schemaInferenceError( + path, + row, + current.describe(), + describeEvidence(newType), + ); + } + + private inferType(value: unknown, path: string[]): DataType | undefined { + if (typeof value === "bigint") { + return new Int64(); + } + if (typeof value === "number") { + return new Float64(); + } + if (typeof value === "string") { + return this.options.dictionaryEncodeStrings + ? new Dictionary(new Utf8(), new Int32()) + : new Utf8(); + } + if (typeof value === "boolean") { + return new Bool(); + } + if (value instanceof Buffer) { + return new Binary(); + } + if (ArrayBuffer.isView(value) && !(value instanceof DataView)) { + const typedArray = typedArrayToArrowType(value); + return typedArray === undefined + ? undefined + : new FixedSizeList( + typedArray.length, + new Field("item", typedArray.elementType, true), + ); + } + if (!Array.isArray(value) || value.length === 0) { + return undefined; + } + + const configuredVector = + path.length === 1 ? this.options.vectorColumns[path[0]] : undefined; + if (configuredVector !== undefined) { + return new FixedSizeList( + value.length, + new Field("item", sanitizeType(configuredVector.type), true), + ); + } + + const itemType = this.inferArrayItemType(value, path); + if (itemType === undefined) { + return undefined; + } + + return nameSuggestsVectorColumn(path[path.length - 1]) + ? new FixedSizeList(value.length, new Field("item", new Float32(), true)) + : new List(new Field("item", itemType, true)); + } + + private inferArrayItemType( + values: unknown[], + path: string[], + ): DataType | undefined { + let itemType: DataType | undefined; + const deferredItems: unknown[] = []; + + for (const value of values) { + const candidate = this.inferType(value, path); + if (candidate === undefined) { + if (!isDeferredValue(value)) { + return undefined; + } + deferredItems.push(value); + } else if (itemType === undefined) { + itemType = candidate; + } else if (!inferredTypesEqual(itemType, candidate)) { + return undefined; + } + } + + if (itemType === undefined) { + return undefined; + } + return deferredItems.every((value) => + deferredValueMatchesType(value, itemType), + ) + ? itemType + : undefined; + } +} + +/** Nulls and empty/all-null lists that do not determine a type by themselves. */ +class DeferredTypeEvidence { + private constructor( + private readonly values: Array<{ value: unknown; row: number }>, + ) {} + + static from(value: unknown, row: number): DeferredTypeEvidence | undefined { + return isDeferredValue(value) + ? new DeferredTypeEvidence([{ value, row }]) + : undefined; + } + + isOnlyNulls(): boolean { + return this.values.every(({ value }) => value == null); + } + + matches(type: DataType): boolean { + return this.values.every(({ value }) => + deferredValueMatchesType(value, type), + ); + } + + merge(other: DeferredTypeEvidence): DeferredTypeEvidence { + return new DeferredTypeEvidence([...this.values, ...other.values]); + } + + describe(): string { + const list = this.values.find(({ value }) => Array.isArray(value)); + return list === undefined + ? "null" + : `List[${(list.value as unknown[]).length}]`; + } + + firstRow(): number { + return this.values[0].row; + } +} + +type FieldNode = DataType | DeferredTypeEvidence | FieldTree; +type LeafNode = Exclude; +type FieldConflict = { path: string[]; value: FieldNode }; + +/** Nested field state, kept separate from Arrow's eventual Struct types. */ +class FieldTree { + private readonly children = new Map(); + + get(path: string[]): FieldNode | undefined { + let current: FieldNode = this; + for (const part of path) { + if (!(current instanceof FieldTree)) { + return undefined; + } + const child = current.children.get(part); + if (child === undefined) { + return undefined; + } + current = child; + } + return current; + } + + set( + path: string[], + value: LeafNode, + canReplaceLeaf: (value: LeafNode) => boolean = () => false, + ): FieldConflict | undefined { + let branch: FieldTree = this; + for (const [index, part] of path.slice(0, -1).entries()) { + const child = branch.children.get(part); + if (child === undefined || (isLeaf(child) && canReplaceLeaf(child))) { + const nextBranch = new FieldTree(); + branch.children.set(part, nextBranch); + branch = nextBranch; + } else if (child instanceof FieldTree) { + branch = child; + } else { + return { path: path.slice(0, index + 1), value: child }; + } + } + + const name = path[path.length - 1]; + const current = branch.children.get(name); + if (current instanceof FieldTree) { + return { path, value: current }; + } + branch.children.set(name, value); + return undefined; + } + + entries(): IterableIterator<[string, FieldNode]> { + return this.children.entries(); + } + + has(name: string): boolean { + return this.children.has(name); + } +} + +function isLeaf(value: FieldNode): value is LeafNode { + return !(value instanceof FieldTree); +} + +function fieldsFromTree(tree: FieldTree, path: string[] = []): Field[] { + const fields: Field[] = []; + for (const [name, value] of tree.entries()) { + if (value instanceof FieldTree) { + fields.push( + new Field( + name, + new Struct(fieldsFromTree(value, [...path, name])), + true, + ), + ); + } else if (value instanceof DeferredTypeEvidence) { + throw typeInferenceError([...path, name], value.firstRow()); + } else { + fields.push(new Field(name, value, true)); + } + } + return fields; +} + +function matchingFields(fields: Field[], tree: FieldTree): Field[] { + const matches: Field[] = []; + for (const field of fields) { + if (!tree.has(field.name)) { + continue; + } + const value = tree.get([field.name]); + if (value instanceof FieldTree) { + const struct = field.type as Struct; + matches.push( + new Field( + field.name, + new Struct(matchingFields(struct.children, value)), + field.nullable, + ), + ); + } else { + matches.push(new Field(field.name, value as DataType, field.nullable)); + } + } + return matches; +} + +function* recordPathsAndValues( + record: Record, + path: string[] = [], +): Generator<[string[], unknown]> { + for (const [name, value] of Object.entries(record)) { + if (isRecord(value)) { + yield* recordPathsAndValues(value, [...path, name]); + } else if (value !== undefined) { + yield [[...path, name], value]; + } + } +} + +function isRecord(value: unknown): value is Record { + return ( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + !(value instanceof RegExp) && + !(value instanceof Date) && + !(value instanceof Set) && + !(value instanceof Map) && + !(value instanceof Buffer) && + !ArrayBuffer.isView(value) + ); +} + +function fieldAtPath(schema: Schema, path: string[]): Field | undefined { + let fields = schema.fields; + let field: Field | undefined; + for (const [index, name] of path.entries()) { + field = fields.find((candidate) => candidate.name === name); + if (field === undefined || index === path.length - 1) { + return field; + } + if (!DataType.isStruct(field.type)) { + return undefined; + } + fields = field.type.children; + } + return field; +} + +function isDeferredValue(value: unknown): boolean { + return ( + value == null || (Array.isArray(value) && value.every(isDeferredValue)) + ); +} + +function deferredValueMatchesType(value: unknown, type: DataType): boolean { + if (value == null) { + return true; + } + if (!Array.isArray(value)) { + return false; + } + if (DataType.isList(type)) { + return value.every((item) => + deferredValueMatchesType(item, type.valueType), + ); + } + if (DataType.isFixedSizeList(type)) { + return ( + value.length === type.listSize && + value.every((item) => deferredValueMatchesType(item, type.valueType)) + ); + } + return false; +} + +function inferredTypesEqual(current: DataType, candidate: DataType): boolean { + if (DataType.isDictionary(current)) { + return ( + DataType.isDictionary(candidate) && + current.isOrdered === candidate.isOrdered && + inferredTypesEqual(current.indices, candidate.indices) && + inferredTypesEqual(current.dictionary, candidate.dictionary) + ); + } + if (DataType.isList(current)) { + return ( + DataType.isList(candidate) && + current.valueField.name === candidate.valueField.name && + current.valueField.nullable === candidate.valueField.nullable && + inferredTypesEqual(current.valueType, candidate.valueType) + ); + } + if (DataType.isFixedSizeList(current)) { + return ( + DataType.isFixedSizeList(candidate) && + current.listSize === candidate.listSize && + current.valueField.name === candidate.valueField.name && + current.valueField.nullable === candidate.valueField.nullable && + inferredTypesEqual(current.valueType, candidate.valueType) + ); + } + return arrowUtil.compareTypes(current, candidate); +} + +function describeEvidence( + evidence: DataType | DeferredTypeEvidence | undefined, +): string { + if (evidence === undefined) { + return "an unsupported value"; + } + return evidence instanceof DeferredTypeEvidence + ? evidence.describe() + : evidence.toString(); +} + +function branchConflictError( + conflict: FieldConflict, + row: number, + candidate: string, +): Error { + return schemaInferenceError( + conflict.path, + row, + conflict.value instanceof FieldTree + ? "Struct" + : describeEvidence(conflict.value), + candidate, + ); +} + +function schemaInferenceError( + path: string[], + row: number, + currentType: string, + newType: string, +): Error { + return new Error( + `Failed to infer schema for data. Previously inferred type ${currentType} ` + + `but found ${newType} for field ${path.join(".")} at row ${row}. ` + + "Consider providing an explicit schema.", + ); +} + +function typeInferenceError(path: string[], row: number): Error { + return new Error( + `Failed to infer data type for field ${path.join(".")} at row ${row}. ` + + "Consider providing an explicit schema.", + ); +} + +function nameSuggestsVectorColumn(name: string): boolean { + const normalized = name.toLowerCase(); + return normalized.includes("vector") || normalized.includes("embedding"); +} From 6ed3074d4cf98b08bec11269cd2c1fba5bf8f7d3 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Mon, 24 Aug 2026 17:16:42 -0700 Subject: [PATCH 29/31] feat: pin the base table version for data loader reads (#3982) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A permutation stores `_rowid`s, which are row addresses unless stable row ids are enabled. Nothing in the data loader pinned a table version, so a compaction between building a permutation and reading it can resolve those ids to different rows. The exposure differs by backend but exists on both: - Remote never pins. `prepare_query_bodies` stamps `"version": current_version()` on every request, but `current_version()` is `None` unless `checkout` was called, so every request means "latest". - Native pins implicitly by holding an `Arc` under `ConsistencyMode::Lazy`, but `StreamingDataset.__setstate__` reopens the table in each DataLoader worker, so each worker pins to whatever is latest at fork time. ## Changes `Table::at_version` returns an independent handle pinned to a version without mutating the receiver. `checkout` cannot serve this: on remote the version cell is an `Arc>>` shared across clones, so pinning through it would silently pin the caller's table too. `PermutationBuilder::build` pins for the whole build and records the version in the permutation table's schema metadata, alongside the existing split names. `PermutationReader` pins the base table to that version before any take. Because the reader pins on construction, the Python worker fork is covered without touching the pickle format — `Permutation.__setstate__` drops the reader and `_ensure_open` rebuilds it, which re-pins. ## Behaviour change A permutation is now bound to the version it was built against, so rows appended to the base table afterwards are not visible through an existing permutation. That is the intended semantics — the permutation only addresses rows that existed when it was built — but it is a change worth flagging. Permutations written before this carry no version key and read exactly as they did before. --- python/python/lancedb/permutation.py | 26 ++- python/python/lancedb/streaming.py | 4 + python/python/tests/test_permutation.py | 25 +++ .../src/dataloader/permutation/builder.rs | 186 ++++++++++++++++-- .../src/dataloader/permutation/reader.rs | 93 ++++++++- rust/lancedb/src/remote/table.rs | 37 ++++ rust/lancedb/src/table/query.rs | 3 + 7 files changed, 350 insertions(+), 24 deletions(-) diff --git a/python/python/lancedb/permutation.py b/python/python/lancedb/permutation.py index 5d7a685ef..8287ef2fe 100644 --- a/python/python/lancedb/permutation.py +++ b/python/python/lancedb/permutation.py @@ -391,6 +391,15 @@ def _table_to_pickle_state(table: Table) -> dict[str, Any]: } +def _drop_base_version(permutation_data: pa.Table) -> pa.Table: + """Strip the recorded base version so the reader leaves the base table unpinned.""" + metadata = dict(permutation_data.schema.metadata or {}) + if metadata.pop(b"base_version", None) is None: + return permutation_data + metadata.pop(b"base_branch", None) + return permutation_data.replace_schema_metadata(metadata) + + def _table_from_pickle_state(state: dict[str, Any]) -> Table: from . import connect @@ -679,11 +688,15 @@ class Permutation: from . import connect connection_factory = state["connection_factory"] + rebuilt_base = False if connection_factory is not None: base_table = connection_factory(state["base_table_name"]) elif "base_table_state" in state: - base_table = _table_from_pickle_state(state["base_table_state"]) + base_state = state["base_table_state"] + rebuilt_base = base_state["kind"] == "memory" + base_table = _table_from_pickle_state(base_state) elif "base_table_data" in state: + rebuilt_base = True # In-memory base table inlined into the pickle; rebuild the same # way we rebuild the in-memory permutation table. mem_db = connect("memory://") @@ -701,11 +714,14 @@ class Permutation: ) permutation_table: Optional[Table] = None - if state["permutation_data"] is not None: + permutation_data = state["permutation_data"] + if permutation_data is not None: + if rebuilt_base: + # The base table was materialized from Arrow, so it is a fresh + # single-version dataset and the recorded pin cannot resolve on it. + permutation_data = _drop_base_version(permutation_data) mem_db = connect("memory://") - permutation_table = mem_db.create_table( - "permutation", state["permutation_data"] - ) + permutation_table = mem_db.create_table("permutation", permutation_data) self.base_table = base_table self.permutation_table = permutation_table diff --git a/python/python/lancedb/streaming.py b/python/python/lancedb/streaming.py index 2c0e2d5c1..76b3702a1 100644 --- a/python/python/lancedb/streaming.py +++ b/python/python/lancedb/streaming.py @@ -41,6 +41,7 @@ from .permutation import ( Permutation, Transforms, permutation_builder, + _drop_base_version, _table_from_pickle_state, _table_to_pickle_state, ) @@ -1327,6 +1328,9 @@ class StreamingDataset(IterableDataset): self._table = self._connection_factory(table_name) else: self._table = _table_from_pickle_state(table_state) + if table_state["kind"] == "memory": + # Rebuilt from Arrow, so the recorded pin cannot resolve on it. + perm_data = _drop_base_version(perm_data) self._perm_table = _connect("memory://").create_table(perm_name, perm_data) def state_dict(self) -> dict: diff --git a/python/python/tests/test_permutation.py b/python/python/tests/test_permutation.py index 135742c84..142fc84f1 100644 --- a/python/python/tests/test_permutation.py +++ b/python/python/tests/test_permutation.py @@ -56,6 +56,31 @@ def test_execute_does_not_reenter_background_loop(tmp_path, monkeypatch): assert permutation_tbl._conn.read_consistency_interval is None +def test_pickled_permutation_reads_pinned_version(tmp_path): + """An unpickled copy must still read the pinned version, which also covers the + version surviving the ``to_arrow()`` round trip in ``__getstate__``.""" + import pickle + + db = connect(tmp_path) + tbl = db.create_table("base", pa.table({"idx": range(20)})) + permutation_tbl = permutation_builder(tbl).execute() + perm = Permutation.from_tables(tbl, permutation_tbl) + + payload = pickle.dumps(perm) + + # Compact so the stored row addresses no longer describe these rows at latest. + tbl.delete("true") + tbl.optimize() + assert tbl.count_rows() == 0 + + # Unpickle after the mutation: __setstate__ reopens at latest, so this only + # passes if the recorded version is applied on reopen. + restored = pickle.loads(payload) + assert len(restored) == 20 + rows = restored.__getitems__(list(range(20))) + assert sorted(row["idx"] for row in rows) == list(range(20)) + + def test_split_random_counts(mem_db): """Test random splitting with absolute counts.""" tbl = mem_db.create_table( diff --git a/rust/lancedb/src/dataloader/permutation/builder.rs b/rust/lancedb/src/dataloader/permutation/builder.rs index 641c191d5..ba0d0e180 100644 --- a/rust/lancedb/src/dataloader/permutation/builder.rs +++ b/rust/lancedb/src/dataloader/permutation/builder.rs @@ -27,6 +27,12 @@ pub const SRC_ROW_ID_COL: &str = "row_id"; pub const SPLIT_NAMES_CONFIG_KEY: &str = "split_names"; +/// Base table version the permutation was built against. +pub const BASE_VERSION_CONFIG_KEY: &str = "base_version"; + +/// Base table branch the permutation was built against. Absent means main. +pub const BASE_BRANCH_CONFIG_KEY: &str = "base_branch"; + pub const DEFAULT_MEMORY_LIMIT: usize = 100 * 1024 * 1024; /// Where to store the permutation table @@ -214,21 +220,11 @@ impl PermutationBuilder { Ok(Box::pin(SimpleRecordBatchStream { schema, stream })) } - fn add_split_names( + fn add_config_metadata( data: SendableRecordBatchStream, - split_names: &[String], + metadata: HashMap, ) -> Result { - let schema = data - .schema() - .as_ref() - .clone() - .with_metadata(HashMap::from([( - SPLIT_NAMES_CONFIG_KEY.to_string(), - serde_json::to_string(split_names).map_err(|e| Error::Other { - message: format!("Failed to serialize split names: {}", e), - source: Some(e.into()), - })?, - )])); + let schema = data.schema().as_ref().clone().with_metadata(metadata); let schema = Arc::new(schema); let schema_clone = schema.clone(); let stream = data.map_ok(move |batch| batch.with_schema(schema.clone()).unwrap()); @@ -269,6 +265,12 @@ impl PermutationBuilder { Err(err) => return Err(err), } + // The handle above is already pinned to one version. Record which one, so a + // reader -- in a DataLoader worker, against a table that has since moved -- + // resolves these row addresses against the same snapshot. + let base_version = self.base_table.version().await?; + let base_branch = self.base_table.current_branch(); + // First pass, apply filter and load row ids. `Shuffler` permutes positions, so // every rank must scan the rows in the same order to build the same permutation. let mut rows = self.base_table.query().select(Select::columns(&[ROW_ID])); @@ -330,11 +332,24 @@ impl PermutationBuilder { // Rename _rowid to row_id let renamed = rename_column(sorted, ROW_ID, SRC_ROW_ID_COL)?; - let streaming_data = if let Some(split_names) = &self.config.split_names { - Self::add_split_names(renamed, split_names)? - } else { - renamed - }; + let mut metadata = HashMap::from([( + BASE_VERSION_CONFIG_KEY.to_string(), + base_version.to_string(), + )]); + // Version numbers are per-branch, so the branch is part of the coordinate. + if let Some(branch) = &base_branch { + metadata.insert(BASE_BRANCH_CONFIG_KEY.to_string(), branch.clone()); + } + if let Some(split_names) = &self.config.split_names { + metadata.insert( + SPLIT_NAMES_CONFIG_KEY.to_string(), + serde_json::to_string(split_names).map_err(|e| Error::Other { + message: format!("Failed to serialize split names: {}", e), + source: Some(e.into()), + })?, + ); + } + let streaming_data = Self::add_config_metadata(renamed, metadata)?; let (name, database) = match &self.config.destination { PermutationDestination::Permanent(database, table_name) => { @@ -533,6 +548,141 @@ mod tests { assert_eq!(*planning_versions.lock().unwrap(), vec![7, 7, 6, 6]); } + #[tokio::test] + async fn test_permutation_records_base_version() { + let temp_dir = tempfile::tempdir().unwrap(); + + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let initial_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(100), BatchCount::from(2)); + let data_table = db + .create_table("base_tbl", initial_data) + .execute() + .await + .unwrap(); + + let build_version = data_table.version().await.unwrap(); + let permutation_table = PermutationBuilder::new(data_table.clone()) + .build() + .await + .unwrap(); + + let recorded = permutation_table + .schema() + .await + .unwrap() + .metadata + .get(BASE_VERSION_CONFIG_KEY) + .expect("permutation should record the base version") + .parse::() + .unwrap(); + assert_eq!(recorded, build_version); + + // Advancing the base table must not move the recorded version. + let more_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(50), BatchCount::from(1)); + data_table.add(more_data).execute().await.unwrap(); + assert!(data_table.version().await.unwrap() > recorded); + assert_eq!( + permutation_table + .schema() + .await + .unwrap() + .metadata + .get(BASE_VERSION_CONFIG_KEY) + .unwrap() + .parse::() + .unwrap(), + recorded, + ); + } + + /// Version numbers are per-branch, so a permutation built on a branch must record + /// it -- a worker reopens by name and lands on main at the same number. + #[tokio::test] + async fn test_permutation_records_base_branch() { + let temp_dir = tempfile::tempdir().unwrap(); + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let initial_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(10), BatchCount::from(1)); + let data_table = db + .create_table("base_tbl", initial_data) + .execute() + .await + .unwrap(); + + let branch = data_table + .create_branch("exp", lance::dataset::refs::Ref::from(("main", 1))) + .await + .unwrap(); + let permutation_table = PermutationBuilder::new(branch.clone()) + .build() + .await + .unwrap(); + + let metadata = permutation_table.schema().await.unwrap().metadata.clone(); + assert_eq!( + metadata.get(BASE_BRANCH_CONFIG_KEY).map(String::as_str), + Some("exp") + ); + + // Main records nothing, so an absent key keeps meaning main. + let main_permutation = PermutationBuilder::new(data_table.clone()) + .build() + .await + .unwrap(); + assert!( + !main_permutation + .schema() + .await + .unwrap() + .metadata + .contains_key(BASE_BRANCH_CONFIG_KEY) + ); + } + + #[tokio::test] + async fn test_build_does_not_pin_the_callers_table() { + let temp_dir = tempfile::tempdir().unwrap(); + + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let initial_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(100), BatchCount::from(1)); + let data_table = db + .create_table("base_tbl", initial_data) + .execute() + .await + .unwrap(); + + PermutationBuilder::new(data_table.clone()) + .build() + .await + .unwrap(); + + // The builder pins its own handle; the caller's must still track latest. + let more_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(50), BatchCount::from(1)); + data_table.add(more_data).execute().await.unwrap(); + assert_eq!(data_table.count_rows(None).await.unwrap(), 150); + } + #[tokio::test] async fn test_permutation_builder() { let temp_dir = tempfile::tempdir().unwrap(); diff --git a/rust/lancedb/src/dataloader/permutation/reader.rs b/rust/lancedb/src/dataloader/permutation/reader.rs index 6da92e986..9757dc552 100644 --- a/rust/lancedb/src/dataloader/permutation/reader.rs +++ b/rust/lancedb/src/dataloader/permutation/reader.rs @@ -8,7 +8,9 @@ //! the rows from a source table that correspond to row IDs stored in a separate table. use crate::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream}; -use crate::dataloader::permutation::builder::SRC_ROW_ID_COL; +use crate::dataloader::permutation::builder::{ + BASE_BRANCH_CONFIG_KEY, BASE_VERSION_CONFIG_KEY, SRC_ROW_ID_COL, +}; use crate::dataloader::permutation::split::SPLIT_ID_COLUMN; use crate::error::Error; use crate::query::{ @@ -23,6 +25,7 @@ use arrow_array::{RecordBatch, UInt64Array}; use arrow_schema::SchemaRef; use datafusion_expr::{Expr, col, lit}; use futures::{StreamExt, TryStreamExt}; +use lance::dataset::refs::MAIN_BRANCH; use lance::dataset::scanner::DatasetRecordBatchStream; use lance::io::RecordBatchStream; use lance_arrow::RecordBatchExt; @@ -69,6 +72,10 @@ impl PermutationReader { permutation_table: Option>, split: u64, ) -> Result { + let base_table = match &permutation_table { + Some(permutation_table) => Self::pin_base_table(base_table, permutation_table).await?, + None => base_table, + }; let mut slf = Self { base_table, permutation_table, @@ -89,6 +96,34 @@ impl PermutationReader { Ok(slf) } + /// Pins the base table to the version the permutation was built against. + /// Permutations written before that was recorded carry no key and stay unpinned. + async fn pin_base_table( + base_table: Arc, + permutation_table: &Arc, + ) -> Result> { + let schema = permutation_table.schema().await?; + let Some(raw) = schema.metadata.get(BASE_VERSION_CONFIG_KEY) else { + return Ok(base_table); + }; + let version = raw.parse::().map_err(|e| Error::InvalidInput { + message: format!( + "Permutation table has an unreadable {} of {:?}: {}", + BASE_VERSION_CONFIG_KEY, raw, e + ), + })?; + // The recorded branch, not the handle's: a worker reopens by name and lands + // on main, and version numbers are per-branch. + let branch = schema + .metadata + .get(BASE_BRANCH_CONFIG_KEY) + .map(String::as_str) + .unwrap_or(MAIN_BRANCH); + base_table + .checkout_branch_version(branch, Some(version)) + .await + } + pub async fn try_from_tables( base_table: Arc, permutation_table: Arc, @@ -511,9 +546,13 @@ mod tests { use lance_datagen::{BatchCount, RowCount}; use rand::seq::SliceRandom; + // Aliased: `test_utils::datagen` exports a trait of the same name. + use crate::arrow::LanceDbDatagenExt as _; use crate::{ Table, arrow::SendableRecordBatchStream, + connect, + dataloader::permutation::builder::PermutationBuilder, query::{ExecutableQuery, QueryBase}, test_utils::datagen::{LanceDbDatagenExt, virtual_table}, }; @@ -545,6 +584,58 @@ mod tests { .await } + /// Compaction moves row addresses, so the reader must read the pinned version. + #[tokio::test] + async fn test_reader_pins_base_version() { + let temp_dir = tempfile::tempdir().unwrap(); + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let data = lance_datagen::gen_batch() + .col("idx", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(20), BatchCount::from(1)); + let base_table = db.create_table("base_tbl", data).execute().await.unwrap(); + + let permutation_table = PermutationBuilder::new(base_table.clone()) + .build() + .await + .unwrap(); + + base_table.delete("true").await.unwrap(); + base_table + .optimize(crate::table::OptimizeAction::All) + .await + .unwrap(); + assert_eq!(base_table.count_rows(None).await.unwrap(), 0); + + let reader = PermutationReader::try_from_tables( + base_table.base_table().clone(), + permutation_table.base_table().clone(), + 0, + ) + .await + .unwrap(); + + let values = collect_from_stream::( + reader + .read( + Select::Columns(vec!["idx".to_string()]), + QueryExecutionOptions::default(), + ) + .await + .unwrap(), + "idx", + ) + .await; + assert_eq!( + values.len(), + 20, + "reader should still see the pinned version" + ); + } + #[tokio::test] async fn test_permutation_reader() { let base_table = lance_datagen::gen_batch() diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 633d0ffce..10980d55a 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -4349,6 +4349,43 @@ mod tests { assert!(!table.base_table().scan_order_is_deterministic()); } + #[tokio::test] + async fn test_checkout_branch_pins_without_touching_the_original() { + let seen = Arc::new(std::sync::Mutex::new(Vec::new())); + let recorder = seen.clone(); + let table = Table::new_with_handler_version( + "my_table", + semver::Version::new(0, 5, 0), + move |request| match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(br#"{"version": 42, "schema": {"fields": []}}"#.to_vec()) + .unwrap(), + "/v1/table/my_table/count_rows/" => { + let body = request_body_json(&request); + recorder.lock().unwrap().push(body["version"].clone()); + http::Response::builder() + .status(200) + .body(b"0".to_vec()) + .unwrap() + } + path => panic!("unexpected request path: {path}"), + }, + ); + + let pinned = table.checkout_branch("main", Some(42)).await.unwrap(); + pinned.count_rows(None).await.unwrap(); + table.count_rows(None).await.unwrap(); + + let seen = seen.lock().unwrap(); + assert_eq!(seen[0], 42, "the pinned handle must send its version"); + assert!( + seen[1].is_null(), + "the original handle must still track latest, got {:?}", + seen[1] + ); + } + #[tokio::test] async fn test_fetch_blobs_sends_the_checked_out_version() { let ipc = one_row_blob_ipc_stream("image"); diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index 9feb9d5ab..9b81786cd 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -70,6 +70,9 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R .contains(&NamespaceClientPushdownOperation::QueryTable) && table.namespace_client.is_some() && table.dataset.current_branch().is_none() + // NsQueryTableRequest has no version field, so a pushed-down query would + // read latest and ignore the pin. + && table.dataset.time_travel_version().is_none() && !requires_local_namespace_execution(query)) { return Ok(false); From 0e65123bd8910548c1ad60b4038f0674c33ed940 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Mon, 24 Aug 2026 23:15:07 -0700 Subject: [PATCH 30/31] fix: package ordinary @udf bodies and emit only the V1 type grammar (#4044) Registering a real (embedding) Function failed on the client for three reasons: - `_package_source` treated `inspect.getclosurevars().unbound` as "unresolved globals"; CPython puts attribute names there, so any body with `np.linalg.norm(...)` or `body.split()` was rejected. Module-scope references now come from Python's own scope analysis (`symtable`) over the function source, recursively, and each is resolved the way the interpreter would: the function's globals first (a module global may shadow a builtin), then builtins. Free variables of nested scopes stay lexical; postponed annotations are not runtime loads. A genuinely missing global still fails. - `_canonical_arrow_type` emitted spellings the server's frozen grammar rejects (`fixed_size_list[n]`, `timestamp[us]`, `struct<...>`, zero-sized lists). It now emits exactly the grammar, with the server's `fixed_size_list` form, and the Rust declaration planner parses that form too. A shared golden (`tests/fixtures/first_class_functions/v1/arrow_types.json`) enumerates every grammar type, nested forms and rejected spellings; the Python emitter and Rust parser are tested against it, and the same file is under test in sophon. Packaging tests execute the shipped artifact in a fresh namespace. Contract changes (hence `breaking-change`): - `@udf` now rejects namespace acquisition structurally (`globals()`/`eval`/... by name, plus `import sys`/`builtins`/`importlib`/`inspect` inside the body), requires the function's captured `__builtins__` to be the standard mapping itself (identity, so neither lookups nor implicit hooks such as `__import__` can differ), rejects module globals that are namespace-bearing modules (`builtins`, `sys`, ...), and treats the function's own name as recursion only when the module binds it to the function or to the exact `UdfDefinition` the decorator produced; it resolves module globals through the function's real namespace (a module global may shadow a builtin) and ships importable classes/functions as imports. - List outputs must declare a non-nullable, metadata-free child named `item` (`pa.list_(pa.field("item", t, nullable=False))`); that is what the grammar means, and pyarrow's default nullable child was being silently collapsed into it. Contract, stated in the `udf` docstring: the artifact is a snapshot of the function source plus exactly the module names it references. Reaching the module namespace by another route is rejected where a static packager can see it and is otherwise unsupported; there is no dynamic-access detection beyond that. --- python/python/lancedb/functions.py | 248 ++++++++--- .../tests/test_first_class_function_slice2.py | 399 +++++++++++++++++- rust/lancedb/src/remote/table.rs | 47 +++ rust/lancedb/src/table/computed_columns.rs | 55 +++ .../first_class_functions/v1/arrow_types.json | 333 +++++++++++++++ ...remote_fixed_size_declaration_request.json | 79 ++++ 6 files changed, 1100 insertions(+), 61 deletions(-) create mode 100644 rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json create mode 100644 rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index c4ff9a9b3..33a67b413 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -12,10 +12,13 @@ expression-backed refresh job. from __future__ import annotations import ast +import builtins import base64 import functools import hashlib +import importlib import inspect +import symtable import json import math import re @@ -489,59 +492,58 @@ _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$") _SECRET_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_GRAMMAR_PRIMITIVES = ( + (pa.bool_(), "bool"), + (pa.int8(), "int8"), + (pa.int16(), "int16"), + (pa.int32(), "int32"), + (pa.int64(), "int64"), + (pa.uint8(), "uint8"), + (pa.uint16(), "uint16"), + (pa.uint32(), "uint32"), + (pa.uint64(), "uint64"), + (pa.float16(), "float16"), + (pa.float32(), "float32"), + (pa.float64(), "float64"), + (pa.string(), "utf8"), + (pa.binary(), "binary"), + (pa.date32(), "date32"), + (pa.date64(), "date64"), +) + + def _canonical_arrow_type(data_type: pa.DataType) -> str: - primitive_types = ( - (pa.bool_(), "bool"), - (pa.int8(), "int8"), - (pa.int16(), "int16"), - (pa.int32(), "int32"), - (pa.int64(), "int64"), - (pa.uint8(), "uint8"), - (pa.uint16(), "uint16"), - (pa.uint32(), "uint32"), - (pa.uint64(), "uint64"), - (pa.float16(), "float16"), - (pa.float32(), "float32"), - (pa.float64(), "float64"), - (pa.string(), "utf8"), - (pa.large_utf8(), "large_utf8"), - (pa.binary(), "binary"), - (pa.large_binary(), "large_binary"), - (pa.date32(), "date32"), - (pa.date64(), "date64"), - ) - for candidate, name in primitive_types: + """The server's V1 Function type grammar. Anything outside it is rejected + here rather than at registration.""" + for candidate, name in _GRAMMAR_PRIMITIVES: if data_type == candidate: return name - if pa.types.is_fixed_size_binary(data_type): - return f"fixed_size_binary[{data_type.byte_width}]" - if pa.types.is_list(data_type): - return f"list<{_canonical_arrow_type(data_type.value_type)}>" - if pa.types.is_large_list(data_type): - return f"large_list<{_canonical_arrow_type(data_type.value_type)}>" - if pa.types.is_fixed_size_list(data_type): + if pa.types.is_list(data_type) or pa.types.is_large_list(data_type): + prefix = "list" if pa.types.is_list(data_type) else "large_list" + return f"{prefix}<{_canonical_list_item(data_type)}>" + if pa.types.is_fixed_size_list(data_type) and data_type.list_size > 0: return ( - f"fixed_size_list<{_canonical_arrow_type(data_type.value_type)}>" - f"[{data_type.list_size}]" + f"fixed_size_list<{_canonical_list_item(data_type)}, {data_type.list_size}>" ) - if pa.types.is_struct(data_type): - fields = ",".join( - f"{field.name}:{_canonical_arrow_type(field.type)}" for field in data_type - ) - return f"struct<{fields}>" - if pa.types.is_timestamp(data_type): - timezone = f",tz={data_type.tz}" if data_type.tz is not None else "" - return f"timestamp[{data_type.unit}{timezone}]" - if pa.types.is_time32(data_type) or pa.types.is_time64(data_type): - return f"time[{data_type.unit}]" - if pa.types.is_duration(data_type): - return f"duration[{data_type.unit}]" - if pa.types.is_decimal(data_type): - bit_width = data_type.bit_width - return f"decimal{bit_width}({data_type.precision},{data_type.scale})" raise TypeError(f"unsupported Arrow type for Function signature: {data_type}") +def _canonical_list_item(data_type: pa.DataType) -> str: + """The grammar names only the item type; it always means a non-nullable + child called `item`, so any other child metadata cannot be represented.""" + child = data_type.value_field + if child.name != "item" or child.nullable or child.metadata: + raise TypeError( + "unsupported Arrow type for Function signature: list items must be a " + f"non-nullable field named 'item', got {child}" + ) + return _canonical_arrow_type(child.type) + + +def _list_of(item: pa.DataType) -> pa.DataType: + return pa.list_(pa.field("item", item, nullable=False)) + + def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]: nullable = False origin = get_origin(annotation) @@ -589,7 +591,7 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]: value_type, value_nullable = _annotation_type(arguments[0]) if value_nullable: raise TypeError("nullable Function list elements are not supported") - return pa.list_(value_type), nullable + return _list_of(value_type), nullable raise TypeError(f"unsupported Function annotation: {annotation!r}") @@ -736,6 +738,104 @@ def _literal_source(value: Any) -> str: ) +_DYNAMIC_NAMESPACE_ACCESS = frozenset( + {"globals", "locals", "vars", "eval", "exec", "compile", "__import__"} +) +# Modules that hand out namespaces (`sys.modules`, `builtins`, importers, +# introspection). The artifact's module namespace holds only the names it was +# packaged with, so reaching around it cannot be represented. +_NAMESPACE_MODULES = frozenset( + {"sys", "builtins", "importlib", "inspect", "gc", "ctypes", "types"} +) + + +def _namespace_acquisition( + definition: ast.FunctionDef, references: set[str] +) -> list[str]: + found = set(references & _DYNAMIC_NAMESPACE_ACCESS) + for node in ast.walk(definition): + if isinstance(node, ast.Import): + found.update( + alias.name + for alias in node.names + if alias.name.split(".")[0] in _NAMESPACE_MODULES + ) + elif isinstance(node, ast.ImportFrom) and node.module: + if node.module.split(".")[0] in _NAMESPACE_MODULES: + found.add(node.module) + return sorted(found) + + +def _module_references(module_source: str) -> set[str]: + """Names any scope in `module_source` binds or loads at module scope. + Python's own scope analysis on the exact text that ships: free variables + belong to an enclosing scope inside the function, and postponed + annotations are not runtime loads.""" + + def visit(table: symtable.SymbolTable, found: set[str]) -> None: + for symbol in table.get_symbols(): + if symbol.is_global() and ( + symbol.is_referenced() or symbol.is_declared_global() + ): + found.add(symbol.get_name()) + for child in table.get_children(): + visit(child, found) + + found: set[str] = set() + for table in symtable.symtable(module_source, "", "exec").get_children(): + visit(table, found) + return found + + +def _global_source(name: str, value: Any) -> str: + """One module-level line that rebinds `name` to `value` in the artifact: + an import for modules and importable classes/functions, a literal otherwise.""" + if isinstance(value, types.ModuleType): + if value.__name__.split(".")[0] in _NAMESPACE_MODULES: + raise ValueError( + f"@udf cannot package dynamic namespace access: {value.__name__!r}" + ) + try: + imported = importlib.import_module(value.__name__) + except ImportError: + imported = None + if imported is not value: + raise TypeError( + f"Function source references module {name!r} that does not import " + f"as {value.__name__!r}" + ) + return f"import {value.__name__} as {name}" + module_name = getattr(value, "__module__", None) + qualname = getattr(value, "__qualname__", None) + if ( + isinstance(module_name, str) + and isinstance(qualname, str) + and module_name != "__main__" + and "." not in qualname + and "<" not in qualname + ): + try: + imported = getattr(importlib.import_module(module_name), qualname) + except (ImportError, AttributeError): + imported = None + if imported is value: + return f"from {module_name} import {qualname} as {name}" + return f"{name} = {_literal_source(value)}" + + +def _is_recursive_reference(function: Callable[..., Any], name: str) -> bool: + """`name` inside the body means the function itself unless the module has + since bound it to something else.""" + if name != function.__name__: + return False + bound = function.__globals__.get(name, function) + if bound is function: + return True + # The decorator's own result is the one wrapper known to call `function` + # unchanged; any other binding may behave differently from a self-call. + return type(bound) is UdfDefinition and bound._function is function + + def _package_source(function: Callable[..., Any]) -> bytes: if not inspect.isfunction(function) or inspect.iscoroutinefunction(function): raise TypeError("@udf requires a synchronous Python function") @@ -760,23 +860,46 @@ def _package_source(function: Callable[..., Any]) -> bytes: closure = inspect.getclosurevars(function) if closure.nonlocals: raise ValueError("@udf cannot package functions that capture closure values") - if closure.unbound: - raise ValueError( - f"@udf source contains unresolved global names: {sorted(closure.unbound)!r}" - ) - globals_source = [] - for name, value in sorted(closure.globals.items()): - if isinstance(value, types.ModuleType): - globals_source.append(f"import {value.__name__} as {name}") - else: - globals_source.append(f"{name} = {_literal_source(value)}") - function_source = ast.unparse(definition) - parts = ["from __future__ import annotations"] + module_header = "from __future__ import annotations" + references = _module_references(f"{module_header}\n\n{function_source}\n") + dynamic = _namespace_acquisition(definition, references) + if dynamic: + raise ValueError(f"@udf cannot package dynamic namespace access: {dynamic!r}") + # Resolve every module-scope reference the way the interpreter would: the + # function's own globals first (a module global may shadow a builtin, and + # nested scopes are not visible to getclosurevars), then its builtins. + # The artifact runs under the standard builtins; only the exact mapping is + # provably equivalent (a subclass or copy can change lookups and hooks). + if function.__builtins__ is not vars(builtins): + raise ValueError("@udf cannot package a non-standard builtins environment") + globals_source = [] + unresolved = [] + for name in sorted(references): + if name == function.__name__: + if not _is_recursive_reference(function, name): + raise ValueError( + f"@udf cannot package {name!r}: the module binds that name to " + "another value, which the artifact's own definition would shadow" + ) + continue + if name in function.__globals__: + globals_source.append(_global_source(name, function.__globals__[name])) + elif hasattr(builtins, name): + pass + else: + unresolved.append(name) + if unresolved: + raise ValueError( + f"@udf source contains unresolved global names: {unresolved!r}" + ) + + parts = [module_header] if globals_source: parts.extend(["", *globals_source]) parts.extend(["", function_source, ""]) - return "\n".join(parts).encode("utf-8") + packaged = "\n".join(parts) + return packaged.encode("utf-8") class UdfDefinition: @@ -923,6 +1046,13 @@ def udf( python_version : str, optional Remote Python major/minor version. Defaults to the client version. + The packaged artifact is a snapshot: the function source plus exactly + the module-level names it references (modules as imports, importable + classes and functions as imports, literals inline). Code that reaches the + module namespace another way -- ``globals()``/``eval``, ``sys.modules``, + ``builtins`` -- is rejected where it can be seen and otherwise + unsupported; closures and a non-standard ``__builtins__`` are rejected. + Returns ------- UdfDefinition diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index a20347771..c67e17520 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -3,7 +3,12 @@ from __future__ import annotations +import base64 import contextlib +import functools +import importlib.util +import types +from datetime import date import http.server import json from pathlib import Path @@ -16,6 +21,9 @@ import pytest import lancedb from lancedb.functions import UdfDefinition, udf +THRESHOLD = 20 +_CACHE = None + FIXTURES = ( Path(__file__).parents[3] @@ -71,9 +79,396 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable(): _assert_no_secret_values(request) +def _run_packaged(definition, *args): + """Execute the shipped artifact in a fresh namespace, as a worker would.""" + source = base64.b64decode(definition.registration_request.artifact.content.data) + namespace: dict = {} + exec(compile(source, "", "exec"), namespace) + return namespace[definition.registration_request.artifact.entrypoint](*args) + + +def test_udf_packages_attribute_access_and_body_imports(): + @udf + def word_norm(body: str) -> float: + import numpy as np + + try: + words = body.split() + except AttributeError as error: + raise ValueError(str(error)) from error + return float(np.linalg.norm([len(w) for w in words])) + + assert _run_packaged(word_norm, "aa bb") == pytest.approx(8**0.5) + + +def test_udf_packages_module_globals_and_global_caches(): + @udf + def label(value: int) -> str: + return "big" if value >= THRESHOLD else "small" + + assert _run_packaged(label, 21) == "big" + + @udf + def cached(value: int) -> int: + global _CACHE + if _CACHE is None: + _CACHE = 40 + return _CACHE + value + + assert _run_packaged(cached, 2) == 42 + + +def test_udf_annotations_are_not_runtime_names(): + @udf + def identity(value: date) -> date: + return value + + assert _run_packaged(identity, date(2026, 8, 25)) == date(2026, 8, 25) + + +def test_udf_nested_scopes_resolve_lexically(): + @udf + def score(value: int) -> int: + offset = 2 + + def add_offset() -> int: + return value + offset + + return add_offset() + sum(v for v in [0]) + + assert _run_packaged(score, 3) == 5 + + +def test_udf_resolves_module_globals_before_builtins(tmp_path): + module_path = tmp_path / "shadowing_udfs.py" + module_path.write_text( + "max = 7\n" + "len = lambda _: 99\n" + "\n" + "def uses_literal_shadow(value: int) -> int:\n" + " def nested() -> int:\n" + " return max\n" + " return nested() + value\n" + "\n" + "def uses_callable_shadow(value: int) -> int:\n" + " def nested() -> int:\n" + " return len([1])\n" + " return nested() + value\n" + ) + spec = importlib.util.spec_from_file_location("shadowing_udfs", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # The module's `max = 7` is what the interpreter would use, so it ships. + assert _run_packaged(udf(module.uses_literal_shadow), 1) == 8 + # A callable global cannot ship; it must not be silently swapped for the builtin. + with pytest.raises(TypeError, match="unsupported global value of type function"): + udf(module.uses_callable_shadow) + + +def test_canonical_arrow_type_is_exactly_the_grammar(): + from lancedb.functions import _GRAMMAR_PRIMITIVES, _canonical_arrow_type + + golden = json.loads( + ( + Path(__file__).parents[3] + / "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json" + ).read_text() + ) + primitives = [ + case["arrow_type"] for case in golden["valid"] if "<" not in case["arrow_type"] + ] + assert [name for _, name in _GRAMMAR_PRIMITIVES] == primitives + for outside in [ + pa.timestamp("us"), + pa.decimal128(10, 2), + pa.large_string(), + pa.large_binary(), + pa.binary(4), + pa.duration("s"), + pa.struct([pa.field("a", pa.int32())]), + pa.list_(pa.float32(), 0), + pa.list_(pa.timestamp("us")), + ]: + with pytest.raises(TypeError, match="unsupported Arrow type"): + _canonical_arrow_type(outside) + + +def test_udf_nested_annotations_are_postponed_in_the_artifact(): + @udf + def score(value: int) -> int: + def identity(item: date) -> date: + return item + + identity(date(2026, 8, 25)) + return value + + assert _run_packaged(score, 3) == 3 + + +def test_udf_ships_globals_the_body_deletes(): + @udf + def clear(value: int) -> int: + global _CACHE + del _CACHE + return value + + assert _run_packaged(clear, 3) == 3 + + +def test_udf_rejects_a_module_global_that_does_not_import_as_itself(tmp_path): + module_path = tmp_path / "fake_module_udfs.py" + module_path.write_text( + "import types\n" + "np = types.ModuleType('numpy')\n" + "np.sqrt = lambda x: 0\n" + "\n" + "def score(value: int) -> int:\n" + " return int(np.sqrt(value))\n" + ) + spec = importlib.util.spec_from_file_location("fake_module_udfs", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + with pytest.raises(TypeError, match="does not import as 'numpy'"): + udf(module.score) + + +def test_udf_rejects_a_module_level_namespace_alias(tmp_path): + module_path = tmp_path / "aliasing_udfs.py" + module_path.write_text( + "import builtins as b\n" + "THRESHOLD = 5\n" + "\n" + "def score(value: int) -> int:\n" + " return value + b.vars(b.__import__('aliasing_udfs'))['THRESHOLD']\n" + ) + spec = importlib.util.spec_from_file_location("aliasing_udfs", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + with pytest.raises(ValueError, match="dynamic namespace access"): + udf(module.score) + + +@pytest.mark.parametrize( + "access", + [ + "globals()['THRESHOLD']", + "eval('THRESHOLD')", + "(lambda g: g()['THRESHOLD'])(globals)", + "__import__('sys').modules[__name__].THRESHOLD", + "sys.modules[__name__].THRESHOLD", + ], +) +def test_udf_rejects_dynamic_namespace_access(access): + namespace: dict = {} + exec( + f"def score(value: int) -> int:\n return value + {access}\n", + {"THRESHOLD": 5}, + namespace, + ) + with pytest.raises(ValueError, match="dynamic namespace access"): + _package_from_text( + "def score(value: int) -> int:\n" + " import sys\n" + f" return value + {access}\n" + ) + + +def _package_from_text(source: str, module_globals: dict | None = None): + """Load `source` as a real module file so the packager can inspect it.""" + import tempfile + + directory = tempfile.mkdtemp() + path = Path(directory) / "generated_udf_module.py" + path.write_text(source) + spec = importlib.util.spec_from_file_location(f"generated_udf_{id(source)}", path) + module = importlib.util.module_from_spec(spec) + if module_globals: + module.__dict__.update(module_globals) + spec.loader.exec_module(module) + functions = [ + value + for value in vars(module).values() + if callable(value) and getattr(value, "__module__", None) == module.__name__ + ] + return udf(functions[0]) + + +def test_udf_rejects_a_non_standard_builtins_environment(): + def score(value: int) -> int: + return len([1]) + value + + score.__globals__ # noqa: B018 -- real function, real globals + import builtins + + patched = types.FunctionType( + score.__code__, + {"__builtins__": {**vars(builtins), "len": lambda _: 99}}, + "score", + ) + patched.__annotations__ = score.__annotations__ + assert patched(3) == 102 + with pytest.raises(ValueError, match="non-standard builtins environment"): + udf(patched) + + class ReportingDict(dict): # reports standard entries, resolves differently + def __missing__(self, key): + return vars(builtins)[key] + + disguised = types.FunctionType( + score.__code__, {"__builtins__": ReportingDict(len=lambda _: 99)}, "score" + ) + disguised.__annotations__ = score.__annotations__ + assert disguised(3) == 102 + with pytest.raises(ValueError, match="non-standard builtins environment"): + udf(disguised) + + hooked = types.FunctionType( + score.__code__, + {"__builtins__": {**vars(builtins), "__import__": lambda *a, **k: None}}, + "score", + ) + hooked.__annotations__ = score.__annotations__ + with pytest.raises(ValueError, match="non-standard builtins environment"): + udf(hooked) + + +def test_udf_recursion_versus_a_rebound_module_name(tmp_path): + module_path = tmp_path / "rebound_udfs.py" + module_path.write_text( + "def fact(value: int) -> int:\n" + " return 1 if value <= 1 else value * fact(value - 1)\n" + "\n" + "def score(value: int) -> int:\n" + " return score + value\n" + ) + spec = importlib.util.spec_from_file_location("rebound_udfs", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + assert _run_packaged(udf(module.fact), 5) == 120 + raw = module.score + module.score = 10 + with pytest.raises(ValueError, match="binds that name to another value"): + udf(raw) + # A wrapper that merely exposes __wrapped__ is not the function. + module.score = functools.wraps(raw)(lambda value: 41) + with pytest.raises(ValueError, match="binds that name to another value"): + udf(raw) + # The decorator's own result is; a subclass of it is not. + module.fact = udf(module.fact) + assert _run_packaged(module.fact, 4) == 24 + + class Twisted(UdfDefinition): + def __call__(self, *args, **kwargs): + return 41 + + raw_fact = module.fact._function + module.fact = Twisted( + raw_fact, + name=None, + input_schema=None, + output_schema=None, + pip=(), + env={}, + secrets=(), + python_version=None, + ) + with pytest.raises(ValueError, match="binds that name to another value"): + udf(raw_fact) + + +def test_canonical_arrow_type_rejects_unrepresentable_list_children(): + from lancedb.functions import _canonical_arrow_type + + for outside in [ + pa.list_(pa.float32()), # pyarrow default: nullable child + pa.list_(pa.field("custom", pa.float32(), nullable=False)), + pa.list_(pa.field("item", pa.float32(), nullable=False, metadata={"k": "v"})), + pa.list_(pa.field("item", pa.float32(), nullable=False), 0), + ]: + with pytest.raises(TypeError, match="unsupported Arrow type"): + _canonical_arrow_type(outside) + assert ( + _canonical_arrow_type( + pa.list_(pa.field("item", pa.float32(), nullable=False), 3) + ) + == "fixed_size_list" + ) + + +def _calls_missing(value: int) -> int: + return missing(value) # noqa: F821 + + +def _shadows_missing_in_a_comprehension(value: int) -> int: + return missing(value) + sum(missing for missing in ()) # noqa: F821 + + +def _shadows_missing_in_a_lambda(value: int) -> int: + return (lambda missing: missing)(value) + missing # noqa: F821 + + +@pytest.mark.parametrize( + "function", + [_calls_missing, _shadows_missing_in_a_comprehension, _shadows_missing_in_a_lambda], +) +def test_udf_rejects_a_truly_unresolved_global(function): + with pytest.raises(ValueError, match=r"unresolved global names: \['missing'\]"): + udf(function) + + +def _arrow_type_from_golden(spec: dict) -> pa.DataType: + kind = spec["type"] + if kind in ("list", "large_list", "fixed_size_list"): + item = _arrow_type_from_golden(spec["fields"][0]["type"]) + field = pa.field("item", item, nullable=False) + if kind == "list": + return pa.list_(field) + if kind == "large_list": + return pa.large_list(field) + return pa.list_(field, spec["length"]) + return { + "null": pa.null(), + "bool": pa.bool_(), + "utf8": pa.string(), + "binary": pa.binary(), + "float16": pa.float16(), + "float32": pa.float32(), + "float64": pa.float64(), + "date32": pa.date32(), + "date64": pa.date64(), + }.get(kind) or getattr(pa, kind)() + + +def test_arrow_type_grammar_matches_the_shared_golden(): + golden = json.loads( + ( + Path(__file__).parents[3] + / "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json" + ).read_text() + ) + from lancedb.functions import _canonical_arrow_type + + emitted = { + case["arrow_type"]: _canonical_arrow_type(_arrow_type_from_golden(case["json"])) + for case in golden["valid"] + } + assert emitted == { + case["arrow_type"]: case["arrow_type"] for case in golden["valid"] + } + assert not set(emitted) & set(golden["invalid"]) + for case in golden["server_only"]: + with pytest.raises(TypeError, match="unsupported Arrow type"): + _canonical_arrow_type(_arrow_type_from_golden(case["json"])) + + def test_explicit_arrow_schema_is_deterministic(): input_schema = pa.schema([pa.field("value", pa.float32(), nullable=True)]) - output_schema = pa.field("embedding", pa.list_(pa.float32(), 3), nullable=False) + output_schema = pa.field( + "embedding", + pa.list_(pa.field("item", pa.float32(), nullable=False), 3), + nullable=False, + ) @udf(input_schema=input_schema, output_schema=output_schema) def explicit(value): @@ -82,7 +477,7 @@ def test_explicit_arrow_schema_is_deterministic(): signature = explicit.registration_request.signature assert signature.inputs[0].arrow_type == "float32" assert signature.inputs[0].nullable is True - assert signature.output.arrow_type == "fixed_size_list[3]" + assert signature.output.arrow_type == "fixed_size_list" assert signature.output.nullable is False diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 10980d55a..35ac56970 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -6802,6 +6802,53 @@ mod tests { assert_eq!(result.version, 8); } + #[tokio::test] + async fn test_add_fixed_size_list_function_column_declares_the_vector_type() { + let table = Table::new_with_handler("my_table", |request| { + match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body( + r#"{"version":1,"schema":{"fields":[{"name":"description","nullable":true,"type":{"type":"string"}}]}}"#, + ) + .unwrap(), + "/v1/table/my_table/add_columns/" => { + let actual: serde_json::Value = serde_json::from_slice( + request.body().unwrap().as_bytes().unwrap(), + ) + .unwrap(); + let expected: serde_json::Value = serde_json::from_str(include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json" + )) + .unwrap(); + assert_eq!(actual, expected); + http::Response::builder() + .status(200) + .body(r#"{"version":8}"#) + .unwrap() + } + path => panic!("Unexpected path: {path}"), + } + }); + let application = crate::function::FunctionApplication::from_json( + r#"{ + "function":{"name":"embed","version":"fv_01K3EXACT"}, + "inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}], + "output":{"kind":"scalar","arrow_type":"fixed_size_list","nullable":false}, + "group_id":"fg_fixed" + }"#, + ) + .unwrap(); + + let result = table + .add_columns() + .function_as("embedding", application) + .execute() + .await + .unwrap(); + assert_eq!(result.version, 8); + } + #[tokio::test] async fn test_add_named_struct_function_expands_one_atomic_sibling_group() { let table = Table::new_with_handler("my_table", |request| match request.url().path() { diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index b1c0b8370..133f37a55 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -586,6 +586,25 @@ fn canonical_input_arrow_type(field: &JsonArrowField) -> Result { } } +/// `fixed_size_list` -> (`item`, `size`); the comma must sit outside +/// any nested `<...>`. +fn split_fixed_size_list(raw: &str) -> Option<(&str, i32)> { + let inner = raw.strip_prefix("fixed_size_list<")?.strip_suffix('>')?; + let mut depth = 0_u32; + let mut separator = None; + for (index, byte) in inner.bytes().enumerate() { + match byte { + b'<' => depth += 1, + b'>' => depth = depth.checked_sub(1)?, + b',' if depth == 0 => separator = Some(index), + _ => {} + } + } + let (item, size) = inner.split_at(separator?); + let size: i32 = size[1..].trim().parse().ok()?; + (size > 0).then_some((item.trim(), size)) +} + fn parse_output_arrow_type(raw: &str) -> Result { fn parse(raw: &str) -> Result { let raw = raw.trim(); @@ -618,6 +637,16 @@ fn parse_output_arrow_type(raw: &str) -> Result { )]); return Ok(data_type); } + if let Some((inner, size)) = split_fixed_size_list(raw) { + let mut data_type = JsonArrowDataType::new("fixed_size_list".to_string()); + data_type.fields = Some(vec![JsonArrowField::new( + "item".to_string(), + false, + parse(inner)?, + )]); + data_type.length = Some(i64::from(size)); + return Ok(data_type); + } let normalized = match raw { "boolean" => "bool", "string" => "utf8", @@ -1322,6 +1351,32 @@ pub(super) async fn add_foreign_kind(table: &crate::Table, name: &str, kind: &st #[cfg(test)] mod tests { + #[test] + fn output_arrow_type_grammar_matches_the_shared_golden() { + let golden: serde_json::Value = serde_json::from_str(include_str!( + "../../tests/fixtures/first_class_functions/v1/arrow_types.json" + )) + .unwrap(); + let valid = golden["valid"].as_array().unwrap().iter(); + for case in valid.chain(golden["server_only"].as_array().unwrap()) { + let raw = case["arrow_type"].as_str().unwrap(); + let parsed = super::parse_output_arrow_type(raw) + .unwrap_or_else(|error| panic!("{raw}: {error}")); + assert_eq!( + serde_json::to_value(&parsed).unwrap(), + case["json"], + "{raw}" + ); + } + for raw in golden["invalid"].as_array().unwrap() { + let raw = raw.as_str().unwrap(); + assert!( + super::parse_output_arrow_type(raw).is_err(), + "{raw:?} should be rejected" + ); + } + } + use arrow_array::record_batch; use arrow_schema::DataType; use futures::TryStreamExt; diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json new file mode 100644 index 000000000..ff26e4c08 --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json @@ -0,0 +1,333 @@ +{ + "valid": [ + { + "arrow_type": "bool", + "json": { + "type": "bool" + } + }, + { + "arrow_type": "int8", + "json": { + "type": "int8" + } + }, + { + "arrow_type": "int16", + "json": { + "type": "int16" + } + }, + { + "arrow_type": "int32", + "json": { + "type": "int32" + } + }, + { + "arrow_type": "int64", + "json": { + "type": "int64" + } + }, + { + "arrow_type": "uint8", + "json": { + "type": "uint8" + } + }, + { + "arrow_type": "uint16", + "json": { + "type": "uint16" + } + }, + { + "arrow_type": "uint32", + "json": { + "type": "uint32" + } + }, + { + "arrow_type": "uint64", + "json": { + "type": "uint64" + } + }, + { + "arrow_type": "float16", + "json": { + "type": "float16" + } + }, + { + "arrow_type": "float32", + "json": { + "type": "float32" + } + }, + { + "arrow_type": "float64", + "json": { + "type": "float64" + } + }, + { + "arrow_type": "utf8", + "json": { + "type": "utf8" + } + }, + { + "arrow_type": "binary", + "json": { + "type": "binary" + } + }, + { + "arrow_type": "date32", + "json": { + "type": "date32" + } + }, + { + "arrow_type": "date64", + "json": { + "type": "date64" + } + }, + { + "arrow_type": "list", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + }, + { + "arrow_type": "large_list", + "json": { + "type": "large_list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + }, + { + "arrow_type": "list", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "int64" + } + } + ] + } + }, + { + "arrow_type": "large_list", + "json": { + "type": "large_list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "int64" + } + } + ] + } + }, + { + "arrow_type": "list", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "utf8" + } + } + ] + } + }, + { + "arrow_type": "large_list", + "json": { + "type": "large_list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "utf8" + } + } + ] + } + }, + { + "arrow_type": "fixed_size_list", + "json": { + "type": "fixed_size_list", + "length": 384, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + }, + { + "arrow_type": "fixed_size_list", + "json": { + "type": "fixed_size_list", + "length": 8, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float16" + } + } + ] + } + }, + { + "arrow_type": "fixed_size_list", + "json": { + "type": "fixed_size_list", + "length": 1, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "uint8" + } + } + ] + } + }, + { + "arrow_type": "list>", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + } + ] + } + }, + { + "arrow_type": "fixed_size_list, 2>", + "json": { + "type": "fixed_size_list", + "length": 2, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "int32" + } + } + ] + } + } + ] + } + }, + { + "arrow_type": "list>", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "fixed_size_list", + "length": 3, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + } + ] + } + } + ], + "server_only": [ + { + "arrow_type": "null", + "json": { + "type": "null" + } + } + ], + "invalid": [ + "", + "list<>", + "list[3]", + "fixed_size_list", + "fixed_size_list", + "fixed_size_list", + "map", + "decimal128(10, 2)", + "timestamp[us]", + "struct" + ] +} \ No newline at end of file diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json new file mode 100644 index 000000000..7d92f1454 --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json @@ -0,0 +1,79 @@ +{ + "new_columns": [ + { + "name": "embedding", + "all_null": true + } + ], + "function": { + "application": { + "function": { + "name": "embed", + "version": "fv_01K3EXACT" + }, + "inputs": [ + { + "parameter": "text", + "kind": "column", + "value": { + "path": "description" + } + } + ], + "output": { + "kind": "scalar", + "arrow_type": "fixed_size_list", + "nullable": false + }, + "group_id": "fg_fixed" + }, + "binding_metadata_version": 1, + "input_bindings": [ + { + "parameter": "text", + "field_path": "description", + "arrow_type": "utf8", + "nullable": true + } + ], + "input_schema": { + "fields": [ + { + "name": "text", + "nullable": true, + "type": { + "type": "utf8" + } + } + ] + }, + "output_schema": { + "fields": [ + { + "name": "embedding", + "nullable": true, + "type": { + "type": "fixed_size_list", + "length": 3, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + } + ] + }, + "outputs": [ + { + "result_field": "$value", + "output_name": "embedding", + "output_ordinal": 0 + } + ] + } +} \ No newline at end of file From 2fea7cd48d86ef149ab7eee93f7751efeb410233 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Tue, 25 Aug 2026 06:26:01 +0000 Subject: [PATCH 31/31] =?UTF-8?q?Bump=20version:=200.38.0-beta.7=20?= =?UTF-8?q?=E2=86=92=200.38.0-beta.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index bcf714002..86a5064f7 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.7" +current_version = "0.38.0-beta.8" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index a60a5ddf1..8d0058569 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5402,7 +5402,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.7" +version = "0.38.0-beta.8" dependencies = [ "ahash", "anyhow", @@ -5490,7 +5490,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.7" +version = "0.38.0-beta.8" dependencies = [ "arrow-array", "arrow-buffer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.7" +version = "0.38.0-beta.8" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 59ee8a7d6..28de11c30 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.7 + 0.38.0-beta.8 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index b24ad61ab..6f4b0744b 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.7 + 0.38.0-beta.8 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 0ee59bcb7..ac6196a45 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.7 + 0.38.0-beta.8 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 8732d1965..e85da66ec 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.7" +version = "0.38.0-beta.8" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 56480c0cb..f920c9e91 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index a1102ee29..ad6926039 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 695862baf..ccd39f18e 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 9099ee4de..50446b597 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index f04c2a884..d4da82dc7 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index f9b35af32..dddc3a470 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index cc5340105..23ac122f7 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 6604da6cd..21deb8dd8 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index 66777d60c..96f4bafd5 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.7", + "version": "0.38.0-beta.8", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index d2ccf3064..bffac2ef1 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.7" +version = "0.38.0-beta.8" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 071c876d9..7290694f1 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.7" +version = "0.38.0-beta.8" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true