From 1765747f916d6eb7f3fbbcc105591e5835ea3684 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Fri, 13 Nov 2026 11:02:07 -0800 Subject: [PATCH] fix: allow updates to Function input columns (#4985) Updating an input column on a table with a Function binding currently fails before the write, although the write path already preserves freshness for bindings whose inputs are unchanged. Allow input updates through the normal update builder while retaining supported binding-metadata validation and the guard against writing computed or internal assignment outputs directly. Lance rewrites only the matched rows. Their affected Function outputs keep their previous values and their changed input signatures make them eligible for refresh; unchanged fragments retain their freshness. Rewrites affecting only unrelated inputs keep the existing freshness-preservation path. For a table whose Function reads `title`, this update now succeeds while retaining the previous computed values until refresh: ```python table.update(where="id = 7", values={"title": "revised"}) ``` Server consumers need this crate update before remote input updates gain the behavior. Adds regressions for a one-row input update, retained output values, freshness scope, independent binding preservation, and atomic rejection of mixed input/output writes. Local builds and tests are disabled by request; CI runs the regressions. --- rust/lancedb/src/table/computed_columns.rs | 54 ++++++++- rust/lancedb/src/table/freshness.rs | 122 +++++++++++++++++++++ rust/lancedb/src/table/update.rs | 33 +++--- 3 files changed, 192 insertions(+), 17 deletions(-) diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index d09a790a3..64e90b246 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -1757,10 +1757,20 @@ pub(crate) fn ensure_not_written<'a>( schema: &ArrowSchema, written: impl IntoIterator, ) -> Result<()> { - let declared: Vec = computed_columns(schema) + ensure_supported_function_metadata(schema)?; + let mut declared: Vec = computed_columns(schema) .into_iter() .map(|declaration| declaration.name) .collect(); + declared.extend( + function_bindings(schema)? + .into_iter() + .filter_map(|binding| { + binding + .assignment() + .map(|assignment| assignment.output_name.clone()) + }), + ); for name in written { if declared.iter().any(|declared| *declared == root(name)) { return Err(Error::InvalidInput { @@ -2532,10 +2542,15 @@ pub(crate) mod test_support { )]) .await .unwrap(); - let inputs = ["title".to_string(), "body".to_string()]; + let schema = dataset.schema(); let outputs = bindings .iter() .flat_map(|binding| { + let inputs = binding + .inputs() + .iter() + .map(|input| input.field_path.clone()) + .collect::>(); let assignment = binding.assignment().map(|assignment| { ( assignment.output_name.clone(), @@ -2547,9 +2562,9 @@ pub(crate) mod test_support { .iter() .map(|output| (output.output_name.clone(), output.output_ordinal)) .chain(assignment) - .map(|(name, ordinal)| { + .map(move |(name, ordinal)| { ( - dataset.schema().field(&name).unwrap().id as u32, + schema.field(&name).unwrap().id as u32, function_computed_column_metadata( binding.binding_id(), ordinal, @@ -4144,6 +4159,37 @@ mod tests { ArrowSchema::new(fields) } + #[test] + fn input_writes_preserve_the_assignment_column_guard() { + let mut raw: Value = serde_json::to_value(test_support::text_features_binding("")).unwrap(); + raw["assignment"] = serde_json::json!({ + "output_name": "__function_assignment_fb_01K3TEXT", "output_field_id": -1, + }); + raw["outputs"][0]["nullable"] = Value::Bool(true); + raw["outputs"][1]["nullable"] = Value::Bool(true); + raw["output_schema"]["fields"].as_array_mut().unwrap().push(serde_json::json!({ + "name": "__function_assignment_fb_01K3TEXT", "nullable": true, "type": {"type": "bool"}, + })); + let binding: FunctionBinding = serde_json::from_value(raw).unwrap(); + let schema = ArrowSchema::new_with_metadata( + valid_function_binding_schema(true, true, &binding) + .fields() + .to_vec(), + HashMap::from([( + FUNCTION_BINDINGS_META_KEY.into(), + function_bindings_metadata(&[binding]).unwrap(), + )]), + ); + ensure_not_written(&schema, ["title", "body"]).unwrap(); + for output in [ + "search_text", + "search_token_count", + "__function_assignment_fb_01K3TEXT", + ] { + assert!(ensure_not_written(&schema, [output]).is_err(), "{output}"); + } + } + #[test] fn test_non_nullable_function_inputs_can_bind_to_nullable_parameters() { let binding = FunctionBinding::from_json(include_str!( diff --git a/rust/lancedb/src/table/freshness.rs b/rust/lancedb/src/table/freshness.rs index 645653a46..563a2a4e5 100644 --- a/rust/lancedb/src/table/freshness.rs +++ b/rust/lancedb/src/table/freshness.rs @@ -2646,6 +2646,8 @@ mod tests { use lance_file::version::ConcreteFileVersion; use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use crate::table::computed_columns::test_support::{stamp_bindings, text_features_binding}; + const COLUMN: &str = "doubled"; const EXPRESSION: &str = "value * 2"; @@ -3126,6 +3128,126 @@ mod tests { } } + #[tokio::test] + async fn input_update_dirties_only_the_updated_rows_and_retains_outputs() { + let table = fresh_function_table().await; + let result = table + .update() + .only_if("title = 'a'") + .column("body", "'changed'") + .execute() + .await + .unwrap(); + assert_eq!(result.rows_updated, 1); + let dataset = table.as_native().unwrap().dataset.get().await.unwrap(); + for output in ["search_text", "search_token_count"] { + let stale = function_plan(&dataset, output).await; + let mut dirty_rows = 0; + for fragment in dataset.get_fragments() { + if stale.needs_recompute(fragment.id() as u32) { + let batch = fragment + .scan() + .project(&["title"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let titles = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(titles.iter().all(|title| title == Some("a")), "{stale:?}"); + dirty_rows += batch.num_rows(); + } + } + assert_eq!(dirty_rows, 1, "{stale:?}"); + } + assert_eq!(table.count_rows(Some("title = 'a' AND body = 'changed' AND search_text = 'a x' AND search_token_count = 2".into())).await.unwrap(), 1); + } + + #[tokio::test] + async fn input_update_keeps_an_independent_function_binding_fresh() { + let batch = arrow_array::record_batch!( + ("title", Utf8, ["a", "b", "c", "d"]), + ("body", Utf8, ["x", "y", "z", "w"]), + ("search_text", Utf8, ["a x", "b y", "c z", "d w"]), + ("search_token_count", Int64, [2, 2, 2, 2]), + ("search_text_other", Utf8, ["a a", "b b", "c c", "d d"]), + ("search_token_count_other", Int64, [2, 2, 2, 2]) + ) + .unwrap(); + let table = crate::connect("memory://") + .execute() + .await + .unwrap() + .create_table("independent_functions", batch) + .write_options(crate::table::WriteOptions { + lance_write_params: Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + }) + .execute() + .await + .unwrap(); + let mut other = serde_json::to_value(text_features_binding("_other")).unwrap(); + other["inputs"][1]["field_path"] = serde_json::json!("title"); + other["inputs"][1]["field_id"] = other["inputs"][0]["field_id"].clone(); + let bindings = [ + text_features_binding(""), + serde_json::from_value(other).unwrap(), + ]; + stamp_bindings(&table, &bindings).await; + let native = table.as_native().unwrap(); + let mut dataset = native.dataset.get().await.unwrap().as_ref().clone(); + for binding in &bindings { + let paths = binding + .inputs() + .iter() + .map(|input| input.field_path.clone()) + .collect::>(); + let inputs = fields_for_paths(dataset.schema(), &paths).unwrap(); + let signatures = signatures_for(&dataset, &[0, 1], &inputs).unwrap(); + for output in binding.outputs() { + write_signatures( + &mut dataset, + &output.output_name, + binding.binding_id(), + signatures.clone(), + ) + .await + .unwrap(); + } + } + native.dataset.update(dataset); + table + .update() + .only_if("title = 'a'") + .column("body", "'changed'") + .execute() + .await + .unwrap(); + let dataset = native.dataset.get().await.unwrap(); + assert!( + function_plan(&dataset, "search_text") + .await + .needs_recompute(2) + ); + let other = &bindings[1]; + let inputs = fields_for_paths(dataset.schema(), &["title".into()]).unwrap(); + for output in other.outputs() { + let stale = + staleness_against(&dataset, &output.output_name, other.binding_id(), &inputs) + .await + .unwrap(); + assert!( + !stale.recompute_all && stale.dirty.is_empty() && stale.row_dirty.is_empty(), + "{stale:?}" + ); + } + } + #[rstest::rstest] #[case::same_definition(false)] #[case::changed_definition(true)] diff --git a/rust/lancedb/src/table/update.rs b/rust/lancedb/src/table/update.rs index 55980c5ff..93e8a08ea 100644 --- a/rust/lancedb/src/table/update.rs +++ b/rust/lancedb/src/table/update.rs @@ -103,11 +103,6 @@ pub(crate) async fn execute_update( // 1. Snapshot the current dataset let dataset = table.dataset.get().await?; crate::materialized_view::ensure_not_a_view(&table.name, &dataset.schema().metadata, "update")?; - super::computed_columns::ensure_not_function_bound( - &arrow_schema::Schema::from(dataset.schema()), - "update", - update.columns.iter().map(|(name, _)| name.as_str()), - )?; super::computed_columns::ensure_not_written( &arrow_schema::Schema::from(dataset.schema()), update.columns.iter().map(|(name, _)| name.as_str()), @@ -473,26 +468,38 @@ mod tests { assert_eq!(0, tbl.count_rows(Some("i == 0".to_string())).await.unwrap()); } - /// A Function binding protects only the columns it reads or writes; any - /// other column updates as on a plain table. #[tokio::test] - async fn update_on_a_function_table_refuses_only_bound_columns() { + async fn update_allows_function_inputs_but_protects_outputs() { let table = crate::table::computed_columns::test_support::bound_table().await; - let result = table.update().column("spare", "7").execute().await.unwrap(); + let result = table + .update() + .column("title", "'changed'") + .execute() + .await + .unwrap(); assert_eq!(result.rows_updated, 1); - for column in ["title", "search_text"] { + assert_eq!( + table + .count_rows(Some("title = 'changed' AND search_text = 'a b'".into())) + .await + .unwrap(), + 1 + ); + for column in ["search_text", "search_token_count"] { + let version = table.version().await.unwrap(); let err = table .update() - .column(column, "'x'") + .column("title", "'must not commit'") + .column(column, "NULL") .execute() .await .unwrap_err(); assert!( matches!(&err, crate::Error::InvalidInput { message } - if message.contains(&format!("update of '{column}'")) - && message.contains("a Function binding reads or writes it")), + if message.contains("is computed") && message.contains("cannot be written directly")), "{column}: {err:?}" ); + assert_eq!(table.version().await.unwrap(), version); } } }