diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index d09a790a3..64e90b246 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -1757,10 +1757,20 @@ pub(crate) fn ensure_not_written<'a>( schema: &ArrowSchema, written: impl IntoIterator, ) -> Result<()> { - let declared: Vec = computed_columns(schema) + ensure_supported_function_metadata(schema)?; + let mut declared: Vec = computed_columns(schema) .into_iter() .map(|declaration| declaration.name) .collect(); + declared.extend( + function_bindings(schema)? + .into_iter() + .filter_map(|binding| { + binding + .assignment() + .map(|assignment| assignment.output_name.clone()) + }), + ); for name in written { if declared.iter().any(|declared| *declared == root(name)) { return Err(Error::InvalidInput { @@ -2532,10 +2542,15 @@ pub(crate) mod test_support { )]) .await .unwrap(); - let inputs = ["title".to_string(), "body".to_string()]; + let schema = dataset.schema(); let outputs = bindings .iter() .flat_map(|binding| { + let inputs = binding + .inputs() + .iter() + .map(|input| input.field_path.clone()) + .collect::>(); let assignment = binding.assignment().map(|assignment| { ( assignment.output_name.clone(), @@ -2547,9 +2562,9 @@ pub(crate) mod test_support { .iter() .map(|output| (output.output_name.clone(), output.output_ordinal)) .chain(assignment) - .map(|(name, ordinal)| { + .map(move |(name, ordinal)| { ( - dataset.schema().field(&name).unwrap().id as u32, + schema.field(&name).unwrap().id as u32, function_computed_column_metadata( binding.binding_id(), ordinal, @@ -4144,6 +4159,37 @@ mod tests { ArrowSchema::new(fields) } + #[test] + fn input_writes_preserve_the_assignment_column_guard() { + let mut raw: Value = serde_json::to_value(test_support::text_features_binding("")).unwrap(); + raw["assignment"] = serde_json::json!({ + "output_name": "__function_assignment_fb_01K3TEXT", "output_field_id": -1, + }); + raw["outputs"][0]["nullable"] = Value::Bool(true); + raw["outputs"][1]["nullable"] = Value::Bool(true); + raw["output_schema"]["fields"].as_array_mut().unwrap().push(serde_json::json!({ + "name": "__function_assignment_fb_01K3TEXT", "nullable": true, "type": {"type": "bool"}, + })); + let binding: FunctionBinding = serde_json::from_value(raw).unwrap(); + let schema = ArrowSchema::new_with_metadata( + valid_function_binding_schema(true, true, &binding) + .fields() + .to_vec(), + HashMap::from([( + FUNCTION_BINDINGS_META_KEY.into(), + function_bindings_metadata(&[binding]).unwrap(), + )]), + ); + ensure_not_written(&schema, ["title", "body"]).unwrap(); + for output in [ + "search_text", + "search_token_count", + "__function_assignment_fb_01K3TEXT", + ] { + assert!(ensure_not_written(&schema, [output]).is_err(), "{output}"); + } + } + #[test] fn test_non_nullable_function_inputs_can_bind_to_nullable_parameters() { let binding = FunctionBinding::from_json(include_str!( diff --git a/rust/lancedb/src/table/freshness.rs b/rust/lancedb/src/table/freshness.rs index 645653a46..563a2a4e5 100644 --- a/rust/lancedb/src/table/freshness.rs +++ b/rust/lancedb/src/table/freshness.rs @@ -2646,6 +2646,8 @@ mod tests { use lance_file::version::ConcreteFileVersion; use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use crate::table::computed_columns::test_support::{stamp_bindings, text_features_binding}; + const COLUMN: &str = "doubled"; const EXPRESSION: &str = "value * 2"; @@ -3126,6 +3128,126 @@ mod tests { } } + #[tokio::test] + async fn input_update_dirties_only_the_updated_rows_and_retains_outputs() { + let table = fresh_function_table().await; + let result = table + .update() + .only_if("title = 'a'") + .column("body", "'changed'") + .execute() + .await + .unwrap(); + assert_eq!(result.rows_updated, 1); + let dataset = table.as_native().unwrap().dataset.get().await.unwrap(); + for output in ["search_text", "search_token_count"] { + let stale = function_plan(&dataset, output).await; + let mut dirty_rows = 0; + for fragment in dataset.get_fragments() { + if stale.needs_recompute(fragment.id() as u32) { + let batch = fragment + .scan() + .project(&["title"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let titles = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(titles.iter().all(|title| title == Some("a")), "{stale:?}"); + dirty_rows += batch.num_rows(); + } + } + assert_eq!(dirty_rows, 1, "{stale:?}"); + } + assert_eq!(table.count_rows(Some("title = 'a' AND body = 'changed' AND search_text = 'a x' AND search_token_count = 2".into())).await.unwrap(), 1); + } + + #[tokio::test] + async fn input_update_keeps_an_independent_function_binding_fresh() { + let batch = arrow_array::record_batch!( + ("title", Utf8, ["a", "b", "c", "d"]), + ("body", Utf8, ["x", "y", "z", "w"]), + ("search_text", Utf8, ["a x", "b y", "c z", "d w"]), + ("search_token_count", Int64, [2, 2, 2, 2]), + ("search_text_other", Utf8, ["a a", "b b", "c c", "d d"]), + ("search_token_count_other", Int64, [2, 2, 2, 2]) + ) + .unwrap(); + let table = crate::connect("memory://") + .execute() + .await + .unwrap() + .create_table("independent_functions", batch) + .write_options(crate::table::WriteOptions { + lance_write_params: Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + }) + .execute() + .await + .unwrap(); + let mut other = serde_json::to_value(text_features_binding("_other")).unwrap(); + other["inputs"][1]["field_path"] = serde_json::json!("title"); + other["inputs"][1]["field_id"] = other["inputs"][0]["field_id"].clone(); + let bindings = [ + text_features_binding(""), + serde_json::from_value(other).unwrap(), + ]; + stamp_bindings(&table, &bindings).await; + let native = table.as_native().unwrap(); + let mut dataset = native.dataset.get().await.unwrap().as_ref().clone(); + for binding in &bindings { + let paths = binding + .inputs() + .iter() + .map(|input| input.field_path.clone()) + .collect::>(); + let inputs = fields_for_paths(dataset.schema(), &paths).unwrap(); + let signatures = signatures_for(&dataset, &[0, 1], &inputs).unwrap(); + for output in binding.outputs() { + write_signatures( + &mut dataset, + &output.output_name, + binding.binding_id(), + signatures.clone(), + ) + .await + .unwrap(); + } + } + native.dataset.update(dataset); + table + .update() + .only_if("title = 'a'") + .column("body", "'changed'") + .execute() + .await + .unwrap(); + let dataset = native.dataset.get().await.unwrap(); + assert!( + function_plan(&dataset, "search_text") + .await + .needs_recompute(2) + ); + let other = &bindings[1]; + let inputs = fields_for_paths(dataset.schema(), &["title".into()]).unwrap(); + for output in other.outputs() { + let stale = + staleness_against(&dataset, &output.output_name, other.binding_id(), &inputs) + .await + .unwrap(); + assert!( + !stale.recompute_all && stale.dirty.is_empty() && stale.row_dirty.is_empty(), + "{stale:?}" + ); + } + } + #[rstest::rstest] #[case::same_definition(false)] #[case::changed_definition(true)] diff --git a/rust/lancedb/src/table/update.rs b/rust/lancedb/src/table/update.rs index 55980c5ff..93e8a08ea 100644 --- a/rust/lancedb/src/table/update.rs +++ b/rust/lancedb/src/table/update.rs @@ -103,11 +103,6 @@ pub(crate) async fn execute_update( // 1. Snapshot the current dataset let dataset = table.dataset.get().await?; crate::materialized_view::ensure_not_a_view(&table.name, &dataset.schema().metadata, "update")?; - super::computed_columns::ensure_not_function_bound( - &arrow_schema::Schema::from(dataset.schema()), - "update", - update.columns.iter().map(|(name, _)| name.as_str()), - )?; super::computed_columns::ensure_not_written( &arrow_schema::Schema::from(dataset.schema()), update.columns.iter().map(|(name, _)| name.as_str()), @@ -473,26 +468,38 @@ mod tests { assert_eq!(0, tbl.count_rows(Some("i == 0".to_string())).await.unwrap()); } - /// A Function binding protects only the columns it reads or writes; any - /// other column updates as on a plain table. #[tokio::test] - async fn update_on_a_function_table_refuses_only_bound_columns() { + async fn update_allows_function_inputs_but_protects_outputs() { let table = crate::table::computed_columns::test_support::bound_table().await; - let result = table.update().column("spare", "7").execute().await.unwrap(); + let result = table + .update() + .column("title", "'changed'") + .execute() + .await + .unwrap(); assert_eq!(result.rows_updated, 1); - for column in ["title", "search_text"] { + assert_eq!( + table + .count_rows(Some("title = 'changed' AND search_text = 'a b'".into())) + .await + .unwrap(), + 1 + ); + for column in ["search_text", "search_token_count"] { + let version = table.version().await.unwrap(); let err = table .update() - .column(column, "'x'") + .column("title", "'must not commit'") + .column(column, "NULL") .execute() .await .unwrap_err(); assert!( matches!(&err, crate::Error::InvalidInput { message } - if message.contains(&format!("update of '{column}'")) - && message.contains("a Function binding reads or writes it")), + if message.contains("is computed") && message.contains("cannot be written directly")), "{column}: {err:?}" ); + assert_eq!(table.version().await.unwrap(), version); } } }