mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
fix: allow updates to Function input columns (#4985)
Updating an input column on a table with a Function binding currently
fails before the write, although the write path already preserves
freshness for bindings whose inputs are unchanged. Allow input updates
through the normal update builder while retaining supported
binding-metadata validation and the guard against writing computed or
internal assignment outputs directly.
Lance rewrites only the matched rows. Their affected Function outputs
keep their previous values and their changed input signatures make them
eligible for refresh; unchanged fragments retain their freshness.
Rewrites affecting only unrelated inputs keep the existing
freshness-preservation path.
For a table whose Function reads `title`, this update now succeeds while
retaining the previous computed values until refresh:
```python
table.update(where="id = 7", values={"title": "revised"})
```
Server consumers need this crate update before remote input updates gain
the behavior.
Adds regressions for a one-row input update, retained output values,
freshness scope, independent binding preservation, and atomic rejection
of mixed input/output writes. Local builds and tests are disabled by
request; CI runs the regressions.
This commit is contained in:
@@ -1757,10 +1757,20 @@ pub(crate) fn ensure_not_written<'a>(
|
||||
schema: &ArrowSchema,
|
||||
written: impl IntoIterator<Item = &'a str>,
|
||||
) -> Result<()> {
|
||||
let declared: Vec<String> = computed_columns(schema)
|
||||
ensure_supported_function_metadata(schema)?;
|
||||
let mut declared: Vec<String> = computed_columns(schema)
|
||||
.into_iter()
|
||||
.map(|declaration| declaration.name)
|
||||
.collect();
|
||||
declared.extend(
|
||||
function_bindings(schema)?
|
||||
.into_iter()
|
||||
.filter_map(|binding| {
|
||||
binding
|
||||
.assignment()
|
||||
.map(|assignment| assignment.output_name.clone())
|
||||
}),
|
||||
);
|
||||
for name in written {
|
||||
if declared.iter().any(|declared| *declared == root(name)) {
|
||||
return Err(Error::InvalidInput {
|
||||
@@ -2532,10 +2542,15 @@ pub(crate) mod test_support {
|
||||
)])
|
||||
.await
|
||||
.unwrap();
|
||||
let inputs = ["title".to_string(), "body".to_string()];
|
||||
let schema = dataset.schema();
|
||||
let outputs = bindings
|
||||
.iter()
|
||||
.flat_map(|binding| {
|
||||
let inputs = binding
|
||||
.inputs()
|
||||
.iter()
|
||||
.map(|input| input.field_path.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let assignment = binding.assignment().map(|assignment| {
|
||||
(
|
||||
assignment.output_name.clone(),
|
||||
@@ -2547,9 +2562,9 @@ pub(crate) mod test_support {
|
||||
.iter()
|
||||
.map(|output| (output.output_name.clone(), output.output_ordinal))
|
||||
.chain(assignment)
|
||||
.map(|(name, ordinal)| {
|
||||
.map(move |(name, ordinal)| {
|
||||
(
|
||||
dataset.schema().field(&name).unwrap().id as u32,
|
||||
schema.field(&name).unwrap().id as u32,
|
||||
function_computed_column_metadata(
|
||||
binding.binding_id(),
|
||||
ordinal,
|
||||
@@ -4144,6 +4159,37 @@ mod tests {
|
||||
ArrowSchema::new(fields)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_writes_preserve_the_assignment_column_guard() {
|
||||
let mut raw: Value = serde_json::to_value(test_support::text_features_binding("")).unwrap();
|
||||
raw["assignment"] = serde_json::json!({
|
||||
"output_name": "__function_assignment_fb_01K3TEXT", "output_field_id": -1,
|
||||
});
|
||||
raw["outputs"][0]["nullable"] = Value::Bool(true);
|
||||
raw["outputs"][1]["nullable"] = Value::Bool(true);
|
||||
raw["output_schema"]["fields"].as_array_mut().unwrap().push(serde_json::json!({
|
||||
"name": "__function_assignment_fb_01K3TEXT", "nullable": true, "type": {"type": "bool"},
|
||||
}));
|
||||
let binding: FunctionBinding = serde_json::from_value(raw).unwrap();
|
||||
let schema = ArrowSchema::new_with_metadata(
|
||||
valid_function_binding_schema(true, true, &binding)
|
||||
.fields()
|
||||
.to_vec(),
|
||||
HashMap::from([(
|
||||
FUNCTION_BINDINGS_META_KEY.into(),
|
||||
function_bindings_metadata(&[binding]).unwrap(),
|
||||
)]),
|
||||
);
|
||||
ensure_not_written(&schema, ["title", "body"]).unwrap();
|
||||
for output in [
|
||||
"search_text",
|
||||
"search_token_count",
|
||||
"__function_assignment_fb_01K3TEXT",
|
||||
] {
|
||||
assert!(ensure_not_written(&schema, [output]).is_err(), "{output}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_non_nullable_function_inputs_can_bind_to_nullable_parameters() {
|
||||
let binding = FunctionBinding::from_json(include_str!(
|
||||
|
||||
@@ -2646,6 +2646,8 @@ mod tests {
|
||||
use lance_file::version::ConcreteFileVersion;
|
||||
use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
|
||||
|
||||
use crate::table::computed_columns::test_support::{stamp_bindings, text_features_binding};
|
||||
|
||||
const COLUMN: &str = "doubled";
|
||||
const EXPRESSION: &str = "value * 2";
|
||||
|
||||
@@ -3126,6 +3128,126 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn input_update_dirties_only_the_updated_rows_and_retains_outputs() {
|
||||
let table = fresh_function_table().await;
|
||||
let result = table
|
||||
.update()
|
||||
.only_if("title = 'a'")
|
||||
.column("body", "'changed'")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.rows_updated, 1);
|
||||
let dataset = table.as_native().unwrap().dataset.get().await.unwrap();
|
||||
for output in ["search_text", "search_token_count"] {
|
||||
let stale = function_plan(&dataset, output).await;
|
||||
let mut dirty_rows = 0;
|
||||
for fragment in dataset.get_fragments() {
|
||||
if stale.needs_recompute(fragment.id() as u32) {
|
||||
let batch = fragment
|
||||
.scan()
|
||||
.project(&["title"])
|
||||
.unwrap()
|
||||
.try_into_batch()
|
||||
.await
|
||||
.unwrap();
|
||||
let titles = batch
|
||||
.column(0)
|
||||
.as_any()
|
||||
.downcast_ref::<arrow_array::StringArray>()
|
||||
.unwrap();
|
||||
assert!(titles.iter().all(|title| title == Some("a")), "{stale:?}");
|
||||
dirty_rows += batch.num_rows();
|
||||
}
|
||||
}
|
||||
assert_eq!(dirty_rows, 1, "{stale:?}");
|
||||
}
|
||||
assert_eq!(table.count_rows(Some("title = 'a' AND body = 'changed' AND search_text = 'a x' AND search_token_count = 2".into())).await.unwrap(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn input_update_keeps_an_independent_function_binding_fresh() {
|
||||
let batch = arrow_array::record_batch!(
|
||||
("title", Utf8, ["a", "b", "c", "d"]),
|
||||
("body", Utf8, ["x", "y", "z", "w"]),
|
||||
("search_text", Utf8, ["a x", "b y", "c z", "d w"]),
|
||||
("search_token_count", Int64, [2, 2, 2, 2]),
|
||||
("search_text_other", Utf8, ["a a", "b b", "c c", "d d"]),
|
||||
("search_token_count_other", Int64, [2, 2, 2, 2])
|
||||
)
|
||||
.unwrap();
|
||||
let table = crate::connect("memory://")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.create_table("independent_functions", batch)
|
||||
.write_options(crate::table::WriteOptions {
|
||||
lance_write_params: Some(WriteParams {
|
||||
max_rows_per_file: 2,
|
||||
..Default::default()
|
||||
}),
|
||||
})
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
let mut other = serde_json::to_value(text_features_binding("_other")).unwrap();
|
||||
other["inputs"][1]["field_path"] = serde_json::json!("title");
|
||||
other["inputs"][1]["field_id"] = other["inputs"][0]["field_id"].clone();
|
||||
let bindings = [
|
||||
text_features_binding(""),
|
||||
serde_json::from_value(other).unwrap(),
|
||||
];
|
||||
stamp_bindings(&table, &bindings).await;
|
||||
let native = table.as_native().unwrap();
|
||||
let mut dataset = native.dataset.get().await.unwrap().as_ref().clone();
|
||||
for binding in &bindings {
|
||||
let paths = binding
|
||||
.inputs()
|
||||
.iter()
|
||||
.map(|input| input.field_path.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let inputs = fields_for_paths(dataset.schema(), &paths).unwrap();
|
||||
let signatures = signatures_for(&dataset, &[0, 1], &inputs).unwrap();
|
||||
for output in binding.outputs() {
|
||||
write_signatures(
|
||||
&mut dataset,
|
||||
&output.output_name,
|
||||
binding.binding_id(),
|
||||
signatures.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
native.dataset.update(dataset);
|
||||
table
|
||||
.update()
|
||||
.only_if("title = 'a'")
|
||||
.column("body", "'changed'")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
let dataset = native.dataset.get().await.unwrap();
|
||||
assert!(
|
||||
function_plan(&dataset, "search_text")
|
||||
.await
|
||||
.needs_recompute(2)
|
||||
);
|
||||
let other = &bindings[1];
|
||||
let inputs = fields_for_paths(dataset.schema(), &["title".into()]).unwrap();
|
||||
for output in other.outputs() {
|
||||
let stale =
|
||||
staleness_against(&dataset, &output.output_name, other.binding_id(), &inputs)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
!stale.recompute_all && stale.dirty.is_empty() && stale.row_dirty.is_empty(),
|
||||
"{stale:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::same_definition(false)]
|
||||
#[case::changed_definition(true)]
|
||||
|
||||
@@ -103,11 +103,6 @@ pub(crate) async fn execute_update(
|
||||
// 1. Snapshot the current dataset
|
||||
let dataset = table.dataset.get().await?;
|
||||
crate::materialized_view::ensure_not_a_view(&table.name, &dataset.schema().metadata, "update")?;
|
||||
super::computed_columns::ensure_not_function_bound(
|
||||
&arrow_schema::Schema::from(dataset.schema()),
|
||||
"update",
|
||||
update.columns.iter().map(|(name, _)| name.as_str()),
|
||||
)?;
|
||||
super::computed_columns::ensure_not_written(
|
||||
&arrow_schema::Schema::from(dataset.schema()),
|
||||
update.columns.iter().map(|(name, _)| name.as_str()),
|
||||
@@ -473,26 +468,38 @@ mod tests {
|
||||
assert_eq!(0, tbl.count_rows(Some("i == 0".to_string())).await.unwrap());
|
||||
}
|
||||
|
||||
/// A Function binding protects only the columns it reads or writes; any
|
||||
/// other column updates as on a plain table.
|
||||
#[tokio::test]
|
||||
async fn update_on_a_function_table_refuses_only_bound_columns() {
|
||||
async fn update_allows_function_inputs_but_protects_outputs() {
|
||||
let table = crate::table::computed_columns::test_support::bound_table().await;
|
||||
let result = table.update().column("spare", "7").execute().await.unwrap();
|
||||
let result = table
|
||||
.update()
|
||||
.column("title", "'changed'")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.rows_updated, 1);
|
||||
for column in ["title", "search_text"] {
|
||||
assert_eq!(
|
||||
table
|
||||
.count_rows(Some("title = 'changed' AND search_text = 'a b'".into()))
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
for column in ["search_text", "search_token_count"] {
|
||||
let version = table.version().await.unwrap();
|
||||
let err = table
|
||||
.update()
|
||||
.column(column, "'x'")
|
||||
.column("title", "'must not commit'")
|
||||
.column(column, "NULL")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, crate::Error::InvalidInput { message }
|
||||
if message.contains(&format!("update of '{column}'"))
|
||||
&& message.contains("a Function binding reads or writes it")),
|
||||
if message.contains("is computed") && message.contains("cannot be written directly")),
|
||||
"{column}: {err:?}"
|
||||
);
|
||||
assert_eq!(table.version().await.unwrap(), version);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user