From e5a5b67212ccc75ca7b69f3ac53de3dc045ffb34 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 14 Nov 2026 20:59:20 +0800 Subject: [PATCH] feat: support nullable scalar Function declarations (#5030) Function declarations currently reject nullable scalar returns. Accept `Optional[T]`, `T | None`, and explicit nullable Arrow fields, preserving `nullable: true` in registration signatures, JSON round trips, and remote `add_columns` requests. The Rust planner accepts nullable scalar outputs, and binding readback recognizes the server-owned assignment marker for a single nullable scalar result. Clients do not allocate that marker. Non-nullable whole results, whole structs, and Blob v2 retain their existing assignment restrictions. Validated with 127 Python tests, 30 documentation tests (1 skipped), and seven focused Rust tests covering planning, binding readback, rejection boundaries, and the remote request body. The final condition-collapse change addresses strict Clippy; its delta check and CI are pending. Python tests use a source overlay on a retained native extension, and Rust remote coverage uses HTTP mocks. Server support and end-to-end validation remain separate. This PR does not claim that server publication distinguishes computed NULL values from pending rows. --- docs/web-tests/py/test_function.py | 28 +- docs/web/curation/create-function.mdx | 11 +- docs/web/management/function.mdx | 42 ++- python/python/lancedb/functions.py | 35 +- .../tests/test_first_class_function_slice2.py | 64 +++- rust/lancedb/src/function.rs | 4 +- rust/lancedb/src/remote/table.rs | 77 +++++ rust/lancedb/src/table/computed_columns.rs | 306 +++++++++++++++++- 8 files changed, 517 insertions(+), 50 deletions(-) diff --git a/docs/web-tests/py/test_function.py b/docs/web-tests/py/test_function.py index 147520b18..ff4c6b9a3 100644 --- a/docs/web-tests/py/test_function.py +++ b/docs/web-tests/py/test_function.py @@ -154,13 +154,27 @@ def test_function_output_nullability(): ("note", True), ] - # A scalar output, and a struct declared as one nullable field, may not be. - for nullable in ( - pa.field("result", pa.float64(), nullable=True), - pa.field("result", pa.struct(fields), nullable=True), - ): - with pytest.raises(ValueError, match="non-nullable"): - function(input_schema=inputs, output_schema=nullable)(describe) + # A scalar output may be nullable. + @function(input_schema=inputs) + def maybe_double(value: float) -> float | None: + return None if value == 0 else value * 2 + + output = maybe_double.registration_request.signature.output + assert (output.kind, output.arrow_type, output.nullable) == ( + "scalar", + "float64", + True, + ) + assert json.loads(maybe_double.registration_request.signature.to_canonical_json())[ + "output" + ] == {"kind": "scalar", "arrow_type": "float64", "nullable": True} + + # A struct declared as one nullable field may not be. + with pytest.raises(ValueError, match="non-nullable"): + function( + input_schema=inputs, + output_schema=pa.field("result", pa.struct(fields), nullable=True), + )(describe) def test_function_registration_is_remote(tmp_path): diff --git a/docs/web/curation/create-function.mdx b/docs/web/curation/create-function.mdx index 9ee7d572a..0342849b4 100644 --- a/docs/web/curation/create-function.mdx +++ b/docs/web/curation/create-function.mdx @@ -295,11 +295,12 @@ extract_frame_version = db.create_function_async(extract_frame).wait( ``` -An empty byte string is a filled result. A scalar output cannot be nullable, -because a physical NULL marks an unfilled computed row. To return a nullable -blob as a value, use a named-struct output schema with a nullable blob field; -the struct itself records whether the result has been filled. Read a nested -blob with `table.fetch_blobs("payload.image", hits)`. +An empty byte string is a filled result. A top-level Blob v2 output cannot be +nullable; to return a nullable blob as a value, use a named-struct output +schema with a nullable blob field. The struct itself records whether the +result has been filled. A nullable non-Blob scalar output is declared with +`Optional[T]` or `T | None`; see [Functions](/management/function#define). +Read a nested blob with `table.fetch_blobs("payload.image", hits)`. A list or large-list output may also contain Blob v2 elements; inputs and fixed-size lists do not support this layout. List-nested values are queryable diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx index 27f45cb6a..57a16dbf2 100644 --- a/docs/web/management/function.mdx +++ b/docs/web/management/function.mdx @@ -66,11 +66,43 @@ your process, without the database, so unit tests stay ordinary tests: Where annotations cannot express the Arrow type precisely, pass `input_schema` and `output_schema` together: one without the other is rejected. A scalar -output must be non-nullable, because a physical NULL is what marks a -computed-column row as unfilled. A named-struct output may declare nullable -fields: whether a row has a result at all is the struct's own validity, which -Enterprise keeps in a separate internal column when it expands the result into -sibling columns, so a null field is a value rather than a missing row. +output is non-nullable by default. Annotate the return as `Optional[T]` or +`T | None`, or pass a nullable output field, to declare a nullable scalar: + +```python +import pyarrow as pa + +from lancedb import function + + +@function( + input_schema=pa.schema([pa.field("value", pa.int64(), nullable=False)]), + output_schema=pa.field("result", pa.int64(), nullable=True), +) +def maybe_double(value: int) -> int | None: + return None if value == 0 else value * 2 +``` + +The signature sent in the registration request then carries the nullable +scalar: + +```json +{ + "inputs": [{"name": "value", "arrow_type": "int64", "nullable": false}], + "output": {"kind": "scalar", "arrow_type": "int64", "nullable": true} +} +``` + + +Nullable scalar outputs require an Enterprise server that supports them; +older servers reject the registration. On a supporting server, a computed +`NULL` counts as a filled result, independently of its stored value. + + +A named-struct output may declare nullable fields: whether a row has a result +at all is the struct's own validity, which Enterprise keeps in a separate +internal column when it expands the result into sibling columns, so a null +field is a value rather than a missing row. ## Connect and create a table {#table} diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index f04d3d481..86230be52 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -312,7 +312,11 @@ class FunctionResultField(_OpenRemoteValue): class FunctionOutput(_OpenRemoteValue): - """Scalar or ordered named-struct output; unknown kinds remain decodable.""" + """Scalar or ordered named-struct output; unknown kinds remain decodable. + + A scalar output may be nullable. A named-struct output has no top-level + nullability; its fields carry their own flags. + """ kind: str arrow_type: Optional[str] = None @@ -1399,12 +1403,12 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp "output_schema must be a PyArrow DataType, Field, or Schema" ) _validate_exact_arrow_field(field) - if field.nullable: - raise ValueError("Function output must be non-nullable") + if field.nullable and _is_blob_v2_field(field): + raise ValueError("Blob v2 Function output must be non-nullable") return FunctionOutput( kind="scalar", arrow_type=_canonical_arrow_field(field, collection_blobs=True), - nullable=False, + nullable=field.nullable, ) if not fields: @@ -1504,9 +1508,9 @@ def _infer_signature( output = _function_output(output_schema) else: output_type, output_nullable = _annotation_type(annotations["return"]) - if output_nullable: - raise ValueError("Function output must be non-nullable") - output = _function_output(pa.field("result", output_type, nullable=False)) + output = _function_output( + pa.field("result", output_type, nullable=output_nullable) + ) if inputs is not None: return FunctionSignature(inputs=inputs, output=output) @@ -2650,9 +2654,13 @@ def function( ``None`` for a null value. PyArrow fields are nullable unless declared otherwise, so most columns need ``Optional``. For Arrow types annotations cannot express precisely, pass ``input_schema``, ``output_schema``, or - both; the one omitted is inferred. Scalar outputs must be non-nullable. - Every named-struct field may be nullable; Enterprise preserves the - struct's validity when the result is expanded into sibling columns. + both; the one omitted is inferred. A scalar output is non-nullable by + default; annotate the return as ``Optional[T]`` or ``T | None``, or pass a + nullable ``output_schema`` field, to declare a nullable scalar. The + deployment must accept nullable scalar outputs; an older server rejects + the registration. Every named-struct field may be nullable; Enterprise + preserves the struct's validity when the result is expanded into sibling + columns. **Large values.** A ``bytes`` parameter receives the whole value, which is held in memory with its row and counts against the server's per-row @@ -2757,9 +2765,10 @@ def function( Required for batch annotations. When omitted, inputs are inferred from the parameter annotations. output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional - Explicit scalar or named-struct output. Scalar outputs must be - non-nullable. Required for batch annotations. When omitted, the output - is inferred from the return annotation. + Explicit scalar or named-struct output. A scalar field's ``nullable`` + flag is preserved; a top-level named-struct field and a Blob v2 output + must be non-nullable. Required for batch annotations. When omitted, the + output is inferred from the return annotation. pip : sequence of str, optional Pip requirements for the remote environment. Syntax is validated at decoration time, including extras and environment markers. Named direct diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index a3a9e3e20..7e089a96d 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -28,6 +28,7 @@ import lancedb from lancedb import BlobReader from lancedb.db import AsyncConnection from lancedb.functions import ( + FunctionSignature, FunctionVersion, HuggingFaceModel, PythonRuntimeSpec, @@ -1214,6 +1215,42 @@ def test_explicit_arrow_schema_is_deterministic(): assert signature.output.nullable is False +def test_nullable_scalar_output_annotation_and_explicit_schema(): + input_schema = pa.schema([pa.field("value", pa.int64(), nullable=False)]) + + @function(input_schema=input_schema) + def optional_annotation(value: int) -> Optional[int]: + return None if value == 0 else value * 2 + + @function(input_schema=input_schema) + def optional_union(value: int) -> int | None: + return None if value == 0 else value * 2 + + @function( + input_schema=input_schema, + output_schema=pa.field("result", pa.int64(), nullable=True), + ) + def explicit_schema(value: int): + return None if value == 0 else value * 2 + + for definition in (optional_annotation, optional_union, explicit_schema): + signature = definition.registration_request.signature + assert (signature.output.kind, signature.output.arrow_type) == ( + "scalar", + "int64", + ) + assert signature.output.nullable is True + assert json.loads(signature.to_canonical_json())["output"] == { + "kind": "scalar", + "arrow_type": "int64", + "nullable": True, + } + assert ( + FunctionSignature.from_json(signature.to_canonical_json()).output.nullable + is True + ) + + def test_blob_fields_use_the_scalar_function_semantic_type(): @function( input_schema=pa.schema([lancedb.blob("image", nullable=False)]), @@ -1848,12 +1885,6 @@ def test_annotation_and_explicit_schema_validation_fail_closed(): def unsupported(value: set[str]) -> str: return "" - with pytest.raises(ValueError, match="output must be non-nullable"): - - @function - def nullable_output(value: int) -> Optional[int]: - return value - with pytest.raises(TypeError, match=r"missing Function annotations: \['return'\]"): @function(input_schema=pa.schema([pa.field("value", pa.int64())])) @@ -1873,10 +1904,25 @@ def test_annotation_and_explicit_schema_validation_fail_closed(): @function( input_schema=pa.schema([pa.field("value", pa.int64())]), - output_schema=pa.field("result", pa.int64(), nullable=True), + output_schema=pa.field( + "result", + pa.struct([pa.field("value", pa.int64(), nullable=False)]), + nullable=True, + ), ) - def nullable_explicit(value): - return value + def nullable_struct(value): + return {"value": value} + + with pytest.raises( + ValueError, match="Blob v2 Function output must be non-nullable" + ): + + @function( + input_schema=pa.schema([lancedb.blob("image", nullable=False)]), + output_schema=lancedb.blob("result", nullable=True), + ) + def nullable_blob(image): + return image for invalid_field in [ pa.field("", pa.int32(), nullable=False), diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index d1b63f2ad..387c80364 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -1032,8 +1032,8 @@ pub struct OutputMapping { pub nullable: bool, } -/// Internal physical column preserving the parent validity of a flattened -/// named-struct result. +/// Internal physical column preserving assignment of a nullable scalar result +/// or the parent validity of a flattened named-struct result. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct AssignmentMapping { pub output_name: String, diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 5c00232f7..72d67fb5a 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -10802,6 +10802,83 @@ mod tests { assert_eq!(result.version, 8); } + #[tokio::test] + async fn test_add_nullable_scalar_function_column_sends_atomic_null_declaration() { + let table = Table::new_with_handler("my_table", |request| { + match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body( + r#"{"version":1,"schema":{"fields":[{"name":"description","nullable":true,"type":{"type":"string"}}]}}"#, + ) + .unwrap(), + "/v1/table/my_table/add_columns/" => { + let actual: serde_json::Value = serde_json::from_slice( + request.body().unwrap().as_bytes().unwrap(), + ) + .unwrap(); + assert_eq!( + actual["new_columns"], + serde_json::json!([ + {"name": "maybe", "all_null": true} + ]) + ); + assert_eq!( + actual["function"]["application"]["output"], + serde_json::json!({ + "kind": "scalar", + "arrow_type": "int64", + "nullable": true + }) + ); + assert_eq!( + actual["function"]["output_schema"], + serde_json::json!({ + "fields": [{ + "name": "maybe", + "nullable": true, + "type": {"type": "int64"} + }] + }) + ); + assert_eq!( + actual["function"]["outputs"], + serde_json::json!([ + { + "result_field": "$value", + "output_name": "maybe", + "output_ordinal": 0 + } + ]) + ); + assert!(actual["function"].get("assignment").is_none()); + assert_eq!(actual["auto_refresh_enabled"], serde_json::json!(true)); + http::Response::builder() + .status(200) + .body(r#"{"version":8}"#) + .unwrap() + } + path => panic!("Unexpected path: {path}"), + } + }); + let application = crate::function::FunctionApplication::from_json( + r#"{ + "function":{"name":"nullable_scalar","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, + "inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}], + "output":{"kind":"scalar","arrow_type":"int64","nullable":true} + }"#, + ) + .unwrap(); + + let result = table + .add_columns() + .function_as("maybe", application) + .execute() + .await + .unwrap(); + assert_eq!(result.version, 8); + } + /// The fixture binding's table: `title` and `body` bound as inputs, its /// two outputs declared, plus an unbound `spare`. fn fixture_bound_schema() -> Schema { diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index 64e90b246..b01f45d06 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -40,7 +40,7 @@ use lance_namespace::models::{JsonArrowDataType, JsonArrowField, JsonArrowSchema use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::function::{FUNCTION_BLOB_V2_TYPE, FunctionApplication, FunctionBinding}; +use crate::function::{FUNCTION_BLOB_V2_TYPE, FunctionApplication, FunctionBinding, OutputMapping}; use crate::materialized_view::without_declarations; use crate::table::FunctionWorkerResourceBudget; use crate::utils::resolve_arrow_field_path; @@ -1289,6 +1289,21 @@ fn function_output_type_matches(expected: &DataType, actual: &DataType) -> bool } } +/// Whether a whole-result binding may carry the server-owned assignment marker. +/// +/// Only a nullable scalar result needs a marker to distinguish a computed NULL +/// from an unfilled row. A non-nullable result, a whole named-struct column, +/// and a Blob v2 value keep their existing rejection. +fn nullable_scalar_allows_assignment(output: &OutputMapping) -> Result { + if !output.nullable || output.arrow_type == FUNCTION_BLOB_V2_TYPE { + return Ok(false); + } + let json = parse_output_arrow_type(&output.arrow_type)?; + let data_type = lance_namespace::schema::convert_json_arrow_type(&json) + .map_err(|e| invalid_function(format!("invalid Function output type: {e}")))?; + Ok(!matches!(data_type, DataType::Struct(_))) +} + fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding) -> Result<()> { let mut input_fields = Vec::with_capacity(binding.inputs().len()); for input in binding.inputs() { @@ -1419,13 +1434,14 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding } let mut declared_output_schema = binding.output_schema().cloned(); if let Some(assignment) = binding.assignment() { - if binding + if let Some(whole_result) = binding .outputs() .iter() - .any(|output| output.result_field == WHOLE_RESULT_FIELD) + .find(|output| output.result_field == WHOLE_RESULT_FIELD) + && (binding.outputs().len() != 1 || !nullable_scalar_allows_assignment(whole_result)?) { return Err(invalid_function(format!( - "Function binding '{}' cannot attach an assignment column to a whole result", + "Function binding '{}' cannot attach an assignment column to a whole result; only a single nullable scalar output permits one", binding.binding_id() ))); } @@ -1552,14 +1568,17 @@ pub(crate) fn plan_function_application( "a scalar Function application must be mapped to one output column", ) })?; - if output.nullable != Some(false) { - return Err(invalid_function( - "Function logical outputs must be non-nullable during NULL assignment", - )); - } + let nullable = output.nullable.ok_or_else(|| { + invalid_function("scalar Function output must declare nullability") + })?; let arrow_type = output.arrow_type.as_deref().ok_or_else(|| { invalid_function("scalar Function output is missing its Arrow type") })?; + if nullable && arrow_type == FUNCTION_BLOB_V2_TYPE { + return Err(invalid_function( + "Blob v2 Function outputs cannot be nullable", + )); + } outputs.push(FunctionOutputTarget { result_field: WHOLE_RESULT_FIELD.to_string(), output_name: name.to_string(), @@ -4054,6 +4073,22 @@ mod tests { .unwrap() } + fn nullable_scalar_application() -> FunctionApplication { + FunctionApplication::from_json( + &serde_json::json!({ + "function": {"name": "nullable_scalar", "version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, + "inputs": [{ + "parameter": "value", + "kind": "column", + "value": {"path": "value"} + }], + "output": {"kind": "scalar", "arrow_type": "int64", "nullable": true} + }) + .to_string(), + ) + .unwrap() + } + fn binding_from_plan(plan: &FunctionDeclarationPlan) -> FunctionBinding { let inputs = plan .input_bindings @@ -4363,6 +4398,259 @@ mod tests { )); } + #[test] + fn test_nullable_scalar_application_plans_one_nullable_output() { + let input = ArrowField::new("value", DataType::Int64, false); + let plan = plan_function_application( + &ArrowSchema::new(vec![input]), + &nullable_scalar_application(), + Some("maybe"), + ) + .unwrap(); + + assert_eq!(plan.application.output().nullable, Some(true)); + assert_eq!(plan.outputs.len(), 1); + assert_eq!(plan.outputs[0].result_field, WHOLE_RESULT_FIELD); + assert_eq!(plan.outputs[0].output_name, "maybe"); + assert_eq!(plan.outputs[0].output_ordinal, 0); + assert_eq!(plan.output_schema.fields.len(), 1); + assert!(plan.output_schema.fields[0].nullable); + assert_eq!(plan.output_schema.fields[0].r#type.r#type, "int64"); + } + + #[test] + fn test_invalid_nullable_scalar_applications_fail_closed() { + let input = ArrowField::new("value", DataType::Int64, false); + let schema = ArrowSchema::new(vec![input]); + + let mut missing_nullability = serde_json::to_value(nullable_scalar_application()).unwrap(); + missing_nullability["output"] + .as_object_mut() + .unwrap() + .remove("nullable"); + let application = FunctionApplication::from_json(&missing_nullability.to_string()).unwrap(); + let err = plan_function_application(&schema, &application, Some("maybe")).unwrap_err(); + assert!( + matches!(&err, Error::InvalidInput { message } if message.contains("declare nullability")), + "{err:?}" + ); + + let nullable_blob = + blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":true}"#); + let err = plan_function_application( + &ArrowSchema::new(vec![crate::blob("image", false)]), + &nullable_blob, + Some("copy"), + ) + .unwrap_err(); + assert!( + matches!(&err, Error::InvalidInput { message } if message.contains("Blob v2") && message.contains("nullable")), + "{err:?}" + ); + } + + #[test] + fn test_nullable_scalar_binding_accepts_server_owned_assignment() { + let input = ArrowField::new("value", DataType::Int64, false); + let plan = plan_function_application( + &ArrowSchema::new(vec![input.clone()]), + &nullable_scalar_application(), + Some("maybe"), + ) + .unwrap(); + let assignment_name = "__function_assignment_nullable_scalar"; + let mut raw = serde_json::to_value(binding_from_plan(&plan)).unwrap(); + raw["outputs"][0]["nullable"] = Value::Bool(true); + raw["assignment"] = serde_json::json!({ + "output_name": assignment_name, + "output_field_id": -1, + }); + raw["output_schema"]["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "name": assignment_name, + "nullable": true, + "type": {"type": "bool"}, + })); + let binding: FunctionBinding = serde_json::from_value(raw).unwrap(); + let output = ArrowField::new("maybe", DataType::Int64, true).with_metadata( + function_computed_column_metadata(binding.binding_id(), 0, &["value".into()]), + ); + let assignment = ArrowField::new(assignment_name, DataType::Boolean, true).with_metadata( + function_computed_column_metadata( + binding.binding_id(), + FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL, + &["value".into()], + ), + ); + + let schema = ArrowSchema::new(vec![input.clone(), output.clone(), assignment]); + ensure_binding_matches_schema(&schema, &binding).unwrap(); + let hidden = ArrowSchema::new(vec![input, output]); + ensure_binding_matches_schema(&hidden, &binding).unwrap(); + + let schema = ArrowSchema::new_with_metadata( + schema.fields().to_vec(), + HashMap::from([( + FUNCTION_BINDINGS_META_KEY.to_string(), + function_bindings_metadata(std::slice::from_ref(&binding)).unwrap(), + )]), + ); + ensure_supported_function_metadata(&schema).unwrap(); + } + + #[test] + fn test_non_nullable_scalar_binding_rejects_server_owned_assignment() { + let input = ArrowField::new("value", DataType::Int64, false); + let plan = plan_function_application( + &ArrowSchema::new(vec![input.clone()]), + &single_input_application("value"), + Some("maybe"), + ) + .unwrap(); + let assignment_name = "__function_assignment_non_nullable_scalar"; + let mut raw = serde_json::to_value(binding_from_plan(&plan)).unwrap(); + raw["assignment"] = serde_json::json!({ + "output_name": assignment_name, + "output_field_id": -1, + }); + raw["output_schema"]["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "name": assignment_name, + "nullable": true, + "type": {"type": "bool"}, + })); + let binding: FunctionBinding = serde_json::from_value(raw).unwrap(); + let output = ArrowField::new("maybe", DataType::Int64, true).with_metadata( + function_computed_column_metadata(binding.binding_id(), 0, &["value".into()]), + ); + let assignment = ArrowField::new(assignment_name, DataType::Boolean, true).with_metadata( + function_computed_column_metadata( + binding.binding_id(), + FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL, + &["value".into()], + ), + ); + + let err = ensure_binding_matches_schema( + &ArrowSchema::new(vec![input, output, assignment]), + &binding, + ) + .unwrap_err(); + assert!( + matches!(&err, Error::InvalidInput { message } if message.contains("nullable scalar")), + "{err:?}" + ); + } + + #[test] + fn test_named_struct_whole_binding_rejects_server_owned_assignment() { + let plan = plan_function_application( + &function_input_schema(), + &named_struct_application("{}"), + Some("features"), + ) + .unwrap(); + let assignment_name = "__function_assignment_whole_struct"; + let mut raw = serde_json::to_value(binding_from_plan(&plan)).unwrap(); + raw["outputs"][0]["nullable"] = Value::Bool(true); + raw["assignment"] = serde_json::json!({ + "output_name": assignment_name, + "output_field_id": -1, + }); + raw["output_schema"]["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "name": assignment_name, + "nullable": true, + "type": {"type": "bool"}, + })); + let binding: FunctionBinding = serde_json::from_value(raw).unwrap(); + let output = + lance_namespace::schema::convert_json_arrow_field(&plan.output_schema.fields[0]) + .unwrap() + .with_metadata(function_computed_column_metadata( + binding.binding_id(), + 0, + &["title".into(), "body".into()], + )); + let assignment = ArrowField::new(assignment_name, DataType::Boolean, true).with_metadata( + function_computed_column_metadata( + binding.binding_id(), + FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL, + &["title".into(), "body".into()], + ), + ); + let mut fields = function_input_schema() + .fields() + .iter() + .map(|field| field.as_ref().clone()) + .collect::>(); + fields.push(output); + fields.push(assignment); + + let err = ensure_binding_matches_schema(&ArrowSchema::new(fields), &binding).unwrap_err(); + assert!( + matches!(&err, Error::InvalidInput { message } if message.contains("nullable scalar")), + "{err:?}" + ); + } + + #[test] + fn test_blob_whole_binding_rejects_server_owned_assignment() { + let input = crate::blob("image", false); + let plan = plan_function_application( + &ArrowSchema::new(vec![input.clone()]), + &blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#), + Some("thumbnail"), + ) + .unwrap(); + let assignment_name = "__function_assignment_whole_blob"; + let mut raw = serde_json::to_value(binding_from_plan(&plan)).unwrap(); + raw["outputs"][0]["nullable"] = Value::Bool(true); + raw["assignment"] = serde_json::json!({ + "output_name": assignment_name, + "output_field_id": -1, + }); + raw["output_schema"]["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "name": assignment_name, + "nullable": true, + "type": {"type": "bool"}, + })); + let binding: FunctionBinding = serde_json::from_value(raw).unwrap(); + let mut metadata = full_blob_field("thumbnail", true).metadata().clone(); + metadata.extend(function_computed_column_metadata( + binding.binding_id(), + 0, + &["image".into()], + )); + let output = full_blob_field("thumbnail", true).with_metadata(metadata); + let assignment = ArrowField::new(assignment_name, DataType::Boolean, true).with_metadata( + function_computed_column_metadata( + binding.binding_id(), + FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL, + &["image".into()], + ), + ); + + let err = ensure_binding_matches_schema( + &ArrowSchema::new(vec![input, output, assignment]), + &binding, + ) + .unwrap_err(); + assert!( + matches!(&err, Error::InvalidInput { message } if message.contains("nullable scalar")), + "{err:?}" + ); + } + #[test] fn test_nullable_function_input_cannot_bind_to_non_nullable_parameter() { let mut raw_binding: Value = serde_json::from_str(include_str!(