diff --git a/docs/web-tests/py/test_function.py b/docs/web-tests/py/test_function.py
index 147520b18..ff4c6b9a3 100644
--- a/docs/web-tests/py/test_function.py
+++ b/docs/web-tests/py/test_function.py
@@ -154,13 +154,27 @@ def test_function_output_nullability():
("note", True),
]
- # A scalar output, and a struct declared as one nullable field, may not be.
- for nullable in (
- pa.field("result", pa.float64(), nullable=True),
- pa.field("result", pa.struct(fields), nullable=True),
- ):
- with pytest.raises(ValueError, match="non-nullable"):
- function(input_schema=inputs, output_schema=nullable)(describe)
+ # A scalar output may be nullable.
+ @function(input_schema=inputs)
+ def maybe_double(value: float) -> float | None:
+ return None if value == 0 else value * 2
+
+ output = maybe_double.registration_request.signature.output
+ assert (output.kind, output.arrow_type, output.nullable) == (
+ "scalar",
+ "float64",
+ True,
+ )
+ assert json.loads(maybe_double.registration_request.signature.to_canonical_json())[
+ "output"
+ ] == {"kind": "scalar", "arrow_type": "float64", "nullable": True}
+
+ # A struct declared as one nullable field may not be.
+ with pytest.raises(ValueError, match="non-nullable"):
+ function(
+ input_schema=inputs,
+ output_schema=pa.field("result", pa.struct(fields), nullable=True),
+ )(describe)
def test_function_registration_is_remote(tmp_path):
diff --git a/docs/web/curation/create-function.mdx b/docs/web/curation/create-function.mdx
index 9ee7d572a..0342849b4 100644
--- a/docs/web/curation/create-function.mdx
+++ b/docs/web/curation/create-function.mdx
@@ -295,11 +295,12 @@ extract_frame_version = db.create_function_async(extract_frame).wait(
```
-An empty byte string is a filled result. A scalar output cannot be nullable,
-because a physical NULL marks an unfilled computed row. To return a nullable
-blob as a value, use a named-struct output schema with a nullable blob field;
-the struct itself records whether the result has been filled. Read a nested
-blob with `table.fetch_blobs("payload.image", hits)`.
+An empty byte string is a filled result. A top-level Blob v2 output cannot be
+nullable; to return a nullable blob as a value, use a named-struct output
+schema with a nullable blob field. The struct itself records whether the
+result has been filled. A nullable non-Blob scalar output is declared with
+`Optional[T]` or `T | None`; see [Functions](/management/function#define).
+Read a nested blob with `table.fetch_blobs("payload.image", hits)`.
A list or large-list output may also contain Blob v2 elements; inputs and
fixed-size lists do not support this layout. List-nested values are queryable
diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx
index 27f45cb6a..57a16dbf2 100644
--- a/docs/web/management/function.mdx
+++ b/docs/web/management/function.mdx
@@ -66,11 +66,43 @@ your process, without the database, so unit tests stay ordinary tests:
Where annotations cannot express the Arrow type precisely, pass `input_schema`
and `output_schema` together: one without the other is rejected. A scalar
-output must be non-nullable, because a physical NULL is what marks a
-computed-column row as unfilled. A named-struct output may declare nullable
-fields: whether a row has a result at all is the struct's own validity, which
-Enterprise keeps in a separate internal column when it expands the result into
-sibling columns, so a null field is a value rather than a missing row.
+output is non-nullable by default. Annotate the return as `Optional[T]` or
+`T | None`, or pass a nullable output field, to declare a nullable scalar:
+
+```python
+import pyarrow as pa
+
+from lancedb import function
+
+
+@function(
+ input_schema=pa.schema([pa.field("value", pa.int64(), nullable=False)]),
+ output_schema=pa.field("result", pa.int64(), nullable=True),
+)
+def maybe_double(value: int) -> int | None:
+ return None if value == 0 else value * 2
+```
+
+The signature sent in the registration request then carries the nullable
+scalar:
+
+```json
+{
+ "inputs": [{"name": "value", "arrow_type": "int64", "nullable": false}],
+ "output": {"kind": "scalar", "arrow_type": "int64", "nullable": true}
+}
+```
+
+
+Nullable scalar outputs require an Enterprise server that supports them;
+older servers reject the registration. On a supporting server, a computed
+`NULL` counts as a filled result, independently of its stored value.
+
+
+A named-struct output may declare nullable fields: whether a row has a result
+at all is the struct's own validity, which Enterprise keeps in a separate
+internal column when it expands the result into sibling columns, so a null
+field is a value rather than a missing row.
## Connect and create a table {#table}
diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py
index f04d3d481..86230be52 100644
--- a/python/python/lancedb/functions.py
+++ b/python/python/lancedb/functions.py
@@ -312,7 +312,11 @@ class FunctionResultField(_OpenRemoteValue):
class FunctionOutput(_OpenRemoteValue):
- """Scalar or ordered named-struct output; unknown kinds remain decodable."""
+ """Scalar or ordered named-struct output; unknown kinds remain decodable.
+
+ A scalar output may be nullable. A named-struct output has no top-level
+ nullability; its fields carry their own flags.
+ """
kind: str
arrow_type: Optional[str] = None
@@ -1399,12 +1403,12 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
"output_schema must be a PyArrow DataType, Field, or Schema"
)
_validate_exact_arrow_field(field)
- if field.nullable:
- raise ValueError("Function output must be non-nullable")
+ if field.nullable and _is_blob_v2_field(field):
+ raise ValueError("Blob v2 Function output must be non-nullable")
return FunctionOutput(
kind="scalar",
arrow_type=_canonical_arrow_field(field, collection_blobs=True),
- nullable=False,
+ nullable=field.nullable,
)
if not fields:
@@ -1504,9 +1508,9 @@ def _infer_signature(
output = _function_output(output_schema)
else:
output_type, output_nullable = _annotation_type(annotations["return"])
- if output_nullable:
- raise ValueError("Function output must be non-nullable")
- output = _function_output(pa.field("result", output_type, nullable=False))
+ output = _function_output(
+ pa.field("result", output_type, nullable=output_nullable)
+ )
if inputs is not None:
return FunctionSignature(inputs=inputs, output=output)
@@ -2650,9 +2654,13 @@ def function(
``None`` for a null value. PyArrow fields are nullable unless declared
otherwise, so most columns need ``Optional``. For Arrow types annotations
cannot express precisely, pass ``input_schema``, ``output_schema``, or
- both; the one omitted is inferred. Scalar outputs must be non-nullable.
- Every named-struct field may be nullable; Enterprise preserves the
- struct's validity when the result is expanded into sibling columns.
+ both; the one omitted is inferred. A scalar output is non-nullable by
+ default; annotate the return as ``Optional[T]`` or ``T | None``, or pass a
+ nullable ``output_schema`` field, to declare a nullable scalar. The
+ deployment must accept nullable scalar outputs; an older server rejects
+ the registration. Every named-struct field may be nullable; Enterprise
+ preserves the struct's validity when the result is expanded into sibling
+ columns.
**Large values.** A ``bytes`` parameter receives the whole value, which
is held in memory with its row and counts against the server's per-row
@@ -2757,9 +2765,10 @@ def function(
Required for batch annotations. When omitted, inputs are inferred from
the parameter annotations.
output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional
- Explicit scalar or named-struct output. Scalar outputs must be
- non-nullable. Required for batch annotations. When omitted, the output
- is inferred from the return annotation.
+ Explicit scalar or named-struct output. A scalar field's ``nullable``
+ flag is preserved; a top-level named-struct field and a Blob v2 output
+ must be non-nullable. Required for batch annotations. When omitted, the
+ output is inferred from the return annotation.
pip : sequence of str, optional
Pip requirements for the remote environment. Syntax is validated at
decoration time, including extras and environment markers. Named direct
diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py
index a3a9e3e20..7e089a96d 100644
--- a/python/python/tests/test_first_class_function_slice2.py
+++ b/python/python/tests/test_first_class_function_slice2.py
@@ -28,6 +28,7 @@ import lancedb
from lancedb import BlobReader
from lancedb.db import AsyncConnection
from lancedb.functions import (
+ FunctionSignature,
FunctionVersion,
HuggingFaceModel,
PythonRuntimeSpec,
@@ -1214,6 +1215,42 @@ def test_explicit_arrow_schema_is_deterministic():
assert signature.output.nullable is False
+def test_nullable_scalar_output_annotation_and_explicit_schema():
+ input_schema = pa.schema([pa.field("value", pa.int64(), nullable=False)])
+
+ @function(input_schema=input_schema)
+ def optional_annotation(value: int) -> Optional[int]:
+ return None if value == 0 else value * 2
+
+ @function(input_schema=input_schema)
+ def optional_union(value: int) -> int | None:
+ return None if value == 0 else value * 2
+
+ @function(
+ input_schema=input_schema,
+ output_schema=pa.field("result", pa.int64(), nullable=True),
+ )
+ def explicit_schema(value: int):
+ return None if value == 0 else value * 2
+
+ for definition in (optional_annotation, optional_union, explicit_schema):
+ signature = definition.registration_request.signature
+ assert (signature.output.kind, signature.output.arrow_type) == (
+ "scalar",
+ "int64",
+ )
+ assert signature.output.nullable is True
+ assert json.loads(signature.to_canonical_json())["output"] == {
+ "kind": "scalar",
+ "arrow_type": "int64",
+ "nullable": True,
+ }
+ assert (
+ FunctionSignature.from_json(signature.to_canonical_json()).output.nullable
+ is True
+ )
+
+
def test_blob_fields_use_the_scalar_function_semantic_type():
@function(
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
@@ -1848,12 +1885,6 @@ def test_annotation_and_explicit_schema_validation_fail_closed():
def unsupported(value: set[str]) -> str:
return ""
- with pytest.raises(ValueError, match="output must be non-nullable"):
-
- @function
- def nullable_output(value: int) -> Optional[int]:
- return value
-
with pytest.raises(TypeError, match=r"missing Function annotations: \['return'\]"):
@function(input_schema=pa.schema([pa.field("value", pa.int64())]))
@@ -1873,10 +1904,25 @@ def test_annotation_and_explicit_schema_validation_fail_closed():
@function(
input_schema=pa.schema([pa.field("value", pa.int64())]),
- output_schema=pa.field("result", pa.int64(), nullable=True),
+ output_schema=pa.field(
+ "result",
+ pa.struct([pa.field("value", pa.int64(), nullable=False)]),
+ nullable=True,
+ ),
)
- def nullable_explicit(value):
- return value
+ def nullable_struct(value):
+ return {"value": value}
+
+ with pytest.raises(
+ ValueError, match="Blob v2 Function output must be non-nullable"
+ ):
+
+ @function(
+ input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
+ output_schema=lancedb.blob("result", nullable=True),
+ )
+ def nullable_blob(image):
+ return image
for invalid_field in [
pa.field("", pa.int32(), nullable=False),
diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs
index d1b63f2ad..387c80364 100644
--- a/rust/lancedb/src/function.rs
+++ b/rust/lancedb/src/function.rs
@@ -1032,8 +1032,8 @@ pub struct OutputMapping {
pub nullable: bool,
}
-/// Internal physical column preserving the parent validity of a flattened
-/// named-struct result.
+/// Internal physical column preserving assignment of a nullable scalar result
+/// or the parent validity of a flattened named-struct result.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AssignmentMapping {
pub output_name: String,
diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs
index 5c00232f7..72d67fb5a 100644
--- a/rust/lancedb/src/remote/table.rs
+++ b/rust/lancedb/src/remote/table.rs
@@ -10802,6 +10802,83 @@ mod tests {
assert_eq!(result.version, 8);
}
+ #[tokio::test]
+ async fn test_add_nullable_scalar_function_column_sends_atomic_null_declaration() {
+ let table = Table::new_with_handler("my_table", |request| {
+ match request.url().path() {
+ "/v1/table/my_table/describe/" => http::Response::builder()
+ .status(200)
+ .body(
+ r#"{"version":1,"schema":{"fields":[{"name":"description","nullable":true,"type":{"type":"string"}}]}}"#,
+ )
+ .unwrap(),
+ "/v1/table/my_table/add_columns/" => {
+ let actual: serde_json::Value = serde_json::from_slice(
+ request.body().unwrap().as_bytes().unwrap(),
+ )
+ .unwrap();
+ assert_eq!(
+ actual["new_columns"],
+ serde_json::json!([
+ {"name": "maybe", "all_null": true}
+ ])
+ );
+ assert_eq!(
+ actual["function"]["application"]["output"],
+ serde_json::json!({
+ "kind": "scalar",
+ "arrow_type": "int64",
+ "nullable": true
+ })
+ );
+ assert_eq!(
+ actual["function"]["output_schema"],
+ serde_json::json!({
+ "fields": [{
+ "name": "maybe",
+ "nullable": true,
+ "type": {"type": "int64"}
+ }]
+ })
+ );
+ assert_eq!(
+ actual["function"]["outputs"],
+ serde_json::json!([
+ {
+ "result_field": "$value",
+ "output_name": "maybe",
+ "output_ordinal": 0
+ }
+ ])
+ );
+ assert!(actual["function"].get("assignment").is_none());
+ assert_eq!(actual["auto_refresh_enabled"], serde_json::json!(true));
+ http::Response::builder()
+ .status(200)
+ .body(r#"{"version":8}"#)
+ .unwrap()
+ }
+ path => panic!("Unexpected path: {path}"),
+ }
+ });
+ let application = crate::function::FunctionApplication::from_json(
+ r#"{
+ "function":{"name":"nullable_scalar","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
+ "inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}],
+ "output":{"kind":"scalar","arrow_type":"int64","nullable":true}
+ }"#,
+ )
+ .unwrap();
+
+ let result = table
+ .add_columns()
+ .function_as("maybe", application)
+ .execute()
+ .await
+ .unwrap();
+ assert_eq!(result.version, 8);
+ }
+
/// The fixture binding's table: `title` and `body` bound as inputs, its
/// two outputs declared, plus an unbound `spare`.
fn fixture_bound_schema() -> Schema {
diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs
index 64e90b246..b01f45d06 100644
--- a/rust/lancedb/src/table/computed_columns.rs
+++ b/rust/lancedb/src/table/computed_columns.rs
@@ -40,7 +40,7 @@ use lance_namespace::models::{JsonArrowDataType, JsonArrowField, JsonArrowSchema
use serde::{Deserialize, Serialize};
use serde_json::Value;
-use crate::function::{FUNCTION_BLOB_V2_TYPE, FunctionApplication, FunctionBinding};
+use crate::function::{FUNCTION_BLOB_V2_TYPE, FunctionApplication, FunctionBinding, OutputMapping};
use crate::materialized_view::without_declarations;
use crate::table::FunctionWorkerResourceBudget;
use crate::utils::resolve_arrow_field_path;
@@ -1289,6 +1289,21 @@ fn function_output_type_matches(expected: &DataType, actual: &DataType) -> bool
}
}
+/// Whether a whole-result binding may carry the server-owned assignment marker.
+///
+/// Only a nullable scalar result needs a marker to distinguish a computed NULL
+/// from an unfilled row. A non-nullable result, a whole named-struct column,
+/// and a Blob v2 value keep their existing rejection.
+fn nullable_scalar_allows_assignment(output: &OutputMapping) -> Result {
+ if !output.nullable || output.arrow_type == FUNCTION_BLOB_V2_TYPE {
+ return Ok(false);
+ }
+ let json = parse_output_arrow_type(&output.arrow_type)?;
+ let data_type = lance_namespace::schema::convert_json_arrow_type(&json)
+ .map_err(|e| invalid_function(format!("invalid Function output type: {e}")))?;
+ Ok(!matches!(data_type, DataType::Struct(_)))
+}
+
fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding) -> Result<()> {
let mut input_fields = Vec::with_capacity(binding.inputs().len());
for input in binding.inputs() {
@@ -1419,13 +1434,14 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
}
let mut declared_output_schema = binding.output_schema().cloned();
if let Some(assignment) = binding.assignment() {
- if binding
+ if let Some(whole_result) = binding
.outputs()
.iter()
- .any(|output| output.result_field == WHOLE_RESULT_FIELD)
+ .find(|output| output.result_field == WHOLE_RESULT_FIELD)
+ && (binding.outputs().len() != 1 || !nullable_scalar_allows_assignment(whole_result)?)
{
return Err(invalid_function(format!(
- "Function binding '{}' cannot attach an assignment column to a whole result",
+ "Function binding '{}' cannot attach an assignment column to a whole result; only a single nullable scalar output permits one",
binding.binding_id()
)));
}
@@ -1552,14 +1568,17 @@ pub(crate) fn plan_function_application(
"a scalar Function application must be mapped to one output column",
)
})?;
- if output.nullable != Some(false) {
- return Err(invalid_function(
- "Function logical outputs must be non-nullable during NULL assignment",
- ));
- }
+ let nullable = output.nullable.ok_or_else(|| {
+ invalid_function("scalar Function output must declare nullability")
+ })?;
let arrow_type = output.arrow_type.as_deref().ok_or_else(|| {
invalid_function("scalar Function output is missing its Arrow type")
})?;
+ if nullable && arrow_type == FUNCTION_BLOB_V2_TYPE {
+ return Err(invalid_function(
+ "Blob v2 Function outputs cannot be nullable",
+ ));
+ }
outputs.push(FunctionOutputTarget {
result_field: WHOLE_RESULT_FIELD.to_string(),
output_name: name.to_string(),
@@ -4054,6 +4073,22 @@ mod tests {
.unwrap()
}
+ fn nullable_scalar_application() -> FunctionApplication {
+ FunctionApplication::from_json(
+ &serde_json::json!({
+ "function": {"name": "nullable_scalar", "version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
+ "inputs": [{
+ "parameter": "value",
+ "kind": "column",
+ "value": {"path": "value"}
+ }],
+ "output": {"kind": "scalar", "arrow_type": "int64", "nullable": true}
+ })
+ .to_string(),
+ )
+ .unwrap()
+ }
+
fn binding_from_plan(plan: &FunctionDeclarationPlan) -> FunctionBinding {
let inputs = plan
.input_bindings
@@ -4363,6 +4398,259 @@ mod tests {
));
}
+ #[test]
+ fn test_nullable_scalar_application_plans_one_nullable_output() {
+ let input = ArrowField::new("value", DataType::Int64, false);
+ let plan = plan_function_application(
+ &ArrowSchema::new(vec![input]),
+ &nullable_scalar_application(),
+ Some("maybe"),
+ )
+ .unwrap();
+
+ assert_eq!(plan.application.output().nullable, Some(true));
+ assert_eq!(plan.outputs.len(), 1);
+ assert_eq!(plan.outputs[0].result_field, WHOLE_RESULT_FIELD);
+ assert_eq!(plan.outputs[0].output_name, "maybe");
+ assert_eq!(plan.outputs[0].output_ordinal, 0);
+ assert_eq!(plan.output_schema.fields.len(), 1);
+ assert!(plan.output_schema.fields[0].nullable);
+ assert_eq!(plan.output_schema.fields[0].r#type.r#type, "int64");
+ }
+
+ #[test]
+ fn test_invalid_nullable_scalar_applications_fail_closed() {
+ let input = ArrowField::new("value", DataType::Int64, false);
+ let schema = ArrowSchema::new(vec![input]);
+
+ let mut missing_nullability = serde_json::to_value(nullable_scalar_application()).unwrap();
+ missing_nullability["output"]
+ .as_object_mut()
+ .unwrap()
+ .remove("nullable");
+ let application = FunctionApplication::from_json(&missing_nullability.to_string()).unwrap();
+ let err = plan_function_application(&schema, &application, Some("maybe")).unwrap_err();
+ assert!(
+ matches!(&err, Error::InvalidInput { message } if message.contains("declare nullability")),
+ "{err:?}"
+ );
+
+ let nullable_blob =
+ blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":true}"#);
+ let err = plan_function_application(
+ &ArrowSchema::new(vec![crate::blob("image", false)]),
+ &nullable_blob,
+ Some("copy"),
+ )
+ .unwrap_err();
+ assert!(
+ matches!(&err, Error::InvalidInput { message } if message.contains("Blob v2") && message.contains("nullable")),
+ "{err:?}"
+ );
+ }
+
+ #[test]
+ fn test_nullable_scalar_binding_accepts_server_owned_assignment() {
+ let input = ArrowField::new("value", DataType::Int64, false);
+ let plan = plan_function_application(
+ &ArrowSchema::new(vec![input.clone()]),
+ &nullable_scalar_application(),
+ Some("maybe"),
+ )
+ .unwrap();
+ let assignment_name = "__function_assignment_nullable_scalar";
+ let mut raw = serde_json::to_value(binding_from_plan(&plan)).unwrap();
+ raw["outputs"][0]["nullable"] = Value::Bool(true);
+ raw["assignment"] = serde_json::json!({
+ "output_name": assignment_name,
+ "output_field_id": -1,
+ });
+ raw["output_schema"]["fields"]
+ .as_array_mut()
+ .unwrap()
+ .push(serde_json::json!({
+ "name": assignment_name,
+ "nullable": true,
+ "type": {"type": "bool"},
+ }));
+ let binding: FunctionBinding = serde_json::from_value(raw).unwrap();
+ let output = ArrowField::new("maybe", DataType::Int64, true).with_metadata(
+ function_computed_column_metadata(binding.binding_id(), 0, &["value".into()]),
+ );
+ let assignment = ArrowField::new(assignment_name, DataType::Boolean, true).with_metadata(
+ function_computed_column_metadata(
+ binding.binding_id(),
+ FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL,
+ &["value".into()],
+ ),
+ );
+
+ let schema = ArrowSchema::new(vec![input.clone(), output.clone(), assignment]);
+ ensure_binding_matches_schema(&schema, &binding).unwrap();
+ let hidden = ArrowSchema::new(vec![input, output]);
+ ensure_binding_matches_schema(&hidden, &binding).unwrap();
+
+ let schema = ArrowSchema::new_with_metadata(
+ schema.fields().to_vec(),
+ HashMap::from([(
+ FUNCTION_BINDINGS_META_KEY.to_string(),
+ function_bindings_metadata(std::slice::from_ref(&binding)).unwrap(),
+ )]),
+ );
+ ensure_supported_function_metadata(&schema).unwrap();
+ }
+
+ #[test]
+ fn test_non_nullable_scalar_binding_rejects_server_owned_assignment() {
+ let input = ArrowField::new("value", DataType::Int64, false);
+ let plan = plan_function_application(
+ &ArrowSchema::new(vec![input.clone()]),
+ &single_input_application("value"),
+ Some("maybe"),
+ )
+ .unwrap();
+ let assignment_name = "__function_assignment_non_nullable_scalar";
+ let mut raw = serde_json::to_value(binding_from_plan(&plan)).unwrap();
+ raw["assignment"] = serde_json::json!({
+ "output_name": assignment_name,
+ "output_field_id": -1,
+ });
+ raw["output_schema"]["fields"]
+ .as_array_mut()
+ .unwrap()
+ .push(serde_json::json!({
+ "name": assignment_name,
+ "nullable": true,
+ "type": {"type": "bool"},
+ }));
+ let binding: FunctionBinding = serde_json::from_value(raw).unwrap();
+ let output = ArrowField::new("maybe", DataType::Int64, true).with_metadata(
+ function_computed_column_metadata(binding.binding_id(), 0, &["value".into()]),
+ );
+ let assignment = ArrowField::new(assignment_name, DataType::Boolean, true).with_metadata(
+ function_computed_column_metadata(
+ binding.binding_id(),
+ FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL,
+ &["value".into()],
+ ),
+ );
+
+ let err = ensure_binding_matches_schema(
+ &ArrowSchema::new(vec![input, output, assignment]),
+ &binding,
+ )
+ .unwrap_err();
+ assert!(
+ matches!(&err, Error::InvalidInput { message } if message.contains("nullable scalar")),
+ "{err:?}"
+ );
+ }
+
+ #[test]
+ fn test_named_struct_whole_binding_rejects_server_owned_assignment() {
+ let plan = plan_function_application(
+ &function_input_schema(),
+ &named_struct_application("{}"),
+ Some("features"),
+ )
+ .unwrap();
+ let assignment_name = "__function_assignment_whole_struct";
+ let mut raw = serde_json::to_value(binding_from_plan(&plan)).unwrap();
+ raw["outputs"][0]["nullable"] = Value::Bool(true);
+ raw["assignment"] = serde_json::json!({
+ "output_name": assignment_name,
+ "output_field_id": -1,
+ });
+ raw["output_schema"]["fields"]
+ .as_array_mut()
+ .unwrap()
+ .push(serde_json::json!({
+ "name": assignment_name,
+ "nullable": true,
+ "type": {"type": "bool"},
+ }));
+ let binding: FunctionBinding = serde_json::from_value(raw).unwrap();
+ let output =
+ lance_namespace::schema::convert_json_arrow_field(&plan.output_schema.fields[0])
+ .unwrap()
+ .with_metadata(function_computed_column_metadata(
+ binding.binding_id(),
+ 0,
+ &["title".into(), "body".into()],
+ ));
+ let assignment = ArrowField::new(assignment_name, DataType::Boolean, true).with_metadata(
+ function_computed_column_metadata(
+ binding.binding_id(),
+ FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL,
+ &["title".into(), "body".into()],
+ ),
+ );
+ let mut fields = function_input_schema()
+ .fields()
+ .iter()
+ .map(|field| field.as_ref().clone())
+ .collect::>();
+ fields.push(output);
+ fields.push(assignment);
+
+ let err = ensure_binding_matches_schema(&ArrowSchema::new(fields), &binding).unwrap_err();
+ assert!(
+ matches!(&err, Error::InvalidInput { message } if message.contains("nullable scalar")),
+ "{err:?}"
+ );
+ }
+
+ #[test]
+ fn test_blob_whole_binding_rejects_server_owned_assignment() {
+ let input = crate::blob("image", false);
+ let plan = plan_function_application(
+ &ArrowSchema::new(vec![input.clone()]),
+ &blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#),
+ Some("thumbnail"),
+ )
+ .unwrap();
+ let assignment_name = "__function_assignment_whole_blob";
+ let mut raw = serde_json::to_value(binding_from_plan(&plan)).unwrap();
+ raw["outputs"][0]["nullable"] = Value::Bool(true);
+ raw["assignment"] = serde_json::json!({
+ "output_name": assignment_name,
+ "output_field_id": -1,
+ });
+ raw["output_schema"]["fields"]
+ .as_array_mut()
+ .unwrap()
+ .push(serde_json::json!({
+ "name": assignment_name,
+ "nullable": true,
+ "type": {"type": "bool"},
+ }));
+ let binding: FunctionBinding = serde_json::from_value(raw).unwrap();
+ let mut metadata = full_blob_field("thumbnail", true).metadata().clone();
+ metadata.extend(function_computed_column_metadata(
+ binding.binding_id(),
+ 0,
+ &["image".into()],
+ ));
+ let output = full_blob_field("thumbnail", true).with_metadata(metadata);
+ let assignment = ArrowField::new(assignment_name, DataType::Boolean, true).with_metadata(
+ function_computed_column_metadata(
+ binding.binding_id(),
+ FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL,
+ &["image".into()],
+ ),
+ );
+
+ let err = ensure_binding_matches_schema(
+ &ArrowSchema::new(vec![input, output, assignment]),
+ &binding,
+ )
+ .unwrap_err();
+ assert!(
+ matches!(&err, Error::InvalidInput { message } if message.contains("nullable scalar")),
+ "{err:?}"
+ );
+ }
+
#[test]
fn test_nullable_function_input_cannot_bind_to_non_nullable_parameter() {
let mut raw_binding: Value = serde_json::from_str(include_str!(