mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat: support nullable scalar Function declarations (#5030)
Function declarations currently reject nullable scalar returns. Accept `Optional[T]`, `T | None`, and explicit nullable Arrow fields, preserving `nullable: true` in registration signatures, JSON round trips, and remote `add_columns` requests. The Rust planner accepts nullable scalar outputs, and binding readback recognizes the server-owned assignment marker for a single nullable scalar result. Clients do not allocate that marker. Non-nullable whole results, whole structs, and Blob v2 retain their existing assignment restrictions. Validated with 127 Python tests, 30 documentation tests (1 skipped), and seven focused Rust tests covering planning, binding readback, rejection boundaries, and the remote request body. The final condition-collapse change addresses strict Clippy; its delta check and CI are pending. Python tests use a source overlay on a retained native extension, and Rust remote coverage uses HTTP mocks. Server support and end-to-end validation remain separate. This PR does not claim that server publication distinguishes computed NULL values from pending rows.
This commit is contained in:
@@ -312,7 +312,11 @@ class FunctionResultField(_OpenRemoteValue):
|
||||
|
||||
|
||||
class FunctionOutput(_OpenRemoteValue):
|
||||
"""Scalar or ordered named-struct output; unknown kinds remain decodable."""
|
||||
"""Scalar or ordered named-struct output; unknown kinds remain decodable.
|
||||
|
||||
A scalar output may be nullable. A named-struct output has no top-level
|
||||
nullability; its fields carry their own flags.
|
||||
"""
|
||||
|
||||
kind: str
|
||||
arrow_type: Optional[str] = None
|
||||
@@ -1399,12 +1403,12 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
|
||||
"output_schema must be a PyArrow DataType, Field, or Schema"
|
||||
)
|
||||
_validate_exact_arrow_field(field)
|
||||
if field.nullable:
|
||||
raise ValueError("Function output must be non-nullable")
|
||||
if field.nullable and _is_blob_v2_field(field):
|
||||
raise ValueError("Blob v2 Function output must be non-nullable")
|
||||
return FunctionOutput(
|
||||
kind="scalar",
|
||||
arrow_type=_canonical_arrow_field(field, collection_blobs=True),
|
||||
nullable=False,
|
||||
nullable=field.nullable,
|
||||
)
|
||||
|
||||
if not fields:
|
||||
@@ -1504,9 +1508,9 @@ def _infer_signature(
|
||||
output = _function_output(output_schema)
|
||||
else:
|
||||
output_type, output_nullable = _annotation_type(annotations["return"])
|
||||
if output_nullable:
|
||||
raise ValueError("Function output must be non-nullable")
|
||||
output = _function_output(pa.field("result", output_type, nullable=False))
|
||||
output = _function_output(
|
||||
pa.field("result", output_type, nullable=output_nullable)
|
||||
)
|
||||
if inputs is not None:
|
||||
return FunctionSignature(inputs=inputs, output=output)
|
||||
|
||||
@@ -2650,9 +2654,13 @@ def function(
|
||||
``None`` for a null value. PyArrow fields are nullable unless declared
|
||||
otherwise, so most columns need ``Optional``. For Arrow types annotations
|
||||
cannot express precisely, pass ``input_schema``, ``output_schema``, or
|
||||
both; the one omitted is inferred. Scalar outputs must be non-nullable.
|
||||
Every named-struct field may be nullable; Enterprise preserves the
|
||||
struct's validity when the result is expanded into sibling columns.
|
||||
both; the one omitted is inferred. A scalar output is non-nullable by
|
||||
default; annotate the return as ``Optional[T]`` or ``T | None``, or pass a
|
||||
nullable ``output_schema`` field, to declare a nullable scalar. The
|
||||
deployment must accept nullable scalar outputs; an older server rejects
|
||||
the registration. Every named-struct field may be nullable; Enterprise
|
||||
preserves the struct's validity when the result is expanded into sibling
|
||||
columns.
|
||||
|
||||
**Large values.** A ``bytes`` parameter receives the whole value, which
|
||||
is held in memory with its row and counts against the server's per-row
|
||||
@@ -2757,9 +2765,10 @@ def function(
|
||||
Required for batch annotations. When omitted, inputs are inferred from
|
||||
the parameter annotations.
|
||||
output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional
|
||||
Explicit scalar or named-struct output. Scalar outputs must be
|
||||
non-nullable. Required for batch annotations. When omitted, the output
|
||||
is inferred from the return annotation.
|
||||
Explicit scalar or named-struct output. A scalar field's ``nullable``
|
||||
flag is preserved; a top-level named-struct field and a Blob v2 output
|
||||
must be non-nullable. Required for batch annotations. When omitted, the
|
||||
output is inferred from the return annotation.
|
||||
pip : sequence of str, optional
|
||||
Pip requirements for the remote environment. Syntax is validated at
|
||||
decoration time, including extras and environment markers. Named direct
|
||||
|
||||
@@ -28,6 +28,7 @@ import lancedb
|
||||
from lancedb import BlobReader
|
||||
from lancedb.db import AsyncConnection
|
||||
from lancedb.functions import (
|
||||
FunctionSignature,
|
||||
FunctionVersion,
|
||||
HuggingFaceModel,
|
||||
PythonRuntimeSpec,
|
||||
@@ -1214,6 +1215,42 @@ def test_explicit_arrow_schema_is_deterministic():
|
||||
assert signature.output.nullable is False
|
||||
|
||||
|
||||
def test_nullable_scalar_output_annotation_and_explicit_schema():
|
||||
input_schema = pa.schema([pa.field("value", pa.int64(), nullable=False)])
|
||||
|
||||
@function(input_schema=input_schema)
|
||||
def optional_annotation(value: int) -> Optional[int]:
|
||||
return None if value == 0 else value * 2
|
||||
|
||||
@function(input_schema=input_schema)
|
||||
def optional_union(value: int) -> int | None:
|
||||
return None if value == 0 else value * 2
|
||||
|
||||
@function(
|
||||
input_schema=input_schema,
|
||||
output_schema=pa.field("result", pa.int64(), nullable=True),
|
||||
)
|
||||
def explicit_schema(value: int):
|
||||
return None if value == 0 else value * 2
|
||||
|
||||
for definition in (optional_annotation, optional_union, explicit_schema):
|
||||
signature = definition.registration_request.signature
|
||||
assert (signature.output.kind, signature.output.arrow_type) == (
|
||||
"scalar",
|
||||
"int64",
|
||||
)
|
||||
assert signature.output.nullable is True
|
||||
assert json.loads(signature.to_canonical_json())["output"] == {
|
||||
"kind": "scalar",
|
||||
"arrow_type": "int64",
|
||||
"nullable": True,
|
||||
}
|
||||
assert (
|
||||
FunctionSignature.from_json(signature.to_canonical_json()).output.nullable
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def test_blob_fields_use_the_scalar_function_semantic_type():
|
||||
@function(
|
||||
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
|
||||
@@ -1848,12 +1885,6 @@ def test_annotation_and_explicit_schema_validation_fail_closed():
|
||||
def unsupported(value: set[str]) -> str:
|
||||
return ""
|
||||
|
||||
with pytest.raises(ValueError, match="output must be non-nullable"):
|
||||
|
||||
@function
|
||||
def nullable_output(value: int) -> Optional[int]:
|
||||
return value
|
||||
|
||||
with pytest.raises(TypeError, match=r"missing Function annotations: \['return'\]"):
|
||||
|
||||
@function(input_schema=pa.schema([pa.field("value", pa.int64())]))
|
||||
@@ -1873,10 +1904,25 @@ def test_annotation_and_explicit_schema_validation_fail_closed():
|
||||
|
||||
@function(
|
||||
input_schema=pa.schema([pa.field("value", pa.int64())]),
|
||||
output_schema=pa.field("result", pa.int64(), nullable=True),
|
||||
output_schema=pa.field(
|
||||
"result",
|
||||
pa.struct([pa.field("value", pa.int64(), nullable=False)]),
|
||||
nullable=True,
|
||||
),
|
||||
)
|
||||
def nullable_explicit(value):
|
||||
return value
|
||||
def nullable_struct(value):
|
||||
return {"value": value}
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="Blob v2 Function output must be non-nullable"
|
||||
):
|
||||
|
||||
@function(
|
||||
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
|
||||
output_schema=lancedb.blob("result", nullable=True),
|
||||
)
|
||||
def nullable_blob(image):
|
||||
return image
|
||||
|
||||
for invalid_field in [
|
||||
pa.field("", pa.int32(), nullable=False),
|
||||
|
||||
Reference in New Issue
Block a user