mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-14 15:52:26 +00:00
feat: support nullable named function outputs (#4123)
Supports fully nullable named Function outputs while preserving the
distinction between a valid all-null struct and a null/unassigned
result.
## Concrete example
This UDF contract is now valid:
```python
@udf(
input_schema=pa.schema([
pa.field("text", pa.string(), nullable=False),
]),
output_schema=pa.schema([
pa.field(
"embedding",
pa.list_(pa.float32(), list_size=1024),
nullable=True,
),
pa.field("embedding_failure_reason", pa.string(), nullable=True),
pa.field("embedding_failure_code", pa.int32(), nullable=True),
]),
)
def embed(text):
...
```
A successful row can return:
```text
embedding = [0.12, ...]
embedding_failure_reason = NULL
embedding_failure_code = NULL
```
If remote inference still fails after retries, it can return:
```text
embedding = NULL
embedding_failure_reason = "HTTP 429: rate limited"
embedding_failure_code = 429
```
An all-null but valid result struct is also assigned; it is not mistaken
for unfinished work.
## Binding shapes
- Mapping the result to one output column stores the `StructArray`
directly, including its parent validity bitmap.
- Flattening the result into top-level columns stores the parent
validity in a reserved internal nullable Boolean assignment column that
is not part of the UDF result mapping.
- An outer null struct remains unassigned/skipped. A valid struct
remains assigned regardless of which child fields are null.
- Scalar Function outputs remain non-nullable.
The contract is preserved through Python registration, Rust application
planning, persisted `FunctionBinding` metadata, schema revalidation, and
Enterprise execution.
This commit is contained in:
@@ -850,6 +850,43 @@ def test_named_struct_function_can_include_a_blob_result_field():
|
||||
]
|
||||
|
||||
|
||||
def test_named_struct_function_preserves_nullable_result_fields():
|
||||
@udf(
|
||||
input_schema=pa.schema([pa.field("value", pa.int64(), nullable=False)]),
|
||||
output_schema=pa.schema(
|
||||
[
|
||||
pa.field("result", pa.int64(), nullable=True),
|
||||
pa.field("failure_code", pa.int32(), nullable=False),
|
||||
]
|
||||
),
|
||||
)
|
||||
def nullable_result(value):
|
||||
return {"result": value, "failure_code": 0}
|
||||
|
||||
output = nullable_result.registration_request.signature.output
|
||||
assert [(field.name, field.nullable) for field in output.fields] == [
|
||||
("result", True),
|
||||
("failure_code", False),
|
||||
]
|
||||
|
||||
@udf(
|
||||
input_schema=pa.schema([pa.field("value", pa.int64(), nullable=False)]),
|
||||
output_schema=pa.schema(
|
||||
[
|
||||
pa.field("result", pa.int64(), nullable=True),
|
||||
pa.field("failure_code", pa.int32(), nullable=True),
|
||||
]
|
||||
),
|
||||
)
|
||||
def all_nullable(value):
|
||||
return {"result": value, "failure_code": None}
|
||||
|
||||
assert all(
|
||||
field.nullable
|
||||
for field in all_nullable.registration_request.signature.output.fields
|
||||
)
|
||||
|
||||
|
||||
def test_metadata_marked_blob_field_uses_the_semantic_type():
|
||||
extension = lancedb.blob("image", nullable=False).type
|
||||
storage = (
|
||||
|
||||
Reference in New Issue
Block a user