rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-09-02 03:28:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support nested blob function signatures (#4109)

Browse Source 
Function signatures currently reject Blob v2 fields nested inside
structs, preventing UDFs from accepting or returning structured values
that contain blobs.

Accept canonical Blob v2 fields as direct or recursive struct children
while preserving exact field metadata and nullability. Blob fields under
list, large-list, fixed-size-list, or map ancestors remain rejected
because collection runtime adaptation is outside the supported Function
ABI.

A whole named struct result can bind directly to one destination column
without introducing an extra wrapper level.
This commit is contained in:
Xuanwo
2026-09-01 23:51:08 +08:00
committed by GitHub
parent 193c5e3458
commit e6867f7d04
3 changed files with 446 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python/tests/test_first_class_function_slice2.py
+161 -16
View File
@@ -668,6 +668,167 @@ def test_blob_fields_use_the_scalar_function_semantic_type():
assert signature.output.arrow_type == "blob_v2"
def test_whole_named_struct_function_can_include_a_blob_result_field():
@udf(
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
output_schema=pa.field(
"payload",
pa.struct(
[
pa.field("mime_type", pa.string(), nullable=False),
lancedb.blob("image", nullable=False),
]
),
nullable=False,
),
)
def inspect_blob(image):
return {"mime_type": "image/png", "image": image}
output = inspect_blob.registration_request.signature.output
assert output.kind == "named_struct"
assert [(field.name, field.arrow_type) for field in output.fields] == [
("mime_type", "utf8"),
("image", "blob_v2"),
]
def test_struct_blob_signature_fields_preserve_exact_metadata_and_nullability():
nested_input = pa.field(
"payload",
pa.struct(
[
pa.field("mime_type", pa.string(), nullable=False),
pa.field(
"nested",
pa.struct([lancedb.blob("image", nullable=True)]),
nullable=True,
),
]
),
nullable=True,
)
nested_output = pa.field(
"result",
pa.struct(
[
pa.field("mime_type", pa.string(), nullable=False),
pa.field(
"nested",
pa.struct([lancedb.blob("image", nullable=True)]),
nullable=False,
),
]
),
nullable=False,
)
@udf(input_schema=pa.schema([nested_input]), output_schema=nested_output)
def copy_payload(payload):
return payload
signature = copy_payload.registration_request.signature
input_type = json.loads(signature.inputs[0].arrow_type)
assert input_type["fields"][1]["nullable"] is True
input_blob = input_type["fields"][1]["type"]["fields"][0]
assert input_blob["nullable"] is True
assert input_blob["metadata"] == {"ARROW:extension:name": "lance.blob.v2"}
assert signature.output.kind == "named_struct"
nested_result = next(
field for field in signature.output.fields if field.name == "nested"
)
output_type = json.loads(nested_result.arrow_type)
output_blob = output_type["fields"][0]
assert output_blob["nullable"] is True
assert output_blob["metadata"] == {"ARROW:extension:name": "lance.blob.v2"}
def test_struct_blob_signature_supports_multiple_struct_levels():
recursive = pa.field(
"value",
pa.struct(
[
pa.field(
"level_1",
pa.struct(
[
pa.field(
"level_2",
pa.struct([lancedb.blob("image", nullable=False)]),
nullable=False,
)
]
),
nullable=False,
)
]
),
nullable=False,
)
@udf(
input_schema=pa.schema([recursive]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(value):
return len(value["level_1"]["level_2"]["image"])
encoded = json.loads(blob_size.registration_request.signature.inputs[0].arrow_type)
blob = encoded["fields"][0]["type"]["fields"][0]["type"]["fields"][0]
assert blob["metadata"]["ARROW:extension:name"] == "lance.blob.v2"
@pytest.mark.parametrize(
"data_type",
[
pa.list_(lancedb.blob("item", nullable=False)),
pa.large_list(lancedb.blob("item", nullable=False)),
pa.list_(lancedb.blob("item", nullable=False), 2),
pa.map_(pa.string(), lancedb.blob("value", nullable=False).type),
],
)
def test_blob_signature_rejects_collection_ancestors(data_type):
with pytest.raises(
TypeError,
match="Blob v2 fields nested under collection types are not supported",
):
@udf(
input_schema=pa.schema([pa.field("value", data_type, nullable=False)]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(value):
return len(value)
def test_blob_signature_rejects_collection_below_a_struct():
nested = pa.field(
"value",
pa.struct(
[
pa.field(
"images",
pa.list_(lancedb.blob("item", nullable=False)),
nullable=False,
)
]
),
nullable=False,
)
with pytest.raises(
TypeError,
match="Blob v2 fields nested under collection types are not supported",
):
@udf(
input_schema=pa.schema([nested]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(value):
return len(value["images"])
def test_named_struct_function_can_include_a_blob_result_field():
@udf(
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
@@ -729,22 +890,6 @@ def test_blob_marker_rejects_invalid_storage_layout():
return len(image)
def test_nested_blob_signature_field_has_a_clear_error():
nested = pa.field(
"value",
pa.struct([lancedb.blob("image", nullable=False)]),
nullable=False,
)
with pytest.raises(TypeError, match="nested Blob v2 fields are not supported"):
@udf(
input_schema=pa.schema([nested]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(value):
return len(value["image"])
def test_nested_non_blob_extension_is_not_silently_unwrapped():
class TestExtension(pa.ExtensionType):
def __init__(self):