mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-20 10:42:26 +00:00
feat(python): accept large_binary in Function signatures (#4577)
udf() rejected a large_binary input or output, and large_binary nested in a list or struct, outside a Blob v2 payload. It is now a grammar primitive like large_utf8, and the shared type golden lists it.
This commit is contained in:
@@ -697,6 +697,7 @@ _GRAMMAR_PRIMITIVES = (
|
||||
(pa.string(), "utf8"),
|
||||
(pa.large_string(), "large_utf8"),
|
||||
(pa.binary(), "binary"),
|
||||
(pa.large_binary(), "large_binary"),
|
||||
(pa.date32(), "date32"),
|
||||
(pa.date64(), "date64"),
|
||||
)
|
||||
@@ -846,11 +847,7 @@ def _exact_blob_storage_type(field: pa.Field) -> dict[str, Any]:
|
||||
{
|
||||
"name": child.name,
|
||||
"nullable": child.nullable,
|
||||
"type": (
|
||||
{"type": "large_binary"}
|
||||
if pa.types.is_large_binary(child.type)
|
||||
else _exact_arrow_type(child.type)
|
||||
),
|
||||
"type": _exact_arrow_type(child.type),
|
||||
}
|
||||
for child in storage
|
||||
],
|
||||
|
||||
@@ -622,7 +622,6 @@ def test_canonical_arrow_type_prefers_the_compact_grammar():
|
||||
for outside in [
|
||||
pa.timestamp("us"),
|
||||
pa.decimal128(10, 2),
|
||||
pa.large_binary(),
|
||||
pa.binary(4),
|
||||
pa.duration("s"),
|
||||
pa.list_(pa.float32(), 0),
|
||||
@@ -1280,6 +1279,28 @@ def test_explicit_large_utf8_schemas_use_the_canonical_function_name():
|
||||
}
|
||||
|
||||
|
||||
def test_large_binary_schemas_use_the_canonical_function_name():
|
||||
payload = pa.field("data", pa.large_binary(), nullable=True)
|
||||
|
||||
@udf(
|
||||
input_schema=pa.schema([payload]),
|
||||
output_schema=pa.field("result", pa.large_binary(), nullable=False),
|
||||
)
|
||||
def preserve(data):
|
||||
return data
|
||||
|
||||
signature = preserve.registration_request.signature
|
||||
assert signature.inputs[0].arrow_type == "large_binary"
|
||||
assert signature.output.arrow_type == "large_binary"
|
||||
cells = pa.large_list(pa.field("item", pa.struct([payload]), nullable=True))
|
||||
assert json.loads(_canonical_arrow_type(cells))["fields"][0]["type"] == {
|
||||
"type": "struct",
|
||||
"fields": [
|
||||
{"name": "data", "nullable": True, "type": {"type": "large_binary"}}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_nested_struct_output_uses_canonical_exact_json():
|
||||
token = pa.struct(
|
||||
[
|
||||
|
||||
@@ -90,6 +90,12 @@
|
||||
"type": "binary"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arrow_type": "large_binary",
|
||||
"json": {
|
||||
"type": "large_binary"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arrow_type": "date32",
|
||||
"json": {
|
||||
@@ -192,6 +198,21 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"arrow_type": "large_list<large_binary>",
|
||||
"json": {
|
||||
"type": "large_list",
|
||||
"fields": [
|
||||
{
|
||||
"name": "item",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": "large_binary"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"arrow_type": "large_list<utf8>",
|
||||
"json": {
|
||||
|
||||
Reference in New Issue
Block a user