mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat(python): support map and decimal Function signatures (#5031)
Python Function declarations reject Arrow map and decimal types before registration. Accept `dict[str, T]`, nullable map values, explicit Arrow maps, and `Annotated[Decimal, pa.decimal128/256(...)]` using the existing canonical JSON representation. Preserve map key/value nullability and decimal precision/scale; reject unsupported map keys, sorted maps, nested Blob values, and a bare Decimal without precision/scale. This is the SDK map/decimal portion of ENT-3414. Server validation and SQL support are separate changes; older servers can still reject these signatures. Nullable scalar output support remains separate in #5030. Validation: 129 Function signature tests passed; 30 documentation tests passed with one existing skip. Ruff format/lint and diff checks passed. Python tests used the existing Python 3.13 environment and prebuilt native extension with this Python source overlaid; no Rust build or remote-server execution was performed.
This commit is contained in:
@@ -30,6 +30,7 @@ import types
|
||||
import warnings
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import date, datetime, timedelta
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
Annotated,
|
||||
@@ -1378,6 +1379,13 @@ def _exact_arrow_type(
|
||||
for candidate, name in _GRAMMAR_PRIMITIVES:
|
||||
if data_type == candidate:
|
||||
return {"type": name}
|
||||
if pa.types.is_decimal128(data_type) or pa.types.is_decimal256(data_type):
|
||||
return {
|
||||
"type": (
|
||||
"decimal128" if pa.types.is_decimal128(data_type) else "decimal256"
|
||||
),
|
||||
"length": data_type.precision * 1000 + data_type.scale,
|
||||
}
|
||||
if pa.types.is_timestamp(data_type):
|
||||
unit = data_type.unit
|
||||
if unit not in _FUNCTION_TIMEUNITS:
|
||||
@@ -1453,8 +1461,29 @@ def _exact_arrow_type(
|
||||
if pa.types.is_fixed_size_list(data_type):
|
||||
value["length"] = data_type.list_size
|
||||
return value
|
||||
if pa.types.is_map(data_type) and _data_type_has_blob_v2(data_type):
|
||||
raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
|
||||
if pa.types.is_map(data_type):
|
||||
if data_type.keys_sorted:
|
||||
raise TypeError(
|
||||
"unsupported Arrow type for Function signature: map keys_sorted=true "
|
||||
"is not supported"
|
||||
)
|
||||
key_field = data_type.key_field
|
||||
if key_field.nullable or key_field.type != pa.string():
|
||||
raise TypeError(
|
||||
"unsupported Arrow type for Function signature: map keys must be "
|
||||
"non-nullable utf8"
|
||||
)
|
||||
if _data_type_has_blob_v2(data_type):
|
||||
raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
|
||||
return {
|
||||
"type": "map",
|
||||
"fields": [
|
||||
_exact_arrow_field(
|
||||
data_type.field(0),
|
||||
inside_collection=True,
|
||||
)
|
||||
],
|
||||
}
|
||||
raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
|
||||
|
||||
|
||||
@@ -1483,9 +1512,15 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
|
||||
raise TypeError(
|
||||
"Annotated Function types require exactly one PyArrow DataType"
|
||||
)
|
||||
# A struct value is written as a dict, and timedelta has no unit of
|
||||
# its own; the Annotated metadata supplies those types.
|
||||
if base is dict or get_origin(base) is dict or base is timedelta:
|
||||
# A struct value is written as a dict, while Decimal and timedelta
|
||||
# carry no precision or unit of their own; the Annotated metadata
|
||||
# supplies those types.
|
||||
if (
|
||||
base is dict
|
||||
or get_origin(base) is dict
|
||||
or base is Decimal
|
||||
or base is timedelta
|
||||
):
|
||||
base_nullable = False
|
||||
else:
|
||||
_, base_nullable = _annotation_type(base)
|
||||
@@ -1514,6 +1549,13 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
|
||||
# Existing default: microsecond precision, no timezone. Other units
|
||||
# and timezones require Annotated or an explicit schema.
|
||||
return pa.timestamp("us"), nullable
|
||||
if annotation is Decimal:
|
||||
raise TypeError(
|
||||
"Decimal Function annotations require precision and scale; use "
|
||||
"Annotated[Decimal, pyarrow.decimal128(precision, scale)], "
|
||||
"Annotated[Decimal, pyarrow.decimal256(precision, scale)], or an "
|
||||
"explicit schema"
|
||||
)
|
||||
if annotation is timedelta:
|
||||
raise TypeError(
|
||||
"timedelta Function annotations require an explicit unit; use "
|
||||
@@ -1527,6 +1569,21 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
|
||||
if value_nullable:
|
||||
raise TypeError("nullable Function list elements are not supported")
|
||||
return _list_of(value_type), nullable
|
||||
if get_origin(annotation) is dict:
|
||||
arguments = get_args(annotation)
|
||||
if len(arguments) != 2:
|
||||
raise TypeError(f"unsupported dict annotation: {annotation!r}")
|
||||
key_type, value_annotation = arguments
|
||||
if key_type is not str:
|
||||
raise TypeError("Function map keys must be str")
|
||||
value_type, value_nullable = _annotation_type(value_annotation)
|
||||
return (
|
||||
pa.map_(
|
||||
pa.string(),
|
||||
pa.field("value", value_type, nullable=value_nullable),
|
||||
),
|
||||
nullable,
|
||||
)
|
||||
raise TypeError(f"unsupported Function annotation: {annotation!r}")
|
||||
|
||||
|
||||
@@ -2847,7 +2904,13 @@ def function(
|
||||
deployment must accept nullable scalar outputs; an older server rejects
|
||||
the registration. Every named-struct field may be nullable; Enterprise
|
||||
preserves the struct's validity when the result is expanded into sibling
|
||||
columns. Bare ``datetime`` keeps its existing ``timestamp[us]`` default
|
||||
columns.
|
||||
``dict[str, T]`` declares an Arrow map with a non-nullable utf8 key; the
|
||||
value is non-nullable for ``T`` and nullable for ``Optional[T]``. A bare
|
||||
``Decimal`` has no precision or scale; use
|
||||
``Annotated[Decimal, pyarrow.decimal128(precision, scale)]``,
|
||||
``Annotated[Decimal, pyarrow.decimal256(precision, scale)]``, or an
|
||||
explicit schema. Bare ``datetime`` keeps its existing ``timestamp[us]`` default
|
||||
without a timezone; other units and timezones require
|
||||
``Annotated[datetime, pyarrow.timestamp(unit, tz=...)]`` or an explicit
|
||||
schema. A bare ``timedelta`` has no unit and is rejected; use
|
||||
@@ -2958,10 +3021,12 @@ def function(
|
||||
output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional
|
||||
Explicit scalar or named-struct output. A scalar field's ``nullable``
|
||||
flag is preserved; a top-level named-struct field and a Blob v2 output
|
||||
must be non-nullable. Temporal fields use canonical Arrow JSON; the
|
||||
microsecond/no-timezone case keeps the legacy timestamp/duration
|
||||
shape. Required for batch annotations. When omitted, the output is
|
||||
inferred from the return annotation.
|
||||
must be non-nullable. Map and decimal fields use canonical Arrow JSON;
|
||||
a map key must be non-nullable utf8 and its value field keeps the
|
||||
declared nullability. Temporal fields use canonical Arrow JSON; the
|
||||
microsecond/no-timezone case keeps the legacy timestamp/duration shape.
|
||||
Required for batch annotations. When omitted, the output is inferred
|
||||
from the return annotation.
|
||||
pip : sequence of str, optional
|
||||
Pip requirements for the remote environment. Syntax is validated at
|
||||
decoration time, including extras and environment markers. Named direct
|
||||
|
||||
@@ -19,6 +19,7 @@ import sys
|
||||
import typing
|
||||
import urllib.parse
|
||||
import warnings
|
||||
from decimal import Decimal
|
||||
from typing import Annotated, Optional
|
||||
|
||||
import pyarrow as pa
|
||||
@@ -1054,7 +1055,6 @@ def test_canonical_arrow_type_prefers_the_compact_grammar():
|
||||
== "large_list<float32>"
|
||||
)
|
||||
for outside in [
|
||||
pa.decimal128(10, 2),
|
||||
pa.binary(4),
|
||||
pa.time32("s"),
|
||||
pa.time64("us"),
|
||||
@@ -1531,6 +1531,157 @@ def test_duration_annotation_and_explicit_schema_serialization():
|
||||
assert explicit_round_trip.output.arrow_type == '{"type":"duration"}'
|
||||
|
||||
|
||||
def test_map_annotation_and_explicit_schema_serialization():
|
||||
@function
|
||||
def map_round_trip(values: dict[str, int]) -> dict[str, int]:
|
||||
return values
|
||||
|
||||
signature = map_round_trip.registration_request.signature
|
||||
expected = {
|
||||
"type": "map",
|
||||
"fields": [
|
||||
{
|
||||
"name": "entries",
|
||||
"nullable": False,
|
||||
"type": {
|
||||
"type": "struct",
|
||||
"fields": [
|
||||
{
|
||||
"name": "key",
|
||||
"nullable": False,
|
||||
"type": {"type": "utf8"},
|
||||
},
|
||||
{
|
||||
"name": "value",
|
||||
"nullable": False,
|
||||
"type": {"type": "int64"},
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
assert json.loads(signature.inputs[0].arrow_type) == expected
|
||||
assert json.loads(signature.output.arrow_type) == expected
|
||||
round_trip = FunctionSignature.from_json(signature.to_canonical_json())
|
||||
assert round_trip.inputs[0].arrow_type == signature.inputs[0].arrow_type
|
||||
assert round_trip.output.arrow_type == signature.output.arrow_type
|
||||
|
||||
@function
|
||||
def nullable_values(
|
||||
values: dict[str, Optional[int]],
|
||||
) -> dict[str, Optional[int]]:
|
||||
return values
|
||||
|
||||
nullable_signature = nullable_values.registration_request.signature
|
||||
nullable_map = json.loads(nullable_signature.inputs[0].arrow_type)
|
||||
assert nullable_map["fields"][0]["type"]["fields"][1] == {
|
||||
"name": "value",
|
||||
"nullable": True,
|
||||
"type": {"type": "int64"},
|
||||
}
|
||||
|
||||
input_schema = pa.schema(
|
||||
[
|
||||
pa.field(
|
||||
"values",
|
||||
pa.map_(pa.string(), pa.field("custom", pa.int64(), nullable=True)),
|
||||
nullable=False,
|
||||
)
|
||||
]
|
||||
)
|
||||
output_schema = pa.field(
|
||||
"result",
|
||||
pa.map_(pa.string(), pa.field("value", pa.int64(), nullable=False)),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
@function(input_schema=input_schema, output_schema=output_schema)
|
||||
def explicit(values):
|
||||
return {}
|
||||
|
||||
explicit_signature = explicit.registration_request.signature
|
||||
explicit_input = json.loads(explicit_signature.inputs[0].arrow_type)
|
||||
assert explicit_input["fields"][0]["type"]["fields"][1] == {
|
||||
"name": "custom",
|
||||
"nullable": True,
|
||||
"type": {"type": "int64"},
|
||||
}
|
||||
explicit_output = json.loads(explicit_signature.output.arrow_type)
|
||||
assert explicit_output["fields"][0]["type"]["fields"][1]["nullable"] is False
|
||||
|
||||
|
||||
def test_map_rejects_unsupported_boundaries():
|
||||
with pytest.raises(TypeError, match="map keys_sorted=true"):
|
||||
_canonical_arrow_type(pa.map_(pa.string(), pa.int64(), keys_sorted=True))
|
||||
with pytest.raises(TypeError, match="map keys must be non-nullable utf8"):
|
||||
_canonical_arrow_type(pa.map_(pa.int64(), pa.int64()))
|
||||
with pytest.raises(TypeError, match="Blob v2 fields nested under"):
|
||||
_canonical_arrow_type(pa.map_(pa.string(), lancedb.blob("image").type))
|
||||
with pytest.raises(TypeError, match="map keys must be str"):
|
||||
|
||||
@function
|
||||
def invalid_keys(values: dict[int, int]) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def test_decimal_annotation_and_explicit_schema_serialization():
|
||||
@function
|
||||
def price(
|
||||
amount: Annotated[Decimal, pa.decimal128(10, 2)],
|
||||
) -> Annotated[Decimal, pa.decimal256(50, 4)]:
|
||||
return Decimal("1.0000")
|
||||
|
||||
signature = price.registration_request.signature
|
||||
assert json.loads(signature.inputs[0].arrow_type) == {
|
||||
"type": "decimal128",
|
||||
"length": 10002,
|
||||
}
|
||||
assert json.loads(signature.output.arrow_type) == {
|
||||
"type": "decimal256",
|
||||
"length": 50004,
|
||||
}
|
||||
round_trip = FunctionSignature.from_json(signature.to_canonical_json())
|
||||
assert round_trip.inputs[0].arrow_type == signature.inputs[0].arrow_type
|
||||
assert round_trip.output.arrow_type == signature.output.arrow_type
|
||||
assert json.loads(_canonical_arrow_type(pa.decimal128(10, -2))) == {
|
||||
"type": "decimal128",
|
||||
"length": 9998,
|
||||
}
|
||||
decimal_map = json.loads(
|
||||
_canonical_arrow_type(pa.map_(pa.string(), pa.decimal128(10, 2)))
|
||||
)
|
||||
assert decimal_map["fields"][0]["type"]["fields"][1]["type"] == {
|
||||
"type": "decimal128",
|
||||
"length": 10002,
|
||||
}
|
||||
|
||||
input_schema = pa.schema([pa.field("amount", pa.decimal128(12, 2), nullable=False)])
|
||||
output_schema = pa.field("result", pa.decimal256(60, 6), nullable=False)
|
||||
|
||||
@function(input_schema=input_schema, output_schema=output_schema)
|
||||
def explicit(amount: Decimal) -> Decimal:
|
||||
return amount
|
||||
|
||||
explicit_signature = explicit.registration_request.signature
|
||||
assert json.loads(explicit_signature.inputs[0].arrow_type) == {
|
||||
"type": "decimal128",
|
||||
"length": 12002,
|
||||
}
|
||||
assert json.loads(explicit_signature.output.arrow_type) == {
|
||||
"type": "decimal256",
|
||||
"length": 60006,
|
||||
}
|
||||
|
||||
with pytest.raises(
|
||||
TypeError, match="Decimal Function annotations require precision and scale"
|
||||
):
|
||||
|
||||
@function
|
||||
def missing(amount: Decimal) -> Decimal:
|
||||
return amount
|
||||
|
||||
|
||||
def test_nullable_scalar_output_annotation_and_explicit_schema():
|
||||
input_schema = pa.schema([pa.field("value", pa.int64(), nullable=False)])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user