feat(python): support map and decimal Function signatures (#5031)

Python Function declarations reject Arrow map and decimal types before
registration. Accept `dict[str, T]`, nullable map values, explicit Arrow
maps, and `Annotated[Decimal, pa.decimal128/256(...)]` using the
existing canonical JSON representation. Preserve map key/value
nullability and decimal precision/scale; reject unsupported map keys,
sorted maps, nested Blob values, and a bare Decimal without
precision/scale.

This is the SDK map/decimal portion of ENT-3414. Server validation and
SQL support are separate changes; older servers can still reject these
signatures. Nullable scalar output support remains separate in #5030.

Validation: 129 Function signature tests passed; 30 documentation tests
passed with one existing skip. Ruff format/lint and diff checks passed.
Python tests used the existing Python 3.13 environment and prebuilt
native extension with this Python source overlaid; no Rust build or
remote-server execution was performed.
This commit is contained in:
Xuanwo
2026-11-14 22:30:47 +08:00
committed by GitHub
parent 2499342fa1
commit dc9eb269b1
3 changed files with 267 additions and 11 deletions
+40
View File
@@ -139,6 +139,46 @@ compound temporal representation cannot read those signatures. A deployment
must include the corresponding Lance Namespace and Sophon support; this
client-side declaration does not add server support.
### Maps and decimals {#maps-decimals}
A `dict[str, T]` annotation declares an Arrow map with a non-nullable `utf8`
key. A plain `T` value is non-nullable; annotate `dict[str, Optional[T]]` for
nullable values. An explicit `pa.map_(...)` schema preserves the value field's
own `nullable` flag:
```python
@function
def word_counts(text: str) -> dict[str, int]:
...
```
Map keys must be non-nullable `utf8`; `keys_sorted=True` is not supported, and
Blob v2 values inside a map are not supported.
`Decimal` carries no precision or scale by itself, so a bare `Decimal`
annotation is rejected. Declare both explicitly with `Annotated`, or use an
explicit Arrow schema:
```python
from decimal import Decimal
from typing import Annotated
import pyarrow as pa
@function
def add_tax(
amount: Annotated[Decimal, pa.decimal128(12, 2)],
) -> Annotated[Decimal, pa.decimal128(12, 2)]:
...
```
`decimal128` and `decimal256` keep the declared precision and scale in the
canonical signature; no default is chosen. Map and decimal signatures are sent
as canonical Arrow JSON. A deployment must support those types; an older
Enterprise server may reject the registration, and this client-side
declaration does not add server support.
## Connect and create a table {#table}
The remaining steps run against your Enterprise deployment. Replace the database
+75 -10
View File
@@ -30,6 +30,7 @@ import types
import warnings
from collections.abc import Mapping, Sequence
from datetime import date, datetime, timedelta
from decimal import Decimal
from pathlib import Path
from typing import (
Annotated,
@@ -1378,6 +1379,13 @@ def _exact_arrow_type(
for candidate, name in _GRAMMAR_PRIMITIVES:
if data_type == candidate:
return {"type": name}
if pa.types.is_decimal128(data_type) or pa.types.is_decimal256(data_type):
return {
"type": (
"decimal128" if pa.types.is_decimal128(data_type) else "decimal256"
),
"length": data_type.precision * 1000 + data_type.scale,
}
if pa.types.is_timestamp(data_type):
unit = data_type.unit
if unit not in _FUNCTION_TIMEUNITS:
@@ -1453,8 +1461,29 @@ def _exact_arrow_type(
if pa.types.is_fixed_size_list(data_type):
value["length"] = data_type.list_size
return value
if pa.types.is_map(data_type) and _data_type_has_blob_v2(data_type):
raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
if pa.types.is_map(data_type):
if data_type.keys_sorted:
raise TypeError(
"unsupported Arrow type for Function signature: map keys_sorted=true "
"is not supported"
)
key_field = data_type.key_field
if key_field.nullable or key_field.type != pa.string():
raise TypeError(
"unsupported Arrow type for Function signature: map keys must be "
"non-nullable utf8"
)
if _data_type_has_blob_v2(data_type):
raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
return {
"type": "map",
"fields": [
_exact_arrow_field(
data_type.field(0),
inside_collection=True,
)
],
}
raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
@@ -1483,9 +1512,15 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
raise TypeError(
"Annotated Function types require exactly one PyArrow DataType"
)
# A struct value is written as a dict, and timedelta has no unit of
# its own; the Annotated metadata supplies those types.
if base is dict or get_origin(base) is dict or base is timedelta:
# A struct value is written as a dict, while Decimal and timedelta
# carry no precision or unit of their own; the Annotated metadata
# supplies those types.
if (
base is dict
or get_origin(base) is dict
or base is Decimal
or base is timedelta
):
base_nullable = False
else:
_, base_nullable = _annotation_type(base)
@@ -1514,6 +1549,13 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
# Existing default: microsecond precision, no timezone. Other units
# and timezones require Annotated or an explicit schema.
return pa.timestamp("us"), nullable
if annotation is Decimal:
raise TypeError(
"Decimal Function annotations require precision and scale; use "
"Annotated[Decimal, pyarrow.decimal128(precision, scale)], "
"Annotated[Decimal, pyarrow.decimal256(precision, scale)], or an "
"explicit schema"
)
if annotation is timedelta:
raise TypeError(
"timedelta Function annotations require an explicit unit; use "
@@ -1527,6 +1569,21 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
if value_nullable:
raise TypeError("nullable Function list elements are not supported")
return _list_of(value_type), nullable
if get_origin(annotation) is dict:
arguments = get_args(annotation)
if len(arguments) != 2:
raise TypeError(f"unsupported dict annotation: {annotation!r}")
key_type, value_annotation = arguments
if key_type is not str:
raise TypeError("Function map keys must be str")
value_type, value_nullable = _annotation_type(value_annotation)
return (
pa.map_(
pa.string(),
pa.field("value", value_type, nullable=value_nullable),
),
nullable,
)
raise TypeError(f"unsupported Function annotation: {annotation!r}")
@@ -2847,7 +2904,13 @@ def function(
deployment must accept nullable scalar outputs; an older server rejects
the registration. Every named-struct field may be nullable; Enterprise
preserves the struct's validity when the result is expanded into sibling
columns. Bare ``datetime`` keeps its existing ``timestamp[us]`` default
columns.
``dict[str, T]`` declares an Arrow map with a non-nullable utf8 key; the
value is non-nullable for ``T`` and nullable for ``Optional[T]``. A bare
``Decimal`` has no precision or scale; use
``Annotated[Decimal, pyarrow.decimal128(precision, scale)]``,
``Annotated[Decimal, pyarrow.decimal256(precision, scale)]``, or an
explicit schema. Bare ``datetime`` keeps its existing ``timestamp[us]`` default
without a timezone; other units and timezones require
``Annotated[datetime, pyarrow.timestamp(unit, tz=...)]`` or an explicit
schema. A bare ``timedelta`` has no unit and is rejected; use
@@ -2958,10 +3021,12 @@ def function(
output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional
Explicit scalar or named-struct output. A scalar field's ``nullable``
flag is preserved; a top-level named-struct field and a Blob v2 output
must be non-nullable. Temporal fields use canonical Arrow JSON; the
microsecond/no-timezone case keeps the legacy timestamp/duration
shape. Required for batch annotations. When omitted, the output is
inferred from the return annotation.
must be non-nullable. Map and decimal fields use canonical Arrow JSON;
a map key must be non-nullable utf8 and its value field keeps the
declared nullability. Temporal fields use canonical Arrow JSON; the
microsecond/no-timezone case keeps the legacy timestamp/duration shape.
Required for batch annotations. When omitted, the output is inferred
from the return annotation.
pip : sequence of str, optional
Pip requirements for the remote environment. Syntax is validated at
decoration time, including extras and environment markers. Named direct
@@ -19,6 +19,7 @@ import sys
import typing
import urllib.parse
import warnings
from decimal import Decimal
from typing import Annotated, Optional
import pyarrow as pa
@@ -1054,7 +1055,6 @@ def test_canonical_arrow_type_prefers_the_compact_grammar():
== "large_list<float32>"
)
for outside in [
pa.decimal128(10, 2),
pa.binary(4),
pa.time32("s"),
pa.time64("us"),
@@ -1531,6 +1531,157 @@ def test_duration_annotation_and_explicit_schema_serialization():
assert explicit_round_trip.output.arrow_type == '{"type":"duration"}'
def test_map_annotation_and_explicit_schema_serialization():
@function
def map_round_trip(values: dict[str, int]) -> dict[str, int]:
return values
signature = map_round_trip.registration_request.signature
expected = {
"type": "map",
"fields": [
{
"name": "entries",
"nullable": False,
"type": {
"type": "struct",
"fields": [
{
"name": "key",
"nullable": False,
"type": {"type": "utf8"},
},
{
"name": "value",
"nullable": False,
"type": {"type": "int64"},
},
],
},
}
],
}
assert json.loads(signature.inputs[0].arrow_type) == expected
assert json.loads(signature.output.arrow_type) == expected
round_trip = FunctionSignature.from_json(signature.to_canonical_json())
assert round_trip.inputs[0].arrow_type == signature.inputs[0].arrow_type
assert round_trip.output.arrow_type == signature.output.arrow_type
@function
def nullable_values(
values: dict[str, Optional[int]],
) -> dict[str, Optional[int]]:
return values
nullable_signature = nullable_values.registration_request.signature
nullable_map = json.loads(nullable_signature.inputs[0].arrow_type)
assert nullable_map["fields"][0]["type"]["fields"][1] == {
"name": "value",
"nullable": True,
"type": {"type": "int64"},
}
input_schema = pa.schema(
[
pa.field(
"values",
pa.map_(pa.string(), pa.field("custom", pa.int64(), nullable=True)),
nullable=False,
)
]
)
output_schema = pa.field(
"result",
pa.map_(pa.string(), pa.field("value", pa.int64(), nullable=False)),
nullable=False,
)
@function(input_schema=input_schema, output_schema=output_schema)
def explicit(values):
return {}
explicit_signature = explicit.registration_request.signature
explicit_input = json.loads(explicit_signature.inputs[0].arrow_type)
assert explicit_input["fields"][0]["type"]["fields"][1] == {
"name": "custom",
"nullable": True,
"type": {"type": "int64"},
}
explicit_output = json.loads(explicit_signature.output.arrow_type)
assert explicit_output["fields"][0]["type"]["fields"][1]["nullable"] is False
def test_map_rejects_unsupported_boundaries():
with pytest.raises(TypeError, match="map keys_sorted=true"):
_canonical_arrow_type(pa.map_(pa.string(), pa.int64(), keys_sorted=True))
with pytest.raises(TypeError, match="map keys must be non-nullable utf8"):
_canonical_arrow_type(pa.map_(pa.int64(), pa.int64()))
with pytest.raises(TypeError, match="Blob v2 fields nested under"):
_canonical_arrow_type(pa.map_(pa.string(), lancedb.blob("image").type))
with pytest.raises(TypeError, match="map keys must be str"):
@function
def invalid_keys(values: dict[int, int]) -> int:
return 0
def test_decimal_annotation_and_explicit_schema_serialization():
@function
def price(
amount: Annotated[Decimal, pa.decimal128(10, 2)],
) -> Annotated[Decimal, pa.decimal256(50, 4)]:
return Decimal("1.0000")
signature = price.registration_request.signature
assert json.loads(signature.inputs[0].arrow_type) == {
"type": "decimal128",
"length": 10002,
}
assert json.loads(signature.output.arrow_type) == {
"type": "decimal256",
"length": 50004,
}
round_trip = FunctionSignature.from_json(signature.to_canonical_json())
assert round_trip.inputs[0].arrow_type == signature.inputs[0].arrow_type
assert round_trip.output.arrow_type == signature.output.arrow_type
assert json.loads(_canonical_arrow_type(pa.decimal128(10, -2))) == {
"type": "decimal128",
"length": 9998,
}
decimal_map = json.loads(
_canonical_arrow_type(pa.map_(pa.string(), pa.decimal128(10, 2)))
)
assert decimal_map["fields"][0]["type"]["fields"][1]["type"] == {
"type": "decimal128",
"length": 10002,
}
input_schema = pa.schema([pa.field("amount", pa.decimal128(12, 2), nullable=False)])
output_schema = pa.field("result", pa.decimal256(60, 6), nullable=False)
@function(input_schema=input_schema, output_schema=output_schema)
def explicit(amount: Decimal) -> Decimal:
return amount
explicit_signature = explicit.registration_request.signature
assert json.loads(explicit_signature.inputs[0].arrow_type) == {
"type": "decimal128",
"length": 12002,
}
assert json.loads(explicit_signature.output.arrow_type) == {
"type": "decimal256",
"length": 60006,
}
with pytest.raises(
TypeError, match="Decimal Function annotations require precision and scale"
):
@function
def missing(amount: Decimal) -> Decimal:
return amount
def test_nullable_scalar_output_annotation_and_explicit_schema():
input_schema = pa.schema([pa.field("value", pa.int64(), nullable=False)])