From dc9eb269b15bae731e3d99ddcf67f61723fc703c Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 14 Nov 2026 22:30:47 +0800 Subject: [PATCH] feat(python): support map and decimal Function signatures (#5031) Python Function declarations reject Arrow map and decimal types before registration. Accept `dict[str, T]`, nullable map values, explicit Arrow maps, and `Annotated[Decimal, pa.decimal128/256(...)]` using the existing canonical JSON representation. Preserve map key/value nullability and decimal precision/scale; reject unsupported map keys, sorted maps, nested Blob values, and a bare Decimal without precision/scale. This is the SDK map/decimal portion of ENT-3414. Server validation and SQL support are separate changes; older servers can still reject these signatures. Nullable scalar output support remains separate in #5030. Validation: 129 Function signature tests passed; 30 documentation tests passed with one existing skip. Ruff format/lint and diff checks passed. Python tests used the existing Python 3.13 environment and prebuilt native extension with this Python source overlaid; no Rust build or remote-server execution was performed. --- docs/web/management/function.mdx | 40 +++++ python/python/lancedb/functions.py | 85 ++++++++-- .../tests/test_first_class_function_slice2.py | 153 +++++++++++++++++- 3 files changed, 267 insertions(+), 11 deletions(-) diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx index efcbcc799..41e47dee1 100644 --- a/docs/web/management/function.mdx +++ b/docs/web/management/function.mdx @@ -139,6 +139,46 @@ compound temporal representation cannot read those signatures. A deployment must include the corresponding Lance Namespace and Sophon support; this client-side declaration does not add server support. +### Maps and decimals {#maps-decimals} + +A `dict[str, T]` annotation declares an Arrow map with a non-nullable `utf8` +key. A plain `T` value is non-nullable; annotate `dict[str, Optional[T]]` for +nullable values. An explicit `pa.map_(...)` schema preserves the value field's +own `nullable` flag: + +```python +@function +def word_counts(text: str) -> dict[str, int]: + ... +``` + +Map keys must be non-nullable `utf8`; `keys_sorted=True` is not supported, and +Blob v2 values inside a map are not supported. + +`Decimal` carries no precision or scale by itself, so a bare `Decimal` +annotation is rejected. Declare both explicitly with `Annotated`, or use an +explicit Arrow schema: + +```python +from decimal import Decimal +from typing import Annotated + +import pyarrow as pa + + +@function +def add_tax( + amount: Annotated[Decimal, pa.decimal128(12, 2)], +) -> Annotated[Decimal, pa.decimal128(12, 2)]: + ... +``` + +`decimal128` and `decimal256` keep the declared precision and scale in the +canonical signature; no default is chosen. Map and decimal signatures are sent +as canonical Arrow JSON. A deployment must support those types; an older +Enterprise server may reject the registration, and this client-side +declaration does not add server support. + ## Connect and create a table {#table} The remaining steps run against your Enterprise deployment. Replace the database diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index ecbc697d1..548129211 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -30,6 +30,7 @@ import types import warnings from collections.abc import Mapping, Sequence from datetime import date, datetime, timedelta +from decimal import Decimal from pathlib import Path from typing import ( Annotated, @@ -1378,6 +1379,13 @@ def _exact_arrow_type( for candidate, name in _GRAMMAR_PRIMITIVES: if data_type == candidate: return {"type": name} + if pa.types.is_decimal128(data_type) or pa.types.is_decimal256(data_type): + return { + "type": ( + "decimal128" if pa.types.is_decimal128(data_type) else "decimal256" + ), + "length": data_type.precision * 1000 + data_type.scale, + } if pa.types.is_timestamp(data_type): unit = data_type.unit if unit not in _FUNCTION_TIMEUNITS: @@ -1453,8 +1461,29 @@ def _exact_arrow_type( if pa.types.is_fixed_size_list(data_type): value["length"] = data_type.list_size return value - if pa.types.is_map(data_type) and _data_type_has_blob_v2(data_type): - raise TypeError(_NESTED_BLOB_COLLECTION_ERROR) + if pa.types.is_map(data_type): + if data_type.keys_sorted: + raise TypeError( + "unsupported Arrow type for Function signature: map keys_sorted=true " + "is not supported" + ) + key_field = data_type.key_field + if key_field.nullable or key_field.type != pa.string(): + raise TypeError( + "unsupported Arrow type for Function signature: map keys must be " + "non-nullable utf8" + ) + if _data_type_has_blob_v2(data_type): + raise TypeError(_NESTED_BLOB_COLLECTION_ERROR) + return { + "type": "map", + "fields": [ + _exact_arrow_field( + data_type.field(0), + inside_collection=True, + ) + ], + } raise TypeError(f"unsupported Arrow type for Function signature: {data_type}") @@ -1483,9 +1512,15 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]: raise TypeError( "Annotated Function types require exactly one PyArrow DataType" ) - # A struct value is written as a dict, and timedelta has no unit of - # its own; the Annotated metadata supplies those types. - if base is dict or get_origin(base) is dict or base is timedelta: + # A struct value is written as a dict, while Decimal and timedelta + # carry no precision or unit of their own; the Annotated metadata + # supplies those types. + if ( + base is dict + or get_origin(base) is dict + or base is Decimal + or base is timedelta + ): base_nullable = False else: _, base_nullable = _annotation_type(base) @@ -1514,6 +1549,13 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]: # Existing default: microsecond precision, no timezone. Other units # and timezones require Annotated or an explicit schema. return pa.timestamp("us"), nullable + if annotation is Decimal: + raise TypeError( + "Decimal Function annotations require precision and scale; use " + "Annotated[Decimal, pyarrow.decimal128(precision, scale)], " + "Annotated[Decimal, pyarrow.decimal256(precision, scale)], or an " + "explicit schema" + ) if annotation is timedelta: raise TypeError( "timedelta Function annotations require an explicit unit; use " @@ -1527,6 +1569,21 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]: if value_nullable: raise TypeError("nullable Function list elements are not supported") return _list_of(value_type), nullable + if get_origin(annotation) is dict: + arguments = get_args(annotation) + if len(arguments) != 2: + raise TypeError(f"unsupported dict annotation: {annotation!r}") + key_type, value_annotation = arguments + if key_type is not str: + raise TypeError("Function map keys must be str") + value_type, value_nullable = _annotation_type(value_annotation) + return ( + pa.map_( + pa.string(), + pa.field("value", value_type, nullable=value_nullable), + ), + nullable, + ) raise TypeError(f"unsupported Function annotation: {annotation!r}") @@ -2847,7 +2904,13 @@ def function( deployment must accept nullable scalar outputs; an older server rejects the registration. Every named-struct field may be nullable; Enterprise preserves the struct's validity when the result is expanded into sibling - columns. Bare ``datetime`` keeps its existing ``timestamp[us]`` default + columns. + ``dict[str, T]`` declares an Arrow map with a non-nullable utf8 key; the + value is non-nullable for ``T`` and nullable for ``Optional[T]``. A bare + ``Decimal`` has no precision or scale; use + ``Annotated[Decimal, pyarrow.decimal128(precision, scale)]``, + ``Annotated[Decimal, pyarrow.decimal256(precision, scale)]``, or an + explicit schema. Bare ``datetime`` keeps its existing ``timestamp[us]`` default without a timezone; other units and timezones require ``Annotated[datetime, pyarrow.timestamp(unit, tz=...)]`` or an explicit schema. A bare ``timedelta`` has no unit and is rejected; use @@ -2958,10 +3021,12 @@ def function( output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional Explicit scalar or named-struct output. A scalar field's ``nullable`` flag is preserved; a top-level named-struct field and a Blob v2 output - must be non-nullable. Temporal fields use canonical Arrow JSON; the - microsecond/no-timezone case keeps the legacy timestamp/duration - shape. Required for batch annotations. When omitted, the output is - inferred from the return annotation. + must be non-nullable. Map and decimal fields use canonical Arrow JSON; + a map key must be non-nullable utf8 and its value field keeps the + declared nullability. Temporal fields use canonical Arrow JSON; the + microsecond/no-timezone case keeps the legacy timestamp/duration shape. + Required for batch annotations. When omitted, the output is inferred + from the return annotation. pip : sequence of str, optional Pip requirements for the remote environment. Syntax is validated at decoration time, including extras and environment markers. Named direct diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 6dcc0bbbe..dbc574e88 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -19,6 +19,7 @@ import sys import typing import urllib.parse import warnings +from decimal import Decimal from typing import Annotated, Optional import pyarrow as pa @@ -1054,7 +1055,6 @@ def test_canonical_arrow_type_prefers_the_compact_grammar(): == "large_list" ) for outside in [ - pa.decimal128(10, 2), pa.binary(4), pa.time32("s"), pa.time64("us"), @@ -1531,6 +1531,157 @@ def test_duration_annotation_and_explicit_schema_serialization(): assert explicit_round_trip.output.arrow_type == '{"type":"duration"}' +def test_map_annotation_and_explicit_schema_serialization(): + @function + def map_round_trip(values: dict[str, int]) -> dict[str, int]: + return values + + signature = map_round_trip.registration_request.signature + expected = { + "type": "map", + "fields": [ + { + "name": "entries", + "nullable": False, + "type": { + "type": "struct", + "fields": [ + { + "name": "key", + "nullable": False, + "type": {"type": "utf8"}, + }, + { + "name": "value", + "nullable": False, + "type": {"type": "int64"}, + }, + ], + }, + } + ], + } + assert json.loads(signature.inputs[0].arrow_type) == expected + assert json.loads(signature.output.arrow_type) == expected + round_trip = FunctionSignature.from_json(signature.to_canonical_json()) + assert round_trip.inputs[0].arrow_type == signature.inputs[0].arrow_type + assert round_trip.output.arrow_type == signature.output.arrow_type + + @function + def nullable_values( + values: dict[str, Optional[int]], + ) -> dict[str, Optional[int]]: + return values + + nullable_signature = nullable_values.registration_request.signature + nullable_map = json.loads(nullable_signature.inputs[0].arrow_type) + assert nullable_map["fields"][0]["type"]["fields"][1] == { + "name": "value", + "nullable": True, + "type": {"type": "int64"}, + } + + input_schema = pa.schema( + [ + pa.field( + "values", + pa.map_(pa.string(), pa.field("custom", pa.int64(), nullable=True)), + nullable=False, + ) + ] + ) + output_schema = pa.field( + "result", + pa.map_(pa.string(), pa.field("value", pa.int64(), nullable=False)), + nullable=False, + ) + + @function(input_schema=input_schema, output_schema=output_schema) + def explicit(values): + return {} + + explicit_signature = explicit.registration_request.signature + explicit_input = json.loads(explicit_signature.inputs[0].arrow_type) + assert explicit_input["fields"][0]["type"]["fields"][1] == { + "name": "custom", + "nullable": True, + "type": {"type": "int64"}, + } + explicit_output = json.loads(explicit_signature.output.arrow_type) + assert explicit_output["fields"][0]["type"]["fields"][1]["nullable"] is False + + +def test_map_rejects_unsupported_boundaries(): + with pytest.raises(TypeError, match="map keys_sorted=true"): + _canonical_arrow_type(pa.map_(pa.string(), pa.int64(), keys_sorted=True)) + with pytest.raises(TypeError, match="map keys must be non-nullable utf8"): + _canonical_arrow_type(pa.map_(pa.int64(), pa.int64())) + with pytest.raises(TypeError, match="Blob v2 fields nested under"): + _canonical_arrow_type(pa.map_(pa.string(), lancedb.blob("image").type)) + with pytest.raises(TypeError, match="map keys must be str"): + + @function + def invalid_keys(values: dict[int, int]) -> int: + return 0 + + +def test_decimal_annotation_and_explicit_schema_serialization(): + @function + def price( + amount: Annotated[Decimal, pa.decimal128(10, 2)], + ) -> Annotated[Decimal, pa.decimal256(50, 4)]: + return Decimal("1.0000") + + signature = price.registration_request.signature + assert json.loads(signature.inputs[0].arrow_type) == { + "type": "decimal128", + "length": 10002, + } + assert json.loads(signature.output.arrow_type) == { + "type": "decimal256", + "length": 50004, + } + round_trip = FunctionSignature.from_json(signature.to_canonical_json()) + assert round_trip.inputs[0].arrow_type == signature.inputs[0].arrow_type + assert round_trip.output.arrow_type == signature.output.arrow_type + assert json.loads(_canonical_arrow_type(pa.decimal128(10, -2))) == { + "type": "decimal128", + "length": 9998, + } + decimal_map = json.loads( + _canonical_arrow_type(pa.map_(pa.string(), pa.decimal128(10, 2))) + ) + assert decimal_map["fields"][0]["type"]["fields"][1]["type"] == { + "type": "decimal128", + "length": 10002, + } + + input_schema = pa.schema([pa.field("amount", pa.decimal128(12, 2), nullable=False)]) + output_schema = pa.field("result", pa.decimal256(60, 6), nullable=False) + + @function(input_schema=input_schema, output_schema=output_schema) + def explicit(amount: Decimal) -> Decimal: + return amount + + explicit_signature = explicit.registration_request.signature + assert json.loads(explicit_signature.inputs[0].arrow_type) == { + "type": "decimal128", + "length": 12002, + } + assert json.loads(explicit_signature.output.arrow_type) == { + "type": "decimal256", + "length": 60006, + } + + with pytest.raises( + TypeError, match="Decimal Function annotations require precision and scale" + ): + + @function + def missing(amount: Decimal) -> Decimal: + return amount + + def test_nullable_scalar_output_annotation_and_explicit_schema(): input_schema = pa.schema([pa.field("value", pa.int64(), nullable=False)])