From ea59f8acb61a6ac93fc940d055efe3c8f823991c Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 14 Nov 2026 21:49:45 +0800 Subject: [PATCH] feat(python): preserve Function timestamp and duration types (#5032) Function declarations currently reject exact timestamp and duration types. Preserve their Arrow units and timezones in canonical signature JSON, including fixed-offset timezones and nested fields. Bare `datetime` and microsecond/no-timezone declarations keep their existing representation; `timedelta` requires an explicit unit. Compound temporal signatures require the Lance Namespace decoder in https://github.com/lance-format/lance/pull/11059 and corresponding server validation support. Older readers reject these signatures; this change does not upgrade server support or dependency pins. Validation: 128 Python signature tests and 30 documentation tests passed (one skipped), using the changed Python source with an existing native extension. No Rust rebuild or end-to-end Function registration/refresh was performed. --- docs/web/management/function.mdx | 35 ++++ python/python/lancedb/functions.py | 48 +++++- .../tests/test_first_class_function_slice2.py | 156 +++++++++++++++++- 3 files changed, 227 insertions(+), 12 deletions(-) diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx index 41e64d482..f6264edbf 100644 --- a/docs/web/management/function.mdx +++ b/docs/web/management/function.mdx @@ -104,6 +104,41 @@ at all is the struct's own validity, which Enterprise keeps in a separate internal column when it expands the result into sibling columns, so a null field is a value rather than a missing row. +### Timestamps and durations {#timestamps-durations} + +Bare `datetime` keeps its existing `timestamp[us]` default without a timezone. +For any other unit or a timezone, declare the Arrow type explicitly: + +```python +from datetime import datetime +from typing import Annotated + +import pyarrow as pa + + +@function +def normalize( + ts: Annotated[datetime, pa.timestamp("ns", tz="UTC")], +) -> Annotated[datetime, pa.timestamp("ns", tz="UTC")]: + ... +``` + +`pa.timestamp` accepts `s`, `ms`, `us`, and `ns`, and a timezone such as +`UTC`, `America/New_York`, or `+08:00`. The canonical signature uses +`{"type":"timestamp::"}`, with `-` for no timezone; the +microsecond/no-timezone case keeps the legacy `{"type":"timestamp"}` shape. +A timezone string of `-` is reserved for "no timezone" and is rejected. + +`pa.duration` accepts the same four units and uses +`{"type":"duration:"}`; the microsecond case keeps the legacy +`{"type":"duration"}` shape. A bare `timedelta` has no unit and is rejected; +use `Annotated[timedelta, pa.duration("s")]` or an explicit schema. + +Other units are never downgraded to microseconds. A reader that predates the +compound temporal representation cannot read those signatures. A deployment +must include the corresponding Lance Namespace and Sophon support; this +client-side declaration does not add server support. + ## Connect and create a table {#table} The remaining steps run against your Enterprise deployment. Replace the database diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 471fc2437..18c504a8f 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -29,7 +29,7 @@ import textwrap import types import warnings from collections.abc import Mapping, Sequence -from datetime import date, datetime +from datetime import date, datetime, timedelta from pathlib import Path from typing import ( Annotated, @@ -1043,6 +1043,7 @@ _GRAMMAR_PRIMITIVES = ( (pa.date32(), "date32"), (pa.date64(), "date64"), ) +_FUNCTION_TIMEUNITS = ("s", "ms", "us", "ns") def _canonical_arrow_type( @@ -1259,6 +1260,26 @@ def _exact_arrow_type( for candidate, name in _GRAMMAR_PRIMITIVES: if data_type == candidate: return {"type": name} + if pa.types.is_timestamp(data_type): + unit = data_type.unit + if unit not in _FUNCTION_TIMEUNITS: + raise TypeError(f"unsupported Function timestamp unit: {unit!r}") + timezone = data_type.tz + if timezone == "-": + raise TypeError( + "unsupported Function timestamp timezone: '-' is reserved for " + "no timezone" + ) + if unit == "us" and timezone is None: + return {"type": "timestamp"} + return {"type": f"timestamp:{unit}:{timezone if timezone is not None else '-'}"} + if pa.types.is_duration(data_type): + unit = data_type.unit + if unit not in _FUNCTION_TIMEUNITS: + raise TypeError(f"unsupported Function duration unit: {unit!r}") + if unit == "us": + return {"type": "duration"} + return {"type": f"duration:{unit}"} if pa.types.is_struct(data_type): fields = list(data_type) names = [field.name for field in fields] @@ -1344,9 +1365,9 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]: raise TypeError( "Annotated Function types require exactly one PyArrow DataType" ) - # A struct value is written as a dict, which has no Arrow type of its - # own; the Annotated metadata supplies it. - if base is dict or get_origin(base) is dict: + # A struct value is written as a dict, and timedelta has no unit of + # its own; the Annotated metadata supplies those types. + if base is dict or get_origin(base) is dict or base is timedelta: base_nullable = False else: _, base_nullable = _annotation_type(base) @@ -1372,7 +1393,14 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]: if annotation is date: return pa.date32(), nullable if annotation is datetime: + # Existing default: microsecond precision, no timezone. Other units + # and timezones require Annotated or an explicit schema. return pa.timestamp("us"), nullable + if annotation is timedelta: + raise TypeError( + "timedelta Function annotations require an explicit unit; use " + "Annotated[timedelta, pyarrow.duration(unit)] or an explicit schema" + ) if get_origin(annotation) is list: arguments = get_args(annotation) if len(arguments) != 1: @@ -2701,7 +2729,11 @@ def function( deployment must accept nullable scalar outputs; an older server rejects the registration. Every named-struct field may be nullable; Enterprise preserves the struct's validity when the result is expanded into sibling - columns. + columns. Bare ``datetime`` keeps its existing ``timestamp[us]`` default + without a timezone; other units and timezones require + ``Annotated[datetime, pyarrow.timestamp(unit, tz=...)]`` or an explicit + schema. A bare ``timedelta`` has no unit and is rejected; use + ``Annotated[timedelta, pyarrow.duration(unit)]`` or an explicit schema. **Large values.** A ``bytes`` parameter receives the whole value, which is held in memory with its row and counts against the server's per-row @@ -2808,8 +2840,10 @@ def function( output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional Explicit scalar or named-struct output. A scalar field's ``nullable`` flag is preserved; a top-level named-struct field and a Blob v2 output - must be non-nullable. Required for batch annotations. When omitted, the - output is inferred from the return annotation. + must be non-nullable. Temporal fields use canonical Arrow JSON; the + microsecond/no-timezone case keeps the legacy timestamp/duration + shape. Required for batch annotations. When omitted, the output is + inferred from the return annotation. pip : sequence of str, optional Pip requirements for the remote environment. Syntax is validated at decoration time, including extras and environment markers. Named direct diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 7e089a96d..1e896f780 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -9,7 +9,7 @@ import functools import importlib.util import inspect import types -from datetime import date +from datetime import date, datetime, timedelta import http.server import json import os @@ -19,7 +19,7 @@ import sys import typing import urllib.parse import warnings -from typing import Optional +from typing import Annotated, Optional import pyarrow as pa import pytest @@ -884,12 +884,11 @@ def test_canonical_arrow_type_prefers_the_compact_grammar(): == "large_list" ) for outside in [ - pa.timestamp("us"), pa.decimal128(10, 2), pa.binary(4), - pa.duration("s"), + pa.time32("s"), + pa.time64("us"), pa.list_(pa.float32(), 0), - pa.list_(pa.timestamp("us")), ]: with pytest.raises(TypeError, match="unsupported Arrow type"): _canonical_arrow_type(outside) @@ -1215,6 +1214,153 @@ def test_explicit_arrow_schema_is_deterministic(): assert signature.output.nullable is False +def test_timestamp_annotation_and_explicit_schema_serialization(): + for unit in ("s", "ms", "us", "ns"): + for timezone in (None, "UTC", "America/New_York", "+08:00", "-05:00"): + encoded = json.loads(_canonical_arrow_type(pa.timestamp(unit, tz=timezone))) + if unit == "us" and timezone is None: + assert encoded == {"type": "timestamp"} + else: + suffix = timezone if timezone is not None else "-" + assert encoded == {"type": f"timestamp:{unit}:{suffix}"} + + with pytest.raises(TypeError, match="reserved for no timezone"): + _canonical_arrow_type(pa.timestamp("us", tz="-")) + + @function + def normalize( + ts: Annotated[datetime, pa.timestamp("ns", tz="UTC")], + ) -> Annotated[datetime, pa.timestamp("ns", tz="UTC")]: + return ts + + signature = normalize.registration_request.signature + assert json.loads(signature.inputs[0].arrow_type) == {"type": "timestamp:ns:UTC"} + assert json.loads(signature.output.arrow_type) == {"type": "timestamp:ns:UTC"} + round_trip = FunctionSignature.from_json(signature.to_canonical_json()) + assert round_trip.inputs[0].arrow_type == signature.inputs[0].arrow_type + assert round_trip.output.arrow_type == signature.output.arrow_type + + @function + def optional_timestamp( + ts: Optional[Annotated[datetime, pa.timestamp("ms", tz="UTC")]], + ) -> int: + return 0 + + optional_input = optional_timestamp.registration_request.signature.inputs[0] + assert optional_input.nullable is True + assert json.loads(optional_input.arrow_type) == {"type": "timestamp:ms:UTC"} + + @function + def legacy(ts: datetime) -> datetime: + return ts + + legacy_signature = legacy.registration_request.signature + assert json.loads(legacy_signature.inputs[0].arrow_type) == {"type": "timestamp"} + assert json.loads(legacy_signature.output.arrow_type) == {"type": "timestamp"} + legacy_round_trip = FunctionSignature.from_json( + legacy_signature.to_canonical_json() + ) + assert legacy_round_trip.inputs[0].arrow_type == '{"type":"timestamp"}' + + input_schema = pa.schema( + [pa.field("ts", pa.timestamp("ms", tz="America/New_York"), nullable=True)] + ) + output_schema = pa.schema( + [ + pa.field("ts", pa.timestamp("ns", tz="+08:00"), nullable=True), + pa.field("elapsed", pa.duration("ms"), nullable=False), + ] + ) + + @function(input_schema=input_schema, output_schema=output_schema) + def nested(ts): + return {"ts": ts, "elapsed": None} + + nested_signature = nested.registration_request.signature + assert nested_signature.inputs[0].nullable is True + assert json.loads(nested_signature.inputs[0].arrow_type) == { + "type": "timestamp:ms:America/New_York" + } + assert nested_signature.output.kind == "named_struct" + assert nested_signature.output.fields[0].nullable is True + assert json.loads(nested_signature.output.fields[0].arrow_type) == { + "type": "timestamp:ns:+08:00" + } + assert json.loads(nested_signature.output.fields[1].arrow_type) == { + "type": "duration:ms" + } + + list_type = pa.list_(pa.field("item", pa.timestamp("ms", tz="UTC"), nullable=False)) + assert json.loads(_canonical_arrow_type(list_type)) == { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": False, + "type": {"type": "timestamp:ms:UTC"}, + } + ], + } + + +def test_duration_annotation_and_explicit_schema_serialization(): + for unit in ("s", "ms", "us", "ns"): + encoded = json.loads(_canonical_arrow_type(pa.duration(unit))) + expected = ( + {"type": "duration"} if unit == "us" else {"type": f"duration:{unit}"} + ) + assert encoded == expected + + @function + def elapsed( + value: Annotated[timedelta, pa.duration("s")], + ) -> Annotated[timedelta, pa.duration("ns")]: + return value + + signature = elapsed.registration_request.signature + assert json.loads(signature.inputs[0].arrow_type) == {"type": "duration:s"} + assert json.loads(signature.output.arrow_type) == {"type": "duration:ns"} + round_trip = FunctionSignature.from_json(signature.to_canonical_json()) + assert round_trip.inputs[0].arrow_type == signature.inputs[0].arrow_type + assert round_trip.output.arrow_type == signature.output.arrow_type + + @function + def optional_elapsed( + value: Optional[Annotated[timedelta, pa.duration("ms")]], + ) -> int: + return 0 + + optional_input = optional_elapsed.registration_request.signature.inputs[0] + assert optional_input.nullable is True + assert json.loads(optional_input.arrow_type) == {"type": "duration:ms"} + + with pytest.raises( + TypeError, match="timedelta Function annotations require an explicit unit" + ): + + @function + def missing(value: timedelta) -> timedelta: + return value + + input_schema = pa.schema([pa.field("elapsed", pa.duration("ms"), nullable=True)]) + output_schema = pa.field("result", pa.duration("us"), nullable=False) + + @function(input_schema=input_schema, output_schema=output_schema) + def explicit(elapsed: timedelta) -> timedelta: + return elapsed + + explicit_signature = explicit.registration_request.signature + assert explicit_signature.inputs[0].nullable is True + assert json.loads(explicit_signature.inputs[0].arrow_type) == { + "type": "duration:ms" + } + assert json.loads(explicit_signature.output.arrow_type) == {"type": "duration"} + explicit_round_trip = FunctionSignature.from_json( + explicit_signature.to_canonical_json() + ) + assert explicit_round_trip.output.arrow_type == '{"type":"duration"}' + + def test_nullable_scalar_output_annotation_and_explicit_schema(): input_schema = pa.schema([pa.field("value", pa.int64(), nullable=False)])