feat(python): preserve Function timestamp and duration types (#5032)

Function declarations currently reject exact timestamp and duration
types. Preserve their Arrow units and timezones in canonical signature
JSON, including fixed-offset timezones and nested fields. Bare
`datetime` and microsecond/no-timezone declarations keep their existing
representation; `timedelta` requires an explicit unit.

Compound temporal signatures require the Lance Namespace decoder in
https://github.com/lance-format/lance/pull/11059 and corresponding
server validation support. Older readers reject these signatures; this
change does not upgrade server support or dependency pins.

Validation: 128 Python signature tests and 30 documentation tests passed
(one skipped), using the changed Python source with an existing native
extension. No Rust rebuild or end-to-end Function registration/refresh
was performed.
This commit is contained in:
Xuanwo
2026-11-14 21:49:45 +08:00
committed by GitHub
parent 12b95b59ab
commit ea59f8acb6
3 changed files with 227 additions and 12 deletions
+35
View File
@@ -104,6 +104,41 @@ at all is the struct's own validity, which Enterprise keeps in a separate
internal column when it expands the result into sibling columns, so a null
field is a value rather than a missing row.
### Timestamps and durations {#timestamps-durations}
Bare `datetime` keeps its existing `timestamp[us]` default without a timezone.
For any other unit or a timezone, declare the Arrow type explicitly:
```python
from datetime import datetime
from typing import Annotated
import pyarrow as pa
@function
def normalize(
ts: Annotated[datetime, pa.timestamp("ns", tz="UTC")],
) -> Annotated[datetime, pa.timestamp("ns", tz="UTC")]:
...
```
`pa.timestamp` accepts `s`, `ms`, `us`, and `ns`, and a timezone such as
`UTC`, `America/New_York`, or `+08:00`. The canonical signature uses
`{"type":"timestamp:<unit>:<timezone>"}`, with `-` for no timezone; the
microsecond/no-timezone case keeps the legacy `{"type":"timestamp"}` shape.
A timezone string of `-` is reserved for "no timezone" and is rejected.
`pa.duration` accepts the same four units and uses
`{"type":"duration:<unit>"}`; the microsecond case keeps the legacy
`{"type":"duration"}` shape. A bare `timedelta` has no unit and is rejected;
use `Annotated[timedelta, pa.duration("s")]` or an explicit schema.
Other units are never downgraded to microseconds. A reader that predates the
compound temporal representation cannot read those signatures. A deployment
must include the corresponding Lance Namespace and Sophon support; this
client-side declaration does not add server support.
## Connect and create a table {#table}
The remaining steps run against your Enterprise deployment. Replace the database
+41 -7
View File
@@ -29,7 +29,7 @@ import textwrap
import types
import warnings
from collections.abc import Mapping, Sequence
from datetime import date, datetime
from datetime import date, datetime, timedelta
from pathlib import Path
from typing import (
Annotated,
@@ -1043,6 +1043,7 @@ _GRAMMAR_PRIMITIVES = (
(pa.date32(), "date32"),
(pa.date64(), "date64"),
)
_FUNCTION_TIMEUNITS = ("s", "ms", "us", "ns")
def _canonical_arrow_type(
@@ -1259,6 +1260,26 @@ def _exact_arrow_type(
for candidate, name in _GRAMMAR_PRIMITIVES:
if data_type == candidate:
return {"type": name}
if pa.types.is_timestamp(data_type):
unit = data_type.unit
if unit not in _FUNCTION_TIMEUNITS:
raise TypeError(f"unsupported Function timestamp unit: {unit!r}")
timezone = data_type.tz
if timezone == "-":
raise TypeError(
"unsupported Function timestamp timezone: '-' is reserved for "
"no timezone"
)
if unit == "us" and timezone is None:
return {"type": "timestamp"}
return {"type": f"timestamp:{unit}:{timezone if timezone is not None else '-'}"}
if pa.types.is_duration(data_type):
unit = data_type.unit
if unit not in _FUNCTION_TIMEUNITS:
raise TypeError(f"unsupported Function duration unit: {unit!r}")
if unit == "us":
return {"type": "duration"}
return {"type": f"duration:{unit}"}
if pa.types.is_struct(data_type):
fields = list(data_type)
names = [field.name for field in fields]
@@ -1344,9 +1365,9 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
raise TypeError(
"Annotated Function types require exactly one PyArrow DataType"
)
# A struct value is written as a dict, which has no Arrow type of its
# own; the Annotated metadata supplies it.
if base is dict or get_origin(base) is dict:
# A struct value is written as a dict, and timedelta has no unit of
# its own; the Annotated metadata supplies those types.
if base is dict or get_origin(base) is dict or base is timedelta:
base_nullable = False
else:
_, base_nullable = _annotation_type(base)
@@ -1372,7 +1393,14 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
if annotation is date:
return pa.date32(), nullable
if annotation is datetime:
# Existing default: microsecond precision, no timezone. Other units
# and timezones require Annotated or an explicit schema.
return pa.timestamp("us"), nullable
if annotation is timedelta:
raise TypeError(
"timedelta Function annotations require an explicit unit; use "
"Annotated[timedelta, pyarrow.duration(unit)] or an explicit schema"
)
if get_origin(annotation) is list:
arguments = get_args(annotation)
if len(arguments) != 1:
@@ -2701,7 +2729,11 @@ def function(
deployment must accept nullable scalar outputs; an older server rejects
the registration. Every named-struct field may be nullable; Enterprise
preserves the struct's validity when the result is expanded into sibling
columns.
columns. Bare ``datetime`` keeps its existing ``timestamp[us]`` default
without a timezone; other units and timezones require
``Annotated[datetime, pyarrow.timestamp(unit, tz=...)]`` or an explicit
schema. A bare ``timedelta`` has no unit and is rejected; use
``Annotated[timedelta, pyarrow.duration(unit)]`` or an explicit schema.
**Large values.** A ``bytes`` parameter receives the whole value, which
is held in memory with its row and counts against the server's per-row
@@ -2808,8 +2840,10 @@ def function(
output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional
Explicit scalar or named-struct output. A scalar field's ``nullable``
flag is preserved; a top-level named-struct field and a Blob v2 output
must be non-nullable. Required for batch annotations. When omitted, the
output is inferred from the return annotation.
must be non-nullable. Temporal fields use canonical Arrow JSON; the
microsecond/no-timezone case keeps the legacy timestamp/duration
shape. Required for batch annotations. When omitted, the output is
inferred from the return annotation.
pip : sequence of str, optional
Pip requirements for the remote environment. Syntax is validated at
decoration time, including extras and environment markers. Named direct
@@ -9,7 +9,7 @@ import functools
import importlib.util
import inspect
import types
from datetime import date
from datetime import date, datetime, timedelta
import http.server
import json
import os
@@ -19,7 +19,7 @@ import sys
import typing
import urllib.parse
import warnings
from typing import Optional
from typing import Annotated, Optional
import pyarrow as pa
import pytest
@@ -884,12 +884,11 @@ def test_canonical_arrow_type_prefers_the_compact_grammar():
== "large_list<float32>"
)
for outside in [
pa.timestamp("us"),
pa.decimal128(10, 2),
pa.binary(4),
pa.duration("s"),
pa.time32("s"),
pa.time64("us"),
pa.list_(pa.float32(), 0),
pa.list_(pa.timestamp("us")),
]:
with pytest.raises(TypeError, match="unsupported Arrow type"):
_canonical_arrow_type(outside)
@@ -1215,6 +1214,153 @@ def test_explicit_arrow_schema_is_deterministic():
assert signature.output.nullable is False
def test_timestamp_annotation_and_explicit_schema_serialization():
for unit in ("s", "ms", "us", "ns"):
for timezone in (None, "UTC", "America/New_York", "+08:00", "-05:00"):
encoded = json.loads(_canonical_arrow_type(pa.timestamp(unit, tz=timezone)))
if unit == "us" and timezone is None:
assert encoded == {"type": "timestamp"}
else:
suffix = timezone if timezone is not None else "-"
assert encoded == {"type": f"timestamp:{unit}:{suffix}"}
with pytest.raises(TypeError, match="reserved for no timezone"):
_canonical_arrow_type(pa.timestamp("us", tz="-"))
@function
def normalize(
ts: Annotated[datetime, pa.timestamp("ns", tz="UTC")],
) -> Annotated[datetime, pa.timestamp("ns", tz="UTC")]:
return ts
signature = normalize.registration_request.signature
assert json.loads(signature.inputs[0].arrow_type) == {"type": "timestamp:ns:UTC"}
assert json.loads(signature.output.arrow_type) == {"type": "timestamp:ns:UTC"}
round_trip = FunctionSignature.from_json(signature.to_canonical_json())
assert round_trip.inputs[0].arrow_type == signature.inputs[0].arrow_type
assert round_trip.output.arrow_type == signature.output.arrow_type
@function
def optional_timestamp(
ts: Optional[Annotated[datetime, pa.timestamp("ms", tz="UTC")]],
) -> int:
return 0
optional_input = optional_timestamp.registration_request.signature.inputs[0]
assert optional_input.nullable is True
assert json.loads(optional_input.arrow_type) == {"type": "timestamp:ms:UTC"}
@function
def legacy(ts: datetime) -> datetime:
return ts
legacy_signature = legacy.registration_request.signature
assert json.loads(legacy_signature.inputs[0].arrow_type) == {"type": "timestamp"}
assert json.loads(legacy_signature.output.arrow_type) == {"type": "timestamp"}
legacy_round_trip = FunctionSignature.from_json(
legacy_signature.to_canonical_json()
)
assert legacy_round_trip.inputs[0].arrow_type == '{"type":"timestamp"}'
input_schema = pa.schema(
[pa.field("ts", pa.timestamp("ms", tz="America/New_York"), nullable=True)]
)
output_schema = pa.schema(
[
pa.field("ts", pa.timestamp("ns", tz="+08:00"), nullable=True),
pa.field("elapsed", pa.duration("ms"), nullable=False),
]
)
@function(input_schema=input_schema, output_schema=output_schema)
def nested(ts):
return {"ts": ts, "elapsed": None}
nested_signature = nested.registration_request.signature
assert nested_signature.inputs[0].nullable is True
assert json.loads(nested_signature.inputs[0].arrow_type) == {
"type": "timestamp:ms:America/New_York"
}
assert nested_signature.output.kind == "named_struct"
assert nested_signature.output.fields[0].nullable is True
assert json.loads(nested_signature.output.fields[0].arrow_type) == {
"type": "timestamp:ns:+08:00"
}
assert json.loads(nested_signature.output.fields[1].arrow_type) == {
"type": "duration:ms"
}
list_type = pa.list_(pa.field("item", pa.timestamp("ms", tz="UTC"), nullable=False))
assert json.loads(_canonical_arrow_type(list_type)) == {
"type": "list",
"fields": [
{
"name": "item",
"nullable": False,
"type": {"type": "timestamp:ms:UTC"},
}
],
}
def test_duration_annotation_and_explicit_schema_serialization():
for unit in ("s", "ms", "us", "ns"):
encoded = json.loads(_canonical_arrow_type(pa.duration(unit)))
expected = (
{"type": "duration"} if unit == "us" else {"type": f"duration:{unit}"}
)
assert encoded == expected
@function
def elapsed(
value: Annotated[timedelta, pa.duration("s")],
) -> Annotated[timedelta, pa.duration("ns")]:
return value
signature = elapsed.registration_request.signature
assert json.loads(signature.inputs[0].arrow_type) == {"type": "duration:s"}
assert json.loads(signature.output.arrow_type) == {"type": "duration:ns"}
round_trip = FunctionSignature.from_json(signature.to_canonical_json())
assert round_trip.inputs[0].arrow_type == signature.inputs[0].arrow_type
assert round_trip.output.arrow_type == signature.output.arrow_type
@function
def optional_elapsed(
value: Optional[Annotated[timedelta, pa.duration("ms")]],
) -> int:
return 0
optional_input = optional_elapsed.registration_request.signature.inputs[0]
assert optional_input.nullable is True
assert json.loads(optional_input.arrow_type) == {"type": "duration:ms"}
with pytest.raises(
TypeError, match="timedelta Function annotations require an explicit unit"
):
@function
def missing(value: timedelta) -> timedelta:
return value
input_schema = pa.schema([pa.field("elapsed", pa.duration("ms"), nullable=True)])
output_schema = pa.field("result", pa.duration("us"), nullable=False)
@function(input_schema=input_schema, output_schema=output_schema)
def explicit(elapsed: timedelta) -> timedelta:
return elapsed
explicit_signature = explicit.registration_request.signature
assert explicit_signature.inputs[0].nullable is True
assert json.loads(explicit_signature.inputs[0].arrow_type) == {
"type": "duration:ms"
}
assert json.loads(explicit_signature.output.arrow_type) == {"type": "duration"}
explicit_round_trip = FunctionSignature.from_json(
explicit_signature.to_canonical_json()
)
assert explicit_round_trip.output.arrow_type == '{"type":"duration"}'
def test_nullable_scalar_output_annotation_and_explicit_schema():
input_schema = pa.schema([pa.field("value", pa.int64(), nullable=False)])