mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat(python): preserve Function timestamp and duration types (#5032)
Function declarations currently reject exact timestamp and duration types. Preserve their Arrow units and timezones in canonical signature JSON, including fixed-offset timezones and nested fields. Bare `datetime` and microsecond/no-timezone declarations keep their existing representation; `timedelta` requires an explicit unit. Compound temporal signatures require the Lance Namespace decoder in https://github.com/lance-format/lance/pull/11059 and corresponding server validation support. Older readers reject these signatures; this change does not upgrade server support or dependency pins. Validation: 128 Python signature tests and 30 documentation tests passed (one skipped), using the changed Python source with an existing native extension. No Rust rebuild or end-to-end Function registration/refresh was performed.
This commit is contained in:
@@ -29,7 +29,7 @@ import textwrap
|
||||
import types
|
||||
import warnings
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import date, datetime
|
||||
from datetime import date, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
Annotated,
|
||||
@@ -1043,6 +1043,7 @@ _GRAMMAR_PRIMITIVES = (
|
||||
(pa.date32(), "date32"),
|
||||
(pa.date64(), "date64"),
|
||||
)
|
||||
_FUNCTION_TIMEUNITS = ("s", "ms", "us", "ns")
|
||||
|
||||
|
||||
def _canonical_arrow_type(
|
||||
@@ -1259,6 +1260,26 @@ def _exact_arrow_type(
|
||||
for candidate, name in _GRAMMAR_PRIMITIVES:
|
||||
if data_type == candidate:
|
||||
return {"type": name}
|
||||
if pa.types.is_timestamp(data_type):
|
||||
unit = data_type.unit
|
||||
if unit not in _FUNCTION_TIMEUNITS:
|
||||
raise TypeError(f"unsupported Function timestamp unit: {unit!r}")
|
||||
timezone = data_type.tz
|
||||
if timezone == "-":
|
||||
raise TypeError(
|
||||
"unsupported Function timestamp timezone: '-' is reserved for "
|
||||
"no timezone"
|
||||
)
|
||||
if unit == "us" and timezone is None:
|
||||
return {"type": "timestamp"}
|
||||
return {"type": f"timestamp:{unit}:{timezone if timezone is not None else '-'}"}
|
||||
if pa.types.is_duration(data_type):
|
||||
unit = data_type.unit
|
||||
if unit not in _FUNCTION_TIMEUNITS:
|
||||
raise TypeError(f"unsupported Function duration unit: {unit!r}")
|
||||
if unit == "us":
|
||||
return {"type": "duration"}
|
||||
return {"type": f"duration:{unit}"}
|
||||
if pa.types.is_struct(data_type):
|
||||
fields = list(data_type)
|
||||
names = [field.name for field in fields]
|
||||
@@ -1344,9 +1365,9 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
|
||||
raise TypeError(
|
||||
"Annotated Function types require exactly one PyArrow DataType"
|
||||
)
|
||||
# A struct value is written as a dict, which has no Arrow type of its
|
||||
# own; the Annotated metadata supplies it.
|
||||
if base is dict or get_origin(base) is dict:
|
||||
# A struct value is written as a dict, and timedelta has no unit of
|
||||
# its own; the Annotated metadata supplies those types.
|
||||
if base is dict or get_origin(base) is dict or base is timedelta:
|
||||
base_nullable = False
|
||||
else:
|
||||
_, base_nullable = _annotation_type(base)
|
||||
@@ -1372,7 +1393,14 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
|
||||
if annotation is date:
|
||||
return pa.date32(), nullable
|
||||
if annotation is datetime:
|
||||
# Existing default: microsecond precision, no timezone. Other units
|
||||
# and timezones require Annotated or an explicit schema.
|
||||
return pa.timestamp("us"), nullable
|
||||
if annotation is timedelta:
|
||||
raise TypeError(
|
||||
"timedelta Function annotations require an explicit unit; use "
|
||||
"Annotated[timedelta, pyarrow.duration(unit)] or an explicit schema"
|
||||
)
|
||||
if get_origin(annotation) is list:
|
||||
arguments = get_args(annotation)
|
||||
if len(arguments) != 1:
|
||||
@@ -2701,7 +2729,11 @@ def function(
|
||||
deployment must accept nullable scalar outputs; an older server rejects
|
||||
the registration. Every named-struct field may be nullable; Enterprise
|
||||
preserves the struct's validity when the result is expanded into sibling
|
||||
columns.
|
||||
columns. Bare ``datetime`` keeps its existing ``timestamp[us]`` default
|
||||
without a timezone; other units and timezones require
|
||||
``Annotated[datetime, pyarrow.timestamp(unit, tz=...)]`` or an explicit
|
||||
schema. A bare ``timedelta`` has no unit and is rejected; use
|
||||
``Annotated[timedelta, pyarrow.duration(unit)]`` or an explicit schema.
|
||||
|
||||
**Large values.** A ``bytes`` parameter receives the whole value, which
|
||||
is held in memory with its row and counts against the server's per-row
|
||||
@@ -2808,8 +2840,10 @@ def function(
|
||||
output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional
|
||||
Explicit scalar or named-struct output. A scalar field's ``nullable``
|
||||
flag is preserved; a top-level named-struct field and a Blob v2 output
|
||||
must be non-nullable. Required for batch annotations. When omitted, the
|
||||
output is inferred from the return annotation.
|
||||
must be non-nullable. Temporal fields use canonical Arrow JSON; the
|
||||
microsecond/no-timezone case keeps the legacy timestamp/duration
|
||||
shape. Required for batch annotations. When omitted, the output is
|
||||
inferred from the return annotation.
|
||||
pip : sequence of str, optional
|
||||
Pip requirements for the remote environment. Syntax is validated at
|
||||
decoration time, including extras and environment markers. Named direct
|
||||
|
||||
@@ -9,7 +9,7 @@ import functools
|
||||
import importlib.util
|
||||
import inspect
|
||||
import types
|
||||
from datetime import date
|
||||
from datetime import date, datetime, timedelta
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
@@ -19,7 +19,7 @@ import sys
|
||||
import typing
|
||||
import urllib.parse
|
||||
import warnings
|
||||
from typing import Optional
|
||||
from typing import Annotated, Optional
|
||||
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
@@ -884,12 +884,11 @@ def test_canonical_arrow_type_prefers_the_compact_grammar():
|
||||
== "large_list<float32>"
|
||||
)
|
||||
for outside in [
|
||||
pa.timestamp("us"),
|
||||
pa.decimal128(10, 2),
|
||||
pa.binary(4),
|
||||
pa.duration("s"),
|
||||
pa.time32("s"),
|
||||
pa.time64("us"),
|
||||
pa.list_(pa.float32(), 0),
|
||||
pa.list_(pa.timestamp("us")),
|
||||
]:
|
||||
with pytest.raises(TypeError, match="unsupported Arrow type"):
|
||||
_canonical_arrow_type(outside)
|
||||
@@ -1215,6 +1214,153 @@ def test_explicit_arrow_schema_is_deterministic():
|
||||
assert signature.output.nullable is False
|
||||
|
||||
|
||||
def test_timestamp_annotation_and_explicit_schema_serialization():
|
||||
for unit in ("s", "ms", "us", "ns"):
|
||||
for timezone in (None, "UTC", "America/New_York", "+08:00", "-05:00"):
|
||||
encoded = json.loads(_canonical_arrow_type(pa.timestamp(unit, tz=timezone)))
|
||||
if unit == "us" and timezone is None:
|
||||
assert encoded == {"type": "timestamp"}
|
||||
else:
|
||||
suffix = timezone if timezone is not None else "-"
|
||||
assert encoded == {"type": f"timestamp:{unit}:{suffix}"}
|
||||
|
||||
with pytest.raises(TypeError, match="reserved for no timezone"):
|
||||
_canonical_arrow_type(pa.timestamp("us", tz="-"))
|
||||
|
||||
@function
|
||||
def normalize(
|
||||
ts: Annotated[datetime, pa.timestamp("ns", tz="UTC")],
|
||||
) -> Annotated[datetime, pa.timestamp("ns", tz="UTC")]:
|
||||
return ts
|
||||
|
||||
signature = normalize.registration_request.signature
|
||||
assert json.loads(signature.inputs[0].arrow_type) == {"type": "timestamp:ns:UTC"}
|
||||
assert json.loads(signature.output.arrow_type) == {"type": "timestamp:ns:UTC"}
|
||||
round_trip = FunctionSignature.from_json(signature.to_canonical_json())
|
||||
assert round_trip.inputs[0].arrow_type == signature.inputs[0].arrow_type
|
||||
assert round_trip.output.arrow_type == signature.output.arrow_type
|
||||
|
||||
@function
|
||||
def optional_timestamp(
|
||||
ts: Optional[Annotated[datetime, pa.timestamp("ms", tz="UTC")]],
|
||||
) -> int:
|
||||
return 0
|
||||
|
||||
optional_input = optional_timestamp.registration_request.signature.inputs[0]
|
||||
assert optional_input.nullable is True
|
||||
assert json.loads(optional_input.arrow_type) == {"type": "timestamp:ms:UTC"}
|
||||
|
||||
@function
|
||||
def legacy(ts: datetime) -> datetime:
|
||||
return ts
|
||||
|
||||
legacy_signature = legacy.registration_request.signature
|
||||
assert json.loads(legacy_signature.inputs[0].arrow_type) == {"type": "timestamp"}
|
||||
assert json.loads(legacy_signature.output.arrow_type) == {"type": "timestamp"}
|
||||
legacy_round_trip = FunctionSignature.from_json(
|
||||
legacy_signature.to_canonical_json()
|
||||
)
|
||||
assert legacy_round_trip.inputs[0].arrow_type == '{"type":"timestamp"}'
|
||||
|
||||
input_schema = pa.schema(
|
||||
[pa.field("ts", pa.timestamp("ms", tz="America/New_York"), nullable=True)]
|
||||
)
|
||||
output_schema = pa.schema(
|
||||
[
|
||||
pa.field("ts", pa.timestamp("ns", tz="+08:00"), nullable=True),
|
||||
pa.field("elapsed", pa.duration("ms"), nullable=False),
|
||||
]
|
||||
)
|
||||
|
||||
@function(input_schema=input_schema, output_schema=output_schema)
|
||||
def nested(ts):
|
||||
return {"ts": ts, "elapsed": None}
|
||||
|
||||
nested_signature = nested.registration_request.signature
|
||||
assert nested_signature.inputs[0].nullable is True
|
||||
assert json.loads(nested_signature.inputs[0].arrow_type) == {
|
||||
"type": "timestamp:ms:America/New_York"
|
||||
}
|
||||
assert nested_signature.output.kind == "named_struct"
|
||||
assert nested_signature.output.fields[0].nullable is True
|
||||
assert json.loads(nested_signature.output.fields[0].arrow_type) == {
|
||||
"type": "timestamp:ns:+08:00"
|
||||
}
|
||||
assert json.loads(nested_signature.output.fields[1].arrow_type) == {
|
||||
"type": "duration:ms"
|
||||
}
|
||||
|
||||
list_type = pa.list_(pa.field("item", pa.timestamp("ms", tz="UTC"), nullable=False))
|
||||
assert json.loads(_canonical_arrow_type(list_type)) == {
|
||||
"type": "list",
|
||||
"fields": [
|
||||
{
|
||||
"name": "item",
|
||||
"nullable": False,
|
||||
"type": {"type": "timestamp:ms:UTC"},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_duration_annotation_and_explicit_schema_serialization():
|
||||
for unit in ("s", "ms", "us", "ns"):
|
||||
encoded = json.loads(_canonical_arrow_type(pa.duration(unit)))
|
||||
expected = (
|
||||
{"type": "duration"} if unit == "us" else {"type": f"duration:{unit}"}
|
||||
)
|
||||
assert encoded == expected
|
||||
|
||||
@function
|
||||
def elapsed(
|
||||
value: Annotated[timedelta, pa.duration("s")],
|
||||
) -> Annotated[timedelta, pa.duration("ns")]:
|
||||
return value
|
||||
|
||||
signature = elapsed.registration_request.signature
|
||||
assert json.loads(signature.inputs[0].arrow_type) == {"type": "duration:s"}
|
||||
assert json.loads(signature.output.arrow_type) == {"type": "duration:ns"}
|
||||
round_trip = FunctionSignature.from_json(signature.to_canonical_json())
|
||||
assert round_trip.inputs[0].arrow_type == signature.inputs[0].arrow_type
|
||||
assert round_trip.output.arrow_type == signature.output.arrow_type
|
||||
|
||||
@function
|
||||
def optional_elapsed(
|
||||
value: Optional[Annotated[timedelta, pa.duration("ms")]],
|
||||
) -> int:
|
||||
return 0
|
||||
|
||||
optional_input = optional_elapsed.registration_request.signature.inputs[0]
|
||||
assert optional_input.nullable is True
|
||||
assert json.loads(optional_input.arrow_type) == {"type": "duration:ms"}
|
||||
|
||||
with pytest.raises(
|
||||
TypeError, match="timedelta Function annotations require an explicit unit"
|
||||
):
|
||||
|
||||
@function
|
||||
def missing(value: timedelta) -> timedelta:
|
||||
return value
|
||||
|
||||
input_schema = pa.schema([pa.field("elapsed", pa.duration("ms"), nullable=True)])
|
||||
output_schema = pa.field("result", pa.duration("us"), nullable=False)
|
||||
|
||||
@function(input_schema=input_schema, output_schema=output_schema)
|
||||
def explicit(elapsed: timedelta) -> timedelta:
|
||||
return elapsed
|
||||
|
||||
explicit_signature = explicit.registration_request.signature
|
||||
assert explicit_signature.inputs[0].nullable is True
|
||||
assert json.loads(explicit_signature.inputs[0].arrow_type) == {
|
||||
"type": "duration:ms"
|
||||
}
|
||||
assert json.loads(explicit_signature.output.arrow_type) == {"type": "duration"}
|
||||
explicit_round_trip = FunctionSignature.from_json(
|
||||
explicit_signature.to_canonical_json()
|
||||
)
|
||||
assert explicit_round_trip.output.arrow_type == '{"type":"duration"}'
|
||||
|
||||
|
||||
def test_nullable_scalar_output_annotation_and_explicit_schema():
|
||||
input_schema = pa.schema([pa.field("value", pa.int64(), nullable=False)])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user