mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-28 22:46:14 +00:00
fix(python)!: reject tuple fields with mixed element types in pydantic models (#4679)
`pydantic_to_schema` maps a `Tuple[...]` field to an Arrow list of its
**first** element type. A tuple with different element types gets the
wrong schema, and the error only surfaces when a row is written:
```python
class Pair(LanceModel):
pair: Tuple[int, str]
pydantic_to_schema(Pair) # pair: list<item: int64>
db.create_table("t", schema=Pair).add([Pair(pair=(1, "a"))])
# pyarrow.lib.ArrowInvalid: Could not convert 'a' with type str: tried to convert to int64
```
An Arrow list has a single element type, so such a tuple cannot be
stored as one. This raises a `TypeError` when the schema is built,
saying the tuple elements must share one type and pointing to a nested
`LanceModel` for a fixed set of differently typed fields. Homogeneous
tuples are unchanged: `Tuple[int, int]` and `Tuple[float, ...]` still
map to lists.
**Breaking change:** a model with a mixed-type tuple field, which used
to get a (wrong) `list<...>` schema from `pydantic_to_schema`, now
raises `TypeError` when the schema is built. Such a schema could not
hold the model's values: writing a row failed with `ArrowInvalid`. The
requirement and the nested-`LanceModel` alternative are documented in
the `pydantic_to_schema` docstring.
Tests (in `test_pydantic.py`): `TypeError` for `Tuple[int, str]` (fails
on `main`), and the list mapping for `Tuple[int, int]` and `Tuple[float,
...]`. The whole file passes; `ruff format` and `ruff check` are clean.
I used an AI assistant (Claude) while working on this; I reviewed and
tested the change.
---------
Co-authored-by: Xuanwo <github@xuanwo.io>
This commit is contained in:
@@ -259,6 +259,15 @@ def _py_type_to_arrow_type(py_type: Type[Any], field: FieldInfo) -> pa.DataType:
|
||||
f"{py_type}. Specify the element type, e.g. List[int] instead "
|
||||
"of a bare List."
|
||||
)
|
||||
# An Arrow list has a single element type, so a tuple such as
|
||||
# Tuple[int, str] cannot be stored as one. Tuple[int, ...] is homogeneous.
|
||||
element_types = {arg for arg in args if arg is not Ellipsis}
|
||||
if len(element_types) > 1:
|
||||
raise TypeError(
|
||||
"Converting Pydantic type to Arrow Type: unsupported type "
|
||||
f"{py_type}. All tuple elements must have the same type; use a "
|
||||
"nested LanceModel for a fixed set of fields with different types."
|
||||
)
|
||||
child = args[0]
|
||||
return _pydantic_list_child_to_arrow(child, field)
|
||||
raise TypeError(
|
||||
@@ -419,6 +428,15 @@ def pydantic_to_schema(model: Type[pydantic.BaseModel]) -> pa.Schema:
|
||||
pyarrow.Schema
|
||||
The Arrow Schema
|
||||
|
||||
Raises
|
||||
------
|
||||
TypeError
|
||||
If a field has a type with no Arrow equivalent. A ``Tuple`` field becomes
|
||||
an Arrow list, so all its elements must have the same type, as in
|
||||
``Tuple[int, int]`` or ``Tuple[float, ...]``; for a fixed set of fields
|
||||
with different types, such as ``Tuple[int, str]``, use a nested
|
||||
``LanceModel`` (an Arrow struct) instead.
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
|
||||
@@ -200,6 +200,27 @@ def test_bare_generic_raises_type_error():
|
||||
pydantic_to_schema(TestModel)
|
||||
|
||||
|
||||
def test_tuple_with_mixed_element_types_raises_type_error():
|
||||
# An Arrow list has one element type, so Tuple[int, str] has no Arrow
|
||||
# equivalent. It used to map to list<int64> and fail later, on write.
|
||||
class TestModel(pydantic.BaseModel):
|
||||
pair: Tuple[int, str]
|
||||
|
||||
with pytest.raises(TypeError, match="same type"):
|
||||
pydantic_to_schema(TestModel)
|
||||
|
||||
|
||||
def test_homogeneous_tuples_map_to_lists():
|
||||
class TestModel(pydantic.BaseModel):
|
||||
pair: Tuple[int, int]
|
||||
many: Tuple[float, ...]
|
||||
|
||||
schema = pydantic_to_schema(TestModel)
|
||||
|
||||
assert schema.field("pair").type == pa.list_(pa.int64())
|
||||
assert schema.field("many").type == pa.list_(pa.float64())
|
||||
|
||||
|
||||
def test_nested_struct_list_optional():
|
||||
class SplitInfo(pydantic.BaseModel):
|
||||
start_frame: int
|
||||
|
||||
Reference in New Issue
Block a user