rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-08-22 05:58:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(python): require pydantic v2 (#3990)

Browse Source 
LanceDB's Python SDK now requires Pydantic `>=2.7.4,<3` and uses the v2
APIs throughout. This removes dual-version behavior from schema
conversion, query serialization, embedding models, and Function wire
models while preserving their existing public and canonical-wire
behavior.

The minimum-dependencies CI job pins Pydantic 2.7.4 so the declared
compatibility floor remains covered.
This commit is contained in:
Xuanwo
2026-08-21 16:50:00 +08:00
committed by GitHub
parent 09843410ec
commit 4ba2421254
13 changed files with 69 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/pyproject.toml
+1 -1
View File
@@ -8,7 +8,7 @@ dependencies = [
"overrides>=0.7; python_version<'3.12'",
"packaging>=23.0",
"pyarrow>=16",
"pydantic>=1.10",
"pydantic>=2.7.4,<3",
"tqdm>=4.27.0",
"lance-namespace>=0.3.2"
]
python/python/lancedb/embeddings/base.py
-1
View File
@@ -26,7 +26,6 @@ class EmbeddingFunction(BaseModel, ABC):
3. ndims() which returns the number of dimensions of the vector column
"""
__slots__ = ("__weakref__",) # pydantic 1.x compatibility
max_retries: int = (
7 # Setting 0 disables retires. Maybe this should not be enabled by default,
)
python/python/lancedb/embeddings/bedrock.py
+2 -9
View File
@@ -7,8 +7,7 @@ from functools import cached_property
from typing import List, Union
import numpy as np
from lancedb.pydantic import PYDANTIC_VERSION
from pydantic import ConfigDict
from ..util import attempt_import_or_raise
from .base import TextEmbeddingFunction
@@ -67,13 +66,7 @@ class BedRockText(TextEmbeddingFunction):
source_input_type: str = "search_document"
query_input_type: str = "search_query"
if PYDANTIC_VERSION.major < 2: # Pydantic 1.x compat
class Config:
keep_untouched = (cached_property,)
else:
model_config = dict()
model_config["ignored_types"] = (cached_property,)
model_config = ConfigDict(ignored_types=(cached_property,))
def ndims(self):
# return len(self._generate_embedding("test"))
python/python/lancedb/embeddings/gemini_text.py
+2 -9
View File
@@ -7,8 +7,7 @@ from functools import cached_property
from typing import List, Optional, Union
import numpy as np
from lancedb.pydantic import PYDANTIC_VERSION
from pydantic import ConfigDict
from ..util import attempt_import_or_raise
from .base import TextEmbeddingFunction
@@ -87,13 +86,7 @@ class GeminiText(TextEmbeddingFunction):
query_task_type: str = "retrieval_query"
source_task_type: str = "retrieval_document"
if PYDANTIC_VERSION.major < 2: # Pydantic 1.x compat
class Config:
keep_untouched = (cached_property,)
else:
model_config = dict()
model_config["ignored_types"] = (cached_property,)
model_config = ConfigDict(ignored_types=(cached_property,))
def ndims(self):
if self.dim:
python/python/lancedb/embeddings/imagebind.py
+2 -9
View File
@@ -7,14 +7,13 @@ from typing import List, Union
import numpy as np
import pyarrow as pa
from pydantic import ConfigDict
from ..util import attempt_import_or_raise
from .base import EmbeddingFunction
from .registry import register
from .utils import AUDIO, IMAGES, TEXT
from lancedb.pydantic import PYDANTIC_VERSION
@register("imagebind")
class ImageBindEmbeddings(EmbeddingFunction):
@@ -31,13 +30,7 @@ class ImageBindEmbeddings(EmbeddingFunction):
device: str = "cpu"
normalize: bool = False
if PYDANTIC_VERSION.major < 2: # Pydantic 1.x compat
class Config:
keep_untouched = (cached_property,)
else:
model_config = dict()
model_config["ignored_types"] = (cached_property,)
model_config = ConfigDict(ignored_types=(cached_property,))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
python/python/lancedb/embeddings/transformers.py
+2 -9
View File
@@ -7,8 +7,7 @@ from typing import List, Any
import numpy as np
from pydantic import PrivateAttr
from lancedb.pydantic import PYDANTIC_VERSION
from pydantic import ConfigDict, PrivateAttr
from ..util import attempt_import_or_raise
from .base import EmbeddingFunction
@@ -59,13 +58,7 @@ class TransformersEmbeddingFunction(EmbeddingFunction):
)
self._model.to(self.device)
if PYDANTIC_VERSION.major < 2: # Pydantic 1.x compat
class Config:
keep_untouched = (cached_property,)
else:
model_config = dict()
model_config["ignored_types"] = (cached_property,)
model_config = ConfigDict(ignored_types=(cached_property,))
def ndims(self):
self._ndims = self._model.config.hidden_size
python/python/lancedb/functions.py
+34 -79
View File
@@ -13,14 +13,14 @@ import json
from collections.abc import Mapping
from typing import Any, Optional
import pydantic
from pydantic import BaseModel, Field, conint
_PYDANTIC_V2 = int(pydantic.VERSION.split(".", 1)[0]) >= 2
if _PYDANTIC_V2:
from pydantic import field_validator, model_validator
else:
from pydantic import root_validator, validator
from pydantic import (
BaseModel,
ConfigDict,
Field,
conint,
field_validator,
model_validator,
)
_Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
_UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
@@ -82,43 +82,25 @@ def _known_wire_value(value):
class _RemoteValue(BaseModel):
if _PYDANTIC_V2:
model_config = {"extra": "ignore", "frozen": True}
else:
model_config = ConfigDict(extra="ignore", frozen=True)
class Config:
allow_mutation = False
extra = "ignore"
if _PYDANTIC_V2:
@model_validator(mode="after")
def _freeze_mappings(self):
for name, value in self.__dict__.items():
object.__setattr__(self, name, _freeze_value(value))
return self
else:
@root_validator
def _freeze_mappings(cls, values):
return {name: _freeze_value(value) for name, value in values.items()}
@model_validator(mode="after")
def _freeze_mappings(self):
for name, value in self.__dict__.items():
object.__setattr__(self, name, _freeze_value(value))
return self
@classmethod
def from_json(cls, payload: str):
if _PYDANTIC_V2:
return cls.model_validate_json(payload)
return cls.parse_raw(payload)
return cls.model_validate_json(payload)
def _known_dict(self) -> dict[str, Any]:
fields = self.__class__.model_fields if _PYDANTIC_V2 else self.__fields__
known = {}
for name, field in fields.items():
for name, field in self.__class__.model_fields.items():
value = getattr(self, name)
if value is None:
continue
required = field.is_required() if _PYDANTIC_V2 else field.required
if not required:
if not field.is_required():
default_factory = field.default_factory
if default_factory is not None and value == default_factory():
continue
@@ -129,9 +111,7 @@ class _RemoteValue(BaseModel):
def _copy(self, *, update: Mapping[str, Any]):
update = {name: _freeze_value(value) for name, value in update.items()}
if _PYDANTIC_V2:
return self.model_copy(update=update)
return self.copy(update=update)
return self.model_copy(update=update)
def to_canonical_json(self) -> str:
return json.dumps(
@@ -199,35 +179,18 @@ class PythonRuntimeSpec(_RemoteValue):
environment: Optional[PythonEnvironmentSpec] = None
env: Optional[Mapping[str, str]] = None
if _PYDANTIC_V2:
@model_validator(mode="after")
def _validate_runtime_kind(self):
if self.kind == "python":
if self.python_version is None:
raise ValueError("python runtime requires python_version")
if self.environment is None:
raise ValueError("python runtime requires environment")
else:
object.__setattr__(self, "python_version", None)
object.__setattr__(self, "environment", None)
object.__setattr__(self, "env", None)
return self
else:
@root_validator
def _validate_runtime_kind(cls, values):
if values.get("kind") == "python":
if values.get("python_version") is None:
raise ValueError("python runtime requires python_version")
if values.get("environment") is None:
raise ValueError("python runtime requires environment")
else:
values["python_version"] = None
values["environment"] = None
values["env"] = None
return values
@model_validator(mode="after")
def _validate_runtime_kind(self):
if self.kind == "python":
if self.python_version is None:
raise ValueError("python runtime requires python_version")
if self.environment is None:
raise ValueError("python runtime requires environment")
else:
object.__setattr__(self, "python_version", None)
object.__setattr__(self, "environment", None)
object.__setattr__(self, "env", None)
return self
class FunctionVersion(_RemoteValue):
@@ -265,18 +228,10 @@ class ApplicationInput(_RemoteValue):
kind: str
value: Any
if _PYDANTIC_V2:
@field_validator("value")
@classmethod
def _validate_value(cls, value):
return _validate_literal(value)
else:
@validator("value")
def _validate_value(cls, value):
return _validate_literal(value)
@field_validator("value")
@classmethod
def _validate_value(cls, value):
return _validate_literal(value)
class FunctionApplication(_RemoteValue):
python/python/lancedb/pydantic.py
+12 -94
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Pydantic (v1 / v2) adapter for LanceDB"""
"""Pydantic adapter for LanceDB."""
from __future__ import annotations
@@ -14,9 +14,6 @@ from enum import Enum
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
Generator,
List,
Type,
Union,
@@ -24,17 +21,9 @@ from typing import (
GenericAlias,
)
import numpy as np
import pyarrow as pa
import pydantic
from packaging.version import Version
PYDANTIC_VERSION = Version(pydantic.__version__)
try:
from pydantic_core import CoreSchema, core_schema
except ImportError:
if PYDANTIC_VERSION.major >= 2:
raise
from pydantic_core import CoreSchema, core_schema
if TYPE_CHECKING:
from pydantic.fields import FieldInfo
@@ -131,25 +120,6 @@ def Vector(
),
)
@classmethod
def __get_validators__(cls) -> Generator[Callable, None, None]:
yield cls.validate
# For pydantic v1
@classmethod
def validate(cls, v):
if not isinstance(v, (list, range, np.ndarray)) or len(v) != dim:
raise TypeError("A list of numbers or numpy.ndarray is needed")
return cls(v)
if PYDANTIC_VERSION.major < 2:
@classmethod
def __modify_schema__(cls, field_schema: Dict[str, Any]):
field_schema["items"] = {"type": "number"}
field_schema["maxItems"] = dim
field_schema["minItems"] = dim
return FixedSizeList
@@ -157,9 +127,8 @@ def _raise_bare_vector_error(*_args):
raise TypeError("Vector must be parameterized with a dimension, e.g. Vector(128).")
# Pydantic v1 and v2 otherwise treat the bare Vector factory as a field validator
# and inspect its signature, which produces misleading errors about internal types.
setattr(Vector, "__get_validators__", _raise_bare_vector_error)
# Pydantic otherwise inspects the bare factory as a field type and produces
# misleading errors about its internal annotations.
setattr(Vector, "__get_pydantic_core_schema__", _raise_bare_vector_error)
@@ -233,31 +202,6 @@ def MultiVector(
),
)
@classmethod
def __get_validators__(cls) -> Generator[Callable, None, None]:
yield cls.validate
# For pydantic v1
@classmethod
def validate(cls, v):
if not isinstance(v, (list, range)):
raise TypeError("A list of vectors is needed")
for vec in v:
if not isinstance(vec, (list, range, np.ndarray)) or len(vec) != dim:
raise TypeError(f"Each vector must be a list of {dim} numbers")
return cls(v)
if PYDANTIC_VERSION.major < 2:
@classmethod
def __modify_schema__(cls, field_schema: Dict[str, Any]):
field_schema["items"] = {
"type": "array",
"items": {"type": "number"},
"minItems": dim,
"maxItems": dim,
}
return MultiVectorList
@@ -303,20 +247,10 @@ def _py_type_to_arrow_type(py_type: Type[Any], field: FieldInfo) -> pa.DataType:
)
if PYDANTIC_VERSION.major < 2:
def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
return [
_pydantic_to_field(name, field) for name, field in model.__fields__.items()
]
else:
def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
return [
_pydantic_to_field(name, field)
for name, field in model.model_fields.items()
]
def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
return [
_pydantic_to_field(name, field) for name, field in model.model_fields.items()
]
def _pydantic_type_to_arrow_type(tp: Any, field: FieldInfo) -> pa.DataType:
@@ -509,8 +443,6 @@ class LanceModel(pydantic.BaseModel):
@classmethod
def safe_get_fields(cls):
if PYDANTIC_VERSION.major < 2:
return cls.__fields__
return cls.model_fields
@classmethod
@@ -548,23 +480,9 @@ def get_extras(field_info: FieldInfo, key: str) -> Any:
"""
Get the extra metadata from a Pydantic FieldInfo.
"""
if PYDANTIC_VERSION.major >= 2:
return (field_info.json_schema_extra or {}).get(key)
return (field_info.field_info.extra or {}).get("json_schema_extra", {}).get(key)
return (field_info.json_schema_extra or {}).get(key)
if PYDANTIC_VERSION.major < 2:
def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
"""
Convert a Pydantic model to a dictionary.
"""
return model.dict()
else:
def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
"""
Convert a Pydantic model to a dictionary.
"""
return model.model_dump()
def model_to_dict(model: pydantic.BaseModel) -> dict[str, Any]:
"""Convert a Pydantic model to a dictionary."""
return model.model_dump()
python/python/lancedb/query.py
+2 -14
View File
@@ -32,8 +32,6 @@ from typing_extensions import Annotated
from lancedb._lancedb import fts_query_to_json
from lancedb.background_loop import LOOP
from lancedb.pydantic import PYDANTIC_VERSION
from . import __version__
from .arrow import AsyncRecordBatchReader
from .dependencies import pandas as pd
@@ -827,12 +825,7 @@ class Query(pydantic.BaseModel):
# This tells pydantic to allow custom types (needed for the `vector` query since
# pa.Array wouln't be allowed otherwise)
if PYDANTIC_VERSION.major < 2: # Pydantic 1.x compat
class Config:
arbitrary_types_allowed = True
else:
model_config = {"arbitrary_types_allowed": True}
model_config = pydantic.ConfigDict(arbitrary_types_allowed=True)
class LanceQueryBuilder(ABC):
@@ -3251,12 +3244,7 @@ class AsyncStandardQuery(AsyncQueryBase):
if ordering is None:
self._inner.order_by(None)
else:
self._inner.order_by(
[
o.model_dump() if hasattr(o, "model_dump") else o.dict()
for o in ordering
]
)
self._inner.order_by([o.model_dump() for o in ordering])
return self
def fast_search(self) -> Self:
python/python/tests/test_pydantic.py
+6 -16
View File
@@ -10,11 +10,10 @@ import pyarrow as pa
import pydantic
import pytest
from lancedb.pydantic import (
PYDANTIC_VERSION,
LanceModel,
MultiVector,
Vector,
pydantic_to_schema,
MultiVector,
)
from pydantic import BaseModel
from pydantic import Field
@@ -432,16 +431,10 @@ def test_fixed_size_list_field():
li: List[int]
data = TestModel(vec=list(range(16)), li=[1, 2, 3])
if PYDANTIC_VERSION.major >= 2:
assert json.loads(data.model_dump_json()) == {
"vec": list(range(16)),
"li": [1, 2, 3],
}
else:
assert data.dict() == {
"vec": list(range(16)),
"li": [1, 2, 3],
}
assert json.loads(data.model_dump_json()) == {
"vec": list(range(16)),
"li": [1, 2, 3],
}
schema = pydantic_to_schema(TestModel)
assert schema == pa.schema(
@@ -451,10 +444,7 @@ def test_fixed_size_list_field():
]
)
if PYDANTIC_VERSION.major >= 2:
json_schema = TestModel.model_json_schema()
else:
json_schema = TestModel.schema()
json_schema = TestModel.model_json_schema()
assert json_schema == {
"properties": {
python/uv.lock
Generated
+1 -1
View File
@@ -2003,7 +2003,7 @@ requires-dist = [
{ name = "pyarrow", specifier = ">=16" },
{ name = "pyarrow", marker = "extra == 'tests'", specifier = "<25" },
{ name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" },
{ name = "pydantic", specifier = ">=1.10" },
{ name = "pydantic", specifier = ">=2.7.4,<3" },
{ name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==9.0.0rc1" },
{ name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },