diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 52582395f..db8919ddc 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -229,7 +229,8 @@ jobs: # Make sure wheels are not included in the Rust cache - name: Delete wheels run: rm -rf target/wheels - pydantic1x: + min-deps: + name: "Minimum dependencies" timeout-minutes: 60 runs-on: "ubuntu-24.04" defaults: @@ -259,8 +260,7 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install lancedb run: | - pip install "pydantic<2" - pip install pyarrow==16 + pip install "pydantic==2.7.4" "pyarrow==16" pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests] - name: Run tests run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests diff --git a/docs/requirements.txt b/docs/requirements.txt index e5f3867cb..89de1cb71 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,5 +5,5 @@ mkdocs-autorefs>=0.5,<=1.0 mkdocstrings[python]>=0.24,<1.0 griffe>=0.40,<1.0 mkdocs-render-swagger-plugin>=0.1.0 -pydantic>=2.0,<3.0 -mkdocs-redirects>=1.2.0 \ No newline at end of file +pydantic>=2.7.4,<3 +mkdocs-redirects>=1.2.0 diff --git a/python/pyproject.toml b/python/pyproject.toml index ce71484de..ae42172c0 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ "overrides>=0.7; python_version<'3.12'", "packaging>=23.0", "pyarrow>=16", - "pydantic>=1.10", + "pydantic>=2.7.4,<3", "tqdm>=4.27.0", "lance-namespace>=0.3.2" ] diff --git a/python/python/lancedb/embeddings/base.py b/python/python/lancedb/embeddings/base.py index f711e5b7d..149b9c089 100644 --- a/python/python/lancedb/embeddings/base.py +++ b/python/python/lancedb/embeddings/base.py @@ -26,7 +26,6 @@ class EmbeddingFunction(BaseModel, ABC): 3. ndims() which returns the number of dimensions of the vector column """ - __slots__ = ("__weakref__",) # pydantic 1.x compatibility max_retries: int = ( 7 # Setting 0 disables retires. Maybe this should not be enabled by default, ) diff --git a/python/python/lancedb/embeddings/bedrock.py b/python/python/lancedb/embeddings/bedrock.py index dc2badceb..dc93a2da9 100644 --- a/python/python/lancedb/embeddings/bedrock.py +++ b/python/python/lancedb/embeddings/bedrock.py @@ -7,8 +7,7 @@ from functools import cached_property from typing import List, Union import numpy as np - -from lancedb.pydantic import PYDANTIC_VERSION +from pydantic import ConfigDict from ..util import attempt_import_or_raise from .base import TextEmbeddingFunction @@ -67,13 +66,7 @@ class BedRockText(TextEmbeddingFunction): source_input_type: str = "search_document" query_input_type: str = "search_query" - if PYDANTIC_VERSION.major < 2: # Pydantic 1.x compat - - class Config: - keep_untouched = (cached_property,) - else: - model_config = dict() - model_config["ignored_types"] = (cached_property,) + model_config = ConfigDict(ignored_types=(cached_property,)) def ndims(self): # return len(self._generate_embedding("test")) diff --git a/python/python/lancedb/embeddings/gemini_text.py b/python/python/lancedb/embeddings/gemini_text.py index 32f2d4d04..d3bf79af4 100644 --- a/python/python/lancedb/embeddings/gemini_text.py +++ b/python/python/lancedb/embeddings/gemini_text.py @@ -7,8 +7,7 @@ from functools import cached_property from typing import List, Optional, Union import numpy as np - -from lancedb.pydantic import PYDANTIC_VERSION +from pydantic import ConfigDict from ..util import attempt_import_or_raise from .base import TextEmbeddingFunction @@ -87,13 +86,7 @@ class GeminiText(TextEmbeddingFunction): query_task_type: str = "retrieval_query" source_task_type: str = "retrieval_document" - if PYDANTIC_VERSION.major < 2: # Pydantic 1.x compat - - class Config: - keep_untouched = (cached_property,) - else: - model_config = dict() - model_config["ignored_types"] = (cached_property,) + model_config = ConfigDict(ignored_types=(cached_property,)) def ndims(self): if self.dim: diff --git a/python/python/lancedb/embeddings/imagebind.py b/python/python/lancedb/embeddings/imagebind.py index 84bb0a123..c8051f14d 100644 --- a/python/python/lancedb/embeddings/imagebind.py +++ b/python/python/lancedb/embeddings/imagebind.py @@ -7,14 +7,13 @@ from typing import List, Union import numpy as np import pyarrow as pa +from pydantic import ConfigDict from ..util import attempt_import_or_raise from .base import EmbeddingFunction from .registry import register from .utils import AUDIO, IMAGES, TEXT -from lancedb.pydantic import PYDANTIC_VERSION - @register("imagebind") class ImageBindEmbeddings(EmbeddingFunction): @@ -31,13 +30,7 @@ class ImageBindEmbeddings(EmbeddingFunction): device: str = "cpu" normalize: bool = False - if PYDANTIC_VERSION.major < 2: # Pydantic 1.x compat - - class Config: - keep_untouched = (cached_property,) - else: - model_config = dict() - model_config["ignored_types"] = (cached_property,) + model_config = ConfigDict(ignored_types=(cached_property,)) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/python/python/lancedb/embeddings/transformers.py b/python/python/lancedb/embeddings/transformers.py index c8a65b9ca..3f42edfdd 100644 --- a/python/python/lancedb/embeddings/transformers.py +++ b/python/python/lancedb/embeddings/transformers.py @@ -7,8 +7,7 @@ from typing import List, Any import numpy as np -from pydantic import PrivateAttr -from lancedb.pydantic import PYDANTIC_VERSION +from pydantic import ConfigDict, PrivateAttr from ..util import attempt_import_or_raise from .base import EmbeddingFunction @@ -59,13 +58,7 @@ class TransformersEmbeddingFunction(EmbeddingFunction): ) self._model.to(self.device) - if PYDANTIC_VERSION.major < 2: # Pydantic 1.x compat - - class Config: - keep_untouched = (cached_property,) - else: - model_config = dict() - model_config["ignored_types"] = (cached_property,) + model_config = ConfigDict(ignored_types=(cached_property,)) def ndims(self): self._ndims = self._model.config.hidden_size diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 4ffb65e2c..9c4b063cd 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -13,14 +13,14 @@ import json from collections.abc import Mapping from typing import Any, Optional -import pydantic -from pydantic import BaseModel, Field, conint - -_PYDANTIC_V2 = int(pydantic.VERSION.split(".", 1)[0]) >= 2 -if _PYDANTIC_V2: - from pydantic import field_validator, model_validator -else: - from pydantic import root_validator, validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + conint, + field_validator, + model_validator, +) _Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1) _UInt32 = conint(strict=True, ge=0, le=2**32 - 1) @@ -82,43 +82,25 @@ def _known_wire_value(value): class _RemoteValue(BaseModel): - if _PYDANTIC_V2: - model_config = {"extra": "ignore", "frozen": True} - else: + model_config = ConfigDict(extra="ignore", frozen=True) - class Config: - allow_mutation = False - extra = "ignore" - - if _PYDANTIC_V2: - - @model_validator(mode="after") - def _freeze_mappings(self): - for name, value in self.__dict__.items(): - object.__setattr__(self, name, _freeze_value(value)) - return self - - else: - - @root_validator - def _freeze_mappings(cls, values): - return {name: _freeze_value(value) for name, value in values.items()} + @model_validator(mode="after") + def _freeze_mappings(self): + for name, value in self.__dict__.items(): + object.__setattr__(self, name, _freeze_value(value)) + return self @classmethod def from_json(cls, payload: str): - if _PYDANTIC_V2: - return cls.model_validate_json(payload) - return cls.parse_raw(payload) + return cls.model_validate_json(payload) def _known_dict(self) -> dict[str, Any]: - fields = self.__class__.model_fields if _PYDANTIC_V2 else self.__fields__ known = {} - for name, field in fields.items(): + for name, field in self.__class__.model_fields.items(): value = getattr(self, name) if value is None: continue - required = field.is_required() if _PYDANTIC_V2 else field.required - if not required: + if not field.is_required(): default_factory = field.default_factory if default_factory is not None and value == default_factory(): continue @@ -129,9 +111,7 @@ class _RemoteValue(BaseModel): def _copy(self, *, update: Mapping[str, Any]): update = {name: _freeze_value(value) for name, value in update.items()} - if _PYDANTIC_V2: - return self.model_copy(update=update) - return self.copy(update=update) + return self.model_copy(update=update) def to_canonical_json(self) -> str: return json.dumps( @@ -199,35 +179,18 @@ class PythonRuntimeSpec(_RemoteValue): environment: Optional[PythonEnvironmentSpec] = None env: Optional[Mapping[str, str]] = None - if _PYDANTIC_V2: - - @model_validator(mode="after") - def _validate_runtime_kind(self): - if self.kind == "python": - if self.python_version is None: - raise ValueError("python runtime requires python_version") - if self.environment is None: - raise ValueError("python runtime requires environment") - else: - object.__setattr__(self, "python_version", None) - object.__setattr__(self, "environment", None) - object.__setattr__(self, "env", None) - return self - - else: - - @root_validator - def _validate_runtime_kind(cls, values): - if values.get("kind") == "python": - if values.get("python_version") is None: - raise ValueError("python runtime requires python_version") - if values.get("environment") is None: - raise ValueError("python runtime requires environment") - else: - values["python_version"] = None - values["environment"] = None - values["env"] = None - return values + @model_validator(mode="after") + def _validate_runtime_kind(self): + if self.kind == "python": + if self.python_version is None: + raise ValueError("python runtime requires python_version") + if self.environment is None: + raise ValueError("python runtime requires environment") + else: + object.__setattr__(self, "python_version", None) + object.__setattr__(self, "environment", None) + object.__setattr__(self, "env", None) + return self class FunctionVersion(_RemoteValue): @@ -265,18 +228,10 @@ class ApplicationInput(_RemoteValue): kind: str value: Any - if _PYDANTIC_V2: - - @field_validator("value") - @classmethod - def _validate_value(cls, value): - return _validate_literal(value) - - else: - - @validator("value") - def _validate_value(cls, value): - return _validate_literal(value) + @field_validator("value") + @classmethod + def _validate_value(cls, value): + return _validate_literal(value) class FunctionApplication(_RemoteValue): diff --git a/python/python/lancedb/pydantic.py b/python/python/lancedb/pydantic.py index 1ab6e6fcc..528d865d7 100644 --- a/python/python/lancedb/pydantic.py +++ b/python/python/lancedb/pydantic.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors -"""Pydantic (v1 / v2) adapter for LanceDB""" +"""Pydantic adapter for LanceDB.""" from __future__ import annotations @@ -14,9 +14,6 @@ from enum import Enum from typing import ( TYPE_CHECKING, Any, - Callable, - Dict, - Generator, List, Type, Union, @@ -24,17 +21,9 @@ from typing import ( GenericAlias, ) -import numpy as np import pyarrow as pa import pydantic -from packaging.version import Version - -PYDANTIC_VERSION = Version(pydantic.__version__) -try: - from pydantic_core import CoreSchema, core_schema -except ImportError: - if PYDANTIC_VERSION.major >= 2: - raise +from pydantic_core import CoreSchema, core_schema if TYPE_CHECKING: from pydantic.fields import FieldInfo @@ -131,25 +120,6 @@ def Vector( ), ) - @classmethod - def __get_validators__(cls) -> Generator[Callable, None, None]: - yield cls.validate - - # For pydantic v1 - @classmethod - def validate(cls, v): - if not isinstance(v, (list, range, np.ndarray)) or len(v) != dim: - raise TypeError("A list of numbers or numpy.ndarray is needed") - return cls(v) - - if PYDANTIC_VERSION.major < 2: - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]): - field_schema["items"] = {"type": "number"} - field_schema["maxItems"] = dim - field_schema["minItems"] = dim - return FixedSizeList @@ -157,9 +127,8 @@ def _raise_bare_vector_error(*_args): raise TypeError("Vector must be parameterized with a dimension, e.g. Vector(128).") -# Pydantic v1 and v2 otherwise treat the bare Vector factory as a field validator -# and inspect its signature, which produces misleading errors about internal types. -setattr(Vector, "__get_validators__", _raise_bare_vector_error) +# Pydantic otherwise inspects the bare factory as a field type and produces +# misleading errors about its internal annotations. setattr(Vector, "__get_pydantic_core_schema__", _raise_bare_vector_error) @@ -233,31 +202,6 @@ def MultiVector( ), ) - @classmethod - def __get_validators__(cls) -> Generator[Callable, None, None]: - yield cls.validate - - # For pydantic v1 - @classmethod - def validate(cls, v): - if not isinstance(v, (list, range)): - raise TypeError("A list of vectors is needed") - for vec in v: - if not isinstance(vec, (list, range, np.ndarray)) or len(vec) != dim: - raise TypeError(f"Each vector must be a list of {dim} numbers") - return cls(v) - - if PYDANTIC_VERSION.major < 2: - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]): - field_schema["items"] = { - "type": "array", - "items": {"type": "number"}, - "minItems": dim, - "maxItems": dim, - } - return MultiVectorList @@ -303,20 +247,10 @@ def _py_type_to_arrow_type(py_type: Type[Any], field: FieldInfo) -> pa.DataType: ) -if PYDANTIC_VERSION.major < 2: - - def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]: - return [ - _pydantic_to_field(name, field) for name, field in model.__fields__.items() - ] - -else: - - def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]: - return [ - _pydantic_to_field(name, field) - for name, field in model.model_fields.items() - ] +def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]: + return [ + _pydantic_to_field(name, field) for name, field in model.model_fields.items() + ] def _pydantic_type_to_arrow_type(tp: Any, field: FieldInfo) -> pa.DataType: @@ -509,8 +443,6 @@ class LanceModel(pydantic.BaseModel): @classmethod def safe_get_fields(cls): - if PYDANTIC_VERSION.major < 2: - return cls.__fields__ return cls.model_fields @classmethod @@ -548,23 +480,9 @@ def get_extras(field_info: FieldInfo, key: str) -> Any: """ Get the extra metadata from a Pydantic FieldInfo. """ - if PYDANTIC_VERSION.major >= 2: - return (field_info.json_schema_extra or {}).get(key) - return (field_info.field_info.extra or {}).get("json_schema_extra", {}).get(key) + return (field_info.json_schema_extra or {}).get(key) -if PYDANTIC_VERSION.major < 2: - - def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]: - """ - Convert a Pydantic model to a dictionary. - """ - return model.dict() - -else: - - def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]: - """ - Convert a Pydantic model to a dictionary. - """ - return model.model_dump() +def model_to_dict(model: pydantic.BaseModel) -> dict[str, Any]: + """Convert a Pydantic model to a dictionary.""" + return model.model_dump() diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index e2bb491ea..bd5e2cea2 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -32,8 +32,6 @@ from typing_extensions import Annotated from lancedb._lancedb import fts_query_to_json from lancedb.background_loop import LOOP -from lancedb.pydantic import PYDANTIC_VERSION - from . import __version__ from .arrow import AsyncRecordBatchReader from .dependencies import pandas as pd @@ -827,12 +825,7 @@ class Query(pydantic.BaseModel): # This tells pydantic to allow custom types (needed for the `vector` query since # pa.Array wouln't be allowed otherwise) - if PYDANTIC_VERSION.major < 2: # Pydantic 1.x compat - - class Config: - arbitrary_types_allowed = True - else: - model_config = {"arbitrary_types_allowed": True} + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) class LanceQueryBuilder(ABC): @@ -3251,12 +3244,7 @@ class AsyncStandardQuery(AsyncQueryBase): if ordering is None: self._inner.order_by(None) else: - self._inner.order_by( - [ - o.model_dump() if hasattr(o, "model_dump") else o.dict() - for o in ordering - ] - ) + self._inner.order_by([o.model_dump() for o in ordering]) return self def fast_search(self) -> Self: diff --git a/python/python/tests/test_pydantic.py b/python/python/tests/test_pydantic.py index db93d7c64..ca03940f2 100644 --- a/python/python/tests/test_pydantic.py +++ b/python/python/tests/test_pydantic.py @@ -10,11 +10,10 @@ import pyarrow as pa import pydantic import pytest from lancedb.pydantic import ( - PYDANTIC_VERSION, LanceModel, + MultiVector, Vector, pydantic_to_schema, - MultiVector, ) from pydantic import BaseModel from pydantic import Field @@ -432,16 +431,10 @@ def test_fixed_size_list_field(): li: List[int] data = TestModel(vec=list(range(16)), li=[1, 2, 3]) - if PYDANTIC_VERSION.major >= 2: - assert json.loads(data.model_dump_json()) == { - "vec": list(range(16)), - "li": [1, 2, 3], - } - else: - assert data.dict() == { - "vec": list(range(16)), - "li": [1, 2, 3], - } + assert json.loads(data.model_dump_json()) == { + "vec": list(range(16)), + "li": [1, 2, 3], + } schema = pydantic_to_schema(TestModel) assert schema == pa.schema( @@ -451,10 +444,7 @@ def test_fixed_size_list_field(): ] ) - if PYDANTIC_VERSION.major >= 2: - json_schema = TestModel.model_json_schema() - else: - json_schema = TestModel.schema() + json_schema = TestModel.model_json_schema() assert json_schema == { "properties": { diff --git a/python/uv.lock b/python/uv.lock index 2cdcb182e..c957a4c06 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -2003,7 +2003,7 @@ requires-dist = [ { name = "pyarrow", specifier = ">=16" }, { name = "pyarrow", marker = "extra == 'tests'", specifier = "<25" }, { name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" }, - { name = "pydantic", specifier = ">=1.10" }, + { name = "pydantic", specifier = ">=2.7.4,<3" }, { name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" }, { name = "pylance", marker = "extra == 'tests'", specifier = "==9.0.0rc1" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },