Improve pydantic integration (#384)

This commit is contained in:
Chang She
2023-07-31 12:16:44 -04:00
committed by GitHub
parent 2d25c263e9
commit cada35d5b7
9 changed files with 108 additions and 7 deletions

View File

@@ -27,12 +27,17 @@ from lance import LanceDataset
from lance.vector import vec_to_table
from .common import DATA, VEC, VECTOR_COLUMN_NAME
from .pydantic import LanceModel
from .query import LanceFtsQueryBuilder, LanceQueryBuilder, Query
from .util import fs_from_uri
def _sanitize_data(data, schema, on_bad_vectors, fill_value):
if isinstance(data, list):
# convert to list of dict if data is a bunch of LanceModels
if isinstance(data[0], LanceModel):
schema = data[0].__class__.to_arrow_schema()
data = [dict(d) for d in data]
data = pa.Table.from_pylist(data)
data = _sanitize_schema(
data, schema=schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value