mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-23 12:12:27 +00:00
fix(python): preserve metadata during vector schema inference (#4451)
Creating a table from an Arrow `list` or `large_list` vector field infers a fixed-size vector type, but currently drops that field's metadata. Descriptions and tags such as `lancedb:description` and `lancedb:tag:model` disappear even though the input field carries them. Preserve the original field metadata while replacing its inferred type. Tests use public synchronous and asynchronous table creation, reopening, and reads for floating-point and integer inference, checking the complete schema and row values. Existing dtype inference and nullability remain unchanged. Validation on macOS arm64 with an official native extension built from main `d864e719` using Rust 1.97.0: - Python 3.11.14 and 3.13.11: full `pytest -m "not slow and not s3_test" -q -rs python/tests`: **1,533 passed, 37 skipped, 68 deselected** each, wall times 331.464 s and 416.308 s. - Added public SDK cases on unchanged main: **12 failed**; after the fix: **12 passed** on each runtime. Final installed distributions passed **8 public SDK round trips** each, including fixed-size fields and fields without metadata. - Documentation tests on Python 3.11.14: **77 passed, 6 skipped**. Ruff lint/format, Pyright (**0 errors**), applicable pre-commit hooks (**4 passed, 2 without matching files**), compile checks and `git diff --check` passed. - Comparable unchanged-main full baselines from the same day, native extension and dependency versions: **1,521 passed, 37 skipped, 68 deselected** on each runtime. These were reused rather than rerun. The final local wheel is an explicitly verified repack of the official main wheel, replacing only `table.py` and `RECORD`; all production Python files match the tested source and the native extension is unchanged. No candidate Rust rebuild was performed. Slow/S3 integration tests, the remaining OS/runtime and minimum-dependency matrices, and the workflow's Linux/Python 3.13 doctest job were not run locally. The existing Rust migration draft #2970 and issues #3216/#3222 do not fix this metadata loss in the current Python path; a future migration should preserve the same metadata. During final verification main advanced to `81d52a98` through #4373 and the beta.13 version bump. Neither modified the two files here; the local native and full-suite validation above remain explicitly based on `d864e719`, and the updated native changes have not been tested locally. AI assistance: developed and verified with **gpt-6.1-sol**.
This commit is contained in:
@@ -5404,6 +5404,7 @@ def _infer_target_schema(
|
||||
field.name, # preserve original field name
|
||||
target_type,
|
||||
nullable=field.nullable,
|
||||
metadata=field.metadata,
|
||||
)
|
||||
|
||||
schema = schema.set(i, new_field)
|
||||
|
||||
@@ -4507,6 +4507,58 @@ async def test_alter_columns_rejects_invalid_async(
|
||||
assert await table.schema() == initial_schema
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("list_type", [pa.list_, pa.large_list])
|
||||
@pytest.mark.parametrize(
|
||||
"value_type, values, inferred_type",
|
||||
[
|
||||
(pa.float64(), [1.25, 2.5], pa.float32()),
|
||||
(pa.int32(), [0, 255], pa.uint8()),
|
||||
(pa.int32(), [-1, 256], pa.float32()),
|
||||
],
|
||||
)
|
||||
async def test_inferred_vector_field_metadata(
|
||||
tmp_db: DBConnection,
|
||||
tmp_db_async: AsyncConnection,
|
||||
asynchronous,
|
||||
list_type,
|
||||
value_type,
|
||||
values,
|
||||
inferred_type,
|
||||
):
|
||||
metadata = {
|
||||
"lancedb:description": "Embedding values",
|
||||
"lancedb:tag:model": "test-model",
|
||||
}
|
||||
schema = pa.schema(
|
||||
[
|
||||
pa.field(
|
||||
"vector", list_type(value_type), nullable=False, metadata=metadata
|
||||
),
|
||||
pa.field("text", pa.string(), metadata={"description": "Source text"}),
|
||||
],
|
||||
metadata={"description": "Source dataset"},
|
||||
)
|
||||
rows = [{"vector": values, "text": "first"}, {"vector": values, "text": "second"}]
|
||||
data = pa.Table.from_pylist(rows, schema=schema)
|
||||
if asynchronous:
|
||||
await tmp_db_async.create_table("vector_metadata", data=data)
|
||||
table = await tmp_db_async.open_table("vector_metadata")
|
||||
actual_schema = await table.schema()
|
||||
actual_data = await table.to_arrow()
|
||||
else:
|
||||
tmp_db.create_table("vector_metadata", data=data)
|
||||
table = tmp_db.open_table("vector_metadata")
|
||||
actual_schema = table.schema
|
||||
actual_data = table.to_arrow()
|
||||
expected_schema = schema.set(
|
||||
0, schema.field("vector").with_type(pa.list_(inferred_type, 2))
|
||||
)
|
||||
assert actual_schema.equals(expected_schema, check_metadata=True)
|
||||
assert actual_data.to_pylist() == rows
|
||||
|
||||
|
||||
def test_update_field_metadata(mem_db: DBConnection):
|
||||
data = pa.table({"id": [0, 1], "category": ["a", "b"]})
|
||||
table = mem_db.create_table("my_table", data=data)
|
||||
|
||||
Reference in New Issue
Block a user