mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-14 15:52:26 +00:00
fix(python): preserve mixed blob values before Arrow conversion (#4304)
## Cause Python list and pandas rows were converted to Arrow before the target blob v2 schema was applied. In a mixed bytes and URI batch, Arrow inferred a binary column and encoded the URI as inline bytes; dict and `lance.Blob` values could fail conversion. ## Fix Build blob v2 columns from the original Python values using the target field's storage type before converting the rest of each row. This preserves inline bytes, external URIs, descriptor dictionaries, blob objects, and nulls for `add`, `merge_insert`, and `create_table`. ## Validation - `python/tests/test_blob.py`: 76 passed, 1 skipped. - Python project Ruff check passed. - Repository-wide Ruff check still reports 29 existing findings in `ci/` files outside this change. Fixes #4271 <!-- lance-gatekeeper-fix:v1 agent=54c95f06f11c88187b07a430aac6be2f generation=1 --> Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
This commit is contained in:
@@ -7,10 +7,11 @@ import sys
|
||||
import textwrap
|
||||
|
||||
import lance
|
||||
import pandas as pd
|
||||
import pyarrow as pa
|
||||
import pyarrow.compute as pc
|
||||
import pytest
|
||||
from lance.blob import BlobType as LanceBlobType
|
||||
from lance.blob import Blob, BlobType as LanceBlobType
|
||||
|
||||
import lancedb
|
||||
from lancedb._blob import (
|
||||
@@ -1432,3 +1433,129 @@ def test_add_external_uri_string_round_trips_with_flag(tmp_path):
|
||||
hits = table.search().to_arrow()
|
||||
blobs = table.fetch_blobs("image", hits)
|
||||
assert blobs[0].as_py() == payload
|
||||
|
||||
|
||||
@pytest.mark.parametrize("as_pandas", [False, True])
|
||||
def test_add_bytes_and_uri_in_one_batch_preserves_external_uri(tmp_path, as_pandas):
|
||||
payload = b"external-payload"
|
||||
blob_path = tmp_path / "payload.bin"
|
||||
blob_path.write_bytes(payload)
|
||||
rows = [
|
||||
{"id": 1, "image": b"inline"},
|
||||
{"id": 2, "image": blob_path.as_uri()},
|
||||
]
|
||||
db = lancedb.connect(tmp_path / "db")
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
table = db.create_table("mixed", schema=schema)
|
||||
|
||||
table.add(
|
||||
pd.DataFrame(rows) if as_pandas else rows,
|
||||
allow_external_blob_outside_bases=True,
|
||||
)
|
||||
|
||||
by_id = _row_ids_by_id(table)
|
||||
assert table.fetch_blobs("image", [by_id[1], by_id[2]]).to_pylist() == [
|
||||
b"inline",
|
||||
payload,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("as_pandas", [False, True])
|
||||
def test_add_mixed_python_blob_values_preserves_external_uris(tmp_path, as_pandas):
|
||||
payload = b"external-payload"
|
||||
blob_path = tmp_path / "payload.bin"
|
||||
blob_path.write_bytes(payload)
|
||||
uri = blob_path.as_uri()
|
||||
rows = [
|
||||
{"id": 1, "image": b"inline"},
|
||||
{"id": 2, "image": uri},
|
||||
{"id": 3, "image": {"data": b"from-dict"}},
|
||||
{"id": 4, "image": Blob.from_bytes(b"from-blob")},
|
||||
{"id": 5, "image": Blob.from_uri(uri)},
|
||||
{"id": 6, "image": {"uri": uri}},
|
||||
{"id": 7, "image": None},
|
||||
]
|
||||
db = lancedb.connect(tmp_path / "db")
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
table = db.create_table("mixed", schema=schema)
|
||||
|
||||
table.add(
|
||||
pd.DataFrame(rows) if as_pandas else rows,
|
||||
allow_external_blob_outside_bases=True,
|
||||
)
|
||||
|
||||
by_id = _row_ids_by_id(table)
|
||||
assert table.fetch_blobs("image", [by_id[i] for i in range(1, 8)]).to_pylist() == [
|
||||
b"inline",
|
||||
payload,
|
||||
b"from-dict",
|
||||
b"from-blob",
|
||||
payload,
|
||||
payload,
|
||||
None,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("as_pandas", [False, True])
|
||||
def test_merge_insert_mixed_python_blobs_validates_external_uri(tmp_path, as_pandas):
|
||||
blob_path = tmp_path / "payload.bin"
|
||||
blob_path.write_bytes(b"external-payload")
|
||||
table = _blob_table("merge_mixed_uri", [{"id": 1, "image": b"before"}])
|
||||
rows = [
|
||||
{"id": 1, "image": b"updated"},
|
||||
{"id": 2, "image": blob_path.as_uri()},
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError, match="allow_external_blob_outside_bases"):
|
||||
(
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute(pd.DataFrame(rows) if as_pandas else rows)
|
||||
)
|
||||
assert table.count_rows() == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("as_pandas", [False, True])
|
||||
def test_merge_insert_python_blob_dicts_and_objects(as_pandas):
|
||||
table = _blob_table("merge_python_blobs", [{"id": 1, "image": b"before"}])
|
||||
rows = [
|
||||
{"id": 1, "image": b"updated"},
|
||||
{"id": 2, "image": {"data": b"from-dict"}},
|
||||
{"id": 3, "image": Blob.from_bytes(b"from-blob")},
|
||||
]
|
||||
|
||||
result = (
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute(pd.DataFrame(rows) if as_pandas else rows)
|
||||
)
|
||||
|
||||
assert result.num_updated_rows == 1
|
||||
assert result.num_inserted_rows == 2
|
||||
by_id = _row_ids_by_id(table)
|
||||
assert table.fetch_blobs("image", [by_id[i] for i in range(1, 4)]).to_pylist() == [
|
||||
b"updated",
|
||||
b"from-dict",
|
||||
b"from-blob",
|
||||
]
|
||||
|
||||
|
||||
def test_create_table_with_python_blob_dict_and_object():
|
||||
db = lancedb.connect("memory:///")
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
table = db.create_table(
|
||||
"create_python_blobs",
|
||||
data=[
|
||||
{"id": 1, "image": {"data": b"from-dict"}},
|
||||
{"id": 2, "image": Blob.from_bytes(b"from-blob")},
|
||||
],
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
by_id = _row_ids_by_id(table)
|
||||
assert table.fetch_blobs("image", [by_id[1], by_id[2]]).to_pylist() == [
|
||||
b"from-dict",
|
||||
b"from-blob",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user