diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 3cb996a15..5f359f4b7 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -223,9 +223,13 @@ tokens = list( Blob columns store large binary values out of line so they can be read lazily instead of being materialized with the rest of the row. -::: lancedb.blob +`lancedb.BlobType` is `lance.blob.BlobType` when pylance is installed. Without +pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns +still work. Queries return descriptors. Call +[`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or +[`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes. -::: lancedb.BlobType +::: lancedb.blob ::: lancedb._blob.BlobFile options: diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 8cb85a3ed..21ffc8860 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -6,7 +6,7 @@ import importlib.metadata import os from concurrent.futures import ThreadPoolExecutor from datetime import timedelta -from typing import Dict, Optional, Union, Any, List, Iterable +from typing import Dict, Optional, Union, Any, List, Iterable, TYPE_CHECKING __version__ = importlib.metadata.version("lancedb") @@ -20,7 +20,7 @@ from .db import AsyncConnection, DBConnection, LanceDBConnection from .remote import ClientConfig from .remote.db import RemoteDBConnection from .expr import Expr, col, lit, func -from .schema import blob, vector, BlobType +from .schema import blob, vector from .job import AsyncJob, Job from .functions import ( FunctionArtifactRequest as FunctionArtifactRequest, @@ -49,6 +49,19 @@ from .namespace import ( ) +if TYPE_CHECKING: + from lance.blob import BlobType as BlobType + + +def __getattr__(name: str): + if name == "BlobType": + from .schema import BlobType + + globals()["BlobType"] = BlobType + return BlobType + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + def _check_s3_bucket_with_dots( uri: str, storage_options: Optional[Dict[str, str]] ) -> None: diff --git a/python/python/lancedb/_blob.py b/python/python/lancedb/_blob.py index 5b4c0c343..dc4ed37df 100644 --- a/python/python/lancedb/_blob.py +++ b/python/python/lancedb/_blob.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Optional, Union import pyarrow as pa from .expr import Expr -from .schema import blob_v2_column_paths +from .schema import row_addressable_blob_v2_paths from .types import BlobMode, QueryProjection, QueryProjectionSpec if TYPE_CHECKING: @@ -119,7 +119,7 @@ def blob_v2_projection_sources( schema: pa.Schema, projection: QueryProjection, ) -> dict[str, str]: - blob_columns = blob_v2_column_paths(schema) + blob_columns = row_addressable_blob_v2_paths(schema) if not blob_columns: return {} columns = set(blob_columns) @@ -140,7 +140,9 @@ def v2_projection_needs_row_id( ) -> bool: if with_row_id: return False - return projection_includes_blob_column(projection, blob_v2_column_paths(schema)) + return projection_includes_blob_column( + projection, row_addressable_blob_v2_paths(schema) + ) def blob_auto_row_id_for_scan( diff --git a/python/python/lancedb/schema.py b/python/python/lancedb/schema.py index 33adbbae3..4dce09f0f 100644 --- a/python/python/lancedb/schema.py +++ b/python/python/lancedb/schema.py @@ -4,30 +4,34 @@ """Schema helpers for Lance blob columns.""" +import importlib +from typing import TYPE_CHECKING + import pyarrow as pa +import pyarrow.ipc + +if TYPE_CHECKING: + from lance.blob import BlobType as BlobType _BLOB_EXTENSION_NAME = "lance.blob.v2" _BLOB_V1_KEY = "lance-encoding:blob" _ARROW_EXT_NAME_KEY = "ARROW:extension:name" +_BLOB_V2_STORAGE_TYPE = pa.struct( + [ + pa.field("data", pa.large_binary(), nullable=True), + pa.field("uri", pa.utf8(), nullable=True), + pa.field("position", pa.uint64(), nullable=True), + pa.field("size", pa.uint64(), nullable=True), + ] +) +_resolved_blob_type = None -class BlobType(pa.ExtensionType): - """PyArrow extension type for a Lance blob v2 column. - - Queries return descriptors; call :meth:`~lancedb.table.Table.fetch_blob_files` - for lazy reads or :meth:`~lancedb.table.Table.fetch_blobs` for eager bytes. - """ +class _FallbackBlobType(pa.ExtensionType): + """lance.blob.v2 extension type used when pylance is not installed.""" def __init__(self) -> None: - storage_type = pa.struct( - [ - pa.field("data", pa.large_binary(), nullable=True), - pa.field("uri", pa.utf8(), nullable=True), - pa.field("position", pa.uint64(), nullable=True), - pa.field("size", pa.uint64(), nullable=True), - ] - ) - super().__init__(storage_type, _BLOB_EXTENSION_NAME) + pa.ExtensionType.__init__(self, _BLOB_V2_STORAGE_TYPE, _BLOB_EXTENSION_NAME) def __arrow_ext_serialize__(self) -> bytes: return b"" @@ -35,23 +39,16 @@ class BlobType(pa.ExtensionType): @classmethod def __arrow_ext_deserialize__( cls, storage_type: pa.DataType, serialized: bytes - ) -> "BlobType": + ) -> "_FallbackBlobType": return cls() def __reduce__(self): - # Ensure pickle round-trips on older pyarrow (apache/arrow#35599). return type(self).__arrow_ext_deserialize__, ( self.storage_type, self.__arrow_ext_serialize__(), ) -try: - pa.register_extension_type(BlobType()) # type: ignore[arg-type] -except pa.ArrowKeyError: - pass - - def _metadata_value(metadata: dict, key: str): return metadata.get(key.encode()) or metadata.get(key) @@ -92,43 +89,105 @@ def is_blob_like_field(field: pa.Field) -> bool: return is_blob_v2_field(field) or _metadata_marks_legacy_blob(field.metadata or {}) -def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[str]: - paths: list[str] = [] +def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[tuple[str, bool]]: + """Walk the schema and return (path, has_list_ancestor) for each blob field.""" + paths: list[tuple[str, bool]] = [] - def walk(fields, prefix: str) -> None: + def walk(fields, prefix: str, has_list_ancestor: bool) -> None: for field in fields: path = f"{prefix}.{field.name}" if prefix else field.name if is_blob(field): - paths.append(path) + paths.append((path, has_list_ancestor)) elif pa.types.is_struct(field.type): - walk(field.type, path) + walk(field.type, path, has_list_ancestor) elif ( pa.types.is_list(field.type) or pa.types.is_large_list(field.type) or pa.types.is_fixed_size_list(field.type) ): - walk([field.type.value_field], path) + walk([field.type.value_field], path, True) - walk(schema, "") + walk(schema, "", False) return paths def blob_column_paths(schema: pa.Schema) -> list[str]: """Dotted paths of blob-like columns (v2 extension or legacy metadata).""" - return _collect_blob_paths(schema, is_blob_like_field) + return [path for path, _ in _collect_blob_paths(schema, is_blob_like_field)] def blob_v2_column_paths(schema: pa.Schema) -> list[str]: - return _collect_blob_paths(schema, is_blob_v2_field) + return [path for path, _ in _collect_blob_paths(schema, is_blob_v2_field)] + + +def row_addressable_blob_v2_paths(schema: pa.Schema) -> list[str]: + """Blob v2 paths with one blob addressable by table row id. + + ``fetch_blobs`` and the descriptor row-id ride-along address one blob per + row, so a blob inside a list container has no row-id slot and no fetch + path. Those columns still store and query as raw descriptors. + """ + return [ + path + for path, has_list_ancestor in _collect_blob_paths(schema, is_blob_v2_field) + if not has_list_ancestor + ] def schema_has_blob_field(schema: pa.Schema) -> bool: return bool(blob_column_paths(schema)) +def _deserialize_registered_type(extension_type: pa.ExtensionType) -> pa.DataType: + """Return the type Arrow reconstructs for this extension name.""" + schema = pa.schema([pa.field("value", extension_type)]) + restored = pa.ipc.read_schema(schema.serialize()) + return restored.field("value").type + + +def _resolve_blob_type(): + """Return the BlobType class this process should use. + + pylance's class when it owns the lance.blob.v2 registry entry, + otherwise LanceDB's fallback. A different registered class is an error. + """ + global _resolved_blob_type + if _resolved_blob_type is not None: + return _resolved_blob_type + try: + blob_module = importlib.import_module("lance.blob") + except ModuleNotFoundError as err: + if err.name not in ("lance", "lance.blob"): + raise + else: + blob_type = getattr(blob_module, "BlobType", None) + if blob_type is not None: + registered_type = _deserialize_registered_type(blob_type()) + if type(registered_type) is not blob_type: + registered_cls = type(registered_type) + raise ValueError( + "lance.blob.v2 is already registered by " + f"{registered_cls.__module__}.{registered_cls.__qualname__}" + ) + _resolved_blob_type = blob_type + return blob_type + try: + pa.register_extension_type(_FallbackBlobType()) # type: ignore[arg-type] + except pa.ArrowKeyError as err: + raise ValueError( + "lance.blob.v2 is already registered by another extension class" + ) from err + _resolved_blob_type = _FallbackBlobType + return _resolved_blob_type + + def blob(name: str, nullable: bool = True) -> pa.Field: - """Create a Lance blob v2 column field.""" - return pa.field(name, BlobType(), nullable=nullable) + """Create a Lance blob v2 column field. + + When pylance is installed this is ``lance.blob.BlobType``. + """ + blob_type = _resolve_blob_type() + return pa.field(name, blob_type(), nullable=nullable) def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType: @@ -155,3 +214,11 @@ def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataTyp ... ]) """ return pa.list_(value_type, dimension) + + +def __getattr__(name: str): + if name == "BlobType": + blob_type = _resolve_blob_type() + globals()["BlobType"] = blob_type + return blob_type + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 765b7fa14..535ed7c0d 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -104,7 +104,12 @@ from .util import ( value_to_sql, ) from .index import lang_mapping -from .schema import blob_v2_column_paths, schema_has_blob_field +from .schema import ( + blob_v2_column_paths, + is_blob_v2_field, + row_addressable_blob_v2_paths, + schema_has_blob_field, +) def _should_push_down_query_table( @@ -426,6 +431,7 @@ def _cast_to_target_schema( def gen(): for batch in reader: + batch = _coerce_blob_write_columns(batch, reordered_schema) # Table but not RecordBatch has cast. cast_batches = ( pa.Table.from_batches([batch]).cast(reordered_schema).to_batches() @@ -438,6 +444,166 @@ def _cast_to_target_schema( return pa.RecordBatchReader.from_batches(reordered_schema, gen()) +def _coerce_blob_write_columns( + batch: pa.RecordBatch, target_schema: pa.Schema +) -> pa.RecordBatch: + """Materialize blob storage structs before the stream leaves Python. + + merge_insert requires its source reader to already match the table's + physical schema. Unlike add and insert, it does not pass through + LanceDB's Rust blob coercion, so preserving binary input here would + reach Lance as binary and fail the schema check. + """ + columns = [] + fields = [] + changed = False + for field, column in zip(batch.schema, batch.columns): + target_field = target_schema.field(field.name) + coerced = _coerce_blob_value(column, target_field) + if coerced is not column: + column = coerced + field = pa.field( + field.name, + coerced.type, + field.nullable, + target_field.metadata, + ) + changed = True + columns.append(column) + fields.append(field) + if not changed: + return batch + return pa.RecordBatch.from_arrays( + columns, schema=pa.schema(fields, metadata=batch.schema.metadata) + ) + + +def _coerce_blob_value(column: pa.Array, target_field: pa.Field) -> pa.Array: + if is_blob_v2_field(target_field) and _can_coerce_to_blob(column.type): + return _coerce_value_to_blob(column, target_field) + + target_type = target_field.type + if pa.types.is_struct(target_type) and pa.types.is_struct(column.type): + children = [] + fields = [] + changed = False + for source_field in column.type: + source_column = column.field(source_field.name) + nested_target = next( + (field for field in target_type if field.name == source_field.name), + None, + ) + if nested_target is None: + children.append(source_column) + fields.append(source_field) + continue + coerced = _coerce_blob_value(source_column, nested_target) + if coerced is not source_column: + changed = True + child_array, child_type = _physical_array_and_type(coerced) + children.append(child_array) + fields.append( + pa.field( + source_field.name, + child_type, + source_field.nullable, + nested_target.metadata, + ) + ) + if not changed: + return column + return pa.StructArray.from_arrays( + children, + fields=fields, + mask=column.is_null() if column.null_count else None, + ) + + if _is_list_like(target_type) and _is_list_like(column.type): + return _coerce_blob_list_values(column, target_type.value_field) + + return column + + +def _coerce_blob_list_values( + column: pa.Array, target_value_field: pa.Field +) -> pa.Array: + """Coerce blob values inside a list column, preserving offsets and nulls. + + Works on the raw child values window instead of ``pc.list_flatten`` because + flatten drops values spanned by null slots, which would misalign offsets. + """ + mask = column.is_null() if column.null_count else None + if pa.types.is_fixed_size_list(column.type): + list_size = column.type.list_size + values = column.values.slice(column.offset * list_size, len(column) * list_size) + coerced = _coerce_blob_value(values, target_value_field) + if coerced is values: + return column + physical_values, _ = _physical_array_and_type(coerced) + return pa.FixedSizeListArray.from_arrays(physical_values, list_size, mask=mask) + offsets = column.offsets + first_offset = offsets[0].as_py() + values = column.values.slice( + first_offset, + offsets[-1].as_py() - first_offset, + ) + coerced = _coerce_blob_value(values, target_value_field) + if coerced is values: + return column + physical_values, _ = _physical_array_and_type(coerced) + if first_offset: + offsets = pc.subtract(offsets, pa.scalar(first_offset, offsets.type)) + if pa.types.is_large_list(column.type): + return pa.LargeListArray.from_arrays(offsets, physical_values, mask=mask) + return pa.ListArray.from_arrays(offsets, physical_values, mask=mask) + + +def _coerce_value_to_blob(values: pa.Array, target_field: pa.Field) -> pa.Array: + if pa.types.is_null(values.type): + data = pa.nulls(len(values), type=pa.large_binary()) + elif pa.types.is_large_binary(values.type): + data = values + else: + data = values.cast(pa.large_binary()) + length = len(values) + storage_type = target_field.type + if isinstance(storage_type, pa.ExtensionType): + storage_type = storage_type.storage_type + storage_fields = list(storage_type) + children = [] + for storage_field in storage_fields: + if storage_field.name == "data": + children.append(data) + else: + children.append(pa.nulls(length, type=storage_field.type)) + storage = pa.StructArray.from_arrays( + children, + fields=storage_fields, + mask=values.is_null() if values.null_count else None, + ) + if isinstance(target_field.type, pa.ExtensionType): + return pa.ExtensionArray.from_storage(target_field.type, storage) + return storage + + +def _physical_array_and_type(array: pa.Array) -> tuple[pa.Array, pa.DataType]: + if isinstance(array.type, pa.ExtensionType): + return array.storage, array.type.storage_type + return array, array.type + + +def _can_coerce_to_blob(data_type: pa.DataType) -> bool: + return _is_binary_like(data_type) or pa.types.is_null(data_type) + + +def _is_binary_like(data_type: pa.DataType) -> bool: + return ( + pa.types.is_binary(data_type) + or pa.types.is_large_binary(data_type) + or pa.types.is_binary_view(data_type) + ) + + def _field_extension_name(field: pa.Field) -> Optional[str]: extension_name = getattr(field.type, "extension_name", None) if extension_name is not None: @@ -464,63 +630,71 @@ def _align_field_types( target_field = next((f for f in target_fields if f.name == field.name), None) if target_field is None: raise ValueError(f"Field '{field.name}' not found in target schema") - # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored - # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the - # input to that storage type here merely relabels the raw JSON bytes as - # JSONB. Lance must see arrow.json so it can perform the JSONB encoding. - if ( - _field_extension_name(field) == "arrow.json" - and _field_extension_name(target_field) == "lance.json" - ): - new_fields.append(field) - continue - if pa.types.is_struct(target_field.type): - if pa.types.is_struct(field.type): - new_type = pa.struct( - _align_field_types( - field.type.fields, - target_field.type.fields, - ) + new_fields.append(_align_field(field, target_field)) + return new_fields + + +def _align_list_value_field( + value_field: pa.Field, target_value_field: pa.Field +) -> pa.Field: + # A list has exactly one child, so the inferred child name ("item") aligns + # positionally and adopts the table's child name; pa.Table.cast renames it. + return _align_field(value_field, target_value_field).with_name( + target_value_field.name + ) + + +def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field: + # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored + # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the + # input to that storage type here merely relabels the raw JSON bytes as + # JSONB. Lance must see arrow.json so it can perform the JSONB encoding. + if ( + _field_extension_name(field) == "arrow.json" + and _field_extension_name(target_field) == "lance.json" + ): + return field + if pa.types.is_struct(target_field.type): + if pa.types.is_struct(field.type): + new_type = pa.struct( + _align_field_types( + field.type.fields, + target_field.type.fields, ) - else: - new_type = target_field.type - elif pa.types.is_list(target_field.type): - if _is_list_like(field.type): - new_type = pa.list_( - _align_field_types( - [field.type.value_field], - [target_field.type.value_field], - )[0] - ) - else: - new_type = target_field.type - elif pa.types.is_large_list(target_field.type): - if _is_list_like(field.type): - new_type = pa.large_list( - _align_field_types( - [field.type.value_field], - [target_field.type.value_field], - )[0] - ) - else: - new_type = target_field.type - elif pa.types.is_fixed_size_list(target_field.type): - if _is_list_like(field.type): - new_type = pa.list_( - _align_field_types( - [field.type.value_field], - [target_field.type.value_field], - )[0], - target_field.type.list_size, - ) - else: - new_type = target_field.type + ) else: new_type = target_field.type - new_fields.append( - pa.field(field.name, new_type, field.nullable, target_field.metadata) - ) - return new_fields + elif pa.types.is_list(target_field.type): + if _is_list_like(field.type): + new_type = pa.list_( + _align_list_value_field( + field.type.value_field, target_field.type.value_field + ) + ) + else: + new_type = target_field.type + elif pa.types.is_large_list(target_field.type): + if _is_list_like(field.type): + new_type = pa.large_list( + _align_list_value_field( + field.type.value_field, target_field.type.value_field + ) + ) + else: + new_type = target_field.type + elif pa.types.is_fixed_size_list(target_field.type): + if _is_list_like(field.type): + new_type = pa.list_( + _align_list_value_field( + field.type.value_field, target_field.type.value_field + ), + target_field.type.list_size, + ) + else: + new_type = target_field.type + else: + new_type = target_field.type + return pa.field(field.name, new_type, field.nullable, target_field.metadata) def _infer_subschema( @@ -589,7 +763,7 @@ def sanitize_create_table( schema = data.schema else: if schema is not None: - data = pa.Table.from_pylist([], schema) + data = pa.Table.from_batches([], schema=schema) if schema is None: if data is None: raise ValueError("Either data or schema must be provided") @@ -2698,7 +2872,7 @@ class LanceTable(Table): arrow_tbl = self.to_arrow() if blob_mode == "descriptions": arrow_tbl = strip_auto_row_ids( - arrow_tbl, blob_v2_column_paths(self.schema) + arrow_tbl, row_addressable_blob_v2_paths(self.schema) ) return arrow_tbl.to_pandas(**kwargs) @@ -5102,7 +5276,9 @@ class AsyncTable: if blob_mode == "descriptions" or not schema_has_blob_field(schema): arrow_tbl = await self.to_arrow() if blob_mode == "descriptions": - arrow_tbl = strip_auto_row_ids(arrow_tbl, blob_v2_column_paths(schema)) + arrow_tbl = strip_auto_row_ids( + arrow_tbl, row_addressable_blob_v2_paths(schema) + ) return arrow_tbl.to_pandas(**kwargs) if blob_mode == "lazy" and get_uri_scheme(await self.uri()) == "memory": diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 351769ff6..c9694277c 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -2,10 +2,15 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors import io +import subprocess +import sys +import textwrap +import lance import pyarrow as pa import pyarrow.compute as pc import pytest +from lance.blob import BlobType as LanceBlobType import lancedb from lancedb._blob import ( @@ -18,6 +23,20 @@ from lancedb.index import FTS from lancedb.schema import blob_column_paths, blob_v2_column_paths +_HIDE_LANCE_BLOB = """\ +import importlib.abc +import sys + +class _MissingLanceBlob(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path, target=None): + if fullname == "lance.blob" or fullname.startswith("lance.blob."): + raise ModuleNotFoundError(fullname, name="lance.blob") + +sys.modules.pop("lance.blob", None) +sys.meta_path.insert(0, _MissingLanceBlob()) +""" + + def _blob_table(name, rows): db = lancedb.connect("memory:///") schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) @@ -51,6 +70,181 @@ def test_blob_factory_declares_v2_field(): field = lancedb.blob("image") assert isinstance(field.type, pa.ExtensionType) assert field.type.extension_name == "lance.blob.v2" + assert lancedb.BlobType is LanceBlobType + assert type(field.type) is LanceBlobType + + +def test_blob_type_works_without_pylance(): + script = _HIDE_LANCE_BLOB + textwrap.dedent( + """\ + import lancedb + import pyarrow as pa + + field = lancedb.blob("image") + if not isinstance(field.type, pa.ExtensionType): + raise SystemExit("expected an extension type") + if field.type.extension_name != "lance.blob.v2": + raise SystemExit(field.type.extension_name) + if lancedb.BlobType is not type(field.type): + raise SystemExit("BlobType is not the field type class") + if lancedb.BlobType.__module__ != "lancedb.schema": + raise SystemExit(lancedb.BlobType.__module__) + + db = lancedb.connect("memory:///") + table = db.create_table( + "images", + schema=pa.schema([pa.field("id", pa.int64()), field]), + ) + table.add([{"id": 1, "image": b"hello"}]) + result = ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}]) + ) + if result.num_updated_rows != 1 or result.num_inserted_rows != 1: + raise SystemExit( + f"merge_insert rows updated={result.num_updated_rows} " + f"inserted={result.num_inserted_rows}" + ) + """ + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def test_blob_resolves_pylance_type_without_eager_import(): + script = textwrap.dedent( + """\ + import sys + import lancedb + + if "lance.blob" in sys.modules: + raise SystemExit("import lancedb imported lance.blob") + field = lancedb.blob("image") + from lance.blob import BlobType + + if type(field.type) is not BlobType: + raise SystemExit(f"{type(field.type)} is not {BlobType}") + import lance + + image = lance.blob_array([b"x"]) + if type(image.type) is not BlobType: + raise SystemExit("blob_array used a different class") + if type(image.type) is not type(field.type): + raise SystemExit("field and array classes differ") + """ + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def test_blob_fallback_fails_if_name_already_registered(): + script = _HIDE_LANCE_BLOB + textwrap.dedent( + """\ + import pyarrow as pa + + class OtherBlobType(pa.ExtensionType): + def __init__(self): + super().__init__( + pa.struct([pa.field("data", pa.large_binary())]), + "lance.blob.v2", + ) + + def __arrow_ext_serialize__(self): + return b"" + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + return cls() + + pa.register_extension_type(OtherBlobType()) + import lancedb + + try: + lancedb.blob("image") + except ValueError as err: + if "already registered" not in str(err): + raise SystemExit(err) + else: + raise SystemExit("expected ValueError") + """ + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def test_blob_type_rejects_competing_registration_with_pylance(): + script = textwrap.dedent( + """\ + import pyarrow as pa + import pyarrow.ipc + + class OtherBlobType(pa.ExtensionType): + def __init__(self): + super().__init__( + pa.struct( + [ + pa.field("data", pa.large_binary()), + pa.field("uri", pa.utf8()), + pa.field("position", pa.uint64()), + pa.field("size", pa.uint64()), + ] + ), + "lance.blob.v2", + ) + + def __arrow_ext_serialize__(self): + return b"" + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + return cls() + + pa.register_extension_type(OtherBlobType()) + + from lance.blob import BlobType + + if BlobType is OtherBlobType: + raise SystemExit("pylance BlobType was replaced") + schema = pa.schema([pa.field("value", BlobType())]) + restored = pa.ipc.read_schema(schema.serialize()) + if type(restored.field("value").type) is not OtherBlobType: + raise SystemExit(type(restored.field("value").type)) + + import lancedb + + try: + lancedb.blob("image") + except ValueError as err: + if "__main__.OtherBlobType" not in str(err): + raise SystemExit(err) + else: + raise SystemExit("expected ValueError") + """ + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr def test_blob_v2_column_paths_include_list_children(): @@ -203,6 +397,292 @@ def test_fetch_blobs_round_trip(): assert [blobs[0].as_py(), blobs[1].as_py()] == [b"alpha", b"beta"] +def test_merge_insert_writes_python_bytes(): + table = _blob_table("merge_bytes", [{"id": 1, "image": b"before"}]) + result = ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}]) + ) + assert result.num_updated_rows == 1 + assert result.num_inserted_rows == 1 + by_id = _row_ids_by_id(table) + blobs = table.fetch_blobs("image", [by_id[1], by_id[2]]) + assert blobs.to_pylist() == [b"updated", b"inserted"] + + +def test_merge_insert_bytes_after_reopen_without_touching_blob_type(tmp_path): + db = lancedb.connect(tmp_path) + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = db.create_table("images", schema=schema) + table.add([{"id": 1, "image": b"hello"}]) + + script = textwrap.dedent( + f"""\ + import lancedb + + db = lancedb.connect({str(tmp_path)!r}) + table = db.open_table("images") + image_type = table.schema.field("image").type + if type(image_type).__name__ != "StructType": + raise SystemExit(f"expected StructType, got {{type(image_type)}}") + result = ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute( + [{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}] + ) + ) + if result.num_updated_rows != 1 or result.num_inserted_rows != 1: + raise SystemExit( + f"rows updated={{result.num_updated_rows}} " + f"inserted={{result.num_inserted_rows}}" + ) + hits = table.search().with_row_id(True).limit(10).to_arrow() + by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist())) + blobs = table.fetch_blobs("image", [by_id[1], by_id[2]]) + if blobs.to_pylist() != [b"updated", b"inserted"]: + raise SystemExit(blobs.to_pylist()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def test_merge_insert_bytes_after_reopen_without_pylance(tmp_path): + db = lancedb.connect(tmp_path) + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = db.create_table("images", schema=schema) + table.add([{"id": 1, "image": b"hello"}]) + + script = _HIDE_LANCE_BLOB + textwrap.dedent( + f"""\ + import lancedb + + db = lancedb.connect({str(tmp_path)!r}) + table = db.open_table("images") + image_type = table.schema.field("image").type + if type(image_type).__name__ != "StructType": + raise SystemExit(f"expected StructType, got {{type(image_type)}}") + result = ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute( + [{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}] + ) + ) + if result.num_updated_rows != 1 or result.num_inserted_rows != 1: + raise SystemExit( + f"rows updated={{result.num_updated_rows}} " + f"inserted={{result.num_inserted_rows}}" + ) + hits = table.search().with_row_id(True).limit(10).to_arrow() + by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist())) + blobs = table.fetch_blobs("image", [by_id[1], by_id[2]]) + if blobs.to_pylist() != [b"updated", b"inserted"]: + raise SystemExit(blobs.to_pylist()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def test_merge_insert_blob_array_into_reopened_unregistered_table(tmp_path): + db = lancedb.connect(tmp_path) + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = db.create_table("images", schema=schema) + table.add([{"id": 1, "image": b"before"}]) + + script = textwrap.dedent( + f"""\ + import pyarrow as pa + import lancedb + + db = lancedb.connect({str(tmp_path)!r}) + table = db.open_table("images") + image_type = table.schema.field("image").type + if type(image_type).__name__ != "StructType": + raise SystemExit( + f"expected StructType before lance import, got {{type(image_type)}}" + ) + + import lance + + updates = pa.Table.from_arrays( + [ + pa.array([1, 2], type=pa.int64()), + lance.blob_array([b"updated", b"inserted"]), + ], + names=["id", "image"], + ) + result = ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(updates) + ) + if result.num_updated_rows != 1 or result.num_inserted_rows != 1: + raise SystemExit( + f"rows updated={{result.num_updated_rows}} " + f"inserted={{result.num_inserted_rows}}" + ) + hits = table.search().with_row_id(True).limit(10).to_arrow() + by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist())) + blobs = table.fetch_blobs("image", [by_id[1], by_id[2]]) + if blobs.to_pylist() != [b"updated", b"inserted"]: + raise SystemExit(blobs.to_pylist()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def test_add_all_null_blob_column(): + db = lancedb.connect("memory:///") + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = db.create_table("all_null", schema=schema) + table.add([{"id": 1, "image": None}, {"id": 2, "image": None}]) + by_id = _row_ids_by_id(table) + blobs = table.fetch_blobs("image", [by_id[1], by_id[2]]) + assert blobs.to_pylist() == [None, None] + + +def test_create_table_nested_blob_schema_without_rows(): + db = lancedb.connect("memory:///") + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("info", pa.struct([lancedb.blob("blob")])), + pa.field("images", pa.list_(lancedb.blob("image"))), + ] + ) + table = db.create_table("nested_empty", schema=schema) + assert table.count_rows() == 0 + + +def test_merge_insert_nested_blob_dicts(): + db = lancedb.connect("memory:///") + info = pa.StructArray.from_arrays( + [ + pa.array(["first"], type=pa.string()), + _blob_array("blob", [b"before"]), + ], + names=["name", "blob"], + ) + data = pa.Table.from_arrays( + [pa.array([1], type=pa.int64()), info], + names=["id", "info"], + ) + table = db.create_table("nested_merge", data=data) + result = ( + table.merge_insert("id") + .when_matched_update_all() + .execute([{"id": 1, "info": {"name": "first", "blob": b"after"}}]) + ) + assert result.num_updated_rows == 1 + by_id = _row_ids_by_id(table) + blobs = table.fetch_blobs("info.blob", [by_id[1]]) + assert blobs.to_pylist() == [b"after"] + + +def _list_blob_table(name): + db = lancedb.connect("memory:///") + blob_field = lancedb.blob("image") + images = pa.ListArray.from_arrays( + pa.array([0, 1], type=pa.int32()), _blob_array("image", [b"before"]) + ) + data = pa.Table.from_arrays( + [pa.array([1], type=pa.int64()), images], + schema=pa.schema( + [pa.field("id", pa.int64()), pa.field("images", pa.list_(blob_field))] + ), + ) + return db.create_table(name, data=data) + + +def test_merge_insert_list_blob_dicts(): + table = _list_blob_table("list_merge") + result = ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute([{"id": 1, "images": [b"one", b"two"]}, {"id": 2, "images": None}]) + ) + assert result.num_updated_rows == 1 + assert result.num_inserted_rows == 1 + hits = table.search().limit(10).to_arrow() + sizes = { + row["id"]: None if row["images"] is None else [d["size"] for d in row["images"]] + for row in hits.to_pylist() + } + assert sizes == {1: [3, 3], 2: None} + + +def test_list_blob_column_queries_as_raw_descriptors(): + table = _list_blob_table("list_query") + hits = table.search().limit(10).to_arrow() + element = hits.schema.field("images").type.value_type + assert pa.types.is_struct(element) + assert "_lance_row_id" not in element.names + with pytest.raises(ValueError, match="expected struct before segment"): + table.fetch_blobs("images.image", [0]) + + +def test_row_addressable_paths_exclude_list_children(): + from lancedb.schema import row_addressable_blob_v2_paths + + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("info", pa.struct([lancedb.blob("blob")])), + pa.field("images", pa.list_(lancedb.blob("image"))), + ] + ) + assert blob_v2_column_paths(schema) == ["info.blob", "images.image"] + assert row_addressable_blob_v2_paths(schema) == ["info.blob"] + + +def test_merge_insert_writes_pylance_blob_array(): + table = _blob_table("merge_pylance", [{"id": 1, "image": b"before"}]) + image = lance.blob_array([b"updated", b"inserted"]) + assert type(image.type) is LanceBlobType + assert type(image.type) is type(lancedb.BlobType()) + updates = pa.Table.from_arrays( + [pa.array([1, 2], type=pa.int64()), image], names=["id", "image"] + ) + + result = ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(updates) + ) + + assert result.num_updated_rows == 1 + assert result.num_inserted_rows == 1 + by_id = _row_ids_by_id(table) + blobs = table.fetch_blobs("image", [by_id[1], by_id[2]]) + assert blobs.to_pylist() == [b"updated", b"inserted"] + + def test_fetch_blobs_accepts_query_result(): table = _blob_table("from_result", [{"id": 1, "image": b"gamma"}]) hits = table.search().limit(10).to_arrow() diff --git a/python/python/tests/test_util.py b/python/python/tests/test_util.py index a9b66b2dd..acdef8eac 100644 --- a/python/python/tests/test_util.py +++ b/python/python/tests/test_util.py @@ -7,6 +7,7 @@ import pathlib from typing import Optional import lance +from lance.blob import BlobType as LanceBlobType from lancedb.conftest import MockTextEmbeddingFunction from lancedb.embeddings.base import EmbeddingFunctionConfig from lancedb.embeddings.registry import EmbeddingFunctionRegistry @@ -907,6 +908,165 @@ def test_cast_to_target_schema(): assert output == expected +def test_cast_to_target_schema_coerces_binary_to_blob_v2(): + data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())}) + target = pa.schema([lancedb.blob("image")]) + + output = _cast_to_target_schema(data.to_reader(), target).read_all() + + image = output["image"].chunk(0) + assert type(image.type) is lancedb.BlobType + assert image.storage.to_pylist() == [ + {"data": b"hello", "uri": None, "position": None, "size": None}, + None, + ] + + +def test_cast_to_target_schema_coerces_binary_to_metadata_blob_struct(): + storage = lancedb.blob("image").type.storage_type + target = pa.schema( + [ + pa.field( + "image", + storage, + metadata={ + b"ARROW:extension:name": b"lance.blob.v2", + b"ARROW:extension:metadata": b"", + }, + ) + ] + ) + data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())}) + + output = _cast_to_target_schema(data.to_reader(), target).read_all() + + image = output["image"].chunk(0) + assert not isinstance(image.type, pa.ExtensionType) + assert image.to_pylist() == [ + {"data": b"hello", "uri": None, "position": None, "size": None}, + None, + ] + + +def test_cast_to_target_schema_coerces_nested_binary_blob(): + data = pa.table( + { + "info": pa.array( + [{"blob": b"hello"}, {"blob": None}], + type=pa.struct([pa.field("blob", pa.binary())]), + ) + } + ) + target = pa.schema([pa.field("info", pa.struct([lancedb.blob("blob")]))]) + + output = _cast_to_target_schema(data.to_reader(), target).read_all() + + blob = output["info"].chunk(0).field("blob") + assert type(blob.type) is lancedb.BlobType + assert blob.storage.to_pylist() == [ + {"data": b"hello", "uri": None, "position": None, "size": None}, + None, + ] + + +def test_cast_to_target_schema_coerces_list_binary_blob_with_inferred_child_name(): + data = pa.table( + {"images": pa.array([[b"a", b"b"], None], type=pa.list_(pa.binary()))} + ) + target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))]) + + output = _cast_to_target_schema(data.to_reader(), target).read_all() + + images = output["images"].chunk(0) + assert images.type.value_field.name == "image" + assert type(images.type.value_type) is lancedb.BlobType + assert images.to_pylist()[1] is None + assert images.values.storage.to_pylist() == [ + {"data": b"a", "uri": None, "position": None, "size": None}, + {"data": b"b", "uri": None, "position": None, "size": None}, + ] + + +def test_list_blob_coercion_preserves_null_slots_with_nonzero_extent(): + child = pa.field("image", pa.binary()) + source = pa.ListArray.from_arrays( + pa.array([0, 2, 4], type=pa.int32()), + pa.array([b"a", b"b", b"dead", b"beef"], type=pa.binary()), + mask=pa.array([False, True]), + ).cast(pa.list_(child)) + target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))]) + + output = _cast_to_target_schema( + pa.table({"images": source}).to_reader(), target + ).read_all() + + images = output["images"].chunk(0) + assert images.to_pylist()[1] is None + assert [b["data"] for b in images.to_pylist()[0]] == [b"a", b"b"] + + +def test_fixed_size_list_blob_coercion_keeps_null_rows(): + child = pa.field("frame", pa.binary()) + source = ( + pa.FixedSizeListArray.from_arrays( + pa.array([b"a", b"b", b"c", b"d"], type=pa.binary()), 2 + ) + .take(pa.array([0, None], type=pa.int32())) + .cast(pa.list_(child, 2)) + ) + target = pa.schema([pa.field("frames", pa.list_(lancedb.blob("frame"), 2))]) + + output = _cast_to_target_schema( + pa.table({"frames": source}).to_reader(), target + ).read_all() + + frames = output["frames"].chunk(0) + assert frames.to_pylist()[1] is None + assert [b["data"] for b in frames.to_pylist()[0]] == [b"a", b"b"] + + +def test_cast_to_target_schema_accepts_pylance_blob_v2(): + target_type = lancedb.BlobType() + source = lance.blob_array([b"hello", None]) + assert type(source.type) is LanceBlobType + assert type(source.type) is type(target_type) + data = pa.table({"image": source}) + target = pa.schema([pa.field("image", target_type)]) + + output = _cast_to_target_schema(data.to_reader(), target).read_all() + + image = output["image"].chunk(0) + assert type(image.type) is LanceBlobType + assert image.type == target_type + assert image.storage.to_pylist() == [ + {"data": b"hello", "uri": None, "position": None, "size": None}, + None, + ] + + +def test_cast_to_target_schema_rejects_different_blob_v2_class(): + class OtherBlobType(pa.ExtensionType): + def __init__(self): + super().__init__(lancedb.BlobType().storage_type, "lance.blob.v2") + + def __arrow_ext_serialize__(self) -> bytes: + return b"" + + @classmethod + def __arrow_ext_deserialize__( + cls, storage_type: pa.DataType, serialized: bytes + ) -> "OtherBlobType": + return cls() + + storage = lance.blob_array([b"hello"]).storage + source = pa.ExtensionArray.from_storage(OtherBlobType(), storage) + data = pa.table({"image": source}) + target = pa.schema([lancedb.blob("image")]) + + with pytest.raises(pa.ArrowTypeError, match="different extension type"): + _cast_to_target_schema(data.to_reader(), target).read_all() + + def test_sanitize_data_stream(): # Make sure we don't collect the whole stream when running sanitize_data schema = pa.schema({"a": pa.int32()}) diff --git a/rust/lancedb/src/table/datafusion/blob_coerce.rs b/rust/lancedb/src/table/datafusion/blob_coerce.rs index cb984f7f4..0596e7a2d 100644 --- a/rust/lancedb/src/table/datafusion/blob_coerce.rs +++ b/rust/lancedb/src/table/datafusion/blob_coerce.rs @@ -36,6 +36,14 @@ pub(super) fn coerce_blob_expr( }; let input_shape = match input_field.data_type() { + DataType::Null => { + let expr: Arc = Arc::new(CastExpr::new( + input_expr, + table_field.data_type().clone(), + None, + )); + return Ok((expr, table_field.clone())); + } DataType::Binary | DataType::LargeBinary | DataType::BinaryView => BlobInputShape::Bytes, DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => BlobInputShape::String, DataType::Struct(children) => { @@ -155,7 +163,7 @@ mod tests { use crate::blob::blob; use arrow_array::{ Array, ArrayRef, BinaryArray, BinaryViewArray, Int32Array, Int64Array, LargeBinaryArray, - RecordBatch, StringArray, StringViewArray, StructArray, UInt8Array, UInt64Array, + NullArray, RecordBatch, StringArray, StringViewArray, StructArray, UInt8Array, UInt64Array, }; use arrow_schema::Schema; use datafusion::prelude::SessionContext; @@ -279,6 +287,18 @@ mod tests { assert_eq!(data.value(0), b"view"); } + #[tokio::test] + async fn null_column_coerces_to_all_null_blob_struct() { + let batch = batch_with_image( + Field::new("image", DataType::Null, true), + Arc::new(NullArray::new(2)), + ); + let coerced = coerce(batch, &blob_table_schema()).await; + let image = image_struct(&coerced); + assert!(image.is_null(0)); + assert!(image.is_null(1)); + } + #[tokio::test] async fn binary_nulls_stay_null_after_coercion() { let batch = batch_with_image(