From 27b65064d0166feabe1c307eba07f66ca2601857 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:37:07 -0700 Subject: [PATCH] fix(python): support custom blob thresholds without pylance (#4634) --- python/python/lancedb/schema.py | 62 ++++++++++------ python/python/tests/test_blob.py | 37 ++++++++++ python/python/tests/test_table.py | 118 ++++++++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 24 deletions(-) diff --git a/python/python/lancedb/schema.py b/python/python/lancedb/schema.py index 5edda5610..8bec17058 100644 --- a/python/python/lancedb/schema.py +++ b/python/python/lancedb/schema.py @@ -4,6 +4,7 @@ """Schema helpers for Lance blob columns.""" +import ctypes import importlib from typing import TYPE_CHECKING, Optional @@ -16,6 +17,7 @@ if TYPE_CHECKING: _BLOB_EXTENSION_NAME = "lance.blob.v2" _BLOB_V1_KEY = "lance-encoding:blob" _ARROW_EXT_NAME_KEY = "ARROW:extension:name" +_MAX_RUST_USIZE = ctypes.c_size_t(-1).value _BLOB_V2_STORAGE_TYPE = pa.struct( [ pa.field("data", pa.large_binary(), nullable=True), @@ -209,32 +211,44 @@ def blob( Maximum size in bytes of a single packed blob sidecar file before a new one is started. Must be positive. - The thresholds are forwarded to ``lance.blob_field``, which validates - them and stores them as field metadata, so pylance is required whenever - any threshold is set. When all thresholds are omitted the behavior is - unchanged. + Thresholds are validated and stored as Arrow field metadata, with the + same format as ``lance.blob_field``. They do not require pylance. """ - if ( - inline_size_threshold is None - and dedicated_size_threshold is None - and pack_file_size_threshold is None + metadata = {} + for threshold, value, allow_zero, key in ( + ( + "inline_size_threshold", + inline_size_threshold, + True, + "lance-encoding:blob-inline-size-threshold", + ), + ( + "dedicated_size_threshold", + dedicated_size_threshold, + False, + "lance-encoding:blob-dedicated-size-threshold", + ), + ( + "pack_file_size_threshold", + pack_file_size_threshold, + False, + "lance-encoding:blob-pack-file-size-threshold", + ), ): - blob_type = _resolve_blob_type() - return pa.field(name, blob_type(), nullable=nullable) - try: - lance = importlib.import_module("lance") - except ModuleNotFoundError as err: - raise ValueError( - "Setting blob tier thresholds requires pylance to be installed" - ) from err - _resolve_blob_type() - return lance.blob_field( - name, - nullable=nullable, - inline_size_threshold=inline_size_threshold, - dedicated_size_threshold=dedicated_size_threshold, - pack_file_size_threshold=pack_file_size_threshold, - ) + if value is None: + continue + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{threshold} must be an int, got {type(value).__name__}") + if value < 0 or (value == 0 and not allow_zero): + bound = "non-negative" if allow_zero else "positive" + raise ValueError(f"{threshold} must be {bound}") + if value > _MAX_RUST_USIZE: + raise OverflowError(f"{threshold} must fit in a Rust usize") + metadata[key] = str(value) + + blob_type = _resolve_blob_type() + field = pa.field(name, blob_type(), nullable=nullable) + return field.with_metadata(metadata) if metadata else field def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType: diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index ec1b8b27e..c2dc2a265 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import ctypes import io import subprocess import sys @@ -14,6 +15,7 @@ import pytest from lance.blob import Blob, BlobType as LanceBlobType import lancedb +from lancedb import schema as schema_module from lancedb._blob import ( _set_blob_column, blob_v2_projection_sources, @@ -139,6 +141,41 @@ def test_blob_without_thresholds_has_no_threshold_metadata(): assert not any(b"threshold" in key for key in metadata) +@pytest.mark.parametrize("nullable", [False, True]) +@pytest.mark.parametrize( + "thresholds", + [ + {}, + {"inline_size_threshold": 0}, + {"inline_size_threshold": 8192}, + {"dedicated_size_threshold": 1 << 20}, + {"pack_file_size_threshold": 1 << 26}, + { + "inline_size_threshold": 8192, + "dedicated_size_threshold": 1 << 20, + "pack_file_size_threshold": 1 << 26, + }, + { + "inline_size_threshold": ctypes.c_size_t(-1).value, + "dedicated_size_threshold": ctypes.c_size_t(-1).value, + "pack_file_size_threshold": ctypes.c_size_t(-1).value, + }, + ], +) +def test_blob_thresholds_match_pylance(monkeypatch, nullable, thresholds): + expected = lance.blob_field("payload", nullable=nullable, **thresholds) + actual = lancedb.blob("payload", nullable=nullable, **thresholds) + assert type(actual.type) is LanceBlobType + assert actual.equals(expected, check_metadata=True) + + # Both extension classes must encode the same Arrow field on the wire. + monkeypatch.setattr( + schema_module, "_resolve_blob_type", lambda: schema_module._FallbackBlobType + ) + fallback = lancedb.blob("payload", nullable=nullable, **thresholds) + assert pa.schema([fallback]).serialize() == pa.schema([expected]).serialize() + + def test_blob_thresholds_are_validated(): with pytest.raises(ValueError, match="inline_size_threshold must be non-negative"): lancedb.blob("payload", inline_size_threshold=-1) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 18d3c16db..6ccba606f 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -59,6 +59,124 @@ def _blob_v2_table(db: DBConnection, name: str): return table +@pytest.mark.parametrize( + "threshold", + [ + "inline_size_threshold", + "dedicated_size_threshold", + "pack_file_size_threshold", + ], +) +@pytest.mark.parametrize( + "value,error", + [ + (-1, ValueError), + (True, TypeError), + (False, TypeError), + (1.5, TypeError), + ("8192", TypeError), + (ctypes.c_size_t(-1).value + 1, OverflowError), + ], +) +def test_blob_thresholds_reject_invalid_values(threshold, value, error): + # Also runs in CI with pylance actually uninstalled. + with pytest.raises(error, match=f"{threshold} must"): + lancedb.blob("blob", **{threshold: value}) + + +@pytest.mark.parametrize( + "threshold", ["dedicated_size_threshold", "pack_file_size_threshold"] +) +def test_blob_thresholds_reject_zero_for_positive_thresholds(threshold): + with pytest.raises(ValueError, match=f"{threshold} must be positive"): + lancedb.blob("blob", **{threshold: 0}) + + +def test_blob_thresholds_accept_integer_boundaries(): + maximum = ctypes.c_size_t(-1).value + field = lancedb.blob( + "blob", + nullable=False, + inline_size_threshold=0, + dedicated_size_threshold=maximum, + pack_file_size_threshold=maximum, + ) + assert not field.nullable + assert field.metadata == { + b"lance-encoding:blob-inline-size-threshold": b"0", + b"lance-encoding:blob-dedicated-size-threshold": str(maximum).encode(), + b"lance-encoding:blob-pack-file-size-threshold": str(maximum).encode(), + } + + +def test_blob_thresholds_control_writes_after_reopen(tmp_path): + # Keep this in test_table.py: CI runs it without pylance or pandas installed. + field = lancedb.blob( + "blob", + inline_size_threshold=4, + dedicated_size_threshold=12, + pack_file_size_threshold=8, + ) + metadata = { + b"lance-encoding:blob-inline-size-threshold": b"4", + b"lance-encoding:blob-dedicated-size-threshold": b"12", + b"lance-encoding:blob-pack-file-size-threshold": b"8", + } + assert field.metadata == metadata + schema = pa.schema([pa.field("id", pa.int64()), field]) + db = lancedb.connect(tmp_path) + table = db.create_table("blobs", schema=schema) + # Every payload fits in the default inline tier; custom thresholds must + # instead send the larger values to packed and dedicated storage. + rows = [ + {"id": 0, "blob": b"a" * 2}, + {"id": 1, "blob": b"b" * 8}, + {"id": 2, "blob": b"c" * 16}, + {"id": 3, "blob": b"d" * 8}, + ] + + def check_writes(table, expected): + assert table.schema.field("blob").metadata.items() >= metadata.items() + hits = table.search().with_row_id(True).limit(10).to_arrow() + actual = {row["id"]: row for row in hits.to_pylist()} + assert actual.keys() == {row["id"] for row in expected} + for row in expected: + size = len(row["blob"]) + descriptor = actual[row["id"]]["blob"] + assert descriptor["size"] == size + assert descriptor["kind"] == (2 if size > 12 else 1 if size > 4 else 0) + row_ids = [actual[row["id"]]["_rowid"] for row in expected] + assert table.fetch_blobs("blob", row_ids).to_pylist() == [ + row["blob"] for row in expected + ] + # Two packed values reach the configured sidecar limit independently. + packed = [ + actual[row["id"]]["blob"] for row in expected if len(row["blob"]) == 8 + ] + assert len({desc["blob_id"] for desc in packed}) == len(packed) + + table.add(rows) + check_writes(table, rows) + table = lancedb.connect(tmp_path).open_table("blobs") + check_writes(table, rows) + updates = [ + {"id": 0, "blob": b"e" * 16}, + {"id": 1, "blob": b"f" * 2}, + {"id": 2, "blob": b"g" * 8}, + {"id": 3, "blob": b"h" * 8}, + {"id": 4, "blob": b"i" * 16}, + ] + result = ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(updates) + ) + assert result.num_updated_rows == 4 + assert result.num_inserted_rows == 1 + check_writes(lancedb.connect(tmp_path).open_table("blobs"), updates) + + async def _blob_v2_table_async(db: AsyncConnection, name: str): schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")]) table = await db.create_table(name, schema=schema)