From 38a62db418498b75a3b412d9a2680df404b16c2e Mon Sep 17 00:00:00 2001 From: SIDDARTHA REDDY <75976672+SIDDARTHAREDDY8@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:36:24 -0400 Subject: [PATCH] feat(python,rust): allow lancedb.blob() to set blob tier thresholds (#4323) --- python/python/lancedb/schema.py | 55 +++++++++++++++++++++++-- python/python/tests/test_blob.py | 31 ++++++++++++++ rust/lancedb/src/blob.rs | 70 +++++++++++++++++++++++++++++++- 3 files changed, 151 insertions(+), 5 deletions(-) diff --git a/python/python/lancedb/schema.py b/python/python/lancedb/schema.py index 4dce09f0f..5edda5610 100644 --- a/python/python/lancedb/schema.py +++ b/python/python/lancedb/schema.py @@ -5,7 +5,7 @@ """Schema helpers for Lance blob columns.""" import importlib -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional import pyarrow as pa import pyarrow.ipc @@ -181,13 +181,60 @@ def _resolve_blob_type(): return _resolved_blob_type -def blob(name: str, nullable: bool = True) -> pa.Field: +def blob( + name: str, + nullable: bool = True, + *, + inline_size_threshold: Optional[int] = None, + dedicated_size_threshold: Optional[int] = None, + pack_file_size_threshold: Optional[int] = None, +) -> pa.Field: """Create a Lance blob v2 column field. When pylance is installed this is ``lance.blob.BlobType``. + + Parameters + ---------- + name: + Field name. + nullable: + Whether the column accepts null values. + inline_size_threshold: + Maximum payload size in bytes to keep inline in the data file + before using packed blob storage. Zero is allowed. + dedicated_size_threshold: + Maximum payload size in bytes to store in packed blob storage + before using dedicated blob storage. Must be positive. + pack_file_size_threshold: + Maximum size in bytes of a single packed blob sidecar file before + a new one is started. Must be positive. + + The thresholds are forwarded to ``lance.blob_field``, which validates + them and stores them as field metadata, so pylance is required whenever + any threshold is set. When all thresholds are omitted the behavior is + unchanged. """ - blob_type = _resolve_blob_type() - return pa.field(name, blob_type(), nullable=nullable) + if ( + inline_size_threshold is None + and dedicated_size_threshold is None + and pack_file_size_threshold is None + ): + blob_type = _resolve_blob_type() + return pa.field(name, blob_type(), nullable=nullable) + try: + lance = importlib.import_module("lance") + except ModuleNotFoundError as err: + raise ValueError( + "Setting blob tier thresholds requires pylance to be installed" + ) from err + _resolve_blob_type() + return lance.blob_field( + name, + nullable=nullable, + inline_size_threshold=inline_size_threshold, + dedicated_size_threshold=dedicated_size_threshold, + pack_file_size_threshold=pack_file_size_threshold, + ) def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType: diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 2bb5b6bea..27e920e50 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -95,6 +95,37 @@ def test_blob_factory_declares_v2_field(): assert type(field.type) is LanceBlobType +def test_blob_thresholds_land_in_field_metadata(): + field = lancedb.blob( + "payload", + inline_size_threshold=8 * 1024, + dedicated_size_threshold=1 << 20, + pack_file_size_threshold=1 << 26, + ) + assert field.type.extension_name == "lance.blob.v2" + metadata = field.metadata + assert metadata[b"lance-encoding:blob-inline-size-threshold"] == b"8192" + assert metadata[b"lance-encoding:blob-dedicated-size-threshold"] == b"1048576" + assert metadata[b"lance-encoding:blob-pack-file-size-threshold"] == b"67108864" + + +def test_blob_without_thresholds_has_no_threshold_metadata(): + field = lancedb.blob("image") + metadata = field.metadata or {} + assert not any(b"threshold" in key for key in metadata) + + +def test_blob_thresholds_are_validated(): + with pytest.raises(ValueError, match="inline_size_threshold must be non-negative"): + lancedb.blob("payload", inline_size_threshold=-1) + with pytest.raises(ValueError, match="dedicated_size_threshold must be positive"): + lancedb.blob("payload", dedicated_size_threshold=0) + with pytest.raises(ValueError, match="pack_file_size_threshold must be positive"): + lancedb.blob("payload", pack_file_size_threshold=0) + with pytest.raises(TypeError, match="inline_size_threshold must be an int"): + lancedb.blob("payload", inline_size_threshold="8192") + + def test_blob_type_works_without_pylance(): script = _HIDE_LANCE_BLOB + textwrap.dedent( """\ diff --git a/rust/lancedb/src/blob.rs b/rust/lancedb/src/blob.rs index cbc0724c6..b7f16974e 100644 --- a/rust/lancedb/src/blob.rs +++ b/rust/lancedb/src/blob.rs @@ -262,6 +262,37 @@ pub fn blob(name: impl AsRef, nullable: bool) -> Field { lance::blob::blob_field(name.as_ref(), nullable) } +/// Options for creating a blob v2 column field with tier thresholds. +/// +/// Re-exported from lance so the thresholds are built exactly the way +/// [`lance::blob::blob_field_with_options`] builds them. Note that lance does +/// not currently expose a pack file size threshold option, so only the inline +/// and dedicated size thresholds are supported here. +pub use lance::blob::BlobFieldOptions as BlobOptions; + +/// Creates an Arrow field for a Lance blob v2 column with tier thresholds. +/// +/// Same as [`blob`], but forwards `options` to +/// [`lance::blob::blob_field_with_options`], storing the inline and dedicated +/// size thresholds as field metadata. With [`BlobOptions::default()`] this is +/// identical to [`blob`]. +/// +/// ``` +/// use std::num::NonZeroUsize; +/// +/// use lancedb::blob::BlobOptions; +/// +/// let options = BlobOptions { +/// inline_size_threshold: Some(8 * 1024), +/// dedicated_size_threshold: NonZeroUsize::new(1 << 20), +/// }; +/// let field = lancedb::blob::blob_with_options("payload", true, options); +/// assert!(lancedb::blob::is_blob(&field)); +/// ``` +pub fn blob_with_options(name: impl AsRef, nullable: bool, options: BlobOptions) -> Field { + lance::blob::blob_field_with_options(name.as_ref(), nullable, options) +} + /// Returns true if `field` is a blob v2 column. /// /// ``` @@ -501,8 +532,13 @@ pub(crate) async fn take_blob_files_aligned( #[cfg(test)] mod tests { use super::*; + use std::num::NonZeroUsize; + use arrow_schema::DataType; - use lance_arrow::ARROW_EXT_NAME_KEY; + use lance_arrow::{ + ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, + BLOB_INLINE_SIZE_THRESHOLD_META_KEY, + }; fn blob_schema() -> Schema { Schema::new(vec![ @@ -521,6 +557,38 @@ mod tests { assert!(matches!(field.data_type(), DataType::Struct(_))); } + #[test] + fn blob_with_options_sets_threshold_metadata() { + let options = BlobOptions { + inline_size_threshold: Some(8 * 1024), + dedicated_size_threshold: NonZeroUsize::new(1 << 20), + }; + let field = blob_with_options("payload", true, options); + assert!(is_blob(&field)); + assert_eq!( + field + .metadata() + .get(BLOB_INLINE_SIZE_THRESHOLD_META_KEY) + .map(String::as_str), + Some("8192") + ); + assert_eq!( + field + .metadata() + .get(BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY) + .map(String::as_str), + Some("1048576") + ); + } + + #[test] + fn blob_with_default_options_matches_plain_blob() { + let field = blob_with_options("payload", true, BlobOptions::default()); + let plain = blob("payload", true); + assert_eq!(field.metadata(), plain.metadata()); + assert_eq!(field.data_type(), plain.data_type()); + } + #[test] fn has_blob_columns_detects_blob_fields() { assert!(has_blob_columns(&blob_schema()));