feat(python,rust): allow lancedb.blob() to set blob tier thresholds (#4323)

python/python/lancedb/schema.py

@@ -5,7 +5,7 @@
 """Schema helpers for Lance blob columns."""
 
 import importlib
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 import pyarrow as pa
 import pyarrow.ipc
@@ -181,13 +181,60 @@ def _resolve_blob_type():
     return _resolved_blob_type
 
 
-def blob(name: str, nullable: bool = True) -> pa.Field:
+def blob(
+    name: str,
+    nullable: bool = True,
+    *,
+    inline_size_threshold: Optional[int] = None,
+    dedicated_size_threshold: Optional[int] = None,
+    pack_file_size_threshold: Optional[int] = None,
+) -> pa.Field:
     """Create a Lance blob v2 column field.
 
     When pylance is installed this is ``lance.blob.BlobType``.
+
+    Parameters
+    ----------
+    name:
+        Field name.
+    nullable:
+        Whether the column accepts null values.
+    inline_size_threshold:
+        Maximum payload size in bytes to keep inline in the data file
+        before using packed blob storage. Zero is allowed.
+    dedicated_size_threshold:
+        Maximum payload size in bytes to store in packed blob storage
+        before using dedicated blob storage. Must be positive.
+    pack_file_size_threshold:
+        Maximum size in bytes of a single packed blob sidecar file before
+        a new one is started. Must be positive.
+
+    The thresholds are forwarded to ``lance.blob_field``, which validates
+    them and stores them as field metadata, so pylance is required whenever
+    any threshold is set. When all thresholds are omitted the behavior is
+    unchanged.
     """
-    blob_type = _resolve_blob_type()
-    return pa.field(name, blob_type(), nullable=nullable)
+    if (
+        inline_size_threshold is None
+        and dedicated_size_threshold is None
+        and pack_file_size_threshold is None
+    ):
+        blob_type = _resolve_blob_type()
+        return pa.field(name, blob_type(), nullable=nullable)
+    try:
+        lance = importlib.import_module("lance")
+    except ModuleNotFoundError as err:
+        raise ValueError(
+            "Setting blob tier thresholds requires pylance to be installed"
+        ) from err
+    _resolve_blob_type()
+    return lance.blob_field(
+        name,
+        nullable=nullable,
+        inline_size_threshold=inline_size_threshold,
+        dedicated_size_threshold=dedicated_size_threshold,
+        pack_file_size_threshold=pack_file_size_threshold,
+    )
 
 
 def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType:

python/python/tests/test_blob.py

@@ -95,6 +95,37 @@ def test_blob_factory_declares_v2_field():
     assert type(field.type) is LanceBlobType
 
 
+def test_blob_thresholds_land_in_field_metadata():
+    field = lancedb.blob(
+        "payload",
+        inline_size_threshold=8 * 1024,
+        dedicated_size_threshold=1 << 20,
+        pack_file_size_threshold=1 << 26,
+    )
+    assert field.type.extension_name == "lance.blob.v2"
+    metadata = field.metadata
+    assert metadata[b"lance-encoding:blob-inline-size-threshold"] == b"8192"
+    assert metadata[b"lance-encoding:blob-dedicated-size-threshold"] == b"1048576"
+    assert metadata[b"lance-encoding:blob-pack-file-size-threshold"] == b"67108864"
+
+
+def test_blob_without_thresholds_has_no_threshold_metadata():
+    field = lancedb.blob("image")
+    metadata = field.metadata or {}
+    assert not any(b"threshold" in key for key in metadata)
+
+
+def test_blob_thresholds_are_validated():
+    with pytest.raises(ValueError, match="inline_size_threshold must be non-negative"):
+        lancedb.blob("payload", inline_size_threshold=-1)
+    with pytest.raises(ValueError, match="dedicated_size_threshold must be positive"):
+        lancedb.blob("payload", dedicated_size_threshold=0)
+    with pytest.raises(ValueError, match="pack_file_size_threshold must be positive"):
+        lancedb.blob("payload", pack_file_size_threshold=0)
+    with pytest.raises(TypeError, match="inline_size_threshold must be an int"):
+        lancedb.blob("payload", inline_size_threshold="8192")
+
+
 def test_blob_type_works_without_pylance():
     script = _HIDE_LANCE_BLOB + textwrap.dedent(
         """\

rust/lancedb/src/blob.rs

@@ -262,6 +262,37 @@ pub fn blob(name: impl AsRef<str>, nullable: bool) -> Field {
     lance::blob::blob_field(name.as_ref(), nullable)
 }
 
+/// Options for creating a blob v2 column field with tier thresholds.
+///
+/// Re-exported from lance so the thresholds are built exactly the way
+/// [`lance::blob::blob_field_with_options`] builds them. Note that lance does
+/// not currently expose a pack file size threshold option, so only the inline
+/// and dedicated size thresholds are supported here.
+pub use lance::blob::BlobFieldOptions as BlobOptions;
+
+/// Creates an Arrow field for a Lance blob v2 column with tier thresholds.
+///
+/// Same as [`blob`], but forwards `options` to
+/// [`lance::blob::blob_field_with_options`], storing the inline and dedicated
+/// size thresholds as field metadata. With [`BlobOptions::default()`] this is
+/// identical to [`blob`].
+///
+/// ```
+/// use std::num::NonZeroUsize;
+///
+/// use lancedb::blob::BlobOptions;
+///
+/// let options = BlobOptions {
+///     inline_size_threshold: Some(8 * 1024),
+///     dedicated_size_threshold: NonZeroUsize::new(1 << 20),
+/// };
+/// let field = lancedb::blob::blob_with_options("payload", true, options);
+/// assert!(lancedb::blob::is_blob(&field));
+/// ```
+pub fn blob_with_options(name: impl AsRef<str>, nullable: bool, options: BlobOptions) -> Field {
+    lance::blob::blob_field_with_options(name.as_ref(), nullable, options)
+}
+
 /// Returns true if `field` is a blob v2 column.
 ///
 /// ```
@@ -501,8 +532,13 @@ pub(crate) async fn take_blob_files_aligned(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::num::NonZeroUsize;
+
     use arrow_schema::DataType;
-    use lance_arrow::ARROW_EXT_NAME_KEY;
+    use lance_arrow::{
+        ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY,
+        BLOB_INLINE_SIZE_THRESHOLD_META_KEY,
+    };
 
     fn blob_schema() -> Schema {
         Schema::new(vec![
@@ -521,6 +557,38 @@ mod tests {
         assert!(matches!(field.data_type(), DataType::Struct(_)));
     }
 
+    #[test]
+    fn blob_with_options_sets_threshold_metadata() {
+        let options = BlobOptions {
+            inline_size_threshold: Some(8 * 1024),
+            dedicated_size_threshold: NonZeroUsize::new(1 << 20),
+        };
+        let field = blob_with_options("payload", true, options);
+        assert!(is_blob(&field));
+        assert_eq!(
+            field
+                .metadata()
+                .get(BLOB_INLINE_SIZE_THRESHOLD_META_KEY)
+                .map(String::as_str),
+            Some("8192")
+        );
+        assert_eq!(
+            field
+                .metadata()
+                .get(BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY)
+                .map(String::as_str),
+            Some("1048576")
+        );
+    }
+
+    #[test]
+    fn blob_with_default_options_matches_plain_blob() {
+        let field = blob_with_options("payload", true, BlobOptions::default());
+        let plain = blob("payload", true);
+        assert_eq!(field.metadata(), plain.metadata());
+        assert_eq!(field.data_type(), plain.data_type());
+    }
+
     #[test]
     fn has_blob_columns_detects_blob_fields() {
         assert!(has_blob_columns(&blob_schema()));