feat(python,rust): allow lancedb.blob() to set blob tier thresholds (#4323)

python/python/lancedb/schema.py

@@ -5,7 +5,7 @@
 """Schema helpers for Lance blob columns."""
 
 import importlib
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 import pyarrow as pa
 import pyarrow.ipc
@@ -181,13 +181,60 @@ def _resolve_blob_type():
     return _resolved_blob_type
 
 
-def blob(name: str, nullable: bool = True) -> pa.Field:
+def blob(
+    name: str,
+    nullable: bool = True,
+    *,
+    inline_size_threshold: Optional[int] = None,
+    dedicated_size_threshold: Optional[int] = None,
+    pack_file_size_threshold: Optional[int] = None,
+) -> pa.Field:
     """Create a Lance blob v2 column field.
 
     When pylance is installed this is ``lance.blob.BlobType``.
+
+    Parameters
+    ----------
+    name:
+        Field name.
+    nullable:
+        Whether the column accepts null values.
+    inline_size_threshold:
+        Maximum payload size in bytes to keep inline in the data file
+        before using packed blob storage. Zero is allowed.
+    dedicated_size_threshold:
+        Maximum payload size in bytes to store in packed blob storage
+        before using dedicated blob storage. Must be positive.
+    pack_file_size_threshold:
+        Maximum size in bytes of a single packed blob sidecar file before
+        a new one is started. Must be positive.
+
+    The thresholds are forwarded to ``lance.blob_field``, which validates
+    them and stores them as field metadata, so pylance is required whenever
+    any threshold is set. When all thresholds are omitted the behavior is
+    unchanged.
     """
-    blob_type = _resolve_blob_type()
-    return pa.field(name, blob_type(), nullable=nullable)
+    if (
+        inline_size_threshold is None
+        and dedicated_size_threshold is None
+        and pack_file_size_threshold is None
+    ):
+        blob_type = _resolve_blob_type()
+        return pa.field(name, blob_type(), nullable=nullable)
+    try:
+        lance = importlib.import_module("lance")
+    except ModuleNotFoundError as err:
+        raise ValueError(
+            "Setting blob tier thresholds requires pylance to be installed"
+        ) from err
+    _resolve_blob_type()
+    return lance.blob_field(
+        name,
+        nullable=nullable,
+        inline_size_threshold=inline_size_threshold,
+        dedicated_size_threshold=dedicated_size_threshold,
+        pack_file_size_threshold=pack_file_size_threshold,
+    )
 
 
 def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType:

python/python/tests/test_blob.py

@@ -95,6 +95,37 @@ def test_blob_factory_declares_v2_field():
     assert type(field.type) is LanceBlobType
 
 
+def test_blob_thresholds_land_in_field_metadata():
+    field = lancedb.blob(
+        "payload",
+        inline_size_threshold=8 * 1024,
+        dedicated_size_threshold=1 << 20,
+        pack_file_size_threshold=1 << 26,
+    )
+    assert field.type.extension_name == "lance.blob.v2"
+    metadata = field.metadata
+    assert metadata[b"lance-encoding:blob-inline-size-threshold"] == b"8192"
+    assert metadata[b"lance-encoding:blob-dedicated-size-threshold"] == b"1048576"
+    assert metadata[b"lance-encoding:blob-pack-file-size-threshold"] == b"67108864"
+
+
+def test_blob_without_thresholds_has_no_threshold_metadata():
+    field = lancedb.blob("image")
+    metadata = field.metadata or {}
+    assert not any(b"threshold" in key for key in metadata)
+
+
+def test_blob_thresholds_are_validated():
+    with pytest.raises(ValueError, match="inline_size_threshold must be non-negative"):
+        lancedb.blob("payload", inline_size_threshold=-1)
+    with pytest.raises(ValueError, match="dedicated_size_threshold must be positive"):
+        lancedb.blob("payload", dedicated_size_threshold=0)
+    with pytest.raises(ValueError, match="pack_file_size_threshold must be positive"):
+        lancedb.blob("payload", pack_file_size_threshold=0)
+    with pytest.raises(TypeError, match="inline_size_threshold must be an int"):
+        lancedb.blob("payload", inline_size_threshold="8192")
+
+
 def test_blob_type_works_without_pylance():
     script = _HIDE_LANCE_BLOB + textwrap.dedent(
         """\