feat: accept blob URI writes (#3954)

#3528 added blob declarations and binary coercion. String values were
still rejected. They now coerce to the blob `uri` child.

```python
table.add([{"id": 1, "image": "s3://bucket/media/cat.jpg"}])

payload = table.fetch_blobs("image", table.search().to_arrow())
```

A URI under a registered base writes with no extra options. An
unregistered URI fails. `allow_external_blob_outside_bases` is a local
escape hatch that stores an absolute URI. It does not register a base.
Remote `add` rejects that flag before making a request. String input
still coerces and is sent as a `uri` struct.

`add_bases` is a follow-up. `merge_insert` does not coerce string blob
input.

### Testing

- `cargo test -p lancedb --test blob_integration`
- `cargo test -p lancedb blob_coerce`
- `cargo test -p lancedb --features remote --lib
add_rejects_external_blob_flag add_string_blob_becomes_uri_struct`
- `cd python && uv run --extra tests pytest python/tests/test_blob.py -k
uri -q`

Co-authored-by: Xuanwo <github@xuanwo.io>

python/python/lancedb/_lancedb.pyi

@@ -283,6 +283,7 @@ class Table:
         mode: Literal["append", "overwrite"],
         progress: Optional[Any] = None,
         write_parallelism: Optional[int] = None,
+        allow_external_blob_outside_bases: bool = False,
     ) -> AddResult: ...
     async def update(
         self, updates: Dict[str, str], where: Optional[str]

python/python/lancedb/remote/table.py

@@ -610,6 +610,7 @@ class RemoteTable(Table):
         fill_value: float = 0.0,
         progress: Optional[Union[bool, Callable, Any]] = None,
         write_parallelism: Optional[int] = None,
+        allow_external_blob_outside_bases: bool = False,
     ) -> AddResult:
         """Add more data to the [Table][lancedb.table.Table].
 
@@ -642,6 +643,8 @@ class RemoteTable(Table):
             data in flight. Defaults to an estimate based on the data size,
             capped at the number of CPU cores. Lower this if bulk ingestion is
             using too much memory.
+        allow_external_blob_outside_bases: bool, default False
+            Not supported on LanceDB Cloud. Setting this raises.
 
         Returns
         -------
@@ -658,6 +661,7 @@ class RemoteTable(Table):
                     fill_value=fill_value,
                     progress=progress,
                     write_parallelism=write_parallelism,
+                    allow_external_blob_outside_bases=allow_external_blob_outside_bases,
                 )
             )
         finally:

python/python/lancedb/table.py

@@ -1269,6 +1269,7 @@ class Table(ABC):
         fill_value: float = 0.0,
         progress: Optional[Union[bool, Callable, Any]] = None,
         write_parallelism: Optional[int] = None,
+        allow_external_blob_outside_bases: bool = False,
     ) -> AddResult:
         """Add more data to the [Table][lancedb.table.Table].
 
@@ -1320,6 +1321,10 @@ class Table(ABC):
             data in flight. Defaults to an estimate based on the data size,
             capped at the number of CPU cores. Lower this if bulk ingestion is
             using too much memory.
+        allow_external_blob_outside_bases: bool, default False
+            Store blob URIs that sit outside registered blob bases. The row
+            keeps a reference, so the object has to stay readable. Local
+            tables only.
 
         Returns
         -------
@@ -3409,6 +3414,7 @@ class LanceTable(Table):
         fill_value: float = 0.0,
         progress: Optional[Union[bool, Callable, Any]] = None,
         write_parallelism: Optional[int] = None,
+        allow_external_blob_outside_bases: bool = False,
     ) -> AddResult:
         """Add data to the table.
         If vector columns are missing and the table
@@ -3436,6 +3442,9 @@ class LanceTable(Table):
             data in flight. Defaults to an estimate based on the data size,
             capped at the number of CPU cores. Lower this if bulk ingestion is
             using too much memory.
+        allow_external_blob_outside_bases: bool, default False
+            Allow blob URIs outside registered bases. See :meth:`Table.add`.
+            Local tables only.
 
         Returns
         -------
@@ -3452,6 +3461,7 @@ class LanceTable(Table):
                     fill_value=fill_value,
                     progress=progress,
                     write_parallelism=write_parallelism,
+                    allow_external_blob_outside_bases=allow_external_blob_outside_bases,
                 )
             )
         finally:
@@ -5365,6 +5375,7 @@ class AsyncTable:
         fill_value: Optional[float] = None,
         progress: Optional[Union[bool, Callable, Any]] = None,
         write_parallelism: Optional[int] = None,
+        allow_external_blob_outside_bases: bool = False,
     ) -> AddResult:
         """Add more data to the [AsyncTable][lancedb.table.AsyncTable].
 
@@ -5395,6 +5406,9 @@ class AsyncTable:
             data in flight. Defaults to an estimate based on the data size,
             capped at the number of CPU cores. Lower this if bulk ingestion is
             using too much memory.
+        allow_external_blob_outside_bases: bool, default False
+            Allow blob URIs outside registered bases. See :meth:`Table.add`.
+            Local tables only.
 
         """
         schema = await self.schema()
@@ -5431,6 +5445,7 @@ class AsyncTable:
                 mode or "append",
                 progress=progress,
                 write_parallelism=write_parallelism,
+                allow_external_blob_outside_bases=allow_external_blob_outside_bases,
             )
         except RuntimeError as e:
             if "Cast error" in str(e):

python/python/tests/test_blob.py

@@ -617,3 +617,71 @@ def test_fetch_blobs_nested_path_survives_sort_after_query():
 def _identifiable_payload(size: int) -> bytes:
     block = 256
     return b"".join(bytes([i % 256]) * block for i in range(size // block))
+
+
+def _external_uri_blob_array(uris):
+    blob_type = lancedb.blob("image").type
+    storage_type = blob_type.storage_type
+    child_names = [field.name for field in storage_type]
+    assert "uri" in child_names, "blob layout no longer has a uri child"
+    children = [
+        pa.array(uris if field.name == "uri" else [None] * len(uris), type=field.type)
+        for field in storage_type
+    ]
+    storage = pa.StructArray.from_arrays(children, fields=list(storage_type))
+    return pa.ExtensionArray.from_storage(blob_type, storage)
+
+
+def _external_uri_table_and_rows(name, uris):
+    db = lancedb.connect("memory:///")
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table(name, schema=schema)
+    rows = pa.Table.from_arrays(
+        [
+            pa.array(range(len(uris)), type=pa.int64()),
+            _external_uri_blob_array(uris),
+        ],
+        schema=schema,
+    )
+    return table, rows
+
+
+def test_add_external_uri_struct_round_trips_with_flag(tmp_path):
+    payload = b"external-uri-bytes"
+    blob_path = tmp_path / "payload.bin"
+    blob_path.write_bytes(payload)
+
+    table, rows = _external_uri_table_and_rows("external_struct", [blob_path.as_uri()])
+    table.add(rows, allow_external_blob_outside_bases=True)
+
+    hits = table.search().to_arrow()
+    blobs = table.fetch_blobs("image", hits)
+    assert blobs[0].as_py() == payload
+
+
+def test_add_external_uri_without_flag_raises(tmp_path):
+    blob_path = tmp_path / "payload.bin"
+    blob_path.write_bytes(b"unreachable")
+
+    table, rows = _external_uri_table_and_rows("external_no_flag", [blob_path.as_uri()])
+    with pytest.raises(ValueError, match="allow_external_blob_outside_bases"):
+        table.add(rows)
+    assert table.count_rows() == 0
+
+
+def test_add_external_uri_string_round_trips_with_flag(tmp_path):
+    payload = b"external-uri-bytes"
+    blob_path = tmp_path / "payload.bin"
+    blob_path.write_bytes(payload)
+
+    db = lancedb.connect("memory:///")
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table("external_string", schema=schema)
+    table.add(
+        [{"id": 1, "image": blob_path.as_uri()}],
+        allow_external_blob_outside_bases=True,
+    )
+
+    hits = table.search().to_arrow()
+    blobs = table.fetch_blobs("image", hits)
+    assert blobs[0].as_py() == payload