feat(python): blob v2 fetch API (#3578)

Python bindings for blob v2 read on **local** tables. Rust read APIs
landed in #3562.

This PR wires `fetch_blob_files`, `fetch_blobs`, v2
query/`to_pandas(blob_mode="bytes")`, and hidden `_rowid` metadata so
`fetch_*` works from query hits without exposing `_rowid` in the column
list.

**Cloud:** `RemoteTable.fetch_blobs` / `fetch_blob_files` raise
`NotImplementedError` until Phalanx ships the server route (separate
track; not blocking local merge).

### Primary path: lazy file handles

```python
table = db.create_table("videos", schema=pa.schema([
    pa.field("id", pa.int64()),
    lancedb.blob("video"),
]))
table.add([{"id": 1, "video": open("clip.mp4", "rb").read()}])

hits = table.search().select(["id", "video"]).to_arrow()
handle = table.fetch_blob_files("video", hits)[0]

# seek + partial read — PyAV / decoders can use the handle
handle.seek(frame_offset)
chunk = handle.read_range(0, 65536)
```

`BlobFile` exposes `seek`, `read`, `read_range`, `read_up_to`, and works
with `BufferedReader`.

### When you want full bytes

```python
blobs = table.fetch_blobs("video", hits)  # eager materialize, null-aligned
df = table.to_pandas(blob_mode="bytes")   # descriptors → bytes in pandas
```

### `_rowid` (join key, not user `id`)

Fetch needs Lance row ids. For v2 blob queries we auto-inject `_rowid`,
stash it in Arrow schema metadata on `to_arrow()`, and drop the visible
column unless you pass `.with_row_id(True)`.

v1 legacy blobs (`lance-encoding:blob`) unchanged; fetch on v1 raises
the migration error.

## Test plan

- [x] `./scripts/test-blob.sh python` (105 passed in worktree)
- [x] `fetch_blob_files` lazy read, seek, partial read, null alignment,
cross-fragment dups
- [x] hybrid query → `fetch_blobs` / `fetch_blob_files`
- [ ] Will re-review after seek/`BlobFile` commit (`d77ab1a6`)

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Drew Gallardo
2026-07-10 12:54:16 -07:00
committed by GitHub
parent 104fc5a08e
commit a548e59d49
13 changed files with 1833 additions and 119 deletions
+46 -3
View File
@@ -45,6 +45,32 @@ def _blob_test_data():
)
def _blob_v2_table(db: DBConnection, name: str):
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")])
table = db.create_table(name, schema=schema)
table.add([{"id": 1, "blob": b"hello"}, {"id": 2, "blob": b"world"}])
return table
async def _blob_v2_table_async(db: AsyncConnection, name: str):
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")])
table = await db.create_table(name, schema=schema)
await table.add([{"id": 1, "blob": b"hello"}, {"id": 2, "blob": b"world"}])
return table
def _blob_table(db: DBConnection, name: str, blob_schema: str):
if blob_schema == "v1":
return db.create_table(name, data=_blob_test_data())
return _blob_v2_table(db, name)
async def _blob_table_async(db: AsyncConnection, name: str, blob_schema: str):
if blob_schema == "v1":
return await db.create_table(name, data=_blob_test_data())
return await _blob_v2_table_async(db, name)
def _assert_lazy_blob(value, expected: bytes):
assert hasattr(value, "readall")
assert value.readall() == expected
@@ -107,6 +133,18 @@ def test_table_to_pandas_blob_modes(tmp_db: DBConnection, blob_mode):
assert not hasattr(first, "readall")
@pytest.mark.parametrize("blob_schema", ["v1", "v2"])
def test_table_to_pandas_blob_bytes(tmp_db: DBConnection, blob_schema):
pytest.importorskip("lance")
table = _blob_table(tmp_db, f"test_to_pandas_blob_{blob_schema}_bytes", blob_schema)
df = table.to_pandas(blob_mode="bytes")
assert list(df.columns) == ["id", "blob"]
assert df["blob"].tolist() == [b"hello", b"world"]
assert "_rowid" not in df.columns
def test_table_to_pandas_kwargs(tmp_db: DBConnection):
pd = pytest.importorskip("pandas")
data = pa.table({"id": pa.array([1, 2], pa.int64())})
@@ -118,15 +156,20 @@ def test_table_to_pandas_kwargs(tmp_db: DBConnection):
@pytest.mark.asyncio
async def test_async_table_to_pandas_blob_bytes(tmp_db_async: AsyncConnection):
@pytest.mark.parametrize("blob_schema", ["v1", "v2"])
async def test_async_table_to_pandas_blob_bytes(
tmp_db_async: AsyncConnection, blob_schema
):
pytest.importorskip("lance")
table = await tmp_db_async.create_table(
"test_async_to_pandas_blob_bytes", data=_blob_test_data()
table = await _blob_table_async(
tmp_db_async, f"test_async_to_pandas_blob_{blob_schema}_bytes", blob_schema
)
df = await table.to_pandas(blob_mode="bytes")
assert list(df.columns) == ["id", "blob"]
assert df["blob"].tolist() == [b"hello", b"world"]
assert "_rowid" not in df.columns
@pytest.mark.asyncio