mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-19 04:28:35 +00:00
feat(python): blob v2 fetch API (#3578)
Python bindings for blob v2 read on **local** tables. Rust read APIs landed in #3562. This PR wires `fetch_blob_files`, `fetch_blobs`, v2 query/`to_pandas(blob_mode="bytes")`, and hidden `_rowid` metadata so `fetch_*` works from query hits without exposing `_rowid` in the column list. **Cloud:** `RemoteTable.fetch_blobs` / `fetch_blob_files` raise `NotImplementedError` until Phalanx ships the server route (separate track; not blocking local merge). ### Primary path: lazy file handles ```python table = db.create_table("videos", schema=pa.schema([ pa.field("id", pa.int64()), lancedb.blob("video"), ])) table.add([{"id": 1, "video": open("clip.mp4", "rb").read()}]) hits = table.search().select(["id", "video"]).to_arrow() handle = table.fetch_blob_files("video", hits)[0] # seek + partial read — PyAV / decoders can use the handle handle.seek(frame_offset) chunk = handle.read_range(0, 65536) ``` `BlobFile` exposes `seek`, `read`, `read_range`, `read_up_to`, and works with `BufferedReader`. ### When you want full bytes ```python blobs = table.fetch_blobs("video", hits) # eager materialize, null-aligned df = table.to_pandas(blob_mode="bytes") # descriptors → bytes in pandas ``` ### `_rowid` (join key, not user `id`) Fetch needs Lance row ids. For v2 blob queries we auto-inject `_rowid`, stash it in Arrow schema metadata on `to_arrow()`, and drop the visible column unless you pass `.with_row_id(True)`. v1 legacy blobs (`lance-encoding:blob`) unchanged; fetch on v1 raises the migration error. ## Test plan - [x] `./scripts/test-blob.sh python` (105 passed in worktree) - [x] `fetch_blob_files` lazy read, seek, partial read, null alignment, cross-fragment dups - [x] hybrid query → `fetch_blobs` / `fetch_blob_files` - [ ] Will re-review after seek/`BlobFile` commit (`d77ab1a6`) --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -45,6 +45,32 @@ def _blob_test_data():
|
||||
)
|
||||
|
||||
|
||||
def _blob_v2_table(db: DBConnection, name: str):
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")])
|
||||
table = db.create_table(name, schema=schema)
|
||||
table.add([{"id": 1, "blob": b"hello"}, {"id": 2, "blob": b"world"}])
|
||||
return table
|
||||
|
||||
|
||||
async def _blob_v2_table_async(db: AsyncConnection, name: str):
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")])
|
||||
table = await db.create_table(name, schema=schema)
|
||||
await table.add([{"id": 1, "blob": b"hello"}, {"id": 2, "blob": b"world"}])
|
||||
return table
|
||||
|
||||
|
||||
def _blob_table(db: DBConnection, name: str, blob_schema: str):
|
||||
if blob_schema == "v1":
|
||||
return db.create_table(name, data=_blob_test_data())
|
||||
return _blob_v2_table(db, name)
|
||||
|
||||
|
||||
async def _blob_table_async(db: AsyncConnection, name: str, blob_schema: str):
|
||||
if blob_schema == "v1":
|
||||
return await db.create_table(name, data=_blob_test_data())
|
||||
return await _blob_v2_table_async(db, name)
|
||||
|
||||
|
||||
def _assert_lazy_blob(value, expected: bytes):
|
||||
assert hasattr(value, "readall")
|
||||
assert value.readall() == expected
|
||||
@@ -107,6 +133,18 @@ def test_table_to_pandas_blob_modes(tmp_db: DBConnection, blob_mode):
|
||||
assert not hasattr(first, "readall")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("blob_schema", ["v1", "v2"])
|
||||
def test_table_to_pandas_blob_bytes(tmp_db: DBConnection, blob_schema):
|
||||
pytest.importorskip("lance")
|
||||
table = _blob_table(tmp_db, f"test_to_pandas_blob_{blob_schema}_bytes", blob_schema)
|
||||
|
||||
df = table.to_pandas(blob_mode="bytes")
|
||||
|
||||
assert list(df.columns) == ["id", "blob"]
|
||||
assert df["blob"].tolist() == [b"hello", b"world"]
|
||||
assert "_rowid" not in df.columns
|
||||
|
||||
|
||||
def test_table_to_pandas_kwargs(tmp_db: DBConnection):
|
||||
pd = pytest.importorskip("pandas")
|
||||
data = pa.table({"id": pa.array([1, 2], pa.int64())})
|
||||
@@ -118,15 +156,20 @@ def test_table_to_pandas_kwargs(tmp_db: DBConnection):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_table_to_pandas_blob_bytes(tmp_db_async: AsyncConnection):
|
||||
@pytest.mark.parametrize("blob_schema", ["v1", "v2"])
|
||||
async def test_async_table_to_pandas_blob_bytes(
|
||||
tmp_db_async: AsyncConnection, blob_schema
|
||||
):
|
||||
pytest.importorskip("lance")
|
||||
table = await tmp_db_async.create_table(
|
||||
"test_async_to_pandas_blob_bytes", data=_blob_test_data()
|
||||
table = await _blob_table_async(
|
||||
tmp_db_async, f"test_async_to_pandas_blob_{blob_schema}_bytes", blob_schema
|
||||
)
|
||||
|
||||
df = await table.to_pandas(blob_mode="bytes")
|
||||
|
||||
assert list(df.columns) == ["id", "blob"]
|
||||
assert df["blob"].tolist() == [b"hello", b"world"]
|
||||
assert "_rowid" not in df.columns
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Reference in New Issue
Block a user