feat: support batched blob range reads (#3703)

## Summary

Lance can now plan multiple byte ranges for the same blob in one
`read_blob_ranges` operation, but LanceDB users currently cannot expose
a complete set of logical ranges to that planner.

This complements `BlobFile`: file-like consumers such as PyAV can
continue to discover ranges dynamically, while callers that already know
the ranges for a batch can submit them together.

## Motivating example

A training table may store a large video blob together with a small
application-level clip index:

```text
video: blob
clips: [{offset, length}, ...]
```

The caller can select the videos and clips for a batch, obtain their row
IDs from the query, and read all of the selected windows together:

```python
rows = (
    table.search()
    .select(["clips"])
    .with_row_id(True)
    .limit(64)
    .to_arrow()
    .to_pylist()
)

requests = []
for row in rows:
    clip = sample_clip(row["clips"])
    requests.append(
        (row["_rowid"], clip["offset"], clip["length"])
    )

chunks = table.fetch_blob_ranges("video", requests)
```

Here, `_rowid` comes from the LanceDB query, while `offset` and `length`
come from the application's clip index and are relative to that row's
video blob. The caller describes only the logical reads; Lance still
handles validation, source grouping, coalescing, scheduling, and byte
backpressure.

Lance v10.0.0-beta.5 returns one logical result per blob selector or
range request and explicitly distinguishes null blobs from valid empty
values. LanceDB consumes that aligned result contract directly and only
adds a cardinality check for unresolved row IDs.

This PR exposes batched blob-range reads on local Rust and Python
tables. Results preserve request identity, duplicates, null slots, and
valid empty ranges while allowing Lance to execute the physical reads
out of order. Scheduler buffer sizing remains an internal Lance concern,
so the LanceDB API does not expose `io_buffer_size`.

Cloud tables continue to report this operation as unsupported until
there is a corresponding remote API.
This commit is contained in:
Xuanwo
2026-07-28 06:08:47 +08:00
committed by GitHub
parent 119b9baf90
commit ff6ff09998
9 changed files with 408 additions and 153 deletions
+5
View File
@@ -297,6 +297,11 @@ class Table:
async def fetch_blobs(
self, column: str, row_ids: list[int]
) -> pa.LargeBinaryArray: ...
async def fetch_blob_ranges(
self,
column: str,
requests: List[Tuple[int, int, int]],
) -> pa.LargeBinaryArray: ...
async def fetch_blob_files(
self, column: str, row_ids: list[int]
) -> list[Optional[BlobFile]]: ...
+5
View File
@@ -1042,6 +1042,11 @@ class RemoteTable(Table):
def fetch_blobs(self, column: str, row_ids) -> pa.LargeBinaryArray:
raise NotImplementedError("fetch_blobs() is not supported on LanceDB Cloud")
def fetch_blob_ranges(self, column: str, requests) -> pa.LargeBinaryArray:
raise NotImplementedError(
"fetch_blob_ranges() is not supported on LanceDB Cloud"
)
def fetch_blob_files(self, column: str, row_ids):
raise NotImplementedError(
"fetch_blob_files() is not supported on LanceDB Cloud"
+35
View File
@@ -20,6 +20,7 @@ from typing import (
List,
Literal,
Optional,
Sequence,
Tuple,
Union,
overload,
@@ -1538,10 +1539,30 @@ class Table(ABC):
) -> pa.LargeBinaryArray:
"""Materialize full blob bytes for ``column`` at the given rows.
The result has the same length and order as ``row_ids``. Null blobs
produce null slots; valid empty blobs produce ``b""``.
Convenience for small payloads. For large values use
:meth:`fetch_blob_files`.
"""
@abstractmethod
def fetch_blob_ranges(
self,
column: str,
requests: Sequence[Tuple[int, int, int]],
) -> pa.LargeBinaryArray:
"""Materialize row-specific byte ranges from a blob v2 column.
Each request is a ``(row_id, offset, length)`` tuple. Requests may be
repeated or reordered, including multiple ranges for the same blob.
The result has the same length and order as ``requests``; null blobs
produce null slots and empty ranges on non-null blobs produce ``b""``.
Row IDs can be obtained from a query with ``with_row_id(True)``. This
API is currently supported only by local tables.
"""
@abstractmethod
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
@@ -2265,6 +2286,13 @@ class LanceTable(Table):
) -> pa.LargeBinaryArray:
return LOOP.run(self._table.fetch_blobs(column, row_ids))
def fetch_blob_ranges(
self,
column: str,
requests: Sequence[Tuple[int, int, int]],
) -> pa.LargeBinaryArray:
return LOOP.run(self._table.fetch_blob_ranges(column, list(requests)))
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
) -> "list[Optional[BlobFile]]":
@@ -5829,6 +5857,13 @@ class AsyncTable:
column, _normalize_blob_row_ids(row_ids, column)
)
async def fetch_blob_ranges(
self,
column: str,
requests: Sequence[Tuple[int, int, int]],
) -> pa.LargeBinaryArray:
return await self._inner.fetch_blob_ranges(column, list(requests))
async def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
) -> "list[Optional[BlobFile]]":
+61 -4
View File
@@ -184,18 +184,75 @@ def test_fetch_blobs_accepts_query_result():
assert {blobs[i].as_py() for i in range(len(blobs))} == {b"gamma"}
def test_fetch_blobs_null_alignment():
def test_fetch_blobs_preserves_null_and_empty_values():
table = _blob_table(
"nulls",
[{"id": 1, "image": b"present"}, {"id": 2, "image": None}],
[
{"id": 1, "image": b"present"},
{"id": 2, "image": None},
{"id": 3, "image": b""},
],
)
by_id = _row_ids_by_id(table)
request = [by_id[1], by_id[2], by_id[1]]
request = [by_id[1], by_id[2], by_id[3], by_id[1]]
blobs = table.fetch_blobs("image", request)
assert len(blobs) == len(request)
assert blobs[0].as_py() == b"present"
assert blobs[1].as_py() is None
assert blobs[2].as_py() == b"present"
assert blobs[2].as_py() == b""
assert blobs[3].as_py() == b"present"
def test_fetch_blob_ranges_aligns_repeated_ranges_and_nulls():
table = _blob_table(
"range_alignment",
[{"id": 1, "image": b"abcdefghij"}, {"id": 2, "image": None}],
)
by_id = _row_ids_by_id(table)
requests = [
(by_id[1], 2, 3),
(by_id[2], 0, 0),
(by_id[1], 0, 2),
(by_id[1], 2, 3),
(by_id[1], 10, 0),
]
ranges = table.fetch_blob_ranges("image", requests)
assert ranges.to_pylist() == [b"cde", None, b"ab", b"cde", b""]
def test_fetch_blob_ranges_validates_requests():
table = _blob_table("range_validation", [{"id": 1, "image": b"abc"}])
row_id = _row_ids_by_id(table)[1]
with pytest.raises(RuntimeError, match="exceeds blob size"):
table.fetch_blob_ranges("image", [(row_id, 2, 2)])
with pytest.raises(RuntimeError, match="offset \\+ length overflowed"):
table.fetch_blob_ranges("image", [(row_id, 2**64 - 1, 1)])
with pytest.raises(ValueError, match="row ids"):
table.fetch_blob_ranges("image", [(2**64 - 1, 0, 1)])
def test_fetch_blob_ranges_empty_requests_returns_empty_array():
table = _blob_table("range_empty", [{"id": 1, "image": b"x"}])
assert table.fetch_blob_ranges("image", []).to_pylist() == []
@pytest.mark.asyncio
async def test_async_fetch_blob_ranges():
db = await lancedb.connect_async("memory:///")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = await db.create_table("range_async", schema=schema)
await table.add([{"id": 1, "image": b"abcdefghij"}])
hits = await table.query().with_row_id().to_arrow()
row_id = hits["_rowid"][0].as_py()
ranges = await table.fetch_blob_ranges("image", [(row_id, 1, 3), (row_id, 6, 2)])
assert ranges.to_pylist() == [b"bcd", b"gh"]
def test_fetch_blobs_nested_path():