feat(remote): add RemoteTable fetch_blobs HTTP client (#3684)

Remote half of the blob read path. #3578 did local Python. This makes
`RemoteTable` hit the server.

- `fetch_blobs(column, row_ids or hits)` → bytes over `POST
/v1/table/{id}/fetch_blobs/`
- `blob_columns()` from the cached schema (describe already has the
metadata, no extra route)
- search then `fetch_blobs` works. row identity rides inside the blob
descriptor so you do not need a public `_rowid`
- `fetch_blob_files` still `NotSupported` on remote. use `fetch_blobs`
for full bytes for now. Range is a follow up

Accepts Binary / LargeBinary / BinaryView on the way back. Empty
`row_ids` short-circuits. Version + branch go in the request body same
as other read calls.

### Example

```python
db = lancedb.connect(uri="db://my-project", api_key=...)
table = db.open_table("clips")

hits = table.search(query_vec).select(["id", "video"]).limit(10).to_arrow()
# hits is just id + video. row ids are stashed on the descriptor
blobs = table.fetch_blobs("video", hits)  # null-aligned, same length as hits
```

Or pass ids yourself:

```python
blobs = table.fetch_blobs("video", [10, 20, 30])
```

### Testing

- `cargo test -p lancedb --features remote --lib`
- `cargo test -p lancedb --features remote --test blob_integration`
- `pytest python/tests/test_remote_db.py -k remote_blob`
- live e2e against a local 0.5.0 remote server (search → fetch, nulls,
nested path, old server gate)

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Drew Gallardo
2026-07-30 12:16:01 -07:00
committed by GitHub
parent b505dc1315
commit 77208fd464
9 changed files with 999 additions and 65 deletions
+181 -1
View File
@@ -1020,7 +1020,7 @@ def query_test_table(query_handler, *, server_version=Version("0.1.0")):
request.send_header("Content-Type", "application/json")
request.send_header("phalanx-version", str(server_version))
request.end_headers()
request.wfile.write(b"{}")
request.wfile.write(b'{"version": 1, "schema": {"fields": []}}')
elif request.path == "/v1/table/test/query/":
content_len = int(request.headers.get("Content-Length"))
body = request.rfile.read(content_len)
@@ -1858,3 +1858,183 @@ def test_inherited_remote_table_reopens_after_fork():
finally:
server.shutdown()
server_thread.join()
BLOB_DESCRIBE_RESPONSE = {
"table": "test",
"version": 1,
"schema": {
"fields": [
{"name": "id", "type": {"type": "int64"}, "nullable": False},
{
"name": "image",
"type": {
"type": "struct",
"fields": [
{
"name": "data",
"type": {"type": "large_binary"},
"nullable": True,
},
{"name": "uri", "type": {"type": "string"}, "nullable": True},
],
},
"nullable": True,
"metadata": {
"ARROW:extension:name": "lance.blob.v2",
"ARROW:extension:metadata": "",
},
},
]
},
}
def blob_query_response_table():
image_field = pa.field(
"image",
pa.struct(
[
pa.field("kind", pa.uint8(), nullable=False),
pa.field("position", pa.uint64(), nullable=False),
pa.field("size", pa.uint64(), nullable=False),
pa.field("blob_id", pa.uint32(), nullable=False),
pa.field("blob_uri", pa.string(), nullable=False),
]
),
metadata={"lance-encoding:blob": "true"},
)
images = pa.StructArray.from_arrays(
[
pa.array([1, 0, 0], type=pa.uint8()),
pa.array([0, 0, 0], type=pa.uint64()),
pa.array([5, 0, 5], type=pa.uint64()),
pa.array([1, 0, 2], type=pa.uint32()),
pa.array(["", "", ""], type=pa.string()),
],
fields=image_field.type,
mask=pa.array([False, True, False]),
)
return pa.Table.from_arrays(
[
pa.array([1, 2, 3], type=pa.int64()),
images,
pa.array([10, 20, 30], type=pa.uint64()),
],
schema=pa.schema(
[
pa.field("id", pa.int64(), nullable=False),
image_field,
pa.field("_rowid", pa.uint64()),
]
),
)
@contextlib.contextmanager
def blob_remote_table(*, server_version=Version("0.5.0")):
def handler(request):
if request.path == "/v1/table/test/describe/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.send_header("phalanx-version", str(server_version))
request.end_headers()
request.wfile.write(json.dumps(BLOB_DESCRIBE_RESPONSE).encode())
elif request.path == "/v1/table/test/query/":
content_len = int(request.headers.get("Content-Length", 0))
body = json.loads(request.rfile.read(content_len))
assert body["columns"] == ["id", "image"]
assert body["with_row_id"] is True
response_table = blob_query_response_table()
request.send_response(200)
request.send_header("Content-Type", "application/vnd.apache.arrow.file")
request.end_headers()
with pa.ipc.new_file(request.wfile, response_table.schema) as writer:
writer.write_table(response_table)
elif request.path == "/v1/table/test/fetch_blobs/":
content_len = int(request.headers.get("Content-Length", 0))
body = json.loads(request.rfile.read(content_len))
assert body["column"] == "image"
assert body["row_ids"] == [10, 20, 30]
response_table = pa.table(
{"image": pa.array([b"alpha", None, b"gamma"], type=pa.large_binary())}
)
request.send_response(200)
request.send_header("Content-Type", "application/vnd.apache.arrow.stream")
request.end_headers()
with pa.ipc.new_stream(request.wfile, response_table.schema) as writer:
writer.write_table(response_table)
else:
request.send_response(404)
request.end_headers()
with mock_lancedb_connection(handler) as db:
yield db.open_table("test")
def test_remote_blob_columns_and_fetch():
with blob_remote_table() as table:
assert table.blob_columns() == ["image"]
blobs = table.fetch_blobs("image", [10, 20, 30])
assert blobs.to_pylist() == [b"alpha", None, b"gamma"]
with pytest.raises(NotImplementedError, match="Use fetch_blobs for full bytes"):
table.fetch_blob_files("image", [10, 20, 30])
def test_remote_blob_fetch_accepts_query_table():
hits = pa.table({"_rowid": pa.array([10, 20, 30], type=pa.uint64())})
with blob_remote_table() as table:
blobs = table.fetch_blobs("image", hits)
assert blobs.to_pylist() == [b"alpha", None, b"gamma"]
def test_remote_blob_query_stashes_row_ids_for_fetch():
with blob_remote_table() as table:
hits = table.search().select(["id", "image"]).limit(3).to_arrow()
assert "_rowid" not in hits.column_names
assert "_lance_row_id" in hits.schema.field("image").type.names
blobs = table.fetch_blobs("image", hits)
assert blobs.to_pylist() == [b"alpha", None, b"gamma"]
def test_remote_blob_query_survives_a_server_that_ignores_the_row_id_request():
def handler(request):
if request.path == "/v1/table/test/describe/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.send_header("phalanx-version", "0.5.0")
request.end_headers()
request.wfile.write(json.dumps(BLOB_DESCRIBE_RESPONSE).encode())
elif request.path == "/v1/table/test/query/":
content_len = int(request.headers.get("Content-Length", 0))
assert json.loads(request.rfile.read(content_len))["with_row_id"] is True
response_table = blob_query_response_table().drop_columns(["_rowid"])
request.send_response(200)
request.send_header("Content-Type", "application/vnd.apache.arrow.file")
request.end_headers()
with pa.ipc.new_file(request.wfile, response_table.schema) as writer:
writer.write_table(response_table)
else:
request.send_response(404)
request.end_headers()
with mock_lancedb_connection(handler) as db:
table = db.open_table("test")
hits = table.search().select(["id", "image"]).limit(3).to_arrow()
assert hits.column_names == ["id", "image"]
assert "_lance_row_id" not in hits.schema.field("image").type.names
with pytest.raises(ValueError, match="pass a list of row ids"):
table.fetch_blobs("image", hits)
def test_remote_blob_byte_apis_not_supported_on_old_server():
with blob_remote_table(server_version=Version("0.1.0")) as table:
assert table.blob_columns() == ["image"]
with pytest.raises(NotImplementedError, match="not supported"):
table.fetch_blobs("image", [1])
with pytest.raises(NotImplementedError, match="not supported"):
table.fetch_blob_files("image", [1])