mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-19 04:28:35 +00:00
feat(remote): add RemoteTable fetch_blobs HTTP client (#3684)
Remote half of the blob read path. #3578 did local Python. This makes `RemoteTable` hit the server. - `fetch_blobs(column, row_ids or hits)` → bytes over `POST /v1/table/{id}/fetch_blobs/` - `blob_columns()` from the cached schema (describe already has the metadata, no extra route) - search then `fetch_blobs` works. row identity rides inside the blob descriptor so you do not need a public `_rowid` - `fetch_blob_files` still `NotSupported` on remote. use `fetch_blobs` for full bytes for now. Range is a follow up Accepts Binary / LargeBinary / BinaryView on the way back. Empty `row_ids` short-circuits. Version + branch go in the request body same as other read calls. ### Example ```python db = lancedb.connect(uri="db://my-project", api_key=...) table = db.open_table("clips") hits = table.search(query_vec).select(["id", "video"]).limit(10).to_arrow() # hits is just id + video. row ids are stashed on the descriptor blobs = table.fetch_blobs("video", hits) # null-aligned, same length as hits ``` Or pass ids yourself: ```python blobs = table.fetch_blobs("video", [10, 20, 30]) ``` ### Testing - `cargo test -p lancedb --features remote --lib` - `cargo test -p lancedb --features remote --test blob_integration` - `pytest python/tests/test_remote_db.py -k remote_blob` - live e2e against a local 0.5.0 remote server (search → fetch, nulls, nested path, old server gate) --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1020,7 +1020,7 @@ def query_test_table(query_handler, *, server_version=Version("0.1.0")):
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.send_header("phalanx-version", str(server_version))
|
||||
request.end_headers()
|
||||
request.wfile.write(b"{}")
|
||||
request.wfile.write(b'{"version": 1, "schema": {"fields": []}}')
|
||||
elif request.path == "/v1/table/test/query/":
|
||||
content_len = int(request.headers.get("Content-Length"))
|
||||
body = request.rfile.read(content_len)
|
||||
@@ -1858,3 +1858,183 @@ def test_inherited_remote_table_reopens_after_fork():
|
||||
finally:
|
||||
server.shutdown()
|
||||
server_thread.join()
|
||||
|
||||
|
||||
BLOB_DESCRIBE_RESPONSE = {
|
||||
"table": "test",
|
||||
"version": 1,
|
||||
"schema": {
|
||||
"fields": [
|
||||
{"name": "id", "type": {"type": "int64"}, "nullable": False},
|
||||
{
|
||||
"name": "image",
|
||||
"type": {
|
||||
"type": "struct",
|
||||
"fields": [
|
||||
{
|
||||
"name": "data",
|
||||
"type": {"type": "large_binary"},
|
||||
"nullable": True,
|
||||
},
|
||||
{"name": "uri", "type": {"type": "string"}, "nullable": True},
|
||||
],
|
||||
},
|
||||
"nullable": True,
|
||||
"metadata": {
|
||||
"ARROW:extension:name": "lance.blob.v2",
|
||||
"ARROW:extension:metadata": "",
|
||||
},
|
||||
},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def blob_query_response_table():
|
||||
image_field = pa.field(
|
||||
"image",
|
||||
pa.struct(
|
||||
[
|
||||
pa.field("kind", pa.uint8(), nullable=False),
|
||||
pa.field("position", pa.uint64(), nullable=False),
|
||||
pa.field("size", pa.uint64(), nullable=False),
|
||||
pa.field("blob_id", pa.uint32(), nullable=False),
|
||||
pa.field("blob_uri", pa.string(), nullable=False),
|
||||
]
|
||||
),
|
||||
metadata={"lance-encoding:blob": "true"},
|
||||
)
|
||||
images = pa.StructArray.from_arrays(
|
||||
[
|
||||
pa.array([1, 0, 0], type=pa.uint8()),
|
||||
pa.array([0, 0, 0], type=pa.uint64()),
|
||||
pa.array([5, 0, 5], type=pa.uint64()),
|
||||
pa.array([1, 0, 2], type=pa.uint32()),
|
||||
pa.array(["", "", ""], type=pa.string()),
|
||||
],
|
||||
fields=image_field.type,
|
||||
mask=pa.array([False, True, False]),
|
||||
)
|
||||
return pa.Table.from_arrays(
|
||||
[
|
||||
pa.array([1, 2, 3], type=pa.int64()),
|
||||
images,
|
||||
pa.array([10, 20, 30], type=pa.uint64()),
|
||||
],
|
||||
schema=pa.schema(
|
||||
[
|
||||
pa.field("id", pa.int64(), nullable=False),
|
||||
image_field,
|
||||
pa.field("_rowid", pa.uint64()),
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def blob_remote_table(*, server_version=Version("0.5.0")):
|
||||
def handler(request):
|
||||
if request.path == "/v1/table/test/describe/":
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.send_header("phalanx-version", str(server_version))
|
||||
request.end_headers()
|
||||
request.wfile.write(json.dumps(BLOB_DESCRIBE_RESPONSE).encode())
|
||||
elif request.path == "/v1/table/test/query/":
|
||||
content_len = int(request.headers.get("Content-Length", 0))
|
||||
body = json.loads(request.rfile.read(content_len))
|
||||
assert body["columns"] == ["id", "image"]
|
||||
assert body["with_row_id"] is True
|
||||
response_table = blob_query_response_table()
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/vnd.apache.arrow.file")
|
||||
request.end_headers()
|
||||
with pa.ipc.new_file(request.wfile, response_table.schema) as writer:
|
||||
writer.write_table(response_table)
|
||||
elif request.path == "/v1/table/test/fetch_blobs/":
|
||||
content_len = int(request.headers.get("Content-Length", 0))
|
||||
body = json.loads(request.rfile.read(content_len))
|
||||
assert body["column"] == "image"
|
||||
assert body["row_ids"] == [10, 20, 30]
|
||||
response_table = pa.table(
|
||||
{"image": pa.array([b"alpha", None, b"gamma"], type=pa.large_binary())}
|
||||
)
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/vnd.apache.arrow.stream")
|
||||
request.end_headers()
|
||||
with pa.ipc.new_stream(request.wfile, response_table.schema) as writer:
|
||||
writer.write_table(response_table)
|
||||
else:
|
||||
request.send_response(404)
|
||||
request.end_headers()
|
||||
|
||||
with mock_lancedb_connection(handler) as db:
|
||||
yield db.open_table("test")
|
||||
|
||||
|
||||
def test_remote_blob_columns_and_fetch():
|
||||
with blob_remote_table() as table:
|
||||
assert table.blob_columns() == ["image"]
|
||||
blobs = table.fetch_blobs("image", [10, 20, 30])
|
||||
assert blobs.to_pylist() == [b"alpha", None, b"gamma"]
|
||||
with pytest.raises(NotImplementedError, match="Use fetch_blobs for full bytes"):
|
||||
table.fetch_blob_files("image", [10, 20, 30])
|
||||
|
||||
|
||||
def test_remote_blob_fetch_accepts_query_table():
|
||||
hits = pa.table({"_rowid": pa.array([10, 20, 30], type=pa.uint64())})
|
||||
|
||||
with blob_remote_table() as table:
|
||||
blobs = table.fetch_blobs("image", hits)
|
||||
|
||||
assert blobs.to_pylist() == [b"alpha", None, b"gamma"]
|
||||
|
||||
|
||||
def test_remote_blob_query_stashes_row_ids_for_fetch():
|
||||
with blob_remote_table() as table:
|
||||
hits = table.search().select(["id", "image"]).limit(3).to_arrow()
|
||||
assert "_rowid" not in hits.column_names
|
||||
assert "_lance_row_id" in hits.schema.field("image").type.names
|
||||
blobs = table.fetch_blobs("image", hits)
|
||||
|
||||
assert blobs.to_pylist() == [b"alpha", None, b"gamma"]
|
||||
|
||||
|
||||
def test_remote_blob_query_survives_a_server_that_ignores_the_row_id_request():
|
||||
def handler(request):
|
||||
if request.path == "/v1/table/test/describe/":
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.send_header("phalanx-version", "0.5.0")
|
||||
request.end_headers()
|
||||
request.wfile.write(json.dumps(BLOB_DESCRIBE_RESPONSE).encode())
|
||||
elif request.path == "/v1/table/test/query/":
|
||||
content_len = int(request.headers.get("Content-Length", 0))
|
||||
assert json.loads(request.rfile.read(content_len))["with_row_id"] is True
|
||||
response_table = blob_query_response_table().drop_columns(["_rowid"])
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/vnd.apache.arrow.file")
|
||||
request.end_headers()
|
||||
with pa.ipc.new_file(request.wfile, response_table.schema) as writer:
|
||||
writer.write_table(response_table)
|
||||
else:
|
||||
request.send_response(404)
|
||||
request.end_headers()
|
||||
|
||||
with mock_lancedb_connection(handler) as db:
|
||||
table = db.open_table("test")
|
||||
hits = table.search().select(["id", "image"]).limit(3).to_arrow()
|
||||
|
||||
assert hits.column_names == ["id", "image"]
|
||||
assert "_lance_row_id" not in hits.schema.field("image").type.names
|
||||
with pytest.raises(ValueError, match="pass a list of row ids"):
|
||||
table.fetch_blobs("image", hits)
|
||||
|
||||
|
||||
def test_remote_blob_byte_apis_not_supported_on_old_server():
|
||||
with blob_remote_table(server_version=Version("0.1.0")) as table:
|
||||
assert table.blob_columns() == ["image"]
|
||||
with pytest.raises(NotImplementedError, match="not supported"):
|
||||
table.fetch_blobs("image", [1])
|
||||
with pytest.raises(NotImplementedError, match="not supported"):
|
||||
table.fetch_blob_files("image", [1])
|
||||
|
||||
Reference in New Issue
Block a user