From 0acb61471c848277097e56bb525e04e71b970bbc Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:24:29 -0700 Subject: [PATCH] fix(python): route async remote blob reads through server (#4306) ## Cause Async remote pandas conversion could call `AsyncTable.to_lance()`, which opened the server's table storage URI in the client process. This bypassed server query and authorization APIs. The first repair routed blob reads through the server, but a query and subsequent blob fetch could resolve different table snapshots after replacement or compaction. ## Fix - Reject `AsyncTable.to_lance()` for remote tables before requesting a table URI. - Convert remote nonblob and blob description results through the server Arrow path. - Reject automatic remote byte and lazy blob materialization until the server can bind the query and fetches to one verified table identity and version. The default lazy mode now raises `NotImplementedError` when a blob column is projected; callers can request `blob_mode="descriptions"`. ## Validation - Mocked remote tests cover direct storage rejection, description results, byte and lazy rejection before any blob fetch, and nonblob projections. - Focused local and remote blob tests passed (19 tests). `ruff check .`, `ruff format .`, and pyright passed. Fixes #4272 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/lancedb/_lancedb.pyi | 1 + python/python/lancedb/query.py | 38 +++++++++-- python/python/lancedb/table.py | 18 ++++- python/python/tests/test_remote_db.py | 98 +++++++++++++++++++++++++-- python/src/table.rs | 5 ++ 5 files changed, 147 insertions(+), 13 deletions(-) diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index af964d459..a8f02e458 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -423,6 +423,7 @@ class Table: def name(self) -> str: ... def __repr__(self) -> str: ... def is_open(self) -> bool: ... + def _is_native(self) -> bool: ... def close(self) -> None: ... async def schema(self) -> pa.Schema: ... async def add( diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index d9755afa1..6f68692a2 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -39,7 +39,7 @@ from .expr import Expr from .rerankers.base import Reranker from .rerankers.rrf import RRFReranker from .rerankers.util import check_reranker_result -from .schema import is_blob_like_field, schema_has_blob_field +from .schema import blob_column_paths, is_blob_like_field, schema_has_blob_field from .util import _validate_query_vector, flatten_columns from . import _wal_hybrid # WAL-PK-FUSION: delete. from ._blob import ( @@ -51,6 +51,7 @@ from ._blob import ( finalize_blob_query_table, replace_v2_blob_columns_with_bytes, replace_v2_blob_columns_with_bytes_sync, + strip_auto_row_ids, validate_blob_mode, ) from .types import BlobMode, QueryProjection @@ -3073,14 +3074,18 @@ class AsyncQueryBase(object): If not specified, no timeout is applied. If the query does not complete within the specified time, an error will be raised. blob_mode: str, default "lazy" - Controls how blob columns are returned for plain scan queries. - Vector, FTS, hybrid, and other non-native query shapes keep the - existing Arrow conversion path and only support blob descriptions. + Controls how blob columns are returned. Remote queries support + "descriptions"; "bytes" and "lazy" are not yet supported. **kwargs Forwarded to pyarrow.Table.to_pandas after query execution and optional flattening. """ validate_blob_mode(blob_mode) + if self._table is not None and not self._table._inner._is_native(): + return await self._remote_to_pandas( + flatten=flatten, timeout=timeout, blob_mode=blob_mode, **kwargs + ) + if hasattr(self._inner, "output_schema"): schema = await self.output_schema() if _blob_mode_requires_native_pandas(blob_mode, schema): @@ -3108,6 +3113,31 @@ class AsyncQueryBase(object): ) return tbl.to_pandas(**kwargs) + async def _remote_to_pandas( + self, + *, + flatten: Optional[Union[int, bool]], + timeout: Optional[timedelta], + blob_mode: BlobMode, + **kwargs, + ) -> "pd.DataFrame": + # A live remote table can be replaced between query and blob fetch. + # Row ids and version numbers do not identify the producing table. + tbl = await self.to_arrow(timeout=timeout) + projected_blob_paths = set(blob_column_paths(tbl.schema)) + if not projected_blob_paths: + return flatten_columns(tbl, flatten).to_pandas(**kwargs) + + if blob_mode == "descriptions": + tbl = strip_auto_row_ids(tbl, self._blob_paths) + return flatten_columns(tbl, flatten).to_pandas(**kwargs) + + raise NotImplementedError( + f"remote to_pandas(blob_mode={blob_mode!r}) cannot safely materialize " + "blob columns without a stable table snapshot; " + "use blob_mode='descriptions'" + ) + async def _plain_scan_to_pandas( self, blob_mode: BlobMode, diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 186d38ad6..2970cb902 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -5602,7 +5602,11 @@ class AsyncTable: return AsyncQuery(self._inner.query(), self) async def to_lance(self, **kwargs) -> lance.LanceDataset: - """Return the Lance dataset backing this table. + """Return the Lance dataset backing a local table. + + Remote tables cannot be opened as Lance datasets by the client. Use + [to_arrow][lancedb.table.AsyncTable.to_arrow] or + [query][lancedb.table.AsyncTable.query] to read them through the server. Parameters ---------- @@ -5619,6 +5623,12 @@ class AsyncTable: >>> async def get_lance_dataset(table): ... return await table.to_lance() """ + if not self._inner._is_native(): + raise NotImplementedError( + "to_lance() is not supported for remote tables; " + "query the server instead" + ) + try: import lance except ImportError: @@ -5648,7 +5658,8 @@ class AsyncTable: Parameters ---------- blob_mode: str, default "lazy" - Controls how Lance blob columns are returned. + Controls how Lance blob columns are returned. Remote tables support + "descriptions"; "bytes" and "lazy" are not yet supported. **kwargs Forwarded to PyArrow / Lance pandas conversion. @@ -5666,6 +5677,9 @@ class AsyncTable: ) return arrow_tbl.to_pandas(**kwargs) + if not self._inner._is_native(): + return await self.query().to_pandas(blob_mode=blob_mode, **kwargs) + if blob_mode == "lazy" and get_uri_scheme(await self.uri()) == "memory": return (await self.to_arrow()).to_pandas(**kwargs) if blob_mode == "bytes" and blob_v2_column_paths(schema): diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 2494d7a80..8e4d7a7f1 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -3213,14 +3213,18 @@ def blob_remote_table(*, server_version=Version("0.5.0")): elif request.path == "/v1/table/test/query/": content_len = int(request.headers.get("Content-Length", 0)) body = json.loads(request.rfile.read(content_len)) - assert body["with_row_id"] is True - if body["columns"] == ["image"]: - # fetch_blob_files sizes its handles from a descriptor take. - assert body["filter"].startswith("_rowid IN") - response_table = blob_descriptor_take_table() + columns = body.get("columns") + assert columns in (None, ["id", "image"], ["id"], ["image"]) + if columns == ["id"]: + response_table = blob_query_response_table().select(["id"]) else: - assert body["columns"] == ["id", "image"] - response_table = blob_query_response_table() + assert body["with_row_id"] is True + if columns == ["image"]: + # fetch_blob_files sizes its handles from a descriptor take. + assert body["filter"].startswith("_rowid IN") + response_table = blob_descriptor_take_table() + else: + response_table = blob_query_response_table() request.send_response(200) request.send_header("Content-Type", "application/vnd.apache.arrow.file") request.end_headers() @@ -3254,6 +3258,86 @@ def test_remote_blob_columns_and_fetch(): assert blobs.to_pylist() == [b"alpha", None, b"gamma"] +@pytest.mark.asyncio +async def test_async_remote_to_lance_rejects_storage_access(monkeypatch): + with blob_remote_table() as remote_table: + table = remote_table._table + + async def fail_uri(): + raise AssertionError("remote table URI must not be requested") + + monkeypatch.setattr(table, "uri", fail_uri) + with pytest.raises(NotImplementedError, match="remote tables"): + await table.to_lance() + + +@pytest.mark.asyncio +async def test_async_remote_blob_descriptions_to_pandas_uses_server(monkeypatch): + with blob_remote_table() as remote_table: + table = remote_table._table + + async def fail_uri(): + raise AssertionError("remote table URI must not be requested") + + async def fail_to_lance(): + raise AssertionError("remote table storage must not be opened") + + monkeypatch.setattr(table, "uri", fail_uri) + monkeypatch.setattr(table, "to_lance", fail_to_lance) + + query_df = await ( + table.query().select(["id", "image"]).to_pandas(blob_mode="descriptions") + ) + table_df = await table.to_pandas(blob_mode="descriptions") + + for df in (query_df, table_df): + assert df["id"].tolist() == [1, 2, 3] + assert "_rowid" not in df.columns + images = df["image"].tolist() + assert images[1] is None + assert images[0]["size"] == 5 + assert "_lance_row_id" not in images[0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("blob_mode", ["bytes", "lazy"]) +async def test_async_remote_blob_pandas_rejects_unpinned_fetch(monkeypatch, blob_mode): + with blob_remote_table() as remote_table: + table = remote_table._table + + async def fail_uri(): + raise AssertionError("remote table URI must not be requested") + + async def fail_to_lance(): + raise AssertionError("remote table storage must not be opened") + + async def fail_fetch(*args, **kwargs): + raise AssertionError("blob fetch must not use an unpinned remote table") + + monkeypatch.setattr(table, "uri", fail_uri) + monkeypatch.setattr(table, "to_lance", fail_to_lance) + monkeypatch.setattr(table, "fetch_blobs", fail_fetch) + monkeypatch.setattr(table, "fetch_blob_files", fail_fetch) + + pandas_kwargs = {} if blob_mode == "lazy" else {"blob_mode": blob_mode} + with pytest.raises(NotImplementedError, match="stable table snapshot"): + await table.query().select(["id", "image"]).to_pandas(**pandas_kwargs) + with pytest.raises(NotImplementedError, match="stable table snapshot"): + await table.to_pandas(**pandas_kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("blob_mode", ["bytes", "lazy"]) +async def test_async_remote_nonblob_projection_to_pandas(blob_mode): + with blob_remote_table() as remote_table: + df = await ( + remote_table._table.query().select(["id"]).to_pandas(blob_mode=blob_mode) + ) + + assert df["id"].tolist() == [1, 2, 3] + assert list(df.columns) == ["id"] + + def test_remote_blob_files_are_lazy_seekable_handles(): with blob_remote_table() as table: files = table.fetch_blob_files("image", [10, 20, 30, 40]) diff --git a/python/src/table.rs b/python/src/table.rs index 7102a8aa7..d82ebbaaf 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -917,6 +917,11 @@ impl Table { self.inner.is_some() } + /// Whether this table has a local Lance dataset that the client may open. + pub fn _is_native(&self) -> PyResult { + Ok(self.inner_ref()?.as_native().is_some()) + } + /// Closes the table, releasing any resources associated with it. pub fn close(&mut self) { self.inner.take();