From dac51e27162c166bfc3f9fd0258108e6881a35b5 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Wed, 14 Oct 2026 12:39:38 -0700 Subject: [PATCH] docs: match the blob guidance to the released Python SDK lancedb 0.40.0, the latest release, does not include #4529: its query builders' to_arrow() and to_list() take no blob_mode, and a vector, FTS, hybrid or ordered query raises when its projection includes a blob column. Restore the earlier guidance and example, with the snippet regenerated from the restored test. On 0.40.0 "descriptions" works on in-memory tables and "bytes" does not, so the restored note names only "bytes". --- docs/web-tests/py/test_multimodal.py | 20 +++++--------------- docs/web/management/table.mdx | 9 ++++----- docs/web/snippets/multimodal.mdx | 2 +- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/docs/web-tests/py/test_multimodal.py b/docs/web-tests/py/test_multimodal.py index bf7ec1bc8..cb4a647c3 100644 --- a/docs/web-tests/py/test_multimodal.py +++ b/docs/web-tests/py/test_multimodal.py @@ -215,28 +215,18 @@ async def test_query_to_pandas_kwargs(db_path_factory): .to_pandas(blob_mode="bytes") ) - # Vector / FTS / hybrid queries return "bytes" or "descriptions". - # Lazy BlobFile handles need a plain scan. + # Vector / FTS / hybrid queries can't materialize blob columns, + # so omit them from the projection df_vec = ( tbl.search(query_vector) .limit(10) - .select(["id", "video"]) - .to_pandas(blob_mode="bytes", split_blocks=True, self_destruct=True) + .select(["id", "vector"]) + .to_pandas(split_blocks=True, self_destruct=True) ) - - # to_arrow and to_list take the same blob_mode - hits = tbl.search(query_vector).limit(2).select(["id", "video"]) - arrow_bytes = hits.to_arrow(blob_mode="bytes") - rows = hits.to_list(blob_mode="bytes") # --8<-- [end:query_to_pandas_kwargs] assert len(df_lazy) == 1 assert len(df_bytes) == 1 assert df_bytes["video"].iloc[0] == b"fake_video_bytes_1" assert len(df_vec) == 10 - assert all( - video == f"fake_video_bytes_{i}".encode() - for i, video in zip(df_vec["id"], df_vec["video"]) - ) - assert arrow_bytes["video"].to_pylist() == [row["video"] for row in rows] - assert all(isinstance(row["video"], bytes) for row in rows) + assert "video" not in df_vec.columns diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index 8826f59fe..0087f101e 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -480,6 +480,8 @@ When you call `to_pandas()` on a local LanceDB table that contains Blob API colu - `"bytes"`: eagerly materializes each blob as `bytes`. Use this when you need the raw payload in the DataFrame, for example to decode an image or audio clip in-process. - `"descriptions"`: returns blob descriptors (offsets, sizes, and positions) instead of the data itself. Use this when you want to plan I/O without paying the cost of loading every blob. +`"bytes"` is not supported on in-memory tables. + Extra keyword arguments are forwarded to the underlying PyArrow / Lance pandas conversion, so you can also pass options like `split_blocks` or `self_destruct`: @@ -488,13 +490,10 @@ Extra keyword arguments are forwarded to the underlying PyArrow / Lance pandas c -Query builders also accept `blob_mode` on `to_pandas()`, `to_arrow()`, and `to_list()`: +Query builders also accept `blob_mode` on their `to_pandas()` method: - Plain scans support `"lazy"`, `"bytes"`, and `"descriptions"` with filters, projections, aliases, `limit`, and `offset`. -- Vector, FTS, hybrid, and take queries support `"bytes"` and `"descriptions"`. Lazy `BlobFile` handles need a plain scan. -- In `to_arrow()` and `to_list()`, `"lazy"` returns blob descriptors that carry the row id `fetch_blobs` reads, and `"descriptions"` returns them without it. -- A plain scan's `to_pandas()` without `timeout` reads the local dataset through Lance's scanner, so it also returns `"bytes"` for legacy blob columns and under namespace query pushdown. In-memory tables have no local dataset, so they take the `to_arrow()` route. -- Every other `"bytes"` route raises `ValueError` for legacy blob columns and blobs inside lists, and `NotImplementedError` on remote tables and namespace query pushdown, where the blob fetch could read a different table version than the query. +- Vector, FTS, hybrid, and ordered queries can't materialize blob columns through `to_pandas()`; omit blob columns from the projection for those query shapes. - This works on both sync and async query builders. Extra PyArrow kwargs like `split_blocks` and `self_destruct` are still forwarded. diff --git a/docs/web/snippets/multimodal.mdx b/docs/web/snippets/multimodal.mdx index 90e275db7..f41af6629 100644 --- a/docs/web/snippets/multimodal.mdx +++ b/docs/web/snippets/multimodal.mdx @@ -16,7 +16,7 @@ export const PyMultimodalImports = "import lancedb\nimport pyarrow as pa\nimport export const PyProcessResults = "# Convert back to PIL Image\nfor _, row in results.iterrows():\n image_bytes = row['image_blob']\n image = Image.open(io.BytesIO(image_bytes))\n print(f\"Retrieved image: {row['filename']}, Size: {image.size}\")\n # You can now use 'image' with other libraries or display it\n"; -export const PyQueryToPandasKwargs = "# Plain scan query: blob_mode is supported end to end\ndf_lazy = (\n tbl.search()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"lazy\")\n)\n\n# Same call shape works on async query builders\ndf_bytes = await (\n tbl_async.query()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\")\n)\n\n# Vector / FTS / hybrid queries return \"bytes\" or \"descriptions\".\n# Lazy BlobFile handles need a plain scan.\ndf_vec = (\n tbl.search(query_vector)\n .limit(10)\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\", split_blocks=True, self_destruct=True)\n)\n\n# to_arrow and to_list take the same blob_mode\nhits = tbl.search(query_vector).limit(2).select([\"id\", \"video\"])\narrow_bytes = hits.to_arrow(blob_mode=\"bytes\")\nrows = hits.to_list(blob_mode=\"bytes\")\n"; +export const PyQueryToPandasKwargs = "# Plain scan query: blob_mode is supported end to end\ndf_lazy = (\n tbl.search()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"lazy\")\n)\n\n# Same call shape works on async query builders\ndf_bytes = await (\n tbl_async.query()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\")\n)\n\n# Vector / FTS / hybrid queries can't materialize blob columns,\n# so omit them from the projection\ndf_vec = (\n tbl.search(query_vector)\n .limit(10)\n .select([\"id\", \"vector\"])\n .to_pandas(split_blocks=True, self_destruct=True)\n)\n"; export const PySearchData = "# Search for similar images\nquery_vector = np.random.rand(128).astype(np.float32)\nresults = tbl.search(query_vector).limit(1).to_pandas()\n";