From 07e8a8db17155850469a06a464a3883efb480f90 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Fri, 9 Oct 2026 13:36:49 -0700 Subject: [PATCH] docs: fix copyable Python examples that fail as published - Full-text search: the multiline query chain was not parenthesized, so pasting it raised an IndentationError. - Pandas async: the example chained .select onto the coroutine an async search returns; it now awaits the search first, and its test asserts the result. - Pydantic: the example searched with a string, which needs a full-text index or an embedding function the table does not have; it searches with a vector, as the page describes, and asserts the model returned. - PyTorch: the column-selection example selected columns the page's table lacks and printed a schema from a batch that is a dict. It now builds its own table and prints the batch's columns. - Data evolution: the setup used the test's tmp_db fixture, an optional data argument and an unimported numpy. It now needs only `db`, which the page says how to open. --- docs/web-tests/py/test_integrations.py | 21 ++++--- docs/web-tests/py/test_tables.py | 60 ++++++++----------- .../feature-engineering/data-evolution.mdx | 3 + docs/web/search/full-text-search.mdx | 4 +- docs/web/snippets/integrations.mdx | 4 +- docs/web/snippets/tables.mdx | 2 +- docs/web/training/torch.mdx | 22 ++++++- 7 files changed, 66 insertions(+), 50 deletions(-) diff --git a/docs/web-tests/py/test_integrations.py b/docs/web-tests/py/test_integrations.py index ff07dfae0..12ef2cb88 100644 --- a/docs/web-tests/py/test_integrations.py +++ b/docs/web-tests/py/test_integrations.py @@ -1390,9 +1390,12 @@ def test_frameworks_pydantic_examples() -> None: {"text": "goodbye world", "vector": [0.0, 1.0]}, ] ) - results = table.search("hello world").limit(1).to_pydantic(LanceDocs) + results = table.search([1.0, 0.0]).limit(1).to_pydantic(LanceDocs) print(results[0].text) + # hello world # --8<-- [end:frameworks_pydantic_base_example] + assert isinstance(results[0], LanceDocs) + assert results[0].text == "hello world" # Platform integrations @@ -1686,7 +1689,7 @@ def test_platforms_pandas_examples() -> None: # --8<-- [end:platforms_pandas_vector_search] # --8<-- [start:platforms_pandas_async_example] - async def run_pandas_async_example() -> None: + async def run_pandas_async_example() -> pd.DataFrame: async_db = await lancedb.connect_async( str(Path(tempfile.mkdtemp()) / "pandas-async") ) @@ -1699,16 +1702,16 @@ def test_platforms_pandas_examples() -> None: async_table = await async_db.create_table( "creatures_async", data=async_df, mode="overwrite" ) - async_results = await ( - async_table.search([0.6, 0.4, 0.8]) - .select(["text", "_distance"]) - .limit(1) - .to_pandas() - ) + # An async search returns its query builder from an await + query = await async_table.search([0.6, 0.4, 0.8]) + async_results = await query.select(["text", "_distance"]).limit(1).to_pandas() print(async_results) + return async_results - asyncio.run(run_pandas_async_example()) + async_results = asyncio.run(run_pandas_async_example()) # --8<-- [end:platforms_pandas_async_example] + assert async_results["text"].tolist() == ["sage"] + assert async_results["_distance"].tolist() == [0.0] def test_platforms_polars_examples() -> None: diff --git a/docs/web-tests/py/test_tables.py b/docs/web-tests/py/test_tables.py index b3a456adc..d17344aec 100644 --- a/docs/web-tests/py/test_tables.py +++ b/docs/web-tests/py/test_tables.py @@ -835,31 +835,31 @@ def test_update_optimize_cleanup_snippet(tmp_db): # ============================================================================ -def _setup_schema_add_table(tmp_db, data=None): +def _setup_schema_add_table(db): # --8<-- [start:schema_add_setup] - table_name = "schema_evolution_add_example" - if data is None: - data = [ - { - "id": 1, - "name": "Laptop", - "price": 1200.00, - "vector": np.random.random(128).tolist(), - }, - { - "id": 2, - "name": "Smartphone", - "price": 800.00, - "vector": np.random.random(128).tolist(), - }, - { - "id": 3, - "name": "Headphones", - "price": 150.00, - "vector": np.random.random(128).tolist(), - }, - ] - table = tmp_db.create_table(table_name, data, mode="overwrite") + import numpy as np + + data = [ + { + "id": 1, + "name": "Laptop", + "price": 1200.00, + "vector": np.random.random(128).tolist(), + }, + { + "id": 2, + "name": "Smartphone", + "price": 800.00, + "vector": np.random.random(128).tolist(), + }, + { + "id": 3, + "name": "Headphones", + "price": 150.00, + "vector": np.random.random(128).tolist(), + }, + ] + table = db.create_table("schema_evolution_add_example", data, mode="overwrite") # --8<-- [end:schema_add_setup] return table @@ -953,17 +953,7 @@ def test_add_columns_default_values(tmp_db): def test_add_columns_nullable(tmp_db): - table = _setup_schema_add_table( - tmp_db, - data=[ - { - "id": 1, - "name": "Laptop", - "price": 1200.00, - "vector": np.random.random(128).tolist(), - } - ], - ) + table = _setup_schema_add_table(tmp_db) # --8<-- [start:add_columns_nullable] # Add a nullable timestamp column diff --git a/docs/web/feature-engineering/data-evolution.mdx b/docs/web/feature-engineering/data-evolution.mdx index 9a82a0af9..79da46859 100644 --- a/docs/web/feature-engineering/data-evolution.mdx +++ b/docs/web/feature-engineering/data-evolution.mdx @@ -38,6 +38,9 @@ In SQL, on a deployment that enables computed columns, this is [`ALTER TABLE ... ## Set up the example table {#set-up-the-example-table} +The examples use `db`, a connection opened as in +[Connect to LanceDB](/basics/connect), and create a small product table in it: + {PySchemaAddSetup} diff --git a/docs/web/search/full-text-search.mdx b/docs/web/search/full-text-search.mdx index 72b26629f..1ea3501d2 100644 --- a/docs/web/search/full-text-search.mdx +++ b/docs/web/search/full-text-search.mdx @@ -112,10 +112,12 @@ Perform full-text search and retrieve results: ```python Python icon="python" -results = table.search("puppy") +results = ( + table.search("puppy") .limit(10) .select(["text"]) .to_list() +) # [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}] ``` diff --git a/docs/web/snippets/integrations.mdx b/docs/web/snippets/integrations.mdx index 22bbde338..f4bc0a805 100644 --- a/docs/web/snippets/integrations.mdx +++ b/docs/web/snippets/integrations.mdx @@ -88,7 +88,7 @@ export const PyFrameworksLlamaindexHybridSearch = "from lancedb.rerankers import export const PyFrameworksLlamaindexQuickStart = "import logging\nimport sys\nimport textwrap\n\nimport openai\n\n# Uncomment to see debug logs\n# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)\n# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))\nfrom llama_index.core import (\n Document,\n SimpleDirectoryReader,\n StorageContext,\n VectorStoreIndex,\n)\nfrom llama_index.vector_stores.lancedb import LanceDBVectorStore\n\nopenai.api_key = \"sk-...\"\n\ndocuments = SimpleDirectoryReader(\"./data/your-data-dir/\").load_data()\nprint(\"Document ID:\", documents[0].doc_id, \"Document Hash:\", documents[0].hash)\n\n## For LanceDB Enterprise :\n# vector_store = LanceDBVectorStore(\n# uri=\"db://db_name\", # your remote DB URI\n# api_key=\"sk_..\", # lancedb enterprise api key\n# region=\"your-region\" # the region you configured\n# host_override=\"https://your-host.com\" # if you have a custom host, otherwise omit this\n# )\n\nvector_store = LanceDBVectorStore(\n uri=\"./lancedb\", mode=\"overwrite\", query_type=\"vector\"\n)\nstorage_context = StorageContext.from_defaults(vector_store=vector_store)\n\nindex = VectorStoreIndex.from_documents(documents, storage_context=storage_context)\nlance_filter = \"metadata.file_name = 'paul_graham_essay.txt' \"\nretriever = index.as_retriever(vector_store_kwargs={\"where\": lance_filter})\nresponse = retriever.retrieve(\"What did the author do growing up?\")\n"; -export const PyFrameworksPydanticBaseExample = "table = db.create_table(\"docs\", schema=LanceDocs, mode=\"overwrite\")\ntable.add(\n [\n {\"text\": \"hello world\", \"vector\": [1.0, 0.0]},\n {\"text\": \"goodbye world\", \"vector\": [0.0, 1.0]},\n ]\n)\nresults = table.search(\"hello world\").limit(1).to_pydantic(LanceDocs)\nprint(results[0].text)\n"; +export const PyFrameworksPydanticBaseExample = "table = db.create_table(\"docs\", schema=LanceDocs, mode=\"overwrite\")\ntable.add(\n [\n {\"text\": \"hello world\", \"vector\": [1.0, 0.0]},\n {\"text\": \"goodbye world\", \"vector\": [0.0, 1.0]},\n ]\n)\nresults = table.search([1.0, 0.0]).limit(1).to_pydantic(LanceDocs)\nprint(results[0].text)\n# hello world\n"; export const PyFrameworksPydanticBaseModel = "class LanceDocs(LanceModel):\n text: str\n vector: Vector(2)\n"; @@ -120,7 +120,7 @@ export const PyPlatformsDuckdbMeanPrice = "duckdb.query(\"SELECT mean(price) FRO export const PyPlatformsDuckdbQueryTable = "import duckdb\n\narrow_table = table.to_lance()\n\nduckdb.query(\"SELECT * FROM arrow_table\")\n"; -export const PyPlatformsPandasAsyncExample = "async def run_pandas_async_example() -> None:\n async_db = await lancedb.connect_async(\n str(Path(tempfile.mkdtemp()) / \"pandas-async\")\n )\n async_df = pd.DataFrame(\n [\n {\"id\": \"10\", \"text\": \"sage\", \"vector\": [0.6, 0.4, 0.8]},\n {\"id\": \"11\", \"text\": \"bard\", \"vector\": [0.2, 0.7, 0.3]},\n ]\n )\n async_table = await async_db.create_table(\n \"creatures_async\", data=async_df, mode=\"overwrite\"\n )\n async_results = await (\n async_table.search([0.6, 0.4, 0.8])\n .select([\"text\", \"_distance\"])\n .limit(1)\n .to_pandas()\n )\n print(async_results)\n\nasyncio.run(run_pandas_async_example())\n"; +export const PyPlatformsPandasAsyncExample = "async def run_pandas_async_example() -> pd.DataFrame:\n async_db = await lancedb.connect_async(\n str(Path(tempfile.mkdtemp()) / \"pandas-async\")\n )\n async_df = pd.DataFrame(\n [\n {\"id\": \"10\", \"text\": \"sage\", \"vector\": [0.6, 0.4, 0.8]},\n {\"id\": \"11\", \"text\": \"bard\", \"vector\": [0.2, 0.7, 0.3]},\n ]\n )\n async_table = await async_db.create_table(\n \"creatures_async\", data=async_df, mode=\"overwrite\"\n )\n # An async search returns its query builder from an await\n query = await async_table.search([0.6, 0.4, 0.8])\n async_results = await query.select([\"text\", \"_distance\"]).limit(1).to_pandas()\n print(async_results)\n return async_results\n\nasync_results = asyncio.run(run_pandas_async_example())\n"; export const PyPlatformsPandasCreateTable = "pandas_df = pd.DataFrame(\n [\n {\"id\": \"1\", \"text\": \"dragon\", \"vector\": [0.9, 0.1, 0.3]},\n {\"id\": \"2\", \"text\": \"griffin\", \"vector\": [0.4, 0.5, 0.2]},\n {\"id\": \"3\", \"text\": \"phoenix\", \"vector\": [0.7, 0.3, 0.6]},\n ]\n)\npandas_db = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"pandas-demo\"))\npandas_table = pandas_db.create_table(\"creatures\", data=pandas_df, mode=\"overwrite\")\n"; diff --git a/docs/web/snippets/tables.mdx b/docs/web/snippets/tables.mdx index 6b9b98280..28b9ce172 100644 --- a/docs/web/snippets/tables.mdx +++ b/docs/web/snippets/tables.mdx @@ -90,7 +90,7 @@ export const PyMergeUpdateInsert = "import pyarrow as pa\n\ntable = db.create_ta export const PyOpenExistingTable = "db = tmp_db\n# Create a table first\ndata = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7}]\ndb.create_table(\"test_table\", data, mode=\"overwrite\")\n\n# List table names\nprint(db.list_tables().tables)\n\n# Open existing table\ntbl = db.open_table(\"test_table\")\n"; -export const PySchemaAddSetup = "table_name = \"schema_evolution_add_example\"\nif data is None:\n data = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 3,\n \"name\": \"Headphones\",\n \"price\": 150.00,\n \"vector\": np.random.random(128).tolist(),\n },\n ]\ntable = tmp_db.create_table(table_name, data, mode=\"overwrite\")\n"; +export const PySchemaAddSetup = "import numpy as np\n\ndata = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 3,\n \"name\": \"Headphones\",\n \"price\": 150.00,\n \"vector\": np.random.random(128).tolist(),\n },\n]\ntable = db.create_table(\"schema_evolution_add_example\", data, mode=\"overwrite\")\n"; export const PySchemaAlterSetup = "table_name = \"schema_evolution_alter_example\"\nif data is None:\n data = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200,\n \"discount_price\": 1080.0,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800,\n \"discount_price\": 720.0,\n \"vector\": np.random.random(128).tolist(),\n },\n ]\nschema = pa.schema(\n {\n \"id\": pa.int64(),\n \"name\": pa.string(),\n \"price\": pa.int32(),\n \"discount_price\": pa.float64(),\n \"vector\": pa.list_(pa.float32(), 128),\n }\n)\ntable = tmp_db.create_table(table_name, data, schema=schema, mode=\"overwrite\")\n"; diff --git a/docs/web/training/torch.mdx b/docs/web/training/torch.mdx index 73dbe987e..c9e7888e2 100644 --- a/docs/web/training/torch.mdx +++ b/docs/web/training/torch.mdx @@ -99,17 +99,35 @@ a subset of columns, you can significantly reduce your I/O requirements by selec `Permutation` class provides a `select_columns` method that provides this functionality. ```py Python icon=Python +import lancedb +import pyarrow as pa +import torch from lancedb.permutation import Permutation -permutation = Permutation.identity(table).select_columns(["id", "prompt"]) +db = lancedb.connect("memory://") +prompts = db.create_table( + "prompts", + pa.table( + { + "id": range(4), + "prompt": ["north", "south", "east", "west"], + "embedding": [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6], [0.7, 0.8]], + } + ), +) + +permutation = Permutation.identity(prompts).select_columns(["id", "prompt"]) dataloader = torch.utils.data.DataLoader( permutation, batch_size=1024, shuffle=True ) for batch in dataloader: - print(batch.schema) + print(sorted(batch)) # ['id', 'prompt']: the embedding is never read ``` +Each batch is a dict of the selected columns, the default `Permutation` format +described above. + ## Using multiple DataLoader workers {#using-multiple-dataloader-workers} Set `num_workers > 0` to read from LanceDB in multiple PyTorch worker processes. `Permutation` objects and tables on a deployment are picklable, so each worker reopens the table after it starts. A local table is not, so wrap it in a `Permutation`, as below, to read it from several workers.