mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-12 14:52:26 +00:00
docs: fix copyable Python examples that fail as published
- Full-text search: the multiline query chain was not parenthesized, so pasting it raised an IndentationError. - Pandas async: the example chained .select onto the coroutine an async search returns; it now awaits the search first, and its test asserts the result. - Pydantic: the example searched with a string, which needs a full-text index or an embedding function the table does not have; it searches with a vector, as the page describes, and asserts the model returned. - PyTorch: the column-selection example selected columns the page's table lacks and printed a schema from a batch that is a dict. It now builds its own table and prints the batch's columns. - Data evolution: the setup used the test's tmp_db fixture, an optional data argument and an unimported numpy. It now needs only `db`, which the page says how to open.
This commit is contained in:
Brendan Clement
@@ -1390,9 +1390,12 @@ def test_frameworks_pydantic_examples() -> None:
|
||||
{"text": "goodbye world", "vector": [0.0, 1.0]},
|
||||
]
|
||||
)
|
||||
results = table.search("hello world").limit(1).to_pydantic(LanceDocs)
|
||||
results = table.search([1.0, 0.0]).limit(1).to_pydantic(LanceDocs)
|
||||
print(results[0].text)
|
||||
# hello world
|
||||
# --8<-- [end:frameworks_pydantic_base_example]
|
||||
assert isinstance(results[0], LanceDocs)
|
||||
assert results[0].text == "hello world"
|
||||
|
||||
|
||||
# Platform integrations
|
||||
@@ -1686,7 +1689,7 @@ def test_platforms_pandas_examples() -> None:
|
||||
# --8<-- [end:platforms_pandas_vector_search]
|
||||
|
||||
# --8<-- [start:platforms_pandas_async_example]
|
||||
async def run_pandas_async_example() -> None:
|
||||
async def run_pandas_async_example() -> pd.DataFrame:
|
||||
async_db = await lancedb.connect_async(
|
||||
str(Path(tempfile.mkdtemp()) / "pandas-async")
|
||||
)
|
||||
@@ -1699,16 +1702,16 @@ def test_platforms_pandas_examples() -> None:
|
||||
async_table = await async_db.create_table(
|
||||
"creatures_async", data=async_df, mode="overwrite"
|
||||
)
|
||||
async_results = await (
|
||||
async_table.search([0.6, 0.4, 0.8])
|
||||
.select(["text", "_distance"])
|
||||
.limit(1)
|
||||
.to_pandas()
|
||||
)
|
||||
# An async search returns its query builder from an await
|
||||
query = await async_table.search([0.6, 0.4, 0.8])
|
||||
async_results = await query.select(["text", "_distance"]).limit(1).to_pandas()
|
||||
print(async_results)
|
||||
return async_results
|
||||
|
||||
asyncio.run(run_pandas_async_example())
|
||||
async_results = asyncio.run(run_pandas_async_example())
|
||||
# --8<-- [end:platforms_pandas_async_example]
|
||||
assert async_results["text"].tolist() == ["sage"]
|
||||
assert async_results["_distance"].tolist() == [0.0]
|
||||
|
||||
|
||||
def test_platforms_polars_examples() -> None:
|
||||
|
||||
@@ -835,31 +835,31 @@ def test_update_optimize_cleanup_snippet(tmp_db):
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _setup_schema_add_table(tmp_db, data=None):
|
||||
def _setup_schema_add_table(db):
|
||||
# --8<-- [start:schema_add_setup]
|
||||
table_name = "schema_evolution_add_example"
|
||||
if data is None:
|
||||
data = [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "Laptop",
|
||||
"price": 1200.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "Smartphone",
|
||||
"price": 800.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "Headphones",
|
||||
"price": 150.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
},
|
||||
]
|
||||
table = tmp_db.create_table(table_name, data, mode="overwrite")
|
||||
import numpy as np
|
||||
|
||||
data = [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "Laptop",
|
||||
"price": 1200.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "Smartphone",
|
||||
"price": 800.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "Headphones",
|
||||
"price": 150.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
},
|
||||
]
|
||||
table = db.create_table("schema_evolution_add_example", data, mode="overwrite")
|
||||
# --8<-- [end:schema_add_setup]
|
||||
return table
|
||||
|
||||
@@ -953,17 +953,7 @@ def test_add_columns_default_values(tmp_db):
|
||||
|
||||
|
||||
def test_add_columns_nullable(tmp_db):
|
||||
table = _setup_schema_add_table(
|
||||
tmp_db,
|
||||
data=[
|
||||
{
|
||||
"id": 1,
|
||||
"name": "Laptop",
|
||||
"price": 1200.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
}
|
||||
],
|
||||
)
|
||||
table = _setup_schema_add_table(tmp_db)
|
||||
|
||||
# --8<-- [start:add_columns_nullable]
|
||||
# Add a nullable timestamp column
|
||||
|
||||
@@ -38,6 +38,9 @@ In SQL, on a deployment that enables computed columns, this is [`ALTER TABLE ...
|
||||
|
||||
## Set up the example table {#set-up-the-example-table}
|
||||
|
||||
The examples use `db`, a connection opened as in
|
||||
[Connect to LanceDB](/basics/connect), and create a small product table in it:
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{PySchemaAddSetup}
|
||||
|
||||
@@ -112,10 +112,12 @@ Perform full-text search and retrieve results:
|
||||
|
||||
<CodeGroup>
|
||||
```python Python icon="python"
|
||||
results = table.search("puppy")
|
||||
results = (
|
||||
table.search("puppy")
|
||||
.limit(10)
|
||||
.select(["text"])
|
||||
.to_list()
|
||||
)
|
||||
# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
|
||||
```
|
||||
|
||||
|
||||
@@ -88,7 +88,7 @@ export const PyFrameworksLlamaindexHybridSearch = "from lancedb.rerankers import
|
||||
|
||||
export const PyFrameworksLlamaindexQuickStart = "import logging\nimport sys\nimport textwrap\n\nimport openai\n\n# Uncomment to see debug logs\n# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)\n# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))\nfrom llama_index.core import (\n Document,\n SimpleDirectoryReader,\n StorageContext,\n VectorStoreIndex,\n)\nfrom llama_index.vector_stores.lancedb import LanceDBVectorStore\n\nopenai.api_key = \"sk-...\"\n\ndocuments = SimpleDirectoryReader(\"./data/your-data-dir/\").load_data()\nprint(\"Document ID:\", documents[0].doc_id, \"Document Hash:\", documents[0].hash)\n\n## For LanceDB Enterprise :\n# vector_store = LanceDBVectorStore(\n# uri=\"db://db_name\", # your remote DB URI\n# api_key=\"sk_..\", # lancedb enterprise api key\n# region=\"your-region\" # the region you configured\n# host_override=\"https://your-host.com\" # if you have a custom host, otherwise omit this\n# )\n\nvector_store = LanceDBVectorStore(\n uri=\"./lancedb\", mode=\"overwrite\", query_type=\"vector\"\n)\nstorage_context = StorageContext.from_defaults(vector_store=vector_store)\n\nindex = VectorStoreIndex.from_documents(documents, storage_context=storage_context)\nlance_filter = \"metadata.file_name = 'paul_graham_essay.txt' \"\nretriever = index.as_retriever(vector_store_kwargs={\"where\": lance_filter})\nresponse = retriever.retrieve(\"What did the author do growing up?\")\n";
|
||||
|
||||
export const PyFrameworksPydanticBaseExample = "table = db.create_table(\"docs\", schema=LanceDocs, mode=\"overwrite\")\ntable.add(\n [\n {\"text\": \"hello world\", \"vector\": [1.0, 0.0]},\n {\"text\": \"goodbye world\", \"vector\": [0.0, 1.0]},\n ]\n)\nresults = table.search(\"hello world\").limit(1).to_pydantic(LanceDocs)\nprint(results[0].text)\n";
|
||||
export const PyFrameworksPydanticBaseExample = "table = db.create_table(\"docs\", schema=LanceDocs, mode=\"overwrite\")\ntable.add(\n [\n {\"text\": \"hello world\", \"vector\": [1.0, 0.0]},\n {\"text\": \"goodbye world\", \"vector\": [0.0, 1.0]},\n ]\n)\nresults = table.search([1.0, 0.0]).limit(1).to_pydantic(LanceDocs)\nprint(results[0].text)\n# hello world\n";
|
||||
|
||||
export const PyFrameworksPydanticBaseModel = "class LanceDocs(LanceModel):\n text: str\n vector: Vector(2)\n";
|
||||
|
||||
@@ -120,7 +120,7 @@ export const PyPlatformsDuckdbMeanPrice = "duckdb.query(\"SELECT mean(price) FRO
|
||||
|
||||
export const PyPlatformsDuckdbQueryTable = "import duckdb\n\narrow_table = table.to_lance()\n\nduckdb.query(\"SELECT * FROM arrow_table\")\n";
|
||||
|
||||
export const PyPlatformsPandasAsyncExample = "async def run_pandas_async_example() -> None:\n async_db = await lancedb.connect_async(\n str(Path(tempfile.mkdtemp()) / \"pandas-async\")\n )\n async_df = pd.DataFrame(\n [\n {\"id\": \"10\", \"text\": \"sage\", \"vector\": [0.6, 0.4, 0.8]},\n {\"id\": \"11\", \"text\": \"bard\", \"vector\": [0.2, 0.7, 0.3]},\n ]\n )\n async_table = await async_db.create_table(\n \"creatures_async\", data=async_df, mode=\"overwrite\"\n )\n async_results = await (\n async_table.search([0.6, 0.4, 0.8])\n .select([\"text\", \"_distance\"])\n .limit(1)\n .to_pandas()\n )\n print(async_results)\n\nasyncio.run(run_pandas_async_example())\n";
|
||||
export const PyPlatformsPandasAsyncExample = "async def run_pandas_async_example() -> pd.DataFrame:\n async_db = await lancedb.connect_async(\n str(Path(tempfile.mkdtemp()) / \"pandas-async\")\n )\n async_df = pd.DataFrame(\n [\n {\"id\": \"10\", \"text\": \"sage\", \"vector\": [0.6, 0.4, 0.8]},\n {\"id\": \"11\", \"text\": \"bard\", \"vector\": [0.2, 0.7, 0.3]},\n ]\n )\n async_table = await async_db.create_table(\n \"creatures_async\", data=async_df, mode=\"overwrite\"\n )\n # An async search returns its query builder from an await\n query = await async_table.search([0.6, 0.4, 0.8])\n async_results = await query.select([\"text\", \"_distance\"]).limit(1).to_pandas()\n print(async_results)\n return async_results\n\nasync_results = asyncio.run(run_pandas_async_example())\n";
|
||||
|
||||
export const PyPlatformsPandasCreateTable = "pandas_df = pd.DataFrame(\n [\n {\"id\": \"1\", \"text\": \"dragon\", \"vector\": [0.9, 0.1, 0.3]},\n {\"id\": \"2\", \"text\": \"griffin\", \"vector\": [0.4, 0.5, 0.2]},\n {\"id\": \"3\", \"text\": \"phoenix\", \"vector\": [0.7, 0.3, 0.6]},\n ]\n)\npandas_db = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"pandas-demo\"))\npandas_table = pandas_db.create_table(\"creatures\", data=pandas_df, mode=\"overwrite\")\n";
|
||||
|
||||
|
||||
@@ -90,7 +90,7 @@ export const PyMergeUpdateInsert = "import pyarrow as pa\n\ntable = db.create_ta
|
||||
|
||||
export const PyOpenExistingTable = "db = tmp_db\n# Create a table first\ndata = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7}]\ndb.create_table(\"test_table\", data, mode=\"overwrite\")\n\n# List table names\nprint(db.list_tables().tables)\n\n# Open existing table\ntbl = db.open_table(\"test_table\")\n";
|
||||
|
||||
export const PySchemaAddSetup = "table_name = \"schema_evolution_add_example\"\nif data is None:\n data = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 3,\n \"name\": \"Headphones\",\n \"price\": 150.00,\n \"vector\": np.random.random(128).tolist(),\n },\n ]\ntable = tmp_db.create_table(table_name, data, mode=\"overwrite\")\n";
|
||||
export const PySchemaAddSetup = "import numpy as np\n\ndata = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 3,\n \"name\": \"Headphones\",\n \"price\": 150.00,\n \"vector\": np.random.random(128).tolist(),\n },\n]\ntable = db.create_table(\"schema_evolution_add_example\", data, mode=\"overwrite\")\n";
|
||||
|
||||
export const PySchemaAlterSetup = "table_name = \"schema_evolution_alter_example\"\nif data is None:\n data = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200,\n \"discount_price\": 1080.0,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800,\n \"discount_price\": 720.0,\n \"vector\": np.random.random(128).tolist(),\n },\n ]\nschema = pa.schema(\n {\n \"id\": pa.int64(),\n \"name\": pa.string(),\n \"price\": pa.int32(),\n \"discount_price\": pa.float64(),\n \"vector\": pa.list_(pa.float32(), 128),\n }\n)\ntable = tmp_db.create_table(table_name, data, schema=schema, mode=\"overwrite\")\n";
|
||||
|
||||
|
||||
@@ -99,17 +99,35 @@ a subset of columns, you can significantly reduce your I/O requirements by selec
|
||||
`Permutation` class provides a `select_columns` method that provides this functionality.
|
||||
|
||||
```py Python icon=Python
|
||||
import lancedb
|
||||
import pyarrow as pa
|
||||
import torch
|
||||
from lancedb.permutation import Permutation
|
||||
|
||||
permutation = Permutation.identity(table).select_columns(["id", "prompt"])
|
||||
db = lancedb.connect("memory://")
|
||||
prompts = db.create_table(
|
||||
"prompts",
|
||||
pa.table(
|
||||
{
|
||||
"id": range(4),
|
||||
"prompt": ["north", "south", "east", "west"],
|
||||
"embedding": [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6], [0.7, 0.8]],
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
permutation = Permutation.identity(prompts).select_columns(["id", "prompt"])
|
||||
dataloader = torch.utils.data.DataLoader(
|
||||
permutation, batch_size=1024, shuffle=True
|
||||
)
|
||||
|
||||
for batch in dataloader:
|
||||
print(batch.schema)
|
||||
print(sorted(batch)) # ['id', 'prompt']: the embedding is never read
|
||||
```
|
||||
|
||||
Each batch is a dict of the selected columns, the default `Permutation` format
|
||||
described above.
|
||||
|
||||
## Using multiple DataLoader workers {#using-multiple-dataloader-workers}
|
||||
|
||||
Set `num_workers > 0` to read from LanceDB in multiple PyTorch worker processes. `Permutation` objects and tables on a deployment are picklable, so each worker reopens the table after it starts. A local table is not, so wrap it in a `Permutation`, as below, to read it from several workers.
|
||||
|
||||
Reference in New Issue
Block a user