From 758f738b8ce401e8ac30edd94afb3330a231fd1b Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Thu, 15 Oct 2026 14:54:08 -0700 Subject: [PATCH] docs: complete ingestion, search and integration examples (#4533) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR is part of the documentation revamp. Once the full set of PRs is merged, we’ll publish the combined site to staging for review before updating docs.lancedb.com. Ingestion, search and integration examples are incomplete or inconsistent across languages, and some guidance misstates SDK behavior. This brings the guides and their example sources together. It includes complete Python, TypeScript and Rust workflows, isolated provider and reranker examples, the Superlinked corrections, and guidance on retries that can duplicate writes. Stacked on #4491. Each MDX file has one PR owner. Rebase onto main after #4491 is squash-merged. Local assembly, Mint validation and link checks pass. Example code is unchanged from the previously validated package. Publication stays held through integration. --- docs/web-tests/py/provider_examples.py | 355 +++++++++ docs/web-tests/py/test_ingestion_examples.py | 739 ++++++++++++++++++ docs/web-tests/py/test_integrations.py | 75 +- docs/web-tests/py/test_provider_examples.py | 160 ++++ docs/web-tests/py/test_reranker_examples.py | 96 +++ docs/web-tests/py/test_search_examples.py | 228 ++++++ docs/web-tests/rs/Cargo.lock | 1 + docs/web-tests/rs/Cargo.toml | 33 + docs/web-tests/rs/ingestion_delete.rs | 72 ++ docs/web-tests/rs/ingestion_examples.rs | 97 +++ docs/web-tests/rs/ingestion_progress.rs | 64 ++ docs/web-tests/rs/search_filter_search.rs | 92 +++ docs/web-tests/rs/search_hybrid_search.rs | 99 +++ docs/web-tests/rs/search_query_plan.rs | 81 ++ docs/web-tests/rs/search_text_search.rs | 85 ++ docs/web-tests/rs/search_vector_search.rs | 93 +++ docs/web-tests/ts/ingestion_examples.test.ts | 117 +++ docs/web-tests/ts/search_examples.test.ts | 222 ++++++ docs/web/indexing/fts-index.mdx | 9 +- docs/web/indexing/index.mdx | 36 +- docs/web/indexing/reindexing.mdx | 52 +- docs/web/indexing/scalar-index.mdx | 23 +- docs/web/indexing/vector-index.mdx | 49 +- docs/web/ingestion/create-table.mdx | 178 +++-- docs/web/ingestion/delete.mdx | 139 ++-- docs/web/ingestion/index.mdx | 198 ++--- docs/web/ingestion/insert.mdx | 342 ++++---- docs/web/ingestion/merge-insert.mdx | 136 +++- docs/web/ingestion/update.mdx | 151 ++-- docs/web/integrations/ai/hermes-agent.mdx | 2 +- docs/web/integrations/ai/huggingface.mdx | 6 + docs/web/integrations/ai/langchain.mdx | 3 + docs/web/integrations/data/duckdb.mdx | 2 +- .../integrations/data/pandas_and_pyarrow.mdx | 6 + docs/web/integrations/data/polars_arrow.mdx | 9 +- docs/web/integrations/data/pydantic.mdx | 7 + docs/web/integrations/embedding/cohere.mdx | 4 + docs/web/integrations/embedding/ibm.mdx | 8 +- docs/web/integrations/embedding/jina.mdx | 4 +- docs/web/integrations/embedding/openai.mdx | 8 +- .../integrations/embedding/superlinked.mdx | 31 +- docs/web/integrations/embedding/voyageai.mdx | 4 + docs/web/integrations/lerobotdataset.mdx | 9 +- docs/web/integrations/reranking/cohere.mdx | 4 +- docs/web/integrations/reranking/jina.mdx | 4 +- docs/web/integrations/reranking/openai.mdx | 5 +- docs/web/integrations/reranking/voyageai.mdx | 5 +- docs/web/integrations/reranking/watsonx.mdx | 4 +- docs/web/reranking/index.mdx | 368 +++++---- docs/web/search/explain.mdx | 183 +++-- docs/web/search/filtering.mdx | 193 +++-- docs/web/search/full-text-search.mdx | 493 +++++++----- docs/web/search/hybrid-search.mdx | 210 +++-- docs/web/search/index.mdx | 152 ++-- docs/web/search/vector-search.mdx | 183 +++-- docs/web/snippets/ingestion_delete.mdx | 4 + docs/web/snippets/ingestion_examples.mdx | 66 ++ docs/web/snippets/ingestion_progress.mdx | 4 + docs/web/snippets/integrations.mdx | 16 +- docs/web/snippets/provider_examples.mdx | 18 + docs/web/snippets/search_examples.mdx | 32 + docs/web/snippets/search_filter_search.mdx | 6 + docs/web/snippets/search_hybrid_search.mdx | 6 + docs/web/snippets/search_query_plan.mdx | 6 + docs/web/snippets/search_text_search.mdx | 6 + docs/web/snippets/search_vector_search.mdx | 6 + .../static/code/ingestion-feedback-rust.zip | Bin 0 -> 38831 bytes docs/web/static/code/ingestion-rust.zip | Bin 0 -> 38092 bytes docs/web/static/code/search-rust.zip | Bin 0 -> 42197 bytes 69 files changed, 4734 insertions(+), 1365 deletions(-) create mode 100644 docs/web-tests/py/provider_examples.py create mode 100644 docs/web-tests/py/test_ingestion_examples.py create mode 100644 docs/web-tests/py/test_provider_examples.py create mode 100644 docs/web-tests/py/test_reranker_examples.py create mode 100644 docs/web-tests/py/test_search_examples.py create mode 100644 docs/web-tests/rs/ingestion_delete.rs create mode 100644 docs/web-tests/rs/ingestion_examples.rs create mode 100644 docs/web-tests/rs/ingestion_progress.rs create mode 100644 docs/web-tests/rs/search_filter_search.rs create mode 100644 docs/web-tests/rs/search_hybrid_search.rs create mode 100644 docs/web-tests/rs/search_query_plan.rs create mode 100644 docs/web-tests/rs/search_text_search.rs create mode 100644 docs/web-tests/rs/search_vector_search.rs create mode 100644 docs/web-tests/ts/ingestion_examples.test.ts create mode 100644 docs/web-tests/ts/search_examples.test.ts create mode 100644 docs/web/snippets/ingestion_delete.mdx create mode 100644 docs/web/snippets/ingestion_examples.mdx create mode 100644 docs/web/snippets/ingestion_progress.mdx create mode 100644 docs/web/snippets/provider_examples.mdx create mode 100644 docs/web/snippets/search_examples.mdx create mode 100644 docs/web/snippets/search_filter_search.mdx create mode 100644 docs/web/snippets/search_hybrid_search.mdx create mode 100644 docs/web/snippets/search_query_plan.mdx create mode 100644 docs/web/snippets/search_text_search.mdx create mode 100644 docs/web/snippets/search_vector_search.mdx create mode 100644 docs/web/static/code/ingestion-feedback-rust.zip create mode 100644 docs/web/static/code/ingestion-rust.zip create mode 100644 docs/web/static/code/search-rust.zip diff --git a/docs/web-tests/py/provider_examples.py b/docs/web-tests/py/provider_examples.py new file mode 100644 index 000000000..2ba24fe96 --- /dev/null +++ b/docs/web-tests/py/provider_examples.py @@ -0,0 +1,355 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Reader examples with explicit placeholders. + +These functions are documentation source. test_provider_examples.py runs the +reranker examples offline, with stand-ins for the model and the provider, to +check that each one writes only to its own new directory. Corresponding opt-in +integration tests remain in test_integrations.py and retain real credential +loading. Replace placeholders before running these examples. +""" + + +def embedding_ibm_usage(): + # --8<-- [start:embedding_ibm_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import EmbeddingFunctionRegistry + from lancedb.pydantic import LanceModel, Vector + + registry = EmbeddingFunctionRegistry.get_instance() + registry.set_var("watsonx-api-key", "your-api-key") + watsonx_embed = registry.get("watsonx").create( + name="ibm/slate-125m-english-rtrvr", + api_key="$var:watsonx-api-key", + project_id="your-project-id", + ) + + class TextModel(LanceModel): + text: str = watsonx_embed.SourceField() + vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-demo")) + tbl = db.create_table("watsonx_test", schema=TextModel, mode="overwrite") + tbl.add(data) + + rs = tbl.search("hello").limit(1).to_pandas() + print(rs.head()) + # --8<-- [end:embedding_ibm_usage] + + +def embedding_jina_text(): + # --8<-- [start:embedding_jina_text] + import os + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import EmbeddingFunctionRegistry + from lancedb.pydantic import LanceModel, Vector + + os.environ["JINA_API_KEY"] = "your-api-key" + + jina_embed = ( + EmbeddingFunctionRegistry.get_instance() + .get("jina") + .create(name="jina-embeddings-v2-base-en") + ) + + class TextModel(LanceModel): + text: str = jina_embed.SourceField() + vector: Vector(jina_embed.ndims()) = jina_embed.VectorField() + + data = [{"text": "hello world"}, {"text": "goodbye world"}] + + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-text")) + tbl = db.create_table("test", schema=TextModel, mode="overwrite") + + tbl.add(data) + # --8<-- [end:embedding_jina_text] + + +def embedding_jina_multimodal(): + # --8<-- [start:embedding_jina_multimodal] + import os + import tempfile + from pathlib import Path + + import lancedb + import pandas as pd + import requests + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + + os.environ["JINA_API_KEY"] = "your-api-key" + + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-images")) + func = get_registry().get("jina").create() + + class Images(LanceModel): + label: str + image_uri: str = func.SourceField() + image_bytes: bytes = func.SourceField() + vector: Vector(func.ndims()) = func.VectorField() + vec_from_bytes: Vector(func.ndims()) = func.VectorField() + + table = db.create_table("images", schema=Images) + labels = ["cat", "cat", "dog", "dog", "horse", "horse"] + uris = [ + "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg", + "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg", + "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg", + "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg", + "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg", + "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg", + ] + image_bytes = [requests.get(uri).content for uri in uris] + table.add( + pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes}) + ) + # --8<-- [end:embedding_jina_multimodal] + + +def reranking_cohere_usage(): + # --8<-- [start:reranking_cohere_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import CohereReranker + + embedder = get_registry().get("sentence-transformers").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "cohere-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + reranker = CohereReranker(api_key="your-api-key") + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_cohere_usage] + + +def reranking_jina_usage(): + # --8<-- [start:reranking_jina_usage] + import os + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import JinaReranker + + os.environ["JINA_API_KEY"] = "your-api-key" + embedder = get_registry().get("jina").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + reranker = JinaReranker(api_key="your-api-key") + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_jina_usage] + + +def reranking_watsonx_usage(): + # --8<-- [start:reranking_watsonx_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import WatsonxReranker + + embedder = get_registry().get("sentence-transformers").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + + # Supply your API key and project ID (or use space_id instead of project_id). + reranker = WatsonxReranker( + api_key="your-api-key", + project_id="your-project-id", + ) + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_watsonx_usage] + + +def reranking_openai_usage(): + # --8<-- [start:reranking_openai_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import OpenaiReranker + + embedder = get_registry().get("sentence-transformers").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "openai-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + reranker = OpenaiReranker(api_key="your-api-key") + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_openai_usage] + + +def reranking_voyageai_usage(): + # --8<-- [start:reranking_voyageai_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import VoyageAIReranker + + embedder = get_registry().get("sentence-transformers").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "voyageai-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + reranker = VoyageAIReranker(api_key="your-api-key", model_name="rerank-2") + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_voyageai_usage] diff --git a/docs/web-tests/py/test_ingestion_examples.py b/docs/web-tests/py/test_ingestion_examples.py new file mode 100644 index 000000000..0dedc1458 --- /dev/null +++ b/docs/web-tests/py/test_ingestion_examples.py @@ -0,0 +1,739 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + + +def test_create_table_from_dicts(): + # --8<-- [start:create_table_from_dicts] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + data = [ + {"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}, + {"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1}, + ] + db.create_table("test_table", data, mode="overwrite") + tbl = db["test_table"] + tbl.head() + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_from_dicts] + assert tbl.count_rows() == 2 + + +def test_create_table_conflict_handling(): + # --8<-- [start:create_table_conflict_handling] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}] + db.create_table("conflict_table", data) + # Idempotent open: reuse the existing table if it exists. + # The provided data is ignored; the schema is validated against the + # existing table and a mismatch raises an error. + tbl = db.create_table("conflict_table", data, exist_ok=True) + + # Overwrite: replace the contents with the provided data. + tbl = db.create_table("conflict_table", data, mode="overwrite") + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_conflict_handling] + + +def test_create_table_custom_schema(): + # --8<-- [start:create_table_custom_schema] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + custom_schema = pa.schema( + [ + pa.field("vector", pa.list_(pa.float32(), 4)), + pa.field("lat", pa.float32()), + pa.field("long", pa.float32()), + ] + ) + + data = [ + {"vector": [1.1, 1.2, 1.3, 1.4], "lat": 45.5, "long": -122.7}, + {"vector": [0.2, 1.8, 0.4, 3.6], "lat": 40.1, "long": -74.1}, + ] + tbl = db.create_table( + "my_table_custom_schema", data, schema=custom_schema, mode="overwrite" + ) + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_custom_schema] + assert tbl.count_rows() == 2 + + +def test_create_table_from_pandas(): + # --8<-- [start:create_table_from_pandas] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pandas as pd + + data = pd.DataFrame( + { + "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]], + "lat": [45.5, 40.1], + "long": [-122.7, -74.1], + } + ) + db.create_table("my_table_pandas", data, mode="overwrite") + db["my_table_pandas"].head() + # --8<-- [end:create_table_from_pandas] + + +def test_create_table_from_polars(): + # --8<-- [start:create_table_from_polars] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import polars as pl + + data = pl.DataFrame( + { + "vector": [[3.1, 4.1], [5.9, 26.5]], + "item": ["foo", "bar"], + "price": [10.0, 20.0], + } + ) + tbl = db.create_table("my_table_pl", data, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_from_polars] + assert tbl.count_rows() == 2 + + +def test_create_table_from_arrow(): + # --8<-- [start:create_table_from_arrow] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import numpy as np + import pyarrow as pa + + dim = 16 + total = 2 + schema = pa.schema( + [ + pa.field("vector", pa.list_(pa.float16(), dim)), + pa.field("text", pa.string()), + ] + ) + data = pa.Table.from_arrays( + [ + pa.array( + [np.random.randn(dim).astype(np.float16) for _ in range(total)], + pa.list_(pa.float16(), dim), + ), + pa.array(["foo", "bar"]), + ], + ["vector", "text"], + ) + tbl = db.create_table("f16_tbl", data, schema=schema, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_from_arrow] + assert tbl.count_rows() == 2 + + +def test_create_table_from_pydantic(): + # --8<-- [start:create_table_from_pydantic] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + from lancedb.pydantic import LanceModel, Vector + + class Content(LanceModel): + movie_id: int + vector: Vector(128) + genres: str + title: str + imdb_id: int + + @property + def imdb_url(self) -> str: + return f"https://www.imdb.com/title/tt{self.imdb_id}" + + tbl = db.create_table("movielens_small", schema=Content, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_from_pydantic] + assert tbl.count_rows() == 0 + + +def test_create_table_nested_schema(): + # --8<-- [start:create_table_nested_schema] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + from lancedb.pydantic import LanceModel, Vector + from pydantic import BaseModel + + class Document(BaseModel): + content: str + source: str + + class NestedSchema(LanceModel): + id: str + vector: Vector(1536) + document: Document + + tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_nested_schema] + assert tbl.count_rows() == 0 + + +def test_create_empty_table(): + # --8<-- [start:create_empty_table] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + schema = pa.schema( + [ + pa.field("vector", pa.list_(pa.float32(), 2)), + pa.field("item", pa.string()), + pa.field("price", pa.float32()), + ] + ) + tbl = db.create_table("test_empty_table", schema=schema, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_empty_table] + assert tbl.count_rows() == 0 + + +def test_create_empty_table_pydantic(): + # --8<-- [start:create_empty_table_pydantic] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + from lancedb.pydantic import LanceModel, Vector + + class Item(LanceModel): + vector: Vector(2) + item: str + price: float + + tbl = db.create_table( + "test_empty_table_new", schema=Item.to_arrow_schema(), mode="overwrite" + ) + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_empty_table_pydantic] + assert tbl.count_rows() == 0 + + +def test_open_existing_table(): + # --8<-- [start:open_existing_table] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + # Create a table first + data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}] + db.create_table("test_table", data, mode="overwrite") + + # List table names + print(db.list_tables().tables) + + # Open existing table + tbl = db.open_table("test_table") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:open_existing_table] + assert tbl.count_rows() == 1 + + +def test_drop_table(): + # --8<-- [start:drop_table] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + # Create a table first + data = [{"vector": [1.1, 1.2], "lat": 45.5}] + db.create_table("my_table", data, mode="overwrite") + + # Drop the table + db.drop_table("my_table") + # --8<-- [end:drop_table] + + +def test_add_from_dataset(): + # --8<-- [start:add_from_dataset] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + from pathlib import Path + + import pyarrow as pa + import pyarrow.parquet as pq + + data_path = str(Path(uri) / "input.parquet") + pq.write_table(pa.table({"id": [1, 2], "name": ["Alice", "Bob"]}), data_path) + import pyarrow.dataset as ds + + dataset = ds.dataset(data_path, format="parquet") + table = db.create_table("my_table", schema=dataset.schema, mode="overwrite") + table.add(dataset) + + print("Rows:", table.count_rows()) + # --8<-- [end:add_from_dataset] + assert table.count_rows() == 2 + + +def test_create_table_from_iterator(): + # --8<-- [start:create_table_from_iterator] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + schema = pa.schema( + [ + pa.field("vector", pa.list_(pa.float32(), 4)), + pa.field("item", pa.utf8()), + pa.field("price", pa.float32()), + ] + ) + + def make_batches(): + for i in range(5): + yield pa.RecordBatch.from_arrays( + [ + pa.array( + [[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]], + pa.list_(pa.float32(), 4), + ), + pa.array(["foo", "bar"]), + pa.array([10.0, 20.0]), + ], + ["vector", "item", "price"], + ) + + table = db.create_table("batched_table", schema=schema) + table.add(make_batches()) + + print("Rows:", table.count_rows()) + # --8<-- [end:create_table_from_iterator] + assert table.count_rows() == 10 + + +def test_update_operation(): + # --8<-- [start:update_operation] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + table.update(where="id = 2", values={"name": "Bobby"}) + + print("Rows:", table.count_rows()) + # --8<-- [end:update_operation] + assert table.count_rows() == 2 + assert table.search().where("id = 2").to_list()[0]["name"] == "Bobby" + + +def test_update_using_sql(): + # --8<-- [start:update_using_sql] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + table.update(where="id = 2", values_sql={"login_count": "login_count + 1"}) + + print("Rows:", table.count_rows()) + # --8<-- [end:update_using_sql] + assert table.count_rows() == 2 + assert table.search().where("id = 2").to_list()[0]["login_count"] == 21 + + +def test_merge_matched_update_only(): + # --8<-- [start:merge_matched_update_only] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + "login_count": [21, 5], + } + ) + + (table.merge_insert("id").when_matched_update_all().execute(incoming_users)) + + print("Rows:", table.count_rows()) + # --8<-- [end:merge_matched_update_only] + assert table.count_rows() == 2 + + +def test_insert_if_not_exists(): + # --8<-- [start:insert_if_not_exists] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + "login_count": [21, 5], + } + ) + + (table.merge_insert("id").when_not_matched_insert_all().execute(incoming_users)) + + print("Rows:", table.count_rows()) + # --8<-- [end:insert_if_not_exists] + assert table.count_rows() == 3 + + +def test_merge_update_insert(): + # --8<-- [start:merge_update_insert] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + "login_count": [21, 5], + } + ) + + ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(incoming_users) + ) + + print("Rows:", table.count_rows()) + # --8<-- [end:merge_update_insert] + assert table.count_rows() == 3 + assert table.search().where("id = 2").to_list()[0]["login_count"] == 21 + + +def test_merge_delete_missing_by_source(): + # --8<-- [start:merge_delete_missing_by_source] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2, 3], + "name": ["Alice", "Bob", "Charlie"], + "login_count": [10, 20, 5], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + "login_count": [21, 5], + } + ) + + ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .when_not_matched_by_source_delete() + .execute(incoming_users) + ) + + print("Rows:", table.count_rows()) + # --8<-- [end:merge_delete_missing_by_source] + assert table.count_rows() == 2 + assert table.count_rows("id = 1") == 0 + + +def test_merge_partial_columns(): + # --8<-- [start:merge_partial_columns] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + } + ) + + ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(incoming_users) + ) + + print("Rows:", table.count_rows()) + # --8<-- [end:merge_partial_columns] + assert table.count_rows() == 3 + assert table.search().where("id = 3").to_list()[0]["login_count"] is None + + +def test_write_workflow(): + # --8<-- [start:write_workflow] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "users", + [ + {"id": 1, "name": "Alice", "login_count": 10}, + {"id": 2, "name": "Bob", "login_count": 20}, + ], + ) + table.add([{"id": 3, "name": "Charlie", "login_count": 5}]) + table.update(where="id = 2", values={"name": "Bobby"}) + table.update(where="id = 2", values_sql={"login_count": "login_count + 1"}) + ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute( + [ + {"id": 3, "name": "Charlie", "login_count": 6}, + {"id": 4, "name": "Dana", "login_count": 1}, + ] + ) + ) + print("Rows selected for deletion:", table.search().where("id = 4").to_list()) + table.delete("id = 4") + rows = sorted(table.search().to_list(), key=lambda row: row["id"]) + print(rows) + # --8<-- [end:write_workflow] + + assert rows == [ + {"id": 1, "name": "Alice", "login_count": 10}, + {"id": 2, "name": "Bobby", "login_count": 21}, + {"id": 3, "name": "Charlie", "login_count": 6}, + ] + + +def test_append_rows(): + # --8<-- [start:append_rows] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + table = db.create_table("users", [{"id": 1, "name": "Alice"}]) + table.add([{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}]) + rows = sorted(table.search().to_list(), key=lambda row: row["id"]) + print(rows) + # --8<-- [end:append_rows] + + assert [row["id"] for row in rows] == [1, 2, 3] + + +def test_delete_rows(): + # --8<-- [start:delete_rows] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "users", + [ + {"id": 1, "name": "Alice"}, + {"id": 2, "name": "Bob"}, + {"id": 3, "name": "Charlie"}, + ], + ) + predicate = "id = 3" + print("Selected:", table.search().where(predicate).to_list()) + table.delete(predicate) + rows = sorted(table.search().to_list(), key=lambda row: row["id"]) + print("Remaining:", rows) + # --8<-- [end:delete_rows] + + assert rows == [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}] + + +def test_add_progress(): + # --8<-- [start:add_progress] + from tempfile import TemporaryDirectory + + import lancedb + + def report_progress(progress): + print(f"Rows written: {progress['output_rows']}") + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + table = db.create_table("users", [{"id": 1, "name": "Alice"}]) + table.add( + [{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}], + progress=report_progress, + ) + rows = sorted(table.search().to_list(), key=lambda row: row["id"]) + print("After insertion:", rows) + # --8<-- [end:add_progress] + + assert rows == [ + {"id": 1, "name": "Alice"}, + {"id": 2, "name": "Bob"}, + {"id": 3, "name": "Charlie"}, + ] diff --git a/docs/web-tests/py/test_integrations.py b/docs/web-tests/py/test_integrations.py index ff07dfae0..577847504 100644 --- a/docs/web-tests/py/test_integrations.py +++ b/docs/web-tests/py/test_integrations.py @@ -740,13 +740,16 @@ def test_reranking_answerdotai_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_answerdotai_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import AnswerdotaiRerankers embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "answerdotai-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -756,18 +759,20 @@ def test_reranking_answerdotai_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = AnswerdotaiRerankers() # Run vector search with a reranker result = tbl.search("hello").rerank(reranker=reranker).to_list() + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + # Run FTS search with a reranker result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) result = ( tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() ) @@ -819,13 +824,16 @@ def test_reranking_colbert_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_colbert_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import ColbertReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "colbert-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -835,18 +843,20 @@ def test_reranking_colbert_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = ColbertReranker() # Run vector search with a reranker result = tbl.search("hello").rerank(reranker=reranker).to_list() + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + # Run FTS search with a reranker result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) result = ( tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() ) @@ -857,13 +867,16 @@ def test_reranking_cross_encoder_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_cross_encoder_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import CrossEncoderReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "cross-encoder-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -873,18 +886,20 @@ def test_reranking_cross_encoder_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = CrossEncoderReranker() # Run vector search with a reranker result = tbl.search("hello").rerank(reranker=reranker).to_list() + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + # Run FTS search with a reranker result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) result = ( tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() ) @@ -936,13 +951,16 @@ def test_reranking_linear_combination_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_linear_combination_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import LinearCombinationReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "linear-combination-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -952,7 +970,7 @@ def test_reranking_linear_combination_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = LinearCombinationReranker() @@ -1007,13 +1025,16 @@ def test_reranking_rrf_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_rrf_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import RRFReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "rrf-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -1023,7 +1044,7 @@ def test_reranking_rrf_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = RRFReranker() @@ -1039,13 +1060,16 @@ def test_reranking_mrr_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_mrr_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import MRRReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "mrr-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -1055,7 +1079,7 @@ def test_reranking_mrr_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = MRRReranker(weight_vector=0.7, weight_fts=0.3) @@ -1390,9 +1414,12 @@ def test_frameworks_pydantic_examples() -> None: {"text": "goodbye world", "vector": [0.0, 1.0]}, ] ) - results = table.search("hello world").limit(1).to_pydantic(LanceDocs) + results = table.search([1.0, 0.0]).limit(1).to_pydantic(LanceDocs) print(results[0].text) + # hello world # --8<-- [end:frameworks_pydantic_base_example] + assert isinstance(results[0], LanceDocs) + assert results[0].text == "hello world" # Platform integrations @@ -1686,7 +1713,7 @@ def test_platforms_pandas_examples() -> None: # --8<-- [end:platforms_pandas_vector_search] # --8<-- [start:platforms_pandas_async_example] - async def run_pandas_async_example() -> None: + async def run_pandas_async_example() -> pd.DataFrame: async_db = await lancedb.connect_async( str(Path(tempfile.mkdtemp()) / "pandas-async") ) @@ -1699,16 +1726,16 @@ def test_platforms_pandas_examples() -> None: async_table = await async_db.create_table( "creatures_async", data=async_df, mode="overwrite" ) - async_results = await ( - async_table.search([0.6, 0.4, 0.8]) - .select(["text", "_distance"]) - .limit(1) - .to_pandas() - ) + # An async search returns its query builder from an await + query = await async_table.search([0.6, 0.4, 0.8]) + async_results = await query.select(["text", "_distance"]).limit(1).to_pandas() print(async_results) + return async_results - asyncio.run(run_pandas_async_example()) + async_results = asyncio.run(run_pandas_async_example()) # --8<-- [end:platforms_pandas_async_example] + assert async_results["text"].tolist() == ["sage"] + assert async_results["_distance"].tolist() == [0.0] def test_platforms_polars_examples() -> None: diff --git a/docs/web-tests/py/test_provider_examples.py b/docs/web-tests/py/test_provider_examples.py new file mode 100644 index 000000000..e1836b6ff --- /dev/null +++ b/docs/web-tests/py/test_provider_examples.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""The provider reranker examples write only to a directory they create. + +Each example runs offline. A stand-in embedding function answers for the model +the example asks the registry for, a stand-in reranker replaces the provider +client, and outbound connections are refused, so no model is downloaded and no +provider is called. Sentinel tables in the owned home directory, where these +examples used to write, and in the temporary directory must be left unchanged. +""" + +import importlib.util +import socket +import tempfile +from pathlib import Path +from types import SimpleNamespace +from typing import ClassVar + +import lancedb +import lancedb.embeddings +import lancedb.rerankers +import pyarrow as pa +import pytest +from lancedb.embeddings import TextEmbeddingFunction, register +from lancedb.rerankers import Reranker + +EXAMPLES = Path(__file__).with_name("provider_examples.py") +RERANKERS = { + "reranking_cohere_usage": "CohereReranker", + "reranking_jina_usage": "JinaReranker", + "reranking_watsonx_usage": "WatsonxReranker", + "reranking_openai_usage": "OpenaiReranker", + "reranking_voyageai_usage": "VoyageAIReranker", +} + + +@register("docs-offline-text") +class OfflineEmbedding(TextEmbeddingFunction): + """Deterministic four-dimensional vectors, computed locally.""" + + def ndims(self): + return 4 + + def generate_embeddings(self, texts): + return [[float(len(text)), 1.0, 0.0, 0.0] for text in texts] + + +class OfflineReranker(Reranker): + """Scores every row 1.0 and records which searches it reranked.""" + + instances: ClassVar[list["OfflineReranker"]] = [] + + def __init__(self, **kwargs): + super().__init__() + self.kwargs = kwargs + self.searches = [] + OfflineReranker.instances.append(self) + + def _score(self, results, search): + self.searches.append(search) + drop = [c for c in ("_distance", "_score") if c in results.column_names] + results = results.drop_columns(drop) + scores = pa.array([1.0] * results.num_rows, pa.float32()) + return results.append_column("_relevance_score", scores) + + def rerank_vector(self, query, vector_results): + return self._score(vector_results, "vector") + + def rerank_fts(self, query, fts_results): + return self._score(fts_results, "fts") + + def rerank_hybrid(self, query, vector_results, fts_results): + merged = self.merge_results(vector_results, fts_results) + return self._score(merged, "hybrid") + + +def snapshot(connect, uri): + db = connect(str(uri)) + tables = {} + for name in db.list_tables().tables: + table = db.open_table(name) + tables[name] = (table.version, table.schema, table.to_arrow().to_pylist()) + return tables + + +def load_examples(): + spec = importlib.util.spec_from_file_location("provider_examples", EXAMPLES) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def owned(tmp_path, monkeypatch): + home, temp_root = tmp_path / "home", tmp_path / "tmp" + home.mkdir() + temp_root.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setattr(tempfile, "tempdir", str(temp_root)) + monkeypatch.chdir(tmp_path) + # Anything the examples resolve from "~" must land in the owned home. + assert Path("~/.lancedb").expanduser().resolve().is_relative_to(home.resolve()) + + def refuse(*args, **kwargs): + raise OSError("network access is disabled in this test") + + monkeypatch.setattr(socket.socket, "connect", refuse) + monkeypatch.setattr(socket, "create_connection", refuse) + + real_connect = lancedb.connect + sentinels = [home / ".lancedb", temp_root / "existing"] + for uri in sentinels: + row = {"text": "keep me", "vector": [0.5, 0.5, 0.5, 0.5]} + real_connect(str(uri)).create_table("test", [row]) + before = [snapshot(real_connect, uri) for uri in sentinels] + + registry = lancedb.embeddings.get_registry() + monkeypatch.setattr(registry, "get", lambda name: OfflineEmbedding) + for name in RERANKERS.values(): + monkeypatch.setattr(lancedb.rerankers, name, OfflineReranker) + monkeypatch.setenv("JINA_API_KEY", "replaced-by-the-example") + OfflineReranker.instances.clear() + + connected = [] + + def recording_connect(uri, *args, **kwargs): + connected.append(Path(uri).expanduser().resolve()) + return real_connect(uri, *args, **kwargs) + + monkeypatch.setattr(lancedb, "connect", recording_connect) + return SimpleNamespace( + home=home, + temp_root=temp_root, + sentinels=sentinels, + before=before, + connect=real_connect, + connected=connected, + ) + + +@pytest.mark.parametrize("example", sorted(RERANKERS)) +def test_reranker_example_writes_only_to_its_own_directory(owned, example): + existing = set(owned.temp_root.iterdir()) + + getattr(load_examples(), example)() + + after = [snapshot(owned.connect, uri) for uri in owned.sentinels] + assert after == owned.before + created = set(owned.temp_root.iterdir()) - existing + assert len(created) == 1 + (own_directory,) = created + assert len(owned.connected) == 1 + assert owned.connected[0].is_relative_to(own_directory.resolve()) + assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"] + + (reranker,) = OfflineReranker.instances + assert reranker.searches == ["vector", "fts", "hybrid"] + assert reranker.kwargs["api_key"] == "your-api-key" + assert reranker.kwargs.get("project_id", "your-project-id") == "your-project-id" diff --git a/docs/web-tests/py/test_reranker_examples.py b/docs/web-tests/py/test_reranker_examples.py new file mode 100644 index 000000000..71505d423 --- /dev/null +++ b/docs/web-tests/py/test_reranker_examples.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""The published reranker examples in test_integrations.py write only to a directory they create. + +Each example runs as readers see it: its marked region is extracted the way the snippet +generator extracts it, saved as a script and run offline. A stand-in embedding function answers for the +sentence-transformers model, stand-in rerankers replace the model-backed AnswerDotAI, ColBERT +and cross-encoder rerankers, and outbound connections are refused. The RRF, linear-combination +and MRR rerankers need no model and run unchanged. Sentinel tables in the owned home directory, +where these examples used to write, and in the temporary directory must be left unchanged. +""" + +import importlib.util +import runpy +import sys +from pathlib import Path + +import lancedb.rerankers +import pytest +import test_provider_examples as provider + +owned = provider.owned +SOURCE = Path(__file__).with_name("test_integrations.py") +GENERATOR = Path(__file__).parents[1] / "mdx_snippets_gen.py" +MODEL_RERANKERS = { + "reranking_answerdotai_usage": "AnswerdotaiRerankers", + "reranking_colbert_usage": "ColbertReranker", + "reranking_cross_encoder_usage": "CrossEncoderReranker", +} +LOCAL_RERANKERS = [ + "reranking_linear_combination_usage", + "reranking_mrr_usage", + "reranking_rrf_usage", +] + + +def load_generator(): + if "mdx_snippets_gen" not in sys.modules: + spec = importlib.util.spec_from_file_location("mdx_snippets_gen", GENERATOR) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module # its dataclasses resolve annotations by module + spec.loader.exec_module(module) + return sys.modules["mdx_snippets_gen"] + + +def reader_code(name): + """The example exactly as the snippet generator publishes it.""" + generator = load_generator() + start, end = generator.markers_for_lang("py") + lines = SOURCE.read_text().splitlines(keepends=True) + snippet = generator.parse_snippets(lines, start, end)[name] + return generator.snippet_text(generator.dedent(snippet)) + + +def run_example(owned, tmp_path, name): + script = tmp_path / "scripts" / f"{name}.py" + script.parent.mkdir(exist_ok=True) + script.write_text(reader_code(name)) + existing = set(owned.temp_root.iterdir()) + namespace = runpy.run_path(str(script)) + + after = [provider.snapshot(owned.connect, uri) for uri in owned.sentinels] + assert after == owned.before + created = set(owned.temp_root.iterdir()) - existing + assert len(created) == 1 + (own_directory,) = created + assert len(owned.connected) == 1 + assert owned.connected[0].is_relative_to(own_directory.resolve()) + assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"] + return namespace + + +@pytest.mark.parametrize("name", sorted(MODEL_RERANKERS)) +def test_model_reranker_example_writes_only_to_its_own_directory( + owned, monkeypatch, tmp_path, name +): + monkeypatch.setattr( + lancedb.rerankers, MODEL_RERANKERS[name], provider.OfflineReranker + ) + + namespace = run_example(owned, tmp_path, name) + + (reranker,) = provider.OfflineReranker.instances + assert reranker.searches == ["vector", "fts", "hybrid"] + assert namespace["result"] + + +@pytest.mark.parametrize("name", LOCAL_RERANKERS) +def test_local_reranker_example_writes_only_to_its_own_directory(owned, tmp_path, name): + namespace = run_example(owned, tmp_path, name) + + assert namespace["result"] + assert all("_relevance_score" in row for row in namespace["result"]) + if name == "reranking_mrr_usage": + assert namespace["combined"].num_rows > 0 diff --git a/docs/web-tests/py/test_search_examples.py b/docs/web-tests/py/test_search_examples.py new file mode 100644 index 000000000..241a0f771 --- /dev/null +++ b/docs/web-tests/py/test_search_examples.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + + +def test_vector_search(): + # --8<-- [start:vector_search] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + "vector": [1.0, 0.0], + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + "vector": [0.9, 0.1], + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + "vector": [0.0, 1.0], + }, + ], + ) + rows = ( + table.search([1.0, 0.0]) + .distance_type("l2") + .select(["id", "text", "_distance"]) + .limit(2) + .to_list() + ) + print(rows) + # --8<-- [end:vector_search] + + assert [row["id"] for row in rows] == [1, 2] + assert rows[0]["_distance"] == 0.0 + + +def test_text_search(): + # --8<-- [start:text_search] + from tempfile import TemporaryDirectory + + import lancedb + from lancedb.index import FTS + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + }, + ], + ) + table.create_index("text", config=FTS()) + rows = ( + table.search("puppy", query_type="fts", fts_columns="text") + .select(["id", "text", "_score"]) + .limit(2) + .to_list() + ) + print(rows) + # --8<-- [end:text_search] + + assert [row["id"] for row in rows] == [1] + assert rows[0]["_score"] > 0 + + +def test_hybrid_search(): + # --8<-- [start:hybrid_search] + from tempfile import TemporaryDirectory + + import lancedb + from lancedb.index import FTS + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + "vector": [1.0, 0.0], + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + "vector": [0.9, 0.1], + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + "vector": [0.0, 1.0], + }, + ], + ) + from lancedb.rerankers import RRFReranker + + table.create_index("text", config=FTS()) + rows = ( + table.search( + query_type="hybrid", vector_column_name="vector", fts_columns="text" + ) + .vector([1.0, 0.0]) + .text("puppy") + .rerank(RRFReranker()) + .limit(2) + .to_list() + ) + print(rows) + # --8<-- [end:hybrid_search] + + assert [row["id"] for row in rows] == [1, 2] + assert rows[0]["_relevance_score"] > rows[1]["_relevance_score"] + + +def test_filter_search(): + # --8<-- [start:filter_search] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + "vector": [1.0, 0.0], + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + "vector": [0.9, 0.1], + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + "vector": [0.0, 1.0], + }, + ], + ) + rows = ( + table.search([0.0, 1.0]) + .where("category = 'animals'", prefilter=True) + .select(["id", "text", "category", "_distance"]) + .limit(2) + .to_list() + ) + print(rows) + # --8<-- [end:filter_search] + + assert {row["id"] for row in rows} == {1, 2} + assert all(row["category"] == "animals" for row in rows) + + +def test_query_plan(): + # --8<-- [start:query_plan] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + "vector": [1.0, 0.0], + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + "vector": [0.9, 0.1], + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + "vector": [0.0, 1.0], + }, + ], + ) + query = table.search([1.0, 0.0]).select(["id", "text", "_distance"]).limit(2) + print("Plan:", query.explain_plan()) + print("Measured execution:", query.analyze_plan()) + rows = query.to_list() + print(rows) + # --8<-- [end:query_plan] + + assert [row["id"] for row in rows] == [1, 2] diff --git a/docs/web-tests/rs/Cargo.lock b/docs/web-tests/rs/Cargo.lock index 915e15e58..447bae7f6 100644 --- a/docs/web-tests/rs/Cargo.lock +++ b/docs/web-tests/rs/Cargo.lock @@ -5455,6 +5455,7 @@ dependencies = [ "arrow-schema", "futures", "futures-util", + "lance-index", "lance-namespace", "lancedb", "polars", diff --git a/docs/web-tests/rs/Cargo.toml b/docs/web-tests/rs/Cargo.toml index 6b68f5fff..8bc594c3c 100644 --- a/docs/web-tests/rs/Cargo.toml +++ b/docs/web-tests/rs/Cargo.toml @@ -10,6 +10,7 @@ arrow-schema = "58.3.0" futures = "0.3.31" futures-util = "0.3.31" lance-namespace = "8.0.0" +lance-index = "=8.0.0" lancedb = { version = "0.31.0", features = ["polars", "remote"] } polars = ">=0.37,<0.40.0" polars-arrow = ">=0.37,<0.40.0" @@ -41,3 +42,35 @@ path = "multimodal.rs" [[example]] name = "search" path = "search.rs" + +[[example]] +name = "ingestion_examples" +path = "ingestion_examples.rs" + +[[example]] +name = "ingestion_delete" +path = "ingestion_delete.rs" + +[[example]] +name = "ingestion_progress" +path = "ingestion_progress.rs" + +[[example]] +name = "search_vector_search" +path = "search_vector_search.rs" + +[[example]] +name = "search_text_search" +path = "search_text_search.rs" + +[[example]] +name = "search_hybrid_search" +path = "search_hybrid_search.rs" + +[[example]] +name = "search_filter_search" +path = "search_filter_search.rs" + +[[example]] +name = "search_query_plan" +path = "search_query_plan.rs" diff --git a/docs/web-tests/rs/ingestion_delete.rs b/docs/web-tests/rs/ingestion_delete.rs new file mode 100644 index 000000000..f23b04f2d --- /dev/null +++ b/docs/web-tests/rs/ingestion_delete.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example] +use std::sync::Arc; + +use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase}; + +fn make_users_reader(rows: &[(i64, &str)]) -> Box { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))), + Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))), + ], + ) + .unwrap(); + Box::new(RecordBatchIterator::new( + vec![Ok(batch)].into_iter(), + schema, + )) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let table = db + .create_table( + "users", + make_users_reader(&[(1, "Alice"), (2, "Bob"), (3, "Charlie")]), + ) + .execute() + .await?; + let predicate = "id = 3"; + let selected: Vec = table + .query() + .only_if(predicate) + .execute() + .await? + .try_collect() + .await?; + println!("Selected: {selected:?}"); + table.delete(predicate).await?; + let rows: Vec = table.query().execute().await?.try_collect().await?; + println!("Remaining: {rows:?}"); + assert_eq!(table.count_rows(None).await?, 2); + assert_eq!( + table + .count_rows(Some("id = 1 AND name = 'Alice'".into())) + .await?, + 1 + ); + assert_eq!( + table + .count_rows(Some("id = 2 AND name = 'Bob'".into())) + .await?, + 1 + ); + assert_eq!(table.count_rows(Some("id = 3".into())).await?, 0); + Ok(()) +} +// --8<-- [end:example] diff --git a/docs/web-tests/rs/ingestion_examples.rs b/docs/web-tests/rs/ingestion_examples.rs new file mode 100644 index 000000000..84748c1b9 --- /dev/null +++ b/docs/web-tests/rs/ingestion_examples.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:write_workflow] +use std::sync::Arc; + +use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase}; + +fn make_users_reader(rows: &[(i64, &str, i64)]) -> Box { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), + Field::new("login_count", DataType::Int64, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))), + Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))), + Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.2))), + ], + ) + .unwrap(); + Box::new(RecordBatchIterator::new( + vec![Ok(batch)].into_iter(), + schema, + )) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let table = db + .create_table( + "users", + make_users_reader(&[(1, "Alice", 10), (2, "Bob", 20)]), + ) + .execute() + .await?; + table + .add(make_users_reader(&[(3, "Charlie", 5)])) + .execute() + .await?; + table + .update() + .only_if("id = 2") + .column("name", "'Bobby'") + .execute() + .await?; + table + .update() + .only_if("id = 2") + .column("login_count", "login_count + 1") + .execute() + .await?; + let mut merge = table.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all(); + merge + .execute(make_users_reader(&[(3, "Charlie", 6), (4, "Dana", 1)])) + .await?; + let selected: Vec = table + .query() + .only_if("id = 4") + .execute() + .await? + .try_collect() + .await?; + println!("Rows selected for deletion: {selected:?}"); + table.delete("id = 4").await?; + let rows: Vec = table.query().execute().await?.try_collect().await?; + println!("Remaining rows: {rows:?}"); + assert_eq!(table.count_rows(None).await?, 3); + assert_eq!( + table + .count_rows(Some( + "id = 2 AND name = 'Bobby' AND login_count = 21".into() + )) + .await?, + 1 + ); + assert_eq!( + table + .count_rows(Some("id = 3 AND login_count = 6".into())) + .await?, + 1 + ); + Ok(()) +} +// --8<-- [end:write_workflow] diff --git a/docs/web-tests/rs/ingestion_progress.rs b/docs/web-tests/rs/ingestion_progress.rs new file mode 100644 index 000000000..afeb4c2ad --- /dev/null +++ b/docs/web-tests/rs/ingestion_progress.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example] +use std::sync::Arc; + +use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::ExecutableQuery; + +fn make_users_reader(rows: &[(i64, &str)]) -> Box { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))), + Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))), + ], + ) + .unwrap(); + Box::new(RecordBatchIterator::new( + vec![Ok(batch)].into_iter(), + schema, + )) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let table = db + .create_table("users", make_users_reader(&[(1, "Alice")])) + .execute() + .await?; + table + .add(make_users_reader(&[(2, "Bob"), (3, "Charlie")])) + .progress(|progress| println!("Rows written: {}", progress.output_rows())) + .execute() + .await?; + let rows: Vec = table.query().execute().await?.try_collect().await?; + println!("After insertion: {rows:?}"); + assert_eq!(table.count_rows(None).await?, 3); + assert_eq!( + table + .count_rows(Some("id = 1 AND name = 'Alice'".into())) + .await?, + 1 + ); + assert_eq!( + table + .count_rows(Some("id = 2 AND name = 'Bob'".into())) + .await?, + 1 + ); + assert_eq!(table.count_rows(Some("id = 3".into())).await?, 1); + Ok(()) +} +// --8<-- [end:example] diff --git a/docs/web-tests/rs/search_filter_search.rs b/docs/web-tests/rs/search_filter_search.rs new file mode 100644 index 000000000..0417b5b15 --- /dev/null +++ b/docs/web-tests/rs/search_filter_search.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + Arc::new( + FixedSizeListArray::from_iter_primitive::( + [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] + .into_iter() + .map(|v| Some(v.into_iter().map(Some).collect::>())), + 2, + ), + ), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + let query = table + .vector_search(&[0.0_f32, 1.0])? + .only_if("category = 'animals'") + .select(Select::columns(&["id", "text", "category", "_distance"])) + .limit(2); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!( + ids.into_iter().collect::>(), + std::collections::BTreeSet::from([1, 2]) + ); + assert!(rows.iter().all(|b| { + b.column_by_name("category") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .all(|v| v == Some("animals")) + })); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/rs/search_hybrid_search.rs b/docs/web-tests/rs/search_hybrid_search.rs new file mode 100644 index 000000000..6a57c259c --- /dev/null +++ b/docs/web-tests/rs/search_hybrid_search.rs @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lance_index::scalar::FullTextSearchQuery; +use lancedb::index::{Index, scalar::FtsIndexBuilder}; +use lancedb::query::{ExecutableQuery, QueryBase}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + Arc::new( + FixedSizeListArray::from_iter_primitive::( + [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] + .into_iter() + .map(|v| Some(v.into_iter().map(Some).collect::>())), + 2, + ), + ), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + table + .create_index(&["text"], Index::FTS(FtsIndexBuilder::default())) + .execute() + .await?; + let query = table + .vector_search(&[1.0_f32, 0.0])? + .full_text_search(FullTextSearchQuery::new("puppy".into()).with_column("text".into())?) + .rerank(Arc::new(lancedb::rerankers::rrf::RRFReranker::default())) + .limit(2); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(ids, vec![1, 2]); + let scores: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("_relevance_score") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert!(scores[0] > scores[1]); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/rs/search_query_plan.rs b/docs/web-tests/rs/search_query_plan.rs new file mode 100644 index 000000000..97290fff1 --- /dev/null +++ b/docs/web-tests/rs/search_query_plan.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + Arc::new( + FixedSizeListArray::from_iter_primitive::( + [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] + .into_iter() + .map(|v| Some(v.into_iter().map(Some).collect::>())), + 2, + ), + ), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + let query = table + .vector_search(&[1.0_f32, 0.0])? + .select(Select::columns(&["id", "text", "_distance"])) + .limit(2); + println!("Plan: {}", query.explain_plan(false).await?); + println!("Measured execution: {}", query.analyze_plan().await?); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(ids, vec![1, 2]); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/rs/search_text_search.rs b/docs/web-tests/rs/search_text_search.rs new file mode 100644 index 000000000..ebb4b0292 --- /dev/null +++ b/docs/web-tests/rs/search_text_search.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lance_index::scalar::FullTextSearchQuery; +use lancedb::index::{Index, scalar::FtsIndexBuilder}; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + table + .create_index(&["text"], Index::FTS(FtsIndexBuilder::default())) + .execute() + .await?; + let query = table + .query() + .full_text_search(FullTextSearchQuery::new("puppy".into()).with_column("text".into())?) + .select(Select::columns(&["id", "text", "_score"])) + .limit(2); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(ids, vec![1]); + let scores: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("_score") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert!(scores[0] > 0.0); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/rs/search_vector_search.rs b/docs/web-tests/rs/search_vector_search.rs new file mode 100644 index 000000000..f2807ebef --- /dev/null +++ b/docs/web-tests/rs/search_vector_search.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + Arc::new( + FixedSizeListArray::from_iter_primitive::( + [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] + .into_iter() + .map(|v| Some(v.into_iter().map(Some).collect::>())), + 2, + ), + ), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + let query = table + .vector_search(&[1.0_f32, 0.0])? + .distance_type(lancedb::DistanceType::L2) + .select(Select::columns(&["id", "text", "_distance"])) + .limit(2); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(ids, vec![1, 2]); + let scores: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("_distance") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(scores[0], 0.0); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/ts/ingestion_examples.test.ts b/docs/web-tests/ts/ingestion_examples.test.ts new file mode 100644 index 000000000..4c6b88412 --- /dev/null +++ b/docs/web-tests/ts/ingestion_examples.test.ts @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors +import { expect, test } from "@jest/globals"; + +test("complete ingestion workflow", async () => { + // --8<-- [start:write_workflow_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + + const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("users", [ + { id: 1, name: "Alice", login_count: 10 }, + { id: 2, name: "Bob", login_count: 20 }, + ]); + await table.add([{ id: 3, name: "Charlie", login_count: 5 }]); + await table.update({ where: "id = 2", values: { name: "Bobby" } }); + await table.update({ + where: "id = 2", + valuesSql: { login_count: "login_count + 1" }, + }); + await table + .mergeInsert("id") + .whenMatchedUpdateAll() + .whenNotMatchedInsertAll() + .execute([ + { id: 3, name: "Charlie", login_count: 6 }, + { id: 4, name: "Dana", login_count: 1 }, + ]); + console.log( + "Rows selected for deletion:", + await table.query().where("id = 4").toArray(), + ); + await table.delete("id = 4"); + const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id); + console.log(rows); + // --8<-- [end:write_workflow_body] + expect(rows.map((row) => [row.id, row.name, row.login_count])).toEqual([ + [1, "Alice", 10], + [2, "Bobby", 21], + [3, "Charlie", 6], + ]); + // --8<-- [start:write_workflow_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:write_workflow_tail] +}); + +test("complete delete_rows", async () => { + // --8<-- [start:delete_rows_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("users", [ + { id: 1, name: "Alice" }, + { id: 2, name: "Bob" }, + { id: 3, name: "Charlie" }, + ]); + const predicate = "id = 3"; + console.log("Selected:", await table.query().where(predicate).toArray()); + await table.delete(predicate); + const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id); + console.log("Remaining:", rows); + // --8<-- [end:delete_rows_body] + expect(rows.map((row) => [row.id, row.name])).toEqual([ + [1, "Alice"], + [2, "Bob"], + ]); + // --8<-- [start:delete_rows_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:delete_rows_tail] +}); + +test("complete add_progress", async () => { + // --8<-- [start:add_progress_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("users", [{ id: 1, name: "Alice" }]); + await table.add( + [ + { id: 2, name: "Bob" }, + { id: 3, name: "Charlie" }, + ], + { + progress: (progress) => + console.log(`Rows written: ${progress.outputRows}`), + }, + ); + const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id); + console.log("After insertion:", rows); + // --8<-- [end:add_progress_body] + expect(rows.map((row) => [row.id, row.name])).toEqual([ + [1, "Alice"], + [2, "Bob"], + [3, "Charlie"], + ]); + // --8<-- [start:add_progress_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:add_progress_tail] +}); diff --git a/docs/web-tests/ts/search_examples.test.ts b/docs/web-tests/ts/search_examples.test.ts new file mode 100644 index 000000000..be4c388ee --- /dev/null +++ b/docs/web-tests/ts/search_examples.test.ts @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { expect, test } from "@jest/globals"; + +test("complete vector_search", async () => { + // --8<-- [start:vector_search_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { + id: 1, + text: "A puppy playing in the park", + category: "animals", + vector: [1.0, 0.0], + }, + { + id: 2, + text: "A dog resting at home", + category: "animals", + vector: [0.9, 0.1], + }, + { + id: 3, + text: "A guide to growing tomatoes", + category: "gardening", + vector: [0.0, 1.0], + }, + ]); + const rows = await table + .vectorSearch([1.0, 0.0]) + .distanceType("l2") + .select(["id", "text", "_distance"]) + .limit(2) + .toArray(); + console.log(rows); + // --8<-- [end:vector_search_body] + expect(rows.map((row) => row.id)).toEqual([1, 2]); + expect(rows[0]._distance).toBe(0); + // --8<-- [start:vector_search_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:vector_search_tail] +}); + +test("complete text_search", async () => { + // --8<-- [start:text_search_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { id: 1, text: "A puppy playing in the park", category: "animals" }, + { id: 2, text: "A dog resting at home", category: "animals" }, + { id: 3, text: "A guide to growing tomatoes", category: "gardening" }, + ]); + await table.createIndex("text", { config: lancedb.Index.fts() }); + const rows = await table + .query() + .fullTextSearch("puppy", { columns: ["text"] }) + .select(["id", "text", "_score"]) + .limit(2) + .toArray(); + console.log(rows); + // --8<-- [end:text_search_body] + expect(rows.map((row) => row.id)).toEqual([1]); + expect(rows[0]._score).toBeGreaterThan(0); + // --8<-- [start:text_search_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:text_search_tail] +}); + +test("complete hybrid_search", async () => { + // --8<-- [start:hybrid_search_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { + id: 1, + text: "A puppy playing in the park", + category: "animals", + vector: [1.0, 0.0], + }, + { + id: 2, + text: "A dog resting at home", + category: "animals", + vector: [0.9, 0.1], + }, + { + id: 3, + text: "A guide to growing tomatoes", + category: "gardening", + vector: [0.0, 1.0], + }, + ]); + await table.createIndex("text", { config: lancedb.Index.fts() }); + const reranker = await lancedb.rerankers.RRFReranker.create(); + const rows = await table + .vectorSearch([1.0, 0.0]) + .fullTextSearch("puppy", { columns: ["text"] }) + .rerank(reranker) + .limit(2) + .toArray(); + console.log(rows); + // --8<-- [end:hybrid_search_body] + expect(rows.map((row) => row.id)).toEqual([1, 2]); + expect(rows[0]._relevance_score).toBeGreaterThan(rows[1]._relevance_score); + // --8<-- [start:hybrid_search_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:hybrid_search_tail] +}); + +test("complete filter_search", async () => { + // --8<-- [start:filter_search_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { + id: 1, + text: "A puppy playing in the park", + category: "animals", + vector: [1.0, 0.0], + }, + { + id: 2, + text: "A dog resting at home", + category: "animals", + vector: [0.9, 0.1], + }, + { + id: 3, + text: "A guide to growing tomatoes", + category: "gardening", + vector: [0.0, 1.0], + }, + ]); + const rows = await table + .vectorSearch([0.0, 1.0]) + .where("category = 'animals'") + .select(["id", "text", "category", "_distance"]) + .limit(2) + .toArray(); + console.log(rows); + // --8<-- [end:filter_search_body] + expect(rows.map((row) => row.id).sort()).toEqual([1, 2]); + expect(rows.every((row) => row.category === "animals")).toBe(true); + // --8<-- [start:filter_search_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:filter_search_tail] +}); + +test("complete query_plan", async () => { + // --8<-- [start:query_plan_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { + id: 1, + text: "A puppy playing in the park", + category: "animals", + vector: [1.0, 0.0], + }, + { + id: 2, + text: "A dog resting at home", + category: "animals", + vector: [0.9, 0.1], + }, + { + id: 3, + text: "A guide to growing tomatoes", + category: "gardening", + vector: [0.0, 1.0], + }, + ]); + const query = table + .vectorSearch([1.0, 0.0]) + .select(["id", "text", "_distance"]) + .limit(2); + console.log("Plan:", await query.explainPlan()); + console.log("Measured execution:", await query.analyzePlan()); + const rows = await query.toArray(); + console.log(rows); + // --8<-- [end:query_plan_body] + expect(rows.map((row) => row.id)).toEqual([1, 2]); + // --8<-- [start:query_plan_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:query_plan_tail] +}); diff --git a/docs/web/indexing/fts-index.mdx b/docs/web/indexing/fts-index.mdx index 4e18d5a86..832b67315 100644 --- a/docs/web/indexing/fts-index.mdx +++ b/docs/web/indexing/fts-index.mdx @@ -3,6 +3,13 @@ title: "Full-Text Search (FTS) Index" sidebarTitle: "FTS Index" description: "Create and tune BM25-based full-text search indexes in LanceDB." icon: "book" +availability: + oss: available + enterprise: available + summary: >- + Full-text indexes are built in the embedded library and in LanceDB Enterprise. + The embedded library builds the index before the call returns; a + deployment builds it in the background. --- import { PyFtsIndexAsync as FtsIndexAsync, PyFtsIndexCreate as FtsIndexCreate, PyFtsIndexNested as FtsIndexNested, PyFtsIndexWait as FtsIndexWait } from '/snippets/indexing.mdx'; @@ -114,7 +121,7 @@ await async_table.create_index("payload.text", config=FTS(with_position=True)) - `jieba/*` is for Chinese word segmentation with Jieba. - `lindera/*` loads a compiled Lindera dictionary, such as `lindera/ipadic` for Japanese or `lindera/ko-dic` for Korean. -Model-backed tokenizers such as `jieba/default`, `lindera/ipadic`, and `lindera/ko-dic` require tokenizer model files in Lance's language model home. Lance looks under the default platform data directory for `lance/language_models`, or you can set `LANCE_LANGUAGE_MODEL_HOME` to point to another model root. For example, `jieba/default` is resolved under `/jieba/default/...`. +Model-backed tokenizers such as `jieba/default`, `lindera/ipadic`, and `lindera/ko-dic` require tokenizer model files in Lance's language model home. Lance looks under the default platform data directory for `lance/language_models`, or you can set `LANCE_LANGUAGE_MODEL_HOME` to point to another model root. For example, `jieba/default` is resolved under `/jieba/default/...`. For a table in LanceDB Enterprise, the files must be present where the deployment builds and queries the index, so ask its administrator whether they are installed. `language` is used by token filters, not by the base tokenizer. Stemming supports Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. Built-in stop-word removal supports Danish, Dutch, English, Finnish, French, German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish, and Swedish. For other stemming languages, set `remove_stop_words=False` or pass `custom_stop_words`. diff --git a/docs/web/indexing/index.mdx b/docs/web/indexing/index.mdx index b40fb7918..78844527f 100644 --- a/docs/web/indexing/index.mdx +++ b/docs/web/indexing/index.mdx @@ -3,20 +3,26 @@ title: "Indexing Data" sidebarTitle: "Overview" description: "Optimize search performance with LanceDB using vector indexes, full-text search, scalar indexes, and more." icon: "list" +availability: + oss: available + enterprise: available + summary: >- + Indexes are built in the embedded library and in LanceDB Enterprise. Enterprise builds them in the background, and builds fewer vector index types than the + embedded library. --- -An **index** is a data structure that facilitates efficient scans and lookups on an embedded dataset. -LanceDB provides a comprehensive suite of indexes to optimize performance across different use cases and data types: +Indexes speed up searches and filters by organizing data for specific query +patterns. LanceDB supports three main index families: - **Vector Index**: Efficiently searches for similar vectors across high-dimensional data (e.g. images, audio, or text embeddings) - **Full-Text Search Index**: Enables fast keyword-based searches by indexing words and phrases -- **Scalar Index**: Accelerates filtering and sorting of structured numeric or categorical data +- **Scalar Index**: Accelerates filtering of structured numeric or categorical data ## Supported Indexes | Index | Use Case | Description | | :--------- | :------- | :---------- | -| `IVF` (Vector) | Large-scale vector search with configurable accuracy/speed trade-offs. | Inverted File Index—a partition-based approximate nearest neighbor algorithm that groups similar vectors into partitions for efficient search.
**Quantizations**: `None/Flat` `PQ` `SQ` `RQ`| +| `IVF` (Vector) | Large-scale vector search with configurable accuracy/speed trade-offs. | Inverted File Index: a partition-based approximate nearest neighbor algorithm that groups similar vectors into partitions for efficient search.
**Quantizations**: `None/Flat` `PQ` `SQ` `RQ`| | `IVF_HNSW` (Vector) | Large-scale vector search requiring both high recall and efficient partitioning. Combines the scalability of IVF with the search quality of HNSW. | Hybrid index combining IVF partitioning with HNSW graphs in each partition. Provides improved search quality over pure IVF while maintaining scalability.
**Quantizations**: `None/Flat` `SQ` `PQ`| | `FTS` (Full-text search) | String columns (e.g., title, description, content) requiring keyword-based search with BM25 ranking. | Full-text search index using BM25 ranking algorithm. Tokenizes text with configurable tokenization, stemming, stop word removal, and language-specific processing. | | `BTree` (Scalar) | Numeric, temporal, and string columns with mostly distinct values. Best for selective equality, inequality, and range predicates. | Sorted index storing sorted copies of scalar columns with block headers in a btree cache. Header entries map to blocks of rows (4096 rows per block) for efficient disk reads. | @@ -24,6 +30,10 @@ LanceDB provides a comprehensive suite of indexes to optimize performance across | `LabelList` (Scalar) | List columns (e.g., tags, categories, keywords) requiring `array_contains_all` or `array_contains_any` filters. | Scalar index for `List` and `LargeList` columns of primitive values, using an underlying bitmap index structure to enable fast array membership lookups. | | `FM` (Scalar) | String or binary columns that need raw substring search. | FM-Index over `Utf8`, `LargeUtf8`, `Binary`, or `LargeBinary` data for filters such as `contains(path, 'needle')`. Use FTS instead for tokenized word search and BM25 ranking. | +A deployment builds the vector indexes `IVF_FLAT`, `IVF_PQ`, `IVF_RQ` and +`IVF_HNSW_SQ`. `IVF_SQ`, `IVF_HNSW_FLAT` and `IVF_HNSW_PQ` are built only by the +embedded library. + ## Quantization LanceDB also supports several [quantization](/indexing/vector-index) methods, used by vector indexes to compress vectors and reduce storage requirements: @@ -32,26 +42,30 @@ LanceDB also supports several [quantization](/indexing/vector-index) methods, us | `PQ` (Product Quantization) | Default choice for most vector search scenarios. Use when you need to balance index size and recall. | Divides vectors into subvectors and quantizes each subvector independently. Provides a good balance between compression ratio and search accuracy. | | `RQ` (RabitQ Quantization) | Use when you need maximum compression or have specific per-dimension requirements. | Per-dimension quantization using a RabitQ codebook. Provides fine-grained control over compression per dimension. For `IVF_RQ`, vector dimensions must be divisible by `8`. | | `SQ` (Scalar Quantization) | Use when you need faster indexing or when vector dimensions have consistent value ranges. | Quantizes each dimension independently. Simpler than PQ but typically provides less compression. | -| `None/Flat` | Use for binary vectors (with `hamming` distance) or when you need maximum recall and have sufficient storage. | No quantization—stores raw vectors. Provides the highest accuracy but requires more storage and memory. | +| `None/Flat` | Use for binary vectors (with `hamming` distance) or when you need maximum recall and have sufficient storage. | No quantization; stores raw vectors. Provides the highest accuracy but requires more storage and memory. | ## Performance {#performance} #### Vector indexes {#vector-indexes} -If vector search latency climbs with table size (i.e., queries that ran in milliseconds on a small table take seconds as it grows to millions of rows), the cause is the default brute-force scan over every vector. That works fine below ~100K vectors, but past that you should build a dedicated vector index. Pick the type by your data shape: +If vector search latency climbs with table size (i.e., queries that ran in milliseconds on a small table take seconds as it grows to millions of rows), check whether the query is scanning every vector without an index. The point at which an index helps depends on vector dimensions, hardware and +your latency target. Measure your workload before choosing an index. Pick the type by your data shape: | Index | When to use | |-------|-------------| -| `IVF_PQ` | General-purpose default; what Enterprise builds automatically. Higher accuracy than RQ at small dimensions (≤ 256). | +| `IVF_PQ` | General-purpose default, and what a deployment builds when it is set to index a column automatically. Higher accuracy than RQ at small dimensions (≤ 256). | | `IVF_RQ` | Maximum compression on high-dim vectors, faster builds than PQ. | | `IVF_HNSW_SQ` | Best recall/latency for unfiltered search; higher latency variance under selective filters. | | `IVF_FLAT` | Required for binary vectors with `hamming`. | -The distance metric is fixed once the index is built. Pick the distance metric based on how the embedding model was trained: `cosine` (unnormalized), `dot` (already-normalized, best performance), `l2` (general-purpose, default), `hamming` (binary). If a query explicitly requests a different metric, the index cannot be used and the query falls back to a brute-force scan; LanceDB Enterprise may reject that scan on a large table. For parameter tuning, see [Vector Indexing](/indexing/vector-index). +The distance metric is fixed once the index is built. Pick the distance metric based on how the embedding model was trained: `cosine` (unnormalized), `dot` (already-normalized, best performance), `l2` (general-purpose, default), `hamming` (binary). If a query explicitly requests a different metric, the index cannot be used and the query falls back to a brute-force scan; a deployment can be configured to reject that scan on a large table. For parameter tuning, see [Vector Indexing](/indexing/vector-index). #### Scalar indexes {#scalar-indexes} -If filtered queries slow down as the table grows — even when the filter is selective — the cause is a full column scan: without a scalar index, LanceDB evaluates the `where(...)` predicate on every row, and the same applies to `merge_insert` join keys. The best practice is to build a scalar index on every column you filter or join on, picking the type by the column's shape: +Scalar indexes can reduce the rows scanned for selective filters and speed up +`merge_insert` key lookups. Start with columns used frequently in those operations, +then use [query plans](/search/explain) to check whether the index helps. Choose +the index type based on the column's values: | Index | Best for | |-------|----------| @@ -63,4 +77,6 @@ See [Scalar Indexing](/indexing/scalar-index). #### Full-text search {#full-text-search} -If your full-text index is much larger than expected, or takes longer than expected to build, the cause is usually phrase-query flags being enabled when they aren't needed: `with_position=True` and `remove_stop_words=False` both significantly inflate index size and build time. The best practice is to keep the defaults for most workloads, and only enable those flags when you actually need to search for phrases. See [FTS Indexing](/indexing/fts-index) configuration options for the full set of options. +Token positions and retained stop words increase the information stored in a +full-text index. Keep the defaults for basic keyword search; enable +`with_position=True` and retain the needed stop words for phrase queries. See [FTS Indexing](/indexing/fts-index) configuration options for the full set of options. diff --git a/docs/web/indexing/reindexing.mdx b/docs/web/indexing/reindexing.mdx index dd50f62fe..fdbfe3b5f 100644 --- a/docs/web/indexing/reindexing.mdx +++ b/docs/web/indexing/reindexing.mdx @@ -4,6 +4,14 @@ sidebarTitle: "Build & Repair" description: "Learn how to keep your indexes up-to-date in LanceDB using incremental indexing, including best practices for adding new records without full reindexing." icon: "refresh" keywords: ["reindexing", "re-index", "incremental indexing", "reindex"] +availability: + oss: available + enterprise: available + summary: >- + Indexes are kept up to date in the embedded library and in LanceDB Enterprise. In + the embedded library you add new rows to them with `optimize()`; a + deployment does it in the background, and `optimize()` is not available + there. --- import { @@ -20,16 +28,19 @@ As data is being added and a reindex operation is running, LanceDB will combine Rather than dropping an existing index entirely and reindexing from scratch, LanceDB supports **incremental indexing**. -**Enterprise job types:** building an index runs as `create_ivf_pq`, `create_btree`, -`create_fts`, and one more per index algorithm -- the type names the algorithm, -not a separate feature. Folding new rows into an existing one runs as the -matching `reindex_*` type instead. +**Enterprise job types:** an index build runs as `reindex_` followed by the +algorithm, such as `reindex_ivf_pq`, `reindex_btree` or `reindex_fts`, and so +does folding new rows into an existing index. A create-only build, one that must +not replace an existing index, runs as `create_` followed by the algorithm +instead; Python's `create_scalar_index` and `create_fts_index` ask for that by +default. The name says which algorithm is building, not a separate feature. ## Incremental Reindexing {#incremental-reindexing} -You can manually trigger an incremental indexing operation on updated data -using the `optimize()` method on a table. +In the embedded library, you trigger an incremental indexing operation on +updated data with the `optimize()` method on a table. A deployment does this +for you and does not support `optimize()`, as described below. Table optimization performs three maintenance operations: @@ -45,10 +56,12 @@ Table optimization performs three maintenance operations: Enterprise -LanceDB Enterprise support incremental reindexing through an automated background process. When new data is added to a table, the system automatically triggers a new index build. As the dataset grows, indexes are asynchronously updated in the background. +LanceDB Enterprise supports incremental reindexing through an automated background process. Once enough new rows have been added to a table, the system automatically adds them to its indexes. As the dataset grows, indexes are asynchronously updated in the background. -- While indexes are being rebuilt, queries use brute force methods on unindexed rows, which may temporarily increase latency. To avoid this, set `fast_search=True` to search only indexed data. +- While indexes are being rebuilt, queries use brute force methods on unindexed rows, which may temporarily increase latency. If you can omit unindexed rows, set `fast_search=True` to search only indexed data. + This can return incomplete results while indexing catches up. - Use `index_stats()` to view the number of unindexed rows. This will be zero when indexes are fully up-to-date. If you call `wait_for_index(...)`, it polls the same status and can time out while continuous writes keep adding unindexed rows. +- Rows below the deployment's threshold stay unindexed until more arrive, so the count can stay above zero without further writes. `FM` indexes are not updated in the background. The benefit of using LanceDB Enterprise is that it automates the reindexing process and operates continuously in the background, minimizing the impact on latency under high loads. @@ -71,9 +84,9 @@ If you need to reclaim space more aggressively in OSS, use a shorter retention w ## Building on GPUs {#gpu} -With LanceDB's GPU-powered vector indexing you can index very large datasets in far less time -than you could with the default CPU-based indexing. In our tests, LanceDB -is capable of indexing billions of rows in under four hours on a 1-8 GPU cluster. +GPU acceleration can reduce the time needed to build vector indexes on large +datasets. Actual build time depends on the index configuration, vector +dimensions, storage throughput and available GPU resources. **Automatic GPU indexing** @@ -82,9 +95,8 @@ is capable of indexing billions of rows in under four hours on a 1-8 GPU cluster Automatic GPU Indexing is currently only available in [LanceDB Enterprise](/enterprise/). Please [contact us](https://www.lancedb.com/contact) to enable this feature for your deployment. -The vector index is created when you call `create_index`. The backend will use GPU resources -to build either the IVF or HNSW indexes. The system automatically selects the optimal GPU -configuration based on your data size and available hardware. +The vector index is created when you call `create_index`. The backend uses GPU resources +to build `IVF_PQ` indexes, with the number of GPUs set by the deployment's configuration. This process is also asynchronous by default, but you can use `wait_for_index` to convert it into a synchronous process by waiting until the index is built. @@ -93,9 +105,9 @@ into a synchronous process by waiting until the index is built. `wait_for_index(...)` waits for the index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`. It can time out if the table is receiving continuous writes while the build is trying to catch up. -GPU acceleration changes how the vector index is built, not the lifecycle after the build. Rows -appended later still need `optimize()` before they are part of the index, and `fast_search()` only -searches indexed rows. +GPU acceleration changes how the vector index is built, not the lifecycle after the build. In the +embedded library, rows appended later still need `optimize()` before they are part of the index; a +deployment adds them in the background. `fast_search()` only searches indexed rows. ### Manual GPU indexing in LanceDB OSS {#gpu-manual-gpu-indexing-in-lancedb-oss} @@ -125,9 +137,9 @@ to enable GPU training on your device. ### Performance considerations {#gpu-performance-considerations} - GPU memory usage scales with `num_partitions` and vector dimensions -- For optimal performance, ensure GPU memory exceeds dataset size -- Batch size is automatically tuned based on available GPU memory -- Indexing speed improves with larger batch sizes +- Size GPU memory for the training batches and index configuration +- Monitor memory use while choosing training batch sizes +- Measure build time and recall before changing training settings ### Troubleshooting {#gpu-troubleshooting} diff --git a/docs/web/indexing/scalar-index.mdx b/docs/web/indexing/scalar-index.mdx index e8dcf6bb7..9f1f9a176 100644 --- a/docs/web/indexing/scalar-index.mdx +++ b/docs/web/indexing/scalar-index.mdx @@ -3,6 +3,13 @@ title: "Scalar Indexes" sidebarTitle: "Scalar Index" description: "Learn how to use scalar indexes in LanceDB for efficient metadata filtering and query optimization." icon: "tree" +availability: + oss: available + enterprise: available + summary: >- + Scalar indexes are built in the embedded library and in LanceDB Enterprise. A + deployment builds them in the background and keeps most of them up to date + itself; in the embedded library you update them with `optimize()`. --- import { @@ -19,7 +26,9 @@ import { PyScalarIndexNestedFields as ScalarIndexNestedFields, } from '/snippets/indexing.mdx'; -Scalar indexes organize data by scalar attributes (e.g., numbers, categories) and enable fast filtering of vector data. They accelerate retrieval of scalar data associated with vectors, thus enhancing query performance. +Scalar indexes accelerate filters on values such as numbers, categories and +strings. Use them for selective table queries or metadata filters combined +with vector and full-text search. LanceDB supports four types of scalar indexes: @@ -29,7 +38,7 @@ LanceDB supports four types of scalar indexes: - `FM`: FM-Index over string or binary columns that accelerates substring search via `contains(col, 'needle')`. -Scalar indices serve as a foundational optimization layer, accelerating filtering across diverse search workloads. They can be combined with: +Use scalar indexes with: - Vector search (prefilter results using metadata) - Full-text search (combining keyword matching with structured filters) @@ -70,11 +79,11 @@ If you are using LanceDB Enterprise, the `create_scalar_index` API returns immed -`wait_for_index(...)` waits until the named scalar indexes exist and `index_stats(...)` reports `num_unindexed_rows == 0`. If a table is receiving steady writes, that fully indexed state may not stabilize before the timeout. +`wait_for_index(...)` waits until the named scalar indexes exist and `index_stats(...)` reports `num_unindexed_rows == 0`. If a table is receiving steady writes, that fully indexed state may not stabilize before the timeout. In LanceDB Enterprise, rows added after the index was built are indexed only once enough have accumulated, so a small remainder can also keep the wait from finishing. ### 3. Update the Index -Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index. +Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. In the embedded library, call `optimize`, which will trigger an update to the existing scalar index. @@ -83,7 +92,7 @@ Updating the table data (adding, deleting, or modifying records) requires that y -New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Enterprise automates the optimize process, minimizing the impact on search speed. +New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Enterprise does not support `optimize()`; instead it adds new rows to `BTREE`, `BITMAP` and `LABEL_LIST` indexes in the background once enough have accumulated. It does not update `FM` indexes automatically. ### 4. Run Indexed Searches @@ -125,7 +134,7 @@ Nested paths follow Lance field-path semantics: dot-separate each struct field f The `FM` index is a scalar index built over string or binary columns that accelerates substring lookups expressed as `contains(col, 'needle')`. Unlike the tokenized [FTS index](/indexing/fts-index), which matches whole words after -tokenization, the FM-Index matches arbitrary substrings of the raw bytes — so it +tokenization, the FM-Index matches arbitrary substrings of the raw bytes. It works well for URLs, file paths, identifiers, log lines, or any column where you search for a fragment rather than a word. @@ -143,6 +152,8 @@ or language-aware tokenization. Build an FM-Index with the async `create_index` API by passing the `Fm` config in Python or `Index.fm()` in TypeScript. In Rust, use `Index::Fm(FmIndexBuilder::default())`. +A deployment builds FM-Indexes too, but does not add new rows to them in the background as it +does for the other scalar indexes. ```python Python icon="python" diff --git a/docs/web/indexing/vector-index.mdx b/docs/web/indexing/vector-index.mdx index b10da3f6f..e9768efa6 100644 --- a/docs/web/indexing/vector-index.mdx +++ b/docs/web/indexing/vector-index.mdx @@ -3,6 +3,14 @@ title: "Vector Indexes" sidebarTitle: "Vector Index" description: "Build and manage LanceDB vector indexes." icon: "arrow-up-right-dots" +availability: + oss: available + enterprise: available + summary: >- + Vector indexes are built in the embedded library and in LanceDB Enterprise. The + embedded library builds an index before `create_index()` returns; a + deployment builds it in the background, keeps it up to date, and builds + only `IVF_FLAT`, `IVF_PQ`, `IVF_RQ` and `IVF_HNSW_SQ`. --- import { @@ -53,7 +61,7 @@ Lance with `index_type`. | Maximum compression | `IVF_RQ` | IVF indexing with RaBitQ quantization | Around `1/32` of raw size | `IvfRq` | | Higher accuracy at small dimensions (`dimension <= 256`) | `IVF_PQ` | IVF indexing with product quantization | Usually `1/64` to `1/16` of raw size (depends on `num_sub_vectors`) | `IvfPq` | | Best recall/latency trade-off | `IVF_HNSW_SQ` | IVF-HNSW indexing with scalar | Typically a little larger than `1/4` of raw size | `IvfHnswSq` | -| Highest recall / no quantization | `IVF_HNSW_FLAT` | IVF-HNSW indexing with no quantization | Around raw vector size plus HNSW graph overhead | `IvfHnswFlat` | +| Highest recall / no quantization | `IVF_HNSW_FLAT` | IVF-HNSW indexing with no quantization; embedded library only | Around raw vector size plus HNSW graph overhead | `IvfHnswFlat` | If your vector search frequently includes metadata filters (`where(...)`), use `IVF_RQ` or `IVF_PQ`. In filtered workloads, HNSW-backed IVF indexes such as `IVF_HNSW_FLAT` and `IVF_HNSW_SQ` can show higher latency variance. @@ -114,7 +122,7 @@ your destination. This iterative series of road networks mimics the layered grap In LanceDB, HNSW is not exposed as a top-level vector index. Instead, it's available as a substructure which further indexes the selected vectors inside each IVF partition. This combines the scalability of IVF with the high recall of HNSW. -LanceDB supports IVF-HNSW-based quantized indexes `IVF_HNSW_FLAT`, `IVF_HNSW_PQ`, and `IVF_HNSW_SQ`. +LanceDB supports IVF-HNSW-based quantized indexes `IVF_HNSW_FLAT`, `IVF_HNSW_PQ`, and `IVF_HNSW_SQ`; a deployment builds only `IVF_HNSW_SQ`. @@ -141,7 +149,7 @@ Learn how to configure, build, and search LanceDB vector indexes, including buil | `limit` | Number of results to return (the `k` in `k-ANN`). | | `nprobes` | Sets both `minimum_nprobes` and `maximum_nprobes` to the same value in Lance. Leave unset to use Lance's adaptive defaults. At very high scales you will likely need to experiment to find your preferred recall / latency balance. | | `minimum_nprobes` | Minimum number of partitions scanned. Leave unset to use Lance's default. | -| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results — useful for narrow filters. Leave unset, or set to `0`, to remove the cap. | +| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results, which is useful for narrow filters. Leave unset, or set to `0`, to remove the cap. | | `ef` | HNSW search-time exploration factor. Start around `1.5 * k` and increase up to `10 * k` for higher recall. | | `refine_factor` | Reads and reranks additional candidates in memory to recover recall lost to quantization. Start between `5` and `50` (a factor of `25` reads and reranks the top `25 * k` candidates) and measure the latency cost, typically a few milliseconds, against the recall gained. | | `distance_range(lower_bound, upper_bound)` | Return only rows whose distance falls within `[lower_bound, upper_bound)`. Either bound is optional. Useful for near-duplicate detection or "close-enough" matching. | @@ -152,8 +160,8 @@ The `distance_type` can be specified at search time in addition to build time. I you do this, set it to the same value used to build the vector index. If the search metric does not match the index metric, LanceDB cannot use the index and falls back to a costly flat search. On LanceDB Cloud, the brute-force KNN safety limit -can reject that search on a large table instead of executing it. LanceDB -Enterprise doesn't have the limit. +can reject that search on a large table instead of executing it. An Enterprise +deployment applies the same limit only when it is configured to. When the search metric is omitted, current SDKs use the index metric if an index is available. Some older SDK versions sent `l2` when it was omitted, so explicitly @@ -173,6 +181,8 @@ older client. **Filtered queries and adaptive `nprobes`.** When a `where(...)` filter is active, LanceDB initially scans `minimum_nprobes` partitions and uses a wider scan if too few rows are found. Set `minimum_nprobes == maximum_nprobes` or call `nprobes(n)` to instead fix the partition count. +Whether a deployment widens the scan this way depends on its version; one that does not scans exactly +`minimum_nprobes` partitions. Here is an example of a vector search exercising several of the above parameters. @@ -211,7 +221,7 @@ Multivector indexing currently requires `distance_type="cosine"`. Use `bypass_ve #### Async API and Config Objects Create vector indexes asynchronously with `await table.create_index("vector", config=...)`. The `config` object -admits the same build-time parameters described above — pass an instance of a Python config class from the table in [Choosing the Right Index](#choosing-the-right-index): +admits the same build-time parameters described above. Pass an instance of a Python config class from the table in [Choosing the Right Index](#choosing-the-right-index): @@ -234,7 +244,7 @@ Specify `vector_column_name` if you have multiple vector columns or non-default TypeScript currently doesn't support `IvfSq` (IVF with Scalar Quantization). -For a vector field nested inside a struct, pass its full dotted path as `vector_column_name` (e.g. `image.embedding`) — the same path is used at query time and is what `list_indices()` reports under `columns`. See [Selecting the vector column](/search/vector-search#selecting-the-vector-column) for the full path syntax. +For a vector field nested inside a struct, pass its full dotted path as `vector_column_name` (e.g. `image.embedding`). The same path is used at query time and is what `list_indices()` reports under `columns`. See [Selecting the vector column](/search/vector-search#selecting-the-vector-column) for the full path syntax. @@ -265,17 +275,18 @@ The snippet below builds and queries an `IVF_HNSW_SQ` index. Enterprise-only In LanceDB Enterprise, vector indexes are managed **automatically**. The system asynchronously updates and optimizes indexes as a background process: - Automatically manages indexing parameters and storage -- Infers vector columns from the schema -- GPU-based indexing is available on request -- [contact support](/deployment/support) to enable it for your deployment. A few billion vectors index in under 4 hours on GPU. +- Can build an `IVF_PQ` index automatically on a vector column the deployment is configured to index +- GPU-based `IVF_PQ` builds are available on request -- [contact support](/deployment/support) to enable them for your deployment. Build time depends on dataset size, dimensions and GPU resources. Open-Source LanceDB OSS users can manually create vector indexes by calling `table.create_index()`. See the above sections for guidance on manually tuning index parameters as data changes. -`create_index()` returns immediately, but the vector index builds asynchronously. -To wait until all data is indexed, specify the `wait_timeout` parameter, or call `wait_for_index(...)` afterward — -it waits for the named index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`. +In LanceDB Enterprise, `create_index()` returns immediately and the vector index builds asynchronously. +To wait until all data is indexed, specify the `wait_timeout` parameter, or call `wait_for_index(...)` afterward. +It waits for the named index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`. +In the embedded library the index is built before `create_index()` returns. @@ -287,11 +298,12 @@ fallback and searches only indexed rows. **Operational checks** -After appends or other writes, use `optimize()` to fold new rows into existing indexes. +In the embedded library, use `optimize()` after appends or other writes to fold new rows into existing indexes. +A deployment does this in the background, and `optimize()` is not available there. #### Check Index Status -Vector index creation runs in the background and may take some time to complete. +In LanceDB Enterprise, vector index creation runs in the background and may take some time to complete. While it is ongoing, you can check its status through the API or the **LanceDB Enterprise UI**. In the LanceDB Enterprise UI, navigate to your table page - the "Index" column reflects each column's index status: it is blank when no index exists, shows an "in progress" label while the index is being built, and shows the index type once the build completes. @@ -318,7 +330,7 @@ Each entry returned by `list_indices()` also carries detailed per-index metadata -These fields are populated for local and embedded tables. On LanceDB Enterprise remote tables they are returned as `None` / `undefined` until the server response surfaces them. +These fields are populated for local tables. In LanceDB Enterprise, `list_indices()` returns them except `num_unindexed_rows`; use `index_stats()` for index coverage. #### Custom Index Names @@ -343,9 +355,10 @@ compactly store an index. #### Product Quantization (PQ) To visualize PQ, assume a vector dataset has $d$ dimensions, with a **chunk** of a vector denoting a contiguous block of entries. Imagine that each vector has - $m$ disjoint chunks of $d/m$ entries each, where the first chunk represents entries $0$ through $d-1$, the second contains entries $d$ through $2d - 1$, and so on. + $m$ disjoint chunks of $d/m$ entries each, where the first chunk contains entries $0$ through $d/m-1$, the second contains +entries $d/m$ through $2d/m-1$, and so on. -Now, for each $i$, let $S_i$ represent the set containing chunk $i$ of each vector (entries $(i-1)d$ to $di - 1$). +Now, for each $i$, let $S_i$ represent the set containing chunk $i$ of each vector (entries $(i-1)d/m$ to $id/m-1$). For each $S_i$ independently, a small set of **centroids** is computed corresponding to an approximate solution to the $k$-means clustering problem. For each original vector in the dataset, every chunk is associated to its nearest centroid; thus, the vector itself is associated with the concatenation of $m$ centroids. @@ -397,7 +410,7 @@ among all integral multiples $\{j \times 10^5: -128 \leq j \leq 127 \}$. | Parameter | Description | | :--- | :--- | -| `num_bits` | Bits per dimension for quantization. Only applies to `IVF_PQ`/`IVF_HNSW_PQ` (default `8`) and `IVF_RQ` (default `1`, RaBitQ) — not used by `IVF_FLAT`/`IVF_SQ`/`IVF_HNSW_FLAT`/`IVF_HNSW_SQ`. Higher values improve accuracy at the cost of additional storage. | +| `num_bits` | Bits per dimension for quantization. Only applies to `IVF_PQ`/`IVF_HNSW_PQ` (default `8`) and `IVF_RQ` (default `1`, RaBitQ); not used by `IVF_FLAT`/`IVF_SQ`/`IVF_HNSW_FLAT`/`IVF_HNSW_SQ`. Higher values improve accuracy at the cost of additional storage. | `max_iterations` and `sample_rate` also affect quantizer training, but since they apply to every IVF/HNSW index type (not just quantized ones), they're documented as general [Build-time Parameters](/indexing/vector-index#build-time-parameters) instead. All three are passed as keyword arguments to `create_index`, alongside `index_type`: diff --git a/docs/web/ingestion/create-table.mdx b/docs/web/ingestion/create-table.mdx index fce23ac65..7d4363be7 100644 --- a/docs/web/ingestion/create-table.mdx +++ b/docs/web/ingestion/create-table.mdx @@ -1,9 +1,14 @@ --- title: "Creating a table" sidebarTitle: "Create Table" -description: "Start here: turn the data you already have into a LanceDB table." +description: "Create a table from records or a schema and choose input formats." icon: table keywords: ["create table", "ingest", "arrow", "pandas", "polars", "pydantic", "schema"] +availability: + oss: available + enterprise: available + summary: >- + Create and open tables in OSS and Enterprise. --- import { @@ -11,65 +16,60 @@ import { RsConnect, } from '/snippets/connection.mdx'; import { - PyCreateTableFromDicts as CreateTableFromDicts, TsCreateTableFromDicts as TsCreateTableFromDicts, RsCreateTableFromDicts as RsCreateTableFromDicts, - PyCreateTableConflictHandling as CreateTableConflictHandling, TsCreateTableConflictHandling as TsCreateTableConflictHandling, RsCreateTableConflictHandling as RsCreateTableConflictHandling, - PyCreateTableFromPandas as CreateTableFromPandas, - PyCreateTableCustomSchema as CreateTableCustomSchema, TsCreateTableCustomSchema as TsCreateTableCustomSchema, RsCreateTableCustomSchema as RsCreateTableCustomSchema, - PyCreateTableFromPolars as CreateTableFromPolars, - PyCreateTableFromArrow as CreateTableFromArrow, TsCreateTableFromArrow as TsCreateTableFromArrow, RsCreateTableFromArrow as RsCreateTableFromArrow, - PyCreateTableFromPydantic as CreateTableFromPydantic, - PyCreateTableNestedSchema as CreateTableNestedSchema, - PyOpenExistingTable as OpenExistingTable, TsOpenExistingTable as TsOpenExistingTable, RsOpenExistingTable as RsOpenExistingTable, - PyCreateEmptyTable as CreateEmptyTable, TsCreateEmptyTable as TsCreateEmptyTable, RsCreateEmptyTable as RsCreateEmptyTable, - PyCreateEmptyTablePydantic as CreateEmptyTablePydantic, - PyDropTable as DropTable, TsDropTable as TsDropTable, RsDropTable as RsDropTable, PyTablesBasicConnect as TablesBasicConnect, PyTablesDocumentModel as TablesDocumentModel, PyTablesTzValidator as TablesTzValidator, } from '/snippets/tables.mdx'; +import { + PyCreateTableFromDicts as CreateTableFromDicts, + PyCreateTableConflictHandling as CreateTableConflictHandling, + PyCreateTableFromPandas as CreateTableFromPandas, + PyCreateTableCustomSchema as CreateTableCustomSchema, + PyCreateTableFromPolars as CreateTableFromPolars, + PyCreateTableFromArrow as CreateTableFromArrow, + PyCreateTableFromPydantic as CreateTableFromPydantic, + PyCreateTableNestedSchema as CreateTableNestedSchema, + PyOpenExistingTable as OpenExistingTable, + PyCreateEmptyTable as CreateEmptyTable, + PyCreateEmptyTablePydantic as CreateEmptyTablePydantic, + PyDropTable as DropTable, +} from '/snippets/ingestion_examples.mdx'; -A table is created from data you already have -- a list of objects, an Arrow -table, a pandas or polars frame, Pydantic models -- or from a schema alone when -the rows come later. +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; + +Create a table from existing records, or define an empty table before data +arrives. Define a schema to specify column types, allow or disallow missing +values, and set the size of your vectors. ## Create a table with data {#create-a-table-with-data} -Initialize a LanceDB connection and create a table +The Python input examples below are complete programs. Install `lancedb` and +any additional libraries imported by the example, save one as a Python file, +and run it. Each uses its own temporary local database. - - - {TablesBasicConnect} - - - - {TsConnect} - - - - {RsConnect} - - - -Depending on the SDK, LanceDB can ingest arrays of records, Arrow tables or record batches, and Arrow batch iterators or readers. Let's take a look at some of the common patterns. +The TypeScript and Rust blocks illustrate the corresponding input formats. +For a complete connection-to-result program in either language, expand +[the write workflow](#complete-example). Table names follow the +[naming rules](/basics/naming). ### From list of objects {#from-list-of-objects} You can provide a list of objects to create a table. The Python and TypeScript SDKs -support lists/arrays of dictionaries, while the Rust SDK supports lists of structs. +support lists/arrays of records. The Rust example converts structs to Arrow batches. In Python, pass a list or other batch-like object; a single bare `dict` or single `LanceModel` is rejected. @@ -87,37 +87,42 @@ In Python, pass a list or other batch-like object; a single bare `dict` or singl -### Handle existing tables {#handle-existing-tables} +### If the table already exists {#handle-existing-tables} -By default, `create_table` raises an error if a table with the same name already exists. -You can change this behavior with two parameters that resolve the conflict in different ways: +By default, creating a table fails if its name is already in use. Choose whether +to open that table or overwrite it. -- **Idempotent open**: return the existing table without modifying it. Use when your - code may run more than once (notebooks, retries, init scripts) and you want to reuse - the table on subsequent runs. The provided data is ignored, but the schema is - validated against the existing table and a mismatch raises an error. -- **Overwrite**: drop the existing table and create a new one with the provided data. - Use this for test fixtures or when you intentionally want to replace prior contents. - This permanently discards the old table's data. +#### Open the existing table + +Use `exist_ok=True` in Python or `existOk: true` in TypeScript to return the +existing table without changing its contents. This is useful for setup code +that may run more than once. Any data supplied to the create call is ignored +when the table already exists. + +Inspect the returned table's schema if your code depends on specific columns +or types. Opening an existing table does not change its schema to match the +data supplied in the create call. + +#### Overwrite the table + +Use `mode="overwrite"` in Python or `mode: "overwrite"` in TypeScript to replace +the existing table with the supplied data and schema. Choose this only when +you intend to replace its previous contents. + +The following Python program demonstrates both choices: - {CreateTableConflictHandling} - - {TsCreateTableConflictHandling} - - - {RsCreateTableConflictHandling} - - + `exist_ok` / `existOk` does not append the provided data to an existing table. Use [`table.add()`](/ingestion/insert) for that. If you need to ensure a table exists *and* -contains specific rows, prefer the [empty-table-then-add pattern](#create-empty-table). +contains specific rows, choose an explicit [merge policy](/ingestion/merge-insert). Creating an empty table +and repeatedly appending the same batch does not deduplicate it. ### From a custom schema {#from-a-custom-schema} @@ -138,7 +143,7 @@ You can define a custom Arrow schema for the table. This is useful when you want -An explicit schema is also where you control nullability. If later writes omit a +An explicit schema also determines which columns accept missing values (`null`). If later writes omit a non-nullable column, or provide actual nulls for it, ingestion fails; nullable columns can be omitted or written with null values. Without an explicit schema, Python infers list-like vector values as fixed-size `float32` vector fields from the observed dimension. @@ -186,10 +191,8 @@ The **`vector`** column needs to be a [Vector](/integrations/data/pydantic#vecto ### From a Polars DataFrame {#from-a-polars-dataframe} Python Only -LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library -written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow -under the hood. A deeper integration between LanceDB Tables and Polars DataFrames -is on the way. +Pass a Polars DataFrame as input. Install `polars` for this example; the +conversion preserves the table structure through Arrow. @@ -263,12 +266,14 @@ for a `created_at` field. -When you run this code it, should raise the `ValidationError`. +The example catches a `ValidationError` for the timestamp with a different +timezone. A validator runs when the Pydantic model is constructed; declaring a +model as the table schema does not make every later write run that validator. -Datasets too large to pass to `create_table` at once are covered separately -- -see [Insert data](/ingestion/insert#large-datasets), which explains why -`table.add()` parallelizes where `create_table` does not. +Datasets too large to pass to `create_table` at once are covered separately; +see [Insert data](/ingestion/insert#large-datasets) for batches, file-backed +input and write parallelism. ## Create empty table {#create-empty-table} @@ -292,9 +297,8 @@ An empty table can be initialized via an Arrow schema. -Alternatively, you can also use Pydantic to specify the schema for the empty table. Note that we do not -directly import `pydantic` but instead use `lancedb.pydantic` which is a subclass of `pydantic.BaseModel` -that has been extended to support LanceDB specific types like `Vector`. +Alternatively, define the schema with `LanceModel`, a Pydantic model class +provided by `lancedb.pydantic`. Its `Vector` type declares a fixed vector dimension. @@ -342,6 +346,46 @@ Use the `drop_table()` method on the database to remove a table. -This permanently removes the table and is not recoverable, unlike deleting rows. -By default, if the table does not exist an exception is raised. To suppress this, -you can pass in `ignore_missing=True`. +Dropping a table removes the table object. Deleting rows keeps the table and +changes its contents; retained versions and cleanup are described in +[Cleanup](/maintenance/cleanup). +On a local database, dropping a table that does not exist raises an error; in +Python, pass `ignore_missing=True` to suppress it. A deployment may report +success instead, depending on how it stores its catalog, so do not rely on the +error to tell you a table was missing. Python's deployment connection does not +take `ignore_missing`. + + + +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/ingestion/delete.mdx b/docs/web/ingestion/delete.mdx index 0e2a4fdfa..b7b1f9432 100644 --- a/docs/web/ingestion/delete.mdx +++ b/docs/web/ingestion/delete.mdx @@ -1,77 +1,110 @@ --- title: "Delete rows" sidebarTitle: "Delete" -description: "Remove rows by predicate, and reclaim their space." +description: "Select and delete records, verify the remaining rows, and understand storage retention." icon: trash +availability: + oss: available + enterprise: available + summary: >- + Delete rows through the SDKs in OSS and Enterprise. Storage reclamation follows compaction and retention policy. --- -These examples use the table set up in -[Ingesting data](/ingestion), and the connection made there. +import { RsExample as RsDeleteRows } from '/snippets/ingestion_delete.mdx'; +import { TsDeleteRowsBody, TsDeleteRowsTail } from '/snippets/ingestion_examples.mdx'; +import { PyDeleteRows } from '/snippets/ingestion_examples.mdx'; +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; -import { - PyDeleteOperation as DeleteOperation, - PyUpdateOptimizeCleanup as UpdateOptimizeCleanup, - TsDeleteOperation, - TsUpdateOptimizeCleanup, - RsDeleteOperation, - RsUpdateOptimizeCleanup, -} from '/snippets/tables.mdx'; +Deleting rows removes matching records from the current table contents. It +keeps the table and schema so you can continue reading and writing. To remove +the table itself, see [Drop a table](/ingestion/create-table#drop-a-table). - -In SQL this is [`DELETE`](/sql/dml/delete). - +## Inspect the rows you intend to delete {#select-rows} -## Delete rows {#delete-rows} +Use the same predicate for inspection and deletion. The example below creates +Alice, Bob and Charlie, prints Charlie's row, and deletes it by `id = 3`. +Inspection and deletion are separate operations; concurrent writes can change +which rows match between them. -Delete operations **soft delete** rows that match a given condition. -The underlying data is not immediately removed, but is marked -for deletion (in the [deletion files](https://lance.org/format/table/#deletion-files) at the Lance format level) and excluded from query results. +## Delete matching records {#delete-rows} + +Each program creates Alice, Bob and Charlie in a temporary local database, +deletes Charlie, and prints the remaining rows: **Alice and Bob**. The temporary +database is removed when the program finishes. + +- **Python:** install `lancedb`, save the code as `delete_rows.py`, and run + `python delete_rows.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `delete_rows.mts`, + and run `node delete_rows.mts` with Node.js 22.18 or later. +- **Rust:** download the [Rust examples](/static/code/ingestion-feedback-rust.zip), + extract the project, and run `cargo run --locked --bin delete`. - - {DeleteOperation} - - - - {TsDeleteOperation} - - - - {RsDeleteOperation} - + +{PyDeleteRows} + + +{TsDeleteRowsBody} +{TsDeleteRowsTail} + + +{RsDeleteRows} + -Expected table contents: +Enterprise also supports [`DELETE`](/sql/dml/delete). -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bob | 20 | +## Verify that the rows are absent {#verify} +Query the same predicate after deletion and check the remaining row count. +A reader still using an older snapshot may see the earlier contents; see +[Consistency](/tables/consistency). - -**Deleting rows removes them from the index** +Deleted rows are excluded from current query results, including indexed +searches. Deletion does not imply immediate physical erasure of every stored +copy of the data. -When rows are deleted, those rows are also excluded from the index segments, so indexed queries will not return them either. If ALL the rows are deleted (i.e., the table is emptied), ensure that you recreate the index after ingesting new data. - +## Understand storage reclamation {#storage} -To permanently remove deleted rows, you can optimize the table, which will run compaction and cleans up the soft-deleted rows, which frees up storage space. +Compaction rewrites the live data layout; cleanup removes eligible retained +versions and files. Retained history can continue to reference data that no +longer appears in the latest table version. -- In LanceDB OSS, compaction and cleanup are manual. Run `table.optimize()` regularly to free up disk space. -- In LanceDB Enterprise, files aren't cleaned up by default. You can configure automatic compaction and cleanup behavior at cluster setup time to suit your organization's retention policy. +In OSS, `optimize()` provides maintenance operations. In Enterprise, managed +maintenance and retention follow the service configuration. Choose retention +according to the history you need, rather than shortening it as part of a +routine delete example. See [Compaction](/maintenance/compaction) and +[Cleanup](/maintenance/cleanup) for the separate maintenance workflow. -By default, table cleanup removes data up to 7 days ago. If you need to reclaim space from deleted rows more aggressively, manually call `table.optimize()` use a shorter retention window as follows: +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. - - {UpdateOptimizeCleanup} - - - - {TsUpdateOptimizeCleanup} - - - - {RsUpdateOptimizeCleanup} - + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/ingestion/index.mdx b/docs/web/ingestion/index.mdx index 2a8de6759..c37b46a62 100644 --- a/docs/web/ingestion/index.mdx +++ b/docs/web/ingestion/index.mdx @@ -1,125 +1,97 @@ --- title: "Ingesting data" sidebarTitle: "Overview" -description: "Getting data into a table: creating one from your data, and the four write operations that follow." +description: "Choose how to create a table, append records or change existing data." icon: inbox keywords: ["ingest", "create table", "add", "insert", "arrow", "pandas", "polars", "pydantic"] +availability: + oss: available + enterprise: available + summary: >- + Create tables and write records through the SDKs in OSS and Enterprise. Enterprise also provides SQL write statements. --- -import { - PyUpdateConnectEnterprise as UpdateConnectEnterprise, - PyUpdateConnectLocal as UpdateConnectLocal, - PyUpdateExampleTableSetup as UpdateExampleTableSetup, - TsUpdateConnectEnterprise, - TsUpdateConnectLocal, - TsUpdateExampleTableSetup, - RsUpdateConnectEnterprise, - RsUpdateConnectLocal, - RsUpdateExampleTableSetup, - RsUpdateMakeUsersReader, -} from '/snippets/tables.mdx'; +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; -In LanceDB, tables store records with a defined schema that specifies column -names and types. Across the SDKs, you can create tables from row-oriented data -and Apache Arrow data structures. The Python SDK additionally supports PyArrow -schemas for explicit schema control, and `LanceModel` for Pydantic-based -validation. +Ingestion brings data into a LanceDB table. Start by choosing a schema and +creating the table, then choose a write operation according to how incoming +records relate to data already stored. -Getting data in has two halves: creating a table from data you already have, and -writing to a table that exists. This page covers the first and introduces the -second. +## Choose a write operation {#choose-a-write-method} - -Every operation here has a SQL form -- see [DML](/sql#dml). - - -Start by [creating a table](/ingestion/create-table); the rest of this -section writes to one that exists. - -## Choose a write method {#choose-a-write-method} - -| Family | Method | Use this when... | -| --------------- | --------------- | ---------------- | -| [`add`](/ingestion/insert) | `add(data)` | You are appending new rows and do not need to match against existing ones. | -| `update` | `update(where=..., values=...)` | You want to edit rows that already exist, using a SQL filter. | -| `merge_insert` | `.when_matched_update_all()` | You have incoming rows and want to update keys that already exist in the table. | -| `merge_insert` | `.when_not_matched_insert_all()` | You have incoming rows and want to insert keys that do not exist yet. | -| `merge_insert` | `.when_matched_update_all()` + `.when_not_matched_insert_all()` | You want both behaviors together (often called **upsert**: update existing keys **and** insert missing keys in the same operation). | -| `merge_insert` | `.when_not_matched_by_source_delete(...)` | You want to remove target rows that are missing from the incoming source set. | - -Write operations return status metadata. For example, `update` reports updated row count and -committed `version`, `delete` reports deleted row count and `version`, and `merge_insert` reports -inserted, updated, deleted, retry-attempt, and `version` fields. Writes require a mutable table -handle; after checking out an older version for reads, call `checkout_latest` / `checkoutLatest` -before modifying data. -The committed `version` advances even for writes that affect zero rows, such as a delete predicate -that matches nothing. - -## Connect to LanceDB {#connect-to-lancedb} - -Connect to your local LanceDB instance: - - - - {UpdateConnectLocal} - - - - {TsUpdateConnectLocal} - - - - {RsUpdateConnectLocal} - - - -Or, connect to LanceDB Enterprise: - - - - {UpdateConnectEnterprise} - - - - {TsUpdateConnectEnterprise} - - - - {RsUpdateConnectEnterprise} - - - -In the Rust snippets, a `make_users_reader` helper is used to build Arrow input data. - - - - {RsUpdateMakeUsersReader} - - - -## Create the example table {#create-the-example-table} - -We'll start by creating a simple table with `id`, `name`, and `login_count` columns. All examples below use the same table. - - - {UpdateExampleTableSetup} - - - - {TsUpdateExampleTableSetup} - - - - {RsUpdateExampleTableSetup} - - - -Expected table contents: - -| id | name | login_count | +| Task | Operation | What happens | | --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bob | 20 | +| [Create a table](/ingestion/create-table) | `create_table` / `createTable` | Create a table from data or an explicit schema. | +| [Append records](/ingestion/insert) | `add` | Add rows without matching existing keys. | +| [Upsert by key](/ingestion/merge-insert) | `merge_insert` / `mergeInsert` | Choose whether matching rows are updated and new keys inserted. | +| [Update selected rows](/ingestion/update) | `update` | Change values in rows selected by a predicate. | +| [Delete selected rows](/ingestion/delete) | `delete` | Remove matching rows from the current table contents. | -The example above shows a PyArrow schema. You can just as well create the table using other -table creation patterns (Pandas, Polars, Pydantic, iterators, etc.) -- see [Create a table](/ingestion/create-table#create-a-table-with-data). +Appending the same input twice can create duplicate records. Use merge insert +with a stable key when the incoming data should update existing records. + +## Connect and prepare the data {#connect-to-lancedb} + +Use the [connection guide](/basics/connect) to connect to OSS or Enterprise. +The SDK write operations in this section work with both. Enterprise also +supports [SQL write statements](/sql#dml); an SDK filter expression does not +require a separate SQL connection. + +Choose the column types, required fields and vector dimensions before a large +load. SDKs support different input representations, including row objects and +Arrow batches. The [table creation guide](/ingestion/create-table) covers +those choices. + +## Try a complete write workflow {#create-the-example-table} + +This example starts with Alice and Bob, appends Charlie, updates Bob's name +and login count, then merges a batch containing Charlie and Dana. Finally, it +selects Dana's row for inspection and deletes it. + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + + +## Verify the result {#verify} + +After a write completes, inspect the affected keys and fields and check the row +count. SDK return values differ by operation and language; use the relevant +API reference when relying on returned counts or version metadata. + +Readers with older snapshots may not see another handle's writes immediately. +See [Consistency](/tables/consistency). A table checked out at a historical +version must return to the latest version before it can be modified. + +## Load larger datasets {#large-loads} + +Use batches or file-backed input when the dataset is too large to hold in +memory. The [insert guide](/ingestion/insert#large-datasets) covers input +formats, progress reporting and write parallelism. Keep retry behavior in mind: +repeating an append is not the same as retrying a key-based upsert. diff --git a/docs/web/ingestion/insert.mdx b/docs/web/ingestion/insert.mdx index 2d411eb3b..f2ee0d7db 100644 --- a/docs/web/ingestion/insert.mdx +++ b/docs/web/ingestion/insert.mdx @@ -1,217 +1,237 @@ --- title: "Insert data" sidebarTitle: "Insert" -description: "Append rows to a table, including datasets too large to hold in memory." +description: "Append records, load file-backed data or batches, and verify ingestion." icon: plus +availability: + oss: available + enterprise: available + summary: >- + Append records through the SDKs in OSS and Enterprise. For large Enterprise writes, see [large upload settings](#bulk-ingests-into-a-remote-table). --- -Rows are appended with `table.add()`. It accepts the same shapes -[table creation](/ingestion) does — lists of objects, Arrow tables, pandas -and polars frames, Pydantic models — and commits a new table version. - -These examples use the table set up in [Ingesting data](/ingestion). - +import { + TsCreateTableFromIterator as TsCreateTableFromIterator, + RsCreateTableFromIterator as RsCreateTableFromIterator, +} from '/snippets/tables.mdx'; import { PyAddFromDataset as AddFromDataset, PyCreateTableFromIterator as CreateTableFromIterator, - TsCreateTableFromIterator as TsCreateTableFromIterator, - RsCreateTableFromIterator as RsCreateTableFromIterator, - TsAddProgress, -} from '/snippets/tables.mdx'; +} from '/snippets/ingestion_examples.mdx'; - -In SQL this is [`INSERT`](/sql/dml/insert), including `INSERT ... SELECT` to move rows server-side. - +import { RsExample as RsAddProgress } from '/snippets/ingestion_progress.mdx'; +import { PyAddProgress, TsAddProgressBody, TsAddProgressTail } from '/snippets/ingestion_examples.mdx'; +import { PyAppendRows } from '/snippets/ingestion_examples.mdx'; +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; -## Ingesting large datasets {#large-datasets} +Use `add()` to append records to an existing table. It does not compare incoming +keys with stored keys. Choose [merge insert](/ingestion/merge-insert) when the +same record may arrive again and should update an existing row. -When ingesting large datasets, use `table.add()` on an existing table rather than -passing all data to `create_table()`. The `add()` method auto-parallelizes large -writes, while `create_table(name, data)` does not. +## Append a first batch {#append} - -For best performance with large datasets, create an empty table first and then call -`table.add()`. This enables automatic write parallelism for materialized data sources. - +Install the Python SDK with `pip install lancedb`. This complete example starts +with Alice, appends Bob and Charlie, and prints all three records. -### From files (Parquet, CSV, etc.) {#from-files-parquet-csv-etc} -Python Only + +{PyAppendRows} + -For file-based data, pass a `pyarrow.dataset.Dataset` to `table.add()`. This streams -data from disk without loading the entire dataset into memory. +The program uses temporary local storage. For a complete TypeScript or Rust +example, see [Complete write workflow](#complete-example). +Enterprise SQL offers [`INSERT`](/sql/dml/insert), including `INSERT ... SELECT`. + +## Prepare the input {#prepare} + +Match the table's schema, including required fields and vector dimensions. +Supported input formats differ by SDK; see +[Create a table](/ingestion/create-table). Inspect conversion errors before +retrying, and verify the final row count and representative values. + +## Choose input for a large load {#large-datasets} + +Avoid building one large in-memory list when records already live in files +or arrive as batches. Create a table with the intended schema, then append +through the appropriate input path. + + +### Read from files {#from-files-parquet-csv-etc} + +Python accepts a `pyarrow.dataset.Dataset`. This example writes a small Parquet +file locally, creates an empty table with its schema, and appends the two records. +For your own data, point the dataset reader at the input files instead. + + +{AddFromDataset} + + +A file-backed source can be scanned without first loading the whole dataset +into application memory. This Python input API is distinct from converting +files to Arrow batches yourself in another language. + + +### Produce batches during ingestion {#from-iterators-custom-batch-generation} + +Use batches when processing the input as it arrives. The Python example creates +an empty table and appends ten records produced by a batch iterator. - - {AddFromDataset} - + +{CreateTableFromIterator} + + +{TsCreateTableFromIterator} + + +{RsCreateTableFromIterator} + - -`pa.dataset()` input is currently Python-only. TypeScript and Rust support for -file-based dataset ingestion is tracked in -[lancedb#3173](https://github.com/lancedb/lancedb/issues/3173). - +The language APIs differ: the TypeScript example appends each constructed Arrow +batch, while the Rust example supplies an Arrow reader when creating the table. +These are input-format examples, not equivalent parallel-write benchmarks. -### From iterators (custom batch generation) {#from-iterators-custom-batch-generation} +### Control write parallelism {#write-parallelism} -When you need custom batch logic — generating embeddings on the fly, transforming -rows from an external source, etc. — use an iterator of `RecordBatch` objects. +Python's `add()` can estimate parallelism for materialized inputs. A streaming +reader may not expose its total size or support replay. Use `write_parallelism` +when controlling the number of concurrent partitions; increasing it can also +increase memory use. TypeScript does not expose that manual option. + +Measure with realistic row sizes and storage. Avoid one write per row when a +batch will do, and avoid a large worker count for a small input. + +## Track completion and progress {#tracking-ingestion-progress} + +Python, TypeScript and Rust support progress callbacks when adding data. +These examples start with Alice, append Bob and Charlie, and print the number +of rows written as progress updates arrive. After the write succeeds, they +query the table to show all three records. + +Each program uses a temporary local database and removes it when finished. + +- **Python:** install `lancedb`, save the code as `insert_progress.py`, and run + `python insert_progress.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `insert_progress.mts`, + and run `node insert_progress.mts` with Node.js 22.18 or later. +- **Rust:** download the [Rust examples](/static/code/ingestion-feedback-rust.zip), + extract the project, and run `cargo run --locked --bin progress`. - - {CreateTableFromIterator} - - - - {TsCreateTableFromIterator} - - - - {RsCreateTableFromIterator} - + +{PyAddProgress} + + +{TsAddProgressBody} +{TsAddProgressTail} + + +{RsAddProgress} + -Use this pattern when: +For a progress bar in Python, install `tqdm` and pass `progress=True` instead +of a callback. -- Your source data already arrives in Arrow batches, readers, datasets, or streams. -- Materializing the entire ingest as one giant in-memory list or array would be too expensive. -- You want to control chunk size explicitly during ingestion. +### Read progress updates -Python can also consume iterators of other supported types like Pandas DataFrames or Python lists. +Updates include row and byte counts, elapsed time, and parallel write activity. +The field names differ by language: -### Write parallelism {#write-parallelism} +| Meaning | Python dictionary key | TypeScript property | Rust method | +|:--------|:----------------------|:--------------------|:------------| +| Rows written so far | `output_rows` | `outputRows` | `output_rows()` | +| Bytes written so far | `output_bytes` | `outputBytes` | `output_bytes()` | +| Expected row count, when known | `total_rows` | `totalRows` | `total_rows()` | +| Elapsed time | `elapsed_seconds` | `elapsedSeconds` | `elapsed()` | +| Active write tasks | `active_tasks` | `activeTasks` | `active_tasks()` | +| Total write tasks | `total_tasks` | `totalTasks` | `total_tasks()` | +| Final update | `done` | `done` | `done()` | - -For materialized data (`pa.Table`, `pd.DataFrame`, `pa.dataset()`), LanceDB -automatically parallelizes large writes — no configuration needed. Auto-parallelism -targets approximately 1M rows or 2GB per write partition. +Python and TypeScript report elapsed time in seconds; Rust returns a `Duration`. +The expected row count may be unknown for streaming input. On the final update, +it falls back to the number of rows written. -For streaming sources (iterators, `RecordBatchReader`), LanceDB cannot determine -total size upfront. Python exposes `write_parallelism` on both synchronous and -asynchronous `table.add()` calls; see the [iterator example](#iterator-ingestion-for-data-you-transform-on-the-fly). -TypeScript does not expose a manual write-parallelism option. - +Keep callbacks short and use the write operation's result to determine success. +The final `done` update means progress reporting has ended; it can also occur +after a write fails or is cancelled. Input errors before writing starts may +produce no updates. -### Tracking ingestion progress {#tracking-ingestion-progress} -TypeScript example +TypeScript delivers callbacks asynchronously. Slow callbacks can queue up, +and the `add()` promise may resolve before every callback has run. Exceptions +from a TypeScript callback are logged and do not abort the write. -In TypeScript, for long-running writes, pass a `progress` callback to `table.add()` to surface -per-batch progress in your UI, logs, or metrics pipeline. Progress updates are dispatched -asynchronously to the JavaScript event loop, including a final `done: true` -update when an active write finishes. +## Verify writes and handle retries {#performance} -Each invocation receives a `WriteProgress` object: +Wait for the write operation's result, then check the expected records. +Progress callbacks describe activity; they do not establish that a write +succeeded. If a request fails after the server may have committed it, blindly +repeating an append can duplicate rows. The client's automatic retries can do +the same: on a remote table, the Python and TypeScript 0.40.0 clients resend an +append after a retryable status such as `504`, even when the server had already +committed it, and `add()` still returns success. Reconcile using your record +IDs, or choose an upsert policy designed for replay. -| Field | Description | -|:------|:------------| -| `outputRows` | Rows written so far. | -| `outputBytes` | Bytes written so far. | -| `totalRows` | Expected total rows when the input source reports one. Always set on the final callback. | -| `elapsedSeconds` | Wall-clock seconds since the write started. | -| `activeTasks` | Parallel write tasks currently in flight. | -| `totalTasks` | Total parallel write tasks (the write parallelism). | -| `done` | `true` only for the final callback. | - - - - {TsAddProgress} - - - -A few things to know before you wire this up: - -- Asynchronous delivery: slow callbacks queue updates without back-pressuring the writer. Keep the callback cheap to avoid accumulating queued updates. -- Completion: an active write emits a final `done: true` update when it finishes, including execution errors or cancellation. Errors before writing starts, such as invalid input, may produce no progress callbacks. Use the `table.add()` promise to determine whether the write succeeded; it does not wait for JavaScript to process every queued callback. -- Errors swallowed: anything your callback throws is logged with `console.warn` and won't abort the write, so keep the callback side-effect-only and don't rely on it for control flow. -- Row totals: `totalRows` is only populated when the input source can report it up front (for example, a materialized `arrow.Table`). For streaming sources it stays `undefined` until the final callback, where it falls back to the actual rows written. - -## Performance {#performance} - -If ingestion is taking longer than expected on a large dataset, the cause is almost always how `add()` is called: each call commits a new version and a new fragment, so a per-row loop pays that per-call overhead at every row. The best practice is to pick the ingestion mode that matches your data shape — bulk ingestion when the data is already materialized, or iterator ingestion when it's streamed or computed on the fly. - - -**Why `merge_insert()` is significantly slower than `add()`** - -A merge has to scan existing data to find matches on the join key (or look them up via a scalar index, if one exists), and then delete-and-reinsert any updated rows in a single transaction; `add()` simply appends new fragments. - -Use `add()` for pure appends, and reach for `merge_insert()` only when you need upsert or conditional-insert logic. When you do use it, build a scalar index on the join column first — otherwise the match step falls back to a full column scan, which is the dominant cost at scale. - - -#### Bulk ingestion: for data you already have {#bulk-ingestion-for-data-you-already-have} - -For materialized inputs (Arrow Tables, DataFrames) and file-backed sources (`pyarrow.dataset(...)`), LanceDB auto-parallelizes the write across workers, estimating the partition count from the data size — more partitions means more concurrent writes and higher throughput, up to the CPU core count. - -```python Python icon="python" -table.add(arrow_table) # in-memory -table.add(df) # pandas -table.add(ds.dataset("data/", format="parquet")) # streams from disk, still parallelized -``` - -Pass `progress=True` to watch it happen: LanceDB shows a live tqdm bar with rows written, throughput in MB/s, and active worker count. - -```python Python icon="python" -table.add(ds.dataset("data/", format="parquet"), progress=True) -``` - -```text -71%|███████▏ | 710000/1000000 [00:12<00:05, 58.8kit/s, 42.3 MB/s | 8/8 workers] -``` - -For larger-than-memory data, prefer scanning a file-backed dataset (`ds.dataset(...)`) over a hand-built `pyarrow.RecordBatchReader`: a `Dataset` can be counted and rescanned, so LanceDB knows the row count upfront (better auto-parallelism) and can retry a failed write from the start — a reader can only be consumed once, so neither is possible. Reach for the iterator path below only when the data genuinely can't be backed by files, e.g. to apply custom data transformations as you ingest. - -For very large initial loads, create the table empty first; passing data directly to `create_table(name, data)` skips the auto-parallel path. - -#### Iterator ingestion: for data you transform on the fly {#iterator-ingestion-for-data-you-transform-on-the-fly} - -If each row needs work before it can be written — applying a custom transformation as you ingest, for example — the data doesn't exist as a file you can point `ds.dataset(...)` at. The best practice is to pass an iterator of `pyarrow.RecordBatch` instead; LanceDB consumes one batch at a time as you produce them. - -```python Python icon="python" -def stream(): - for raw in source: - vectors = model.encode(raw["text"]) - yield pa.RecordBatch.from_pydict({**raw, "vector": vectors}) - -table.add(stream()) -``` - -Use decently large chunks of several thousand rows or more, rather than yielding single-row batches. - - -**Set `write_parallelism` yourself for large inputs** - -A reader can't be counted or rescanned the way a `Dataset` can (see above), so LanceDB can't auto-size parallelism for it — set `write_parallelism` explicitly for large inputs: - -```python Python icon="python" -table.add(stream(), write_parallelism=4) -``` - -Each partition becomes its own fragment, so don't over-allocate on a small input — budget one unit of parallelism per ~100K rows or ~1 GB of data as a rule of thumb. - - -#### Bulk ingests into a remote table {#bulk-ingests-into-a-remote-table} +## Configure large uploads {#bulk-ingests-into-a-remote-table} Enterprise This section applies only to remote tables (`db://` connections). Embedded (local) writes are unaffected. -When you write to a remote LanceDB Enterprise table, the client splits each write partition into one or more HTTP `insert` parts and uploads them under a single upload id. Each part is streamed rather than buffered, so peak memory stays bounded. But each part still has to finish within the client read timeout while the server writes it to object storage, so on very large ingests an oversized part can run past that timeout and surface as: +When you write to a remote LanceDB Enterprise table, the client sends the write as one HTTP `insert` request unless it splits the data into more than one partition, which it does for a large input it can measure or when you set [`write_parallelism`](#write-parallelism). A partitioned write goes up as a multipart write: the client cuts each partition into one or more `insert` parts and uploads them under a single upload id. Each part is streamed. Its size and duration still affect resource use and +timeouts: it has to finish within the client read timeout while the server writes it to object storage, so on very large ingests an oversized part can run past that timeout and surface as: ```text lancedb.remote.errors.HttpError: operation timed out ``` -Two environment variables control how parts are cut. Both are picked up automatically by the Python and TypeScript clients, and are the only way to tune this from those SDKs: +Two environment variables control how a multipart write's parts are cut. Both are picked up automatically by the Python and TypeScript clients, for clients that support these settings: | Variable | Default | What it controls | |----------|---------|------------------| | `LANCE_CLIENT_MAX_BYTES_PER_REQUEST` | 8 GiB | Maximum size of a single insert part, in LZ4-compressed Arrow IPC bytes. Set to `0` to disable splitting and send one request per partition. | | `LANCE_CLIENT_MAX_REQUEST_DURATION` | Half the read timeout | Maximum wall-clock time, in whole seconds, that any one insert part may stay open. Set to `0` to disable the time-based cut. | -A part is cut when it hits either limit, whichever comes first. Lower the byte budget when large ingests hit the read timeout; lower the duration when uploads are slow or throttled and the byte budget isn't the limiting factor. +A part is cut after the input batch that reaches either limit, so cuts fall between batches. Lower the byte budget, with smaller input batches, when large ingests hit the read timeout; lower the duration when uploads are slow or throttled and the byte budget isn't the limiting factor. ```bash export LANCE_CLIENT_MAX_BYTES_PER_REQUEST=1073741824 # 1 GiB export LANCE_CLIENT_MAX_REQUEST_DURATION=120 # 2 minutes ``` -Splitting into more parts does not change the final table. The server stages every part under the shared upload id and merges them atomically when the write completes. +Splitting into more parts does not change the final table. The server stages the parts under their shared upload ID before committing +the write. This upload protocol is separate from repeating an application-level +`add()` after a timeout. + +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/ingestion/merge-insert.mdx b/docs/web/ingestion/merge-insert.mdx index 8bfb6084a..6fa59fd9c 100644 --- a/docs/web/ingestion/merge-insert.mdx +++ b/docs/web/ingestion/merge-insert.mdx @@ -1,19 +1,16 @@ --- title: "Merge insert" sidebarTitle: "Merge Insert" -description: "Reconcile incoming rows against a key: update what matches, insert what does not." +description: "Match incoming records by key and choose which rows to update, insert or remove." icon: code-merge +availability: + oss: available + enterprise: available + summary: >- + Merge insert is available through the SDKs in OSS and Enterprise. Use it for key-based updates and inserts. --- -These examples use the table set up in -[Ingesting data](/ingestion), and the connection made there. - import { - PyMergeMatchedUpdateOnly as MergeMatchedUpdateOnly, - PyInsertIfNotExists as InsertIfNotExists, - PyMergeUpdateInsert as MergeUpdateInsert, - PyMergeDeleteMissingBySource as MergeDeleteMissingBySource, - PyMergePartialColumns as MergePartialColumns, TsMergeMatchedUpdateOnly, TsInsertIfNotExists, TsMergeUpdateInsert, @@ -25,15 +22,24 @@ import { RsMergeDeleteMissingBySource, RsMergePartialColumns, } from '/snippets/tables.mdx'; +import { + PyMergeMatchedUpdateOnly as MergeMatchedUpdateOnly, + PyInsertIfNotExists as InsertIfNotExists, + PyMergeUpdateInsert as MergeUpdateInsert, + PyMergeDeleteMissingBySource as MergeDeleteMissingBySource, + PyMergePartialColumns as MergePartialColumns, +} from '/snippets/ingestion_examples.mdx'; - -Merge insert has **no SQL form** -- it needs a source relation and a per-match policy that one statement cannot express. See [DML](/sql/dml/insert#next). - +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; + +Use merge insert when incoming records should be compared with existing rows. +Combine update and insert policies for an **upsert**. Use [append](/ingestion/insert) +when every incoming row should simply be added. ## Merge incoming rows by key {#merge-incoming-rows-by-key} Merging is different from updating because it involves comparing incoming rows to existing rows by key, and then choosing what to do based on whether the key exists in the target table or not. -The `merge_insert(""..."")` method lets you do this. +Use `merge_insert("id")` in Python or the corresponding SDK builder. In merge operations, rows are split into three groups: @@ -45,37 +51,28 @@ Conditional merge clauses can compare old and new values. Use the `target.` pref existing table row and `source.` for the incoming row, for example `target.last_update < source.last_update`. - -**Use scalar indexes to speed up merge insert** +Choose a key that identifies the same record in the source and target. Check +for duplicate and null keys before merging. A primary-key declaration does not +enforce uniqueness on ordinary writes, and append operations do not deduplicate +records for you. -The merge insert command performs a join between the input data and the target table `on` the key you provide. This requires scanning that entire column, which can be expensive for large tables. To speed up this operation, create a scalar index on the join column, which will allow LanceDB to find matches without scanning the whole table. +A scalar index on the key can help matching. Keep it current and measure the +actual workload rather than assuming every merge uses an index. Enterprise may +reject a merge when the unindexed scan exceeds its configured limit; inspect +that error and the index state before retrying. -Read more about scalar indices in the [Scalar Index](/indexing/scalar-index/) guide. - +## Choose a merge policy {#policies} - -If you see this HTTP 400 error from `merge_insert`: `Bad request: Merge insert cannot be performed because the number of unindexed rows exceeds the maximum of 10000`. Verify that the scalar index on the join column is up to date before retrying. - +| Source and target relationship | Available action | +| --- | --- | +| Key appears in both | Update matching target rows, optionally with a condition. | +| Key appears only in the source | Insert it, or leave it out. | +| Key appears only in the target | Keep it, or explicitly delete it under a synchronization policy. | - -**Rust: build merge predicates with DataFusion expressions** - -In the Rust SDK, you can pass a `datafusion_expr::Expr` directly instead of a SQL string by using -`when_matched_update_all_expr` and `when_not_matched_by_source_delete_expr` on `MergeInsertBuilder`. -This is useful when you are constructing predicates programmatically and want to avoid building a SQL string. - -These methods are only supported on local tables. Calling them against a remote table returns a -`NotSupported` error — use the SQL string variants (`when_matched_update_all` / -`when_not_matched_by_source_delete`) for remote tables. - - -Like the create table and add APIs, the merge insert API will automatically compute embeddings based on the [embedding registry](/embedding/index#embedding-registry) if the table has an embedding definition in its schema. - -During `merge_insert`, if the input data doesn't contain the source column (i.e., the original field used to generate embeddings, such as text for a text embedding model or `image_uri` for an image model), or if a vector value is already provided, LanceDB skips embedding generation for that row. Embeddings are only auto-generated when that source field is present in the incoming data, **and** the vector field is empty. - -Primary keys in LanceDB are metadata used by operations such as `merge_insert`; they are not -enforced as uniqueness constraints on ordinary writes. Keep using `merge_insert` or your own -deduplication logic when you need key-based upsert semantics. +The following examples are independent. Each starts from the table shown in +its code, so the expected results do not depend on running the previous example. +Python blocks include their own temporary connection. For a full TypeScript or +Rust program, see [Complete write workflow](#complete-example). ### Update matched rows only {#update-matched-rows-only} @@ -160,7 +157,10 @@ Expected table contents: ### Delete target rows that are missing from source {#delete-target-rows-that-are-missing-from-source} -Use `when_not_matched_by_source_delete()` when you want to remove any target row that does not appear in the incoming source data. +Use `when_not_matched_by_source_delete()` only when the incoming data is the +complete set for the target scope you intend to synchronize. An incremental +batch is usually not a complete source: deleting everything absent from it +could remove unrelated target records. Inspect the unmatched target keys first. @@ -183,7 +183,8 @@ Expected table contents: | 2 | Bobby | 21 | | 3 | Charlie | 5 | -In the example above, LanceDB matches rows by `id`. Rows with `id=2` and `id=3` exist in both the table and incoming data, so they are updated. Row `id=1` exists only in the target, so it is deleted. +Here, IDs 2 and 3 are updated and ID 1 is deleted. This policy synchronizes +the target to the supplied source set. ### Use partial columns in merge updates {#use-partial-columns-in-merge-updates} @@ -212,3 +213,54 @@ Expected table contents: | 3 | Charlie | null | Note that in the example above, when `merge_insert` creates a new row, any missing columns are written as `null`. If a missing column is non-nullable in your schema, the insert will fail. + +## Check the merged records {#verify} + +Inspect the keys that should have changed, confirm that unrelated rows remain, +and check inserted or deleted counts when the SDK returns them. Repeating a +merge is only safe when its keys, conditions and update values produce the +intended result on retry. + +### Embeddings and expression predicates {#advanced} + +When a table uses an [embedding definition](/embedding), behavior depends on +which source and vector fields the incoming records provide. Do not assume a +partial metadata update recomputes embeddings. Inspect the resulting vector +when changing the input used to generate it. + +Rust's `when_matched_update_all_expr` and +`when_not_matched_by_source_delete_expr` accept DataFusion expressions for +local tables. Use the string predicate variants for Enterprise tables. + +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/ingestion/update.mdx b/docs/web/ingestion/update.mdx index e77587ea8..d44a2561e 100644 --- a/docs/web/ingestion/update.mdx +++ b/docs/web/ingestion/update.mdx @@ -1,82 +1,123 @@ --- title: "Update rows" sidebarTitle: "Update" -description: "Change values in rows that already exist, by predicate or SQL expression." +description: "Select existing rows, change values or expressions, and verify the changes." icon: pen +availability: + oss: available + enterprise: available + summary: >- + Update rows through the SDKs in OSS and Enterprise. Enterprise also supports the SQL UPDATE statement. --- -These examples use the table set up in -[Ingesting data](/ingestion), and the connection made there. - import { - PyUpdateOperation as UpdateOperation, - PyUpdateUsingSql as UpdateUsingSql, TsUpdateOperation, TsUpdateUsingSql, RsUpdateOperation, RsUpdateUsingSql, } from '/snippets/tables.mdx'; +import { + PyUpdateOperation as UpdateOperation, + PyUpdateUsingSql as UpdateUsingSql, +} from '/snippets/ingestion_examples.mdx'; - -In SQL this is [`UPDATE`](/sql/dml/update). - +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; -## Update rows {#update-rows} +Use `update` to change rows selected by a predicate. Use +[merge insert](/ingestion/merge-insert) instead when matching an incoming dataset +against existing keys. -Use `update` when you already know which target rows to modify and you do not need to compare against an incoming dataset. +## Select the rows to change {#select-rows} + +Check the predicate with a [filtered query](/search/filtering) before applying +the update. These examples target the row with `id = 2`. Review the scope +carefully when omitting a predicate, since that can update the entire table. + +Each example below starts from Alice with 10 logins and Bob with 20. The +Python programs create and remove their own temporary database. TypeScript +and Rust operation blocks use `db`; a complete program is provided below. + +## Assign a new value {#update-rows} - - {UpdateOperation} - - - - {TsUpdateOperation} - - - - {RsUpdateOperation} - + +{UpdateOperation} + + +{TsUpdateOperation} + + +{RsUpdateOperation} + -Expected table contents: +The result contains Alice and **Bobby**, with their login counts unchanged. +The Python `values` and TypeScript `values` arguments take literal values. +Rust's `column` argument is an expression, so the string value includes SQL quotes. -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bobby | 20 | - - -Updating nested columns is not yet supported. - - -## Update rows with SQL expressions {#update-rows-with-sql-expressions} - -Use `values_sql` when you want to use SQL-like expressions to update rows. This is useful for operations like incrementing a counter, or setting a column value based on another column. +## Calculate a value from the existing row {#update-rows-with-sql-expressions} - - {UpdateUsingSql} - - - - {TsUpdateUsingSql} - - - - {RsUpdateUsingSql} - + +{UpdateUsingSql} + + +{TsUpdateUsingSql} + + +{RsUpdateUsingSql} + -Expected table contents: +In this separate example, Bob keeps his name and his login count becomes **21**. +Python's `values_sql` and TypeScript's `valuesSql` accept expressions. +These SDK expressions work with embedded OSS tables; they do not require an +Enterprise SQL endpoint. Enterprise also offers [`UPDATE`](/sql/dml/update). -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bob | 21 | +## Verify the affected data {#verify} - -See the [SQL queries](/sql#connecting) page for more information on the supported SQL syntax. - +Read the selected rows again and inspect the changed fields. For another +connection or table handle, account for its +[read consistency](/tables/consistency) settings. -When rows are updated, they are moved out of any existing index. The row will still show up in search queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider triggering an index rebuild afterwards. +Nested-field updates are not supported by these operations. For schema changes, +see [schema evolution](/management/table#schema). Treat derived values separately: +changing an input field is not a guarantee that every stored feature has been +recomputed. See [Refresh a column](/feature-engineering/refresh-column). + +Updates can change index coverage and query cost. After a large update, +inspect the workload and follow [index maintenance](/indexing/reindexing) +guidance for the backend you use. + +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/integrations/ai/hermes-agent.mdx b/docs/web/integrations/ai/hermes-agent.mdx index a5521a9d8..621b53129 100644 --- a/docs/web/integrations/ai/hermes-agent.mdx +++ b/docs/web/integrations/ai/hermes-agent.mdx @@ -29,7 +29,7 @@ prompt, plus lexical (keyword) search over past sessions. Both are useful, but k misses paraphrases of what you originally typed — the exact thing you need when recalling a fact you phrased differently months ago. -LanceDB is an embedded retrieval library, which makes it a natural fit here: +LanceDB OSS is an embedded retrieval library, which makes it a natural fit here: - **No server to stand up** — it reads and writes a table on local disk, so the plugin ships as a dependency rather than a service to operate. diff --git a/docs/web/integrations/ai/huggingface.mdx b/docs/web/integrations/ai/huggingface.mdx index 62ac7e8cd..0a38c208b 100644 --- a/docs/web/integrations/ai/huggingface.mdx +++ b/docs/web/integrations/ai/huggingface.mdx @@ -2,6 +2,12 @@ title: "Hugging Face Hub" sidebarTitle: "Hugging Face" description: "Use LanceDB directly on Lance datasets hosted on the Hugging Face Hub for multimodal search and retrieval." +availability: + oss: available + enterprise: unavailable + summary: >- + The embedded library opens Lance datasets on the Hugging Face Hub in place + through `hf://` URIs. A deployment does not read `hf://` datasets. --- [Hugging Face Hub](https://huggingface.co/datasets?format=format:lance&sort=trending) is a popular platform for sharing machine learning datasets, models, and other resources. diff --git a/docs/web/integrations/ai/langchain.mdx b/docs/web/integrations/ai/langchain.mdx index 363331093..14ed628a8 100644 --- a/docs/web/integrations/ai/langchain.mdx +++ b/docs/web/integrations/ai/langchain.mdx @@ -80,6 +80,9 @@ It returns list of ids of the added texts. {PyFrameworksLangchainAddTexts} +With LanceDB Enterprise, `tbl.to_pandas()` raises in the synchronous Python client; +read the rows with a query instead, such as `tbl.search().limit(100).to_pandas()`. + ------ diff --git a/docs/web/integrations/data/duckdb.mdx b/docs/web/integrations/data/duckdb.mdx index 704a47c60..23c4891f9 100644 --- a/docs/web/integrations/data/duckdb.mdx +++ b/docs/web/integrations/data/duckdb.mdx @@ -6,7 +6,7 @@ description: "Learn how to use the DuckDB-Lance extension to query Lance tables LanceDB integrates with [DuckDB](https://duckdb.org/) through the [Lance extension](https://github.com/lance-format/lance-duckdb) for DuckDB. In this page, we'll show how LanceDB manages table lifecycle, and DuckDB provides SQL analytics (including joins) and search over those tables. -Note that earlier versions of LanceDB used to recommend converting Lance tables to Arrow tables via `table.to_arrow()`. Although this method is still available (because DuckDB [natively scans Arrow tables](https://duckdb.org/2021/12/03/duck-arrow)), it is no longer the recommended workflow for working with Lance tables in DuckDB. This page shows how to use the Lance extension with namespace-attached LanceDB tables, allowing you to pushdown SQL queries directly to the Lance layer. +Note that earlier versions of LanceDB used to recommend converting Lance tables to Arrow tables via `table.to_arrow()`. Although this method is still available for local tables (because DuckDB [natively scans Arrow tables](https://duckdb.org/2021/12/03/duck-arrow)), it is no longer the recommended workflow for working with Lance tables in DuckDB. This page shows how to use the Lance extension with namespace-attached LanceDB tables, allowing you to pushdown SQL queries directly to the Lance layer. ## Install {#install} diff --git a/docs/web/integrations/data/pandas_and_pyarrow.mdx b/docs/web/integrations/data/pandas_and_pyarrow.mdx index 170ccf772..c646e7f3b 100644 --- a/docs/web/integrations/data/pandas_and_pyarrow.mdx +++ b/docs/web/integrations/data/pandas_and_pyarrow.mdx @@ -1,6 +1,12 @@ --- title: "Pandas and PyArrow" sidebarTitle: "Pandas & PyArrow" +availability: + oss: available + enterprise: available + summary: >- + DataFrames go into `create_table` and come back from query results the same + way for local and deployment tables. --- diff --git a/docs/web/integrations/data/polars_arrow.mdx b/docs/web/integrations/data/polars_arrow.mdx index 4cb85737e..3acfcfea5 100644 --- a/docs/web/integrations/data/polars_arrow.mdx +++ b/docs/web/integrations/data/polars_arrow.mdx @@ -1,6 +1,13 @@ --- title: "Polars" sidebarTitle: "Polars & Arrow" +availability: + oss: available + enterprise: available + summary: >- + Polars data goes into `create_table`, and query results come back as Polars + DataFrames, for local and deployment tables alike; a LazyFrame over a whole + table is local only. --- @@ -36,7 +43,7 @@ Run vector search and keep the results as a Polars `DataFrame` for further proce ## Work with LazyFrames {#work-with-lazyframes} -When you want to operate on the entire table (potentially larger than RAM), convert to a Polars `LazyFrame` so you can chain transformations without loading everything at once. +When you want to operate on the entire table (potentially larger than RAM), convert to a Polars `LazyFrame` so you can chain transformations without loading everything at once. This works for local tables only; with LanceDB Enterprise, `search(...).to_polars()` returns an eager DataFrame of query results. {PyPlatformsPolarsLazyframe} diff --git a/docs/web/integrations/data/pydantic.mdx b/docs/web/integrations/data/pydantic.mdx index b6f80aca3..3ac660774 100644 --- a/docs/web/integrations/data/pydantic.mdx +++ b/docs/web/integrations/data/pydantic.mdx @@ -1,6 +1,13 @@ --- title: "Pydantic" sidebarTitle: "Pydantic" +availability: + oss: available + enterprise: available + summary: >- + The SDK converts `LanceModel` schemas to Arrow, and query rows back to + models, in your process, so they work the same with local and deployment + tables. --- diff --git a/docs/web/integrations/embedding/cohere.mdx b/docs/web/integrations/embedding/cohere.mdx index 8051462a9..b9a8e3378 100644 --- a/docs/web/integrations/embedding/cohere.mdx +++ b/docs/web/integrations/embedding/cohere.mdx @@ -8,6 +8,10 @@ import { PyEmbeddingCohereUsage } from '/snippets/integrations.mdx'; Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification. You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API. +```shell +export COHERE_API_KEY="your-api-key" +``` + Supported models are: - embed-english-v3.0 diff --git a/docs/web/integrations/embedding/ibm.mdx b/docs/web/integrations/embedding/ibm.mdx index cec6c32bf..6dd3d57e3 100644 --- a/docs/web/integrations/embedding/ibm.mdx +++ b/docs/web/integrations/embedding/ibm.mdx @@ -3,7 +3,7 @@ title: IBM watsonx sidebarTitle: IBM WatsonX --- -import { PyEmbeddingIbmUsage } from '/snippets/integrations.mdx'; +import { PyEmbeddingIbmUsage } from '/snippets/provider_examples.mdx'; Generate text embeddings using IBM's watsonx.ai platform. @@ -54,9 +54,13 @@ Optionally set environment variables (if not passing credentials to `create` dir export WATSONX_API_KEY="YOUR_WATSONX_API_KEY" # Provide exactly one of the following: export WATSONX_PROJECT_ID="YOUR_WATSONX_PROJECT_ID" -export WATSONX_SPACE_ID="YOUR_WATSONX_SPACE_ID" +# export WATSONX_SPACE_ID="YOUR_WATSONX_SPACE_ID" ``` +Replace the credential placeholders with your provider credentials. The registry +variable keeps the API key out of stored embedding configuration. In application +code, load its value from environment variables or a secrets manager. + {PyEmbeddingIbmUsage} diff --git a/docs/web/integrations/embedding/jina.mdx b/docs/web/integrations/embedding/jina.mdx index 70457180d..588500d36 100644 --- a/docs/web/integrations/embedding/jina.mdx +++ b/docs/web/integrations/embedding/jina.mdx @@ -6,7 +6,7 @@ sidebarTitle: Jina import { PyEmbeddingJinaText, PyEmbeddingJinaMultimodal, -} from '/snippets/integrations.mdx'; +} from '/snippets/provider_examples.mdx'; ## Text Embedding Models {#text-embedding-models} @@ -23,6 +23,8 @@ Supported parameters (to be passed in `create` method) are: Usage Example: +Replace `your-api-key` with your Jina API key. This provider reads it from `JINA_API_KEY`; the example sets it explicitly. In application code, supply it through your environment or a secrets manager. + {PyEmbeddingJinaText} diff --git a/docs/web/integrations/embedding/openai.mdx b/docs/web/integrations/embedding/openai.mdx index cdd60d7cd..fbfa62d9d 100644 --- a/docs/web/integrations/embedding/openai.mdx +++ b/docs/web/integrations/embedding/openai.mdx @@ -10,9 +10,15 @@ LanceDB registers the OpenAI embeddings function in the registry by default, as | Parameter | Type | Default Value | Description | |---|---|---|---| | `name` | `str` | `"text-embedding-ada-002"` | The name of the model. | -| `dim` | `int` | Model default | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it | +| `dim` | `int` | Model default | Requested output dimensions for models that support shortening embeddings. Use `func.ndims()` when defining the vector field. | | `use_azure` | bool | `False` | Set true to use Azure OpenAI SDK | +Set your OpenAI API key before running the example: + +```shell +export OPENAI_API_KEY="your-api-key" +``` + {PyEmbeddingOpenaiBasic} diff --git a/docs/web/integrations/embedding/superlinked.mdx b/docs/web/integrations/embedding/superlinked.mdx index f35796230..bb2768d74 100644 --- a/docs/web/integrations/embedding/superlinked.mdx +++ b/docs/web/integrations/embedding/superlinked.mdx @@ -36,6 +36,9 @@ Supported parameters on `.create()`: ## Usage {#usage} ```py Python icon=Python +import tempfile +from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector @@ -47,22 +50,23 @@ sie = get_registry().get("sie").create( ) class Documents(LanceModel): + id: int text: str = sie.SourceField() vector: Vector(sie.ndims()) = sie.VectorField() -db = lancedb.connect("~/.lancedb") -table = db.create_table("docs", schema=Documents, mode="overwrite") +db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "superlinked-demo")) +table = db.create_table("docs", schema=Documents) table.add([ - {"text": "Machine learning is a subset of AI."}, - {"text": "Neural networks use multiple layers."}, - {"text": "Python is popular for ML development."}, + {"id": 1, "text": "Machine learning is a subset of AI."}, + {"id": 2, "text": "Neural networks use multiple layers."}, + {"id": 3, "text": "Python is popular for ML development."}, ]) results = table.search("What is deep learning?").limit(3).to_list() ``` -LanceDB handles embedding generation for both inserts and queries automatically, based on the `SourceField` / `VectorField` declarations on the schema. +LanceDB handles embedding generation for both inserts and queries automatically, based on the `SourceField` / `VectorField` declarations on the schema. The `id` column gives each document a stable key, which [entity extraction](#entity-extraction) uses to merge its results back. ## Hybrid search with reranker {#hybrid-search-with-reranker} @@ -88,7 +92,7 @@ for r in results: The reranker also works with pure vector or pure FTS search via `.rerank()`. ## ColBERT and multivector {#colbert-and-multivector} -`SIEMultiVectorEmbeddingFunction` (registered as `"sie-multivector"`) works with LanceDB's native `MultiVector` type and MaxSim scoring for ColBERT and ColPali models: +`SIEMultiVectorEmbeddingFunction` (registered as `"sie-multivector"`) works with LanceDB's native `MultiVector` type and MaxSim scoring for ColBERT and ColPali models. In LanceDB Python 0.40.0, a multivector that LanceDB generates on insert is stored as null unless the document's token count happens to equal the per-token dimension, so this example computes the document multivectors with `compute_source_embeddings()` and passes them in the `vector` column. LanceDB still generates the query multivector automatically: ```py Python icon=Python from lancedb.pydantic import MultiVector @@ -102,8 +106,12 @@ class ColBERTDocs(LanceModel): text: str = sie_colbert.SourceField() vector: MultiVector(sie_colbert.ndims()) = sie_colbert.VectorField() -table = db.create_table("colbert_docs", schema=ColBERTDocs, mode="overwrite") -table.add([{"text": "Machine learning is a subset of AI."}]) +table = db.create_table("colbert_docs", schema=ColBERTDocs) + +# Compute the document multivectors and store them with their text +texts = ["Machine learning is a subset of AI."] +vectors = sie_colbert.compute_source_embeddings(texts) +table.add([{"text": text, "vector": vector} for text, vector in zip(texts, vectors)]) # MaxSim search - query and document multivectors compared token-by-token results = table.search("What is ML?").limit(5).to_list() @@ -111,7 +119,7 @@ results = table.search("What is ML?").limit(5).to_list() ## Entity extraction {#entity-extraction} -`SIEExtractor` adds entity extraction to LanceDB's data-enrichment workflows. Extract entities from a text column and merge the results back as a structured Arrow column - enabling filtered search on extracted entities: +`SIEExtractor` adds entity extraction to LanceDB's data-enrichment workflows. Extract entities from a text column and merge the results back as a structured Arrow column - enabling filtered search on extracted entities. This example enriches the `docs` table from [Usage](#usage), matching rows on its `id` column: ```py Python icon=Python from sie_lancedb import SIEExtractor @@ -121,8 +129,9 @@ extractor = SIEExtractor( model="urchade/gliner_multi-v2.1", ) +docs = db.open_table("docs") extractor.enrich_table( - table, + docs, source_column="text", target_column="entities", labels=["person", "technology", "organization"], diff --git a/docs/web/integrations/embedding/voyageai.mdx b/docs/web/integrations/embedding/voyageai.mdx index 54d155d1b..8babfc214 100644 --- a/docs/web/integrations/embedding/voyageai.mdx +++ b/docs/web/integrations/embedding/voyageai.mdx @@ -11,6 +11,10 @@ Voyage AI provides cutting-edge embedding and rerankers. Using voyageai API requires voyageai package, which can be installed using `pip install voyageai`. Voyage AI embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification. You also need to set the `VOYAGE_API_KEY` environment variable to use the VoyageAI API. +```shell +export VOYAGE_API_KEY="your-api-key" +``` + Supported models are: - voyage-4-large (best retrieval quality, 1024 default dimensions, supports 256/512/1024/2048) diff --git a/docs/web/integrations/lerobotdataset.mdx b/docs/web/integrations/lerobotdataset.mdx index 08cf8bd15..cfb66b69b 100644 --- a/docs/web/integrations/lerobotdataset.mdx +++ b/docs/web/integrations/lerobotdataset.mdx @@ -143,9 +143,12 @@ class EmbedFromBlob: # SigLIP2 loads once per worker **Curation** is one query across all of it. Semantic search with a predicate on the derived score, no join, no second system: ```python -tbl.search(vec("a gripper closing on an object"), vector_column_name="emb_siglip2") \ - .where("jerk_score > 1.2875") # the roughest 1% of frames - .limit(4) +results = ( + tbl.search(vec("a gripper closing on an object"), vector_column_name="emb_siglip2") + .where("jerk_score > 1.2875") # the roughest 1% of frames + .limit(4) + .to_arrow() +) ``` **Mining** is the same query pointed at the rare cases. `tbl.search().where("jerk_score > 1.2875 AND success = false")` pulls the rough, failed frames out of 27.6M rows in milliseconds, ready to become a fine-tuning set or a labeling queue. diff --git a/docs/web/integrations/reranking/cohere.mdx b/docs/web/integrations/reranking/cohere.mdx index e484f3ee0..26b98a7c8 100644 --- a/docs/web/integrations/reranking/cohere.mdx +++ b/docs/web/integrations/reranking/cohere.mdx @@ -5,7 +5,7 @@ description: Integrate Cohere's powerful reranking API with LanceDB for enhanced --- -import { PyRerankingCohereUsage } from '/snippets/integrations.mdx'; +import { PyRerankingCohereUsage } from '/snippets/provider_examples.mdx'; # Cohere Reranker @@ -18,6 +18,8 @@ This reranker uses the [Cohere](https://cohere.ai/) API to rerank the search res pip install cohere ``` +Replace the credential placeholders with your provider credentials. In application code, load credentials from environment variables or a secrets manager. + {PyRerankingCohereUsage} diff --git a/docs/web/integrations/reranking/jina.mdx b/docs/web/integrations/reranking/jina.mdx index 4624149d2..450a18ecf 100644 --- a/docs/web/integrations/reranking/jina.mdx +++ b/docs/web/integrations/reranking/jina.mdx @@ -5,7 +5,7 @@ description: Integrate Jina's multilingual reranking API with LanceDB for improv --- -import { PyRerankingJinaUsage } from '/snippets/integrations.mdx'; +import { PyRerankingJinaUsage } from '/snippets/provider_examples.mdx'; # Jina Reranker @@ -15,6 +15,8 @@ This reranker uses the [Jina](https://jina.ai/reranker/) API to rerank the searc > **Note:** Supported query types – Hybrid, Vector, and FTS. +Replace the credential placeholders with your provider credentials. In application code, load credentials from environment variables or a secrets manager. + {PyRerankingJinaUsage} diff --git a/docs/web/integrations/reranking/openai.mdx b/docs/web/integrations/reranking/openai.mdx index 413f95e67..22d2af2cc 100644 --- a/docs/web/integrations/reranking/openai.mdx +++ b/docs/web/integrations/reranking/openai.mdx @@ -5,7 +5,7 @@ description: Explore experimental search reranking using OpenAI's GPT models in --- -import { PyRerankingOpenaiUsage } from '/snippets/integrations.mdx'; +import { PyRerankingOpenaiUsage } from '/snippets/provider_examples.mdx'; # OpenAI Reranker (Experimental) @@ -14,6 +14,9 @@ This reranker uses OpenAI chat model to rerank the search results. You can use t > **Warning:** This reranker is experimental. OpenAI does not have a dedicated reranking model, so it uses a chat model under the hood. +Replace `your-api-key` with your provider API key. In application code, load +credentials from environment variables or a secrets manager. + {PyRerankingOpenaiUsage} diff --git a/docs/web/integrations/reranking/voyageai.mdx b/docs/web/integrations/reranking/voyageai.mdx index 62e9a16da..80a666f56 100644 --- a/docs/web/integrations/reranking/voyageai.mdx +++ b/docs/web/integrations/reranking/voyageai.mdx @@ -5,7 +5,7 @@ description: Integrate VoyageAI's cutting-edge reranking models with LanceDB. Fe --- -import { PyRerankingVoyageaiUsage } from '/snippets/integrations.mdx'; +import { PyRerankingVoyageaiUsage } from '/snippets/provider_examples.mdx'; # VoyageAI Reranker @@ -17,6 +17,9 @@ This reranker uses the [VoyageAI](https://docs.voyageai.com/docs/) API to rerank > **Note:** Supported query types – Hybrid, Vector, and FTS. +Replace `your-api-key` with your provider API key. In application code, load +credentials from environment variables or a secrets manager. + {PyRerankingVoyageaiUsage} diff --git a/docs/web/integrations/reranking/watsonx.mdx b/docs/web/integrations/reranking/watsonx.mdx index 775af4f15..5e4c6322c 100644 --- a/docs/web/integrations/reranking/watsonx.mdx +++ b/docs/web/integrations/reranking/watsonx.mdx @@ -5,7 +5,7 @@ description: Rerank LanceDB search results with the IBM watsonx.ai text rerank A --- -import { PyRerankingWatsonxUsage } from '/snippets/integrations.mdx'; +import { PyRerankingWatsonxUsage } from '/snippets/provider_examples.mdx'; # Watsonx Reranker @@ -17,6 +17,8 @@ This reranker uses the [IBM watsonx.ai](https://cloud.ibm.com/docs/apis/watsonx- pip install ibm-watsonx-ai ``` +Replace the credential placeholders with your provider credentials. In application code, load credentials from environment variables or a secrets manager. + {PyRerankingWatsonxUsage} diff --git a/docs/web/reranking/index.mdx b/docs/web/reranking/index.mdx index e17b43552..26db30442 100644 --- a/docs/web/reranking/index.mdx +++ b/docs/web/reranking/index.mdx @@ -1,96 +1,100 @@ --- title: "Reranking Search Results" sidebarTitle: "Reranking" -description: "Use a reranker to improve search relevance." +description: "Choose a reranker, apply it to search candidates and evaluate relevance and latency." icon: "sort-amount-down" keywords: ["re-ranking", "reranker", "rerank"] +availability: + oss: available + enterprise: available + summary: >- + Reranking is available with OSS and Enterprise. --- import { PyRerankingLinearCombinationUsage, PyRerankingRrfUsage, - PyRerankingCohereUsage, PyRerankingMrrUsage, PyRerankingCrossEncoderUsage, } from '/snippets/integrations.mdx'; +import { PyRerankingCohereUsage } from '/snippets/provider_examples.mdx'; -Reranking is the process of re-ordering search results to improve relevance, often using a -different model than the one used for the initial search. LanceDB has built-in support for reranking -with models from Cohere, Sentence-Transformers, and more. +import { PyHybridSearch, TsHybridSearchBody, TsHybridSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsHybridSearchBody, RsExampleTail as RsHybridSearchTail } from '/snippets/search_hybrid_search.mdx'; -### Quickstart {#quickstart} +Reranking changes the order of records returned by an initial search. It can +combine several ranked lists or use a model to score each candidate against +the query. It cannot recover records that the initial searches did not retrieve. -To use a reranker, you run a search and pass the results to the `rerank()` method. The examples below -move from the simplest, model-free rerankers to a model-based one. Each is a complete, runnable script. +## Choose a reranker -**1. Linear combination (simplest).** `LinearCombinationReranker` normalizes the vector and full-text -scores and blends them with a single `weight` (default `0.7`, favoring the vector score). It runs no -model, and reranks [hybrid search](/search/hybrid-search) results. +- **Rank fusion:** start with RRF for hybrid search. It combines vector and + full-text rankings without a model or a provider API call. +- **Model-based reranking:** try a cross encoder or hosted reranking model + when you need a different relevance judgment over the retrieved text. + Compare the relevance gain with added latency and provider cost. +- **Custom ranking:** implement the reranker interface when your application + needs its own scoring policy. + + + + + +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. + +- **Python:** Install with `pip install lancedb`, save as `rerank_search.py`, then run `python rerank_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `rerank_search.mts`, then run `node rerank_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin hybrid_search` from the extracted directory. - - {PyRerankingLinearCombinationUsage} - + +{PyHybridSearch} + + +{TsHybridSearchBody + "\n" + TsHybridSearchTail} + + +{RsHybridSearchBody + "\n" + RsHybridSearchTail} + -**2. Reciprocal Rank Fusion.** `RRFReranker` fuses results by rank position instead of raw score, so it -sidesteps having to make vector and full-text scores comparable. It loads no model either, and it's the -default reranker for hybrid search. +The result includes `_relevance_score` in descending order. RRF is already the +hybrid-search default; passing it explicitly shows where to substitute another +reranker. These scores express a ranking, not a probability of correctness. - - - {PyRerankingRrfUsage} - - + -**3. Cohere (model-based).** For higher relevance, a model-based reranker scores each result against the -query with a trained model. `CohereReranker` works with vector, full-text, or hybrid search, and needs -the `cohere` package plus either `COHERE_API_KEY` in the environment or an `api_key` argument. +## Supported rerankers {#supported-rerankers} - - - {PyRerankingCohereUsage} - - +Use rank fusion to combine search results without a model, or a model-based +reranker to score each document against the query. Reranking runs in your +application after retrieval, including when the table is in Enterprise. -Reach for the model-free rerankers (`LinearCombinationReranker`, `RRFReranker`) when cost and latency -matter most; reach for a model-based one like `CohereReranker` or `CrossEncoderReranker` when you need -higher relevance and can afford to score every query and document pair with a model. +| Reranker | Method | SDK | +|:---------|:-------|:----| +| `RRFReranker` | Reciprocal rank fusion | Python, TypeScript, Rust | +| `MRRReranker` | Weighted reciprocal rank fusion | Python | +| `LinearCombinationReranker` (deprecated) | Blend normalized scores | Python | +| `CrossEncoderReranker` | Score query-document pairs with a local cross encoder | Python | +| `ColbertReranker`, `AnswerdotaiRerankers` | Local model reranking | Python | +| Provider rerankers | Call a hosted model, such as Cohere, Jina or Voyage AI | Python | -### Supported Rerankers {#supported-rerankers} - -LanceDB supports the following rerankers out of the box. The first three are score-based and run no -model; the rest are model-based. The built-in rerankers are documented in this section; the hosted -providers that need an API key live under [integrations](/integrations/reranking). - -| Reranker | Default model | -| --------------------------- | -------------------------------------- | -| `RRFReranker` | None (reciprocal rank fusion) | -| `LinearCombinationReranker` | None (weighted score blend) | -| `MRRReranker` | None (weighted reciprocal rank) | -| `CohereReranker` | `rerank-english-v3.0` | -| `CrossEncoderReranker` | `cross-encoder/ms-marco-TinyBERT-L-6` | -| `ColbertReranker` | `colbert-ir/colbertv2.0` | -| `AnswerdotaiRerankers` | `answerdotai/answerai-colbert-small-v1`| -| `JinaReranker` | `jina-reranker-v2-base-multilingual` | -| `OpenaiReranker` | `gpt-4-turbo-preview` | -| `VoyageAIReranker` | No default (model name required) | -| `WatsonxReranker` | `cross-encoder/ms-marco-minilm-l-12-v2`| - -The model-based rerankers need their provider package installed, and the hosted ones -(`CohereReranker`, `JinaReranker`, `OpenaiReranker`, `VoyageAIReranker`, `WatsonxReranker`) also need an API key, passed -as an `api_key` argument or set in the provider-specific environment variable. +Local model rerankers need their model runtime packages. Hosted rerankers need +provider credentials and make API calls. See [reranking integrations](/integrations/reranking) +for provider setup and model selection; specify a model supported by your provider. Rerankers add `_relevance_score` and return rows ordered by descending relevance. Python rerankers -accept `return_score="relevance"` or `return_score="all"`; use `"all"` when you want to keep the +accept a `return_score` option; use `"all"` where supported to keep the original vector distance or FTS score columns for debugging. Model-based rerankers read from `column="text"` by default, so either return that column in the search results or pass a different column. -Use `refine_factor` on vector or hybrid queries when you're reranking approximate IVF-PQ results and -want a larger candidate pool before the final ranking step. A value of `3` asks LanceDB to fetch -`limit * 3` candidates, refine them with the full vectors, and keep the requested `limit`. Higher -values can improve recall, but they also increase query latency. +`refine_factor` expands the vector index's internal candidate pool and rescores +those candidates using full vectors before keeping the requested limit. It +can improve the candidates available for reranking, but it does not multiply +the final result count. Compare retrieval recall and total query latency. **SDK coverage differs across languages** @@ -106,7 +110,9 @@ yourself. ### Multi-vector reranking {#multi-vector-reranking} -Most rerankers support reranking based on multiple vectors. To rerank based on multiple vectors, you can pass a list of vectors to the `rerank` method. Here's an example of how to rerank based on multiple vector columns using the `CrossEncoderReranker`: +Use `rerank_multivector()` to combine result sets from several vector queries. +The example below assumes `table` has three vector columns with embedding +metadata and uses a cross encoder to rerank their candidates. ```python Python icon="python" @@ -132,15 +138,24 @@ input result sets is missing the `_rowid` column. Therefore, it's recommended to - `RRFReranker.rerank_multivector(...)` always requires `_rowid` on its inputs, regardless of the `deduplicate` flag. -## Creating Custom Rerankers {#creating-custom-rerankers} +## Implement a custom ranking policy {#creating-custom-rerankers} -LanceDB also allows you to create custom rerankers by extending the base `Reranker` class. The custom reranker -should implement the `rerank` method that takes a list of search results and returns a reranked list of -search results. This is covered in more detail in the [creating custom rerankers](/reranking) section. +Python custom rerankers implement `rerank_hybrid()` and can optionally implement +`rerank_vector()` and `rerank_fts()`. TypeScript and Rust expose hybrid reranking +interfaces. See [the interface requirements](#custom) before implementing one. + +## Algorithm options + +The following sections describe the Python constructor options and examples. +Provider and local-model examples require their additional packages and model +or credential setup. They are separate programs, not steps to concatenate. ## Reciprocal rank fusion {#rrf} -## Reciprocal Rank Fusion Reranker + + + + **Reciprocal Rank Fusion (RRF)** is a model-free way to merge several ranked result lists into a single ordering. Rather than comparing raw similarity scores (which aren't directly comparable @@ -162,10 +177,10 @@ why it's the default reranker for LanceDB hybrid search. The implementation foll -### Accepted Arguments {#rrf-accepted-arguments} +### Python arguments {#rrf-accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | -| `K` | `int` | `60` | A constant used in the RRF formula (default is 60). Experiments indicate that k = 60 was near-optimal, but that the choice is not critical. | +| `K` | `int` | `60` | A constant used in the RRF formula (default is 60). Larger values reduce the difference in contribution between nearby rank positions. | | `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score`. If "all", will return all scores from the vector and FTS search along with the relevance score. | `K` must be greater than `0`. In TypeScript, construct the built-in reranker with @@ -185,18 +200,24 @@ You can specify the type of scores you want the reranker to return. The followin #### Hybrid Search {#rrf-hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Returned rows only have the `_relevance_score` column. | -| `all` | ✅ Supported | Returned rows have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | +| `relevance` | ✅ Supported | Keeps `_relevance_score` as the score column alongside the selected data columns. | +| `all` | ✅ Supported | Returned rows have vector(`_distance`) and FTS(`_score`) along with Hybrid Search score(`_relevance_score`). | + + + ## Linear combination {#linear-combination} -## Linear Combination Reranker + -> **Note:** This reranker is deprecated. Use the `RRFReranker` if you need a score-based reranker. + + + +> **Note:** This reranker is deprecated. Use `RRFReranker` to combine vector and full-text rankings. The Linear Combination Reranker combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified, and defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search. -> **Note:** Supported query type – Hybrid search only. +> **Note:** Supported query type: Hybrid search only. @@ -205,7 +226,7 @@ The Linear Combination Reranker combines the results of semantic and full-text s -### Accepted Arguments {#linear-combination-accepted-arguments} +### Python arguments {#linear-combination-accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `weight` | `float` | `0.7` | The weight to use for the semantic search score. The weight for the full-text search score is `1 - weight`. | @@ -222,16 +243,22 @@ You can specify the type of scores you want the reranker to return. The followin #### Hybrid Search {#linear-combination-hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column | -| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_distance`) | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns | +| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`_score`) along with Hybrid Search score(`_relevance_score`) | + + + ## MRR {#mrr} -## MRR Reranker + + + + This reranker uses the Mean Reciprocal Rank (MRR) algorithm to combine and rerank search results from vector and full-text search. You can use this reranker by passing `MRRReranker()` to the `rerank()` method. The MRR algorithm calculates the average of reciprocal ranks across different search results, providing a balanced way to merge results from multiple ranking systems. -> **Note:** Supported query types – Hybrid and Multivector search. +> **Note:** Supported query types: Hybrid and Multivector search. @@ -239,7 +266,7 @@ This reranker uses the Mean Reciprocal Rank (MRR) algorithm to combine and reran -### Accepted Arguments {#mrr-accepted-arguments} +### Python arguments {#mrr-accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `weight_vector` | `float` | `0.5` | Weight for vector search results (0.0 to 1.0). | @@ -258,21 +285,27 @@ You can specify the type of scores you want the reranker to return. The followin #### Hybrid Search {#mrr-hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | +| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`_score`) along with Hybrid Search score(`_relevance_score`). | #### Multivector Search {#mrr-multivector-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | | `all` | ✅ Supported | Results have vector distances from all searches along with `_relevance_score`. | + + + ## Cross encoder {#cross-encoder} -## Cross Encoder Reranker + + + + This reranker uses Cross Encoder models from sentence-transformers to rerank the search results. You can use this reranker by passing `CrossEncoderReranker()` to the `rerank()` method. -> **Note:** Supported query types – Hybrid, Vector, and FTS. +> **Note:** Supported query types: Hybrid, Vector, and FTS. @@ -281,7 +314,7 @@ This reranker uses Cross Encoder models from sentence-transformers to rerank the -### Accepted Arguments {#cross-encoder-accepted-arguments} +### Python arguments {#cross-encoder-accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `model_name` | `str` | `"cross-encoder/ms-marco-TinyBERT-L-6"` | The name of the reranker model to use.| @@ -299,20 +332,23 @@ You can specify the type of scores you want the reranker to return. The followin #### Hybrid Search {#cross-encoder-hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | +| `all` | ✅ Supported | Keeps `_distance`, `_score` and `_relevance_score` alongside the data columns. | #### Vector Search {#cross-encoder-vector-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | | `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | #### FTS Search {#cross-encoder-fts-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | +| `all` | ✅ Supported | Results have FTS(`_score`) along with Hybrid Search score(`_relevance_score`). | + + + ## Building a custom reranker {#custom} @@ -321,11 +357,9 @@ minimum, you need to implement `rerank_hybrid()`, which is the logic that combin full-text search results. Beyond that, you can optionally implement `rerank_vector()` and `rerank_fts()` if you want your reranker to also handle pure vector or pure full-text searches. -Decide up front which surfaces — hybrid, pure vector, or pure full-text — your reranker should -cover, and only override the ones you need. The base class leaves `rerank_vector()` and -`rerank_fts()` unimplemented, so calling `.rerank(...)` on a single-modality search you haven't -overridden raises `NotImplementedError` rather than silently returning unsorted results. That's a -useful guard, but worth knowing about before you wire up a query path you didn't plan for. +Implement the hooks your query paths call. The base class raises +`NotImplementedError` for vector-only or FTS-only reranking unless you +implement those methods. The Python base class exposes hybrid, vector-only, and FTS-only rerank hooks. TypeScript and Rust currently expose the custom reranker interface for hybrid reranking. In Rust, a custom reranker must @@ -350,55 +384,28 @@ For vector-only reranking in Python, pass a text query to `.rerank(..., query_st The vector query itself may be numeric, but `rerank_vector(query, vector_results)` still receives a string query so your reranker can score each candidate against the user's text intent. -Below, we show the pseudocode of a custom reranker that combines the results of semantic and full-text -search using a linear combination of the scores: +The method contract is: - -```python Python icon="python" -from lancedb.rerankers import Reranker -import pyarrow as pa +| Method | Input | Required output | +|:-------|:------|:----------------| +| `rerank_hybrid` | Query text, vector results and FTS results | A merged Arrow table ordered by descending `_relevance_score` | +| `rerank_vector` | Query text and vector results | An Arrow table ordered by descending `_relevance_score` | +| `rerank_fts` | Query text and FTS results | An Arrow table ordered by descending `_relevance_score` | -class MyReranker(Reranker): - def __init__(self, param1, param2, ..., return_score="relevance"): - super().__init__(return_score) - self.param1 = param1 - self.param2 = param2 +Implement only the single-search hooks your application uses. The base class +raises `NotImplementedError` for those hooks until they are implemented. - def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table): - # Use the built-in merging function - combined_result = self.merge_results(vector_results, fts_results) +### Extend an existing reranker {#custom-example} - # Do something with the combined results - # ... - - # Return the combined results - return combined_result - - def rerank_vector(self, query: str, vector_results: pa.Table): - # Do something with the vector results - # ... - - # Return the vector results - return vector_results - - def rerank_fts(self, query: str, fts_results: pa.Table): - # Do something with the FTS results - # ... - - # Return the FTS results - return fts_results -``` - - -### Example {#custom-example} - -As an example, let's build custom reranker that enhances the Cohere Reranker by accepting a filter -query, and accepts any other `CohereReranker` params as `kwargs`. +This class definition extends `CohereReranker` to exclude text patterns after +reranking. It requires the Cohere package and credentials when instantiated. +It can return fewer results because exclusions happen after retrieval; use a +query predicate instead when a metadata filter expresses the same restriction. ```python Python icon="python" from typing import List, Union -import pandas as pd +import pyarrow as pa from lancedb.rerankers import CohereReranker class ModifiedCohereReranker(CohereReranker): @@ -433,58 +440,42 @@ class ModifiedCohereReranker(CohereReranker): ``` - -Under the hood, `vector_results` and `fts_results` are PyArrow tables. You can learn more about -PyArrow tables [here](https://arrow.apache.org/docs/python). The advantage of PyArrow tables is their -interoperability -- you can easily convert them to Pandas/Polars DataFrames, `PyDict`, `PyList`, etc. +The reranker receives and returns PyArrow tables. Use `to_pandas()` for a +DataFrame transformation and `pa.Table.from_pandas()` to convert the result +back, preserving `_relevance_score` and the desired row order. -The benefits are also bidirectional -- just as you can easily convert PyArrow tables _to_ Pandas -DataFrames using the `to_pandas()` method -- you can perform DataFrame transformations -and just as easily convert the DataFrame back to PyArrow tables using `pa.Table.from_pandas()` method -as shown in the example above. - -## Evaluating hybrid search {#eval} +## Evaluate relevance, latency and cost {#eval} -Hybrid search is an often misused and/or misunderstood term. In this section, we're using -the definition of "hybrid search" to mean using a combination of keyword-based and vector search. -Because the vector search operates in a dense embedding space and keyword-based search operate -in a sparse embedding space, their relevance scores cannot be directly compared. -Combining results from multiple searches thus requires a reranking step. - -Before evaluating hybrid search, build an FTS index on the text column and use a table with embedding -metadata or explicit query vectors for the vector side. Otherwise the evaluation is measuring setup -fallbacks or errors rather than the reranker. +Compare rerankers on the same candidate inputs so you can distinguish ranking +changes from retrieval changes. For hybrid retrieval, prepare the text index +and query vectors before collecting those candidates. ### Reranking strategies {#eval-reranking-strategies} -There are two common approaches for reranking search results from multiple sources. +Compare these approaches: -- **Score-based**: Calculate final relevance scores from the individual search algorithm scores. Examples: Reciprocal Rank Fusion (the default in LanceDB), mean reciprocal rank fusion, and weighted linear combination of semantic and keyword-based search scores. +- **Rank fusion:** RRF and MRR combine rank positions from multiple result lists. +- **Score blending:** linear combination mixes normalized vector and text scores. +- **Model-based reranking:** a model scores each query-document pair. -- **Relevance-based**: Discards the existing scores and calculates the relevance of each search result-query pair. Example: Cross Encoder models - - -If you call `.rerank()` on a hybrid query without passing a reranker, LanceDB defaults to -`RRFReranker()` — a score-based reranker that uses Reciprocal Rank Fusion. This is the -score-based path most readers encounter first; `LinearCombinationReranker` is an alternative -score-based strategy you opt into explicitly. - +Hybrid search uses RRF by default. You do not need to call `rerank()` to select +it. Pass a reranker when you want a different algorithm or configuration. By default, rerankers return `_relevance_score`. Pass `return_score="all"` when a reranker supports it, and you also need the original vector or FTS scores for debugging. Evaluation code can rely on returned rows being ordered by descending `_relevance_score`. Empty reranked result sets still include the `_relevance_score` column. -The hybrid `rerank(...)` method also accepts a `normalize` argument that controls how the raw +The synchronous Python hybrid `rerank(...)` method also accepts a `normalize` argument that controls how the raw vector and FTS scores are made comparable before reranking: -- `normalize="score"` (the default) — normalizes the raw vector and FTS scores directly. -- `normalize="rank"` — converts each result list to ranks first, then normalizes. +- `normalize="score"` (the default); normalizes the raw vector and FTS scores directly. +- `normalize="rank"`; converts each result list to ranks first, then normalizes. -This choice materially affects score-based rerankers (such as `LinearCombinationReranker`), so -when you evaluate score-based strategies, treat `normalize` as a tunable hyperparameter -alongside the reranker itself. +Normalization matters for algorithms that use raw scores, such as linear +combination. RRF uses rank positions instead. Compare normalization settings +only when the chosen reranker uses those scores. For score-based evaluation, the main built-in knobs are: @@ -492,17 +483,18 @@ For score-based evaluation, the main built-in knobs are: - `MRRReranker(weight_vector=0.5, weight_fts=0.5)`: weighted reciprocal rank fusion where the weights must sum to `1.0`. - `LinearCombinationReranker(weight=0.7, fill=1.0)`: blends vector and FTS scores directly; `fill` controls how strongly to penalize a result missing from one side. -Even though there are many more strategies for reranking, there are no "universally best" -ones that work well for all cases, because reranking quality is dataset and application specific. -Evaluating whether a reranking strategy is a good fit is also a challenge. In the next -section, we discuss an example evaluation of different reranking strategies on a sample dataset. +A result on one dataset does not establish which reranker will work best on +another. Include representative queries and measure both retrieval quality +and ranking quality. -### Example evaluation {#eval-example-evaluation} +### Historical evaluation example {#eval-example-evaluation} -The table below shows our evaluation results from an experiment comparing multiple rerankers on +The tables below preserve an earlier experiment comparing multiple rerankers on ~800 hybrid search queries. This is a modified version of an evaluation script by [LlamaIndex](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures -hit-rate \@ top-k. +hit-rate \@ top-k. The recorded results do not establish performance for +current models, SDK releases or your dataset; treat them as an example of an +evaluation format. #### Using OpenAI `text-embedding-ada-002` {#eval-using-openai-text-embedding-ada-002} @@ -531,14 +523,14 @@ Vector Search baseline: **0.59** -### Conclusion {#eval-conclusion} +### Compare results on your workload {#eval-conclusion} -The results show that the reranking methods can significantly improve the search relevance. However, -the improvement we saw was not consistent across all rerankers. In reality, the choice of reranker -likely depends on the dataset and the application. +Build a set of representative queries with known relevant records. Compare +retrieval alone with each reranker using the same candidate inputs and result +limit. Record a relevance metric such as recall or nDCG alongside latency and +provider cost. Include queries with no useful matches and queries containing +exact names or identifiers. -It's also important to note that the reranking methods are not a -replacement for the search methods they supplement. They are complementary and it's likely that you'd -have to tune them together to get the best results. The latency vs. recall tradeoff is also an -important factor to consider when choosing the reranker. Hopefully this evaluation -gives you a starting point for your own experiments with hybrid search in LanceDB! +If the relevant records never reach the reranker, adjust retrieval first. +Then change one ranking setting at a time and retain the configuration that +meets your application's quality and latency requirements. diff --git a/docs/web/search/explain.mdx b/docs/web/search/explain.mdx index b2fc12c04..383745e5b 100644 --- a/docs/web/search/explain.mdx +++ b/docs/web/search/explain.mdx @@ -1,40 +1,72 @@ --- title: "Explain a query plan" sidebarTitle: "Query Plans" -description: "Read the plan LanceDB resolved for a query, and measure it with analyze." +description: "Inspect query execution, identify expensive work and compare changes." icon: "map" +availability: + oss: available + enterprise: available + summary: >- + Query plans are available in OSS and Enterprise. --- -`explain_plan` prints the plan LanceDB resolved for a query without executing -it. It answers structural questions -- did the filter push into the scan, is the -vector index being used, which columns are actually read -- and costs nothing to -run, so it is the first thing to reach for when a query behaves unexpectedly. +import { PyQueryPlan, TsQueryPlanBody, TsQueryPlanTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsQueryPlanBody, RsExampleTail as RsQueryPlanTail } from '/snippets/search_query_plan.mdx'; -To measure a query rather than read it, use -[`analyze_plan`](/search/explain), which executes it and annotates every -operator with runtime metrics. +Use query plans to understand how LanceDB reads data, applies filters and uses +indexes. `explain_plan()` shows the planned operations. `analyze_plan()` runs +the query and adds measurements, including execution time and rows processed. - -The SQL equivalent is [`EXPLAIN`](/sql/dql/explain). - +## Inspect a plan or measure execution {#inspect-a-query-plan} -## What it shows {#inspect-a-query-plan} +| Operation | Python / Rust | TypeScript | +|:----------|:--------------|:-----------| +| Inspect the plan without executing the query | `explain_plan()` | `explainPlan()` | +| Execute the query and report runtime measurements | `analyze_plan()` | `analyzePlan()` | -Reveals the logical query plan before execution, helping you identify potential issues with query structure and index usage. This tool is useful for: +Use the first to inspect query structure and the second to investigate work +actually performed. Operator names and available measurements can differ +between OSS and Enterprise. SQL provides [EXPLAIN and EXPLAIN ANALYZE](/sql/dql/explain). -- Verifying query optimization strategies -- Validating index selection -- Understanding query execution order -- Detecting missing indices + -Metadata filters are prefiltered by default, which usually shows the filter pushed into the -`LanceScan` or index scan. If you set `prefilter=False`, expect a separate `FilterExec` after -search instead; that can be useful for some expensive filters, but it changes both latency and -the number of rows available after filtering. + -## Reading the Execution Plan {#reading-the-execution-plan} +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. -To demonstrate query performance analysis, we'll use a table containing 1.2M rows sampled from the [Wikipedia dataset](https://huggingface.co/datasets/wikimedia/wikipedia). Initially, the table has no indices, allowing us to observe the impact of optimization. +- **Python:** Install with `pip install lancedb`, save as `query_plan.py`, then run `python query_plan.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `query_plan.mts`, then run `node query_plan.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin query_plan` from the extracted directory. + + + +{PyQueryPlan} + + +{TsQueryPlanBody + "\n" + TsQueryPlanTail} + + +{RsQueryPlanBody + "\n" + RsQueryPlanTail} + + + +This tiny table demonstrates the API, not a performance benchmark. The plan +uses a scan because the example has no vector index. The following larger +example illustrates how to read scans, filters and index operations. + + + +## Read scans, filters and result retrieval {#reading-the-execution-plan} + +The recorded example below uses 1.2 million rows sampled from the +[Wikipedia dataset](https://huggingface.co/datasets/wikimedia/wikipedia). +The counts describe that run. The table initially has no indexes, so the plan +shows a scan. This Enterprise plan includes `RemoteTake` and hides storage paths +as `uri=***`. A local plan shows the dataset path instead, and fetches the result +rows with Lance's `Take` or, depending on the library version, a second +`LanceRead` whose `source` is `stream(_rowid)`. Let's examine a vector search query that: @@ -67,6 +99,9 @@ const explainPlan = await table ### Execution Plan Components {#execution-plan-components} + + + The execution plan reveals the sequence of operations performed to execute your query. Let's examine each component: ``` @@ -103,7 +138,7 @@ FilterExec: identifier@1 > 0 AND identifier@1 < 1000000 #### 3\. Vector Search {#3-vector-search} -- Computes L2 (Euclidean) distances between query vector and all vectors that passed the filter +- Computes squared L2 distances between query vector and all vectors that passed the filter ``` KNNVectorDistance: metric=l2 @@ -113,7 +148,7 @@ KNNVectorDistance: metric=l2 - Filters out null distance results - Sorts by distance and takes top 100 results -- Processes in batches of 1024 for optimal memory usage +- Coalesces output into batches targeting 1,024 rows in this plan ``` SortExec: TopK(fetch=100) @@ -126,7 +161,7 @@ CoalesceBatchesExec: target_batch_size=1024 #### 5\. Data Retrieval {#5-data-retrieval} -- `RemoteTake` is a key component of Lance's I/O cache +- `RemoteTake` is the deployment's operator for this step; a local plan shows Lance's `Take` or a `LanceRead` over the selected row IDs, depending on the library version - Handles efficient data retrieval from remote storage locations - Fetches specific rows and columns needed for the final output - Optimizes network bandwidth by only retrieving required data @@ -139,19 +174,21 @@ RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title" - Returns only requested columns and maintains column ordering -```python +```text ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance] ``` This plan demonstrates a basic search without index optimizations: it performs a full scan and filter before vector search. + + + ## Non-vector queries {#analyzing-non-vector-queries} -`explain_plan` is not vector-specific -- it is available on every query builder, -including FTS and hybrid. The most common reason to read the plan for a -non-vector query is to confirm whether your `where` clause pushed into the scan -(good) or ran as a separate `FilterExec` on top of the search results (often -slower, and a hint that the filter column needs a scalar index). +Query plans also cover filters and full-text search. Python can show both +sub-plans of a hybrid query. Check where a filter runs and how many rows flow +through each operator; a separate `FilterExec` does not by itself establish +that the plan is inefficient or that an index is missing. ### FTS queries {#fts-queries} @@ -176,7 +213,10 @@ const plan = await table ``` -In an indexed FTS plan you should see a `MatchQuery` (or other FTS execution node) reading from the inverted index, with the metadata filter pushed down. If the plan shows a `LanceScan` followed by `FilterExec` over the entire text column, the FTS index either isn't covering the column or the filter isn't using a scalar index — both worth investigating. +Look for the text-index operation and any scan of unindexed rows. A scan can +be needed to include records written after the index was built. Use index +coverage and execution measurements to determine whether that extra work +matters. Check filter placement separately from text-index use. ### Hybrid queries {#hybrid-queries} @@ -197,21 +237,19 @@ print(plan) ``` -Reading the plan tells you the shape. `analyze_plan` executes both sub-queries -and labels them `Vector Search Plan:` and `FTS Search Plan:`, which is how you -see whether the filter pushed into both halves uniformly and which half -dominates latency. +`analyze_plan` executes both sub-queries and labels them `Vector Search Plan:` +and `FTS Search Plan:`. Compare their filter placement and execution measurements. `analyze_plan` executes the query and returns the physical plan annotated with -runtime metrics -- elapsed time, rows produced, bytes read, index comparisons. +runtime metrics such as elapsed time, rows produced, bytes read and index comparisons. Where [`explain_plan`](#inspect-a-query-plan) tells you what a query intends to do, -this tells you what it cost. +analysis shows the work performed. The SQL equivalent is [`EXPLAIN ANALYZE`](/sql/dql/explain). -## What it measures {#analyze-a-query-plan} +## Read execution measurements {#analyze-a-query-plan} Executes the query and provides detailed runtime metrics, including: @@ -220,7 +258,7 @@ Executes the query and provides detailed runtime metrics, including: - Index effectiveness (`_index_comparisons_`, `_indices_loaded_`) - Resource utilization (`_iops_`, `_requests_`) -## Performance Analysis {#performance-analysis} +## Measure the example query {#performance-analysis} Let's use `analyze_plan` to run the query and analyze the query performance, which will help us identify potential bottlenecks: @@ -249,6 +287,9 @@ const analyzePlan = await table ### Performance Metrics Analysis {#performance-metrics-analysis} + + + ``` ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance], metrics=[output_rows=100, elapsed_compute=1.424µs] RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title", metrics=[output_rows=100, elapsed_compute=175.53097ms, output_batches=1, remote_takes=100] @@ -284,11 +325,14 @@ ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier - Remote take operation for 100 results - Final projection of required columns + + + ### Distributed metrics on remote tables {#distributed-metrics-on-remote-tables} Enterprise -When you call `analyze_plan` against a LanceDB Enterprise table, the query runs across a pool of workers. By default the returned plan aggregates each operator's metrics into a single value, matching the local single-node output. Pass a `distributed_metrics` mode when you need to see how work was split across workers: +When you call `analyze_plan` against a LanceDB Enterprise deployment that distributes its queries across a pool of workers, the query runs on those workers. By default the returned plan aggregates each operator's metrics into a single value, matching the local single-node output. Pass a `distributed_metrics` mode when you need to see how work was split across workers: | Mode | What it shows | When to use it | | :------------ | :---------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------ | @@ -296,7 +340,9 @@ When you call `analyze_plan` against a LanceDB Enterprise table, the query runs | `per_worker` | Metrics reported separately for each worker that participated in the query. | Diagnosing stragglers or uneven work distribution across the cluster. | | `full` | Both the aggregate summary and the per-worker breakdown. | Deep investigations where you need the totals *and* the per-worker detail in a single output. | -The parameter only affects remote plans — local queries always return the aggregate output regardless of what you pass. +Python exposes this option through `distributed_metrics`. In TypeScript, pass +the same mode to `analyzePlan()` as its `distributedMetrics` argument. Local +Python queries ignore the distributed mode and return local measurements. ```python Python icon="python" @@ -310,30 +356,26 @@ plan = ( print(plan) ``` -```typescript TypeScript icon="square-js" -// Show a per-worker breakdown for a remote query -const plan = await table - .search(queryEmbed) - .where("identifier > 0 AND identifier < 1000000") - .limit(100) - .analyzePlan("per_worker"); -``` + -Use `per_worker` or `full` sparingly — the plan output grows with the size of your worker pool. Stick with the default `aggregate` mode for routine tuning. +Use `per_worker` or `full` sparingly; the plan output grows with the size of your worker pool. Stick with the default `aggregate` mode for routine tuning. -### Key Observations {#key-observations} +### Identify expensive operations {#key-observations} -- Vector search is the primary bottleneck (1,099,508 vector comparisons) -- Significant I/O overhead (1.86GB data read) +- The query computes distances for 1,099,508 vectors; inspect timings to assess its contribution to latency +- The scan reads 1.86 GB in this recorded run - Full table scan due to lack of indices -- Substantial optimization potential through proper index implementation +- Compare this scan with an indexed plan using the same query -## Optimized Query Execution {#optimized-query-execution} +## Compare the query after adding indexes {#optimized-query-execution} -After creating vector and scalar indices, the execution plan shows: + + + +The recorded query after adding vector and scalar indexes produced this plan: ``` ProjectionExec: expr=[chunk_index@3 as chunk_index, title@4 as title, identifier@2 as identifier, _distance@0 as _distance] @@ -442,7 +484,7 @@ ANNSubIndex: - Before: L2 calculations on 1,099,508 vectors - After: - - 99.8% reduction in vector comparisons + - About 97.6% fewer vector comparisons (25,893 versus 1,099,508) - Decreased output batches from 1,076 to 20 #### 3\. Data Retrieval Optimization {#3-data-retrieval-optimization} @@ -455,7 +497,10 @@ RemoteTake: - RemoteTake operation remains consistent -## Performance Optimization Guide {#performance-optimization-guide} + + + +## Choose a change to measure {#performance-optimization-guide} ### 1\. Index Implementation {#1-index-implementation} @@ -470,13 +515,14 @@ RemoteTake: | Data Type | Recommended Index | Use Case | | ----------- | ------------------ | ---------------------------------------- | | Vector | IVF_PQ/IVF_HNSW_SQ/IVF_HNSW_FLAT | Approximate nearest neighbor search | -| Scalar | B-Tree | Range queries and sorting | +| Scalar | B-Tree | Equality and range filters | | Categorical | Bitmap | Multi-value filters and set operations | | `List` | Label_list | Multi-label classification and filtering | Use `table.index_stats()` to monitor index coverage. -A well-optimized table should have `num_unindexed_rows ~ 0`. +Unindexed rows can add scan work. Use their count alongside query measurements +to decide when index maintenance is needed. ### 2\. Query Plan Optimization {#2-query-plan-optimization} @@ -485,10 +531,10 @@ A well-optimized table should have `num_unindexed_rows ~ 0`. | Plan Pattern | Optimization | | ------------------------------------------ | -------------------------------------------- | -| LanceScan with high _bytes_read_ or _iops_ | Add missing index | +| Scan with high bytes read or I/O count | Check index coverage and whether the query can use an index | | | Use `select()` to limit returned columns | | | Check whether the dataset has been compacted | -| Multiple sequential filters | Reorder filter conditions | +| Multiple filters | Check their placement and row counts; compare the plan after changing the query | Regularly analyze your query plans to identify and address performance @@ -498,7 +544,8 @@ optimization efforts. ### 3\. Getting Started with Optimization {#3-getting-started-with-optimization} -For vector search performance: - -- Create ANN index on your vector column(s) as described in the [index guide](/indexing/vector-index/) -- If you often filter by metadata, create [scalar indices](/indexing/scalar-index/) on those columns +Keep the query, data and result limit fixed while changing one factor. Compare +rows processed, bytes read, timings and returned results. For ANN changes, +measure recall too: doing less work is useful only if retrieval quality meets +your application's needs. See [vector indexes](/indexing/vector-index) and +[scalar indexes](/indexing/scalar-index) for index choices. diff --git a/docs/web/search/filtering.mdx b/docs/web/search/filtering.mdx index d5bce618b..50d37acc0 100644 --- a/docs/web/search/filtering.mdx +++ b/docs/web/search/filtering.mdx @@ -1,28 +1,103 @@ --- title: "Metadata Filtering in LanceDB" sidebarTitle: Filtering -description: Filter search results in LanceDB based on metadata fields. +description: "Use predicates to select rows and control when filters apply to search." icon: "filter" +availability: + oss: available + enterprise: available + summary: >- + Metadata filtering is available in OSS and Enterprise. --- -LanceDB supports filtering features of query results based on metadata fields. -While joint vector and metadata search at scale presents a significant challenge, -LanceDB achieves sub-100ms latency at thousands of QPS, enabling efficient vector search -with filtering capabilities even on datasets containing billions of records. +import { PyFilterSearch, TsFilterSearchBody, TsFilterSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsFilterSearchBody, RsExampleTail as RsFilterSearchTail } from '/snippets/search_filter_search.mdx'; -**Pre-filtering** means LanceDB applies the metadata `where(...)` condition before running vector search, so the search only considers rows that already match the filter. **Post-filtering** means LanceDB runs vector search first and only then filters the returned candidates. Pre-filtering is enabled by default. In practice, pre-filtering is better when the filter is part of the result contract; post-filtering can be lower-latency for expensive or non-indexable filters, but it can return fewer than `limit` rows, or even zero, if the nearest neighbors do not pass the filter. +A filter restricts results to rows that satisfy a condition. Use it on its +own to select records, or combine it with vector, full-text or hybrid search. +For example, search for similar documents only within a particular category. -On hybrid queries, the same `where(...)` filter is applied to both the vector and full-text halves of the query. The prefilter or postfilter choice controls whether that happens before each subquery scores candidates or after the subquery top-k is produced. + - -The same predicates work in SQL -- see [SELECT](/sql/dql/select#filtering) and [Operators](/sql/operators). - + -## Chaining `where` clauses {#chaining-where-clauses} +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. -In more recent LanceDB SDK versions (see the callout box below for the exact version numbers), you can call `where(...)` (Python and TypeScript) or `only_if(...)` (Rust) more than once on the same query builder. Each additional filter is combined with the previous one using logical `AND`, so `where("a > 0").where("b < 10")` is equivalent to `where("(a > 0) AND (b < 10)")`. +- **Python:** Install with `pip install lancedb`, save as `filtered_search.py`, then run `python filtered_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `filtered_search.mts`, then run `node filtered_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin filter_search` from the extracted directory. -For a fixed predicate, writing one `where(...)` clause with `AND` is just as valid and often clearer. Chaining is mainly useful when code composes filters incrementally, such as applying a shared base predicate in a helper and then adding a per-call predicate at the query site. + + +{PyFilterSearch} + + +{TsFilterSearchBody + "\n" + TsFilterSearchTail} + + +{RsFilterSearchBody + "\n" + RsFilterSearchTail} + + + +The query vector is closest to the gardening record, but the filter excludes +that category. The results contain only the two animal records. + + + +## Prefiltering and postfiltering {#choose-when-the-filter-runs} + +- **Prefiltering**, the default, limits the rows eligible for retrieval before + search chooses its candidates. +- **Postfiltering** removes nonmatching rows after candidates are selected. + It can return fewer than `limit` rows, including zero, even when matching + records exist elsewhere in the table. + +Both modes exclude rows that fail the predicate. They differ in which matching +rows the search can find. On hybrid queries, the filter applies to both the +vector and full-text searches. + +SDK filters use SQL expressions without requiring a SQL connection. For +Enterprise SQL queries, see [SELECT](/sql/dql/select#filtering). + +## Filtering examples by language {#example-metadata-filtering} + +The following excerpts use a connected `db` handle. Create this sample table +before running the queries beneath it; each query uses the same four records. + + +```python Python icon="python" +data = [ + {"vector": [3.1, 4.1], "item": "foo", "price": 10.0}, + {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}, + {"vector": [10.2, 100.8], "item": "baz", "price": 30.0}, + {"vector": [1.4, 9.5], "item": "fred", "price": 40.0}, +] +table = db.create_table("metadata_filter_example", data=data, mode="overwrite") +``` + +```typescript TypeScript icon="square-js" +const data = [ + { vector: [3.1, 4.1], item: "foo", price: 10.0 }, + { vector: [5.9, 26.5], item: "bar", price: 20.0 }, + { vector: [10.2, 100.8], item: "baz", price: 30.0 }, + { vector: [1.4, 9.5], item: "fred", price: 40.0 }, +]; + +const tableName = "metadata_filter_example"; +const table = await db.createTable(tableName, data, { + mode: "overwrite", +}); +``` + + +## Combine predicates {#chaining-where-clauses} + +Repeated filter calls combine predicates with logical `AND`. For example, `where("a > 0").where("b < 10")` is equivalent to `where("(a > 0) AND (b < 10)")`. + +Use a single expression for a fixed filter. Chain calls when your application +builds the filter incrementally. ```python Python icon="python" @@ -51,36 +126,6 @@ const result = await table In Python SDK versions before `0.34.0` and Rust/TypeScript SDK versions before `0.31.0`, a second `where(...)` or `only_if(...)` call replaced the first filter, so only the last predicate was applied. If your code needs to run on those older versions, write a single predicate with `AND` instead of chaining calls. When upgrading to the latest SDKs, review any existing chained filters and drop earlier calls you no longer want to apply. -## Example: Metadata Filtering {#example-metadata-filtering} - -To illustrate filtering capabilities, let's try four data points with combinations of vectors and metadata: - - -```python Python icon="python" -data = [ - {"vector": [3.1, 4.1], "item": "foo", "price": 10.0}, - {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}, - {"vector": [10.2, 100.8], "item": "baz", "price": 30.0}, - {"vector": [1.4, 9.5], "item": "fred", "price": 40.0}, -] -table = db.create_table("metadata_filter_example", data=data, mode="overwrite") -``` - -```typescript TypeScript icon="square-js" -const data = [ - { vector: [3.1, 4.1], item: "foo", price: 10.0 }, - { vector: [5.9, 26.5], item: "bar", price: 20.0 }, - { vector: [10.2, 100.8], item: "baz", price: 30.0 }, - { vector: [1.4, 9.5], item: "fred", price: 40.0 }, -]; - -const tableName = "metadata_filter_example"; -const table = await db.createTable(tableName, data, { - mode: "overwrite", -}); -``` - - ### Filtering Without Vector Search {#filtering-without-vector-search} You can always filter your data without search. This is useful when you need to query based on metadata: @@ -104,9 +149,8 @@ const filteredResult = await table ``` - -If your table is large, this could potentially return a very large amount of data. Please be sure to use a `limit` clause unless you're sure you want to return the whole result set. - +Set `limit()` explicitly to control how many matching records your application +receives. ### Pre-Filtering with Vector Search {#pre-filtering-with-vector-search} @@ -124,6 +168,7 @@ filtered_result = ( const results = await table .search([100, 102]) .where("(item IN ('foo', 'bar')) AND (price > 15.0)") + .limit(3) .toArray(); ``` @@ -141,7 +186,7 @@ post_filtered_result = ( ``` ```typescript TypeScript icon="square-js" -const postFilteredResult = await (table.search([100, 102]) as VectorQuery) +const postFilteredResult = await table.vectorSearch([100, 102]) .where("(item IN ('foo', 'bar')) AND (price > 15.0)") .postfilter() .limit(3) @@ -149,14 +194,13 @@ const postFilteredResult = await (table.search([100, 102]) as VectorQuery) ``` - -When querying large tables, omitting a `limit` clause may overwhelm resources and return excessive data. It's always recommended -to be mindful of the potential impact on performance and costs when working with really large tables. - +Postfiltering can return fewer than the requested number of records because +the predicate removes candidates after search. -## Filtering with SQL {#filtering-with-sql} +## Write SQL predicates {#filtering-with-sql} -Because it's built on top of DataFusion, LanceDB embraces the utilization of standard SQL expressions as predicates for filtering operations. SQL can be used during vector search, update, and deletion operations. +Pass a SQL expression as the filter string. These expressions also select rows +for SDK update and delete operations; they are not standalone SQL statements. LanceDB supports a growing list of SQL expressions: @@ -178,7 +222,7 @@ For example, the following filter string is acceptable: ```python Python icon="python" -tbl.search([100, 102]).where( +table.search([100, 102]).where( "(item IN ('foo', 'baz')) AND (price > 20.0)" ).to_arrow() ``` @@ -202,9 +246,10 @@ path must be wrapped in backticks. AND `nested with space`.`inner with space` < 2 ``` - -Field names containing periods (.) are NOT supported. - + +Periods separate nested fields. LanceDB does not allow a literal period in a +top-level column name; backticks do not remove that restriction. + ### Dates, Timestamps, Decimals {#dates-timestamps-decimals} @@ -231,7 +276,8 @@ parameter. Microsecond precision (6) is the default. ## Apache Arrow Mapping {#apache-arrow-mapping} -LanceDB internally stores data in [Apache Arrow](https://arrow.apache.org/) format. +LanceDB uses Arrow types for schemas and in-memory data. On disk, tables use +the [Lance format](/basics/lance). The mapping from SQL types to Arrow types is: | SQL type | Arrow type | @@ -245,26 +291,27 @@ The mapping from SQL types to Arrow types is: | `double` | `Float64` | | `decimal(precision, scale)` | `Decimal128` | | `date` | `Date32` | -| `timestamp` | `Timestamp` [^1] | +| `timestamp` | `Timestamp` | | `string` | `Utf8` | | `binary` | `Binary` | -## Best Practices {#best-practices} +## Speed up frequently used filters {#best-practices} -**Scalar Indexes**: We strongly recommend creating scalar indices on columns used for filtering, whether combined with a search operation or applied independently (e.g., for updates or deletions). +Consider a [scalar index](/indexing/scalar-index) on frequently filtered columns. +Use the query plan to check whether a predicate uses that index. Functions or +casts applied to a column can change which optimizations are available. +For list columns, a label-list index supports membership filters such as +`array_has_any` and `array_has_all`. -For best performance with large tables or high query volumes: +## Understand missing matches {#limitations} -- Build a scalar index on frequently filtered columns -- Use exact column names in filters (e.g., `user_id` instead of `USER_ID`) -- Avoid complex transformations in filter expressions (keep them simple) -- When running concurrent queries, use connection pooling for better throughput +A restrictive predicate can legitimately leave fewer rows than the requested +limit. Approximate vector search can also miss qualifying neighbors, and +postfiltering can discard candidates without retrieving replacements. -For a column of type LIST(T), you can use `LABEL_LIST` to create a scalar index. Then you should leverage DataFusion's [array functions](https://datafusion.apache.org/user-guide/sql/scalar_functions.html#array-functions) like `array_has_any` or `array_has_all` for optimized filtering. - -## Limitations {#limitations} - -Both **pre-filtering** and **post-filtering** can yield false positives. For pre-filtering, if the filter is too selective, it might eliminate relevant items that the vector search would have otherwise identified as a good match. In this case, increasing `nprobes` parameter will help reduce such false positives. It is recommended to call `bypass_vector_index()` if you know that the filter is highly selective. - -Similarly, a highly selective post-filter can lead to false positives. Increasing both `nprobes` and `refine_factor` can mitigate this issue. When deciding between pre-filtering and post-filtering, pre-filtering is generally the safer choice if you're uncertain. +For a recall comparison, run the same query and predicate while bypassing the +vector index on a suitably sized test table. Then compare the indexed query's +results and latency while changing probe or refinement settings. Increasing +those settings does not guarantee a full result set. See +[vector search](/search/vector-search) and [query plans](/search/explain). diff --git a/docs/web/search/full-text-search.mdx b/docs/web/search/full-text-search.mdx index 0008183dc..0dc0096f7 100644 --- a/docs/web/search/full-text-search.mdx +++ b/docs/web/search/full-text-search.mdx @@ -1,8 +1,13 @@ --- title: Full-Text Search (FTS) sidebarTitle: Full-Text Search (FTS) -description: Learn how to implement full-text search in LanceDB using BM25 for keyword-based retrieval. +description: "Find records by words, phrases or partial matches using a full-text index." icon: "book" +availability: + oss: available + enterprise: available + summary: >- + Full-text search is available in OSS and Enterprise. --- import { @@ -12,17 +17,51 @@ import { } from '/snippets/search.mdx' import { PyFtsBooleanQueries } from '/snippets/doc_function_examples.mdx' -LanceDB provides support for Full-Text Search via Lance, allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. +import { PyTextSearch, TsTextSearchBody, TsTextSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsTextSearchBody, RsExampleTail as RsTextSearchTail } from '/snippets/search_text_search.mdx'; - -In SQL, full-text search is the [`fts()`](/sql/functions#fts) table function, or the [`contains_tokens`](/sql/scalar-functions#contains-tokens) predicate when you do not need a score. - +Full-text search finds documents by matching words and ranks them with BM25. +Use it to search for names, product codes, technical terms or phrases in your +text columns. Create an FTS index on the column before running a query. -## Basic Usage {#basic-usage} +You can also combine keyword matches with vector similarity using +[hybrid search](/search/hybrid-search). -Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search, the FTS index must be created before you can search via keywords. + -### Table Setup {#table-setup} + + +Each example creates a temporary local table, runs the query and removes the +table when finished. No embedding model is needed for a text query. + +- **Python:** Install with `pip install lancedb`, save as `text_search.py`, then run `python text_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `text_search.mts`, then run `node text_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin text_search` from the extracted directory. + + + +{PyTextSearch} + + +{TsTextSearchBody + "\n" + TsTextSearchTail} + + +{RsTextSearchBody + "\n" + RsTextSearchTail} + + + +Each example searches the indexed `text` column and returns the puppy record. +Higher `_score` values rank first; they are relevance scores, not probabilities. + + + +## Basic usage {#basic-usage} + +The steps below create a table, index its `text` column and search for a word. +For a standalone program with imports and sample data, expand the complete +example above. + +### Create the sample table {#table-setup} First, open or create the table you want to search: @@ -66,19 +105,16 @@ let tbl = db ``` -### Construct FTS Index {#construct-fts-index} +### Build the text index {#construct-fts-index} Create a full-text search index on your text column: - -In Python, this page shows the synchronous `create_fts_index(...)` form. For the -asynchronous equivalent (`await table.create_index("text", config=FTS(...))`), see -[FTS index](/indexing/fts-index). - +For asynchronous Python index creation, use +`await table.create_index("text", config=FTS())`. See [FTS index](/indexing/fts-index). ```python Python icon="python" -table.create_fts_index("text") +table.create_index("text", config=FTS()) ``` ```typescript TypeScript icon="square-js" @@ -95,17 +131,24 @@ tbl ``` -### Full-text Search {#full-text-search} +OSS builds the index before the call returns. Enterprise builds it in the +background. Wait for completion with Python’s `table.wait_for_index(["text_idx"])` +or TypeScript’s `await tbl.waitForIndex(["text_idx"], 300)` before querying it. +Use your actual index name if you chose a different one. + +### Run the query {#full-text-search} Perform full-text search and retrieve results: ```python Python icon="python" -results = table.search("puppy") +results = ( + table.search("puppy", query_type="fts", fts_columns="text") .limit(10) - .select(["text"]) + .select(["text", "_score"]) .to_list() -# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}] +) +# Returns the puppy record with its BM25 score. ``` ```typescript TypeScript icon="square-js" @@ -127,19 +170,26 @@ let results = tbl ``` -The search is conducted on all indexed columns by default, so it's useful when there are multiple indexed columns. - -If you want to specify which columns to search use `fts_columns="text"` +Specify the text column with `fts_columns="text"` in synchronous Python, +especially when the table contains multiple text indexes. -LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead. +A string query can also invoke an embedding function when the table has +embedding metadata. Use `query_type="fts"` in Python when you specifically +want full-text search. If a table has more than one FTS index, specify the indexed text column in the query. In Python you can use `fts_columns` or the query builder's `nearest_to_text(..., columns=...)`; in TypeScript, use `query().nearestToText(..., columns)`. The newer Lance-native FTS does not accept legacy Tantivy-only index parameters. +BM25 scores depend on the indexed corpus and its term statistics. Compare +rankings on the dataset you will search; treat scores as relative relevance +values rather than probabilities. + ### Keeping the index up to date {#keeping-the-index-up-to-date} -Rows you add after building an FTS index aren't part of the index until you optimize the table. Until then, queries fall back to a flat scan over the unindexed fragments to keep results complete, which slows them down as the unindexed tail grows. Call `table.optimize()` to fold new rows into the existing index — it's the same operation used for vector indexes: +In OSS, new rows can remain outside the FTS index until you optimize the table. +Search includes those rows by scanning them, but that work can increase query +latency. Call `optimize()` to update the index: @@ -157,13 +207,35 @@ tbl.optimize(OptimizeAction::All).await?; ``` -A useful rule of thumb is to call `optimize()` after roughly 100,000 row changes or 20 data-modification operations, whichever comes first. For tables with continuous ingest, schedule it on a cadence that keeps `num_unindexed_rows` (from `table.index_stats(...)`) close to zero. If you want to skip the flat scan over unindexed rows entirely — for example, on a hot read path where stale results are acceptable — call `.fast_search()` on the query so the search returns only indexed results. +Choose an OSS maintenance schedule based on write volume and query latency. +Inspect index coverage with `table.index_stats(...)` and optimize when the +unindexed data adds too much work. `.fast_search()` skips unindexed rows, so +use it only when omitting recent records is acceptable. -## Advanced Usage {#advanced-usage} +Enterprise manages index updates automatically. You do not call `optimize()` +on an Enterprise table. + +## Choose the matching behavior + +| You need | Use | +|:---------|:----| +| Words in any order | A terms query | +| Words in a particular sequence | A [phrase query](#phrase-vs-terms-queries) and an index with token positions | +| Tolerance for typing errors | [Fuzzy search](#fuzzy-search) | +| Parts of words | An [n-gram index](#search-for-substring) | +| Required, optional or excluded terms | [Boolean queries](#boolean-queries) | + +Use a metadata predicate to restrict qualifying records. See +[filtering options](#filtering-options) for prefiltering and postfiltering. +Enterprise SQL provides [`fts()`](/sql/functions#fts) for scored text retrieval. + +## Configure tokenization and matching {#advanced-usage} ### Tokenize Table Data {#tokenize-table-data} -By default, the text is tokenized by splitting on punctuation and whitespaces, and would filter out words that are longer than 40 characters. All words are converted to lowercase. +The default tokenizer splits text at whitespace and punctuation, lowercases +tokens and removes tokens longer than 40 characters. Stemming, stop-word +removal and accent folding are also enabled by default. Stemming is useful for improving search results by reducing words to their root form, e.g. "running" to "run". LanceDB supports stemming for Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. You should set the `base_tokenizer` parameter rather than `tokenizer_name` because you cannot customize the tokenizer if `tokenizer_name` is specified. @@ -177,7 +249,8 @@ table.create_fts_index("text", language="English", replace=True) ``` -The tokenizer is customizable, you can specify how the tokenizer splits the text, and how it filters out words, etc. +Configure tokenization when you create the index. Rebuild the index to change +how existing text is tokenized. **Default index parameters:** - `base_tokenizer`: `"simple"` @@ -188,7 +261,7 @@ The tokenizer is customizable, you can specify how the tokenizer splits the text - `stem`: true - `remove_stop_words`: true - `ascii_folding`: true -- `custom_stop_words`: `None` — pass a `list[str]` to drop additional words beyond the language defaults. Requires `remove_stop_words=True`. +- `custom_stop_words`: `None`; pass a `list[str]` to drop additional words beyond the language defaults. Requires `remove_stop_words=True`. For multilingual use cases, use `base_tokenizer="icu"` for unicode-aware word segmentation on mixed-language text. ICU stands for [International Components for Unicode](https://icu.unicode.org/). The ICU tokenizer uses bundled ICU4X segmenter data, so it does not require external tokenizer model files. It is a good default when documents mix languages or include scripts where the simple tokenizer would keep an unspaced span as one large token. @@ -215,11 +288,15 @@ export LANCE_LANGUAGE_MODEL_HOME=/path/to/lance/language_models For example, `jieba/default` is resolved under `/jieba/default/...`, `lindera/ipadic` under `/lindera/ipadic/...`, and `lindera/ko-dic` under `/lindera/ko-dic/...`. +Model files must be installed where indexing and queries run: on your machine +for OSS, or on the Enterprise cluster for a remote table. + Built-in stop-word removal supports Danish, Dutch, English, Finnish, French, German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish, and Swedish. If you use another stemming language, such as Arabic, Greek, Romanian, Tamil, or Turkish, set `remove_stop_words=False` or pass `custom_stop_words`. -For example, for language with accents, you can specify the tokenizer to use `ascii_folding` to remove accents, e.g. 'é' to 'e': +Use `ascii_folding=True` to match accented and unaccented forms, such as +“é” and “e”: ```python Python icon="python" @@ -235,9 +312,9 @@ table.create_fts_index( ### Filtering Options {#filtering-options} -LanceDB full text search supports to filter the search results by a condition, both pre-filtering and post-filtering are supported. - -This can be invoked via the familiar `where` syntax. +Use `where(...)` to restrict text results to rows that match a predicate. +Prefiltering applies the restriction before retrieval; postfiltering removes +nonmatching candidates afterward and can leave fewer results. With pre-filtering: @@ -308,15 +385,17 @@ Lance-based FTS doesn't support queries using boolean operators `OR`, `AND` in t For full-text search you can specify either a **phrase** query like `"the old man and the sea"`, or a **terms** search query like `old man sea`. -To search for a phrase, the index must be created with `with_position=True` and `remove_stop_words=False`: +Enable token positions to search for phrases. Set `remove_stop_words=False` if you also need to match common words such as “the” and “and”: ```python Python icon="python" -table.create_fts_index("text", with_position=True, replace=True) +from lancedb.index import FTS + +table.create_index("text", config=FTS(with_position=True, remove_stop_words=False), replace=True) ``` -This will allow you to search for phrases, but it will also significantly increase the index size and indexing time. +Storing token positions increases the index size and adds indexing work. ### Fuzzy Search {#fuzzy-search} @@ -326,7 +405,7 @@ to find similar terms within a specified edit distance. | Parameter | Type | Default | Description | | -------------- | ---- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -| fuzziness | int | 0 | Maximum edit distance allowed for each term. If not specified, automatically set based on term length: 0 for length ≤ 2, 1 for length ≤ 5, 2 for length > 5 | +| fuzziness | int | 0 | Maximum edit distance allowed for each term. The default, `0`, requires an exact match; set `1` or `2` to allow typos. | | max_expansions | int | 50 | Maximum number of terms to consider for fuzzy matching. Higher values may improve recall but increase search time | For a complete walkthrough that creates a sample table and demonstrates fuzzy search and relevance boosting, see [the worked example below](#fuzzy-search-and-boosting-example). @@ -344,65 +423,41 @@ LanceDB supports searching for substrings in the text column, you can set the `b ## Fuzzy Search and Boosting Example {#fuzzy-search-and-boosting-example} + + + ### Generate Data {#generate-data} -First, let's create a table with sample text data for testing fuzzy search: +Run the following steps in order. They use a temporary local database with four records, so the same data is available for each query. ```python Python icon="python" +from tempfile import TemporaryDirectory + import lancedb -import numpy as np import pandas as pd -import random +from lancedb.index import FTS +from lancedb.query import MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery -# Connect to LanceDB -db = lancedb.connect( - uri="db://your-project-slug", - api_key="your-api-key", - region="us-east-1" -) - -# Generate 100 rows of random " " text -table_name = "fts-fuzzy-boosting-test" -vectors = [np.random.randn(128) for _ in range(100)] -verbs = ("runs", "hits", "jumps", "drives", "barfs") -adv = ("crazily.", "dutifully.", "foolishly.", "merrily.", "occasionally.") -adj = ("adorable", "clueless", "dirty", "odd", "stupid") - -def sentence(nouns): - return " ".join(random.choice(words) for words in (nouns, verbs, adv, adj)) - -text = [sentence(("puppy", "car")) for _ in range(100)] -text2 = [sentence(("rabbit", "girl", "monkey")) for _ in range(100)] -count = [random.randint(1, 10000) for _ in range(100)] +directory = TemporaryDirectory() +db = lancedb.connect(directory.name) +table_name = "fts_examples" +text = ["puppy runs merrily", "car drives crazily", "puppy jumps crazily", "car runs slowly"] +text2 = ["rabbit jumps crazily", "girl runs merrily", "monkey jumps slowly", "rabbit runs merrily"] ``` ```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb" +import * as lancedb from "@lancedb/lancedb"; +import { Index, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, Occur } from "@lancedb/lancedb"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; -const db = await lancedb.connect({ - uri: "db://your-project-slug", - apiKey: "your-api-key", - region: "us-east-1" -}); - -// Generate 100 rows of random " " text -const tableName = "fts-fuzzy-boosting-test-ts"; -const n = 100; -const verbs = ["runs", "hits", "jumps", "drives", "barfs"]; -const adverbs = ["crazily", "dutifully", "foolishly", "merrily", "occasionally"]; -const adjectives = ["adorable", "clueless", "dirty", "odd", "stupid"]; - -const pick = (words: string[]) => words[Math.floor(Math.random() * words.length)]; -const sentence = (nouns: string[]) => - [nouns, verbs, adverbs, adjectives].map(pick).join(" "); - -const vectors = Array.from({ length: n }, () => - Array.from({ length: 128 }, () => Math.random() * 2 - 1) -); -const text = Array.from({ length: n }, () => sentence(["puppy", "car"])); -const text2 = Array.from({ length: n }, () => sentence(["rabbit", "girl", "monkey"])); -const count = Array.from({ length: n }, () => Math.floor(Math.random() * 10000) + 1); +const directory = await mkdtemp(join(tmpdir(), "lancedb-fts-")); +const db = await lancedb.connect(directory); +const tableName = "fts_examples"; +const text = ["puppy runs merrily", "car drives crazily", "puppy jumps crazily", "car runs slowly"]; +const text2 = ["rabbit jumps crazily", "girl runs merrily", "monkey jumps slowly", "rabbit runs merrily"]; ``` @@ -414,11 +469,9 @@ const count = Array.from({ length: n }, () => Math.floor(Math.random() * 10000) table = db.create_table( table_name, data=pd.DataFrame({ - "vector": vectors, - "id": [i % 2 for i in range(100)], + "id": range(len(text)), "text": text, "text2": text2, - "count": count, }), mode="overwrite" ) @@ -426,15 +479,7 @@ table = db.create_table( ```typescript TypeScript icon="square-js" // Create table with sample data -const data = makeArrowTable( - vectors.map((vector, i) => ({ - vector, - id: i % 2, - text: text[i], - text2: text2[i], - count: count[i], - })) -); +const data = text.map((value, i) => ({ id: i, text: value, text2: text2[i] })); const table = await db.createTable(tableName, data, { mode: "overwrite" }); ``` @@ -447,14 +492,14 @@ Create a full-text search index on the first text column: ```python Python icon="python" # Create FTS index on first text column -table.create_fts_index("text") -wait_for_index(table, "text_idx") +table.create_index("text", config=FTS(with_position=True, remove_stop_words=False)) +table.wait_for_index(["text_idx"]) ``` ```typescript TypeScript icon="square-js" // Create FTS index on first text column -await table.createIndex("text", { config: Index.fts() }); -await waitForIndex(table, "text_idx"); +await table.createIndex("text", { config: Index.fts({ withPosition: true, removeStopWords: false }) }); +await table.waitForIndex(["text_idx"], 300); ``` @@ -463,14 +508,14 @@ Then, create an index on the second text column: ```python Python icon="python" # Create FTS index on second text column -table.create_fts_index("text2") -wait_for_index(table, "text2_idx") +table.create_index("text2", config=FTS(with_position=True, remove_stop_words=False)) +table.wait_for_index(["text2_idx"]) ``` ```typescript TypeScript icon="square-js" // Create FTS index on second text column -await table.createIndex("text2", { config: Index.fts() }); -await waitForIndex(table, "text2_idx"); +await table.createIndex("text2", { config: Index.fts({ withPosition: true, removeStopWords: false }) }); +await table.waitForIndex(["text2_idx"], 300); ``` @@ -487,19 +532,17 @@ from lancedb.query import MatchQuery # Basic match (exact search) basic_match_results = ( table.search(MatchQuery("crazily", "text")) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" -import { MatchQuery } from "@lancedb/lancedb"; - // Basic match (exact search) const basicMatchResults = await table.query() .fullTextSearch(new MatchQuery("crazily", "text")) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` @@ -512,9 +555,9 @@ const basicMatchResults = await table.query() # Fuzzy match (allows typos) fuzzy_results = ( table.search(MatchQuery("craziou", "text", fuzziness=2)) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` @@ -524,34 +567,35 @@ const fuzzyResults = await table.query() .fullTextSearch(new MatchQuery("craziou", "text", { fuzziness: 2, })) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` -#### Prefix based Match {#prefix-based-match} +#### Require an exact prefix in fuzzy matching {#prefix-based-match} -Prefix-based match allows you to search for documents containing words that start with a specific prefix. +`prefix_length` keeps the first characters unchanged when matching a misspelled term. It does not turn a short query into a prefix-completion search. ```python Python icon="python" # Fuzzy match (allows typos) -fuzzy_results = ( - table.search(MatchQuery("cra", "text", prefix_length=3)) - .select(["id", "text"]) +prefix_results = ( + table.search(MatchQuery("craziou", "text", fuzziness=2, prefix_length=3)) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" // Fuzzy match (allows typos) -const fuzzyResults = await table.query() - .fullTextSearch(new MatchQuery("cra", "text", { +const prefixResults = await table.query() + .fullTextSearch(new MatchQuery("craziou", "text", { + fuzziness: 2, prefixLength: 3, })) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` @@ -580,26 +624,24 @@ from lancedb.query import PhraseQuery phrase_results = ( table.search(PhraseQuery("puppy runs", "text")) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" -import { PhraseQuery } from "@lancedb/lancedb"; - // Exact phrase match const phraseResults = await table.query() .fullTextSearch(new PhraseQuery("puppy runs", "text")) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` #### Flexible Phrase Match {#flexible-phrase-match} -To provide more flexible phrase matching, LanceDB supports the `slop` parameter. This allows you to match phrases where the terms appear close to each other, even if they are not directly adjacent or in the exact order, as long as they are within the specified `slop` value. +To provide more flexible phrase matching, LanceDB supports the `slop` parameter. This allows you to match phrases where the terms appear close to each other, with gaps between them. The `slop` value limits the extra positions allowed in the phrase. For example, the phrase query "puppy merrily" would not return any results by default. However, if you set `slop=1`, it will match phrases like "puppy jumps merrily", "puppy runs merrily", and similar variations where one word appears between "puppy" and "merrily". @@ -608,21 +650,19 @@ For example, the phrase query "puppy merrily" would not return any results by de # Flexible phrase match with slop=1 for 'puppy merrily' from lancedb.query import PhraseQuery -phrase_results = ( +slop_results = ( table.search(PhraseQuery("puppy merrily", "text", slop=1)) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" -import { PhraseQuery } from "@lancedb/lancedb"; - -// Flexible phrase match with slop=1 for 'puppy runs' -const phraseResults = await table.query() - .fullTextSearch(new PhraseQuery("puppy runs", "text", { slop: 1 })) - .select(["id", "text"]) +// Flexible phrase match with slop=1 for 'puppy merrily' +const slopResults = await table.query() + .fullTextSearch(new PhraseQuery("puppy merrily", "text", { slop: 1 })) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` @@ -656,17 +696,17 @@ boosting_results = ( negative_boost=0.2, ), ) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) # Search across both text and text2 multi_match_results = ( table.search(MultiMatchQuery("crazily", ["text", "text2"])) - .select(["id", "text", "text2"]) + .select(["id", "text", "text2", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) # Search with field boosting @@ -674,28 +714,26 @@ multi_match_boosting_results = ( table.search( MultiMatchQuery("crazily", ["text", "text2"], boosts=[1.0, 2.0]), ) - .select(["id", "text", "text2"]) + .select(["id", "text", "text2", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" -import { MatchQuery, BoostQuery, MultiMatchQuery } from "@lancedb/lancedb"; - // Boosting Example const boostingResults = await table.query() .fullTextSearch(new BoostQuery(new MatchQuery("runs", "text"), new MatchQuery("puppy", "text"), { negativeBoost: 0.2, })) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); // Search across both text fields const multiMatchResults = await table.query() .fullTextSearch(new MultiMatchQuery("crazily", ["text", "text2"])) - .select(["id", "text", "text2"]) + .select(["id", "text", "text2", "_score"]) .limit(100) .toArray(); @@ -704,24 +742,12 @@ const multiMatchBoostingResults = await table.query() .fullTextSearch(new MultiMatchQuery("crazily", ["text", "text2"], { boosts: [1.0, 2.0], })) - .select(["id", "text", "text2"]) + .select(["id", "text", "text2", "_score"]) .limit(100) .toArray(); ``` - -- Use fuzzy search when handling user input that may contain typos or variations -- Apply field boosting to prioritize matches in more important columns -- Combine fuzzy search with boosting for robust and precise search results - -**Recommendations for optimal FTS performance:** - -- Create full-text search indices on text columns that will be frequently searched -- For hybrid search combining text and vectors, see our [hybrid search guide](/search/hybrid-search/) -- For performance benchmarks, check our [benchmark results](/enterprise/benchmarks/) -- For complex queries, use SQL to combine FTS with other filter conditions - ### Boolean Queries {#boolean-queries} LanceDB supports boolean logic in full-text search, allowing you to combine multiple queries using `and` and `or` operators. This is useful when you want to match documents that satisfy multiple conditions (intersection) or at least one of several conditions (union). @@ -734,16 +760,7 @@ In Python, combine two `MatchQuery` objects with `&` for intersection or `|` for {PyFtsBooleanQueries} -In TypeScript, boolean queries are constructed using the `BooleanQuery` class with a list of [Occur, subquery] pairs. For example, to perform an AND query: - -```sql SQL icon="code" -BooleanQuery([ -[Occur.Must, new MatchQuery("puppy", "text")], -[Occur.Must, new MatchQuery("merrily", "text")], -]) -``` - -This approach allows you to specify complex boolean logic by combining multiple subqueries with different Occur values (such as `Must`, `Should`, or `MustNot`). +In TypeScript, pass `[Occur, subquery]` pairs to `BooleanQuery`. Use `Must` for required matches, `Should` for alternatives, and `MustNot` to exclude matches. **Which queries are allowed?** @@ -759,24 +776,22 @@ from lancedb.query import MatchQuery and_query = MatchQuery("puppy", "text") & MatchQuery("merrily", "text") and_results = ( table.search(and_query) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) # Example: Find documents containing either "puppy" or "merrily" or_query = MatchQuery("puppy", "text") | MatchQuery("merrily", "text") or_results = ( table.search(or_query) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" expandable=true -import { MatchQuery, BooleanQuery, Occur } from "@lancedb/lancedb"; - // Flexible boolean queries with MatchQuery // Find documents containing both "puppy" and "merrily" @@ -787,7 +802,7 @@ const mustResults = await table [Occur.Must, new MatchQuery("merrily", "text")], ]), ) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); @@ -799,7 +814,7 @@ const shouldResults = await table [Occur.Should, new MatchQuery("merrily", "text")], ]), ) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` @@ -808,12 +823,33 @@ const shouldResults = await table **How to use booleans?** -- Use `and`/`&`(Python), `Occur.Must`(Typescript) for intersection (documents must match all queries). -- Use `or`/`|`(Python), `Occur.Should`(Typescript) for union (documents must match at least one query). +- Use `&` (Python), `Occur.Must`(Typescript) for intersection (documents must match all queries). +- Use `|` (Python), `Occur.Should`(Typescript) for union (documents must match at least one query). + +### Clean up the example + +Remove the temporary database when you have finished running the queries. + + +```python Python icon="python" +directory.cleanup() +``` +```typescript TypeScript icon="square-js" +table.close(); +db.close(); +await rm(directory, { recursive: true, force: true }); +``` + + + + ## Substring Search Example {#substring-search-example} + + + LanceDB supports searching for substrings in text columns using n-gram tokenization. This is useful for finding partial matches within text content. ### Setting Up the Table {#setting-up-the-table} @@ -825,11 +861,15 @@ First, create a table with sample text data and configure n-gram tokenization: import pyarrow as pa import lancedb -db = lancedb.connect(":memory:") +from tempfile import TemporaryDirectory +from lancedb.index import FTS + +directory = TemporaryDirectory() +db = lancedb.connect(directory.name) data = pa.table({"text": ["hello world", "lance database", "lance is cool"]}) table = db.create_table("test", data=data) -table.create_fts_index("text", base_tokenizer="ngram") +table.create_index("text", config=FTS(base_tokenizer="ngram")) ``` @@ -868,12 +908,10 @@ You can customize the n-gram behavior by adjusting the minimum length and using ```python Python icon="python" -table.create_fts_index( +table.create_index( "text", - base_tokenizer="ngram", + config=FTS(base_tokenizer="ngram", ngram_min_length=2, prefix_only=True), replace=True, - ngram_min_length=2, - prefix_only=True, ) ``` @@ -899,34 +937,47 @@ assert set(r["text"] for r in results) == {"lance database", "lance is cool"} ``` + +```python Python icon="python" +directory.cleanup() +``` + + + ## Full-Text Search on Array Fields {#full-text-search-on-array-fields} + + + LanceDB supports full-text search on string array columns, enabling efficient keyword-based search across multiple values within a single field (e.g., tags, keywords). ### Setting Up the Connection {#setting-up-the-connection} -Connect to your LanceDB instance: +This example uses a temporary local database. Run the steps in order. ```python Python icon="python" -import lancedb +from tempfile import TemporaryDirectory -# Connect to LanceDB -db = lancedb.connect( - uri="db://your-project-slug", - api_key="your-api-key", - region="us-east-1" -) +import lancedb +import pyarrow as pa +from lancedb.index import FTS +from lancedb.query import MatchQuery, PhraseQuery + +directory = TemporaryDirectory() +db = lancedb.connect(directory.name) ``` -```typescript TypeScript icon="square-js" expandable=true -import * as lancedb from "@lancedb/lancedb" +```typescript TypeScript icon="square-js" +import * as lancedb from "@lancedb/lancedb"; +import { Index, MatchQuery, PhraseQuery, makeArrowTable } from "@lancedb/lancedb"; +import { Schema, Field, List, Utf8 } from "apache-arrow"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; -const db = await lancedb.connect({ - uri: "db://your-project-slug", - apiKey: "your-api-key", - region: "us-east-1" -}); +const directory = await mkdtemp(join(tmpdir(), "lancedb-array-fts-")); +const db = await lancedb.connect(directory); ``` @@ -1048,25 +1099,25 @@ console.log(`Created table: ${tableName}`); ### Building the Full-Text Search Index {#building-the-full-text-search-index} -Create an FTS index on the tags column to enable efficient text search: +Store token positions for phrase queries. This example disables stemming so fuzzy matching compares the original word forms: ```python Python icon="python" # Create FTS index -table.create_fts_index("tags") -wait_for_index(table, "tags_idx") +table.create_index("tags", config=FTS(with_position=True, stem=False)) +table.wait_for_index(["tags_idx"]) ``` ```typescript TypeScript icon="square-js" // Create FTS index console.log("Creating FTS index on 'tags' column..."); await table.createIndex("tags", { - config: Index.fts() + config: Index.fts({ withPosition: true, stem: false }) }); // Wait for index const ftsIndexName = "tags_idx"; -await waitForIndex(table, ftsIndexName); +await table.waitForIndex([ftsIndexName], 300); ``` @@ -1080,24 +1131,27 @@ Search for terms with typos using fuzzy matching: print("\nSearching for 'learning' in tags with a typo:") result = ( table.search(MatchQuery("learnin", column="tags", fuzziness=1)) - .select(['id', 'tags', 'description']) + .select(["id", "tags", "description", "_score"]) .to_arrow() ) ``` -```typescript TypeScript icon="square-js"> +```typescript TypeScript icon="square-js" // Search examples console.log("\nSearching for 'learning' in tags with a typo:"); const fuzzyResults = await table.query() .fullTextSearch(new MatchQuery("learnin", "tags", { - fuzziness: 2, + fuzziness: 1, })) - .select(["id", "tags", "description"]) + .select(["id", "tags", "description", "_score"]) .toArray(); console.log(fuzzyResults); ``` +A string-array query can return a row more than once when several elements match. +Deduplicate by your record ID if your application needs one result per row. + ### Performing Phrase Search {#performing-phrase-search} Search for exact phrases within the array fields: @@ -1107,7 +1161,7 @@ Search for exact phrases within the array fields: print("\nSearching for 'machine learning' in tags:") result = ( table.search(PhraseQuery("machine learning", column="tags")) - .select(['id', 'tags', 'description']) + .select(["id", "tags", "description", "_score"]) .to_arrow() ) ``` @@ -1116,8 +1170,25 @@ result = ( console.log("\nSearching for 'machine learning' in tags:"); const phraseResults = await table.query() .fullTextSearch(new PhraseQuery("machine learning", "tags")) - .select(["id", "tags", "description"]) + .select(["id", "tags", "description", "_score"]) .toArray(); console.log(phraseResults); ``` + +### Clean up the array example + +Remove the temporary database when you have finished running the queries. + + +```python Python icon="python" +directory.cleanup() +``` +```typescript TypeScript icon="square-js" +table.close(); +db.close(); +await rm(directory, { recursive: true, force: true }); +``` + + + diff --git a/docs/web/search/hybrid-search.mdx b/docs/web/search/hybrid-search.mdx index 4a2d2dead..c3e453145 100644 --- a/docs/web/search/hybrid-search.mdx +++ b/docs/web/search/hybrid-search.mdx @@ -1,26 +1,77 @@ --- title: Hybrid Search sidebarTitle: "Hybrid Search" -description: Learn how to perform hybrid search in LanceDB by combining vector and full-text search techniques with reranking. +description: "Combine vector similarity and keyword matches in a single ranked result set." icon: "search" +availability: + oss: available + enterprise: available + summary: >- + Hybrid search is available in OSS and Enterprise. --- -In certain cases, you may want to retrieve documents that are semantically similar to a given query, -but also prioritize specific keywords. This is an example of **hybrid search**, a query method that combines -multiple search techniques. +import { PyConnectEnterprisePlaceholders, TsConnectEnterprisePlaceholders } from '/snippets/connection.mdx'; -For detailed examples, look at this [Python Notebook](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/saas_examples/python_notebook/Hybrid_search.ipynb) or the [**TypeScript Example**](https://github.com/lancedb/vectordb-recipes/tree/main/examples/saas_examples/ts_example/hybrid-search) +import { PyHybridSearch, TsHybridSearchBody, TsHybridSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsHybridSearchBody, RsExampleTail as RsHybridSearchTail } from '/snippets/search_hybrid_search.mdx'; -## Example: Hybrid Search {#example-hybrid-search} +Hybrid search combines vector similarity with keyword matching. It is useful +when a query needs both: for example, finding products with a particular model +number whose descriptions match what the customer is looking for. + +LanceDB runs a vector search and a full-text search, then merges their results +with a reranker. Reciprocal Rank Fusion (RRF) is the default. It combines the +positions of records in the two ranked lists. + +Before you query, store vectors and create an FTS index on the text column. + + + + + +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. + +- **Python:** Install with `pip install lancedb`, save as `hybrid_search.py`, then run `python hybrid_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `hybrid_search.mts`, then run `node hybrid_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin hybrid_search` from the extracted directory. + + + +{PyHybridSearch} + + +{TsHybridSearchBody + "\n" + TsHybridSearchTail} + + +{RsHybridSearchBody + "\n" + RsHybridSearchTail} + + + +The puppy record ranks first because both searches place it first. The query +returns `_relevance_score`, with higher values first. This small example shows +the mechanics; evaluate retrieval quality with embeddings and queries from +your own application. + + + +## Generate query embeddings automatically {#example-hybrid-search} ### 1\. Setup {#1-setup} -Import the necessary libraries and dependencies for working with LanceDB, OpenAI embeddings, and reranking. +The following walkthrough uses Sentence Transformers in Python and OpenAI in +TypeScript. Install the corresponding provider packages before starting. +Python downloads a local model; TypeScript calls OpenAI. Install +`lancedb sentence-transformers pandas` for Python, or +`@lancedb/lancedb apache-arrow openai` for TypeScript. Set your OpenAI API key +as described in the [OpenAI embedding setup](/integrations/embedding/openai). +Run the steps for your chosen language in order, choosing one connection option. ```python Python icon="python" -import os import lancedb -import openai +from lancedb.index import FTS +from lancedb.rerankers import RRFReranker from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector ``` @@ -33,7 +84,7 @@ import { Utf8 } from "apache-arrow"; ### 2\. Connect to LanceDB {#2-connect-to-lancedb} -Establish a connection to your LanceDB instance, with different options for Enterprise setups or open source. +Choose a local OSS connection or use your Enterprise connection details. OSS @@ -44,9 +95,6 @@ db = lancedb.connect(uri) ``` ```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; -import * as arrow from "apache-arrow"; - const databaseDir = "data/sample-lancedb"; const db = await lancedb.connect(databaseDir); ``` @@ -54,41 +102,21 @@ const db = await lancedb.connect(databaseDir); Enterprise -For LanceDB Enterprise, set the `db://` URI, region and the host override to your private cloud endpoint: +Replace the placeholders with your Enterprise database URI, API key, region and +endpoint. In application code, load credentials from environment variables or a +secrets manager. -```python Python icon="python" -host_override = os.environ.get("LANCEDB_HOST_OVERRIDE") - -db = lancedb.connect( - uri=uri, - api_key=api_key, - region=region, - host_override=host_override -) -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; -import * as arrow from "apache-arrow"; - -const uri = "db://my-lancedb-instance/my-database"; -const apiKey = process.env.LANCEDB_API_KEY; -const region = process.env.LANCEDB_REGION; -const hostOverride = process.env.LANCEDB_HOST_OVERRIDE; - -const db = await lancedb.connect(uri, { - apiKey, - region - hostOverride, -}); -``` + + {PyConnectEnterprisePlaceholders} + + + {TsConnectEnterprisePlaceholders} + - - ### 3\. Configure Embedding Model {#3-configure-embedding-model} -Set up the any embedding model that will convert text into vector representations for semantic search. +Configure the model used for both document and query embeddings. ```python Python icon="python" @@ -108,6 +136,7 @@ Define the data structure for your documents, including both the text content an ```python Python icon="python" class Documents(LanceModel): + category: str text: str = embeddings.SourceField() vector: Vector(embeddings.ndims()) = embeddings.VectorField() @@ -117,6 +146,7 @@ table = db.create_table(table_name, schema=Documents, mode="overwrite") ```typescript TypeScript icon="square-js" const documentSchema = lancedb.embedding.LanceSchema({ + category: new Utf8(), text: embedFunc.sourceField(new Utf8()), vector: embedFunc.vectorField(), }); @@ -134,20 +164,20 @@ Insert sample documents into your table, which will be used for both semantic an ```python Python icon="python" data = [ - {"text": "rebel spaceships striking from a hidden base"}, - {"text": "have won their first victory against the evil Galactic Empire"}, - {"text": "during the battle rebel spies managed to steal secret plans"}, - {"text": "to the Empire's ultimate weapon the Death Star"}, + {"text": "rebel spaceships striking from a hidden base", "category": "film"}, + {"text": "have won their first victory against the evil Galactic Empire", "category": "film"}, + {"text": "during the battle rebel spies managed to steal secret plans", "category": "film"}, + {"text": "to the Empire's ultimate weapon the Death Star", "category": "film"}, ] table.add(data=data) ``` ```typescript TypeScript icon="square-js" const data = [ - { text: "rebel spaceships striking from a hidden base" }, - { text: "have won their first victory against the evil Galactic Empire" }, - { text: "during the battle rebel spies managed to steal secret plans" }, - { text: "to the Empire's ultimate weapon the Death Star" }, + { text: "rebel spaceships striking from a hidden base", category: "film" }, + { text: "have won their first victory against the evil Galactic Empire", category: "film" }, + { text: "during the battle rebel spies managed to steal secret plans", category: "film" }, + { text: "to the Empire's ultimate weapon the Death Star", category: "film" }, ]; await table.add(data); console.log(`Created table: ${tableName} with ${data.length} rows`); @@ -159,8 +189,8 @@ Create a full-text search index on the text column to enable keyword-based searc ```python Python icon="python" -table.create_fts_index("text") -wait_for_index(table, "text_idx") +table.create_index("text", config=FTS()) +table.wait_for_index(["text_idx"]) ``` ```typescript TypeScript icon="square-js" @@ -168,12 +198,13 @@ console.log("Creating full-text search index..."); await table.createIndex("text", { config: lancedb.Index.fts(), }); -await waitForIndex(table as any, "text_idx"); +await table.waitForIndex(["text_idx"], 300); ``` ### 7\. Set Reranker [Optional] {#7-set-reranker-optional} -Initialize the reranker that will combine and rank results from both semantic and keyword search. By default, lancedb uses RRF reranker, but you can choose other rerankers like `Cohere`, `CrossEncoder`, or others lister in integrations section. +RRF is the default. The examples below configure it explicitly. Other +rerankers and their language support are described in [Reranking](/reranking). ```python Python icon="python" @@ -192,7 +223,7 @@ Perform a hybrid search query that combines semantic similarity with keyword mat ```python Python icon="python" results = ( table.search( - "flower moon", + "rebel spaceships", query_type="hybrid", vector_column_name="vector", fts_columns="text", @@ -208,10 +239,10 @@ print(results) ```typescript TypeScript icon="square-js" console.log("Performing hybrid search..."); -const queryVector = await embedFunc.computeQueryEmbeddings("full moon in May"); +const queryVector = await embedFunc.computeQueryEmbeddings("rebel spaceships"); const hybridResults = await table .query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .rerank(reranker) .select(["text"]) @@ -223,13 +254,16 @@ console.log(hybridResults); ``` -### 9\. Hybrid Search - Explicit Vector and Text Query pattern {#9-hybrid-search-explicit-vector-and-text-query-pattern} -You can also pass the vector and text query explicitly. This is useful if you're not using the embedding API or if you're using a separate embedder service. +### 9\. Supply a vector and text separately {#9-hybrid-search-explicit-vector-and-text-query-pattern} +You can supply the vector and text separately. The vector must use the same +model and dimensions as the stored embeddings. This example reuses the model +configured above; the complete example at the top also demonstrates this pattern +in TypeScript and Rust. ```python Python icon="python" -vector_query = [0.1, 0.2, 0.3, 0.4, 0.5] -text_query = "flower moon" +text_query = "rebel spaceships" +vector_query = embeddings.compute_query_embeddings(text_query)[0] ( table.search(query_type="hybrid") .vector(vector_query) @@ -240,13 +274,15 @@ text_query = "flower moon" ``` -## Query controls {#query-controls} +## Control the candidate results {#query-controls} -Hybrid queries inherit the same builder API as vector and FTS queries, so the same knobs for filtering, distance bounds, and row identity apply. These compose with `.rerank(...)` and the explicit `.vector()` / `.text()` form shown above. +Use a result limit, metadata filters and vector distance bounds to control +which candidates are available to the reranker. The examples below show each +option on the query builder. -Always set `.limit(...)` on production hybrid queries. LanceDB's default search limit is 10, but an -explicit cap gives you a clear top-k contract to tune before reranking. +Set `limit()` to the number of results your application needs. The default is 10. +A larger candidate set can help a reranker find relevant records, but adds work. ### Returning row IDs {#returning-row-ids} @@ -256,7 +292,7 @@ Pass `with_row_id(True)` (Python) or `withRowId()` (TypeScript) to include the i ```python Python icon="python" results = ( - table.search("flower moon", query_type="hybrid") + table.search("rebel spaceships", query_type="hybrid") .with_row_id(True) .limit(10) .to_pandas() @@ -267,7 +303,7 @@ results = ( ```typescript TypeScript icon="square-js" const results = await table .query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .withRowId() .limit(10) @@ -282,7 +318,7 @@ const results = await table ```python Python icon="python" results = ( - table.search("flower moon", query_type="hybrid") + table.search("rebel spaceships", query_type="hybrid") .distance_range(lower_bound=0.0, upper_bound=0.4) .limit(10) .to_pandas() @@ -292,7 +328,7 @@ results = ( ```typescript TypeScript icon="square-js" const results = await table .query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .distanceRange(0.0, 0.4) .limit(10) @@ -304,18 +340,21 @@ Either bound can be omitted to leave that side unbounded. ### Prefilter vs. postfilter {#prefilter-vs-postfilter} -When the query carries a metadata filter via `where(...)`, you can choose whether the filter runs before or after the vector and FTS sub-queries. **Prefiltering** (the default) applies `where` to the candidate set before scoring, which is usually what you want — it shrinks the working set and benefits from any scalar indexes on the filter columns. **Postfiltering** runs the filter on the already-ranked top-k from each sub-query; this can be faster when the filter is non-selective or unindexed, but it may return fewer than `limit` rows because some of the top-k may be filtered out. +A hybrid query applies its metadata filter to both the vector and full-text +searches. Prefiltering, the default, restricts eligible rows before retrieval. +Postfiltering removes candidates after retrieval and can leave fewer results +than requested. See [Filtering](/search/filtering). ```python Python icon="python" # Prefilter (default): filter applied before scoring -table.search("flower moon", query_type="hybrid") \ +table.search("rebel spaceships", query_type="hybrid") \ .where("category = 'film'", prefilter=True) \ .limit(10) \ .to_pandas() # Postfilter: filter applied after the sub-queries return top-k -table.search("flower moon", query_type="hybrid") \ +table.search("rebel spaceships", query_type="hybrid") \ .where("category = 'film'", prefilter=False) \ .limit(10) \ .to_pandas() @@ -324,7 +363,7 @@ table.search("flower moon", query_type="hybrid") \ ```typescript TypeScript icon="square-js" // Prefilter (default): just call .where(...) await table.query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .where("category = 'film'") .limit(10) @@ -332,7 +371,7 @@ await table.query() // Postfilter: chain .postfilter() after .where(...) await table.query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .where("category = 'film'") .postfilter() @@ -341,15 +380,20 @@ await table.query() ``` -The choice gets baked into both sub-queries, so the vector and FTS halves see the filter applied the same way. Use [`explain_plan`](/search/explain#analyzing-non-vector-queries) on a hybrid query to see whether the filter pushed into the scan or ran as a separate `FilterExec` step. +In Python, use [`explain_plan`](/search/explain#analyzing-non-vector-queries) on a hybrid query to see whether the filter pushed into the scan or ran as a separate `FilterExec` step. -## More on Reranking {#more-on-reranking} +## Choose and evaluate a reranker {#more-on-reranking} -You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB comes with [**built-in rerankers**](https://docs.lancedb.com/reranking) and you can implement your own **custom reranker** as well. +Start with RRF, then compare alternatives on representative queries. Review +which relevant rows reach the candidate set before evaluating how a reranker +orders them. Record latency as well as relevance: increasing candidate counts +or applying a model-based reranker adds work. -By default, LanceDB uses `RRFReranker()`, which uses reciprocal rank fusion score, to combine and rerank the results of semantic and full-text search. You can customize the hyperparameters as needed or write your own custom reranker. Here's how you can use any of the available rerankers: +Reranking runs in the application for both OSS and Enterprise connections. +See [Reranking](/reranking) for model-free fusion, model-based rerankers, +custom implementations and score interpretation. -| Argument | Type | Default | Description | -|:---------|:-----|:--------|:------------| -| `normalize` | `str` | `"score"` | The method to normalize the scores. Can be `rank` or `score`. If `rank`, the scores are converted to ranks and then normalized. If `score`, the scores are normalized directly. | -| `reranker` | `Reranker` | `RRF()` | The reranker to use. If not specified, the default reranker is used. | +In synchronous Python, `.rerank(...)` accepts a reranker and a `normalize` +option (`"score"` or `"rank"`). Normalization matters for methods that blend +raw scores; RRF uses rank positions. These are Python query options, not a +shared signature across every SDK. diff --git a/docs/web/search/index.mdx b/docs/web/search/index.mdx index 886751331..9d439b204 100644 --- a/docs/web/search/index.mdx +++ b/docs/web/search/index.mdx @@ -1,88 +1,118 @@ --- title: Search sidebarTitle: Overview -description: "Comprehensive guide to all search capabilities in LanceDB including vector search, full-text search, hybrid search, and more." +description: "Choose vector, full-text or hybrid search and return the records your application needs." icon: "list" +availability: + oss: available + enterprise: available + summary: >- + Vector, full-text and hybrid search are available in OSS and Enterprise. --- -| Feature | Description | -|:---------------|:------------| -| [Vector Search](/search/vector-search/) | Semantic similarity search with multiple distance metrics | -| [Multivector Search](/search/vector-search) | Search using multiple vector embeddings per document | -| [Full-Text Search](/search/full-text-search/) | Keyword-based search with BM25 and pre-filtering | -| [Hybrid Search](/search/hybrid-search/) | Combines vector and full-text search with reranking | -| [Filtering](/search/filtering/) | Filter results based on metadata fields | -| [SQL Queries](/sql) | SQL query capabilities for data exploration and analytics | +import { PyVectorSearch, TsVectorSearchBody, TsVectorSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsVectorSearchBody, RsExampleTail as RsVectorSearchTail } from '/snippets/search_vector_search.mdx'; + +LanceDB supports vector, full-text and hybrid search. Use vector search to find +similar content, full-text search to match words, or hybrid search to combine +both. Filters narrow the results to records that match your criteria. + +## Choose a search method + +| You want to find | Use | Prepare | +|:----------------|:----|:--------| +| Items similar to a query, such as related passages or images | [Vector search](/search/vector-search) | Stored vectors and a compatible query vector | +| Documents represented by several embeddings | [Multivector search](/search/vector-search#multivector) | A multivector column and compatible query vectors | +| Records containing particular words or phrases | [Full-text search](/search/full-text-search) | An FTS index on the text column | +| Relevant meaning alongside keyword matches | [Hybrid search](/search/hybrid-search) | Vectors and an FTS index | +| Rows matching a condition, without similarity ranking | [Filtering](/search/filtering) | A predicate over table columns | + +Vector search can run without an index. An approximate nearest-neighbor index +reduces the work needed for larger tables, with a tradeoff between recall and +latency. Full-text search needs an FTS index before you query it. ## Before you search {#before-you-search} -- Vector search can run without an ANN index as an exhaustive scan. That's useful while prototyping, but build a vector index before relying on low-latency searches over larger tables. -- Full-text and hybrid text search require an FTS index on the text column you query. If a table has multiple FTS indexes, specify the target column. FTS also supports phrase, boolean, boosted, multi-match, and fuzzy query forms when you need more than plain terms. -- Phrase FTS queries require an index created with token positions enabled. -- Multivector search currently uses cosine similarity and accepts either one query vector or a matrix of query vectors; every query vector must match the inner dimension of the multivector column. -- Set an explicit `.limit(...)` for production queries. The default top-k is 10 for search builders, - but spelling it out makes latency and result-count assumptions visible. Query builders also - support controls such as prefilter/postfilter, distance ranges, row-id inclusion, offset - pagination, and Arrow/Pandas/list result materialization. +For vector search, generate stored and query vectors with the same embedding +model and compatible settings. Their dimensions must match. Keep the fields +you want to return, such as document text or an item ID, in the same table. -## Performance {#performance} +For full-text search, choose the text column and build its index. Enterprise +builds indexes in the background, so wait for the index to be ready before +querying it. Phrase queries also need token positions in the index. See +[FTS indexes](/indexing/fts-index). -Three knobs materially affect query latency, memory use, and recall. Be deliberate about each one on every query: + - - - If queries return more data than you need or take longer than expected, the cause is usually projecting more columns than necessary, or letting the result count run unbounded. The best practice is to always pass both `select()` and `limit()`: + - ```python Python icon="python" - table.search(emb).select(["id", "title"]).limit(20) - ``` - - - If a query with `prefilter=False` returns fewer than `limit` results — sometimes zero — the cause is that post-filter applies the predicate after the top-k is selected, so only candidates that already passed the search are filtered. Pre-filter is the default and guarantees every result satisfies the predicate. Switch to `prefilter=False` only when fewer-than-`limit` results are acceptable. See [Filtering](/search/filtering). - - - If recall is lower than expected, the cause is that the search-time knobs limit how many candidates the index considers. Adjust based on your index type: +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. - - Quantized indexes (PQ, RQ, SQ): `refine_factor` pulls extra candidates and re-scores them on full vectors. - - HNSW-backed indexes: `ef` at search time. Start at `1.5 × k`, raise toward `10 × k` if recall is short. - - IVF candidate breadth: `nprobes` is auto-tuned; override only when a selective pre-filter leaves too few neighbors. - - +- **Python:** Install with `pip install lancedb`, save as `search.py`, then run `python search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `search.mts`, then run `node search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin vector_search` from the extracted directory. -For hybrid search, the default `RRFReranker()` combines vector and FTS results into a single ranking via reciprocal rank fusion. See [Hybrid Search](/search/hybrid-search). + + +{PyVectorSearch} + + +{TsVectorSearchBody + "\n" + TsVectorSearchTail} + + +{RsVectorSearchBody + "\n" + RsVectorSearchTail} + + -#### Avoid materializing the whole table {#avoid-materializing-the-whole-table} +This query returns the puppy and dog records, ordered by vector distance. +It searches the three-row table directly, without an ANN index or an embedding +service. For connection options, see [Connect](/basics/connect). -If you need every row in the table (for training, export, or migration), calling `to_pandas()` or `to_arrow()` will run you out of memory on any non-trivial dataset — both materialize the full table at once. The best practice is to iterate via `table.search(...)` or `table.query(...)`, which work the same way in both LanceDB OSS and Enterprise. + -In OSS, you can also stream batches through the underlying Lance dataset directly — useful when you need filter pushdown or fragment-level parallelism via `ds.scanner(...)`: +## Read the results -```python Python icon="python" -ds = table.to_lance() -for batch in ds.to_batches(columns=["id", "text"], batch_size=10000): - process(batch) -``` +Use `select()` to choose the columns your application needs and `limit()` to +set the maximum number of results. The search also returns a ranking value: - -**API disparity between OSS and Enterprise.** +| Search method | Ranking column | Order | +|:--------------|:---------------|:------| +| Vector | `_distance` | Smaller distance first | +| Full-text | `_score` | Higher text relevance first | +| Hybrid | `_relevance_score` | Higher combined relevance first | -LanceDB OSS exposes the `to_pandas()`, `to_arrow()`, and `table.to_lance()` methods for direct Lance dataset access. Enterprise's `RemoteTable` exposes none of these. Only `table.search(...)` and `table.query(...)`. In general, it's always best to go through `search()` and `query()` to keep your code portable across both OSS and Enterprise. - +These values have different meanings. Do not compare a vector distance directly +with a full-text score or treat a reranker score as a probability. -#### Diagnostics {#diagnostics} +## Add filters and reranking {#performance} -To analyze a slow query, inspect what the query engine actually did and the state of the indexes it touched. These two tools surface that information — use them in this order: +[Filters](/search/filtering) restrict which rows qualify, such as documents +from a particular collection. Prefiltering searches within those rows. +Postfiltering removes nonmatching rows from the candidates already retrieved, +so it can leave fewer results than requested. -```python Python icon="python" -print(table.search(emb).where("year > 2000").limit(10).analyze_plan()) -print(table.index_stats("vector_idx")) # num_unindexed_rows should be ~0 -``` +[Reranking](/reranking) reorders the candidates returned by retrieval. Start +with representative queries and expected results, then compare changes to +relevance and latency. A reranker cannot recover a relevant row that retrieval +never returned. -`analyze_plan()` returns the execution plan with per-stage timings, so you can see where the query actually spent its time. `index_stats()` shows how many rows are still unindexed — you want `num_unindexed_rows` to be ~0. In `analyze_plan`, look for: +## Read large result sets in batches {#avoid-materializing-the-whole-table} -| Plan pattern | Fix | -|--------------|-----| -| `LanceScan` with high `bytes_read` / `iops` | Add a missing index, project columns with `select()`, or check that the dataset has been compacted | -| Multiple sequential filters | Reorder filter conditions | +Materializing all rows at once can exhaust application memory. For exports and +training reads, use bounded queries or the batch-reading interfaces described +in [Data loading](/training/data-loading). The SDK buffers Enterprise +query responses before exposing batches. Set a query limit rather than relying +on batch iteration alone to bound memory. -[Explain Analyze](/search/explain) walks through a fully worked before/after example, including how `KNNVectorDistance` and `output_batches` change once indexes are in place. +Direct Lance dataset access through Python's `table.to_lance()` is an OSS +interface. Use the SDK query builders for reads that need to work with both +OSS and Enterprise. + +## Investigate a slow query {#diagnostics} + +Use [query plans](/search/explain) to see the scans, filters and indexes selected +for a query. `explain_plan()` inspects the plan; `analyze_plan()` executes the +query and reports measurements. Compare one change at a time, such as adding +an index, returning fewer columns, or adjusting search parameters. diff --git a/docs/web/search/vector-search.mdx b/docs/web/search/vector-search.mdx index c7324701a..3519df553 100644 --- a/docs/web/search/vector-search.mdx +++ b/docs/web/search/vector-search.mdx @@ -1,8 +1,13 @@ --- title: "Vector Search" sidebarTitle: "Vector Search" -description: "Learn how to run vector search queries in LanceDB. Includes best practices, tips and examples." +description: "Search by vector similarity, choose a distance metric and tune your queries." icon: "arrow-up-right-dots" +availability: + oss: available + enterprise: available + summary: >- + Vector search is available in OSS and Enterprise. --- import { @@ -43,31 +48,68 @@ import { RsBinarySearch, } from '/snippets/search.mdx'; -Vector search is a technique used to search for similar items based on their vector representations, called embeddings. It is also known as similarity search, nearest neighbor search, or approximate nearest neighbor search. +import { PyVectorSearch, TsVectorSearchBody, TsVectorSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsVectorSearchBody, RsExampleTail as RsVectorSearchTail } from '/snippets/search_vector_search.mdx'; -![](/static/assets/images/search/vector-db-basics.png) +Vector search finds similar items by comparing their embeddings: numerical +representations of text, images, audio or other data. An embedding model maps +related items to nearby points in vector space, so you can search by meaning +or similarity instead of matching exact words. -Raw data (e.g. text, images, audio, etc.) is converted into embeddings via an embedding model, which are then stored in a multimodal lakehouse like LanceDB. To perform similarity search at scale, an index is created on the stored embeddings, which can then used to perform fast lookups. +![An embedding model maps data into vectors for similarity search.](/static/assets/images/search/vector-db-basics.png) - -Vector search has **no SQL form**. Run it here and use [SQL](/sql) over the result. - +To run a search, pass a query vector to LanceDB and specify how many matches +to return. Generate the query vector with the same model and settings used for +your stored embeddings. LanceDB ranks the results by distance from the query. -## Supported distance metrics +For small tables, you can search the vectors directly. A [vector index](/indexing/vector-index) +speeds up searches on larger tables by narrowing the candidates to examine. -Distance metrics determine how LanceDB compares vectors to find similar matches. Euclidean or `l2` is the default, and used for general-purpose similarity, `cosine` for unnormalized embeddings, `dot` for normalized embeddings (best performance), or `hamming` for binary vectors. + - -Ensure you always use the same distance metric that your embedding model was trained with. Most modern embedding models use cosine similarity, so `cosine` is often the best choice. However, if your vectors are normalized, you should use `dot` for best performance. - + -The right metric improves both search accuracy and query performance. Currently, LanceDB supports the following metrics: +These examples create a three-row table and find the two closest matches to a +query vector. They use hand-written vectors, so you can try the query without +an embedding model. The temporary table is removed when the program finishes. + +- **Python:** Install with `pip install lancedb`, save as `vector_search.py`, then run `python vector_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `vector_search.mts`, then run `node vector_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin vector_search` from the extracted directory. + + + +{PyVectorSearch} + + +{TsVectorSearchBody + "\n" + TsVectorSearchTail} + + +{RsVectorSearchBody + "\n" + RsVectorSearchTail} + + + +The first result is the puppy record, whose vector exactly matches the query. +The dog record is next. `_distance` is smaller for closer matches. This example +uses an exhaustive scan; the indexing section explains how to search larger tables. + + + +## Supported distance metrics {#supported-distance-metrics} + +A distance metric determines how LanceDB compares vectors. Choose the metric +recommended by your embedding model. Use the same metric when building an index +and querying it. + +LanceDB supports `l2`, `cosine`, `dot` and `hamming`. An unindexed search uses +`l2` by default; Hamming distance is for binary vectors. The table below shows +the value returned in the `_distance` column. Smaller values rank first. | Distance metric | Mathematical form | Notes | |---|---|---| -| `l2` | $$\|x-y\|_2= \sqrt{\sum_i (x_i-y_i)^2}$$ | The $\ell_2$ norm. Measures Euclidean distance between two points in geometric space. | +| `l2` | $\sum_i (x_i-y_i)^2$ | Squared Euclidean distance returned in `_distance`. | | `cosine` | $1-\frac{x\cdot y}{\|x\|_2\|y\|_2}$ | Measures directional difference between vectors. Insensitive to nonzero scaling of vectors. | -| `dot` | $x\cdot y= \underset{i}{\sum} x_i y_i$ | The standard dot product of real vectors. Sensitive to nonzero scaling of vectors; use on normalized vectors. | +| `dot` | $1-\sum_i x_i y_i$ | Dot-product distance. For unit-normalized vectors, this equals cosine distance. | | `hamming` | $\underset{i}{\sum} \,\: \mathbf{1}_{\, [x_i\neq y_i]}$ | Counts the number of bitwise-differing bits between binary vectors. | For indexed search, supported distance metrics vary by index type: @@ -111,12 +153,10 @@ non-`l2` index, explicitly set the matching metric or upgrade the SDK. -Here you can see the same search but using `cosine` similarity instead of `l2` -distance. If the table has a vector index, make sure it was also built with -`cosine`. The result focuses on vector direction rather than absolute distance, -which works better for unnormalized embeddings. +These examples use cosine distance and return up to ten matches. If the table +has a vector index, build it with the same metric. -Set `.limit(...)` on vector searches you run in applications. You can page through results with `.offset(...)` and include LanceDB's internal row id with `.with_row_id()` / `.withRowId()` when you need a stable handle for follow-up operations. +Set `.limit(...)` on vector searches you run in applications. You can page through results with `.offset(...)` and include LanceDB's internal row id with `.with_row_id()` / `.withRowId()` when an operation requires internal row IDs. Use your own ID column for application identifiers. ## Selecting the vector column @@ -153,13 +193,12 @@ The same field-path syntax works when creating an index on a nested vector colum When several columns share a name across structs (for example, `image.embedding` and `text.embedding`), LanceDB still picks the one whose dimension matches your query vector. If two candidates have the same dimension, you must pass the column name explicitly. -## Vector Search With ANN Index +## Search with a vector index {#vector-search-with-ann-index} -Instead of performing an exhaustive search on the entire database for each and every query, approximate nearest neighbour (ANN) algorithms use an index to narrow down the search space, which significantly reduces query latency. - -The trade-off is that the results are not guaranteed to be the true nearest neighbors of the query, but are usually "good enough" for most use cases. - -Use ANN search for large-scale applications where speed matters more than perfect recall. LanceDB uses approximate nearest neighbor algorithms to deliver fast results without examining every vector in your dataset. +An approximate nearest-neighbor (ANN) index narrows the set of vectors examined +for a query. It can reduce query latency, but may miss neighbors that an +exhaustive search would find. Compare recall and latency on queries that +represent your application before choosing an index and its search settings. When a vector index is used, `_distance` is not always the true distance between full vectors. On quantized ANN indexes, LanceDB may compute `_distance` from the compressed representation for speed. Use `refine_factor` when you want reranking on full vectors. @@ -193,8 +232,8 @@ The table below summarizes the behavior of `_distance` in search results based o -For deeper tuning guidance on indexing and performance estimation, see the [vector indexes](/indexing/vector-index) page, -For tuning `nprobes`, see below. +See [vector indexes](/indexing/vector-index) for index configuration. The +following sections cover query-time settings. ### Tuning `approx_mode` @@ -208,13 +247,13 @@ The supported values are: | `normal` | Use the default balance between query latency and recall. This only has an effect for RQ indexes built with `num_bits > 1`. | | `accurate` | Prefer higher recall, which can increase query latency. | -You can change `approx_mode` per query without rebuilding the index. For RQ indexes built with `num_bits=1`, `normal` uses the same one-bit scoring path as `fast`. If you also set `refine_factor`, LanceDB first uses `approx_mode` while finding candidates, then reranks the selected candidates on full vectors. +Only the Rust SDK sets `approx_mode`; the Python and TypeScript SDKs do not expose it. You can change `approx_mode` per query without rebuilding the index. For RQ indexes built with `num_bits=1`, `normal` uses the same one-bit scoring path as `fast`. If you also set `refine_factor`, LanceDB first uses `approx_mode` while finding candidates, then reranks the selected candidates on full vectors. ### Tuning `nprobes` - `nprobes` controls how many partitions are searched at query time. - `nprobes` improves candidate recall, but does not by itself make `_distance` exact. -- By default, LanceDB automatically tunes `nprobes` to achieve the best performance without noticeably sacrificing accuracy. +- The default probe selection is a starting point. Measure recall and latency for your data before overriding it. - In most cases, leave `nprobes` unset and use the auto-tuned value. - Only tune `nprobes` manually when recall is below your target, or when you need even higher performance for your workload. - If recall is too low, increase `nprobes` gradually, but after a certain threshold, increasing `nprobes` yields only marginal accuracy gains. @@ -223,6 +262,8 @@ You can change `approx_mode` per query without rebuilding the index. For RQ inde For filtered ANN searches, you can also set `minimum_nprobes` and `maximum_nprobes`. LanceDB starts with the minimum and can scan more partitions up to the maximum if the filter leaves too few candidates. Calling `nprobes(n)` fixes both values to `n`, which disables that adaptive behavior. +Whether a deployment honors this range depends on its version; one that does not searches exactly +`minimum_nprobes` partitions. ### Vector Search with Prefiltering @@ -242,15 +283,15 @@ This is the default vector search setting. You can use prefiltering to boost que -This filters out rows where label ≤ 2 before doing vector search, then picks specific columns from the top 5 matches. - -The `.where("label > 2")` applies a filter before vector search, `.select(["text", "keywords", "label"])` chooses specific columns to return, and `.limit(5)` restricts results to the top `5` most similar vectors. - -As a result, you'll see a result with just the data you want from the most similar vectors. +The filter limits which rows can match. Use `select()` to choose the returned +columns and `limit()` to set the maximum number of results. ### Vector Search with Postfiltering -Use postfiltering to prioritize vector similarity by searching the full dataset first, then applying metadata filters to the top results. This approach ensures you get the most similar vectors before filtering, which can be crucial when similarity is more important than metadata constraints. +Postfiltering selects vector-search candidates before applying the metadata +predicate. It can return fewer matches than requested, even when other rows in +the table satisfy the predicate. Use prefiltering when you want the nearest +records within the matching subset. @@ -266,11 +307,8 @@ Use postfiltering to prioritize vector similarity by searching the full dataset -Here you can see how to do vector search first to get the most similar vectors, then filter by label > 1 on those results. - -The `prefilter=False` parameter tells LanceDB to apply the filter after vector search instead of before, `.where("label > 1")` filters the top results by metadata, and `.select()` chooses which columns to include. - -In the end, you receive a query result with the best matches that also meet your metadata requirements. +Only retrieved candidates that satisfy the predicate remain in the result. +Increase the candidate limit or use prefiltering if too few records are returned. [Post-filtering](/search/filtering/#post-filtering-with-vector-search) in LanceDB applies @@ -282,7 +320,7 @@ the filter condition after obtaining the nearest neighbors based on vector simil Use multivector search when your documents contain multiple embeddings and you need sophisticated matching between query and document vector pairs. The late interaction approach finds the most relevant combinations across all available embeddings and provides nuanced similarity scoring. Only `cosine` similarity is supported as the distance metric for multivector search operations. -Every query vector must match the inner dimension of the multivector column; LanceDB rejects mismatched query dimensions rather than guessing how to reshape them. +Every query vector must match the inner dimension of the multivector column. @@ -292,7 +330,7 @@ Every query vector must match the inner dimension of the multivector column; Lan Here you can see how to take 2 query vectors and find the best matching pairs between them and document vectors using late interaction. The `np.random.random(size=(2, 256))` creates a 2×256 array with two random query vectors, `.limit(5)` returns the top 5 best document-query combinations, and `.to_pandas()` provides results in a DataFrame format. -**Read more:** [Multivector search](/search/vector-search) +**Read more:** [Multivector search walkthrough](#multivector) ## Advanced Search Scenarios @@ -350,9 +388,11 @@ The search produces an Arrow table with binary vectors ranked by how many bits d ## Scaling Vector Search -### Batch Search +### Query multiple vectors in a batch {#batch-search} -Use batch search to handle multiple query vectors simultaneously. This gives you significant efficiency gains over individual queries. LanceDB processes all vectors in parallel and organizes results with a `query_index` field that maps each result set back to its originating query. +Batch search accepts multiple query vectors in one request. The `query_index` +column identifies which query produced each result. Measure throughput and +latency with your workload when choosing a batch size. @@ -368,26 +408,16 @@ Use batch search to handle multiple query vectors simultaneously. This gives you -This takes 5 query embeddings and finds the top 5 matches for each one in a single batch operation. +Each input vector has its own nearest-neighbor results. Use `query_index` to +associate each row with the corresponding input vector. -The `load_dataset()` loads embeddings from a Hugging Face dataset, `query_embeds` contains `5` query vectors, and `.search(query_embeds)` processes all queries simultaneously. - -The final query result contains all results, including a `query_index` to tell you which query each result came from. - - -When processing batch queries, the results include a `query_index` field -to explicitly associate each result set with its corresponding query in -the input batch. - - -### Search With Asynchronous Indexing +### Include or skip unindexed rows {#search-with-asynchronous-indexing} To optimize for speed over completeness, enable the `fast_search` flag in your query to skip searching unindexed data. -While vector indexing occurs asynchronously, newly added vectors are immediately -searchable through a fallback brute-force search mechanism. This ensures zero -latency between data insertion and searchability, though it may temporarily -increase query response times. +By default, search includes both indexed vectors and unindexed rows visible to +the query. LanceDB scans the unindexed rows directly. `fast_search` skips that +scan, which can reduce query time but omit recent records. @@ -403,13 +433,9 @@ increase query response times. -Here you can see how to turn on fast search mode to skip unindexed vectors and only look through indexed data for speed. +Use this option only when results from indexed data are sufficient. -The `fast_search=True` parameter tells LanceDB to only search indexed vectors, skipping any recently added data that hasn't been indexed yet. - -You'll obtain a query result with the top `5` matches from indexed vectors, but might miss data that was just added. - -## Brute Force Search +## Search without a vector index {#brute-force-search} ### Search With No Index @@ -433,11 +459,13 @@ Choose brute force search when you need guaranteed 100% recall, typically with s -This carries out a brute force search through every vector in the table to find the 3 closest matches to a random 1536-dimensional query. You'll get back a list of the most similar vectors with exact distances. +These queries return the three nearest matches. Without a vector index, LanceDB +calculates distances directly from the stored vectors. ![](/static/assets/images/search/knn_search.png) -As you can imagine, the brute force approach is not scalable for datasets larger than a few hundred thousand vectors, as the latency of the search grows linearly with the size of the dataset. This is where approximate nearest neighbour (ANN) algorithms come in. +Exhaustive search does more work as the table grows. When query latency becomes +too high, add an ANN index and compare its results with the exhaustive baseline. ### Bypass the Vector Index @@ -461,7 +489,8 @@ This skips the approximate index and checks every single vector for exact, groun The `.bypass_vector_index()` method forces LanceDB to perform an exhaustive search through all vectors instead of using the approximate nearest neighbor index, ensuring exact results but at the cost of slower performance. -The output is a query result with the top 5 exact matches, guaranteeing 100% recall but taking longer to run. +The query returns the nearest five rows within its filters and distance bounds. +Use an exhaustive scan as a baseline when measuring ANN recall. This approach is particularly useful when: - Evaluating ANN index quality @@ -596,7 +625,7 @@ tbl.create_index(metric="cosine", vector_column_name="vector") A brute-force scan over a multivector column has to compare every query vector to every document vector in every row, so the cost grows with both the row count and the number of vectors per row. In LanceDB OSS, the query will just run, so a large unindexed multivector table can stall a process for a long time before returning results. -On LanceDB Cloud, the brute-force KNN safety check applies a stricter row threshold to multivector columns — roughly 10× lower than for single-vector columns. So an unindexed multivector table can be rejected with a "vector search would use brute-force KNN" error well before a comparable single-vector table would. LanceDB Enterprise doesn't have the limit. Build the index before you start hitting it from production traffic, even if the dataset is small enough that you'd skip indexing for a single-vector workload. +On LanceDB Cloud, the brute-force KNN safety check applies a stricter row threshold to multivector columns; roughly 10× lower than for single-vector columns. So an unindexed multivector table can be rejected with a "vector search would use brute-force KNN" error well before a comparable single-vector table would. LanceDB Enterprise doesn't have the limit. Build the index before you start hitting it from production traffic, even if the dataset is small enough that you'd skip indexing for a single-vector workload. #### 6. Query a Single Vector @@ -644,7 +673,7 @@ import lancedb from pylate import models ## 1) Load a late-interaction model via PyLate -## PyLate docs show ColBERT() + encode(..., is_query=...) :contentReference[oaicite:2]{index=2} +## PyLate docs show ColBERT() + encode(..., is_query=...) model = models.ColBERT(model_name_or_path="lightonai/GTE-ModernColBERT-v1") ## You can discover dim from one embedding (avoid guessing) @@ -655,7 +684,7 @@ db = lancedb.connect("./pylate_lancedb") schema = pa.schema([ pa.field("doc_id", pa.string()), pa.field("text", pa.string()), - # multivector: list> :contentReference[oaicite:3]{index=3} + # multivector: list> pa.field("mv", pa.list_(pa.list_(pa.float32(), dim))), ]) @@ -667,7 +696,7 @@ docs = [ ## 3) Encode documents with PyLate (token vectors per doc) doc_texts = [d["text"] for d in docs] -doc_embs = model.encode(doc_texts, is_query=False) # list/array of (T, dim) per doc :contentReference[oaicite:4]{index=4} +doc_embs = model.encode(doc_texts, is_query=False) # list/array of (T, dim) per doc rows = [] for d, emb in zip(docs, doc_embs): @@ -678,14 +707,14 @@ tbl = db.create_table("docs", data=rows, schema=schema, mode="overwrite") ## 4) Build an index + query using a query matrix. ## Multivector brute-force scales with rows × vectors-per-row, so build the index -## at much smaller dataset sizes than you would for single-vector search — and +## at much smaller dataset sizes than you would for single-vector search; and ## always before exposing the table to remote traffic. tbl.create_index(vector_column_name="mv", metric="cosine") query = "Tell me about ramen in Japan" -q_emb = np.asarray(model.encode([query], is_query=True)[0], dtype=np.float32) # (Tq, dim) :contentReference[oaicite:5]{index=5} +q_emb = np.asarray(model.encode([query], is_query=True)[0], dtype=np.float32) # (Tq, dim) -out = tbl.search(q_emb).limit(5).to_pandas() # multivector search accepts a matrix :contentReference[oaicite:6]{index=6} +out = tbl.search(q_emb).limit(5).to_pandas() # multivector search accepts a matrix print(out[["doc_id", "text"]]) ``` diff --git a/docs/web/snippets/ingestion_delete.mdx b/docs/web/snippets/ingestion_delete.mdx new file mode 100644 index 000000000..f6f6222ae --- /dev/null +++ b/docs/web/snippets/ingestion_delete.mdx @@ -0,0 +1,4 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExample = "use std::sync::Arc;\n\nuse arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase};\n\nfn make_users_reader(rows: &[(i64, &str)]) -> Box {\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"name\", DataType::Utf8, false),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),\n Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),\n ],\n )\n .unwrap();\n Box::new(RecordBatchIterator::new(\n vec![Ok(batch)].into_iter(),\n schema,\n ))\n}\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let table = db\n .create_table(\n \"users\",\n make_users_reader(&[(1, \"Alice\"), (2, \"Bob\"), (3, \"Charlie\")]),\n )\n .execute()\n .await?;\n let predicate = \"id = 3\";\n let selected: Vec = table\n .query()\n .only_if(predicate)\n .execute()\n .await?\n .try_collect()\n .await?;\n println!(\"Selected: {selected:?}\");\n table.delete(predicate).await?;\n let rows: Vec = table.query().execute().await?.try_collect().await?;\n println!(\"Remaining: {rows:?}\");\n assert_eq!(table.count_rows(None).await?, 2);\n assert_eq!(\n table\n .count_rows(Some(\"id = 1 AND name = 'Alice'\".into()))\n .await?,\n 1\n );\n assert_eq!(\n table\n .count_rows(Some(\"id = 2 AND name = 'Bob'\".into()))\n .await?,\n 1\n );\n assert_eq!(table.count_rows(Some(\"id = 3\".into())).await?, 0);\n Ok(())\n}\n"; + diff --git a/docs/web/snippets/ingestion_examples.mdx b/docs/web/snippets/ingestion_examples.mdx new file mode 100644 index 000000000..0268a906a --- /dev/null +++ b/docs/web/snippets/ingestion_examples.mdx @@ -0,0 +1,66 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PyAddFromDataset = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n from pathlib import Path\n\n import pyarrow as pa\n import pyarrow.parquet as pq\n\n data_path = str(Path(uri) / \"input.parquet\")\n pq.write_table(pa.table({\"id\": [1, 2], \"name\": [\"Alice\", \"Bob\"]}), data_path)\n import pyarrow.dataset as ds\n\n dataset = ds.dataset(data_path, format=\"parquet\")\n table = db.create_table(\"my_table\", schema=dataset.schema, mode=\"overwrite\")\n table.add(dataset)\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyAddProgress = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\ndef report_progress(progress):\n print(f\"Rows written: {progress['output_rows']}\")\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\"users\", [{\"id\": 1, \"name\": \"Alice\"}])\n table.add(\n [{\"id\": 2, \"name\": \"Bob\"}, {\"id\": 3, \"name\": \"Charlie\"}],\n progress=report_progress,\n )\n rows = sorted(table.search().to_list(), key=lambda row: row[\"id\"])\n print(\"After insertion:\", rows)\n"; + +export const PyAppendRows = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\"users\", [{\"id\": 1, \"name\": \"Alice\"}])\n table.add([{\"id\": 2, \"name\": \"Bob\"}, {\"id\": 3, \"name\": \"Charlie\"}])\n rows = sorted(table.search().to_list(), key=lambda row: row[\"id\"])\n print(rows)\n"; + +export const PyCreateEmptyTable = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n schema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n pa.field(\"item\", pa.string()),\n pa.field(\"price\", pa.float32()),\n ]\n )\n tbl = db.create_table(\"test_empty_table\", schema=schema, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateEmptyTablePydantic = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n from lancedb.pydantic import LanceModel, Vector\n\n class Item(LanceModel):\n vector: Vector(2)\n item: str\n price: float\n\n tbl = db.create_table(\n \"test_empty_table_new\", schema=Item.to_arrow_schema(), mode=\"overwrite\"\n )\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableConflictHandling = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n data = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7}]\n db.create_table(\"conflict_table\", data)\n # Idempotent open: reuse the existing table if it exists.\n # The provided data is ignored; the schema is validated against the\n # existing table and a mismatch raises an error.\n tbl = db.create_table(\"conflict_table\", data, exist_ok=True)\n\n # Overwrite: replace the contents with the provided data.\n tbl = db.create_table(\"conflict_table\", data, mode=\"overwrite\")\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableCustomSchema = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n custom_schema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 4)),\n pa.field(\"lat\", pa.float32()),\n pa.field(\"long\", pa.float32()),\n ]\n )\n\n data = [\n {\"vector\": [1.1, 1.2, 1.3, 1.4], \"lat\": 45.5, \"long\": -122.7},\n {\"vector\": [0.2, 1.8, 0.4, 3.6], \"lat\": 40.1, \"long\": -74.1},\n ]\n tbl = db.create_table(\n \"my_table_custom_schema\", data, schema=custom_schema, mode=\"overwrite\"\n )\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableFromArrow = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import numpy as np\n import pyarrow as pa\n\n dim = 16\n total = 2\n schema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float16(), dim)),\n pa.field(\"text\", pa.string()),\n ]\n )\n data = pa.Table.from_arrays(\n [\n pa.array(\n [np.random.randn(dim).astype(np.float16) for _ in range(total)],\n pa.list_(pa.float16(), dim),\n ),\n pa.array([\"foo\", \"bar\"]),\n ],\n [\"vector\", \"text\"],\n )\n tbl = db.create_table(\"f16_tbl\", data, schema=schema, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableFromDicts = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n data = [\n {\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7},\n {\"vector\": [0.2, 1.8], \"lat\": 40.1, \"long\": -74.1},\n ]\n db.create_table(\"test_table\", data, mode=\"overwrite\")\n tbl = db[\"test_table\"]\n tbl.head()\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableFromIterator = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n schema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 4)),\n pa.field(\"item\", pa.utf8()),\n pa.field(\"price\", pa.float32()),\n ]\n )\n\n def make_batches():\n for i in range(5):\n yield pa.RecordBatch.from_arrays(\n [\n pa.array(\n [[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],\n pa.list_(pa.float32(), 4),\n ),\n pa.array([\"foo\", \"bar\"]),\n pa.array([10.0, 20.0]),\n ],\n [\"vector\", \"item\", \"price\"],\n )\n\n table = db.create_table(\"batched_table\", schema=schema)\n table.add(make_batches())\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyCreateTableFromPandas = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pandas as pd\n\n data = pd.DataFrame(\n {\n \"vector\": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],\n \"lat\": [45.5, 40.1],\n \"long\": [-122.7, -74.1],\n }\n )\n db.create_table(\"my_table_pandas\", data, mode=\"overwrite\")\n db[\"my_table_pandas\"].head()\n"; + +export const PyCreateTableFromPolars = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import polars as pl\n\n data = pl.DataFrame(\n {\n \"vector\": [[3.1, 4.1], [5.9, 26.5]],\n \"item\": [\"foo\", \"bar\"],\n \"price\": [10.0, 20.0],\n }\n )\n tbl = db.create_table(\"my_table_pl\", data, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableFromPydantic = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n from lancedb.pydantic import LanceModel, Vector\n\n class Content(LanceModel):\n movie_id: int\n vector: Vector(128)\n genres: str\n title: str\n imdb_id: int\n\n @property\n def imdb_url(self) -> str:\n return f\"https://www.imdb.com/title/tt{self.imdb_id}\"\n\n tbl = db.create_table(\"movielens_small\", schema=Content, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableNestedSchema = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n from lancedb.pydantic import LanceModel, Vector\n from pydantic import BaseModel\n\n class Document(BaseModel):\n content: str\n source: str\n\n class NestedSchema(LanceModel):\n id: str\n vector: Vector(1536)\n document: Document\n\n tbl = db.create_table(\"nested_table\", schema=NestedSchema, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyDeleteRows = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"users\",\n [\n {\"id\": 1, \"name\": \"Alice\"},\n {\"id\": 2, \"name\": \"Bob\"},\n {\"id\": 3, \"name\": \"Charlie\"},\n ],\n )\n predicate = \"id = 3\"\n print(\"Selected:\", table.search().where(predicate).to_list())\n table.delete(predicate)\n rows = sorted(table.search().to_list(), key=lambda row: row[\"id\"])\n print(\"Remaining:\", rows)\n"; + +export const PyDropTable = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n # Create a table first\n data = [{\"vector\": [1.1, 1.2], \"lat\": 45.5}]\n db.create_table(\"my_table\", data, mode=\"overwrite\")\n\n # Drop the table\n db.drop_table(\"my_table\")\n"; + +export const PyInsertIfNotExists = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n )\n\n (table.merge_insert(\"id\").when_not_matched_insert_all().execute(incoming_users))\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyMergeDeleteMissingBySource = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2, 3],\n \"name\": [\"Alice\", \"Bob\", \"Charlie\"],\n \"login_count\": [10, 20, 5],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n )\n\n (\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .when_not_matched_by_source_delete()\n .execute(incoming_users)\n )\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyMergeMatchedUpdateOnly = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n )\n\n (table.merge_insert(\"id\").when_matched_update_all().execute(incoming_users))\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyMergePartialColumns = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n }\n )\n\n (\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .execute(incoming_users)\n )\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyMergeUpdateInsert = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n )\n\n (\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .execute(incoming_users)\n )\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyOpenExistingTable = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n # Create a table first\n data = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7}]\n db.create_table(\"test_table\", data, mode=\"overwrite\")\n\n # List table names\n print(db.list_tables().tables)\n\n # Open existing table\n tbl = db.open_table(\"test_table\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyUpdateOperation = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n table.update(where=\"id = 2\", values={\"name\": \"Bobby\"})\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyUpdateUsingSql = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n table.update(where=\"id = 2\", values_sql={\"login_count\": \"login_count + 1\"})\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyWriteWorkflow = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"users\",\n [\n {\"id\": 1, \"name\": \"Alice\", \"login_count\": 10},\n {\"id\": 2, \"name\": \"Bob\", \"login_count\": 20},\n ],\n )\n table.add([{\"id\": 3, \"name\": \"Charlie\", \"login_count\": 5}])\n table.update(where=\"id = 2\", values={\"name\": \"Bobby\"})\n table.update(where=\"id = 2\", values_sql={\"login_count\": \"login_count + 1\"})\n (\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .execute(\n [\n {\"id\": 3, \"name\": \"Charlie\", \"login_count\": 6},\n {\"id\": 4, \"name\": \"Dana\", \"login_count\": 1},\n ]\n )\n )\n print(\"Rows selected for deletion:\", table.search().where(\"id = 4\").to_list())\n table.delete(\"id = 4\")\n rows = sorted(table.search().to_list(), key=lambda row: row[\"id\"])\n print(rows)\n"; + +export const TsAddProgressBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-ingestion-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"users\", [{ id: 1, name: \"Alice\" }]);\n await table.add(\n [\n { id: 2, name: \"Bob\" },\n { id: 3, name: \"Charlie\" },\n ],\n {\n progress: (progress) =>\n console.log(`Rows written: ${progress.outputRows}`),\n },\n );\n const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);\n console.log(\"After insertion:\", rows);\n"; + +export const TsAddProgressTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsDeleteRowsBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-ingestion-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"users\", [\n { id: 1, name: \"Alice\" },\n { id: 2, name: \"Bob\" },\n { id: 3, name: \"Charlie\" },\n ]);\n const predicate = \"id = 3\";\n console.log(\"Selected:\", await table.query().where(predicate).toArray());\n await table.delete(predicate);\n const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);\n console.log(\"Remaining:\", rows);\n"; + +export const TsDeleteRowsTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsWriteWorkflowBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\n\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-ingestion-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"users\", [\n { id: 1, name: \"Alice\", login_count: 10 },\n { id: 2, name: \"Bob\", login_count: 20 },\n ]);\n await table.add([{ id: 3, name: \"Charlie\", login_count: 5 }]);\n await table.update({ where: \"id = 2\", values: { name: \"Bobby\" } });\n await table.update({\n where: \"id = 2\",\n valuesSql: { login_count: \"login_count + 1\" },\n });\n await table\n .mergeInsert(\"id\")\n .whenMatchedUpdateAll()\n .whenNotMatchedInsertAll()\n .execute([\n { id: 3, name: \"Charlie\", login_count: 6 },\n { id: 4, name: \"Dana\", login_count: 1 },\n ]);\n console.log(\n \"Rows selected for deletion:\",\n await table.query().where(\"id = 4\").toArray(),\n );\n await table.delete(\"id = 4\");\n const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);\n console.log(rows);\n"; + +export const TsWriteWorkflowTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const RsWriteWorkflow = "use std::sync::Arc;\n\nuse arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase};\n\nfn make_users_reader(rows: &[(i64, &str, i64)]) -> Box {\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"name\", DataType::Utf8, false),\n Field::new(\"login_count\", DataType::Int64, false),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),\n Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),\n Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.2))),\n ],\n )\n .unwrap();\n Box::new(RecordBatchIterator::new(\n vec![Ok(batch)].into_iter(),\n schema,\n ))\n}\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let table = db\n .create_table(\n \"users\",\n make_users_reader(&[(1, \"Alice\", 10), (2, \"Bob\", 20)]),\n )\n .execute()\n .await?;\n table\n .add(make_users_reader(&[(3, \"Charlie\", 5)]))\n .execute()\n .await?;\n table\n .update()\n .only_if(\"id = 2\")\n .column(\"name\", \"'Bobby'\")\n .execute()\n .await?;\n table\n .update()\n .only_if(\"id = 2\")\n .column(\"login_count\", \"login_count + 1\")\n .execute()\n .await?;\n let mut merge = table.merge_insert(&[\"id\"]);\n merge\n .when_matched_update_all(None)\n .when_not_matched_insert_all();\n merge\n .execute(make_users_reader(&[(3, \"Charlie\", 6), (4, \"Dana\", 1)]))\n .await?;\n let selected: Vec = table\n .query()\n .only_if(\"id = 4\")\n .execute()\n .await?\n .try_collect()\n .await?;\n println!(\"Rows selected for deletion: {selected:?}\");\n table.delete(\"id = 4\").await?;\n let rows: Vec = table.query().execute().await?.try_collect().await?;\n println!(\"Remaining rows: {rows:?}\");\n assert_eq!(table.count_rows(None).await?, 3);\n assert_eq!(\n table\n .count_rows(Some(\n \"id = 2 AND name = 'Bobby' AND login_count = 21\".into()\n ))\n .await?,\n 1\n );\n assert_eq!(\n table\n .count_rows(Some(\"id = 3 AND login_count = 6\".into()))\n .await?,\n 1\n );\n Ok(())\n}\n"; + diff --git a/docs/web/snippets/ingestion_progress.mdx b/docs/web/snippets/ingestion_progress.mdx new file mode 100644 index 000000000..d910468f6 --- /dev/null +++ b/docs/web/snippets/ingestion_progress.mdx @@ -0,0 +1,4 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExample = "use std::sync::Arc;\n\nuse arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::ExecutableQuery;\n\nfn make_users_reader(rows: &[(i64, &str)]) -> Box {\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"name\", DataType::Utf8, false),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),\n Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),\n ],\n )\n .unwrap();\n Box::new(RecordBatchIterator::new(\n vec![Ok(batch)].into_iter(),\n schema,\n ))\n}\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let table = db\n .create_table(\"users\", make_users_reader(&[(1, \"Alice\")]))\n .execute()\n .await?;\n table\n .add(make_users_reader(&[(2, \"Bob\"), (3, \"Charlie\")]))\n .progress(|progress| println!(\"Rows written: {}\", progress.output_rows()))\n .execute()\n .await?;\n let rows: Vec = table.query().execute().await?.try_collect().await?;\n println!(\"After insertion: {rows:?}\");\n assert_eq!(table.count_rows(None).await?, 3);\n assert_eq!(\n table\n .count_rows(Some(\"id = 1 AND name = 'Alice'\".into()))\n .await?,\n 1\n );\n assert_eq!(\n table\n .count_rows(Some(\"id = 2 AND name = 'Bob'\".into()))\n .await?,\n 1\n );\n assert_eq!(table.count_rows(Some(\"id = 3\".into())).await?, 1);\n Ok(())\n}\n"; + diff --git a/docs/web/snippets/integrations.mdx b/docs/web/snippets/integrations.mdx index 22bbde338..67bb654cf 100644 --- a/docs/web/snippets/integrations.mdx +++ b/docs/web/snippets/integrations.mdx @@ -88,7 +88,7 @@ export const PyFrameworksLlamaindexHybridSearch = "from lancedb.rerankers import export const PyFrameworksLlamaindexQuickStart = "import logging\nimport sys\nimport textwrap\n\nimport openai\n\n# Uncomment to see debug logs\n# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)\n# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))\nfrom llama_index.core import (\n Document,\n SimpleDirectoryReader,\n StorageContext,\n VectorStoreIndex,\n)\nfrom llama_index.vector_stores.lancedb import LanceDBVectorStore\n\nopenai.api_key = \"sk-...\"\n\ndocuments = SimpleDirectoryReader(\"./data/your-data-dir/\").load_data()\nprint(\"Document ID:\", documents[0].doc_id, \"Document Hash:\", documents[0].hash)\n\n## For LanceDB Enterprise :\n# vector_store = LanceDBVectorStore(\n# uri=\"db://db_name\", # your remote DB URI\n# api_key=\"sk_..\", # lancedb enterprise api key\n# region=\"your-region\" # the region you configured\n# host_override=\"https://your-host.com\" # if you have a custom host, otherwise omit this\n# )\n\nvector_store = LanceDBVectorStore(\n uri=\"./lancedb\", mode=\"overwrite\", query_type=\"vector\"\n)\nstorage_context = StorageContext.from_defaults(vector_store=vector_store)\n\nindex = VectorStoreIndex.from_documents(documents, storage_context=storage_context)\nlance_filter = \"metadata.file_name = 'paul_graham_essay.txt' \"\nretriever = index.as_retriever(vector_store_kwargs={\"where\": lance_filter})\nresponse = retriever.retrieve(\"What did the author do growing up?\")\n"; -export const PyFrameworksPydanticBaseExample = "table = db.create_table(\"docs\", schema=LanceDocs, mode=\"overwrite\")\ntable.add(\n [\n {\"text\": \"hello world\", \"vector\": [1.0, 0.0]},\n {\"text\": \"goodbye world\", \"vector\": [0.0, 1.0]},\n ]\n)\nresults = table.search(\"hello world\").limit(1).to_pydantic(LanceDocs)\nprint(results[0].text)\n"; +export const PyFrameworksPydanticBaseExample = "table = db.create_table(\"docs\", schema=LanceDocs, mode=\"overwrite\")\ntable.add(\n [\n {\"text\": \"hello world\", \"vector\": [1.0, 0.0]},\n {\"text\": \"goodbye world\", \"vector\": [0.0, 1.0]},\n ]\n)\nresults = table.search([1.0, 0.0]).limit(1).to_pydantic(LanceDocs)\nprint(results[0].text)\n# hello world\n"; export const PyFrameworksPydanticBaseModel = "class LanceDocs(LanceModel):\n text: str\n vector: Vector(2)\n"; @@ -120,7 +120,7 @@ export const PyPlatformsDuckdbMeanPrice = "duckdb.query(\"SELECT mean(price) FRO export const PyPlatformsDuckdbQueryTable = "import duckdb\n\narrow_table = table.to_lance()\n\nduckdb.query(\"SELECT * FROM arrow_table\")\n"; -export const PyPlatformsPandasAsyncExample = "async def run_pandas_async_example() -> None:\n async_db = await lancedb.connect_async(\n str(Path(tempfile.mkdtemp()) / \"pandas-async\")\n )\n async_df = pd.DataFrame(\n [\n {\"id\": \"10\", \"text\": \"sage\", \"vector\": [0.6, 0.4, 0.8]},\n {\"id\": \"11\", \"text\": \"bard\", \"vector\": [0.2, 0.7, 0.3]},\n ]\n )\n async_table = await async_db.create_table(\n \"creatures_async\", data=async_df, mode=\"overwrite\"\n )\n async_results = await (\n async_table.search([0.6, 0.4, 0.8])\n .select([\"text\", \"_distance\"])\n .limit(1)\n .to_pandas()\n )\n print(async_results)\n\nasyncio.run(run_pandas_async_example())\n"; +export const PyPlatformsPandasAsyncExample = "async def run_pandas_async_example() -> pd.DataFrame:\n async_db = await lancedb.connect_async(\n str(Path(tempfile.mkdtemp()) / \"pandas-async\")\n )\n async_df = pd.DataFrame(\n [\n {\"id\": \"10\", \"text\": \"sage\", \"vector\": [0.6, 0.4, 0.8]},\n {\"id\": \"11\", \"text\": \"bard\", \"vector\": [0.2, 0.7, 0.3]},\n ]\n )\n async_table = await async_db.create_table(\n \"creatures_async\", data=async_df, mode=\"overwrite\"\n )\n # An async search returns its query builder from an await\n query = await async_table.search([0.6, 0.4, 0.8])\n async_results = await query.select([\"text\", \"_distance\"]).limit(1).to_pandas()\n print(async_results)\n return async_results\n\nasync_results = asyncio.run(run_pandas_async_example())\n"; export const PyPlatformsPandasCreateTable = "pandas_df = pd.DataFrame(\n [\n {\"id\": \"1\", \"text\": \"dragon\", \"vector\": [0.9, 0.1, 0.3]},\n {\"id\": \"2\", \"text\": \"griffin\", \"vector\": [0.4, 0.5, 0.2]},\n {\"id\": \"3\", \"text\": \"phoenix\", \"vector\": [0.7, 0.3, 0.6]},\n ]\n)\npandas_db = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"pandas-demo\"))\npandas_table = pandas_db.create_table(\"creatures\", data=pandas_df, mode=\"overwrite\")\n"; @@ -152,23 +152,23 @@ export const PyPlatformsVoxel51LoadDataset = "import fiftyone as fo\nimport fift export const PyPlatformsVoxel51SortBySimilarity = "# Step 4: Query your data\nquery = dataset.first().id # query by sample ID\nview = dataset.sort_by_similarity(\n query,\n brain_key=\"lancedb_index\",\n k=10, # limit to 10 most similar samples\n)\n"; -export const PyRerankingAnswerdotaiUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import AnswerdotaiRerankers\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = AnswerdotaiRerankers()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingAnswerdotaiUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import AnswerdotaiRerankers\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"answerdotai-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = AnswerdotaiRerankers()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; export const PyRerankingCohereUsage = "import os\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CohereReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = CohereReranker(api_key=os.environ[\"COHERE_API_KEY\"])\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingColbertUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import ColbertReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = ColbertReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingColbertUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import ColbertReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"colbert-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = ColbertReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingCrossEncoderUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CrossEncoderReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = CrossEncoderReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingCrossEncoderUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CrossEncoderReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"cross-encoder-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = CrossEncoderReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; export const PyRerankingJinaUsage = "import os\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import JinaReranker\n\nembedder = get_registry().get(\"jina\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = JinaReranker(api_key=os.environ[\"JINA_API_KEY\"])\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingLinearCombinationUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import LinearCombinationReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = LinearCombinationReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingLinearCombinationUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import LinearCombinationReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"linear-combination-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = LinearCombinationReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingMrrUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import MRRReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = MRRReranker(weight_vector=0.7, weight_fts=0.3)\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n\n# Run multivector search across multiple vector columns\nrs1 = tbl.search(\"hello\").limit(10).with_row_id(True).to_arrow()\nrs2 = tbl.search(\"greeting\").limit(10).with_row_id(True).to_arrow()\ncombined = MRRReranker().rerank_multivector([rs1, rs2])\n"; +export const PyRerankingMrrUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import MRRReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"mrr-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = MRRReranker(weight_vector=0.7, weight_fts=0.3)\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n\n# Run multivector search across multiple vector columns\nrs1 = tbl.search(\"hello\").limit(10).with_row_id(True).to_arrow()\nrs2 = tbl.search(\"greeting\").limit(10).with_row_id(True).to_arrow()\ncombined = MRRReranker().rerank_multivector([rs1, rs2])\n"; export const PyRerankingOpenaiUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import OpenaiReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = OpenaiReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingRrfUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import RRFReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = RRFReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingRrfUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import RRFReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"rrf-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = RRFReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; export const PyRerankingVoyageaiUsage = "import os\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import VoyageAIReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = VoyageAIReranker(model_name=\"rerank-2\")\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; diff --git a/docs/web/snippets/provider_examples.mdx b/docs/web/snippets/provider_examples.mdx new file mode 100644 index 000000000..13768e265 --- /dev/null +++ b/docs/web/snippets/provider_examples.mdx @@ -0,0 +1,18 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PyEmbeddingIbmUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import EmbeddingFunctionRegistry\nfrom lancedb.pydantic import LanceModel, Vector\n\nregistry = EmbeddingFunctionRegistry.get_instance()\nregistry.set_var(\"watsonx-api-key\", \"your-api-key\")\nwatsonx_embed = registry.get(\"watsonx\").create(\n name=\"ibm/slate-125m-english-rtrvr\",\n api_key=\"$var:watsonx-api-key\",\n project_id=\"your-project-id\",\n)\n\nclass TextModel(LanceModel):\n text: str = watsonx_embed.SourceField()\n vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"watsonx-demo\"))\ntbl = db.create_table(\"watsonx_test\", schema=TextModel, mode=\"overwrite\")\ntbl.add(data)\n\nrs = tbl.search(\"hello\").limit(1).to_pandas()\nprint(rs.head())\n"; + +export const PyEmbeddingJinaMultimodal = "import os\nimport tempfile\nfrom pathlib import Path\n\nimport lancedb\nimport pandas as pd\nimport requests\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\nos.environ[\"JINA_API_KEY\"] = \"your-api-key\"\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"jina-images\"))\nfunc = get_registry().get(\"jina\").create()\n\nclass Images(LanceModel):\n label: str\n image_uri: str = func.SourceField()\n image_bytes: bytes = func.SourceField()\n vector: Vector(func.ndims()) = func.VectorField()\n vec_from_bytes: Vector(func.ndims()) = func.VectorField()\n\ntable = db.create_table(\"images\", schema=Images)\nlabels = [\"cat\", \"cat\", \"dog\", \"dog\", \"horse\", \"horse\"]\nuris = [\n \"http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg\",\n \"http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg\",\n \"http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg\",\n \"http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg\",\n \"http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg\",\n \"http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg\",\n]\nimage_bytes = [requests.get(uri).content for uri in uris]\ntable.add(\n pd.DataFrame({\"label\": labels, \"image_uri\": uris, \"image_bytes\": image_bytes})\n)\n"; + +export const PyEmbeddingJinaText = "import os\nimport tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import EmbeddingFunctionRegistry\nfrom lancedb.pydantic import LanceModel, Vector\n\nos.environ[\"JINA_API_KEY\"] = \"your-api-key\"\n\njina_embed = (\n EmbeddingFunctionRegistry.get_instance()\n .get(\"jina\")\n .create(name=\"jina-embeddings-v2-base-en\")\n)\n\nclass TextModel(LanceModel):\n text: str = jina_embed.SourceField()\n vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()\n\ndata = [{\"text\": \"hello world\"}, {\"text\": \"goodbye world\"}]\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"jina-text\"))\ntbl = db.create_table(\"test\", schema=TextModel, mode=\"overwrite\")\n\ntbl.add(data)\n"; + +export const PyRerankingCohereUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CohereReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"cohere-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = CohereReranker(api_key=\"your-api-key\")\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + +export const PyRerankingJinaUsage = "import os\nimport tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import JinaReranker\n\nos.environ[\"JINA_API_KEY\"] = \"your-api-key\"\nembedder = get_registry().get(\"jina\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"jina-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = JinaReranker(api_key=\"your-api-key\")\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + +export const PyRerankingOpenaiUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import OpenaiReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"openai-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = OpenaiReranker(api_key=\"your-api-key\")\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + +export const PyRerankingVoyageaiUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import VoyageAIReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"voyageai-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = VoyageAIReranker(api_key=\"your-api-key\", model_name=\"rerank-2\")\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + +export const PyRerankingWatsonxUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import WatsonxReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"watsonx-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\n\n# Supply your API key and project ID (or use space_id instead of project_id).\nreranker = WatsonxReranker(\n api_key=\"your-api-key\",\n project_id=\"your-project-id\",\n)\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + diff --git a/docs/web/snippets/search_examples.mdx b/docs/web/snippets/search_examples.mdx new file mode 100644 index 000000000..a9aeb5e7f --- /dev/null +++ b/docs/web/snippets/search_examples.mdx @@ -0,0 +1,32 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PyFilterSearch = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n \"vector\": [1.0, 0.0],\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n \"vector\": [0.9, 0.1],\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n \"vector\": [0.0, 1.0],\n },\n ],\n )\n rows = (\n table.search([0.0, 1.0])\n .where(\"category = 'animals'\", prefilter=True)\n .select([\"id\", \"text\", \"category\", \"_distance\"])\n .limit(2)\n .to_list()\n )\n print(rows)\n"; + +export const PyHybridSearch = "from tempfile import TemporaryDirectory\n\nimport lancedb\nfrom lancedb.index import FTS\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n \"vector\": [1.0, 0.0],\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n \"vector\": [0.9, 0.1],\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n \"vector\": [0.0, 1.0],\n },\n ],\n )\n from lancedb.rerankers import RRFReranker\n\n table.create_index(\"text\", config=FTS())\n rows = (\n table.search(\n query_type=\"hybrid\", vector_column_name=\"vector\", fts_columns=\"text\"\n )\n .vector([1.0, 0.0])\n .text(\"puppy\")\n .rerank(RRFReranker())\n .limit(2)\n .to_list()\n )\n print(rows)\n"; + +export const PyQueryPlan = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n \"vector\": [1.0, 0.0],\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n \"vector\": [0.9, 0.1],\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n \"vector\": [0.0, 1.0],\n },\n ],\n )\n query = table.search([1.0, 0.0]).select([\"id\", \"text\", \"_distance\"]).limit(2)\n print(\"Plan:\", query.explain_plan())\n print(\"Measured execution:\", query.analyze_plan())\n rows = query.to_list()\n print(rows)\n"; + +export const PyTextSearch = "from tempfile import TemporaryDirectory\n\nimport lancedb\nfrom lancedb.index import FTS\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n },\n ],\n )\n table.create_index(\"text\", config=FTS())\n rows = (\n table.search(\"puppy\", query_type=\"fts\", fts_columns=\"text\")\n .select([\"id\", \"text\", \"_score\"])\n .limit(2)\n .to_list()\n )\n print(rows)\n"; + +export const PyVectorSearch = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n \"vector\": [1.0, 0.0],\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n \"vector\": [0.9, 0.1],\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n \"vector\": [0.0, 1.0],\n },\n ],\n )\n rows = (\n table.search([1.0, 0.0])\n .distance_type(\"l2\")\n .select([\"id\", \"text\", \"_distance\"])\n .limit(2)\n .to_list()\n )\n print(rows)\n"; + +export const TsFilterSearchBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n {\n id: 1,\n text: \"A puppy playing in the park\",\n category: \"animals\",\n vector: [1.0, 0.0],\n },\n {\n id: 2,\n text: \"A dog resting at home\",\n category: \"animals\",\n vector: [0.9, 0.1],\n },\n {\n id: 3,\n text: \"A guide to growing tomatoes\",\n category: \"gardening\",\n vector: [0.0, 1.0],\n },\n ]);\n const rows = await table\n .vectorSearch([0.0, 1.0])\n .where(\"category = 'animals'\")\n .select([\"id\", \"text\", \"category\", \"_distance\"])\n .limit(2)\n .toArray();\n console.log(rows);\n"; + +export const TsFilterSearchTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsHybridSearchBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n {\n id: 1,\n text: \"A puppy playing in the park\",\n category: \"animals\",\n vector: [1.0, 0.0],\n },\n {\n id: 2,\n text: \"A dog resting at home\",\n category: \"animals\",\n vector: [0.9, 0.1],\n },\n {\n id: 3,\n text: \"A guide to growing tomatoes\",\n category: \"gardening\",\n vector: [0.0, 1.0],\n },\n ]);\n await table.createIndex(\"text\", { config: lancedb.Index.fts() });\n const reranker = await lancedb.rerankers.RRFReranker.create();\n const rows = await table\n .vectorSearch([1.0, 0.0])\n .fullTextSearch(\"puppy\", { columns: [\"text\"] })\n .rerank(reranker)\n .limit(2)\n .toArray();\n console.log(rows);\n"; + +export const TsHybridSearchTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsQueryPlanBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n {\n id: 1,\n text: \"A puppy playing in the park\",\n category: \"animals\",\n vector: [1.0, 0.0],\n },\n {\n id: 2,\n text: \"A dog resting at home\",\n category: \"animals\",\n vector: [0.9, 0.1],\n },\n {\n id: 3,\n text: \"A guide to growing tomatoes\",\n category: \"gardening\",\n vector: [0.0, 1.0],\n },\n ]);\n const query = table\n .vectorSearch([1.0, 0.0])\n .select([\"id\", \"text\", \"_distance\"])\n .limit(2);\n console.log(\"Plan:\", await query.explainPlan());\n console.log(\"Measured execution:\", await query.analyzePlan());\n const rows = await query.toArray();\n console.log(rows);\n"; + +export const TsQueryPlanTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsTextSearchBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n { id: 1, text: \"A puppy playing in the park\", category: \"animals\" },\n { id: 2, text: \"A dog resting at home\", category: \"animals\" },\n { id: 3, text: \"A guide to growing tomatoes\", category: \"gardening\" },\n ]);\n await table.createIndex(\"text\", { config: lancedb.Index.fts() });\n const rows = await table\n .query()\n .fullTextSearch(\"puppy\", { columns: [\"text\"] })\n .select([\"id\", \"text\", \"_score\"])\n .limit(2)\n .toArray();\n console.log(rows);\n"; + +export const TsTextSearchTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsVectorSearchBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n {\n id: 1,\n text: \"A puppy playing in the park\",\n category: \"animals\",\n vector: [1.0, 0.0],\n },\n {\n id: 2,\n text: \"A dog resting at home\",\n category: \"animals\",\n vector: [0.9, 0.1],\n },\n {\n id: 3,\n text: \"A guide to growing tomatoes\",\n category: \"gardening\",\n vector: [0.0, 1.0],\n },\n ]);\n const rows = await table\n .vectorSearch([1.0, 0.0])\n .distanceType(\"l2\")\n .select([\"id\", \"text\", \"_distance\"])\n .limit(2)\n .toArray();\n console.log(rows);\n"; + +export const TsVectorSearchTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + diff --git a/docs/web/snippets/search_filter_search.mdx b/docs/web/snippets/search_filter_search.mdx new file mode 100644 index 000000000..efe325aae --- /dev/null +++ b/docs/web/snippets/search_filter_search.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::types::Float32Type;\nuse arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n true,\n ),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]\n .into_iter()\n .map(|v| Some(v.into_iter().map(Some).collect::>())),\n 2,\n ),\n ),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n let query = table\n .vector_search(&[0.0_f32, 1.0])?\n .only_if(\"category = 'animals'\")\n .select(Select::columns(&[\"id\", \"text\", \"category\", \"_distance\"]))\n .limit(2);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/snippets/search_hybrid_search.mdx b/docs/web/snippets/search_hybrid_search.mdx new file mode 100644 index 000000000..492e14f54 --- /dev/null +++ b/docs/web/snippets/search_hybrid_search.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::types::Float32Type;\nuse arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lance_index::scalar::FullTextSearchQuery;\nuse lancedb::index::{Index, scalar::FtsIndexBuilder};\nuse lancedb::query::{ExecutableQuery, QueryBase};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n true,\n ),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]\n .into_iter()\n .map(|v| Some(v.into_iter().map(Some).collect::>())),\n 2,\n ),\n ),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n table\n .create_index(&[\"text\"], Index::FTS(FtsIndexBuilder::default()))\n .execute()\n .await?;\n let query = table\n .vector_search(&[1.0_f32, 0.0])?\n .full_text_search(FullTextSearchQuery::new(\"puppy\".into()).with_column(\"text\".into())?)\n .rerank(Arc::new(lancedb::rerankers::rrf::RRFReranker::default()))\n .limit(2);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/snippets/search_query_plan.mdx b/docs/web/snippets/search_query_plan.mdx new file mode 100644 index 000000000..fbfe156c9 --- /dev/null +++ b/docs/web/snippets/search_query_plan.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::types::Float32Type;\nuse arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n true,\n ),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]\n .into_iter()\n .map(|v| Some(v.into_iter().map(Some).collect::>())),\n 2,\n ),\n ),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n let query = table\n .vector_search(&[1.0_f32, 0.0])?\n .select(Select::columns(&[\"id\", \"text\", \"_distance\"]))\n .limit(2);\n println!(\"Plan: {}\", query.explain_plan(false).await?);\n println!(\"Measured execution: {}\", query.analyze_plan().await?);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/snippets/search_text_search.mdx b/docs/web/snippets/search_text_search.mdx new file mode 100644 index 000000000..86745b652 --- /dev/null +++ b/docs/web/snippets/search_text_search.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lance_index::scalar::FullTextSearchQuery;\nuse lancedb::index::{Index, scalar::FtsIndexBuilder};\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n table\n .create_index(&[\"text\"], Index::FTS(FtsIndexBuilder::default()))\n .execute()\n .await?;\n let query = table\n .query()\n .full_text_search(FullTextSearchQuery::new(\"puppy\".into()).with_column(\"text\".into())?)\n .select(Select::columns(&[\"id\", \"text\", \"_score\"]))\n .limit(2);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/snippets/search_vector_search.mdx b/docs/web/snippets/search_vector_search.mdx new file mode 100644 index 000000000..47966f5d4 --- /dev/null +++ b/docs/web/snippets/search_vector_search.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::types::Float32Type;\nuse arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n true,\n ),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]\n .into_iter()\n .map(|v| Some(v.into_iter().map(Some).collect::>())),\n 2,\n ),\n ),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n let query = table\n .vector_search(&[1.0_f32, 0.0])?\n .distance_type(lancedb::DistanceType::L2)\n .select(Select::columns(&[\"id\", \"text\", \"_distance\"]))\n .limit(2);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/static/code/ingestion-feedback-rust.zip b/docs/web/static/code/ingestion-feedback-rust.zip new file mode 100644 index 0000000000000000000000000000000000000000..a9643851ce999d6fb760335c6d92ee8acc607f43 GIT binary patch literal 38831 zcmV(zK<2+tO9KQH000080000?U8eYY#yyV!0Q_45015yA07GGNXKyZSZ)0ohy;*M~ zNzyI&UcVwy_wly!^L-g$fC1)d-skP&LU4C8&t#V*s!6J3z8BUjG&~fV9kQ* zulq}JNoBaZE$5iq^54GvpWj#8FMajEU+`yn*{vVTZq>@e!;dfj+c$pVP1!MhsXxB_ z?`89C{q>js_T~TB$ycA&_~U8EPlS*Adh_M6JiU~MFHEc5>gn6pUw-G!cC~)`^1ps@ zzx?t){>O7^zm;$NKmX@1Pvw#E?yrStV;)vd{Oiege^p!Ssy8XIZ->vXSfByPQ`_ApR z?aQP53SB)}6|GU#>ZB=9%-(Sbwo+^{x{_?F$r#mH`D|MXlzUaVyPT@4zy3n}EK7ZA zE8gNa{*Pb2{Q7P6_^|r*U%&j%x#d0Z#+PjR~yHAE0dIMITrFJcYJC}Eje~9(Ns&dC?Xwo40&2x`9CjZ zx4yc);`o?IlNK^e{6UBc>lavKl5gXEf}pxfAtG5DCx|2zD$*HAkZ=>U9K7? zRCU80!WwaSF3a&y;F5KSVw59`gVEFa_OPng&g^MoU18yMx#~iz)dpx%iqI;fppXHWS&wn-|qeP?j_oVy}uz zE)-LG@8r4rSaN3^SW|4c8P=|6d@}~6)a>*B_`d$*^6}Nuz7K9q8R{vx!PF(#Fp6A? z_mO){o_q^90S_~PlTxA_SVK|K*b4L4=AM7k?zVrF=g|t{;>aW99(!swt5Acf(Yr?8 zngHESa7G%&TT|;WcsgT&T)e47*=6gBbw18M|EB!da39IP{hF8Fe+UXEZNyAZl?gj{Q@z)<)D@^5^>z8AO{0F-K|rp~Qid;Uu|85h^z*dRXISn3M4rZ?(%aGS9V z9j%96H(!Bkxmf_JmQ=}DS8GuxND_2q{`p7Z?-O=^X+LNo;a0Kt*x;C%if!6)gWntA}6owOYgind83u>?)2sM z_mfvtcID*d>bae~`uBGIbn#Veew?&jwM zd9OwrG7XF(Z~_wZx_Nmkra?Wbf-lc-2EAtjR@Yn;Ih?1e(Wa~C^UisUw)|x0fKk^^ z>*2?zm&ZF``PFXw{*KANck>-Htoe!TWXmcX&Z@f25?KJaUgJdT}bN!@ZbPH)#S$4 zANU8)w@X%DSKqK1pWKMSzTA8pT=Ru{^>AzO3}6;UgbWNCsMV*Md>7i^6+m1!;7bj* zH7gZZH?XVN7u48uJ9yRd?bfT|hg!CvLKlL&e(XQ)cBKMPo}C4xQ=(MDy7}@nu%AC( z-aQqV`I{dKXUD1a$R5-ba1`2}fmO8$;1Qs)%>lof0@Ug~MdLiMw)LPF0OdK-j8T|8 zl#v&ku>=R-hCko-4(lA0{pxlNM|^u&&{8rIuCT>auwk8YC>?CXs$8rxsW+&21w+=p z7n2L1g2LA-&{<-h*5w(YJ=vi%9sfCNC&1w5+hZ)uTIwoU=aP*eJ}F^`s0P8=QQK6F zv5L4hs{y8(vcQ2LYk_SGZb==#rq%tS?Dl_uuZ<6Xx%+-}D1U8t^xkF;xa#)1seCI& zgWm@VF@tbvi*4$_r2(64B5(|WfPu7#4#yC|&q8i7fj-(W9ccPF%=vMPd7{j(Qah~e zu{?hAgdgHQoFSbp_Q= zos5s!q)|wYL3FZ%l~5{6zyPey)Y==nu5;IP&0s925m&eAAE8^dRF4ga zR)b3gWt&9MLpC8w1kS6T6r?&v*T8wf;6v9vS-CXO+5@xRW}6NIe06&cU%&Yd^#Jr< zJ9V33HFKc?(ULH13s8R@xJKP;jRf`YLE8+hDGr;laYZvtqsY|`w(#3l6jL1(^O|y9 z--*3A-Fzn!C|vN(I~`5IY9oM3H$6%CGqNJv2~eA<7KjPpba$<3P;;NMGx*Pkh|J%R znveVB1EapZBgsUu5g0|4tE=vSUf7k^zyc-MDEk9QS8JeTtRdFkI8T6;PqIEA`cL*blSEA|dZR@A@vf1#{?(RP?{IXX@{#{-_;DP1pTjU3i^=-fZ z{bl#E;q99reO^7?J#W_k#`fZ`?D*a70;|R^ke#}?X0l612Xo@)JFs6*z=9?;SpQZ* z3Y~5Z&XP#(&q^=Jd+eD7DXi5KrNF%u8)x?Hbp==raoZkQ+R598-TBCNeff2_{%y5> zZ{r@;qia=*9a}n_4LJg)Q%V}T@4CXGfz|5RGW0AL=qYLMb5Wcv z=xj@kfudcTxdg*f+udVno3$C1|DTujK`gMkFGIG3j_>WkV8t#j%hJ-f1zA~N$dJmR zvZRmr0<*&$SqPVpj=)XHHd|EJDYOcbkBw@PnLO2LuCT-g>@yd}*SdD;;F61_np9vU ztn(e}kl@f1ynbXdRI|mv*_%W?u}O(xg{Ch-*F#pu-0CDWG)|vc9QENT zbLN(6=VEd&LARny*U@z&H$NJvk4p~0!CZp%;b1FHT>%op9!!V1KLk*A>&I2Q`=k6e z3XzKTE`8Ogz48@eXi4jkqTy(wE`c0tAQG^kmV8txhL#A_S7k_XE)q|rR7#t@Dtg(i zTmLqC{N??BQz@t*ca5zAY%(NF1glr2w3BFO3V8a?lG9a6O5P?{z-w@LSSqKP#|KUP z4ad~R3263fT-qDMwOWnmidh$<#WV&}tfJBt5hv?mQZ*i$w1n&sT8IG z;%frk;Dq0@o({&bt2lhy~@r+Z_QbK7Bg`SO36+um9M>^F00b zMU04^7vDp;c4T(U;(jNZYK;e37bYrMdb$)1Y^~TfnB1;(&Jc{?asHJ7a475U0TC4} z72cma`qej}m(^n#TTZw5HM0*2NK{S}U;;oZ)Mz1?R#>Bp+>Gu5&SLQ{c<5qx)m5uQ zjLD>IrxSjTns+$u$GQ87&xSIOe08MKc5#bT84^#<>tF#ZvZoy zlB%z^gF{-Ybv6+k-L&FDkKmV+i|a2>yFF0f6k5IPwQg6Q_tx{rUsqQB06#xYzJpXQ zp3?cmr*oLqu0P!LPl&H_lDBM?lioIDVM%1sC^d}wiHHqyJ# zHu~4`g-GY%Putk74#~vjX?MR{J@P&O=h}{du}VBAit*0dgB~RztQcXEPj3 z$7Z;kLumyDs^P#j%#l$f-ZF6no+qztnMtJ?a{mEPkMXCum2Q0+t#JAD{n{b9MT-&x zL^Yc;AUG^$hl@QU8Z0Q6to+>BgHzCGrL-1TfZ}>ok*#vw zdC;9C_juSCWrOD{GX3~+QDkA3&(qhvQuj$oY}uw7gmfhDjV?p3D4Rf9wU$$I5-=?h zmdGZ{+Jc#Qu{pB~Sa0Zp^Dj@Dbtl!0Pfi2U1j$r)Nx@!%DWxE}Cmu4_d$h`PC0!UM zR4E1y3b6+_m)^S1iy8$imX9}TjxI%OYlP>DIk9%uL5!L=uEWsxZc249EAaB#D-AN5 zW5Z%&ac}{1Ht`}8OWh>375F584S>FDljSN%H?8oqaguMJdhRwBQ*{uz@;OP~26u|p zMy`JD*~h5ae09r5IeoH9Ww0y}b*-Ve-8q*bwzbrTV=zt99SHch8whCCKEq;=7U&D0W#;U2`(E&0qb^bV z()b_%$O86(qU9D@dw5BX%oYFu78yoN6$3J3jIpVhqe(79> zJGi0NQ6DiA4Y>zjhDBY8@E!7u#^Euyv@F8ifWUy6uwwYlBC#MiA-Q~G;Cbi4Wh01` z00bX{lidK|1fY`p^*T-KR^_SO2~E5EYki21{(F13{jr_8{2nqe_8h)!C4}}wU_YM@ ziM}OS-<<=s_TseK`tr^m!?Ms(4=zo&NGo$Kxxxv-fVW;Gc+Wa# zPOyfsaWLKt7v)r+;X>mhyq-+f9!4c`4y9ogAhdnBN2RltU&vegm%Y|XV$ z0GN>5od))v8A2P$Bxe%)=}3$``F(v^JrG|hUq1WRc%4h5qo|$Knk!v<2o4|-TE}1* zApLMuT11fKCcMFBS0g|}s~Aj&z4i*qJROQV6psGuxaR9n@c51K-Ra^;Fp-m@Kwt<4 z@c@5vIm6OukL3cw6Zk7geXa(eD)}yVH{SNzTc!-CKc{{tl?|s$h&Q7ro5SGzZv^2YrToY?N) z-G=rm~?P@EGPi3-3c4u^C6Iunmh}vVwww;MD++p-fTa44`x_ zc&BNsx+|L$Y0ALi{^a5(p3_nicU;hTX061YIji=UDd9nZh?VSxdy&f9JnHE>uhNDja5o8USELnIKZ?u#oc>w&_Kqh+XF=i^oWVj@g zn@+Z~rB>#0BgE$1_4UgWmEC2ks$TxCR-$E%G7PS@9D6XeI7uM1jFXkn3rKz{Qq3U+ z#z0YkhN`8QD`_A43==+I#Vhx_-gsl`Z;dYs;?l<$Ex2vX(y7T)FscXFjTB>u`4T)I zG_+j7G<$Fw$yM`FLgo?nIHkEjZF_#ce@*WhyO5vAX{e9^0*c9Jg|FfLX&roLd7B2uJPyd;;wnVycbaZ68o+F&Wx#V42VCz{ zk!y0_obffv<#N{EVW}l}$fsoJ`I+S9ccpW)+VSSnfG&U2DEBrlz={IfXVWvNS?Iny z{A48=LX!St;2|r{1I;#z|3qDgoG_GW|8QLK-SfkG z|3!IY2>MGn$sP)zM&Y;h`lLVGGdA&lyZY;J-HYF|I~I{{%8x?<;#;-KTj*|7qbMGG z$B)kfyUz*@B*^#h8?CMZ@bh)t`uIpxtQedZOo^A|4R*15ARU}mvt-1V*8<}TS8o~S zMQKjb8U-FRbD5UT<55kV!0fV*KzFLHXUOYVytS68lHZp?sy5^hf!-8wIk<+T+eby5 z8<(DR}k_R&I5?q;!4!tgdpV#gCK@mT_1N*-`PB~nhwZOH&0K0<4 zwUV24L1myFWfW_t3=_dhS-2enDvDy8!NIjFaH8IsxmfP!mwU-a2dBDt=(gkdBonb` z4J59!fFZdTR;oe9qlaD24Nee%)&tK<;JU(5Npnka#7ixA``hYSDvmA~+R|kr)J-OK zPJwte$#Q|)5y%QFW~__egO*9dtzMzpWNMc}$e2o&R8y~~Gl3Qo@%B$Q4WHZ=~vc~!VX^`mLT(QD2v33k}m%dtpM)u0E zyQU4jH?5^;@StL;6ugj?&a28>0_=6JOTbs+3@Lj9ou%32h0TxWois^(e7vNT%Tp?l zV3UpVo&fx)tJYZ3uoTuqE#Z|kzlHUU67QlKTCAF3ZU^V0rd{edBNCY62fp-jrG#nR zVn*Mfp{O_D&J>0wGra|(4&Ei)DoWadEg2tbOA>hpW@wy~iG>w3Q<@>pH0nBQ5A)hd zVzNEN{L-hQml_~s>BCloKM9iSnv{#+EZImYg`+A`>?OQebEv6A@F_OZKDTiIWUe`; zX%O*Vkd{hm-`AV%IHN$vKZpJ+L&acJ zSprtAI-bc%-#`8_m0|zf|2p|#h)AA%GJB4b+i-&x0lqNQu$ZCiCr*@rb~U84*P>MB z7U*g3%A3i*(>aw=fE zWYhvhAAu!Pk*jbOoX*fE8?BLxr3^@sq5@6^e4OEH`wqg>Eon|#HvH?C(=MAQ*=KJX zaM$qT!}{CI+d~(PeaqbQkM}HN^4&Y|D7l~;e=0xjw|4%MySJ51e}3$)IQ{8saPs71 z{*&9waLsn{!SiP7y_v;zutWJZ+u0B9%C~Qjm}5M2@+(ig-7(F7{?>?d^5cEB%yHhf3v-Iw0qZ+EZlaNZ2UugA>XFA{X~;~4jU9`N1m;khNKE#rDdLG72#{&`+r zR#%!UPH5K6O^l6=J8@8QZ3ZULd(aa5=@K8Y)|eqSW-7=4MagyU9N^5@SL(ga+$^1# zn2|*}`kb4ZgimSh$He?S*;`J~-f6jX#mwGZR&)fGHCBNPc?uE)wHp5^u1RuT?_dJ> zDe5Fu3eD=wHY!Q;=-q)9zw3@JWaV8)GbuLzs9-E+|6V2RXGQ4)0&^MOyNR5@P!tG2 zajs;~S|trkLblYJ6u3ZiIRL-J(y9w!)zXyRO7}eg!0y{jH=^{di4wk zhC`LbbU)Uqojw*mUD4j{5ptjVfw%iiu6GCX_0bc0wRkr_sgxx-y)c;^qT=Q{1?6pU zwt%&{+`&eiG$YWBbxqBOn5!`@MAC+M8k(O9@NwflZq-h-_vXvz+`&#&s(Xw+xzO$8H$-Sxk zuF##&L75f7{|BP4CFN;Sj4s6IXUx*&WP)2riC4Cp%*rRzbZX(hpE_ zIV0Fva>+KpDmZW0j23!O@Mq$yeF=_O8a&vU7Q6qyG+aupA0`qJ6JAPZj?`# zzHa{jegktaO;Oh&$mkN63fY#EbaSaz`V&e)K{OqhrfLGhksi6d#NbMbJlDr#@|zy|XjE!0#DrdZDs+mho2I2~rKx5xL5h4R(e+5>fltNER`Wq zb?r^EfwHTG6iwfE%p0oD;Am0EiIaOTV*3`DefOLD0PB19Y5b3Vn2nxr&C$e<6YBXf zLHNf)0RScqT(VoL0%$d7!&Nf9gA6_7Bw4dQ+md?3OD&_8U<8ON2v>!LdPg3n5B<|S z`u?W91@Ldr4@@ese$v`elJ9`-oYXAHum`ESw%lX2x|HCp${va&X0WP)EpA@FPfEvT z@MN^tp%3Egk9MqN=acGwPUV)lQP~%=?qfU${C1T&KGNOZcwU*ZFdO3kUiO=Etd-sJ zw?}^3mDhEAIIcf+-*a<`QVP;ybYve?P8B-FO3M6ol(|%a$F1^-eEeJhU!A;;x=%}NWK)bu~sOc@;$d!b>r;7=d-lkj#1FHPbD_vNV2^% zTkCrFn%{*>OWm-2?5cFkVt!`d4>=3Bzp`?i61Pk~Ht?6^NhT`G*a>M@91KMhwDhed zY9Kbr$UIAcuWUSll)eYLewLj4B-dEd`TZTOIe*C8M?++4bIsQ4Ty@DhYa5L%T3gk6 z!BMj9(HNyj?jv#DS(nh8cTq8cpUx!Py>oOw?XnK1bn|U7Y(wNA)f8Tn;wo;anhqh? zM)_QAs+x3{2J0a!nIC$<=*zj=Gl!hR+o@kuv)slFe-$3E!rXXnVx{_8hC zCQx1usYs=rtJ1PS@?)tR51Jdgl=D-PJbb4ist&phwCarlfpMq#@F;l3)5i^{%Gi;LYfhk*Qrz5or zL3(<3ZlPr6+%wgg2`zt50A?Za&-vJi^ZV>^{9`Oqx#~O@xo9X2xG3RJiYZa@Mpy-( zb<4rz7E4f8nxO`qS8$nIcQ$-Hx978#^nG&hbBgncgZKyCSOhJwGEPI~;tV!D$;}^i zPSV}ZT4}poESP~($`ufMU7b@-C&}TgdAfA@Q%bQ6xBNMM`RKLZ^XBF0n1KHGbZq$T zx4%7MYRUkud6`58>*afOtvOJ=QoHZ5g7J}~*0UtwW+jbBdj}y5)LQ(w*L!cO`4ygc z>l~%eL>m^ok@>LfonGKAm1Z~>RvbC-pP0}(p$s>d;?iW^O$v&w_cG_tdzEyP5$7Q_ zmbR6#NXmjt;EX`xGC#03nE(Z*;@=C5z0akuBpv*dhUm&!zkgcWiAlt_u{ll~N#ZUp zv2jM`Xqup9d|Ea*r%)rq1i|Mk?K~OZ7*-0=>ls4ySp)KO=^!(qrVs4ye;)pU zi@c%;PMWC0<5pYKaD~@0*tL{!CCd@-WpEela%A}g6#3h*L zWSP;F3A-edAGv?#`tofx9t@F|ho{`VHypZ8iJkPX0WhlI^`MKTX(u;Zv4Sxcnd{k< z*UU=EXl+wvh~ea(zjM3C8$1?Y4!u~q9TUTL4j1Q;8@YG@Ht2FooSN~fM1yr#nMz=z z?HaEkTrL&4bbxC6G<|{dy#9kX{c*3RAL$&ipU*Ve%X%owUA#HDiqxyhpd2;!5`c7D zA0`-E>B%K#%Db4ImoEF>N=w7u+2$U4c6sL7>Uh5@^cM0xf+qrrN%N-!g{B_7G|!Bx+NkH|GvF}S zB#3`=*)wkOIRw!wtKc+b5+VUmMV|sukl-8WDi+8x?;bveTTNJ$WoT;5%~umcnCam9 zUW3|`72aQrv01(+n;zVx7!(-MQl;^(s8mie(#iv85GkzDIqC5U*bGpOZzQJxK21r} zYdk%r`Gx-HrR>&YBTG_fjxXj7cbZUaE}Cd+)Pt^tw(mXkEO z(hfMWqxIEG!75cM z0mlrUBU5e!9xx!#bqMeoAJ4n*Pon{1(*Aqt27ylD|FHQUE7jZ_RwdUM(kfAM7Gkps zk}CbRIxs!-6Q-C6CaMeI_p`_KyG%Bb-U!kd@;urDom;x;6+_klzFX_wSgpHNMLM(_ z?V?!Fl9Puefg2iPvP`NbqFAhpJ%G$O?O~c~p7AT^XZyLGke>E; z{b5~p<^sDeE!$njSq9fU@T##AJ%!EbcuSK89g)d-(O4<|g+hbG7HgW|J>5fvS)K!L zvINOiA9c69#jM<{vdmPF6yRujRz5cfIweUBhH)~zt6*B@?348A2SaHvvDSOAt~ys= zHXb=U1~*FsIACEFgVPFII>cXfGdWe|ND{*$pr%s2D90d*El08)s~`rGC{l$nH;q)5 zvHcF)da;7#of@cYW^6vG9$6(7B%!$CryP<6?hMX44$3G&H%503wTA)mbR`3$r|rw3 zyG8ly$Ng5y84j;cyv4p<+MFt_{$#4F-Im9J55GJ>1>knTCvbt#9bIZjx|l>c6IPgI zFgCeTd{qWePVx>aw23qsI2C+r?hGHwZXfqJKSbld=C5XHd!VumXU5}Zj@C&kUdH)> z3c)GBR6wh`&QWv5<0m}0QCUOV0=wv$Y5Lk82Y^+4+e(mL9fI`oJnjClo+}5qpL;eQ z(zo~^w+=|nM2W$Yv`pW2-m#5c$RTz@C94QxY^4haoMGnLVRmA#T&3o7rtkQ1K#45) zarLzN>wf*`>e^gTX+t_jpG(WBl_zho=CC5hsQ^3DYAzjA*EPTwcmO4Xf{iP9mn5mM zC@?DE0XhG<;mv*hK*L4;t!*V`@LbSb@OAB*aa?oG%>*rTykK)c%jL?-1$b?T_ic-i zQWs)v$rc&9_nGH6%V);K#zt}glQ9X15lmJ&Edyk#OovP|eN6*K%MCS`z|yL@M0_Q5 zAAmDwpMUds|FU@)DDCL7N8ey3E3jg##(CItFOSxfH!-H@8U1TMAf@ zBFFT0-ZhU>>G0ha#qwEHoFipw7+2b`VcOZ&o- zfo7#awC@BdZmeSp(hfa%8!WX>CflG~QnkpuoDR%5r*|*Y6SfwgvKf4BDpp}64My5G z>CdjUYA%KrZOYZbg4YsFkq#_LW>1F_zm zy?JOqaeTR5J^gm}#;$D7-;f@-*JR=Rfo{I7hB{nd?J?o>qlW1PPL&%D71^pL;g^ya z)L+COWXKhOSWd;j_iI?}>)Sfs!o&8wxA@Bo+#9T1L4XFnq#7Kg^zY?Fpn=c@7=fh7 zP|HriYev|$BHcAr4n2ug48hKFr?Ir$d?A3LNyf|O*uAubcZF*M%mGgVj5QXzp^Y|j zIR&kkbx61dw3h-Lg0DV0WiGt&voAq#9!c=hh7RMP3GnBBl3eE)Wlcb|2f(k6e3?gMr5WGjR z2E~-R=w+93*9Omgll!zY{k{1P$gI6yY6Oo|@K4fE3Fx(Fm~*UO+oZukJn)o(Q!_&s z%cQg9Zgow-UhkaaAi+0Zg!EF0B$8zeXRnp(t*Y#yq8VOG1N;JTlu^*Z zG@T;%DW^N~ug80wk9gdZP7;%;x$@>)NBZqUsq+djOfQu&cTLaqziEs?_kUA;SHH2Y+affrFT4YL7nHlPL=)n5cl>ef2syyCXJmU2x;+-vd z9q2Toz02eC1cxO>WV*wHnIT)qz$=5%MpsB?!K9(np0*@Yn!#3p0dod=?;&%Z#^-M% zsJ{GqSit@8k6v5$ZgNPuS$Y~hWNbvMc zNzfjK1y*K86y9Xu-`nwAXBqFgT^)aTysLOBob%^*!!CbOkuw2@HXEf2uB1@_y|{#9 zhh>t~OZWwLfIJ>VK#)MOPRjGy5Mh_5Ge@TsVspoR^~HZZPVJB#dA@QDH@5}L-y17p zFJ+g}G0Z@$GQ>0*sgNcaaSDYXZY4^$fuIOR8z9{TC4FAw|O*yoBJ>H2Z7 z$uP|PY>N&*+RsSP>hWPUmY(HrdN`O2cAvpzEYZ*=zZFwS^^`4eBIGAPqbMsdp`0Dq zrl)|t?nZ#s9HtoB;$FDnrO&-df4!)BV4W2Ff-2dhN-a6!qOdF{!&yL#lc0ScCBLAS z?qaPp49}i?_V45SHMjf-&u*KlA-68G$aIA2hJpZ;P{iuLo4Yddf#x*CIn<+VFWIdN2SzX`F|5rEX$3=iQY2AjYDjY~>b376Mc zIHt}(3xEuu4QUo^&#u5NUp(M}W1O+9BWPdl%|LHBGEld-rspcT#dI zP_l#t_ql6{*%d5b{*aAvi_hs2&Y^4`ci=NNppFb6i3 z8Ln`+miV&2O8LC3g~fF)IZZNUu(He`G$26MXzJi|No*6>EnUHi>rU-=mN~Jco>i}kc|Yw6>t$T1j1OF5L_rb7!Sm~@OqU!mvQaegjEPX1mQPfyl+KDIExga@<4 z5rkUf2M?8$i|v9W69ahK{fc}=|Xw&zX$8tGB=N1`q(7~!< zHLw7-jg^lj!&ieRWiH;-mOUJj<}8(o-9U3a+*#D2g8(`+msrXE*^rEIY2?a9&yz3<=Lc?%m zz;Xkl0IR6rg{4bj>_86W;q72orb|BG_HhuY*Pq^~=3#knKi)l+-RgI~YrNT=z5o2% z>P|wM+ez+v@_TktReZPQ&F`yr&Ca|%$i*D`ym)KPn3SiWU~B-!VyH>(?Vyt?vv8dv zHC85O*E95z+#vlD*%D{$r*AP&duM-6%15`G^YZ?BYOCib-kk@}Ch~b^bYZ)0zwypo zoZI;^eI)E4&-1YUhW|c4yo{A%c{mi{I=RCAJ8Gp_%fO%!fQ@1x>j+3$(v5?ZAUU{= zs#3Gc04jwBY{Ap{<3pYM{2Eu~sSGpQ?*1B=_=#wS#Y;Z0s@P>}qN+duQAKYw*V225 zDMRC10X*$F_}Gff0IDjR>XRm?WQ5tY8GfS8dPVo!AKM*2-a{gAaJ$uU02$=)uCHI7 zXlP>o8kap^`rY@k`t9W+c6+$d8O2}Uic79H5SaWY#(TVcoKPndi&2v0SHP$l)DnO{ zkS9Yb4Rg1oO-(jF1B%(MLj{KmE_)MZau|kNuenvWm;1JUc;Jb#0p}FW3;yXE=h`74%Q7xtFZRinBjgr0AN}I zPRo*y6oa;{lXe-$pIxJtTP_CpeR7w^NByr~PS^C-M&RT<>AUj!9#r%96&>qXF7|l0 zdV&gXKD(EfZ7OcDq1hOnYcLgP9nf5|nZ4E-KwY_}RhQJw+9PLQgHvRDl~N2Zmm|$I zGu^RlBzEG&j>CA4miv5L{@CCvufwlDvq{52++ONBNxo-`a;Z{ z?NaX;n>HQhCMwrOc{J%lJwlaQcgbXDIk{qLAv<@KFxs43ucERvW z3aY1B!6k!YJ6KZfY%-;5J>1q5*Mr1DGD#m06DnL+t4;DS^<4PiY$tBL25ob54W_K6 zDs6fRycKTKfm`+5urf&-=*@J+3|UDObq9v!UYwT-*^l^uDKU8S1>0df!#x(u{r;`% z+__trM<<^Xk;Cwxe_&&MQ=p^ZG?JAbbhZUR#HAP#^lq3T>z#9nRf?)midjXHcs(n^ zH)m@8t}EHJ9-S@wP2b+bgP+^Ze@5XLcC>JH{JBjw@7Jo>u87(|7e~TlX8`x%Ry^UF>WB{S7 z3`}M-L(A}1b?|O7BnEOTlUad;_xba6C_tWy(!A`}t?%P0XHA49hvn4CeP;ECM5CoA z!*hO;)ngWvDfXlyFVWfQU7oC zfSoo{`+qGZc_sCQf)?&KTrQh_4rcZPm6#W8#?NNqmM0mkw5Rr`^KIX0a4k7=n{iuf zC8=L_rp&DTj9FCNulO7T#0hMbRi;y}uYN2*GEY;!R>dfSWsoMR^ zq~W=>4>tlM&A-7c7U{gBYHmR6KC+dlg^_*)v6WV307R!&oMRV*cNvaQxT@Bf99M1U zvME-Zm9(LmI=iLsejM0n1QFGgasX|uc&_BdNEa(L-!T?zz+^L+X$s)r*7yqAT>?oK z#&i$alk$)EjP!}xUfh=~-hUuCB@i%Gm}z#_Cr638PL+LcGAI^K+ck!_>RwlGt+wK` zq@f7@#&n4DB<<+z0_rpEq+Z*hkIl#OAlZRAbEMv+b*3&34yZBLNVTP6Z8sE#Haj3! zxo2Z~O#m|R)4D6IXR?A19TPvlzU6%}v@4qylTkMILX@S8jR(%RfXFG8Qirq(Lq+-p zd(Z-a7rY6M1HzN06ZPL;-+BzNCC_u|qbnZjSCdbGQV>*zy-7Vt;7IofJk>eso?{2x z0_c{0g*j{RgD8~8%w?VZd&RX(Oqdo(IrzT2bZE?@rn^4X<~9QI&ZJJAr>^nDBYM8Ny?C`^227_ujG)&LH;Z?EC3s z-F#y+1h1#+tW1~paB$XTz%m^oWKlHC{S%&+Hg z^Bz4xGRWY+do;qeO@rcb5)~$OPb-=(eEY}KLlJY(%YLpeU#sEddk8!OlPpX76pFU z*FwHe7XCfe!Q6gLi?{O(uGpw@i`~POLqs)u8EsO6Box}(La8N$Y`sh;D_vy=tlfJh zf(Q_jEvBJN9Z`V&_Rb0L*#o(BL8iNj&te37OPGYbI{+*?UMivS3Z(GZdv_UG}II~xZWJ2?}t9D17kkvU3nG&7~I(q0XW=ijH;=I-^R56yL%^j+7}5g zf4odaTVxmK_xQK+R5q(}Vec2edsaPD4&HdDUoUyfF}fAcJ*bpO_CNN)dfFT*FV*C#I?B-nBKmqnSj&!wIIh7&2+ zzp;D0tUbJ5|He6#(BNZ=lRJ_T9>k2zDpJ;s>uw;$;`cyc~?u<}qwUCM9L za#341pMMOnIqTandH)HZ6RVv}3^1nMSAoPugalssxv9UE6=+(Zb_{^p~Mt8*q5A9F7i7Kc)Rp7sobv9E^`CEGVDF-@iscFJ07^i$zq++>f@E7s&b2CZ z5OEMx$2xZ9Vrt52HkDe3xB3{6$amHXYl!5=>En#WQ+IqURea{LL9 zek$s3j{d`t`dAh78i)Nb7(J1{_sX%(|G8U>k#w<4+B*)_NZ+&MbI&TJz@1X7rCFC+ zE5_q%HF+-sYE%ZGaGfdY>D-_9#AqZnCvtKk2Ga!TcpoqZ^G|e7t7GxN@bb^~_WSC6 zPjWD0H*buse4t`DI@2ZZMiQx@w<$^|M6#?n=ZZ~@K2%m-aEIjEv%z*V5cMsh+ap?d z>n#8R4ITz46%JaeB5aVpA3c|xvs59}9Bi#NO94|&Iz=0sx3;(JrCse+cl!6@9#_2W z6sLH6X}?|dy+g0{3n4zp?zrCalYVy;Dj6HAgK2vB%T1A(c?&htD4g}nK#;;AYi-Jc zefjmzKgB%`b;85zzqY$~>FeN>Z!K&xeWwa&2O{&VVkK6Jt;t?h?+qX=`v43_#${V< zgsf@fqq8C4)l6%lpA@!FNY_U~XP*On7a3XaIl753(J>Bml9F~Bh`vB`ACAd<+ASVV z`qrTjP-!X04P7w+6~oi1K*|@ueQLgYmlgg`TPf-?(I*eHRjk$0TUCdd>fOH+xcw}! zIxSTHkW7u@udCfLS^2}k*U!SXtC7>U4_BL`Nxc#`oiv}Xi4z||d#B-ss~GSita=6v zKN@Jg8d&{o5Or0zWwkx7^i@mgO^BA$*mr@~x7WhXlL)Kiaoj=VcI5!y#x(BjrxQYV z*4|@=&7Fo&KM*<{^_0Jb@d&`vM8GuUYicPv-JP}+;BHhgsdu9+WhaS>tyYrRh-6E* zTyiyYc??Tm^!!6EX%5c6Hqtz%`hM6ujV>MDzdtCu@Asu1(9cK13@0_oACSBE;{pd2 zzd6Egsw6VS)D}VLf-$*NfF3Z2K$(qlApTOMtda{!GD(zfU0Ksx)G5!jPW_mGoszhA zZk^vb*q@HGzsbU_H&%uSQ1sd4R=7DG0SbI-3=~jxQ9)NxZXH_Hij*|t*GBrMYGb4Z ze7ckLiB3GZ%_Y={4}o~^WZOi6j8h6eww$#h61 zBWaCotc^5#GBN_(7jg-D*ma&(!NVHf!6C!T4L_e;1mFL0^uHgl{s(7zbM@-B7#)mF zzBrqWjRC}^L_5r)EYkhNnp&lfzpDML7-yA(>6c#6c8;}t*80z3fh!@7SB*K9+Yd6r zy>i?6y}t%%$caa!!b@s#hGRMTiF>unSHU7!nuDSTncOFORQdo!b=KmR5dDZ zl_u3fNslX6AU+&kdwE?B_2cnx!0yOVCxU}1C4aNh)D98JW@^-qGfH|~1t z4I~|`kwAMOTczZ`G^dIhTCh3>etKJF2g3y1w}h&-k;li>NxeDB{GO-Y}~TDTrJ8wC`` zp6j)K=|isS99SMbSNfXw{%SfB@ln_Hw$k&0zvcVn%LJJ!!;W*1x=bppLNP3@w$tJ;*dk+73J=TbwIww$FI3R#sFr1hK6 z7D7^L<>IHKYaiaa4{gNp@85bWa*0LR+*7Z)>D+P#f{NAj*4QfBfJgzOWROWK-Z_?5 zOHJy46WCL=*QM)Y3%+PEE$dvm^4_wOV=@Y2U$Cu`+60oii>=!LlTXLBg%Zii@-= z$5(<4(--3JsP~7}`26d)-U_F(jAsryxhTDYeT1iq(umo5o0PH6w_37kDMZS>Hsaj6 z<02_`j;3>QJ}AACDS1&!o0eW-NTp`2bc70q77ScdNKUb9$+j#B4qZC=#ROMvvqUT` z1-axUYm^TRq#=)mE-E!<8XBA(Im29##I{W<1ljNRBTl_ zx+IMQBF*B5RoyVH$D?Xw2FBvODngGOHFmeEpp=rP!$F8AT&;}YEaa3n1qdT&e8B`= zWcICsR{GYagGEzqWv-Wpp4^7d5wbZzzqJ|MbAC1#2m8j@vjB2Xc4py{%%)@)B?Ojl82 zFw#O*!`CTmZLqK*dgfB?>Bsj2wK|X0we9Z*%2wL<^>Hf3u_*H-#BfKefg6_1kMp?O zSzYbqIDu1Ia+QnxD31^K$6(trAuJsS9?Bc{`{oU&?;m3s^2YVRh$20|sO`QIl0P4Fc_hAZa4cuuQ)CXcJMdHplE0*Z@Tvyq!t6{+tx- zQ&sPW!0EA&_;9ZGhX~Wj>-z&i9OjLlv0Hw8EQi^v=V|$8qI1%GRvBxNl zkE?V(m$6n|&GxQ5b~qnji!DA(!$Q7P1ZC^0G9DW492^>wTBQ~bfZY1zJDg1EMn$jT zj~LF9NSI)pMX1o-`|>;dpuXfTjA)lKRU zqMB;$uq*Hs!Dp2$*4iUrS@bCztbG1>E@nr^#RqIS>97Lkrd$BcaS_HSMy}Zx*3p$F z?Hy`_4}?qfy8A$q9+^0UCUp?M_a?;-)8J)*<=$e*knlJ>P=4i(BOC_6c?!LY-!Md9>GJw`=J8FL&P`E+%6Ry{#4osN5AfXkG{skx9t}u1YjFf>O9r zQa2vQ8>GdChu89Ad>Tgs{mnCl+NY&&Gb?;7J3Piez%EF8iDVX&;DTK*s)JU*6X?Rx zdsrQVZzu`Ju=3t!mUdhz!cS9xv6{}P+~=f9v);%3Ez*D%UmkX==Z7ETE4Ssz zV^M+FDjhiY0{k(wvjowtMOzJb-d%=msZP}h16*A+(l^fvdyl%PnI5z!J&CVDzVgTO zdi8p*HpGCDIg>ne50 zr|Ze+25*;>L^gDbGH9%6fb8rN*-kNeucp1k<1v{~czoGjGZU+))nE7PKUXw9t+aS= zVBJd$MZ&i|DhDG8O(HnQ;H7PwB!vLCS?LyBlWkRjae!PTxf!d$&P8nKfj4{7yj{WQ z4X^E$9#fB_S_2tgusDzlSt6yX5d46ce`6hGCx*l7q(YU3?a)UJd)P8$wSfha%v;B5 zW1k)o_tpxyZ1>H9KhraExH}*D1LslwCc0ZQCN* z)D%!aW(1HKL5CeZd>#~vK0VLxaXG%}labH&&&@cOD*_IP zW_dz!4O!VyvUac}55@sfF&a1$mTD`^6QdZ(cv%`}?&p4yT-R72=&DUB4*+cn)dtr5150ODEuK;=VyCa!r>?<-U=6)}#jNkJ!=rzC zN9g4AL)RW0Yj_clTlq^+2YAKeGf0s-xj7$iq;S?$Fe4F|^G3UUaovHkkx;6Ln6 zf9YR4Tc2t7^$EhH17rx81mv&;*Crrk8w#WuNho-40q9L#4SX|`F;YmL_k}=R>w`a| z4KKw+c&z&`&&R6#`t-v8ytwjLM1J2O_pJ%NJtw0s{d@1@ya;BdGAU^|$EX4X)NJym zV~3lb1svRtnMJ0VG4bYgHgOI3P^?V*&fV9Gtm7{@DttRTDZ5Ig>8v@~oMXur>t+^@ ztb=u<^iVW2YYf;wh9rEXQUd7tWTZGt5^rf~|plzmac`w+A1Bb4^sUyQ*)l zF}KPR7chXUUYQ)V&6W0-jF*d2m+ZghQ!zk(~cJKF8%+eNEM z*qv-jR6u3c@Pl}PxD3Gh6jDx};#Nps(sRIPEMlf)nYGgz3F}%Y3f|Vw8!qnl3uAtX z7}p+j_Op4PXhz%LBvH{o%2qna-lkJeFjHuY(yJ}!x!xnh&kr$upsOP4mmOCIe$lc z2Uh{A9W?{!C!dj$cyTqfrEGW_udyQ^2l~dP6h;=`W|XQ3JP9k-^>uoE$g(f+@mYQN z!mC;<(&71f*nL4(Ft&t2V%-&q;A!-SNWG|)0f-eSFy65B*nT-&NxZjJd>w}m}U zPy3z~*|vvzJPSM{s(F7&4U981B*z?u%2CO;OfhX$wOTZ%cyJZ1dD6d~Ki7f;L!0T` z+(vWSx`)liSdYcq=SM|(_e?ow2H~{RAR&~JutkeVp4A|nNSM(}Yf~)mWb0_bM6P`CX zgS62yw{qB!IQU2x#tUUWCFHN)d#TD_@ak8+feZS|FNSOj4!%}Fb6UpmUQgMd^5yq& z`|xt2@+xzGbJ~LOr?#8t=h}M1uP60q*MoNVG*f5pUPIG{^;jY)A+I&X=IU&T*&Kp! zLJ;=Ommcw|z3n{{+xr%`+`;bdj%TJHD|2bc2X@7MKycWC!d8Lo@*xGVsO1GQ0~^Ca78TUR)z`C@VGy?0;(MurxL&bxiIt004}{%P=<8s4d62|s9e00ydi+5x z?k~$T_LpOeFSp-*KWfNdnk8ymU)^Ib=57Yqd1w*HI@E#1jvZ`-(F$}n4Xy2vuEB44 zegh64#^|Ep{ljc8?q~I+bI+Z{%JBbs%;Ru?Mx{*^r z6B-6rmA87{Z!p{Qa@7;X>c^+?>Ha~#e1Gl7p5M!bw|EW$&pqMyUg~&RGr8QGZ7#C= zzk1;n1j6qCD=aLVp9OTu6j1*Wry5}^UyJ}d&TT=kQ#Xx8W#{gPqRzNx7>im_PdjB$ zKRl|-cz66~0?QQ3#!5f!XtifET+ixlL~_|2!qevmx&Li(i@d*&nXxSvMW#b;M=FTT zVtPH`Ny1p1fWHMljixaXjGGDF75dPEzavJkABfG{zjQR@eO41iqQgYBbx*A!fT(*yJI zH$lk#*YiXF${_czrt20cZ}d!!FFzXSY{yQE;!^<8C07W!Ia<*I83N~#wl|$AAe2to zXbyWjNPlL7zCGMmchIX3JJiD=#Q=$Swsmlb)P;8E+7M(5*qyjk0^&^sPV9AJB1R*r zA8~*^Z4UcwmG9a_Tp0e(U%vVTSPz^8lADv&m@ve``#v^J+YE%CS~0Bw3*vJ+LAAo* zDi1FBR%-U^i<{Sn(5wWB3nJ`^iuL%0c7*F&e7PUaT*E)l$SQ#T4KiM3?3iUdz&t6&K&DGlGV6WQb z1wzL*4+X42JJ^Gib#ibrg2U1O10BylGTxt`zW%_q+M;330+ti1(spLr(7c4!OE0Zg zKeV_SY*Jy152nSLEQs7#&sV8lIGqXh0$txf*Ed>FS4Zt4n@w#AvAL=YMs|BRjRthi zHn>q-(H10u)#e0nZR@OU3vXc{$D*cd%Wjlb*hAS_T9PJ$QT18^%bo)^XaQvz#_6yY z5Ha@{4wddAGBeVV-K-P3|NS%X7w7GHyO003-y61P9b+6clibKaJ)q;Pftb=Fz%7tO z=%Z3nM60l@rn)NbV0>v2PQ#Yf`8_!H)3Vau?_cdCim4BVzSW^xtFs|oveMePJXt{M zYKv`xPEMqR*}KC)kr#pLK|a@$;CWrk^I&co#;Gs4@#UM3e|%n?l)s*kUu-W~{Jqzn ztTBT_X6*KGL1`2H&}Z-^wJObGescA`T0jYv06Grn0KyPmLR_-hZF26@skr(Jw+&fN zfY6cu@%++3X2g$_fO-6(6|Yd)!XS{^Swm#Mp$8G?{c_6dZLKSBcxMSArhBi7MA)SVP-;J{T9c}>+iQc4Evn8oU)_^_d%Be` zpT0jWXxZ*#Z5ew9yT{Fl*Ohh5bh#?v4k3n-lP-J}l*^t{|@9tExx#0|vLTiHYExqal0Sgor!|Yhq{F zrq)2st74V0(aDAfLP`}v!Wn^%(xjE;A&dR`ki@kEcmQ3>PuWMu-A-n~8TeKkzD z{cEKs>dndTBhb(@2K?IYzF?;h{fa4gQDu80(e{3;`~CeN2XILJm-FJM-K$)Q^l1_| zC>c8X)a+95cr#6B_hzs#J0OuE!B@l+XZp~CM3~k*Wc|*bo21{}|F|&32Yq__asNfx z9dbTMzk3~XMgS}_kj6&tTBeT=p!}drY7gt|l_v6^%6!|T>fTUNDsV!)o29N_ve~V`Jphpua)5$?p8n1SHe?B}YLWJ^wi|2Pa|0_-*9v9fIu%!Z1aOYh z%P17eix$yz>!AOit>ypye%pV)`RnU~GwomNk^z32?a4O~b!(NEFlC+ z!nW~Amr84}k{xEr=3cS$|5kM-#`kS;hhul|du_D%5v2B-K#i*9#T#N*3D~bOSYzs# z4$L68C}HxP(0R{t|4v;2w@&6y$}s2WPXOKs{0ldw;}Pfa&10zi_S4N ze>y7j%ja&%$GiT!KDiH9SH52`dHN)aso%elTXV$Xkr=0jX~5aU#$g(@6@i{v-kfXY zi6(G&PgW38#itQ~I(4@7s^6b;|5=}o#gqR!$P^uu^^4Tau>tI$JZ;UhUFX4=?Y=(? zzP&2HZO;Gkk2m+f-=1Xf*T4Pjjof~FZ5a4F?}V#iQBbmbuLP2GPXeYRqX;%|5>|m0 z8>dm3cu#d4T3I45cEJ)RKqE4jrwb^8|5mZkn=SClA0&;eptt=_=2E}YF@&B| zhp>0yP_WY_TWd-tn#3Y>H(U#0rWAmLSlmlToqV_fA4wGfV~pvui?vm^E(P>99c&En z;YR0MeX;m&zrO%b?V|y>S=x0`rF%?7eUOLHhOfXZl|zkru);C7F()ParbkMQp_#dH zc(Q!>bPWrTGJ3B)BT(}^65xKY@wR7U)eKW=OtZRX)PV@4K0{}5P0Ye|U)$!x%eei% z)DPReE3$)nQEs4Sswx3SII7Hw0BIW=inoA`7)%^Ozw0qW31)cK+lD&Kma-lqTVECg z>@Tfgk_Y2-Y=#3rZ~_`;5y^Wo_&*SBUbKyj6dORqddI(d&(6^!l!775BEUC_pRcA#5@LzNqkR~)`aF{vi_EbU( zvY{Y!7_Kv$Ir?-G4$=x2XW?b~&fov;JJTDiUaIlVWDx(=BJ2&Sq$Q;>P8^CnAWD=B zYGf%gq+xjV6X|h7p!no4^Os-$dI9m?OQ2TJIPg-WI0Q;;Rz%7oNNw3Q=WK2%VLvyO z-X_L9kR&3q1GakcdHD4sq|>+V{~dmPoL5lXz~Q?h zvUWNIAFBzD@3+@l#P@U|^1hh;Ub`MR3D{X2hl6S#rg%Fuoz+rhyKb)`_b>+CH;nHT zwQs%|ysm8K%ptAU1OE-9(N`F~cxQW;4s;PiqhPS|4sPy{%t0CueggJtX2(@gnza@? zIaF-YG+b}0(^Txa_3E8p3_pL}J^psPkQ><_8O&^u=F{O4ZxD-H>_~8@8N5t@msya4 z*c_Q>i>XnSeir28)P?|0T*^Nm3pk$_+PwA(!$cfultjqhgNh`V4+AwwLI9l}tb=8w zF_;9*sSY#+=nfE-LL>mHd8-=aMC)TwRJ8XtG~y!#!79iBfM-zpY)+mVlo@?Me~IE)MqlI^lFQhQC9 zjJ)Lte0=$3lhwT;AAedmWUH<0Gd~6z?Keezw*#D3Tv2P&C(XVe< z9R*>16tLTxQuk`KiyF2tR|BwB6!G3zMh;uzcH`s2vVb)1zO=jGQFVHoyE~XOM4@ia z96gj-HQQWN>D3MLGrM{Z;9@}9QxO((tS@ce8T98{{>?XZbUk0Rmh2z55^bh?Dq5h;mhGEbi5>i{^`#Lxqlh;$weIhbUOayH(tm8#L~#0f4X^{ z85i1P2a_ht^cdCDc&#&hkF zz0x_Zq{6mV;L$lOHhUF|HLi}L7r-mjKCK{4y{})SaD9LMesM3KcOx6BH|>9$7BCP{ zCg*BEqR|4Dnkd1WQ)$2-JpfL(5?#|Y5MEuVrCA#$J2(7B>*uqN!W&DFbp(3+;P-D& zeB9^t`Svm`x{J`94n~Vyw7-NNIEI>D*_R$>1yai_%RSbSTVL(O0w_b1RXY$E0cB*y zBo$FTZK$80#`DvF_oj0=KX;U_bq*hOa{uq!+lSX{lw)3h?!w?Sop#BeUi4m86F&XT ztIu;BP7l81bRMURB@c!ua?KhJ8(0ACQe=rvwh5au6rMI=Ut(o;>Fas(pQ7PAS=);H zSHfGw=N?l8`BR>Hjuv&s>0OP$eVC9Lc)aat8ev>Fex?TekYeq=hmAVklQ{F%ifn7N zeAZWM`!hkR-On{NByq+kW2wGbN+FbJSwb%%1KTFLu7VEL4E$GvN^w191-eR}{{0?>A)_HYBU9W#RZ4=}))2G|ZireHv3)Ywhk z5>Gm*0C}rnvHuM!PY#VAyqb=UXAwPCJT6`l!d`2(r39j$>6idYQllo*n*t*yhu~-~ zj^I@Q{n{!PePWYQ_y`e13&s%|j-ntqau&pf3v~HrC#= z8~lZ}cDvx#`(HT>1W#}Y(8Oq^gb@MPN4I@|TpOe^ycOxTiZ|J@tr(jP>?E6vjR->d zVV_ccST<4T-)jtotf^O7H2Im`|~$}aq<0Qxcc6nPI{Cm5cbYIy`=}|>3MhQ znpzCYC9y9|H<99XwMjHq>2SwBpf+dsBbgLzh*(f;@@2yE9$iE-HhVs|^dNfKQTNax;CCp2oc z6n^!{Ziw``)`W>%KfO7r!29^>cwSrQH=O#5}R;Dh~P}gWzzz(7c`~pgiaFIvk*)D*~ z00A^MS+f|7*ew7z}y`m1n;`BoC_&e8!+pTfb{_&!`~|FgmA}cc(rw1v_bBE zQ`Bb)9>4>TwM;s+Aj9TC3dkB%?B&cxe>tdHdF<<9r6TGYHWk(_{g4wc;cFA!3P9wi z=jyel&gwcl`G)H=YOJ*E3>ByWaG+a+BLpZUOl7Y@&^p6b-PQ^aIvt6>Lj6|r$mTwH zMjCG>0t>J**q``1#2&gH5;H_I$bxKIH&U=rY3tb#kk|L-8nxj0{>{Am{%!p9jz9a} z&$IX!^0Azd%Lm9CpFbne`}fY{HBWHjNjaxbfD{~Ucv+Y@ka9a6UOi3bYnq^^xeHZ) z2(Vk}R9ap?b-g@0n^SP0tP6~>yQm5tA5luA^-^f9Ov^1Fv6gH;l&X#cSB-UG4wB^| zS07Eg05<_uy3H-{zc*`ET7Gwofz3%-Zfhp(*lV3w_FZiEgs{WRh-D7W2&|+H30b3a zsfba8>HxOARrGa?x=VPvH+(?kR34#agdr|7*Bkx~!e1mvMR-cKm>wIAXM?pGN{ax4 z>MVk&*7Y-iGog)-9k`wR()EW8mhVdh%MYE+2(iwd=?nvK?rXuj1FtXCjusaYvJiFY zhc-0C!k?|zlKjdzSI*9_Gg1~D(sLC+sbrgh2Jpo!B}z*v(YXn*H)bRhjMO4Q`baMT z83V6>Y<0W-v%~WfAhA?R-u*PW03d1zSizpI;2>5qRKd!yJG(jo6~_dPUSlqTU_{?q zw(j*D`rL4LdiKqEQ!a51lwQ~Wf4<)S&wG>cp!drjm)#j1)-WCGi=je)aoJ~rC9fc# z-W!%xv+=p$L%s8oZ)n99@ELQb66^XQ#4d~BIc)^KrQW9#~ zs0E7zV44Rr#v5gMiM0k%P*_zcj>lH(%Aa=+U(avUUl$44`soU}I1_(jKxcT(UEz*J z^Fb{IXp-hwMne`lbu!IHM1+c@j66`WHbO}b+{^eeT&8$F3)bJQ)Od{L2+n?*x%~Dl z5Ak|yX`c(X*Do^v=XTDQQ@#VW`qz+GOQK!ub#LkiBN2=pz4eM8(Lms}|e*65d!;$$e zVid=j`5|Awd|KXt?%x-Q_%m8^7<6gup>`DNrFc=Fgo+uA!?hMGtLc&CMzv{7sXC?2 z5$rmX?5D@Ccgrf#-8t96R*HHXgE|~W%$(EuR5SR$ z(wH*Fm<)1xNWS>M+?Ht{2WX9Dwso|K0h~L>Y>);^IYHTU*tbO zK0Q+N{}Q|V?c8G0UZEG;AZ0qsmqLhJJ%MOgV69Q+$Px55BDwh9$^;Dc;*Ik& zc}M^5tp$4f`I~&X`R(rR_nVieJR8gYj>$Qt!1jSwG+6{@N}|oX~L}zbl9}HU2 zcr>FNAIOyU>2cX+5(RiCoGCo?awvxq99))+jmjJ9A z56%9G4o_3EZ8>eZXZ_UtzIHT^RArbMvOqj8U`reu%?%~DJy5oz;bRXj;{9MJiI!fC z!>cHp2bnKgjAv)+SLi>@4a-1$JIJ}v7?wbSmGg1-86x7YvK2M6<`X~}TR+a0Gm!FZ z!)!IaJ${q<{cQTn?zuFv^#^O@xslm+HIWv^G~EPvbak-OrdRwKAt@N7R;1N|)ikYI z*{E6JqdzOM{prsYM?aL z)2=AT;Wcjbww^fOZ4K^!zue2ie2OL=yFXFLTQLsQjt%oW$gi3 zW_O1}s@N(Le;E--4dHyf#QvrI z&%{yyVUrQ?A?(984OQasAHhcDDND^|^WpFAZ~nY{7z>%!D+I|@Fzv}3?`akf1`V7Y zWAojnR7?Zg#!;u6 zK&PH}DWj^xxMJdio8al?5DIvz0jns5Hrum>%@Vq&hlQYbM&U7a>x;5x7qJZ!0CLz(m~}%u@_eA>QpeiqhnFvQ@1~;A zQ&TNq;^IP$5wtu2CMJYu5uZy3=$gez2Mo&zJFwd5hI82{OAFU;{XYPyPfKrqAvteP z&HhV48m~WDplwubjwR~|fSBx$u%=Z_rQjz#AHZ4(QK|u2AwNT!PHDXZU^ljL>;-sR zh;#+A;XB0+wfC5tF9BQK(ow+2GI`XQ-4p~ec9O6^0$U+daWl7};$7W5416)sz@vOU zzxxL~+27OEdq-KZYODoSR32C+HokOj>^f5%G$4z8TH8XGj(y)fI7Hy>XWzK$9ooKs zz8x>u|6k?)*(YKNBHwSMTpEj9e^#n>#^!9h$K+XZl8R=%Ar^5e$X8386x3hZXDx!K`p1V4ju zK7dfNE-{Be&#PK{WHDY5q(THnfGJ>&R)3X=e9&LxeC+-Th4CdmD^uH8ft3Jmhbg{i z2{A~Cy#c$2=`)>|5+#KSQr;{b00s1POYC%qk^fX6SX4&s?g9WX0}>XMl2JYzM{Bns zVBnY>02mNZ05-h(VTqfCZqH_TfHIfehT`Cv3dQy~nbSHyyYxcc-J{UaC~VT(7CBqP*W zO%xi&j!szcw)NUa@u{WRZMMnzHVlBQcLD5MrVDn*|U>7A7wgVz-U^^Q~ znGpy2&o+HWywLJFK{%W`BPt~(U4%)pA zmDD!2&XW;my2)ZK)~Jv4C#je%5uOd2)+rq!GVsR&rU>c_ez>6`^7{Pmh?<>!?&^b8 zI#$tVBO|MVZKW`Podru885sWEDD8AwqR0=B3Jx~PI^?O9L@ zNU^~x4m!1Gm7|m}?swn*IFiQ~2g;RlM~?jF?N_gLju$3tsZ`2CyuCL*W+mw1#ID*| z*s|wRjZWG$wWrzz%o%Y&El?M4xe6A-G)QJas1&?)h<7^5g`eWyFWo6(R_IZT8YzPP z>SbW*RgjDdf~&0tAR;-w%I3!`1r*1F z+Rx4^I4p6?#u#@n1zR7*K@9M$%5wkHS3tjH*rZQSU`yPK2X$oKBT#{k-TOnl6pcWQ zBL*LVlp@k`PbHXj%Z8sKYOe(kU!Ro?@8#ilsz`phd-k+kQ({~#I6eH})c5HiiFndm z@Jo#2^q0?1cXvPGp1~YUFbqH_Ko>TFM!H^jOLXGY@vH~C9J5VAf(_DovRNx40(R0> zVEC!Ryi~=!XViV4T{~gu1j(z zkNMtUkCK+lc(D5^Cp#QRZcT#ZnDOcXPF)YdW-_l?k@FftA?p~~FW~T=BbaDM9smMc z&*KZZ)zyu2t{{3@Gvnklix#l`(Hx?Es6eF2WC|tFFCgm#h{Js4mhy}TlQe51&%}zy ze$il(0u98vt={>L8tvolx8L8NugwYib5WSIztlW1Km~cRu>ue@fZjgYL)u)-9t3A% zW?c7LKwOJpr--9ecalgX2%Pw_^|FxXU5~t%G-)X<3$*n{dG@E&?4>z83e=vL?Csup z8IDDIw|D%tYoFHG12af{NJzDq*39g~lG0C^ zi56lVwBA@hlwD^ujY^betUcf>vsK*mcBI!H1Tq@Xx&p*H_Vy#!)i$!N-I!r(Bo1sn z5A7j`Q3FVMuB8JIq^hcYtFHaUGImR(JdUvUgs-{SGf*{_^?M_}xp3KutT~n0A)_ zh7?oDfv!EQ{4aK9V|(pU1%bQAq9*3)o0LBBZp(E%BHql_Vg~l^`N@DM>cSK(j#7I9 zqtR9(fLz6FObVyiq0x>hffNF84@d`8-cDh&(+9*F|Bx@=d8_~akXdkJuN0N?5+x;l%K}1+0IhYpY_T=W0A2@pb3Jx3U0h|w=enb}JmG)i z*-PNt_Z#{4czg4t4`0`Am6nDfZ`wu=-+1XZGPY5j(>i3IKsf>LeWZ;a z0^sa1uwiC%2yfI!7M(r&_d|Px-B?-YE3HVy8tRl&HP|^8tEnYm%G{+PP*m!yq=?Kf zv3rN?VmF8JqRxD6vDl)$r6I21;XNoo&Uoa4L7qo8$bdD z#MgUCE)2YS@PW2fKfob;x;^*R5ZX!5PH6^)r~$;=S*c zNn9M$^wN=5Q|D0(I0mniuzL>&zb&(bTJfhiy7Yl{x&@k^pZe3$4C*{y?rwb1s59jK z;2C~rn8bDpdb`pZmV<`=9dd z+-GrrD+%DcoRbk9mOV*nsO!0d5rIcU!!AQCLqCu&CKVnV8@a$gI%{@JGwUZHuO|u@ z#`d^yjlgdDKm7j9BPQqZx4WmC`v?8wR{z}Hm#0sfsvmr7C>U%35Pk2$;{28eFh~J% zj3B^Zk+ZqULdya6LRplsc@Dv6zjurH|2!x{tG!bJIJgTA!2prcXX>#I*9pWyd>yD_ zYm;Xybo2^Nv8bJ$xn$%!p`Gb^H2w4T=_MdCo~BEkhKC{hg(i@B8M6Pkxq0X}&bV}A z?2Xyo%?+cUrOi^<28(@JCg8nJM;*$PDI4BOv4}p|AnByhMrDIMGAN+iM)|?p35fQc z3HX^l7D#&c@f!BVg0j%fHHgC=^uzjK3eZ4EYQevdbczc14P14 zVhdKm84>U`I|W_>GR&YVn%58RPUh_dZM>brr8Vh$hNCN!eEoG5O6Tc!^4v|x1q}?+ z=?EdPC^-ds9N3hQtt*=^WMUTkEL|Sg)>c-@o3=m*-ls58jRa zlil_F;s3&HVY&EXB4}`JopnrH-PXouaCf%?MT<;vXDAekLy^JV-QA%`DaCatRtjZ` zQ?$t7uEmPGTd@w#$9wNh-h1DC=VWD{ot$SU`;T9qWXoFlU#^E$?ah+-n(xgm!jwl- z(F=cO(G6na=U`P+@Xh-JhtY5E@SaN&O~~TwV%!QLbS#KEQA^T=detr^k<%lZYfI>O zAS&2e`+4Hs<63jBR=Z3a{-@rp`CA`q*bj;*@}B$8;QkP|pRbNDGPo0nS_-`=*}~J{ zUDy?5jca;Z@oHB{-T;`BtXlJ>*$6>w{bp$&_S$I=wpdq!@4OK^rx&XW8eqgo_I=TN zzd%7>Ni~ju6g_H{i$a{(M~AOz-BVe^{Fvu5>a-~*el^4|wKuwN)=uAY-dO9NgjD~q z8zHzOkE;MDTSKN{71e>#y`%k(vNgDGYcxpF9n-eDRIt7N#KkA?pyZZ|$HaDu;VR5f zImRVq%eL_d#z#}vUG`k&e%KzmOsXR{T7Z6_TvdD~4WxlzyOV6!!fLh| z5tKOis#LDr6xo-0Gf4w~lAB&G%EBiFaK3*2y=>#Jn1K9E$A(wsm0~ZV8Bun9%&VRWV9M3NbyPD+oZeHgbc~Lr z%A11|2=YxCeLo&M6q;HzD&qs0eXZ$OX|DXoYjAkNcS7!{5LaNr1a=L-nyI`;`|h=G z$Q>5+tjPcsPS=i!XCUa1N|8+Z&M%+djg~&F0Vk*iZESwM!4sAnT}7g{2u4w&HfB?z z70vZz)XNU_f-pv!1)*!=0I!3duJpe?vVE$;N0Uq@D4Zq=D!`)_7^uaXmlGb4O7Q?RL;hv?g4q1dV7D zTbE`c-^5;$T#4jKdy`Rj|DpHyYpU`yhSO1w4fwz%_PSCZ3bue4SNB5;D_}Cc5YrBE zP$#O@;w_|x6%*osiRboR6_@Ll?s>s?yB79%)1_#S%U!?rmq5xeT&HXmIlaD!Y%vFm z-1s!lKI%cA;;kJ65d$Ng#AM)*d+hIsJ2YzrR`xFlOqs1m_W znd})9eYy>-*U@Q>Lsq^sLK+D21?V!1BRM&r+`fiGE!Z`x23kDx)*pyf@s>4Qu_1}b zzL;r8LM>>+pMTDdsxdbmbs)gvjqA}GBkx2EH(TZ(0LnKz;mPo?8a!1s;gni*y#hK? zgRh_(AnQ$i!ISa&KG#SmJSb5P@@~-p@`oa3kxM&8S$*49Esh|R)pyXN)gY>RabWFZJ_ z`X=BgUCL^>)>M#L_if=!ms^qtDL$B}BrZ!lh1jKUb6;*MmgRZHh&m5sK>Xe6!gkte zx7=fYtaku&#+kKzE`2V2R4IATjAVvN0z;DJYDCS`1HM3QCLJ zUpE5xrNQRk&T%YyL=9LCqCsihdT-4>AS4F}@_G}D75NkUIA*ZJ#rU#V| z*OTS&yK7>X#~U~Ta(`SW8&{ zp#>RRo4CxJ_GBChNuKfA?SR2Sa|Ymro9304!}DitL30b>rJJEKbmzxnYf8dp|orI2U*JO_RrDD2=3=!A)B zhf^*S_5HwjBP)_4blfj&ibG9L3Ona#aC#d?omOs@XZKOH_U(Pg>7j`Y%c`S1r>0_I zwR$DP7?JPGO@Rarfq?2c3q*V-nw-)O-s&#B9l$rxQ{1p`Hbf+f57Z<;?RP;*OXrI9 zGq%C=YV@fdfE1txwsa+Gtr~xMq}4-Vrm24ZJfcY&)s(hLm`hhT-ueALhrEzP!S2UY zi#j>#2P^1z!7sMc?`VAaEavds*8mSHp3M&i9nXcx>M=t3jgNpqaaU zN>QC8`J?4Q>@80AVxMRgrS8wSRX?Gy+Mq+uB&(S7EsTz#oSLp}Cw^;&O3aYO3BLcysMUYtqX+te{Fu4$JS9ud(803+uE(6Ko`|a4 z!Vxe1DiI-4V&l(6^tNC2xU?C?C>(IsRbCL@1(G-tL%)T$!}*f)2=;G4k)m^idq!VI z1Xad7sdu^3siJ9ppNSd?U^voS`5KK(bE6AYMx-E$KaStZG)dzev;jVROo||BL8r^b zTvCs<#FW1<&8{tRo1Vn8il34$eX6pWPqZJve#TXG@8Xz2xFBOkTIL)fOcyNVV6iwXZj42OISyIxwVr|!I4SF#49W3zJ025Hb&ErvXlU6UypdG4-v+&%t zkJDoad9QEjWT(_VB$8FB=-pLi=y!T526t(Z#vm=ZZ*?L$k@t|F=Hxyqb?yrXM zYh!Y;Nk;e0i>bY=mZ3MbM>f$hz3>SCCh>Kb}29ywTo@;e;jb15f7;i2tqxx(e7zo^H z=u)qZ%#fpPAG%*o9~#4H_G!psJsK;fsdfP@vGJM_z5 z-NF>FJBPs)TZ^FnT9Pd<%XX>~c7EXFaQTUj;LTCUs93_E1pjjDLklZEHY5|ADM4j@ z{ge7WDzgcvRgyt3|lq*b^dek^4 zw^`^z@>9gEKH4}I?Q}P%Nok-;ni%*Lmy@IAWew91o$7Vogw75Rp_BR31(fQYUNEE9 zHBk)mYuo=Y!xXE3TRl-zwX05xFG$_^!{*Stt!IvTi)W@ZJfw5`L)YW@vE=olIj?sm z0Y<;d_gm3<(-o~sEk5~7+%E{_9A|%m$t@<+$&62djOKWf+B@qjKOH6K`<%zxXBQsz z_7#tVO;n~nFZOSmDspI4c8IlQ6~Z2}BTO=)>_;y0_c&Vx5f8-*+ClP8^} ze)s)*1m~pdS{+meX1{To>ns8J_l~HkIxOfUy(JTL-`ZbgeP3gci}iK#&FgCTp&AyS z5CY4)z9h!p8Mp|!;`=gtb<&yx{>|&8-K|4OJ5De(PHjT!LUcQpg7AXX>z~NIK%(MF z4mI|4gI&?gY`g{cuHUY}Htw93>25V2o~bJ#qmTdq0F0+lHd5kvJy!Pg_4pK6Pa$LG zZsWr3>Eh(5oiKqH{t`!a1wpEmXGNL+9u$blQ0$CK_Din5fTuTvT)nW3zx8YDo|8lY zRaIe>NNqOCnry_2S<2?&r$2T04z* z3-cX?Lsdz50(N8~uUNu>CCmgI5e5E5{J(SRZQQ9PMg{;b(E$L$r#X4JTY${%oI#dW zj#i#l-0mLbT25*UF9{x&P0V;PX}CW45<_t~?1flPBki~r$hBlEJdC)Az6#?pBR$-X zszNhJnTold(Ja`25sOP6-0t*E?#d1oQLk!}1-ZY!$nCx%xO&Zr`x3G}Tw55qQaBX1t zAS_MWe#+(%mj1y4z3RT!tQ4Y>^1y`T(hcx7yNr2w?}h#>Dr9!*{zOsTgd#& zY=dR_GGbHB}gbLQr|&bmla7#ugULKO1%G9PC~@lcIhDcNvPPJ1Bq zOR_Nx&IF|L_Bn7^O}FBM;z)Ax(&@&k_gTW5Y4t^j^a_+M)h0LsJp~hiB8zPkmc=IV zieX%%{7PXER7v{6Qc!nWTH`N|{rPi4P~rZ)r1VkaoOsFn4Y*49uv$v{kuSP@(W`~F zZdHXB;eZi1ZhNVIkM_R=Wv(TR-Xj43=1&9>{}(~7?k+a&Rvu3bJTbKL7em1(|6vFs zWy@Pug2Qs!S(|=@s-2W=2$MA@?<;uPmHmsMXQ}ALT=?%d#6o-!OFn-wq(CvG!)xtW zvN35~*{)Jir!RN4&CzRIQ6wTlH=h?8M-LHgT4l-g39fj8DD~JeK9n`oz@jj+#RnsM z5)vw&E7Qx<$t-l|T^AY}NS?^z+o|_rN6!okSHz;hKT^H@d#Q~kRD;_>zBo>TET3n# z)FuYI>nrfN$vLhVqD`R`9j_hK(pk>Fh`F$Brk!s2_dHS5Cti?w< z@R!9I{Lpgv;KPSXi2iPb+QD$Kpo?;u_7gspj2|Fx*v|1cv=s8o`r>I5;0uyUCD=ua}6fapj=d(X0%c1XI} zvbvq!F63?_Omz89f$huNx0Is3vO1*WCr z*~T%mw`%f3eo6}FW(AI@FD5Er=ta@_C4oF=&Z6PiU{nx!v!jxb!v16GEFmq`H*sdC z3suO&5)b>8*M_-Lya_o;I5pW}(D$AU!%PT?kRkAz{tez$@OJU>t zP~R@C?$xK$XW*I$=d9#+ZI~f3zsc?Q8CoweW>>Zizpi^#MM4n1L7SefM&$!YX*L>7 zZ(0cl<2dVHqA@)OQFB@ykbUdq#3I&XW^@bbGxY1Zi@GW+f`>Vka&dbiNb8H_z5%4g zimMeh7gXR8^PA$I3IyLL%109p79V}iacNP!6>liUphMVFazi4^u$nsDR*=v)Z4s}H zG#hZOPN+Xs54Iay>uTiTxNyI3E&9FFE^ycGp*dAh-lNF}xp&WhkRglC;Z!Go{M3c# zQ*QeP*H^_F9^PL1E?uYl$wPriB*=feXz`VQuS5a>aA$z@FBg46=3h@J@UQ0Ip6cjg z;qXu0Ki&7=DgYo9X!SJO-(LJr-9H`fZ=Fcu|KoT6l>Za&zvTfbf6M<1c6B9Gw7-5r Od3t|8wXpQRdjAJM+uj`j literal 0 HcmV?d00001 diff --git a/docs/web/static/code/ingestion-rust.zip b/docs/web/static/code/ingestion-rust.zip new file mode 100644 index 0000000000000000000000000000000000000000..6e71439d18d16e0c99bf873c4d04d04547349014 GIT binary patch literal 38092 zcmV)2K+L~TO9KQH000080000?T}vxO&#V9d0RI30015yA07GGNXKyZaZ*6QNhgV5CiYlR_4IV64_Ok<;Xqq1dnyog#f+7b+>r&4eNNYqnIp zZTxT)<(Eg*lf}P5JEu8NeBCX9C9+d%#x3P4{b2mk-}z6#9Co}V{1|JnsDh6Rg^)5k zkI7`IUq6F}KiCscO9KQH000080000?U8eYY#yyV!0Q_45015yA07GGNXKyZSZ)0oh zy;*M~NzyI&UcVwy_wly!^L-g$fC1)d-skP&LU4C8&t#V*s!6J3z8BUjG&~f zV9kQ*ulq}JNoBaZE$5iq^54GvpWj#8FMajEU+`yn*{vVTZq>@e!;dfj+c$pVP1!Mh zsXxB_?`89C{q>js_T~TB$ycA&_~U8EPlS*Adh_M6JiU~MFHEc5>gn6pUw-G!cC~)` z^1ps@zx?t){>O7^zm;$NKmX@1Pvw#E?yrStV;)vd{Oiege^p!Ssy8XIZ->vXSfByPQ z`_ApR?aQP53SB)}6|GU#>ZB=9%-(Sbwo+^{x{_?F$r#mH`D|MXlzUaVyPT@4zy3n} zEK7ZAE8gNa{*Pb2{Q7P6_^|r*U%&j%x#d0Z#+PjR~yHAE0dIMITrFJcYJC}Eje~9(Ns&dC?Xwo40&2x z`9CjZx4yc);`o?IlNK^e{6UBc>lavKl5gXEf}pxfAtG5DCx|2zD$*HAkZ=> zU9K7?RCU80!WwaSF3a&y;F5KSVw59`gVEFa_OPng&g^MoU18yMx#~iz)dpx%iqI;f zppXHWS&wn-|qeP?j_o zVy}uzE)-LG@8r4rSaN3^SW|4c8P=|6d@}~6)a>*B_`d$*^6}Nuz7K9q8R{vx!PF(# zFp6A?_mO){o_q^90S_~PlTxA_SVK|K*b4L4=AM7k?zVrF=g|t{;>aW99(!swt5Acf z(Yr?8ngHESa7G%&TT|;WcsgT&T)e47*=6gBbw18M|EB!da39IP{hF8Fe+UXEZNyA< zDw=yL_>Zl?gj{Q@z)<)D@^5^>z8AO{0F-K|rp~Qid;Uu|85h^z*dRXISn3M4rZ?(% zaGS9V9j%96H(!Bkxmf_JmQ=}DS8GuxND_2q{`p7Z?-O=^X+LNo;a0Kt*x;C%if!6) zgWntA}6owOYgind83u> z?)2sM_mfvtcID*d>bae~`uBGIbn#Veew z?&jwMd9OwrG7XF(Z~_wZx_Nmkra?Wbf-lc-2EAtjR@Yn;Ih?1e(Wa~C^UisUw)|x0 zfKk^^>*2?zm&ZF``PFXw{*KANck>-Htoe!TWXmcX&Z@f25?KJaUgJdT}bN!@ZbPH z)#S$4ANU8)w@X%DSKqK1pWKMSzTA8pT=Ru{^>AzO3}6;UgbWNCsMV*Md>7i^6+m1! z;7bj*H7gZZH?XVN7u48uJ9yRd?bfT|hg!CvLKlL&e(XQ)cBKMPo}C4xQ=(MDy7}@n zu%AC(-aQqV`I{dKXUD1a$R5-ba1`2}fmO8$;1Qs)%>lof0@Ug~MdLiMw)LPF0OdK- zj8T|8l#v&ku>=R-hCko-4(lA0{pxlNM|^u&&{8rIuCT>auwk8YC>?CXs$8rxsW+&2 z1w+=p7n2L1g2LA-&{<-h*5w(YJ=vi%9sfCNC&1w5+hZ)uTIwoU=aP*eJ}F^`s0P8= zQQK6Fv5L4hs{y8(vcQ2LYk_SGZb==#rq%tS?Dl_uuZ<6Xx%+-}D1U8t^xkF;xa#)1 zseCI&gWm@VF@tbvi*4$_r2(64B5(|WfPu7#4#yC|&q8i7fj-(W9ccPF%=vMPd7{j( zQah~eu{?hAgdgHQoFS zbp_Q=os5s!q)|wYL3FZ%l~5{6zyPey)Y==nu5;IP&0s925m&eAAE8^d zRF4gaR)b3gWt&9MLpC8w1kS6T6r?&v*T8wf;6v9vS-CXO+5@xRW}6NIe06&cU%&Yd z^#JrFkI8T6;PqIEA`cL*blSEA|dZR@A@vf1#{?(RP?{IXX@{#{-_;DP1pTjU3i z^=-fZ{bl#E;q99reO^7?J#W_k#`fZ`?D*a70;|R^ke#}?X0l612Xo@)JFs6*z=9?; zSpQZ*3Y~5Z&XP#(&q^=Jd+eD7DXi5KrNF%u8)x?Hbp==raoZkQ+R598-TBCNeff2_ z{%y5>Z{r@;qia=*9a}n_4LJg)Q%V}T@4CXGfz|5RGW0AL=qYLM zb5Wcv=xj@kfudcTxdg*f+udVno3$C1|DTujK`gMkFGIG3j_>WkV8t#j%hJ-f1zA~N z$dJmRvZRmr0<*&$SqPVpj=)XHHd|EJDYOcbkBw@PnLO2LuCT-g>@yd}*SdD;;F61_ znp9vUtn(e}kl@f1ynbXdRI|mv*_%W?u}O(xg{Ch-*F#pu-0CDWG)|vc z9QENTbLN(6=VEd&LARny*U@z&H$NJvk4p~0!CZp%;b1FHT>%op9!!V1KLk*A>&I2Q z`=k6e3XzKTE`8Ogz48@eXi4jkqTy(wE`c0tAQG^kmV8txhL#A_S7k_XE)q|rR7#t@ zDtg(iTmLqC{N??BQz@t*ca5zAY%(NF1glr2w3BFO3V8a?lG9a6O5P?{z-w@LSSqKP z#|KUP4ad~R3263fT-qDMwOWnmidh$<#WV&}tfJBt5hv?mQZ*i$w1n& zsT8IG;%frk;Dq0@o({&bt2lhy~@r+Z_QbK7Bg`SO36+um9M> z^F00bMU04^7vDp;c4T(U;(jNZYK;e37bYrMdb$)1Y^~TfnB1;(&Jc{?asHJ7a475U z0TC4}72cma`qej}m(^n#TTZw5HM0*2NK{S}U;;oZ)Mz1?R#>Bp+>Gu5&SLQ{c<5qx z)m5uQjLD>IrxSjTns+$u$GQ87&xSIOe08MKc5#bT84^#<>tF#ZvZoylB%z^gF{-Ybv6+k-L&FDkKmV+i|a2>yFF0f6k5IPwQg6Q_tx{rUsqQB06#xY zzJpXQp3?cmr*oLqu0P!LPl&H_lDBM?lioIDVM%1sC^d}wiH zHqyJ#Hu~4`g-GY%Putk74#~vjX?MR{J@P&O=h}{du}VBAit*0dgB~RztQc zXEPj3$7Z;kLumyDs^P#j%#l$f-ZF6no+qztnMtJ?a{mEPkMXCum2Q0+t#JAD{n{b9 zMT-&xL^Yc;AUG^$hl@QU8Z0Q6to+>BgHzCGrL-1TfZ}>o zk*#vwdC;9C_juSCWrOD{GX3~+QDkA3&(qhvQuj$oY}uw7gmfhDjV?p3D4Rf9wU$$I z5-=?hmdGZ{+Jc#Qu{pB~Sa0Zp^Dj@Dbtl!0Pfi2U1j$r)Nx@!%DWxE}Cmu4_d$h`P zC0!UMR4E1y3b6+_m)^S1iy8$imX9}TjxI%OYlP>DIk9%uL5!L=uEWsxZc249EAaB# zD-AN5W5Z%&ac}{1Ht`}8OWh>375F584S>FDljSN%H?8oqaguMJdhRwBQ*{uz@;OP~ z26u|pMy`JD*~h5ae09r5IeoH9Ww0y}b*-Ve-8q*bwzbrTV=zt99SHch8whCCKEq;=7U&D0W#;U2`(E&0 zqb^bV()b_%$O86(qU9D@dw5BX%oYFu78yoN6$3J3jIpVhq ze(79>JGi0NQ6DiA4Y>zjhDBY8@E!7u#^Euyv@F8ifWUy6uwwYlBC#MiA-Q~G;Cbi4 zWh01`00bX{lidK|1fY`p^*T-KR^_SO2~E5EYki21{(F13{jr_8{2nqe_8h)!C4}}w zU_YM@iM}OS-<<=s_TseK`tr^m!?Ms(4=zo&NGo$Kxxxv-fVW;G zc+Wa#POyfsaWLKt7v)r+;X>mhyq-+f9!4c`4y9ogAhdnBN2RltU&vegm% zY|XV$0GN>5od))v8A2P$Bxe%)=}3$``F(v^JrG|hUq1WRc%4h5qo|$Knk!v<2o4|- zTE}1*ApLMuT11fKCcMFBS0g|}s~Aj&z4i*qJROQV6psGuxaR9n@c51K-Ra^;Fp-m@ zKwt<4@c@5vIm6OukL3cw6Zk7geXa(eD)}yVH{SNzTc!-CKc{{tl?|s$h&Q7ro5SGzZv^2YrT zoY?N)-G=rm~?P@EGPi3-3c4u^C6Iunmh}vVwww;MD++p-fTa z44`x_c&BNsx+|L$Y0ALi{^a5(p3_nicU;hTX061YIji=UDd9nZh?VSxdy&f9JnHE>uhNDja5o8USELnIKZ?u#oc>w&_Kqh+XF=i^o zWVj@gn@+Z~rB>#0BgE$1_4UgWmEC2ks$TxCR-$E%G7PS@9D6XeI7uM1jFXkn3rKz{ zQq3U+#z0YkhN`8QD`_A43==+I#Vhx_-gsl`Z;dYs;?l<$Ex2vX(y7T)FscXFjTB>u z`4T)IG_+j7G<$Fw$yM`FLgo?nIHkEjZF_#ce@*WhyO5vAX{e9^0*c9Jg|FfLX&roLd7B2uJPyd;;wnVycbaZ68o+F&Wx#V4 z2VCz{k!y0_obffv<#N{EVW}l}$fsoJ`I+S9ccpW)+VSSnfG&U2DEBrlz={IfXVWvN zS?Iny{A48=LX!St;2|r{1I;#z|3qDgoG_GW|8QLK z-SfkG|3!IY2>MGn$sP)zM&Y;h`lLVGGdA&lyZY;J-HYF|I~I{{%8x?<;#;-KTj*|7 zqbMGG$B)kfyUz*@B*^#h8?CMZ@bh)t`uIpxtQedZOo^A|4R*15ARU}mvt-1V*8<}T zS8o~SMQKjb8U-FRbD5UT<55kV!0fV*KzFLHXUOYVytS68lHZp?sy5^hf!-8wIk<+T z+eby58<(DR}k_R&I5?q;!4!tgdpV#gCK@mT_1N*-`PB~nhwZOH& z0K0<4wUV24L1myFWfW_t3=_dhS-2enDvDy8!NIjFaH8IsxmfP!mwU-a2dBDt=(gkd zBonb`4J59!fFZdTR;oe9qlaD24Nee%)&tK<;JU(5Npnka#7ixA``hYSDvmA~+R|kr z)J-OKPJwte$#Q|)5y%QFW~__egO*9dtzMzpWNMc}$e2o&R8y~~Gl3Qo@%B$Q4WHZ=~vc~!VX^`mLT(QD2v33k}m%dtp zM)u0EyQU4jH?5^;@StL;6ugj?&a28>0_=6JOTbs+3@Lj9ou%32h0TxWois^(e7vNT z%Tp?lV3UpVo&fx)tJYZ3uoTuqE#Z|kzlHUU67QlKTCAF3ZU^V0rd{edBNCY62fp-j zrG#nRVn*Mfp{O_D&J>0wGra|(4&Ei)DoWadEg2tbOA>hpW@wy~iG>w3Q<@>pH0nBQ z5A)hdVzNEN{L-hQml_~s>BCloKM9iSnv{#+EZImYg`+A`>?OQebEv6A@F_OZKDTiI zWUe`;X%O*Vkd{hm-`AV%IHN$vKZpJ+ zL&acJSprtAI-bc%-#`8_m0|zf|2p|#h)AA%GJB4b+i-&x0lqNQu$ZCiCr*@rb~U84 z*P>MB7U*g3%A3i*(> zaw=fEWYhvhAAu!Pk*jbOoX*fE8?BLxr3^@sq5@6^e4OEH`wqg>Eon|#HvH?C(=MAQ z*=KJXaM$qT!}{CI+d~(PeaqbQkM}HN^4&Y|D7l~;e=0xjw|4%MySJ51e}3$)IQ{8s zaPs71{*&9waLsn{!SiP7y_v;zutWJZ+u0B9%C~Qjm}5M2@+(ig-7(F7{?>?d^5cEB z%yHhf3v-Iw0qZ+EZlaNZ2UugA>XFA{X~;~4jU9`N1m;khNKE#rDdLG72# z{&`+rR#%!UPH5K6O^l6=J8@8QZ3ZULd(aa5=@K8Y)|eqSW-7=4MagyU9N^5@SL(ga z+$^1#n2|*}`kb4ZgimSh$He?S*;`J~-f6jX#mwGZR&)fGHCBNPc?uE)wHp5^u1RuT z?_dJ>De5Fu3eD=wHY!Q;=-q)9zw3@JWaV8)GbuLzs9-E+|6V2RXGQ4)0&^MOyNR5@ zP!tG2ajs;~S|trkLblYJ6u3ZiIRL-J(y9w!)zXyRO7}eg!0y{jH=^{ zdi4wkhC`LbbU)Uqojw*mUD4j{5ptjVfw%iiu6GCX_0bc0wRkr_sgxx-y)c;^qT=Q{ z1?6pUwt%&{+`&eiG$YWBbxqBOn5!`@MAC+M8k(O9@NwflZq-h-_vXvz+`&#&s(Xw+xzO$8H z$-SxkuF##&L75f7{|BP4CFN;Sj4s6IXUx*&WP)2riC4Cp%*rRzbZX z(hpE_IV0Fva>+KpDmZW0j23!O@Mq$yeF=_O8a&vU7Q6qyG+aupA0`qJ6JAP zZj?`#zHa{jegktaO;Oh&$mkN63fY#EbaSaz`V&e)K{OqhrfLGhksi6d#NbMbJlDr# z@|zy|XjE!0#DrdZDs+mho2I2~rKx5xL5h4R(e+5>fltN zER`Wqb?r^EfwHTG6iwfE%p0oD;Am0EiIaOTV*3`DefOLD0PB19Y5b3Vn2nxr&C$e< z6YBXfLHNf)0RScqT(VoL0%$d7!&Nf9gA6_7Bw4dQ+md?3OD&_8U<8ON2v>!LdPg3n z5B<|S`u?W91@Ldr4@@ese$v`elJ9`-oYXAHum`ESw%lX2x|HCp${va&X0WP)EpA@F zPfEvT@MN^tp%3Egk9MqN=acGwPUV)lQP~%=?qfU${C1T&KGNOZcwU*ZFdO3kUiO=E ztd-sJw?}^3mDhEAIIcf+-*a<`QVP;ybYve?P8B-FO3M6ol(|%a$F1^-eEeJhU!A;;x=%}NWK)bu~sOc@;$d!b>r;7=d-lkj#1FHPbD_v zNV2^%TkCrFn%{*>OWm-2?5cFkVt!`d4>=3Bzp`?i61Pk~Ht?6^NhT`G*a>M@91KMh zwDhedY9Kbr$UIAcuWUSll)eYLewLj4B-dEd`TZTOIe*C8M?++4bIsQ4Ty@DhYa5L% zT3gk6!BMj9(HNyj?jv#DS(nh8cTq8cpUx!Py>oOw?XnK1bn|U7Y(wNA)f8Tn;wo;a znhqh?M)_QAs+x3{2J0a!nIC$<=*zj=Gl!hR+o@kuv)slFe-$3E!rXXnVx z{_8hCCQx1usYs=rtJ1PS@?)tR51Jdgl=D-PJbb4ist&phwCarlfpMq#@F;l3)5i^{%Gi;LYfhk*Q zrz5orL3(<3ZlPr6+%wgg2`zt50A?Za&-vJi^ZV>^{9`Oqx#~O@xo9X2xG3RJiYZa@ zMpy-(b<4rz7E4f8nxO`qS8$nIcQ$-Hx978#^nG&hbBgncgZKyCSOhJwGEPI~;tV!D z$;}^iPSV}ZT4}poESP~($`ufMU7b@-C&}TgdAfA@Q%bQ6xBNMM`RKLZ^XBF0n1KHG zbZq$Tx4%7MYRUkud6`58>*afOtvOJ=QoHZ5g7J}~*0UtwW+jbBdj}y5)LQ(w*L!cO z`4ygc>l~%eL>m^ok@>LfonGKAm1Z~>RvbC-pP0}(p$s>d;?iW^O$v&w_cG_tdzEyP z5$7Q_mbR6#NXmjt;EX`xGC#03nE(Z*;@=C5z0akuBpv*dhUm&!zkgcWiAlt_u{ll~ zN#ZUpv2jM`Xqup9d|Ea*r%)rq1i|Mk?K~OZ7*-0=>ls4ySp)KO=^!(qrVs4y ze;)pUi@c%;PMWC0<5pYKaD~@0*tL{!CCd@-WpEela%A}g6 z#3h*LWSP;F3A-edAGv?#`tofx9t@F|ho{`VHypZ8iJkPX0WhlI^`MKTX(u;Zv4Sxc znd{k<*UU=EXl+wvh~ea(zjM3C8$1?Y4!u~q9TUTL4j1Q;8@YG@Ht2FooSN~fM1yr# znMz=z?HaEkTrL&4bbxC6G<|{dy#9kX{c*3RAL$&ipU*Ve%X%owUA#HDiqxyhpd2;! z5`c7DA0`-E>B%K#%Db4ImoEF>N=w7u+2$U4c6sL7>Uh5@^cM0xf+qrrN%N-!g{B_7G|!Bx+NkH| zGvF}SB#3`=*)wkOIRw!wtKc+b5+VUmMV|sukl-8WDi+8x?;bveTTNJ$WoT;5%~umc znCam9UW3|`72aQrv01(+n;zVx7!(-MQl;^(s8mie(#iv85GkzDIqC5U*bGpOZzQJx zK21r}Ydk%r`Gx-HrR>&YBTG_fjxXj7cbZUaE}Cd+)Pt^tw( zmXkEO(hfMWqxIEG z!75cM0mlrUBU5e!9xx!#bqMeoAJ4n*Pon{1(*Aqt27ylD|FHQUE7jZ_RwdUM(kfAM z7Gkpsk}CbRIxs!-6Q-C6CaMeI_p`_KyG%Bb-U!kd@;urDom;x;6+_klzFX_wSgpHN zMLM(_?V?!Fl9Puefg2iPvP`NbqFAhpJ%G$O?O~c~p7AT^XZyLG zke>E;{b5~p<^sDeE!$njSq9fU@T##AJ%!EbcuSK89g)d-(O4<|g+hbG7HgW|J>5fv zS)K!LvINOiA9c69#jM<{vdmPF6yRujRz5cfIweUBhH)~zt6*B@?348A2SaHvvDSOA zt~ys=HXb=U1~*FsIACEFgVPFII>cXfGdWe|ND{*$pr%s2D90d*El08)s~`rGC{l$n zH;q)5vHcF)da;7#of@cYW^6vG9$6(7B%!$CryP<6?hMX44$3G&H%503wTA)mbR`3$ zr|rw3yG8ly$Ng5y84j;cyv4p<+MFt_{$#4F-Im9J55GJ>1>knTCvbt#9bIZjx|l>c z6IPgIFgCeTd{qWePVx>aw23qsI2C+r?hGHwZXfqJKSbld=C5XHd!VumXU5}Zj@C&k zUdH)>3c)GBR6wh`&QWv5<0m}0QCUOV0=wv$Y5Lk82Y^+4+e(mL9fI`oJnjClo+}5q zpL;eQ(zo~^w+=|nM2W$Yv`pW2-m#5c$RTz@C94QxY^4haoMGnLVRmA#T&3o7rtkQ1 zK#45)arLzN>wf*`>e^gTX+t_jpG(WBl_zho=CC5hsQ^3DYAzjA*EPTwcmO4Xf{iP9 zmn5mMC@?DE0XhG<;mv*hK*L4;t!*V`@LbSb@OAB*aa?oG%>*rTykK)c%jL?-1$b?T z_ic-iQWs)v$rc&9_nGH6%V);K#zt}glQ9X15lmJ&Edyk#OovP|eN6*K%MCS`z|yL@ zM0_Q5AAmDwpMUds|FU@)DDCL7N8ey3E3jg##(CItFOSxfH!-H@8U1 zTMAf@BFFT0-ZhU>>G0ha#qwEHoFipw7+2b`Vc zOZ&o-fo7#awC@BdZmeSp(hfa%8!WX>CflG~QnkpuoDR%5r*|*Y6SfwgvKf4BDpp}6 z4My5G>CdjUYA%KrZOYZbg4YsFkq#_LW> z1F_zmy?JOqaeTR5J^gm}#;$D7-;f@-*JR=Rfo{I7hB{nd?J?o>qlW1PPL&%D71^pL z;g^ya)L+COWXKhOSWd;j_iI?}>)Sfs!o&8wxA@Bo+#9T1L4XFnq#7Kg^zY?Fpn=c@ z7=fh7P|HriYev|$BHcAr4n2ug48hKFr?Ir$d?A3LNyf|O*uAubcZF*M%mGgVj5QXz zp^Y|jIR&kkbx61dw3h-Lg0DV0WiGt&voAq#9!c=hh7RMP3GnBBl3eE)Wlcb|2f(k6e3?gMr z5WGjR2E~-R=w+93*9Omgll!zY{k{1P$gI6yY6Oo|@K4fE3Fx(Fm~*UO+oZukJn)o( zQ!_&s%cQg9Zgow-UhkaaAi+0Zg!EF0B$8zeXRnp(t*Y#yq8VOG1N;JT zlu^*ZG@T;%DW^N~ug80wk9gdZP7;%;x$@>)NBZqUsq+djOfQu&cTLaqziEs?_kUA;SHH2Y+affrFT4YL7nHlPL=)n5cl>ef2syyCXJmU2x z;+-vd9q2Toz02eC1cxO>WV*wHnIT)qz$=5%MpsB?!K9(np0*@Yn!#3p0dod=?;&%Z z#^-M%sJ{GqSit@8k6v5$ZgNPuS$Y~hW zNbvMcNzfjK1y*K86y9Xu-`nwAXBqFgT^)aTysLOBob%^*!!CbOkuw2@HXEf2uB1@_ zy|{#9hh>t~OZWwLfIJ>VK#)MOPRjGy5Mh_5Ge@TsVspoR^~HZZPVJB#dA@QDH@5}L z-y17pFJ+g}G0Z@$GQ>0*sgNcaaSDYXZY4^$fuIOR8z9{TC4FAw|O*yoBJ z>H2Z7$uP|PY>N&*+RsSP>hWPUmY(HrdN`O2cAvpzEYZ*=zZFwS^^`4eBIGAPqbMsd zp`0Dqrl)|t?nZ#s9HtoB;$FDnrO&-df4!)BV4W2Ff-2dhN-a6!qOdF{!&yL#lc0Sc zCBLAS?qaPp49}i?_V45SHMjf-&u*KlA-68G$aIA2hJpZ;P{iuLo4Yddf#x*CIn<+VFWIdN2SzX`F|5rEX$3=iQY2AjYDjY~>b z376McIHt}(3xEuu4QUo^&#u5NUp(M}W1O+9BWPdl%|LHBGEld-rspcT#dIP_l#t_ql6{*%d5b{*aAvi_hs2&Y^4`ci=NNpp zFb6i38Ln`+miV&2O8LC3g~fF)IZZNUu(He`G$26MXzJi|No*6>EnUHi>rU-=mN~Jco>i}kc|Yw6>t$T1j1OF5L_rb7!Sm~@OqU!mvQaegjEPX1mQPfyl+KDIEx zga@<45rkUf2M?8$i|v9W69ahK{fc}=|Xw&zX$8tGB=N1`q z(7~!rXE*^rEIY2?a9&yz3<= zLc?%mz;Xkl0IR6rg{4bj>_86W;q72orb|BG_HhuY*Pq^~=3#knKi)l+-RgI~YrNT= zz5o2%>P|wM+ez+v@_TktReZPQ&F`yr&Ca|%$i*D`ym)KPn3SiWU~B-!VyH>(?Vyt? zvv8dvHC85O*E95z+#vlD*%D{$r*AP&duM-6%15`G^YZ?BYOCib-kk@}Ch~b^bYZ)0 zzwypooZI;^eI)E4&-1YUhW|c4yo{A%c{mi{I=RCAJ8Gp_%fO%!fQ@1x>j+3$(v5?Z zAUU{=s#3Gc04jwBY{Ap{<3pYM{2Eu~sSGpQ?*1B=_=#wS#Y;Z0s@P>}qN+duQAKYw z*V225DMRC10X*$F_}Gff0IDjR>XRm?WQ5tY8GfS8dPVo!AKM*2-a{gAaJ$uU02$=) zuCHI7XlP>o8kap^`rY@k`t9W+c6+$d8O2}Uic79H5SaWY#(TVcoKPndi&2v0SHP$l z)DnO{kS9Yb4Rg1oO-(jF1B%(MLj{KmE_)MZau|kNuenvWm;1JUc;Jb#0p}FW3;yXE=h`74%Q7xtFZRinBjgr z0AN}IPRo*y6oa;{lXe-$pIxJtTP_CpeR7w^NByr~PS^C-M&RT<>AUj!9#r%96&>qX zF7|l0dV&gXKD(EfZ7OcDq1hOnYcLgP9nf5|nZ4E-KwY_}RhQJw+9PLQgHvRDl~N2Z zmm|$IGu^RlBzEG&j>CA4miv5L{@CCvufwlDvq{52++ONBNxo-`a;Z{?NaX;n>HQhCMwrOc{J%lJwlaQcgbXDIk{qLAv<@KFxs43ucERvW3aY1B!6k!YJ6KZfY%-;5J>1q5*Mr1DGD#m06DnL+t4;DS^<4PiY$tBL25ob5 z4W_K6Ds6fRycKTKfm`+5urf&-=*@J+3|UDObq9v!UYwT-*^l^uDKU8S1>0df!#x(u z{r;`%+__trM<<^Xk;Cwxe_&&MQ=p^ZG?JAbbhZUR#HAP#^lq3T>z#9nRf?)midjXH zcs(n^H)m@8t}EHJ9-S@wP2b+bgP+^Ze@5XLcC>JH{JBjw@7Jo>u87(|7e~TlX8`x%Ry^UF> zWB{S73`}M-L(A}1b?|O7BnEOTlUad;_xba6C_tWy(!A`}t?%P0XHA49hvn4CeP;EC zM5CoA!*hO;)ngWvDfXlyFVWf zQU7oCfSoo{`+qGZc_sCQf)?&KTrQh_4rcZPm6#W8#?NNqmM0mkw5Rr`^KIX0a4k7= zn{iufC8=L_rp&DTj9FCNulO7T#0hMbRi;y}uYN2*GEY;!R>dfSW zsoMR^q~W=>4>tlM&A-7c7U{gBYHmR6KC+dlg^_*)v6WV307R!&oMRV*cNvaQxT@Bf z99M1UvME-Zm9(LmI=iLsejM0n1QFGgasX|uc&_BdNEa(L-!T?zz+^L+X$s)r*7yqA zT>?oK#&i$alk$)EjP!}xUfh=~-hUuCB@i%Gm}z#_Cr638PL+LcGAI^K+ck!_>RwlG zt+wK`q@f7@#&n4DB<<+z0_rpEq+Z*hkIl#OAlZRAbEMv+b*3&34yZBLNVTP6Z8sE# zHaj3!xo2Z~O#m|R)4D6IXR?A19TPvlzU6%}v@4qylTkMILX@S8jR(%RfXFG8Qirq( zLq+-pd(Z-a7rY6M1HzN06ZPL;-+BzNCC_u|qbnZjSCdbGQV>*zy-7Vt;7IofJk>es zo?{2x0_c{0g*j{RgD8~8%w?VZd&RX(Oqdo(IrzT2bZE?@rn^4X<~9QI&ZJJAr>^nDBYM8Ny?C`^227_ujG)&LH;Z z?EC3s-F#y+1h1#+tW1~paB$XTz%m^oWKlHC{S z%&+Hg^Bz4xGRWY+do;qeO@rcb5)~$OPb-=(eEY}KLlJY(%YLpeU#sEddk8!OlPpX z76pFU*FwHe7XCfe!Q6gLi?{O(uGpw@i`~POLqs)u8EsO6Box}(La8N$Y`sh;D_vy= ztlfJhf(Q_jEvBJN9Z`V&_Rb0L*#o(BL8iNj&te37OPGYbI{+*?UMivS3Z(GZd zv_UG}II~xZWJ2?}t9D17kkvU3nG&7~I(q0XW=ijH;=I-^R56yL%^j z+7}5gf4odaTVxmK_xQK+R5q(}Vec2edsaPD4&HdDUoUyfF}fAcJ*bpO_CNN)dfFT*FV*C#I?B-nBKmqnSj&!wII zh7&2+zp;D0tUbJ5|He6#(BNZ=lRJ_T9>k2zDpJ;s>uw;$;`cyc~?u<}qw zUCM9La#341pMMOnIqTandH)HZ6RVv}3^1nMRa-hWE>V8)LrZ|GYPMm4Xq=Iik1mp(j%lq&wop^MyhmWlJ6W=dUs@`Ow1hH#!o@cSH?fGRpe3xB3{6$amHXYl!5=>En#WQ+IqURe za{LL9ek$s3j{d`t`dAh78i)Nb7(J1{_sX%(|G8U>k#w<4+B*)_NZ+&MbI&TJz@1X7 zrCFC+E5_q%HF+-sYE%ZGaGfdY>D-_9#AqZnCvtKk2Ga!TcpoqZ^G|e7t7GxN@bb^~ z_WSC6PjWD0H*buse4t`DI@2ZZMiQx@w<$^|M6#?n=ZZ~@K2%m-aEIjEv%z*V5cMsh z+ap?d>n#8R4ITz46%JaeB5aVpA3c|xvs59}9Bi#NO94|&Iz=0sx3;(JrCse+cl!6@ z9#_2W6sLH6X}?|dy+g0{3n4zp?zrCalYVy;Dj6HAgK2vB%T1A(c?&htD4g}nK#;;A zYi-JcefjmzKgB%`b;85zzqY$~>FeN>Z!K&xeWwa&2O{&VVkK6Jt;t?h?+qX=`v43_ z#${VBml9F~Bh`vB`ACAd< z+ASVV`qrTjP-!X04P7w+6~oi1K*|@ueQLgYmlgg`TPf-?(I*eHRjk$0TUCdd>fOH+ zxcw}!IxSTHkW7u@udCfLS^2}k*U!SXtC7>U4_BL`Nxc#`oiv}Xi4z||d#B-ss~GSi zta=6vKN@Jg8d&{o5Or0zWwkx7^i@mgO^BA$*mr@~x7WhXlL)Kiaoj=VcI5!y#x(Bj zrxQYV*4|@=&7Fo&KM*<{^_0Jb@d&`vM8GuUYicPv-JP}+;BHhgsdu9+WhaS>tyYrR zh-6E*TyiyYc??Tm^!!6EX%5c6Hqtz%`hM6ujV>MDzdtCu@Asu1(9cK13@0_oACSBE z;{pd2zd6Egsw6VS)D}VLf-$*NfF3Z2K$(qlApTOMtda{!GD(zfU0Ksx)G5!jPW_mG zoszhAZk^vb*q@HGzsbU_H&%uSQ1sd4R=7DG0SbI-3=~jxQ9)NxZXH_Hij*|t*GBrM zYGb4Ze7ckLiB3GZ%_Y={4}o~^WZOi6j8h6eww z$#h61BWaCotc^5#GBN_(7jg-D*ma&(!NVHf!6C!T4L_e;1mFL0^uHgl{s(7zbM@-B z7#)mFzBrqWjRC}^L_5r)EYkhNnp&lfzpDML7-yA(>6c#6c8;}t*80z3fh!@7SB*K9 z+Yd6ry>i?6y}t%%$caa!!b@s#hGRMTiF>unSHU7!nuDSTncOFORQdo!b=Km zR5dDZl_u3fNslX6AU+&kdwE?B_2cnx!0yOVCxU}1C4aNh)D98JW@^-qGf zH|~1t4I~|`kwAMOTczZ`G^dIhTCh3>etKJF2g3y1w}h&-k;li>NxeDB{GO-Y}~TDTrJ z8wC``p6j)K=|isS99SMbSNfXw{%SfB@ln_Hw$k&0zvcVn%LJJ!!;W*1x=bppLNP3@ zw$tJ;*dk+73J=TbwIww$FI3R#sF zr1hK67D7^L<>IHKYaiaa4{gNp@85bWa*0LR+*7Z)>D+P#f{NAj*4QfBfJgzOWROWK z-Z_?5OHJy46WCL=*QM)Y3%+PEE$dvm^4_wOV=@Y2U$Cu`+60oii>=!LlTXLBg%Z zii@-=$5(<4(--3JsP~7}`26d)-U_F(jAsryxhTDYeT1iq(umo5o0PH6w_37kDMZS> zHsaj6<02_`j;3>QJ}AACDS1&!o0eW-NTp`2bc70q77ScdNKUb9$+j#B4qZC=#ROMv zvqUT`1-axUYm^TRq#=)mE-E!<8XBA(Im29##I{W<1ljN zRBTl_x+IMQBF*B5RoyVH$D?Xw2FBvODngGOHFmeEpp=rP!$F8AT&;}YEaa3n1qdT& ze8B`=WcICsR{GYagGEzqWv-Wpp4^7d5wbZzzqJ|MbAC1#2m8j@vjB2Xc4py{%%)@)B? zOjl82Fw#O*!`CTmZLqK*dgfB?>Bsj2wK|X0we9Z*%2wL<^>Hf3u_*H-#BfKefg6_1 zkMp?OSzYbqIDu1Ia+QnxD31^K$6(trAuJsS9?Bc{`{oU&?;m3s^2YVRh$20|sO`QIl0P4Fc_hAZa4cuuQ)CXcJMdHplE0*Z@Tvyq!t6 z{+tx-Q&sPW!0EA&_;9ZGhX~Wj>-z&i9OjLlv0Hw8EQi^v=V|$8qI1%GR zvBxNlkE?V(m$6n|&GxQ5b~qnji!DA(!$Q7P1ZC^0G9DW492^>wTBQ~bfZY1zJDg1E zMn$jTj~LF9NSI)pMX1o-`|>;dpuXfTjA z)lKRUqMB;$uq*Hs!Dp2$*4iUrS@bCztbG1>E@nr^#RqIS>97Lkrd$BcaS_HSMy}Zx z*3p$F?Hy`_4}?qfy8A$q9+^0UCUp?M_a?;-)8J)*<=$e*knlJ>P=4i(BOC_6c?!LY-!Md9>GJw`=J8FL&P`E+%6Ry{#4osN5AfXkG{skx9t}u1YjF zf>O9rQa2vQ8>GdChu89Ad>Tgs{mnCl+NY&&Gb?;7J3Piez%EF8iDVX&;DTK*s)JU* z6X?RxdsrQVZzu`Ju=3t!mUdhz!cS9xv6{}P+~=f9v);%3Ez*D%UmkX==Z7ET zE4SszV^M+FDjhiY0{k(wvjowtMOzJb-d%=msZP}h16*A+(l^fvdyl%PnI5z!J&CVD zzVgTOdi8p*HpGCDIg>ne50r|Ze+25*;>L^gDbGH9%6fb8rN*-kNeucp1k<1v{~czoGjGZU+))nE7PKUXw9 zt+aS=VBJd$MZ&i|DhDG8O(HnQ;H7PwB!vLCS?LyBlWkRjae!PTxf!d$&P8nKfj4{7 zyj{WQ4X^E$9#fB_S_2tgusDzlSt6yX5d46ce`6hGCx*l7q(YU3?a)UJd)P8$wSfha z%v;B5W1k)o_tpxyZ1>H9KhraExH}*D1LslwCc0 zZQCN*)D%!aW(1HKL5CeZd>#~vK0VLxaXG%}labH&&&@cO zD*_IPW_dz!4O!VyvUac}55@sfF&a1$mTD`^6QdZ(cv%`}?&p4yT-R72=&DUB4*+cn)dtr5150ODEuK;=VyCa!r>?<-U=6)}#jNkJ z!=rzCN9g4AL)RW0Yj_clTlq^+2YAKeGf0s-xj7$iq;S?$Fe4F|^G3UUaovHkkx z;6Ln6f9YR4Tc2t7^$EhH17rx81mv&;*Crrk8w#WuNho-40q9L#4SX|`F;YmL_k}=R z>w`a|4KKw+c&z&`&&R6#`t-v8ytwjLM1J2O_pJ%NJtw0s{d@1@ya;BdGAU^|$EX4X z)NJymV~3lb1svRtnMJ0VG4bYgHgOI3P^?V*&fV9Gtm7{@DttRTDZ5Ig>8v@~oMXur z>t+^@tb=u<^iVW2YYf;wh9rEXQUd7tWTZGt5^rf~|plzmac`w+A1Bb4^sU zyQ*)lF}KPR7chXUUYQ)V&6W0-jF*d2m+ZghQ!zk(~cJKF8% z+eNEM*qv-jR6u3c@Pl}PxD3Gh6jDx};#Nps(sRIPEMlf)nYGgz3F}%Y3f|Vw8!qnl z3uAtX7}p+j_Op4PXhz%LBvH{o%2qna-lkJeFjHuY(yJ}!x!xnh&kr$upsOP4mmOC zIe$lc2Uh{A9W?{!C!dj$cyTqfrEGW_udyQ^2l~dP6h;=`W|XQ3JP9k-^>uoE$g(f+ z@mYQN!mC;<(&71f*nL4(Ft&t2V%-&q;A!-SNWG|)0f-eSFy65B*nT-&NxZjJd> zw}m}UPy3z~*|vvzJPSM{s(F7&4U981B*z?u%2CO;OfhX$wOTZ%cyJZ1dD6d~Ki7f; zL!0T`+(vWSx`)liSdYcq=SM|(_e?ow2H~{RAR&~JutkeVp4A|nNSM(}Yf~)mWb0_b zM z6P`CXgS62yw{qB!IQU2x#tUUWCFHN)d#TD_@ak8+feZS|FNSOj4!%}Fb6UpmUQgMd z^5yq&`|xt2@+xzGbJ~LOr?#8t=h}M1uP60q*MoNVG*f5pUPIG{^;jY)A+I&X=IU&T z*&Kp!LJ;=Ommcw|z3n{{+xr%`+`;bdj%TJHD|2bc2X@7MKycWC!d8Lo@*xGVsO1GQ0~^Ca78TUR)z`C@VGy?0;(MurxL&bxiIt004}{%P=<8s4d62|s9e00y zdi+5x?k~$T_LpOeFSp-*KWfNdnk8ymU)^Ib=57Yqd1w*HI@E#1jvZ`-(F$}n4Xy2v zuEB44egh64#^|Ep{ljc8?q~I+bI+Z{%JBbs%;Ru?M zx{*^r6B-6rmA87{Z!p{Qa@7;X>c^+?>Ha~#e1Gl7p5M!bw|EW$&pqMyUg~&RGr8QG zZ7#C=zk1;n1j6qCD=aLVp9OTu6j1*Wry5}^UyJ}d&TT=kQ#Xx8W#{gPqRzNx7>im_ zPdjB$KRl|-cz66~0?QQ3#!5f!XtifET+ixlL~_|2!qevmx&Li(i@d*&nXxSvMW#b; zM=FTTVtPH`Ny1p1fWHMljixaXjGGDF75dPEzavJkABfG{zjQR@eO41iqQgYBbx*A!fT z(*yJIH$lk#*YiXF${_czrt20cZ}d!!FFzXSY{yQE;!^<8C07W!Ia<*I83N~#wl|$A zAe2toXbyWjNPlL7zCGMmchIX3JJiD=#Q=$Swsmlb)P;8E+7M(5*qyjk0^&^sPV9AJ zB1R*rA8~*^Z4UcwmG9a_Tp0e(U%vVTSPz^8lADv&m@ve``#v^J+YE%CS~0Bw3*vJ+ zLAAo*Di1FBR%-U^i<{Sn(5wWB3nJ`^iuL%0c7*F&e7PUaT*E)l$SQ#T4KiM3?3iUdz&t6&K&DGlG zV6WQb1wzL*4+X42JJ^Gib#ibrg2U1O10BylGTxt`zW%_q+M;330+ti1(spLr(7c4! zOE0ZgKeV_SY*Jy152nSLEQs7#&sV8lIGqXh0$txf*Ed>FS4Zt4n@w#AvAL=YMs|BR zjRthiHn>q-(H10u)#e0nZR@OU3vXc{$D*cd%Wjlb*hAS_T9PJ$QT18^%bo)^XaQvz z#_6yY5Ha@{4wddAGBeVV-K-P3|NS%X7w7GHyO003-y61P9b+6clibKaJ)q;Pftb=F zz%7tO=%Z3nM60l@rn)NbV0>v2PQ#Yf`8_!H)3Vau?_cdCim4BVzSW^xtFs|oveMeP zJXt{MYKv`xPEMqR*}KC)kr#pLK|a@$;CWrk^I&co#;Gs4@#UM3e|%n?l)s*kUu-W~ z{JqzntTBT_X6*KGL1`2H&}Z-^wJObGescA`T0jYv06Grn0KyPmLR_-hZF26@skr(J zw+&fNfY6cu@%++3X2g$_fO-6(6|Yd)!XS{^Swm#Mp$8G?{c_6dZLKSBcxMSArhBi7MA)SVP-;J{T9c}>+iQc4Evn8oU)_^_ zd%Be`pT0jWXxZ*#Z5ew9yT{Fl*Ohh5bh#?v4k3n-lP-J}l*^t{|@9tExx#0|vLTiHYExqal0Sgor!| zYhq{Frq)2st74V0(aDAfLP`}v!Wn^%(xjE;A&dR`ki@kEcmQ3>PuWMu-A-n~8T zeKkzD{cEKs>dndTBhb(@2K?IYzF?;h{fa4gQDu80(e{3;`~CeN2XILJm-FJM-K$)Q z^l1_|C>c8X)a+95cr#6B_hzs#J0OuE!B@l+XZp~CM3~k*Wc|*bo21{}|F|&32Yq__ zasNfx9dbTMzk3~XMgS}_kj6&tTBeT=p!}drY7gt|l_v6^%6!|T>fTUNDsV!)o29N_ve~V`Jphpua)5$?p8n1SHe?B}YLWJ^wi|2Pa|0_-*9v9fIu%!Z z1aOYh%P17eix$yz>!AOit>ypye%pV)`RnU~GwomNk^z32?a4O~b!(NEFlC+!nW~Amr84}k{xEr=3cS$|5kM-#`kS;hhul|du_D%5v2B-K#i*9#T#N*3D~bO zSYzs#4$L68C}HxP(0R{t|4v;2w@&6y$}s2WPXOKs{0ldw;}Pfa&10zi_S4Ne>y7j%ja&%$GiT!KDiH9SH52`dHN)aso%elTXV$Xkr=0jX~5aU#$g(@6@i{v z-kfXYi6(G&PgW38#itQ~I(4@7s^6b;|5=}o#gqR!$P^uu^^4Tau>tI$JZ;UhUFX4= z?Y=(?zP&2HZO;Gkk2m+f-=1Xf*T4Pjjof~FZ5a4F?}V#iQBbmbuLP2GPXeYRqX;%| z5>|m08>dm3cu#d4T3I45cEJ)RKqE4jrwb^8|5mZkn=SClA0&;eptt=_=2E}Y zF@&B|hp>0yP_WY_TWd-tn#3Y>H(U#0rWAmLSlmlToqV_fA4wGfV~pvui?vm^E(P>9 z9c&En;YR0MeX;m&zrO%b?V|y>S=x0`rF%?7eUOLHhOfXZl|zkru);C7F()ParbkMQ zp_#dHc(Q!>bPWrTGJ3B)BT(}^65xKY@wR7U)eKW=OtZRX)PV@4K0{}5P0Ye|U)$!x z%eei%)DPReE3$)nQEs4Sswx3SII7Hw0BIW=inoA`7)%^Ozw0qW31)cK+lD&Kma-lq zTVECg>@Tfgk_Y2-Y=#3rZ~_`;5y^Wo_&*SBUbKyj6dORqddI(d&(6^!l!775BEUC_pRcA#5@LzNqkR~)`aF{vi z_EbU(vY{Y!7_Kv$Ir?-G4$=x2XW?b~&fov;JJTDiUaIlVWDx(=BJ2&Sq$Q;>P8^Cn zAWD=BYGf%gq+xjV6X|h7p!no4^Os-$dI9m?OQ2TJIPg-WI0Q;;Rz%7oNNw3Q=WK2% zVLvyO-X_L9kR&3q1GakcdHD4sq|>+V{~dmPoL5lXz~Q?hvUWNIAFBzD@3+@l#P@U|^1hh;Ub`MR3D{X2hl6S#rg%Fuoz+rhyKb)`_b>+C zH;nHTwQs%|ysm8K%ptAU1OE-9(N`F~cxQW;4s;PiqhPS|4sPy{%t0CueggJtX2(@g znza@?IaF-YG+b}0(^Txa_3E8p3_pL}J^psPkQ><_8O&^u=F{O4ZxD-H>_~8@8N5t@ zmsya4*c_Q>i>XnSeir28)P?|0T*^Nm3pk$_+PwA(!$cfultjqhgNh`V4+AwwLI9l} ztb=8wF_;9*sSY#+=nfE-LL>mHd8-=aMC)TwRJ8XtG~y!#!79iBfM-zpY)+mVlo@?< zcr6t<0q9vwrWg*sM2J-^-fng4y8Ni@zPRk){pvB5NxM7~+bk1+*gDL0OLXk;ntA}J z1}U&uiFgSNnH3|+J*J#xtAkwe82aLN{^Q{Db*W3{Me~IE)MqlI^lF zQhQC9jJ)Lte0=$3lhwT;AAedmWUH<0Gd~6z?Keezw*#D3Tv2P&C z(XVe<9R*>16tLTxQuk`KiyF2tR|BwB6!G3zMh;uzcH`s2vVb)1zO=jGQFVHoyE~XO zM4@ia96gj-HQQWN>D3MLGrM{Z;9@}9QxO((tS@ce8T98{{>?XZbUk0Rmh2z55^bh< ze-`LEHiB=XTKw0IT(u<$*#ndrCt--rUMVtfv*sW?Dq5h;mhGEbi5>i{^`#Lxqlh;$weIhbUOayH(tm8#L~#0 zf4X^{85i1P2a_ht^cdCDc& z#&hkFz0x_Zq{6mV;L$lOHhUF|HLi}L7r-mjKCK{4y{})SaD9LMesM3KcOx6BH|>9$ z7BCP{Cg*BEqR|4Dnkd1WQ)$2-JpfL(5?#|Y5MEuVrCA#$J2(7B>*uqN!W&DFbp(3+ z;P-D&eB9^t`Svm`x{J`94n~Vyw7-NNIEI>D*_R$>1yai_%RSbSTVL(O0w_b1RXY$E z0cB*yBo$FTZK$80#`DvF_oj0=KX;U_bq*hOa{uq!+lSX{lw)3h?!w?Sop#BeUi4m8 z6F&XTtIu;BP7l81bRMURB@c!ua?KhJ8(0ACQe=rvwh5au6rMI=Ut(o;>Fas(pQ7PA zS=);HSHfGw=N?l8`BR>Hjuv&s>0OP$eVC9Lc)aat8ev>Fex?TekYeq=hmAVklQ{F% zifn7NeAZWM`!hkR-On{NByq+kW2wGbN+FbJSwb%%1KTFLu7VEL4E$GvN^w191-eR} z{{0?>A)_HYBU9W#RZ4=}))2G|ZireHv3 z)Ywhk5>Gm*0C}rnvHuM!PY#VAyqb=UXAwPCJT6`l!d`2(r39j$>6idYQllo*n*t*y zhu~-~j^I@Q{n{!PePWYQ_y`e13&s%|j-ntqau&pf3v~ zHrC#=8~lZ}cDvx#`(HT>1W#}Y(8Oq^gb@MPN4I@|TpOe^ycOxTiZ|J@tr(jP>?E6v zjR->dVV_ccST<4T-)jtotf^O7H2Im`|~$}aq<0Qxcc6nPI{Cm5cbYIy`=}| z>3MhQnpzCYC9y9|H<99XwMjHq>2SwBpf+dsBbgLzh*(f;@@2yE9$iE-HhVs|^dNfKQTNax;C zCp2oc6n^!{Ziw``)`W>%KfO7r!29^>cwSrQH=O#5}R;Dh~P}gWzzz(7c`~pgiaFIvk z*)D*~00A^MS+f|7*ew7z}y`m1n;`BoC_&e8!+pTfb{_&!`~|FgmA}cc(rw1 zv_bBEQ`Bb)9>4>TwM;s+Aj9TC3dkB%?B&cxe>tdHdF<<9r6TGYHWk(_{g4wc;cFA! z3P9wi=jyel&gwcl`G)H=YOJ*E3>ByWaG+a+BLpZUOl7Y@&^p6b-PQ^aIvt6>Lj6|r z$mTwHMjCG>0t>J**q``1#2&gH5;H_I$bxKIH&U=rY3tb#kk|L-8nxj0{>{Am{%!p9 zjz9a}&$IX!^0Azd%Lm9CpFbne`}fY{HBWHjNjaxbfD{~Ucv+Y@ka9a6UOi3bYnq^^ zxeHZ)2(Vk}R9ap?b-g@0n^SP0tP6~>yQm5tA5luA^-^f9Ov^1Fv6gH;l&X#cSB-UG z4wB^|S07Eg05<_uy3H-{zc*`ET7Gwofz3%-Zfhp(*lV3w_FZiEgs{WRh-D7W2&|+H z30b3asfba8>HxOARrGa?x=VPvH+(?kR34#agdr|7*Bkx~!e1mvMR-cKm>wIAXM?pG zN{ax4>MVk&*7Y-iGog)-9k`wR()EW8mhVdh%MYE+2(iwd=?nvK?rXuj1FtXCjusaY zvJiFYhc-0C!k?|zlKjdzSI*9_Gg1~D(sLC+sbrgh2Jpo!B}z*v(YXn*H)bRhjMO4Q z`baMT83V6>Y<0W-v%~WfAhA?R-u*PW03d1zSizpI;2>5qRKd!yJG(jo6~_dPUSlqT zU_{?qw(j*D`rL4LdiKqEQ!a51lwQ~Wf4<)S&wG>cp!drjm)#j1)-WCGi=je)aoJ~r zC9fc#-W!%xv+=p$L%s8oZ)n99@ELQb66^XQ#4d~BIc)^Kr zQW9#~s0E7zV44Rr#v5gMiM0k%P*_zcj>lH(%Aa=+U(avUUl$44`soU}I1_(jKxcT( zUEz*J^Fb{IXp-hwMne`lbu!IHM1+c@j66`WHbO}b+{^eeT&8$F3)bJQ)Od{L2+n?* zx%~Dl5Ak|yX`c(X*Do^v=XTDQQ@#VW`qz+GOQK!ub#LkiBN2=pz4eM8(Lms}|e*65d z!;$$eVid=j`5|Awd|KXt?%x-Q_%m8^7<6gup>`DNrFc=Fgo+uA!?hMGtLc&CMzv{7 zsXC?25$rmX?5D@Ccgrf#-8t96R*HHXgE|~W%$(Eu zR5SR$(wH*Fm<)1xNWS>M+?Ht{2WX9Dwso|K0h~L>Y>);^IYHT zU*tbOK0Q+N{}Q|V?c8G0UZEG;AZ0qsmqLhJJ%MOgV69Q+$Px55BDwh9$^;Dc z;*Ik&c}M^5tp$4f`I~&X`R(rR_nVieJR8gYj>$Qt!1jSwG+6{@N}|oX~L}zbl z9}HU2cr>FNAIOyU>2cX+5(RiCoGCo?awvxq99))+j zmjJ9A56%9G4o_3EZ8>eZXZ_UtzIHT^RArbMvOqj8U`reu%?%~DJy5oz;bRXj;{9MJ ziI!fC!>cHp2bnKgjAv)+SLi>@4a-1$JIJ}v7?wbSmGg1-86x7YvK2M6<`X~}TR+a0 zGm!FZ!)!IaJ${q<{cQTn?zuFv^#^O@xslm+HIWv^G~EPvbak-OrdRwKAt@N7R;1N| z)ikYI*{E6JqdzOM{prs zYM?aL)2=AT;Wcjbww^fOZ4K^!zue2ie2OL=yFXFLTQLsQjt%o zW$gi3W_O1}s@N(Le;E--4dHyf#QvrI&%{yyVUrQ?A?(984OQasAHhcDDND^|^WpFAZ~nY{7z>%!D+I|@Fzv}3?`akf z1`V7YWAojnR7?Zg z#!;u6K&PH}DWj^xxMJdio8al?5DIvz0jns5Hrum>%@Vq&hlQYbM&U7a>x;5x7qJZ!0CLz(m~}%u@_eA>QpeiqhnFvQ z@1~;AQ&TNq;^IP$5wtu2CMJYu5uZy3=$gez2Mo&zJFwd5hI82{OAFU;{XYPyPfKrq zAvteP&HhV48m~WDplwubjwR~|fSBx$u%=Z_rQjz#AHZ4(QK|u2AwNT!PHDXZU^ljL z>;-sRh;#+A;XB0+wfC5tF9BQK(ow+2GI`XQ-4p~ec9O6^0$U+daWl7};$7W5416)s zz@vOUzxxL~+27OEdq-KZYODoSR32C+HokOj>^f5%G$4z8TH8XGj(y)fI7Hy>XWzK$ z9ooKsz8x>u|6k?)*(YKNBHwSMTpEj9e^#n>#^!9h$K+XZl8R=%Ar^5e$X8386x3hZXDx!K`p z1V4juK7dfNE-{Be&#PK{WHDY5q(THnfGJ>&R)3X=e9&LxeC+-Th4CdmD^uH8ft3Jm zhbg{i2{A~Cy#c$2=`)>|5+#KSQr;{b00s1POYC%qk^fX6SX4&s?g9WX0}>XMl2JYz zM{BnsVBnY>02mNZ05-h(VTqfCZqH_TfHIfehT`Cv3dQy~nbSHyyYxcc-J{UaC~VT( z7C zBqP*WO%xi&j!szcw)NUa@u{WRZMMnzHVlBQcLD5MrVDn*|U>7A7wgVz- zU^^Q~nGpy2&o+HWywLJFK{%W`BPt~(U z4%)pAmDD!2&XW;my2)ZK)~Jv4C#je%5uOd2)+rq!GVsR&rU>c_ez>6`^7{Pmh?<>! z?&^b8I#$tVBO|MVZKW`Podru885sWEDD8AwqR0=B3Jx~PI^ z?O9L@NU^~x4m!1Gm7|m}?swn*IFiQ~2g;RlM~?jF?N_gLju$3tsZ`2CyuCL*W+mw1 z#ID*|*s|wRjZWG$wWrzz%o%Y&El?M4xe6A-G)QJas1&?)h<7^5g`eWyFWo6(R_IZT z8YzPP>SbW*RgjDdf~&0tAR;-w%I3!` z1r*1F+Rx4^I4p6?#u#@n1zR7*K@9M$%5wkHS3tjH*rZQSU`yPK2X$oKBT#{k-TOnl z6pcWQBL*LVlp@k`PbHXj%Z8sKYOe(kU!Ro?@8#ilsz`phd-k+kQ({~#I6eH})c5Hi ziFndm@Jo#2^q0?1cXvPGp1~YUFbqH_Ko>TFM!H^jOLXGY@vH~C9J5VAf(_DovRNx4 z0(R0>VEC!Ryi~=!XViV4T{~g zu1j(zkNMtUkCK+lc(D5^Cp#QRZcT#ZnDOcXPF)YdW-_l?k@FftA?p~~FW~T=BbaDM z9smMc&*KZZ)zyu2t{{3@Gvnklix#l`(Hx?Es6eF2WC|tFFCgm#h{Js4mhy}TlQe51 z&%}zye$il(0u98vt={>L8tvolx8L8NugwYib5WSIztlW1Km~cRu>ue@fZjgYL)u)- z9t3A%W?c7LKwOJpr--9ecalgX2%Pw_^|FxXU5~t%G-)X<3$*n{dG@E&?4>z83e=vL z?Csup8IDDIw|D%tYoFHG12af{NJzDq*39g~ zlG0C^i56lVwBA@hlwD^ujY^betUcf>vsK*mcBI!H1Tq@Xx&p*H_Vy#!)i$!N-I!r( zBo1sn5A7j`Q3FVMuB8JIq^hcYtFHaUGImR(JdUvUgs-{SGf*{_^?M_}xp3KutT~ zn0A)_h7?oDfv!EQ{4aK9V|(pU1%bQAq9*3)o0LBBZp(E%BHql_Vg~l^`N@DM>cSK( zj#7I9qtR9(fLz6FObVyiq0x>hffNF84@d`8-cDh&(+9*F|Bx@=d8_~akXdkJuN0N?5+x;l%K}1+0IhYpY_T=W0A2@pb3Jx3U0h|w=enb} zJmG)i*-PNt_Z#{4czg4t4`0`Am6nDfZ`wu=-+1XZGPY5j(>i3IKsf>L zeWZ;a0^sa1uwiC%2yfI!7M(r&_d|Px-B?-YE3HVy8tRl&HP|^8tEnYm%G{+PP*m!y zq=?Kfv3rN?VmF8JqRxD6vDl)$r6I21;XNoo&Uoa4L7qo z8$bdD#MgUCE)2YS@PW2fKfob;x;^*R5ZX!5PH6^)r~$;=S*cNn9M$^wN=5Q|D0(I0mniuzL>&zb&(bTJfhiy7Yl{x&@k^pZe3$4C*{y?rwb1 zs59jK;2C~rn8bDpdb`pZmV< z`=9dd+-GrrD+%DcoRbk9mOV*nsO!0d5rIcU!!AQCLqCu&CKVnV8@a$gI%{@JGwUZH zuO|u@#`d^yjlgdDKm7j9BPQqZx4WmC`v?8wR{z}Hm#0sfsvmr7C>U%35Pk2$;{28e zFh~J%j3B^Zk+ZqULdya6LRplsc@Dv6zjurH|2!x{tG!bJIJgTA!2prcXX>#I*9pWy zd>yD_Ym;Xybo2^Nv8bJ$xn$%!p`Gb^H2w4T=_MdCo~BEkhKC{hg(i@B8M6Pkxq0X} z&bV}A?2Xyo%?+cUrOi^<28(@JCg8nJM;*$PDI4BOv4}p|AnByhMrDIMGAN+iM)|?p z35fQc3HX^l7D#&c@f!BVg0j%fHHgC=^uzjK3eZ4EYQev&O^-Jp#$UykZ{ zp!+0DpnPReS>d{jSQ^aQGO8k3O1Mp1Jil4J?{u4i=*_d$U3n~GsSz~oXcWlMz++J zf&}%{33PVzWV*+6X5Fl}+1CS(eSYR|e5~f)E2ha~3qTSz%Q(zcB^E<&M57nO&Pq3k zwT0%l#W>@e9)7&qU#G4EEGbrOgfnc#_3Zp-81Hu48TU3gmqTuRP+P}mEA!fNsNw9} z;t&2oVt%r3cmmT5>DAAQ2;=S@zh!h!eI63QJ5|(VOgr+gCV!^8-hI7#ti^k6V{jB& z)oedZdP5ywA(v{SHw~+(3R37D>3349Ceo_bro?f|*yvIx@Vy##X zjihsmPs&zm6BJ8~p|3mdInVvLIe5eORonL{&vl7XA3Rckv!@0xIZ*&IAg(+pHfv$k zo2)2WLJ|!+H-4Jzb3Ft8j=pgcgl3#^M{VHnYxM7$HLv&t z>L>et9VWbk!Y@teA{}t`tn}ASbi3b^($_6ONhZZbDPi9A>;QAV2EK#pN%G8|g2h7| zd<~%-!XN>^w2=>Eaf4y$#UqNodNVKII#pV#U3mW*8uuFq9~2RZj+?=*5SPfxTkIO| zU1R?6V5TMr76R0cM+^~jOs7eueD9yn?9Rv>-ar^!jXgTI*5C!pje%3VS&+k2r8nhL zWt7VGVl~VT^VVaHvIxd`O9;FQekkdGe*m4f2>>SQ>`Gnv`vY$e2dNKQ8-nKtjsqBO zPZF#yLQc#7$%Mb3DVRGLZa@Uze!sXf!q8zmwXCf$-b8H$P$l( z;qx1)vFHWn>ID^(YJ18kV!G|SuSeWC)h{s~7HU5;>A0F361<@)hju#}Cfht-qZJs| zCATU2gmE2rPH`!jr{F_H-+jRx;NJxIH${MGM+bZn&v&}g?+P}6xRoM?7Np8Vw>oG>EOB#=ez1(R?w+^L*RC{TYxpoX9y_9cS3^sXS^+@(3N6kEm4*~D zhZ6R!=Wl}714L%+=^Ar0u?7OIUb!8tvI>pI@^gK<0APN%mza$BrY#7sj-b^6^@{4t z|9Sz{7O?qaBz81b-{%(PObjK%0i_P*`9F2U(+(Rx-*v}SircPG%XeX)7Z%fdO! zIYl6*%r|rGfmygzXWx`kWiocP3N5AgNhL{$TOh&^o)&HE2{?nY_CA_ca=^2%jn$7P z`?+^59ctSu@L_+BQ(6$%q>srWh+8Q;`z4@=y%AH>T`>2y^Q;y71XU$I=jKyfpT!6Y z4@3Y3Mwr*`_U0*PL^P66zcUU%otJUjkJ=Ayo!Ko6Tb#$wH)Ehf5pwj#47#mm2;Bhn znv=hVdCC^7Jb&(bV=I%bzh#fp5UtD+*9u9m-8!TTQR+S)Y%&~H;(6teW9UXNHITU` ze`I7P7i~aWh0vJ_w&+F|A-mjDJSj=!$V%fs%cPOJ_WjufPsMStRSati>J7-eUzy*` zIPM1D_s98U5ekgh*fxDfwPlossRz|Haflksx9=^|y(>sgVZ$)xWu_De=b@J-7M+4J z67<(0p#p1XZ&*og8z^MM%|fqE`eyRV0y*NCF1*3{YuZ!s6fm$|vXd36O^{F(a8F1@s@V6#Cd@na z9UME@)W*yedEAqAATE0%WWOaR7d#7*o4;;ZZrwj+Y73s7ms`9Z95t=KAEOGf*4X0m zLj`_ISFr5)G;#N{&X7?0J+DbIFaqc^Z3L*szUj9-9Kq)MoL(vJl{m*kk|z+p_e$W1 zoqdZJ{0ZyBz*r;aBPZyXfA|#78$)@5oQaUkHrzU$T)0>FL6t80w&Qr;%$8%tNtySp zN>P<&CCez8-?KlWN!p@;RkK#8#822@)h*(cZDxCbUyzrKao?8hAn5$ZZga7OsyPojT;1l2U42gv!?m`@J(=dx5p$l)25PQ8p zyC1Hs{b7evlx3kVe%ZHO3G7qqCa2E%*c!RvWy}n_`VzrHj`+1Mm}M9BLo%(<7_0%STF; zN)UIB?%^Poa(YnPR^kSdPTH8m8L|7tF%{al+^6w9h0xa zV(Oz_^xJ%ybTN#6Oj0JIxK7O0ekQ}y{5ay3k!h%sgfX;YlLFyh8{lI?N+d-q4k#CI zQ8Vrxp7NP_c5SKq^dzx$;*>(!gUV_?*=`{B2_O8{)hX-oj=~(w%i81->>c*&$vS|Z zU@d9XDuQofd?v*Dh$89RxEBJ}O*-1Xe@+il08)l??8O*si{yC7B z?{%XqZe&$1XroROh_}8+MVO4i-*K?ElZFXEvw;ADC@F`kD&|i0ELO_b1|6p1J<&L( z$mn?T22AQVPv=IHG*DTXU&K@k=txD&q6H01G71Il?KFV?CB9EBu3v**wJ(^OHOcYU``ED+Mc&daX{m5<+}P>3z41p> zuH0foL(_j5lO1KR;?po2BOe~+NRF#x3$mqI}> ziRl9c1w@WrhWj+Aj(=@ziEl$DyJC+7Vey&O;1eyB$3ios`yxIHB1kSK;WXf2s~pkr z17r1V*a0?8pPks7hLF)3v75TMZuI|DGr-cS3~}#_PSK(FhzzWg6Lpp5#o!iW@tp6(A)3Q z49jA)IunkUXZ39~grxh#?QVNsN(_I-PHq^Sy5563%VXxGbil^4moE1@E~ zczj&>&Zqd*?-#Q=6af37)9?eC;Wl^P`7fjzhKQE?Dr4%FyqyGw^poys3x5)jRxgd6 z;f{8O;0ro7iICbzoB*taAt)6Q5z>T@ULjT>DfBeg`uYl|Qr0-pQbJLKX$=?z+-m6W zvVx)Y^ zOn}3IypI$)DitM^S>`JIGd21V9ebv^cZx^XXOS}?z9i&2py2g{n&@+DKzG>Jox1rc zArBtNC0DDM(Q1mF5XWY^DnWkG{ZRRlzS#9a=!kUEMN&Yy&AyejKi4BO!YMH|Bcr4G zK02;7fl2M}rja@cZCP3YOj?|b+YO0&mzQry8U~>|i*=yQ4)NLfakUEb#hy33lbamO zq4{Yt)(N&w#arDi8S>g#vStw9l5%S7yw5`nWXFb`*I_e5gE&+HpujT2O4Ih`yrKa$EvPCpXy5C$qi;v04(#>TazqPdLfW^|_3;&&=N&?y4My zn5j<{F7*E~SK-mF?2vj@B(gufn_5X6FF$Kz4td#J=BXi{%OC3tp}#ubY1LmktSqRQ ze)<;HjoG{{w5Y&c8PA-*Q~OD$OyfmM9m8!qWn@RERT#(T`;sf=) zanysTscJ2vB}mdJ$j!V-*r(Q?$Qa$E5i zEJO@p<(yCr5TwNZgw53jCL^oTpFP21`K7n!(~TFHf2a#qdai2p=amPB(U+7yypDqQ zlxUKxjC@YoNqC7kpQ7CTqpKz7jRVTPP)p4Q4`Pf++eokhXh~V z?`J`yTic?mwGSz#y|_U}Kyz!>Se|=HdX9!Q#2EC~`iZ z3VHJLDsOzwcfM3FA}K22q4!RPZ&7c#Rp1h~1!DR;v~mvy1fo zo6XRD0LATtBm92+?3%X$N#L2^($esE7QBxe-6rh7AZfCr+(N%(ef#8a&@#dGFk9Ql z>uC4U65VILZ3IG>^&cu#9XAAn0%BVUwPHpCYpbZV#ZI-=dus#8JuXkcTb1tdbIsdl z3!l~uU!4|}Kk>LbPQ9GyQGzsCI2y=vi6wNN3dW`|9s4zP%sFdh+iu7KUEM$~K=0O$ zGxcYGUwj=_M2Tyv0v}Oe{GV5=^1K}V@!-w(5BA{L|7~Hwf2N8S9=5LhUarng|HhSE z2J}8p0sw9xkN$<5`Y+DO)ynbTxPM2H|Gh8_D4G2KVv43J7WO}f7!MuxFcPKzv-%I6 C&#c@4 literal 0 HcmV?d00001 diff --git a/docs/web/static/code/search-rust.zip b/docs/web/static/code/search-rust.zip new file mode 100644 index 0000000000000000000000000000000000000000..f08531a91d323fad6e28a5412120440a6c2f22a6 GIT binary patch literal 42197 zcmV(+K;6GkO9KQH000080000?UCT=bZdH!}00~?I015yA07GGNXKyZSZ)0ohy;*M~ zNzyI&UcVwy_wly!^L-g$fC1)d-skP&LU4C8&t#V*s!6J3z8BUjG&~fV9kQ* zulq}JNoBaZE$5iq^54GvpWj#8FMajEU+`yn*{vVTZq>@e!;dfj+c$pVP1!MhsXxB_ z?`89C{q>js_T~TB$ycA&_~U8EPlS*Adh_M6JiU~MFHEc5>gn6pUw-G!cC~)`^1ps@ zzx?t){>O7^zm;$NKmX@1Pvw#E?yrStV;)vd{Oiege^p!Ssy8XIZ->vXSfByPQ`_ApR z?aQP53SB)}6|GU#>ZB=9%-(Sbwo+^{x{_?F$r#mH`D|MXlzUaVyPT@4zy3n}EK7ZA zE8gNa{*Pb2{Q7P6_^|r*U%&j%x#d0Z#+PjR~yHAE0dIMITrFJcYJC}Eje~9(Ns&dC?Xwo40&2x`9CjZ zx4yc);`o?IlNK^e{6UBc>lavKl5gXEf}pxfAtG5DCx|2zD$*HAkZ=>U9K7? zRCU80!WwaSF3a&y;F5KSVw59`gVEFa_OPng&g^MoU18yMx#~iz)dpx%iqI;fppXHWS&wn-|qeP?j_oVy}uz zE)-LG@8r4rSaN3^SW|4c8P=|6d@}~6)a>*B_`d$*^6}Nuz7K9q8R{vx!PF(#Fp6A? z_mO){o_q^90S_~PlTxA_SVK|K*b4L4=AM7k?zVrF=g|t{;>aW99(!swt5Acf(Yr?8 zngHESa7G%&TT|;WcsgT&T)e47*=6gBbw18M|EB!da39IP{hF8Fe+UXEZNyAZl?gj{Q@z)<)D@^5^>z8AO{0F-K|rp~Qid;Uu|85h^z*dRXISn3M4rZ?(%aGS9V z9j%96H(!Bkxmf_JmQ=}DS8GuxND_2q{`p7Z?-O=^X+LNo;a0Kt*x;C%if!6)gWntA}6owOYgind83u>?)2sM z_mfvtcID*d>bae~`uBGIbn#Veew?&jwM zd9OwrG7XF(Z~_wZx_Nmkra?Wbf-lc-2EAtjR@Yn;Ih?1e(Wa~C^UisUw)|x0fKk^^ z>*2?zm&ZF``PFXw{*KANck>-Htoe!TWXmcX&Z@f25?KJaUgJdT}bN!@ZbPH)#S$4 zANU8)w@X%DSKqK1pWKMSzTA8pT=Ru{^>AzO3}6;UgbWNCsMV*Md>7i^6+m1!;7bj* zH7gZZH?XVN7u48uJ9yRd?bfT|hg!CvLKlL&e(XQ)cBKMPo}C4xQ=(MDy7}@nu%AC( z-aQqV`I{dKXUD1a$R5-ba1`2}fmO8$;1Qs)%>lof0@Ug~MdLiMw)LPF0OdK-j8T|8 zl#v&ku>=R-hCko-4(lA0{pxlNM|^u&&{8rIuCT>auwk8YC>?CXs$8rxsW+&21w+=p z7n2L1g2LA-&{<-h*5w(YJ=vi%9sfCNC&1w5+hZ)uTIwoU=aP*eJ}F^`s0P8=QQK6F zv5L4hs{y8(vcQ2LYk_SGZb==#rq%tS?Dl_uuZ<6Xx%+-}D1U8t^xkF;xa#)1seCI& zgWm@VF@tbvi*4$_r2(64B5(|WfPu7#4#yC|&q8i7fj-(W9ccPF%=vMPd7{j(Qah~e zu{?hAgdgHQoFSbp_Q= zos5s!q)|wYL3FZ%l~5{6zyPey)Y==nu5;IP&0s925m&eAAE8^dRF4ga zR)b3gWt&9MLpC8w1kS6T6r?&v*T8wf;6v9vS-CXO+5@xRW}6NIe06&cU%&Yd^#Jr< zJ9V33HFKc?(ULH13s8R@xJKP;jRf`YLE8+hDGr;laYZvtqsY|`w(#3l6jL1(^O|y9 z--*3A-Fzn!C|vN(I~`5IY9oM3H$6%CGqNJv2~eA<7KjPpba$<3P;;NMGx*Pkh|J%R znveVB1EapZBgsUu5g0|4tE=vSUf7k^zyc-MDEk9QS8JeTtRdFkI8T6;PqIEA`cL*blSEA|dZR@A@vf1#{?(RP?{IXX@{#{-_;DP1pTjU3i^=-fZ z{bl#E;q99reO^7?J#W_k#`fZ`?D*a70;|R^ke#}?X0l612Xo@)JFs6*z=9?;SpQZ* z3Y~5Z&XP#(&q^=Jd+eD7DXi5KrNF%u8)x?Hbp==raoZkQ+R598-TBCNeff2_{%y5> zZ{r@;qia=*9a}n_4LJg)Q%V}T@4CXGfz|5RGW0AL=qYLMb5Wcv z=xj@kfudcTxdg*f+udVno3$C1|DTujK`gMkFGIG3j_>WkV8t#j%hJ-f1zA~N$dJmR zvZRmr0<*&$SqPVpj=)XHHd|EJDYOcbkBw@PnLO2LuCT-g>@yd}*SdD;;F61_np9vU ztn(e}kl@f1ynbXdRI|mv*_%W?u}O(xg{Ch-*F#pu-0CDWG)|vc9QENT zbLN(6=VEd&LARny*U@z&H$NJvk4p~0!CZp%;b1FHT>%op9!!V1KLk*A>&I2Q`=k6e z3XzKTE`8Ogz48@eXi4jkqTy(wE`c0tAQG^kmV8txhL#A_S7k_XE)q|rR7#t@Dtg(i zTmLqC{N??BQz@t*ca5zAY%(NF1glr2w3BFO3V8a?lG9a6O5P?{z-w@LSSqKP#|KUP z4ad~R3263fT-qDMwOWnmidh$<#WV&}tfJBt5hv?mQZ*i$w1n&sT8IG z;%frk;Dq0@o({&bt2lhy~@r+Z_QbK7Bg`SO36+um9M>^F00b zMU04^7vDp;c4T(U;(jNZYK;e37bYrMdb$)1Y^~TfnB1;(&Jc{?asHJ7a475U0TC4} z72cma`qej}m(^n#TTZw5HM0*2NK{S}U;;oZ)Mz1?R#>Bp+>Gu5&SLQ{c<5qx)m5uQ zjLD>IrxSjTns+$u$GQ87&xSIOe08MKc5#bT84^#<>tF#ZvZoy zlB%z^gF{-Ybv6+k-L&FDkKmV+i|a2>yFF0f6k5IPwQg6Q_tx{rUsqQB06#xYzJpXQ zp3?cmr*oLqu0P!LPl&H_lDBM?lioIDVM%1sC^d}wiHHqyJ# zHu~4`g-GY%Putk74#~vjX?MR{J@P&O=h}{du}VBAit*0dgB~RztQcXEPj3 z$7Z;kLumyDs^P#j%#l$f-ZF6no+qztnMtJ?a{mEPkMXCum2Q0+t#JAD{n{b9MT-&x zL^Yc;AUG^$hl@QU8Z0Q6to+>BgHzCGrL-1TfZ}>ok*#vw zdC;9C_juSCWrOD{GX3~+QDkA3&(qhvQuj$oY}uw7gmfhDjV?p3D4Rf9wU$$I5-=?h zmdGZ{+Jc#Qu{pB~Sa0Zp^Dj@Dbtl!0Pfi2U1j$r)Nx@!%DWxE}Cmu4_d$h`PC0!UM zR4E1y3b6+_m)^S1iy8$imX9}TjxI%OYlP>DIk9%uL5!L=uEWsxZc249EAaB#D-AN5 zW5Z%&ac}{1Ht`}8OWh>375F584S>FDljSN%H?8oqaguMJdhRwBQ*{uz@;OP~26u|p zMy`JD*~h5ae09r5IeoH9Ww0y}b*-Ve-8q*bwzbrTV=zt99SHch8whCCKEq;=7U&D0W#;U2`(E&0qb^bV z()b_%$O86(qU9D@dw5BX%oYFu78yoN6$3J3jIpVhqe(79> zJGi0NQ6DiA4Y>zjhDBY8@E!7u#^Euyv@F8ifWUy6uwwYlBC#MiA-Q~G;Cbi4Wh01` z00bX{lidK|1fY`p^*T-KR^_SO2~E5EYki21{(F13{jr_8{2nqe_8h)!C4}}wU_YM@ ziM}OS-<<=s_TseK`tr^m!?Ms(4=zo&NGo$Kxxxv-fVW;Gc+Wa# zPOyfsaWLKt7v)r+;X>mhyq-+f9!4c`4y9ogAhdnBN2RltU&vegm%Y|XV$ z0GN>5od))v8A2P$Bxe%)=}3$``F(v^JrG|hUq1WRc%4h5qo|$Knk!v<2o4|-TE}1* zApLMuT11fKCcMFBS0g|}s~Aj&z4i*qJROQV6psGuxaR9n@c51K-Ra^;Fp-m@Kwt<4 z@c@5vIm6OukL3cw6Zk7geXa(eD)}yVH{SNzTc!-CKc{{tl?|s$h&Q7ro5SGzZv^2YrToY?N) z-G=rm~?P@EGPi3-3c4u^C6Iunmh}vVwww;MD++p-fTa44`x_ zc&BNsx+|L$Y0ALi{^a5(p3_nicU;hTX061YIji=UDd9nZh?VSxdy&f9JnHE>uhNDja5o8USELnIKZ?u#oc>w&_Kqh+XF=i^oWVj@g zn@+Z~rB>#0BgE$1_4UgWmEC2ks$TxCR-$E%G7PS@9D6XeI7uM1jFXkn3rKz{Qq3U+ z#z0YkhN`8QD`_A43==+I#Vhx_-gsl`Z;dYs;?l<$Ex2vX(y7T)FscXFjTB>u`4T)I zG_+j7G<$Fw$yM`FLgo?nIHkEjZF_#ce@*WhyO5vAX{e9^0*c9Jg|FfLX&roLd7B2uJPyd;;wnVycbaZ68o+F&Wx#V42VCz{ zk!y0_obffv<#N{EVW}l}$fsoJ`I+S9ccpW)+VSSnfG&U2DEBrlz={IfXVWvNS?Iny z{A48=LX!St;2|r{1I;#z|3qDgoG_GW|8QLK-SfkG z|3!IY2>MGn$sP)zM&Y;h`lLVGGdA&lyZY;J-HYF|I~I{{%8x?<;#;-KTj*|7qbMGG z$B)kfyUz*@B*^#h8?CMZ@bh)t`uIpxtQedZOo^A|4R*15ARU}mvt-1V*8<}TS8o~S zMQKjb8U-FRbD5UT<55kV!0fV*KzFLHXUOYVytS68lHZp?sy5^hf!-8wIk<+T+eby5 z8<(DR}k_R&I5?q;!4!tgdpV#gCK@mT_1N*-`PB~nhwZOH&0K0<4 zwUV24L1myFWfW_t3=_dhS-2enDvDy8!NIjFaH8IsxmfP!mwU-a2dBDt=(gkdBonb` z4J59!fFZdTR;oe9qlaD24Nee%)&tK<;JU(5Npnka#7ixA``hYSDvmA~+R|kr)J-OK zPJwte$#Q|)5y%QFW~__egO*9dtzMzpWNMc}$e2o&R8y~~Gl3Qo@%B$Q4WHZ=~vc~!VX^`mLT(QD2v33k}m%dtpM)u0E zyQU4jH?5^;@StL;6ugj?&a28>0_=6JOTbs+3@Lj9ou%32h0TxWois^(e7vNT%Tp?l zV3UpVo&fx)tJYZ3uoTuqE#Z|kzlHUU67QlKTCAF3ZU^V0rd{edBNCY62fp-jrG#nR zVn*Mfp{O_D&J>0wGra|(4&Ei)DoWadEg2tbOA>hpW@wy~iG>w3Q<@>pH0nBQ5A)hd zVzNEN{L-hQml_~s>BCloKM9iSnv{#+EZImYg`+A`>?OQebEv6A@F_OZKDTiIWUe`; zX%O*Vkd{hm-`AV%IHN$vKZpJ+L&acJ zSprtAI-bc%-#`8_m0|zf|2p|#h)AA%GJB4b+i-&x0lqNQu$ZCiCr*@rb~U84*P>MB z7U*g3%A3i*(>aw=fE zWYhvhAAu!Pk*jbOoX*fE8?BLxr3^@sq5@6^e4OEH`wqg>Eon|#HvH?C(=MAQ*=KJX zaM$qT!}{CI+d~(PeaqbQkM}HN^4&Y|D7l~;e=0xjw|4%MySJ51e}3$)IQ{8saPs71 z{*&9waLsn{!SiP7y_v;zutWJZ+u0B9%C~Qjm}5M2@+(ig-7(F7{?>?d^5cEB%yHhf3v-Iw0qZ+EZlaNZ2UugA>XFA{X~;~4jU9`N1m;khNKE#rDdLG72#{&`+r zR#%!UPH5K6O^l6=J8@8QZ3ZULd(aa5=@K8Y)|eqSW-7=4MagyU9N^5@SL(ga+$^1# zn2|*}`kb4ZgimSh$He?S*;`J~-f6jX#mwGZR&)fGHCBNPc?uE)wHp5^u1RuT?_dJ> zDe5Fu3eD=wHY!Q;=-q)9zw3@JWaV8)GbuLzs9-E+|6V2RXGQ4)0&^MOyNR5@P!tG2 zajs;~S|trkLblYJ6u3ZiIRL-J(y9w!)zXyRO7}eg!0y{jH=^{di4wk zhC`LbbU)Uqojw*mUD4j{5ptjVfw%iiu6GCX_0bc0wRkr_sgxx-y)c;^qT=Q{1?6pU zwt%&{+`&eiG$YWBbxqBOn5!`@MAC+M8k(O9@NwflZq-h-_vXvz+`&#&s(Xw+xzO$8H$-Sxk zuF##&L75f7{|BP4CFN;Sj4s6IXUx*&WP)2riC4Cp%*rRzbZX(hpE_ zIV0Fva>+KpDmZW0j23!O@Mq$yeF=_O8a&vU7Q6qyG+aupA0`qJ6JAPZj?`# zzHa{jegktaO;Oh&$mkN63fY#EbaSaz`V&e)K{OqhrfLGhksi6d#NbMbJlDr#@|zy|XjE!0#DrdZDs+mho2I2~rKx5xL5h4R(e+5>fltNER`Wq zb?r^EfwHTG6iwfE%p0oD;Am0EiIaOTV*3`DefOLD0PB19Y5b3Vn2nxr&C$e<6YBXf zLHNf)0RScqT(VoL0%$d7!&Nf9gA6_7Bw4dQ+md?3OD&_8U<8ON2v>!LdPg3n5B<|S z`u?W91@Ldr4@@ese$v`elJ9`-oYXAHum`ESw%lX2x|HCp${va&X0WP)EpA@FPfEvT z@MN^tp%3Egk9MqN=acGwPUV)lQP~%=?qfU${C1T&KGNOZcwU*ZFdO3kUiO=Etd-sJ zw?}^3mDhEAIIcf+-*a<`QVP;ybYve?P8B-FO3M6ol(|%a$F1^-eEeJhU!A;;x=%}NWK)bu~sOc@;$d!b>r;7=d-lkj#1FHPbD_vNV2^% zTkCrFn%{*>OWm-2?5cFkVt!`d4>=3Bzp`?i61Pk~Ht?6^NhT`G*a>M@91KMhwDhed zY9Kbr$UIAcuWUSll)eYLewLj4B-dEd`TZTOIe*C8M?++4bIsQ4Ty@DhYa5L%T3gk6 z!BMj9(HNyj?jv#DS(nh8cTq8cpUx!Py>oOw?XnK1bn|U7Y(wNA)f8Tn;wo;anhqh? zM)_QAs+x3{2J0a!nIC$<=*zj=Gl!hR+o@kuv)slFe-$3E!rXXnVx{_8hC zCQx1usYs=rtJ1PS@?)tR51Jdgl=D-PJbb4ist&phwCarlfpMq#@F;l3)5i^{%Gi;LYfhk*Qrz5or zL3(<3ZlPr6+%wgg2`zt50A?Za&-vJi^ZV>^{9`Oqx#~O@xo9X2xG3RJiYZa@Mpy-( zb<4rz7E4f8nxO`qS8$nIcQ$-Hx978#^nG&hbBgncgZKyCSOhJwGEPI~;tV!D$;}^i zPSV}ZT4}poESP~($`ufMU7b@-C&}TgdAfA@Q%bQ6xBNMM`RKLZ^XBF0n1KHGbZq$T zx4%7MYRUkud6`58>*afOtvOJ=QoHZ5g7J}~*0UtwW+jbBdj}y5)LQ(w*L!cO`4ygc z>l~%eL>m^ok@>LfonGKAm1Z~>RvbC-pP0}(p$s>d;?iW^O$v&w_cG_tdzEyP5$7Q_ zmbR6#NXmjt;EX`xGC#03nE(Z*;@=C5z0akuBpv*dhUm&!zkgcWiAlt_u{ll~N#ZUp zv2jM`Xqup9d|Ea*r%)rq1i|Mk?K~OZ7*-0=>ls4ySp)KO=^!(qrVs4ye;)pU zi@c%;PMWC0<5pYKaD~@0*tL{!CCd@-WpEela%A}g6#3h*L zWSP;F3A-edAGv?#`tofx9t@F|ho{`VHypZ8iJkPX0WhlI^`MKTX(u;Zv4Sxcnd{k< z*UU=EXl+wvh~ea(zjM3C8$1?Y4!u~q9TUTL4j1Q;8@YG@Ht2FooSN~fM1yr#nMz=z z?HaEkTrL&4bbxC6G<|{dy#9kX{c*3RAL$&ipU*Ve%X%owUA#HDiqxyhpd2;!5`c7D zA0`-E>B%K#%Db4ImoEF>N=w7u+2$U4c6sL7>Uh5@^cM0xf+qrrN%N-!g{B_7G|!Bx+NkH|GvF}S zB#3`=*)wkOIRw!wtKc+b5+VUmMV|sukl-8WDi+8x?;bveTTNJ$WoT;5%~umcnCam9 zUW3|`72aQrv01(+n;zVx7!(-MQl;^(s8mie(#iv85GkzDIqC5U*bGpOZzQJxK21r} zYdk%r`Gx-HrR>&YBTG_fjxXj7cbZUaE}Cd+)Pt^tw(mXkEO z(hfMWqxIEG!75cM z0mlrUBU5e!9xx!#bqMeoAJ4n*Pon{1(*Aqt27ylD|FHQUE7jZ_RwdUM(kfAM7Gkps zk}CbRIxs!-6Q-C6CaMeI_p`_KyG%Bb-U!kd@;urDom;x;6+_klzFX_wSgpHNMLM(_ z?V?!Fl9Puefg2iPvP`NbqFAhpJ%G$O?O~c~p7AT^XZyLGke>E; z{b5~p<^sDeE!$njSq9fU@T##AJ%!EbcuSK89g)d-(O4<|g+hbG7HgW|J>5fvS)K!L zvINOiA9c69#jM<{vdmPF6yRujRz5cfIweUBhH)~zt6*B@?348A2SaHvvDSOAt~ys= zHXb=U1~*FsIACEFgVPFII>cXfGdWe|ND{*$pr%s2D90d*El08)s~`rGC{l$nH;q)5 zvHcF)da;7#of@cYW^6vG9$6(7B%!$CryP<6?hMX44$3G&H%503wTA)mbR`3$r|rw3 zyG8ly$Ng5y84j;cyv4p<+MFt_{$#4F-Im9J55GJ>1>knTCvbt#9bIZjx|l>c6IPgI zFgCeTd{qWePVx>aw23qsI2C+r?hGHwZXfqJKSbld=C5XHd!VumXU5}Zj@C&kUdH)> z3c)GBR6wh`&QWv5<0m}0QCUOV0=wv$Y5Lk82Y^+4+e(mL9fI`oJnjClo+}5qpL;eQ z(zo~^w+=|nM2W$Yv`pW2-m#5c$RTz@C94QxY^4haoMGnLVRmA#T&3o7rtkQ1K#45) zarLzN>wf*`>e^gTX+t_jpG(WBl_zho=CC5hsQ^3DYAzjA*EPTwcmO4Xf{iP9mn5mM zC@?DE0XhG<;mv*hK*L4;t!*V`@LbSb@OAB*aa?oG%>*rTykK)c%jL?-1$b?T_ic-i zQWs)v$rc&9_nGH6%V);K#zt}glQ9X15lmJ&Edyk#OovP|eN6*K%MCS`z|yL@M0_Q5 zAAmDwpMUds|FU@)DDCL7N8ey3E3jg##(CItFOSxfH!-H@8U1TMAf@ zBFFT0-ZhU>>G0ha#qwEHoFipw7+2b`VcOZ&o- zfo7#awC@BdZmeSp(hfa%8!WX>CflG~QnkpuoDR%5r*|*Y6SfwgvKf4BDpp}64My5G z>CdjUYA%KrZOYZbg4YsFkq#_LW>1F_zm zy?JOqaeTR5J^gm}#;$D7-;f@-*JR=Rfo{I7hB{nd?J?o>qlW1PPL&%D71^pL;g^ya z)L+COWXKhOSWd;j_iI?}>)Sfs!o&8wxA@Bo+#9T1L4XFnq#7Kg^zY?Fpn=c@7=fh7 zP|HriYev|$BHcAr4n2ug48hKFr?Ir$d?A3LNyf|O*uAubcZF*M%mGgVj5QXzp^Y|j zIR&kkbx61dw3h-Lg0DV0WiGt&voAq#9!c=hh7RMP3GnBBl3eE)Wlcb|2f(k6e3?gMr5WGjR z2E~-R=w+93*9Omgll!zY{k{1P$gI6yY6Oo|@K4fE3Fx(Fm~*UO+oZukJn)o(Q!_&s z%cQg9Zgow-UhkaaAi+0Zg!EF0B$8zeXRnp(t*Y#yq8VOG1N;JTlu^*Z zG@T;%DW^N~ug80wk9gdZP7;%;x$@>)NBZqUsq+djOfQu&cTLaqziEs?_kUA;SHH2Y+affrFT4YL7nHlPL=)n5cl>ef2syyCXJmU2x;+-vd z9q2Toz02eC1cxO>WV*wHnIT)qz$=5%MpsB?!K9(np0*@Yn!#3p0dod=?;&%Z#^-M% zsJ{GqSit@8k6v5$ZgNPuS$Y~hWNbvMc zNzfjK1y*K86y9Xu-`nwAXBqFgT^)aTysLOBob%^*!!CbOkuw2@HXEf2uB1@_y|{#9 zhh>t~OZWwLfIJ>VK#)MOPRjGy5Mh_5Ge@TsVspoR^~HZZPVJB#dA@QDH@5}L-y17p zFJ+g}G0Z@$GQ>0*sgNcaaSDYXZY4^$fuIOR8z9{TC4FAw|O*yoBJ>H2Z7 z$uP|PY>N&*+RsSP>hWPUmY(HrdN`O2cAvpzEYZ*=zZFwS^^`4eBIGAPqbMsdp`0Dq zrl)|t?nZ#s9HtoB;$FDnrO&-df4!)BV4W2Ff-2dhN-a6!qOdF{!&yL#lc0ScCBLAS z?qaPp49}i?_V45SHMjf-&u*KlA-68G$aIA2hJpZ;P{iuLo4Yddf#x*CIn<+VFWIdN2SzX`F|5rEX$3=iQY2AjYDjY~>b376Mc zIHt}(3xEuu4QUo^&#u5NUp(M}W1O+9BWPdl%|LHBGEld-rspcT#dI zP_l#t_ql6{*%d5b{*aAvi_hs2&Y^4`ci=NNppFb6i3 z8Ln`+miV&2O8LC3g~fF)IZZNUu(He`G$26MXzJi|No*6>EnUHi>rU-=mN~Jco>i}kc|Yw6>t$T1j1OF5L_rb7!Sm~@OqU!mvQaegjEPX1mQPfyl+KDIExga@<4 z5rkUf2M?8$i|v9W69ahK{fc}=|Xw&zX$8tGB=N1`q(7~!< zHLw7-jg^lj!&ieRWiH;-mOUJj<}8(o-9U3a+*#D2g8(`+msrXE*^rEIY2?a9&yz3<=Lc?%m zz;Xkl0IR6rg{4bj>_86W;q72orb|BG_HhuY*Pq^~=3#knKi)l+-RgI~YrNT=z5o2% z>P|wM+ez+v@_TktReZPQ&F`yr&Ca|%$i*D`ym)KPn3SiWU~B-!VyH>(?Vyt?vv8dv zHC85O*E95z+#vlD*%D{$r*AP&duM-6%15`G^YZ?BYOCib-kk@}Ch~b^bYZ)0zwypo zoZI;^eI)E4&-1YUhW|c4yo{A%c{mi{I=RCAJ8Gp_%fO%!fQ@1x>j+3$(v5?ZAUU{= zs#3Gc04jwBY{Ap{<3pYM{2Eu~sSGpQ?*1B=_=#wS#Y;Z0s@P>}qN+duQAKYw*V225 zDMRC10X*$F_}Gff0IDjR>XRm?WQ5tY8GfS8dPVo!AKM*2-a{gAaJ$uU02$=)uCHI7 zXlP>o8kap^`rY@k`t9W+c6+$d8O2}Uic79H5SaWY#(TVcoKPndi&2v0SHP$l)DnO{ zkS9Yb4Rg1oO-(jF1B%(MLj{KmE_)MZau|kNuenvWm;1JUc;Jb#0p}FW3;yXE=h`74%Q7xtFZRinBjgr0AN}I zPRo*y6oa;{lXe-$pIxJtTP_CpeR7w^NByr~PS^C-M&RT<>AUj!9#r%96&>qXF7|l0 zdV&gXKD(EfZ7OcDq1hOnYcLgP9nf5|nZ4E-KwY_}RhQJw+9PLQgHvRDl~N2Zmm|$I zGu^RlBzEG&j>CA4miv5L{@CCvufwlDvq{52++ONBNxo-`a;Z{ z?NaX;n>HQhCMwrOc{J%lJwlaQcgbXDIk{qLAv<@KFxs43ucERvW z3aY1B!6k!YJ6KZfY%-;5J>1q5*Mr1DGD#m06DnL+t4;DS^<4PiY$tBL25ob54W_K6 zDs6fRycKTKfm`+5urf&-=*@J+3|UDObq9v!UYwT-*^l^uDKU8S1>0df!#x(u{r;`% z+__trM<<^Xk;Cwxe_&&MQ=p^ZG?JAbbhZUR#HAP#^lq3T>z#9nRf?)midjXHcs(n^ zH)m@8t}EHJ9-S@wP2b+bgP+^Ze@5XLcC>JH{JBjw@7Jo>u87(|7e~TlX8`x%Ry^UF>WB{S7 z3`}M-L(A}1b?|O7BnEOTlUad;_xba6C_tWy(!A`}t?%P0XHA49hvn4CeP;ECM5CoA z!*hO;)ngWvDfXlyFVWfQU7oC zfSoo{`+qGZS@P00{Y0)!{|z%yfaN`uNMl+_|+6QTLH{-C*8{bhlA8H=tr4*-B)t2rBxZS(5V&Y z*u~&ohO-eqsC6dCRol6Yh1F&ytzM?iZt43K2gVmc_B5p&WEMT1D|s=5#Y$aujKvzT z#SHeB0(h=9zJl78K$7V&-KX`W#Ns_ec%m8?cM^-&_y?y1W~B;K$j*}r5R{VlA?ZDrd4kMnV_MBbve5Pg7Ys>Mm{aYT0IDQZ9XT#Y6pS@(FMSg37Qr zsRs$p=pKPtI)kF;*a2w(Y^5h)&Kejz3av16nO*;0aV=9*&D1S2M;x4N0m#ZVSm&)T zN$SH|Pcdq#TFkcf?k)UtZJFStRg=Dk60&5-;wUlXvm^Z1enY+t=3|e=P6micgPFSnTAAIb-p5%J%U3bEp$G>jD>wOo3F17P1?P}q zQJ{^b%3Me5pp-^##)=6Y!eR|P58&CfqROM+owP(6Uhn>*Dl-Rm0;H=AlWjaL2qK+!r+@rU}88X&_0D#y_ z;S1cp>LpA|-`^}0em#eq_rVF0r3L@pqY*x98WgVt1^dC(rsAZ0$2nt~YqeA*8JJy) z08)fT`8L-FXF`x}oqm|2FCOOs%#0`1C1oA8Zr#f`0S1m?3Egu^%KMX{FUJLKO?CTBRCky|c>Jo0hro~%k23KrUxyA0`$|0hfy^IejL6Y`t zZK2c>LbhHefR(PY1J>@n62T!(pXfPR{8(|_zqjkBu_`WK`C)KOK#f5MOV&$g5GQd? z1OLuary`34!M3es$Z<}oSHKnp%;_C=W*TaW8(eRW5$;2;(;+kKq(9^t$sXLgFGu+><*dNusxnPMR7g_LDFW8FWC4?-?}56R8~)nvwtpP*6UHOfaB{ovXRnWa zsrIi=MlS5%XvcbUMCm{|3t)&;E^J9Mgu|%Uk{Zctq3i`c40F zaB@n=q{4G;dCPS?nK~qxaa*}Op7g4HdvE*O>RB4L4sR1+<1wS-Z3NNix`sWpqe&uFeHpgu_;~8QwAAjfbfY8W_ z;^h;zdiceA(lX9dh7o~8p1d(pbo=Gkm(BjY>%*T;5C7KIE5m5{UZwpjV5basOq5&;Wq`$Ntc4|`5uP%(cx80;`)29&&i8$@bj=W5BuNiHp8~Cc$DFC) z9%D@S+mCeiJ2@XbSa~R;PT)6bYbf~*X&96nJE#-&n6%cR2M-JbhCmEBro2>zIB7*} zn-)Gw+A(11%tg^pxsTDY&3cT(W`AT8q59j~?;()`%e1zEi*z8@kfbGz(K<08hm{dO z9v|l>k=9^)%D{w?no1!`L%~n%;pH|=ip@K_bK);=Z3mhU&FErt3aS-R@~BO9Y+RL* z4k1AMn~yRQ&Y4tv%)Q8rSYlrjOvl{blph=3FaK>+_?G|k-fW8#e|PJZyVhE);B~rb zU8zRbv&qmcqn-D*O7SU!1-3M_XCx|GresMUjuJipEzKIK(v@ksb6Dx!v5Yd6a%>wv z0Xg>L%T8(P@WXke>eaEsnBKSNm+g#~pY`fvuYKaH-}A%#`231*e$FZH{#gI{1_t&n z`l?u-;I3N>CrGxH1X!y=2N4HBb*y7oE~cieW>Z;aHx>i7ZnZAnsxBS)!T={g*uT)Y zq+ID{`#1Q*$6fPSiYMoZeoT%(0n$%J{ms#T7*ZcAPhR7&9|of*^7mfJ)A>JlYcY~e zkx6^Up&IEemVEA6r4+bRYPB@$QftL{e61$$WeAMQ02Hn>MLnHg^PU)uq~=6UPQ+lE zARX@m#$f)5?rC*Q&>vp@x!!(Xz3)j5X6)w8r3`b|Wa3vayzK%l|H0Hwk~OXYzL(p#hFl5>_!|C)oX)n+LY zs!6A4WAoPbmc6uuz3NW?Ufkn~x1Hh?k1y@FtG;*WXMQ2X2iYChG=9?WjzT44V|6f1 z4}ZBS5;Je1MjC~)ei;Z-IAn%RS+Fm^{`sf4$DtN?c>ULQ_bz=Mobs)OO(xt_0qsC! zo>i>GO0hNBtLnW0q-7s~;mEjbi;a*qZG3b#1iYGQ-}95g_6h0wNa*ZyfbUcSKji2p z!bHb7&`H|YX(0Ln&3!l~^J%wuIO$u5K0u|V+KW(LO z%0!<$%vP~hOaD?GW-4U=PT=;l!0NOZ{6jJ|iodRQ$AsVy2VXx6+pb1V-#%Pzjwba= z+;q}>z9vq51nr%M8?Iu&i?He$F#Kqs^=e@Cvq98V-Imq%xYAcGr8gm3PGjE%Uf*7e zCr=`*lE-lek=vC6d>hlax1UZ3-B}Bb88&wsLj6GKbkw)~7RDn0PZI&tkgutw=yZ45 zQh>Wr#iZVivXq@9Dz;il<`0rB-Ezs*%;hmGebMs|xuiKb|JtnbnCkms?=-q}c>n&O z?7rWZdO$xP4Ktk7{eD31-j53$RQ%=$yQz}M6jNISp$o?3QUQ9vAOdAJ%7OSxA+Sm= zB*`REx^-nuZ&9Z_)1LHW0(MH`+PQUp=U{(2&i*C~x87K3)KAf8lUw2DbOb2ysWDJM z)kOteMY(loRVz}`@Ln6~QL2rRI_v3f%_llNp-xxa(CfOPmH+vJv4ZxnIuCM%v+hTMuHu>UgHZ}$jmlEwT@3BaS5o>CdTKTH>^H`i!4yIrF zE!#QP@>%OYhXt;LI9@g8RBk`W2=~fy^YEPoT+7y!M>@{LGl_Pe7rRd>iUxI%jG=hN) z()hE2u|bo`&o>QuYsnx>KMONF$a8G=TRA+YbVGaO!>6Y$B-YP&e?Ud|AG|!Rqz>fn zSd@Isce=Kd7fI=M`8q+FC;^eGRW~AOq|_mkGMHwVFVVWRi~!X2woytPY)+{G9A{WV zZ#&Q2{lDI*{@v@wh0=uPk>ST~CGq3Y>U|*UC3E!t;q=3IF8$Am+Q&V@a^bMQ4UxxG zW)eJ~neY8Oqe1O-?e*LkA9_&rPCy;0jXRAIDT< zd8;R5aS0#~mXBQq(U0tUFd6QcOl}gXbL2wC8da&O6}V;@*X2@4n(ACF&IA8BsO3fO zl2lzTO_!umFAQ@rGEz%L_IUURwC_C7sPta0GOe#_*lQ;P0$7}&HmXu^#pmgJ{o@LT zmdGcC2j}mv;2OUjW7CIEUejxiZw$?nCVs{K{kLB+C(jd4a_cSirbw3_)3Hb0dsUm# zHWGHS=Ui%t()zMALm{iug0xrj*+NK4tz7(cbnU}i_o0nA{{35TMJ}-@n|tatH=SF~ zKv1!o-WpqF8xSdAlne`L#XHB+&ZtQpZ~}X(_PTU^Y{3_8oMoL$SKgbT96^r)5Rmi` z=z!xI$6rFwkbnpikh@G{trlt)@kfcyG~jfSX8Uvf;-{40=F%(UgycR#q*hCxAniNY zDpp2~t8>O>DOi>SF-W-8SaFed;P^_gVfsS+9rgaO8lQjt)?49JmJ!H7Cl{q3uaEFl zQJNiFZ<8|C`BqCdErm$A*G8OMcU&aJ&e3!(&IhGeG9@ocY17gx45`$tm5xxs(1L+$ z3dt#UE!mbO!J$jHyqMssZI+0IrJ%WquOm3&Ik}be&WopCy8f`eN%$ZocS4HTNqsH!56_ira z*f$9AgsYV?mxY|trT}5&j4zm=i_Di*&`RIhbg*cut<3fE(35-NTNm>=C4%OtN*;ZY z3FI78(MC5QJC^xLVB%T%VW6=LzF{lF%*^l^q;xtsd$>QIUEZE*pRUc`zz0OvsKiWB zLqoC*Cj=@44m4fX)tc>zjOi*$3`W|NYWO;3tqm47M9*BRJ^lE8pjPLxy0-oOK-o(B zzCKQ#I2L7|gc$B65F}0HWo*5# zKH5aotIaWc1vWs@25)E5tv@FP`&8BYA#i#uBtD$${UO41^7{Tj5QmwbXY7_AAIo9B z>3LfIndqD}-}vpv@9&7qM}=eeeTBceSzp5htEZPg?>6O+iQ@Jfw8A3TB-?EDmP&A~ z`v}aJjET_RB%|9~5UdcHTkJ7PbKxr8#btC3<<_#=k1BoZbVXAvrN_r5%j$G*9*dg)#Yz$P|lj;^LC z{d_&PTx-gSdQZThN(ckVWNbhJ;P4FkEv=v}RD*Mv?x(VC>u3J`hLNqdJH;RE3kz3x7cq(>&sph+FX@4acPZ$s$+bv!b< zdmZU>T86X#`EL7Cw{5ekR~g0Tw^87>x?UT+oPVU%Nq>F>d)RJ=H zB25@n44In+v5(ym`_gCV`(N%bvt{mR4qq5`&BcHJyL_bgxOQRlSSd9{;6TzM7R;f- z8=I71NK%BEX|7gyX;{$`68MI+)YLw~u0yEPOfQf2TI_ZWegEa|`@_Zb$)UH^!T^=K zA_vV2K_W6KnZQ+v=0;EoS4!%}<9LI#*zoXLUW`xUXrRA&rcnE|^lfH^k7b9)_y^bp zX)lq?ViH`i>qT|Y3U~rtIC>AOWAF_n;TTrl+sx9AD@FKe3NTjF8I}8-RB4F&xW7f3 zU*%@?=i|%6ZuR`|V|?YdJb5fC5L=}K=U#w6hIW=9y0vJl;m*6uur1Z88exE|i$?n9 zSz+%{7d6x8^`!6bHON=~cwVnwZ#H+|H(X$nH+-cN<|Ezt%hq&D2Q(-FA}}jB9j4wK z$uJ9LbaJ3rIT$B>K$&|DRH~8MV*oV`r6vO=+GYkSz4;3Z8T7m{>kZJH z_}RivCX0bonkwxCL)F@fy1Gzzz?Fn7)VcPil&Zx!X$m^yRE~)M{Fw6c!W*^3 z)|52MRT>V_ux=K|0|wN9nKU49lJ43S0n}JkX|AbkQ&K=kVt{&r-sZXa%JT5{-(Gk-9-*QZ z?`qX!bFRasfwn3ogUwayl26x@(GA`%Cy8w67G;=M(*W7oC9<7j@?K4QiN|9yq44;! zy=Ep>PpiM~*MF{Pd|GMo-oUz-7>a~%dsGfa5}HJCj=@XYHc1KrZnM%YxF*}G0^WZC!g z6|>Ik+4REw@---#T`Dci z-~zabAdgtVf65k9`tt4TujjEO?0)VS$#sndg09-6@&M4LP;Fq%Kd^Lm)#53|B6j+k zed-!a2-eU)SIqhjJ3RWQcZ5z(KXmO;hL}hadT%2{LE4JR_|bjvED+Ehi9vGomepRo z(Qsh=tssZce%h~J4*tXL^q2m%v-O#FU!NdMIzWbiNk9%uaBTuowxK|pk%WTx7J%N= z)xbA186$<{d0zajk#5pxPWO?hYb<1TcicS_x%ioRiCVLnKFwh9>c_Y$L~0IU0&!1V;y6jC=*{CtQNvw`g3ZeVJ3k8=C2R<0*oB*BE zFyH`_bcYPmCr{;Fa$DHr^tA6;k!^dZ$FsmQqMG-Y)WA4HLvqYfs2r7i%M{aARjWmF ziU;S;nkPNd`ExBuFtl>c&22QNt$WyPjP+Q&eSTDwch8h_W)Mz04H7~r30t&?0 zYo?1Z-Nz2LH=;;HGU0iHGe{dPb1R1piGz=HVZ2c0Q$qgwy_c%|1+RY98@Qma{9<~x z;NWW&G^b?@@AZ`ZDW88Iw+}BTDz7s4H>WKae`>pVey*)I{CZMdeCZLd+S}eUvAu6`%N^|Q?kKzab^!JYW*Q^q3VOlo zqM5J`Y0@);quM<0)qS8IBGP3lzLab{0qb+vX{#PCd z7!L8#S)Y*Wr{tfU!jR4IKbo;oH3eOCoB(*wCA40me;Z;TJ#XairN86;1v z3ZTYnux64A!E~Ps7XLx&8tipQ?zTIVjE;ebXcEPi)kl0W-Fh45M_|Z3zWnrf`=9fb zmiAY-z-z(#peBv=w%S7}gRp5l;HCKr29J#rra0i>+9DRE(4Ya+qQQ*yLXT5JITjLK z80W)6$JkxLNp|`&A|gPyX%5BGYmF7pGejfc<4G_BBiSn1fy)AbI8Yn)IZv{kwsnPb znlBc|-g^f&V1x)H=m7&2Of5H>n@^UXfEuWza&A|i|Z9Dmsq)2^+4Dy zg}x4!mj_9F)^YdOr^oNq;{LomV}Cie_K0V(w;uore~ItV11G z?AXCZ7_C5O)6m)u=^Ff&=QrT+VT>*c-apLt;(k_7I``b!dvhXUPUDcQ1Ky(naF&W3 z5NA!Jv6DFA8;+nEsT(;3G@)T|Re7uD{RXo=FIPQLtbTkNpY9*@^S9T2?D?%+c#G#C z@Z5KO@1>5HHIvJ|+2$g<|Em{XK_L7Nu)@N!`B^}hOab*DajFrv^2G?S^|UMo^~0mOjCaR>Ca_GgY^?Otj#hg%!}YA*MkJTbAw2#1Aoss5 zZjtx*F*CNsqR4c}?MMZ&Sxm17JV_Xf6Y#g-r_nSff^jpUyFwpY@OQ-M^#iea`5${(G zl>v}zd$4_U@0x<^Z+c)p{w4^y|9XDtpBd!-)pXqg<&B=H@#RMYo$c6ZQG5y@y5tHW zH%BX4AVc6h()Ok^1%%Qm8_i*F2kFnO#JEDKVTXD+q!=La&bAH?k-E@QTpNOH z0lO2IN+foIsKsbXDQ>n@>_k=p zLeM@Z;6K2@RJC5G74V-rOR5pjB1@}Bp}Y0edwb8&cIi5DdyM*_$K+mskmN!FWs=-(}p<22O`J0f;8J`<)(>lHyC8RLmQ#G)2x_gZIA6)R&z<0qRV zSfO60_1P;cyt!K29PCw_yg=yK=AnQ!Xa{?cvQ7?8MsPU#f1u;}N5=c()7KxkR$Da8 zS-^5aRoc!>8=9BUdg-P0>W3CrgH0-I@xin>lLe6*>-j3x3#T)|UZCsy=lVtq>guRn zWV5L)AvRZ)!N_h8r_q4U*#5NYjrk+OIBJNmnREIU2U;V(8-CEFnf0xDDomuJ;>*J5}f*kQwnKC14(ZXvHg3wlD~!cGeKtZ|Fh9dB2?U zdRyzt8{T?nXEjdC)}Xx)0ZVWR_?=?CV0KI!o@#m}c;6jz{uGetUY!&R_6Jgz=~`{0 zPb_@#TkmYGF6%tv?|n@(inxh@IZQAkmjLWyYDYLM)w-dLt?AyYA`y1!0hHPgtJb8d z?DpEAV~eV@%~$uNU!QK}^QUi53tG1OSX;*4!R~Q0V!4`{Vx|MLi=Zk|EoH51FGvX{ zmM%UYt{n`yi;0+#^|2mLg4q_^RrW{eZ!3 zY+@of=V-{@HX$NU!kXAwwy8BZGc1vvj@ASS9RX--%^t)X&tBaAYGA&Iu|q;m2l>E5 zzoO9SPpHVU;r-~Rel$IYur1xClaS3R!{oOq(g z->8J~e6liuN$=jC_P!b>-2S!F6ZPig_Yr9583TT4cb~D-hkn5nyr{Chk!X9r)&2hd zj{`WQ{>yps)9zKSMEW#|8rPFkl-ugi8FoZK_X0R9<&2}q~E=cIU@iT8AxLzcP-P-Lz8aRjxauzRzF}w0U>c1 z7Mr+(iWfa>%S>*)%H*60d<_4e{L$#W=#!k|dijWQJC9Z%eV_A&6{LrKwP zWV~iX2l5*b%P>a4ml!D5n+9LBIc~?Xhhcl9kBh6Xu={cBaEkmET|m++$kZ0JP6NmR zrN2cpd(fka6l5y}rLh1NL}Lz{N=0tx?WIdjbv<(O@82$dd6fuxX%0D7z&xM0ALr?0 zKfg}g$rPT*IRB8Dd9!2Q#L0^jz2ePxm6>+AC*#B6zsLVkUN+ zg0ay*c?{L!JoeC#+02Hz_YZNG^+Kb6c))Nn+gG2UQZW&sx6!ccVzYLdW&pCLx{{DQ zG&(=D@~9B8r*IZQcBq!v5oyN;)O=n!TLXN0`1D1e&YQ{hKO8i)L+ul|!y*&NOcbfI z7_78Ii*1cIfyD<2-a5t=Z)+%%2?Nn+!$AAx=|8<2QqDej?a^i*GIi~YWVJ|pSlf-Y z?YV&!sB49?aGi=PJ_0yL>17lOY1&(`w4zuorVZ~pqS;7t41x@3T#W_$7t zMBQ5D<#?ulI&zAaVLH3$ov>|u(xuWGtYn8-vbk66{J&M5iSd11+~L^W`(7I@egvt# zCQzeldGUtWRRZ>F4Az)BrUNs`ElQX?Cv@Jk+`m&-z^#+{lQPWt>n8wj1pbAa((#D% z_~tQGe*5X}xBVJzI>;0qll6<# z&9MROpge8Ovt8%GnC-ql3%r(3>sr$|27< zf z0b`8mvx~J=w=Mup0FW=mNQk*zNa0``|yFv){)IyS?BA2$EvfU z0r)RE4M-E3GC0hfbbBhH1=&y#It4nsYX{l(3(hN^cY69!L@q*#TR<_&ogj5z^^f_rHXnALkVmH}Kwy&B+?i zs}UaaGJ#_U2ohg35jz||w5**D!N+QXNFL5ZoPWvC&SNQcaOi_F62h`M+P$+r1^BX z#2duo7CRE$X$CJ7;AIx1AT~$l*=a-#LIC@R`}8yfMEf?yTo z0KhXSeKsf04a$tZIlPvNoB;GJCQ}RtUn0aR7H_w@bzOc`c3)if?|$`|%A{SMiEWk% zKx`f6x+OYxcuhS3RD%>)tVFy7hRlkQSXAN(TLA@ z$X#uX{!JszB59Pp%RE6{S)0p^#>*nl`=2Ruh@W7#HP;j(QeLrcHg*b`S|(5+(VtHn zy?RUIg^Ppz_P8w6io0hE0X{bcLB$t3@L*_cs6fDEv)4onu|a`b9c`e3L%dbfCR?Gj zTbE^Bmg*GB910Ylk2sFD86UjUe$aWe={rBVaJwrSde9SNgr;FRO-qj~|8sxbKG*8~P)Rwj_uiCB;SaDb0Y86er(CQIfB5y> z4x;=L_ukAQH&3|)+jy=$vR69Cl~mZ)3OqW8#b&Q!vBuR=^a6N=+NTwysrU7Z6t3?t z-!AUu^KN8A^``xA(*gzp%H&)PNHki&QWGV3b1Dt^qX)q0R-$W~2EwZgwKQwPWaoz8 zX#ITlQFvnsvW`HHAN=<9iI4ldKHpxZMRyUJ)4^zwi}sh$1IJL)EBn&JtUzj+Wx2;X za_g&|SO8^cvT6qcBcP1Tn4}`Arw#S<(|CRw@ZNL|=jV>nwa($APVWD_y?uDSMmgs7 z=PnFR(`lFd=|%5lHR03Wy!t%H;q>53PUmsDSn^~8E=88;WSg)lL*Z!? z_9a$km%g4i|0x>2leMk5eEOGO%r;>ni9_ z&A@*(s1(;zR-mio=|B5(;t&^xd{XYcuov39l(|KNp{6?OWTK!PAOLMwY7aLs+c6`k z{{RD=X@KqUV+sajMvdLnE%Bt23Xr!N7W?0z^5oF?!K>-mcoxxP#pB`?A?&qgTS_46 znT`pdBsFR>y(utaatMy*;s{;^(66mx(I++;g^v(Hv|t<&;ae3D6?31wM>=l}n=61< z%nnxN*t$@i0Q#~(Vq@(+yTM;rYqtw-z5kWNK=1^Y08NZmN*EDveRSIg$hAQ#!&{MV zt9X+g+lsN-z)rHs*oYvMANDELhh@{n8<)S8;+TtKz|YY0?D*U7Umjzye1YYFa|V5O z9CAzVK81ta8Zx=>Mk{Lx!Qc>#rRIY+u=OSs%C)Ud(V6MaX>ry9vp;_m7#H6^hO2Mw z>7+-A0%7mW(_4CQo}PD?uBpYaToU`jbQ39FSDQp*l@53818Q@2KaxqohKL2lCSN8j z?*WF_QP5lIy={hbstft3{oBLs)AyVCfa&=2?%@)QKL6VmhxNPH;hNITtRv1Cni!WI zAa=LInaUtfPmeZzNr-(LgM0ypk` z2UaSD9)LV3hpibIT`lmVrLm}wph19JV4WpkXJzU#4Rwun1?(W&67u<(+>D_DBrop3 z_OD_Jz%QWG2p4%op6vp-3=lwLL(VQbBNPLliAG)3niyH=ZzIV;v?;D%?~d-~l`US<9qD3o>jTq=2kJ#a_;A^p}IGmB+pwRw|;d zVN+q<(hoWD623OktpG%Rdahn;>a4D_lW({_qsB_h&QO6G00+87I6{Cz!c_Je1g$e{ z)onehJoMi_(adp0xPQZ%{Nds5&zqwI&;PuBeO%P1=jlf@<1A6^>Jhu+K>4||OT>2iwEhZ(X3Yb3_~025y_txB0QI1$;_DE5XwOJYWz8U6v%B6%!H%N;XhY^$ z4nAjOp*+u{%?KqI?SO?}HS)RuA(X%_9} zV!J2S9%e=?bC6qrI&DA>8l6i;j3TFmDt)Uu?U){xIC^iKgAlJg>&u9jTxPB}{5#uP zBuGUZO}3aG8y#|kwHiu`K&|R5qP^Dj!;3RTkdK`Vp8V4FhaJEWOYC4zom~^L&Yo#u z0~XqRd=GtQMm{_BjCm60@u1)?k2X22GFF-wWk zQcAR#0zQu!nGYki$mBlKv_Qtd>mOTfx&Q2x{{-gza1dU!}E`nf0gI)F*_8j`$u%UW3_<2*;agOL-*Z+UM-2U%-lkuSUOH8pl z^~4%h^_US_2^5!oCaUuamh8P@{Wm+b3qI64ZzG3RYyqD!cPg>2pO);h7*1}@&&#HF zS5)?a;`|}o*o8N-7$>!>rj1&#NPzlzf@Hi=*1=e75Cw(hkK%Z~wk}b6_weQX>i>0- z@L1t=gjKzI7cZkB3oS*NW+Ng(MN&o{s8}1JBnOgc{1`4% z<)4-H@0OH2#&QH_KU`;id&ae|*Q@rq!hHS4^?z^Yd^wIhaKL{Z>F=9w9jV~TrdzGe z;M^X)K%WaPF1Fb&f9LA2rLf2WtTWe6#5^hsiwEFzL&`PjVD@B%XDW5toWMXGk0T;w z{IiDjJJ}%*pKrhZ`me*;{w}H+$C>#qUp{|YUe@m4z=`-XT5=e4X>9g(R1~JlQlEs1 z8H~fV7AwpCk>p0TX-ug)rOgrSI=fg~a+^DrC!mi&0XIO+9QK8&`AzJn$1iuwa@O5B z*TMga!XSf!Ax6xc)B03%XkO`{8DmTaxjZCad|+S>r#X}z$1y>U_k)9=4cl>@gv z7Qq}PbH?qjdPyyXO3Db|M?rw|bi6nBncyb^ZnB!%?U2CyWh^eFzpqZ#tl-Y;}tY7rg?jXxYZMgh6UCd zWsV#{ZzGb6@2yP0P%qv%Ka+Pf8sA!=x4(Xs&o{r_-Ti*^@|0&|+27hZrxe)d(26FD zz)a27!COmY&w%LcEq#tbD;m#`l;Z=L(w<$xdfFfi?@;jDFXKkWXMOPJ8`A%>d(Ve= zMpBPLV_}!AP}PJ5Ffy<*T_)*4>fw7bAU=Z>1irTeR!BU)a6G|4n3&NHt0AsmK)x2h{!{$FJrn!C_c*Qb9Yap3 zl}xSG8eDBfV&NUFP*!9h{Da2!JZ70C0%6!Oa;cPjyzWdqYZRTE3-0c#lC7&s@fz9A$f>#DhZU6$;=Na=eDvaCIY%u6 zRJghk6HEQ8P~CV4RrEWMWv2>JObwK#dipBmIK0M8N~@s|;S)DX_sdjVehWM0Sd{&DpA zC(qlpc5mlTV+6u~L^yVB)HYkowi*qt^Y&pTz%mNRfcFi)f5f}26n~>X`E?GY4`S_{ z0iB04HS@KnxIveM}m3j2P62A9(m0OX^|)Z+qJW z-3J%J^QYgN(PzXF*e_AO2q`xmTL_M#=tAqs8tKr7hvn*ob19a z);r}WPY*m}=)W&`-R{f+=qI^STB?AtX_z@2uf#2t*GsClQcgBH0ARLlxX9z`qQfPn zUiDUEDxM0|bVbl|@5difz=u4*XdHFA33Tdtmoln4j4LKSxCx%76QO{o8nB8|=m$Pq z*sS$?dRQpWxwOAtF;cF@AEfSwxWrWUy3;AkFGx4tNgixJx} z0U(FngjqLKF3$&AF16O3et7v}_wIfQJvG$=CN3`27(vSeU}8de7V)`sfUa4bbilBj zumh`&Za9~XvIcVf*8c;L`m{#)CzA8_)a<_$r1AQbmE=a%=2)_h0Eo#(4r^M~R0@8= z^8u`t5TzQR74kEr>6F%66?S7A$EJn1g-BN*8@^MOQG1WM`4X_zEv*lHER#o_*-b$p zW6KL0Ij|Knr9pEWD&E!2!@w7}4?N13^Sgh*ll{$Sy?4|Dc$(gF^(~e)f&4-l6T=ueamn`u~gEKO0&sLFD@#pGyan>(5HH&e)tS z8{LMp)tiVMi={lPAk@WPN5_=#lvHAO_(KYs53cB-U3o*`=>GoG8yA&Ib@m)+OdJXwFq@k1WP3f>emW2rvb#(dy4K zkq;VyoR8gKp@P4}X9;Z^E3gv4?J&jnEFlIdu{U7%Fny*)RHF7#LCTw@1E7GOZmF{F zF!CQ#35&|8-R%khW) zlgrp1Cv#foXP2g}yPGLGI+{(IFe6V%%abMTmWBjbFnj^c2`$D@@p|L2sD!%gE0vS+1a>H`wk;De!>E8rXLSQf9<~{!>o?+}l*pwiILv+k<$q3Sv?5m78{* zJG?Q1UP{I;dP0v4z(3m-`$L{>k%M-xLnXD1t@C8WnQpRJi#6&ajZ`XTON3{Grgdsu zhz$I(fGL9df*)?koxDE(JL+p^pS${CmDXA`1j)#%U|T5+U}wRSMh3<|#-^TnjdObB z<=S!$c3^?WM7_NcuM&xVV?W2u(}ONOWA?^fbkahi0dfu?T3)b~V;F#;UTr8Y2%CI8 z(rk9BeJ0>To^y@{nDKxW`!QbsF5PruU-b$?!;3kP$&&EjEqp#M;(J&zXx%6E#wQ% zuCM1eulMv7Zoaaw-*0|LT#(-v;d*b}=s*Km033{=>?cB1kC+5M47Cu`$b?QoS#Oxt z&#sL|dcek33rN(4R@K9|ul=_RMwcAnQW<@J4%~}(-kac@?GBX3Mw@tz?o*H1kqauK zRRv%~R*DXJh+--T(g0zjrj3(<+sFIQ@_pR(wJrLt6)wk4fc9AymNom|KQ#@=t#ML= zgvD*F#wm!z3LeDD0&50Ht^qp$H<(31evq*XaHq$A+j zM*}csMU)w>mu}pMy--DC#+bp5n4PTSG6yIPu}s@cLPB1zAb5WIPw$7>H=n!qg^8WM z_a@z2J5`X33WBSx1|T9izRKpuECm$DgWAu|DmW~0%f=XYa4%aQ#i8Z&tjco#(^o*h zWT+-U^I}WfiwAXN-6K#ck1_*8ycCT>~~BE$fmMVZtEU4GoIjv@W1>rsMhEV2|3H%XqN+DJMG|M{Z4m>kgPr6E3;MH^me4z z9t1KP(7FP|I`;M>*VQ(%t=*VmYa|YAJrC_6hfxDad9I}c5TvTAeXFkh#f|ekd4=6A z#mC|LRzO0!tyGYTZiHM1#XJFH5W|_;LPRtlYHV*ZC$06Ow6IOTo z_Of?gzWoj_U;gs>)cD;?i$F~~;Fxxn{)QA&$$|Dlto$!_W@CHpQ3Zj!$D$_Y>6?^3 z@ovj?%q-r_)?x`Xy>#`j!QR76Dr8 zblGBSm;t;F@aB5#V!F7>iqCcHae2c3#|fAc#uEA z9Owy#fMceX)Fk_wtPO1|f&6***SdW2?mDU}-tw9k4<545NYAt^#)?Qse-|0P`{B%# zk(1_0qpwO|V5<6TNM3xz^N`}1U(bpc>6pHIREP;Fu~?H*Lkdh0I|HFGQXneGWT@zr zT6W2s2qYshwvsg2kq-_pTd#;k8}fKwY*zL!k;UVLFsNqWRjXw|P7U6bn!Rb-%Rvmw zAkKxDY&zVk9CYh5mRa;$H{CBaCym==nT~hw!E4qb#SW|-WMglWm)csQG8ZNu|Ja5d zfgwt*fi7BDeZ|^2d`>K>d}kEL%9*Rns#b5^#Il)b&T3KxjEbQa?DDRqfzA{IT9`7+ z5j<^Pqf3fF+#U63XMATAU+*3e7jOS_lx|z7-RvE^;v5(Wz&OGZ4IzP#nlW_<@sBOO zMXHWrpJT+p_k)eXcq59TQIu|-fc-FY*x2_kgMHLrzu=`U$7-)pMb`knqRH4sbx!M$eFEhKy!VlQg9w1L$H0b}%^|!|A6azv?B5UV5q4u`ov*Ya z6>F$d_tjwMSgfX&fGKmAhCorNvyvh*zr^kxvWwju#*2*fwN-74_LhdYf`|8@06F84 z3zido-mrXZ)>KT!LxbD1X_{q^qm1E-04i?|;NiSZChmT2G-slrHr=L?NxIiboi^(P zv+Z3f&SB>sI(jhE#eln0=-QVieo(UNk+=Lx@Wy3Rz+2Zn?uRMQj&IeqBr~O zfHlZT8ols(NU7!OqblHiN~{h77_|W;P(XaWm*m30s|O!wTlE7R!l&DF17Hqd?+b^9 zCj>Wl2KaKe=3I9}6{!@P8t*nptS!#j9bSBiRpK-w)X?IHl!4khOFER6pIsF1pIX4M zu#0?G?am@+WS13R6gJ9%WzilG6P_yKaR%2)A!hdE&la3PyjwqmIW67`zZ?%6PW;}^ z#k^icl@2#+Z%xiN7U(N=zxItQ$&N||0!rfIn5LJGw3<4PV!$zYorK+cIQVUuCDe*P z#nGh?tkW&f^!(JHj;>Yb@p5;Ejz*m!?+4HDI~#8&-7u#s9$86-1GyNnSp%|XV{p{M z6C2e5r&|-Zw%DKY`S)>qUP!rrQ!z~sObtBpU?Xkqae$8k9t53ZsMygBUWiVuA%e>0 zkpN+l&D2X&($uwThF3-X=lk6E1?vBlujhu4`+Hsh-{qW)=&pW13k%0eL-9xG=WIg}nuK)Bof5Zyqr@kH6hL-P}LuAGiAF z?gm1A(*6BlutULM3xMc*7Z&HYJb*z8kYfY^28*1{O%_@Xuoudrgw1mZKKs2}#Q*0( z5nAn?3c$f#a0mv7ls;3Bb+}F-4&v)T6aEe9k?93%2-wAzJ*Q4p5w@)tt zk?}NL>NGqI*)KGK%*&AdFXrZ<-#Fvajj=aociTCPewH>%VH+&=Wto8YIvsT=Q>JWq zE5#!EWP_xWMjMq4^2nfoZX4wXZzmwycP8LR`dA?8-N$R#8w<)p+u0xvd(aQ-gDF4* zA*lub%0ltG2qcXkm59$^o6>+7&~zQnohaQcOWsdAEZ_Kl9u=_{-{$G%C1;ma57G1W z6L;@X@@at8x{(mmRkiYzo9Nxr{R|KZKZz|^1!qLS*X$H{3CJ*ms%Tz6xI3A*6SVPm z3YQ+L?>VEcO!D>DRVbaO-^p_~Ar~|-NT(x&z@p?7XvSbuLbk4KzL1Go?6Y*Wq}FZy zGT_I3&vHyRkKgX^?;b8Ra9#MF>(56=*yn9KdzPykb#w}^hg5tGb#H>dnhll2L z`^D`sbC(8+j)QK)BcYf$l?fxHOx-EJ@HkQT8H}18LopD6uDB!aB}A}z^+-S)tGbbr zuvj^o>)8BrPaoA0Pp9+T`_x%!6-s?Bk%g6wdlPeiM;`aD-1sL=3>TFr8+b+?Xs{n# zmK@@7CpF$KV7ow0=O=MX0Il*@G8p#eMKr;_NJc@1DEg+4eal)?&t=zjy)TAzElk|qu_C?KV~6!3w_rCA*5i?VNX3g- zqLT?^fuh3MIHRC!573&>__SS^l+!>W8sU52Gd}KY^K#Y8J9*YWvi3wsPr6$to+1Xq z!Ko1|M)5zMSM>D!XZubC37b>rh|mL*?Xy-QCXKG*H^=H}J7 zFDsCzqT#o*7gBwjhH6@A^>6!ZZF0T<=s%mr{w7?F+lqVklavCmSn_+`Go_=nRHCAe z930H@m$i9vW>z%~;)x5tf?eF9(WXatx|19rOCP#`#+k6Cu7x7azyrguwo2I~Y9O4= zg2=ig^i^i;t-s0pggvD~dEWsJ^fa_aeSZTcXSZ0Zk>3|nPCN<;6?zN?*JgYW&+0ah z*;y!k!viRE+kPccUp^PoIGVgDKm#s+D6d1M`gqKpVgNgIrd&`!Vzp4tHdAj`ptJy- z7(qC2&E~hUep4>O`EgvF&)dS;6vZK1q&y8h*QPI0ja; zHUzL5R!k^@!!FJ4$F{a9%Ba|N8^G5rU)67a4U9?K!6EfEjK;&-HLbfjwZlsEKZM6z zGl|+7G@B#S8S!TcN1kvquS{dm>c^i{D=+SvctEeMTO@9TnHm`h9J_k_J>(|W@pL0e zRCJ9*izLuxo4xV+S~K(riLmzrg^qG(y(x}YvRYHxE+pOA!ffdd8!l6JN7nwsF<(ZR z_ylAS+z_ml*@_j@Fc;zKss`SbQ+WaRxAZHawpA-=!;|xSdmVbV;fqkA!QD@VG_jA-YYR0!-`o=dJ-h`r4)C!&2M%=?|xU}FzPJvaP2z;&kyj~?n zrK~ttmytgAz22ckQdzVnF>*#VbSK#@!l}l2Y`P4x}{P;!>6|O z3PDVG98RXqeh8%t)aKfn9R=Zvr5fMwmK6`Aa&KhyXL4I;_c!|!KAsOXEy+X=m-phj zPn<|v_fN^*jW@1D-W@ksMDpN`uSZ1uYmXz@{1>-XZyvUDXF6Kwi{`qfAE@a>$0TAc z0MI}-movQH>JR4=df<#sqZZKWUnHR|C-;qSZ%hbuJ(s+9KGVm$yk5;Dx8KxNbUuxZ z@V#xVB)7lJRCGRrjGvIKx77BWqWoPuU8ShVR3oTPHnh_FtA&TARd9*1_0%e$h%LLK zY4%R7CgUU@+=chw_Z&ZeKyCKEzLXPk7&icD#H4(T^dRjjg3H8`XunK=s-L7fF+B`d zkp2xPExt;q)v!T?Kk@2p0x=%c$5hCIi*SQ`N#h42taC-FQiqk!>v9%iu2=7L3KoSJ z2kkp3KG^H*`I2nNaTTXk7b~7UwFO>GT;V1ieD>WT68r;k`f_BI;oh$LhMY!cZkbWil5%!oyqtXo^;K9w%f9DA4&KXt~2DI^6E z>qZJ0&lI)efMMvwO?hP==`TvrXhP6p>?6|2$>mAbxc{)N%GgVP>(Nn!;w0>FQkK2a z{_vgJ|1|*H9qt?Qq(Wt#rqSbZCnnYb%5=-tmdB~OO+`(w@$_xk_|jY% z`#Ohfb>uZoP=2PqQd`?=&BEnIs4freG3WC4VyJXSWyURXO?ZE?v~?x&+JKq?;`IX5 z?g{a3w%k%`*k|s~58%0#Z%QTc1hMO)%T_bjGEIK#2Br3kes{SuDgj;4NvD3RnVg48G(j;qFdqOd1HTOYOw22GvY zkxaS)vGO+Hf)h(ukSV1$PaY#>62qd6pobxhposmtLQ3G}1v{Sra&s_nPN5Z_i~eN# z)?`Ql1z*u(lz`({!sphweYe`mK23GF$ICXKGLQf_l%%;+b2S)rwGnExNPxCg z20*Ob43WL84r%rWq{OL6N(L7a1WT_YelYr2tQhvyAN|OK@u5fk)`xx(>XWnyDa1$c z`@CN`VATpG`IvYJq^nL;O0fV85Qe&y+orMg5Qn|A-9 z7Y|6*Tn29$0|pW1!YzVABZHo3F{rzQyC-lOXt@ql@&!~y%HkQ+dXzI?YSYNFS6Bis|uG@*7(ig>c7G+uYu@_LS! zMCBitn)?eCT1C=oLcJM-SwEmRsoW)q%x#1VnGb5w#@9Z=ocBw@jkVXR^5X z>@GWz3nx8rvx>Fz?aD_Lb7+K}>d@ux#$oOH(V*=SKi$T}+HtpBQb^Hv&&;oME_}}_ zAeJY;JIley#LCtoe+ZxH*UV+h^ehmczUbl$^+{9F)fjafM?OErYo|gOqC)|MiI(B@ z7+yLO>el)^Zo-idAWD4*76oqpm9xKgyIx4!M(#2ch0`Ic$O%*vcvz%K(`iGgyGGT+*% zNZjw>*A;%XegULl%B=xj+w-6CYS8OjV&>{)!7g6-5!z?g>{)FU$3R6Sc5}OelMaIC z)KlQYDpT9g2Zy6Wg~IB$Eb-}>Bjh^|?7Ns|DD@ATuaDuZXO?ia&jzmaW>Erj74GHs zSn$n)H@r#@t4*k2-?a_)jBdVK2MYZJE5>dV-;C1OOM;ocYu%Von|mdFS(+^!4~Juc z@A*8Sm=Rg2G0DmV(Zxw5MQAU@8XTaRjE0vQbLCtq-0~@%5Ak5n=t;4{hi}zu$vd*R z)1D8l)sn#s&W|0YJwYK%b_ky(ICLPrA9)=NID}6v9*y3Bc{>qOu zuO0P9PLhmTZ`U`kUvw_~GkBbozeCGKEgL-#UqWgWK;M2voht(v5-D%Rc z6`{P^qDNQGJ{cQdPQ6w2+>zv{k_v*#BSi@km@=KQarN~f)jtHu=mguv2-Ry}(PYN> z1T3`kyacb`9KuxbY7$lqf(ykLq*}NSgN1;P2I+0(q&F+%SC9# zX6HPyc;XpeCp~Oft_JnyJBAhfmb>P4`K#Xuq(g!nlWub7T;JVMs#9tb!+r%)0Nby2tgA7^No_yFmq#1 z-k4jdz)xai)FMC(B1Ke}`C7uwdr#=j+@g&@wX4V28IrxPeWm`a^SzrmE|x$rfu|*S z$|@N|$_T}L-fw;ZOs2iU5$G4o=X}Ao&Y2JdL~ly`SI1v4$Kwy*E=(n=E|S|b`OlfG zRr=I$2quw7Ch>GJY*C*k;_sYG8#T|ws3C9}<9v0T>`U+Q=XVtO2ezL!N_U=}*9o7L zU-=cJAwQslfq}ukUozU#tk)lZz2BjOgT4Q44+bWz?_g?6=VWVbr4l`5)5GvlbmIym zHrqU*QD!@kzVs)M>=bv3EHNGtd3(bgex?y?gl}QK{nUiT^f^!G!!=nnDk@3(MLZlW zV~Fkct^sA3YQr?~P|Gt#q|Gg{h^cvz9u>BDDHa;E@a;FuD9wfy6sAGx2Gus}=bfBo z-KIS_7P&rT&J_5q@yXMWEmLel=utkfA1(UF9>AvD)t% zPv~~f5VgtS3Sd?Go#3It%ZL9?aG(+m+y@d2to41q*zXf`bTFhhFt?#MF}HFucF=V+ z)^{*8qjPXfRME0sVMF)6tI}iDOJtChm^L^_85Q&=7x0A;J+7n=BAJ}7krg3jJsPS> z+v~iGQKA_hhX%kJ9IaLnrQ@bBi18O(>(*x&Y6*@#Sm#1@a6d`u?#J&b99*cL5@!R? z&K2j~j6)e)mG1|iUSFY@tS>G7mQ1eggY`aGpw;2vmd?vP1Md_}@hG8w-D~AkpBlP+ z%^i~LwRhiA35cxm^Jb+@g-~1vRU2Xm8OP9bzBPdx2DC2;&gZCOo-ZjG?0Hv;$~KxR z2n4lH;W~~Vs~TtMrh+_~9*m$3Ixyt9>ToRv34vHL-qXLOgU z&Z@p~j2Rn?*@p?da@d&=F~6r{gsnAiv4knh5@~#Y%CT<)rhY#V|00X^VoFsFL+@?( zR)SjvDEf0%#InST@nS0mD(=!O#`})o3q_L=0z@XH)od?Ca2zW)NiuE-s_$rxZ1+5j z7eb;?hIm4j9HTj0$D)^{&{;(;!f|1_@eiK^%8=fw;@|+RzaP08c*nBzj)yV&J|F2> zvjDd`VDrC~a_7T7{tm%OgC@0j(O_Y-J%lVYrOPv;e44g_R9Rd3lm;eb+k4z&M zgVny8!zP^fsVkUv*VEW(so^}qAYzFdp2S;YOiNtXmJBrfqyJ~oPfb;fJn~Nz7`>mW zrBa>(aPPK%=`b&UtM7qW5=%a>-E4{FWSF&*<m5QTJdg$ht=OgAeay&P+% z;GuN<|$+$qo|0WAs?n48ww20VU#|!b+;PJdU$5x3|*3Rs0~?qCVQA! zmv7~s3jL+tR$?YkigD6;b8>PYs~}l@Zuk^1R&u*FzhF-z9Tg{og$Yk}X&a+mFhfNm z>9k7eF&+hkiCf>#YkeFNv&#&?k$<}y+1wg0ewwV1tUniyda5{20_t<8jwC~l8}EJ~ zaW01x8FHRa$T_#GhV9}$)AmF`yCIwvpUB3#O{PoSI3Q7cm|b~L^Q-nK1qJBk6ZCN% zNLeCNtN=Qf{Up>dT|o*W$pJa4qUpnPU@1n&*8E4dkQ4SNS*_I`H=ukamcpMvr{Mhw z>=Q`am_3%GTbx=CrW9&WNwBF7UWpK^sf&9Xge(<$^14DW-%DSQHovonfobHegnsTi zpgW)0{Q*6OXfQ9gT7GrL=o0O}MtdS8ST_G|wEb}ZHCnSD1`g&%|1w%tYs+Ic^w%ui z-1P41RMX{kMz#iNa}CJAZuOZHX;`tEXf#Ew+ua>99ox63D5Hg>_|=*)CFy-x@%`VN zrC;@gjtz*We5UN8WZ^LOuga0{AC6Mx7UCb(51i?cVrX{x&(q7#$B`j#OU?(0T3*ol z0_9Gj23DrFPy`WZoVqYe3YXO{*zba<*M;Z~!gQhF9lg5tP-_7!G{tCgmgNoh*Bb0b z@FK5vT}g>>7{X*K5HRCq>PA8{9!E4DL6CF;2X0T|pIYn-;#v1IdyexTpv%?zhLuh& zQ^|8SqUvASk|d6|!0aOpGW{9|7@mG%VL0JpQ0&U)8;qYhdGe-u*7a?RVrJ|M{hSlQ zCX~it7N0QPn4q*#$=r(EhXHkPtdd1qRDeN+@pv#~FGaM=VOM;NAy*QXnX+D_|42fw zoC;TymBn)AWQo4d0U7|>Dv9J8ol}E89BT}X!@gN!$RJ*=Y}T3*AGty=lF4p z2%+P&)^vvy?u`5Jx9fQ$l}l&Dul1lr>o~1}Zxl1W?eELIwA8_+MOFQXGUf!NP9*l_ zWqclO6V-{W4l0P0kIhYFUD>|R=!p=NN4E4G2Qau(8=LM5F1UdUYX``ec3uwdee%k2 z8cjSy($`Iw%P-;f5?9GdH@o&_wxCq;=q+Z!R^4_=wO* z62UmFz6%gmAPDujM^Vl-lKU;h>C}Xmi>6S(@Y2$c#xF0}&`y;{I;B`@Os9BSB4&f{ zgs3j!Q>SccLYthtCE7*T6Ry&ggtawc;8*SktCz9kERB3gNjT8g#zHWZpD2@u? z?(B4HlzmP-c9?VyoRPtt0e-I~w4*DMMkXEM5&{&29OI)CVqt>2d3YRWPl(I~Jet*W z+3+{WKSa?AkTVzWzp=w=A;~{C#*aU_EamMD%h+%~?oA+^nXTGq2~eBCrvkiSpHB2x z9L+#N;VFv7gA5M>3clPSwQap_w++iZDZWsY;7 zf?Ve-cB+cv6zCx-?|Q3^eTz0QHH#BsiXW84AEj8BW>VG@m01!d!kCqh<+x1IKj9^k zcxulIL5^!jU-!-9I=A=8cwH7%%^#esB%IdJQ0p_tY*57efdAJQ9AVIiX1)6Yn0H@* z@h^beI~zOv(6zJDxA~ji*6;L2em&Lg{@F#W|1+jQc#1JK;DfX;1VQ86S_gKJfV63P zh=|23u<@PVXA;y&My8NL9~hn6?vnS@{=)Z}(j&d*cuhsl-gCz!#D=bCVbzGF
  • ; z{awt&F8e)=Wm`>OikH&;R`KcI_}2Oh-xqM*u(K2<%q&V31Wz-QLYdY%=tkRWem!Z? zOYH#CFK~C$j8(W+5zg0T=8_m8mp0L_rO4C)p1=qGPuRzxHNslx{#svtT5^-qebw0l z4`?W(MXCzM{Wcqwfrm&S-Iw?#$5|vLM;-W5Of>zCMJdK795K*ZfefjGjCModYZkSq&O0jH+h`5LI;am%ld0F}2>A*3kMW zQcorFD@n``>9$h9N?{{_+6<33 z3RL~pr?b;cbBm9ERvdGcI8{i$#rc3;ceg9_T)U`;AMEhiBzYW9l~rY~_9rr}Btt2- zjju`%5B~3F!-<=;se(LX%ptZGKr+rk*Np2dRGBy#H zee$xp5h@oQ-XHFNOQtmSthEw=!>>;bhuLAqRZxzvaHJ)5tvIqeoaQ9&&M!WED zerr;C(8&SCGXf?8I}u|eH@RS#Pha=P7`ieyh=kZFrVI`oR%QG3Qt%t&TRfL`zKe+= zE8RH(V1JFLp`sJAV}ds~oxX&plpy8sY5G7mb>1s@7l`Iv9_sDsLiN}vF9qtgpDgh> z6|%y{TlWxInebXR`PmL@Bot&zY_UAad9@|YJ5QECxoo@1 zN|9yFI*2$P$d>ll>FBadL50D<&mwo)@#LjjhMHnGUTJ#$)nYvR?34Gi-HEQ!E!cDm zNG#|3jm{}bhQP5Bn^^HEQ;97-^jx*(D%S0^zfMkqk86xHMJ0UB-rrro3K&A+lU$hsx$rlndLwCrl2@L24}|^2qt` zA|)=!CSIYJ>m*p`R&b5n6ie7fWlTc$yn_3|9Kmfw4FfU7IWY(Clihy3^COg#_}X`X zrydHp;b>v+>Fxm92^3$3rFM)n3P1xkbal%ts=n}IDC>`dW_?*O5mC@asSdl7EoojR zus>=Pa5?VSy{=%Y0^Edp?)i{enzSOw4ZyV^^T*N%6 zZMg+B%y>;BNGA*{StSM)$#3C^DL&DCFvmv9FDO!X$mz@lA^8OY znm#QbP*yC!<1mGtb$)>Bag3z^Uc}y`>y`?O zI?!tDfuHpqk|*HPxcywg^hv0UcaEQ_r`vt;l|TuqDfT+7)W4sX3$WsW^wP6;?~sHo zRs9Q+V(*XyfbKZ^Rp&T$p!NG~uD6&hzm-8zV)s5&yrfPn?Et2L;2IR4aAu42eV71w zt3&vd$4>dl#BD9u91DHZF!2_J5)22wca%2iU=3#_1Icg}j=DpbV1^tc-4UdwMmwPOGk@PJStK4EALTOLfZTp+!Z0Vd_Sk2 zHUD^fBP|tJR4T7RJ2ttQxOFwa;J82hMP}@WFId-yb-X!YuL9QhcGo_ zu+H)LHzX}p=Z%-DnXo?n1<6z2#GX~6zaV*?^uLgFY4J_QX=)ht%{9^b8%G4I4#~2)cS!dA4axP0YzB-E2>iGFi~bHoATOLdBcOi9(W(&1k5~4B zWT2;<1Zo!MaMlvlL&(h2_XahhCD=-4hMiNN^^b(7&(1{6jBsilBZ}-8>kOP$U!0fR zyM84^5R;uA!lE!`&O=f1*w8?n&$}P%4*eYjzx=TAu@-)a#0(wYTOaL&2dov@epi%1-Sr^ zg;b34p>9p9vnVEhp2U3I{&jMtw>RfIt!!_iH8?V!Vc<~nN%FuTns(lqb^%SxnneWO z@rELWkE>8kUimKyeg=7ayi*WTK^hza9rB;j+lVx@jY>2a7~FS=ze{iLDE#~IeJ{oR zze1dqt)b