diff --git a/docs/web-tests/py/test_geneva_defaults.py b/docs/web-tests/py/test_geneva_defaults.py deleted file mode 100644 index af4a42b00..000000000 --- a/docs/web-tests/py/test_geneva_defaults.py +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -# These tests assert the default resource values for Geneva KubeRay cluster nodes. -# If any of these values change, update the tables in: -# docs/geneva/jobs/performance.mdx (the "Geneva defaults" section) - -import pytest - - -def test_head_node_defaults(): - # If these change, update the Head Node table in performance.mdx - from geneva.cluster.builder import KubeRayClusterBuilder - - builder = KubeRayClusterBuilder() - assert builder._head_cpus == 4 - assert builder._head_memory == "8Gi" - assert builder._head_gpus == 0 - assert builder._head_node_selector == {"geneva.lancedb.com/ray-head": "true"} - assert builder._head_service_account == "geneva-service-account" - - -def test_cpu_worker_defaults(): - # If these change, update the CPU Workers table in performance.mdx - from geneva.cluster.builder import CpuWorkerBuilder - - worker = CpuWorkerBuilder() - assert worker._num_cpus == 4 - assert worker._memory == "8Gi" - assert worker._node_selector == {"geneva.lancedb.com/ray-worker-cpu": "true"} - assert worker._replicas == 1 - assert worker._min_replicas == 0 - assert worker._max_replicas == 100 - assert worker._idle_timeout_seconds == 60 - - # Confirm build produces 0 GPUs - config = worker.build() - assert config.num_gpus == 0 - - -def test_gpu_worker_defaults(): - # If these change, update the GPU Workers table in performance.mdx - from geneva.cluster.builder import GpuWorkerBuilder - - worker = GpuWorkerBuilder() - assert worker._num_cpus == 8 - assert worker._memory == "16Gi" - assert worker._num_gpus == 1 - assert worker._node_selector == {"geneva.lancedb.com/ray-worker-gpu": "true"} - assert worker._replicas == 1 - assert worker._min_replicas == 0 - assert worker._max_replicas == 100 - assert worker._idle_timeout_seconds == 60 diff --git a/docs/web-tests/py/test_geneva_dependency_verification.py b/docs/web-tests/py/test_geneva_dependency_verification.py deleted file mode 100644 index fc00aba6c..000000000 --- a/docs/web-tests/py/test_geneva_dependency_verification.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -import os - -import pytest - - -def test_quick_fix_manifest(): - # --8<-- [start:quick_fix_manifest] - from geneva.manifest.builder import PipManifestBuilder - - manifest = PipManifestBuilder.create("fix").pip(["numpy==1.26.4"]).build() - # --8<-- [end:quick_fix_manifest] - assert manifest.pip == ["numpy==1.26.4"] - - -def test_env_vars_via_cluster(monkeypatch): - monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-key-id") - monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret") - - # --8<-- [start:env_vars_via_cluster] - from geneva.cluster.builder import KubeRayClusterBuilder - import os - - cluster = ( - KubeRayClusterBuilder.create("my-cluster") - .ray_init_kwargs({ - "runtime_env": { - "env_vars": { - "AWS_ACCESS_KEY_ID": os.environ["AWS_ACCESS_KEY_ID"], - "AWS_SECRET_ACCESS_KEY": os.environ["AWS_SECRET_ACCESS_KEY"], - } - } - }) - .build() - ) - # --8<-- [end:env_vars_via_cluster] - assert cluster.kuberay.ray_init_kwargs["runtime_env"]["env_vars"]["AWS_ACCESS_KEY_ID"] == "test-key-id" - - -def test_pip_manifest(monkeypatch): - import geneva - from unittest.mock import MagicMock, create_autospec - mock_conn = create_autospec(geneva.db.Connection, instance=True) - monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_conn)) - - # --8<-- [start:pip_manifest] - import geneva - from geneva.manifest.builder import PipManifestBuilder - - manifest = ( - PipManifestBuilder.create("my-manifest") - .pip([ - "numpy==1.26.4", - "torch==2.0.1", - "attrs==23.2.0", - ]) - .build() - ) - - conn = geneva.connect("s3://my-bucket/my-db") - conn.define_manifest("my-manifest", manifest) - with conn.context(cluster="my-cluster", manifest="my-manifest"): - conn.open_table("my-table").backfill("my-column") - # --8<-- [end:pip_manifest] - assert "numpy==1.26.4" in manifest.pip - - -def test_conda_cluster_path(): - # --8<-- [start:conda_cluster_path] - from geneva.cluster.builder import KubeRayClusterBuilder - - cluster = ( - KubeRayClusterBuilder.create("my-cluster") - .ray_init_kwargs({ - "runtime_env": {"conda": "environment.yml"} - }) - .build() - ) - # --8<-- [end:conda_cluster_path] - assert cluster.kuberay.ray_init_kwargs["runtime_env"]["conda"] == "environment.yml" - - -def test_conda_cluster_inline(): - # --8<-- [start:conda_cluster_inline] - from geneva.cluster.builder import KubeRayClusterBuilder - - cluster = ( - KubeRayClusterBuilder.create("my-cluster") - .ray_init_kwargs({ - "runtime_env": { - "conda": { - "channels": ["conda-forge"], - "dependencies": [ - "python=3.10", - "ffmpeg<8", - "torchvision=0.22.1", - ], - }, - "config": {"eager_install": True}, - } - }) - .build() - ) - # --8<-- [end:conda_cluster_inline] - assert cluster.kuberay.ray_init_kwargs["runtime_env"]["conda"]["channels"] == ["conda-forge"] diff --git a/docs/web-tests/py/test_geneva_profiling_memory.py b/docs/web-tests/py/test_geneva_profiling_memory.py deleted file mode 100644 index 7ca45db04..000000000 --- a/docs/web-tests/py/test_geneva_profiling_memory.py +++ /dev/null @@ -1,248 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -"""Snippets for docs/geneva/udfs/profiling-memory.mdx.""" - -from unittest.mock import MagicMock - - -def test_stateful_udf_class(): - import geneva - import pyarrow as pa - - load_model = MagicMock(return_value=MagicMock(embed=MagicMock(return_value=[0.1] * 512))) - - # --8<-- [start:stateful_udf_class] - @geneva.udf(data_type=pa.list_(pa.float32(), 512)) - class MyEmbedding: - def __init__(self): - self.model = None - - def setup(self): - self.model = load_model() # allocated once per actor - - def __call__(self, text: str) -> list[float]: - if self.model is None: - self.setup() - return self.model.embed(text) - # --8<-- [end:stateful_udf_class] - - assert MyEmbedding is not None - - -def test_memray_tracker_udf(monkeypatch): - import sys - - load_model = MagicMock(return_value=MagicMock(embed=MagicMock(return_value=[0.1] * 512))) - monkeypatch.setitem(sys.modules, "memray", MagicMock()) - - # --8<-- [start:memray_tracker_udf] - import os, pathlib, uuid - from typing import Any - import memray - import geneva - import pyarrow as pa - - _MEMRAY_OUT_DIR_ENV = "MY_UDF_MEMRAY_OUT_DIR" - - - @geneva.udf(data_type=pa.list_(pa.float32(), 512)) - class MyEmbedding: - def __init__(self): - self.model = None - self._tracker: Any = None # memray.Tracker, when profiling is on - - def setup(self): - # Open a memray tracker per worker process, if requested. Each - # worker writes its own .bin file so traces don't collide. - out_dir = os.environ.get(_MEMRAY_OUT_DIR_ENV) - if out_dir: - pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True) - bin_path = pathlib.Path(out_dir) / ( - f"memray-{os.getpid()}-{uuid.uuid4().hex}.bin" - ) - self._tracker = memray.Tracker( - str(bin_path), native_traces=False, follow_fork=False - ) - self._tracker.__enter__() - self.model = load_model() - - def __call__(self, text: str) -> list[float]: - if self.model is None: - self.setup() - return self.model.embed(text) - # --8<-- [end:memray_tracker_udf] - - assert MyEmbedding is not None - - -def test_ray_cluster_profile(monkeypatch): - from contextlib import contextmanager - - table = MagicMock() - - @contextmanager - def _mock_cluster(*args, **kwargs): - yield - - monkeypatch.setattr("geneva.runners.ray._mgr.ray_cluster", _mock_cluster) - - # --8<-- [start:ray_cluster_profile] - from geneva.runners.ray._mgr import ray_cluster - - with ray_cluster( - local=True, - extra_env={"MY_UDF_MEMRAY_OUT_DIR": "/tmp/my-udf-profile"}, - ): - table.backfill("embedding", concurrency=1) - # --8<-- [end:ray_cluster_profile] - - table.backfill.assert_called_once_with("embedding", concurrency=1) - - -def test_log_memory(capsys): - # --8<-- [start:log_memory] - import resource, pyarrow as pa - - def log_memory(seq: int) -> None: - rss_bytes = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - # ru_maxrss is bytes on macOS, KiB on Linux: - import sys - if sys.platform != "darwin": - rss_bytes *= 1024 - arrow_live = pa.total_allocated_bytes() - print( - f"seq={seq} " - f"rss_mb={rss_bytes // 1024**2} " - f"arrow_live_mb={arrow_live // 1024**2} " - f"gap_mb={(rss_bytes - arrow_live) // 1024**2}", - flush=True, - ) - # --8<-- [end:log_memory] - - log_memory(0) - captured = capsys.readouterr() - assert "seq=0" in captured.out - assert "rss_mb=" in captured.out - - -def test_leaky_cache(): - # --8<-- [start:leaky_cache] - class BadEmbedding: - def __init__(self): - self.cache: dict[str, list[float]] = {} - - def __call__(self, text: str) -> list[float]: - if text not in self.cache: - self.cache[text] = self.model.embed(text) - return self.cache[text] - # --8<-- [end:leaky_cache] - - obj = BadEmbedding() - obj.model = MagicMock(embed=MagicMock(return_value=[0.1, 0.2])) - obj("hello") - obj("hello") - assert obj.model.embed.call_count == 1 # second call served from cache - assert len(obj.cache) == 1 - - -def test_bounded_cache(): - load_model = MagicMock(return_value=MagicMock(embed=MagicMock(return_value=[0.1, 0.2]))) - - # --8<-- [start:bounded_cache] - from functools import lru_cache - - class GoodEmbedding: - def __init__(self): - self._embed = None - - def setup(self): - model = load_model() - self._embed = lru_cache(maxsize=1024)(model.embed) - - def __call__(self, text: str) -> list[float]: - if self._embed is None: - self.setup() - return self._embed(text) - # --8<-- [end:bounded_cache] - - obj = GoodEmbedding() - assert obj("hello") == [0.1, 0.2] - load_model.assert_called_once() - - -def test_leaky_aggregator(): - import pyarrow as pa - - # --8<-- [start:leaky_aggregator] - class BadAggregator: - def __init__(self): - self.history = [] - - def __call__(self, batch: pa.RecordBatch) -> pa.Array: - self.history.append(batch) # holds every batch ever processed - ... - # --8<-- [end:leaky_aggregator] - - obj = BadAggregator() - assert isinstance(obj.history, list) - - -def test_leaky_closure(): - import pyarrow as pa - - def expensive(x): - return x - - # --8<-- [start:leaky_closure] - class BadDeferred: - def __init__(self): - self.work_queue = [] - - def __call__(self, x: pa.Array) -> pa.Array: - # Lambda captures `x` by reference — the whole Array stays alive - self.work_queue.append(lambda: expensive(x)) - ... - # --8<-- [end:leaky_closure] - - obj = BadDeferred() - assert isinstance(obj.work_queue, list) - - -def test_torch_inference_mode(monkeypatch): - import sys - - mock_torch = MagicMock() - monkeypatch.setitem(sys.modules, "torch", mock_torch) - - class TorchUDF: - def __init__(self): - self.model = MagicMock(encode=MagicMock(return_value=[0.1])) - - # --8<-- [start:torch_inference_mode] - def __call__(self, text: str) -> list[float]: - with torch.inference_mode(): # <-- prevents autograd graph retention - return self.model.encode(text) - # --8<-- [end:torch_inference_mode] - - import torch # resolves to monkeypatched mock above - obj = TorchUDF() - assert obj("hello") == [0.1] - - -def test_confidence_check(): - class DebuggingUDF: - def __init__(self): - self._scratches = [] - - # --8<-- [start:confidence_check] - def __call__(self, x): - scratch = bytearray(8 * 1024 * 1024) # 8 MiB - self._scratches.append(scratch) # <-- deliberate leak - return ... - # --8<-- [end:confidence_check] - - obj = DebuggingUDF() - obj(None) - assert len(obj._scratches) == 1 - assert len(obj._scratches[0]) == 8 * 1024 * 1024 diff --git a/docs/web-tests/py/test_geneva_scalar_udtfs.py b/docs/web-tests/py/test_geneva_scalar_udtfs.py deleted file mode 100644 index 30aec525a..000000000 --- a/docs/web-tests/py/test_geneva_scalar_udtfs.py +++ /dev/null @@ -1,311 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -import pytest -from unittest.mock import MagicMock - - -def test_scalar_udtf_iterator(): - def extract_video_segment(path, start, end): - return bytes(f"clip:{start}-{end}", "utf-8") - - # --8<-- [start:scalar_udtf_iterator] - from geneva import chunker - from typing import Iterator, NamedTuple - - class Clip(NamedTuple): - clip_start: float - clip_end: float - clip_bytes: bytes - - @chunker - def extract_clips(video_path: str, duration: float) -> Iterator[Clip]: - """Yields multiple clips per video.""" - clip_length = 10.0 - for start in range(0, int(duration), int(clip_length)): - end = min(start + clip_length, duration) - clip_data = extract_video_segment(video_path, start, end) - yield Clip(clip_start=start, clip_end=end, clip_bytes=clip_data) - # --8<-- [end:scalar_udtf_iterator] - - from geneva.transformer import Chunker - assert isinstance(extract_clips, Chunker) - assert extract_clips.input_columns == ["video_path", "duration"] - assert set(extract_clips.output_schema.names) == {"clip_start", "clip_end", "clip_bytes"} - - clips = list(extract_clips.func("/v/a.mp4", 30.0)) - assert len(clips) == 3 - assert clips[0].clip_start == 0.0 - assert clips[2].clip_end == 30.0 - - -def test_scalar_udtf_list_return(): - from geneva import chunker - from typing import NamedTuple - - class Clip(NamedTuple): - clip_start: float - clip_end: float - clip_bytes: bytes - - # --8<-- [start:scalar_udtf_list] - @chunker - def extract_clips(video_path: str, duration: float) -> list[Clip]: - clips = [] - for start in range(0, int(duration), 10): - end = min(start + 10, duration) - clips.append(Clip(clip_start=start, clip_end=end, clip_bytes=b"...")) - return clips - # --8<-- [end:scalar_udtf_list] - - from geneva.transformer import Chunker - assert isinstance(extract_clips, Chunker) - clips = extract_clips.func("/v/a.mp4", 30.0) - assert len(clips) == 3 - assert all(c.clip_bytes == b"..." for c in clips) - - -def test_scalar_udtf_batch(): - import pyarrow as pa - from geneva import chunker - - clip_schema = pa.schema([ - ("clip_start", pa.float64()), - ("clip_end", pa.float64()), - ]) - - # --8<-- [start:scalar_udtf_batch] - @chunker(batch=True, output_schema=clip_schema) - def extract_clips(batch: pa.RecordBatch) -> pa.RecordBatch: - """Process rows in batches. Same 1:N semantic per row.""" - ... - # --8<-- [end:scalar_udtf_batch] - - from geneva.transformer import Chunker - assert isinstance(extract_clips, Chunker) - assert extract_clips.batch is True - - -def test_create_scalar_udtf_view(monkeypatch): - from typing import Iterator, NamedTuple - from geneva.transformer import chunker - - class Clip(NamedTuple): - clip_start: float - clip_end: float - clip_bytes: bytes - - @chunker - def extract_clips(video_path: str, duration: float) -> Iterator[Clip]: - for start in range(0, int(duration), 10): - yield Clip(clip_start=start, clip_end=min(start + 10.0, duration), clip_bytes=b"") - - import geneva - from unittest.mock import create_autospec - mock_clips = MagicMock() - mock_db = create_autospec(geneva.db.Connection, instance=True) - mock_db.create_udtf_view.return_value = mock_clips - monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db)) - - # --8<-- [start:create_scalar_udtf_view] - import geneva - - db = geneva.connect("/data/mydb") - videos = db.open_table("videos") - - # Create the 1:N materialized view - clips = db.create_udtf_view( - "clips", - source=videos.search(None).select(["video_path", "metadata"]), - udtf=extract_clips, - ) - - # Populate — runs the UDTF on every source row - clips.refresh() - # --8<-- [end:create_scalar_udtf_view] - - call_kwargs = mock_db.create_udtf_view.call_args - assert call_kwargs.args[0] == "clips" - assert call_kwargs.kwargs["udtf"] is extract_clips - mock_clips.refresh.assert_called_once() - - -def test_add_columns_scalar_udtf(): - import pyarrow as pa - from geneva.transformer import udf - - clips = MagicMock() - embed_model = MagicMock() - embed_model.encode.return_value = [0.1] * 512 - - # --8<-- [start:add_columns_scalar_udtf] - @udf(data_type=pa.list_(pa.float32(), 512)) - def clip_embedding(clip_bytes: bytes) -> list[float]: - return embed_model.encode(clip_bytes) - - # Add an embedding column to the clips table - clips.add_columns({"embedding": clip_embedding}) - - # Backfill computes embeddings for all existing clips - clips.backfill("embedding") - # --8<-- [end:add_columns_scalar_udtf] - - clips.add_columns.assert_called_once_with({"embedding": clip_embedding}) - clips.backfill.assert_called_once_with("embedding") - - -def test_incremental_refresh(): - videos = MagicMock() - clips = MagicMock() - new_video_data = [{"video_path": "/v/c.mp4", "duration": 45.0}] - - # --8<-- [start:incremental_refresh] - # Add new videos to the source table - videos.add(new_video_data) - - # Incremental refresh — only processes the new videos - clips.refresh() - # --8<-- [end:incremental_refresh] - - videos.add.assert_called_once_with(new_video_data) - clips.refresh.assert_called_once() - - -def test_chaining_udtf_views(monkeypatch): - from typing import Iterator, NamedTuple - from geneva.transformer import chunker - - class Clip(NamedTuple): - clip_start: float - clip_end: float - - class Frame(NamedTuple): - frame_index: int - frame_bytes: bytes - - @chunker - def extract_clips(video_path: str, duration: float) -> Iterator[Clip]: - for start in range(0, int(duration), 10): - yield Clip(clip_start=start, clip_end=min(start + 10.0, duration)) - - @chunker - def extract_frames(clip_start: float, clip_end: float) -> Iterator[Frame]: - yield Frame(frame_index=0, frame_bytes=b"") - - import geneva - from unittest.mock import create_autospec - mock_db = create_autospec(geneva.db.Connection, instance=True) - monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db)) - - db = geneva.connect("/data/mydb") - videos = db.open_table("videos") - - # --8<-- [start:chaining_udtf_views] - # videos → clips (1:N) - clips = db.create_udtf_view( - "clips", source=videos.search(None), udtf=extract_clips - ) - - # clips → frames (1:N) - frames = db.create_udtf_view( - "frames", source=clips.search(None), udtf=extract_frames - ) - # --8<-- [end:chaining_udtf_views] - - assert mock_db.create_udtf_view.call_count == 2 - first_call = mock_db.create_udtf_view.call_args_list[0] - second_call = mock_db.create_udtf_view.call_args_list[1] - assert first_call.kwargs["udtf"] is extract_clips - assert second_call.kwargs["udtf"] is extract_frames - - -def test_document_chunking_udtf(): - # --8<-- [start:document_chunking_udtf] - from geneva import chunker - from typing import Iterator, NamedTuple - - class Chunk(NamedTuple): - chunk_index: int - chunk_text: str - - @chunker - def chunk_document(text: str) -> Iterator[Chunk]: - """Split a document into overlapping chunks.""" - words = text.split() - chunk_size = 500 - overlap = 50 - for i, start in enumerate(range(0, len(words), chunk_size - overlap)): - chunk_words = words[start:start + chunk_size] - yield Chunk(chunk_index=i, chunk_text=" ".join(chunk_words)) - # --8<-- [end:document_chunking_udtf] - - sample_text = " ".join(["word"] * 600) - chunks = list(chunk_document.func(sample_text)) - assert len(chunks) == 2 - assert chunks[0].chunk_index == 0 - assert chunks[1].chunk_index == 1 - assert len(chunks[0].chunk_text.split()) == 500 - - -def test_document_chunking_full(monkeypatch): - import pyarrow as pa - from unittest.mock import MagicMock - - import geneva - from unittest.mock import create_autospec - mock_chunks_table = MagicMock() - mock_db = create_autospec(geneva.db.Connection, instance=True) - mock_db.create_udtf_view.return_value = mock_chunks_table - monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db)) - - embedding_model = MagicMock() - embedding_model.encode.return_value = [0.1] * 1536 - - # --8<-- [start:document_chunking_full] - from geneva import connect, chunker, udf - from typing import Iterator, NamedTuple - import pyarrow as pa - - class Chunk(NamedTuple): - chunk_index: int - chunk_text: str - - @chunker - def chunk_document(text: str) -> Iterator[Chunk]: - """Split a document into overlapping chunks.""" - words = text.split() - chunk_size = 500 - overlap = 50 - for i, start in enumerate(range(0, len(words), chunk_size - overlap)): - chunk_words = words[start:start + chunk_size] - yield Chunk(chunk_index=i, chunk_text=" ".join(chunk_words)) - - db = connect("/data/mydb") - docs = db.open_table("documents") - - # Create chunked view — inherits doc_id, title, etc. from source - chunks = db.create_udtf_view( - "doc_chunks", - source=docs.search(None).select(["doc_id", "title", "text"]), - udtf=chunk_document, - ) - chunks.refresh() - - # Add embeddings to chunks for semantic search - @udf(data_type=pa.list_(pa.float32(), 1536)) - def embed_text(chunk_text: str) -> list[float]: - return embedding_model.encode(chunk_text) - - chunks.add_columns({"embedding": embed_text}) - chunks.backfill("embedding") # Backfills embeddings on all existing chunks - - # Query — parent columns available alongside chunk columns - chunks.search(None).select(["doc_id", "title", "chunk_text", "embedding"]).to_pandas() - # --8<-- [end:document_chunking_full] - - call_kwargs = mock_db.create_udtf_view.call_args - assert call_kwargs.args[0] == "doc_chunks" - assert call_kwargs.kwargs["udtf"] is chunk_document - mock_chunks_table.refresh.assert_called_once() - mock_chunks_table.add_columns.assert_called_once() - mock_chunks_table.backfill.assert_called_once_with("embedding") diff --git a/docs/web-tests/py/test_geneva_udfs_index.py b/docs/web-tests/py/test_geneva_udfs_index.py deleted file mode 100644 index dffe00d5d..000000000 --- a/docs/web-tests/py/test_geneva_udfs_index.py +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -from unittest.mock import MagicMock, create_autospec - - -def test_registration_udf(monkeypatch): - # Verifies that Table.add_columns exists with the expected signature. - # If this fails, update the Registration row in docs/geneva/udfs/index.mdx. - import geneva - mock_table = create_autospec(geneva.table.Table, instance=True) - my_udf = MagicMock() - - # --8<-- [start:registration_udf] - mock_table.add_columns({"col": my_udf}) - # --8<-- [end:registration_udf] - - mock_table.add_columns.assert_called_once() - - -def test_registration_scalar_udtf(monkeypatch): - # Verifies that Connection.create_udtf_view accepts a chunker. - # If this fails, update the Registration row in docs/geneva/udfs/index.mdx. - import geneva - mock_db = create_autospec(geneva.db.Connection, instance=True) - monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db)) - my_source = MagicMock() - my_chunker = MagicMock() - - # --8<-- [start:registration_scalar_udtf] - db = geneva.connect("/data/mydb") - db.create_udtf_view("my_view", source=my_source, udtf=my_chunker) - # --8<-- [end:registration_scalar_udtf] - - mock_db.create_udtf_view.assert_called_once() - - -def test_registration_udtf(monkeypatch): - # Verifies that Connection.create_udtf_view exists with the expected signature. - # If this fails, update the Registration row in docs/geneva/udfs/index.mdx. - import geneva - mock_db = create_autospec(geneva.db.Connection, instance=True) - monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db)) - my_source = MagicMock() - my_udtf = MagicMock() - - # --8<-- [start:registration_udtf] - db = geneva.connect("/data/mydb") - db.create_udtf_view("my_view", source=my_source, udtf=my_udtf) - # --8<-- [end:registration_udtf] - - mock_db.create_udtf_view.assert_called_once() diff --git a/docs/web/basics/agentic-skills.mdx b/docs/web/basics/agentic-skills.mdx new file mode 100644 index 000000000..d45bc8117 --- /dev/null +++ b/docs/web/basics/agentic-skills.mdx @@ -0,0 +1,61 @@ +--- +title: "Agentic skills" +sidebarTitle: "Agentic Skills" +description: "Give a coding agent a current, maintained reference for the LanceDB APIs." +icon: robot +keywords: ["agents", "plugin", "claude code", "codex", "cursor", "copilot"] +--- + +A coding agent writing LanceDB code from memory writes whatever was in its +training data -- often an older API, sometimes one that never existed. The +LanceDB agent plugin fixes that by supplying the current reference at the point +the agent needs it. + +## The plugin {#plugin} + +It ships as the `lancedb` plugin in the +[lancedb-agent-plugins](https://github.com/lancedb/lancedb-agent-plugins) +repository, which doubles as a plugin marketplace. It covers the Python and +TypeScript APIs, which code is portable between a local database and a +deployment, ingestion performance, and branch operations. + + +```text Claude Code +/plugin marketplace add lancedb/lancedb-agent-plugins +/plugin install lancedb@lancedb +``` + +```bash Codex icon="terminal" +codex plugin marketplace add lancedb/lancedb-agent-plugins +codex plugin add lancedb@lancedb +``` + +```bash Other agents icon="terminal" +npx plugins add lancedb/lancedb-agent-plugins +``` + + +The cross-tool installer detects which agent CLIs are on your `PATH` -- Cursor, +GitHub Copilot CLI, VS Code, Grok Build, Kimi Code, Claude Code, Codex -- and +installs through each one's native plugin system. + +## What agents are good at here {#workflows} + +Two things about LanceDB make it unusually agent-friendly, and both have a +worked tutorial. + + + + Install the plugin and have an agent build a multimodal ingestion pipeline + end to end. + + + Isolate agent-driven experiments on a branch, evaluate them on a fixed test + set, and promote only the winner. + + + +The second is the more interesting pattern. [Branches](/tables/versioning) +give an agent somewhere to be wrong safely: it can rewrite a schema, recompute a +column, or try three embedding models without touching `main`, and you keep only +what measured better. diff --git a/docs/web/basics/apis.mdx b/docs/web/basics/apis.mdx new file mode 100644 index 000000000..3839d8584 --- /dev/null +++ b/docs/web/basics/apis.mdx @@ -0,0 +1,94 @@ +--- +title: "Python, TypeScript, and SQL" +sidebarTitle: "Python vs SQL" +description: "Three ways to reach the same data, what each one can do, and where each one runs." +icon: code-compare +keywords: ["python", "typescript", "sql", "api", "flight sql", "client"] +--- + +The same table can be reached three ways: the Python SDK, the TypeScript SDK, +or SQL. They are views onto one engine, not three products -- a table written +from Python is queried from SQL without an export step. + +They are not interchangeable, though, and the differences are worth knowing +before you pick one. + +## What each one can do {#matrix} + +| | Python | TypeScript | SQL | +| --- | :---: | :---: | :---: | +| Create and manage catalog objects | yes | yes | partly | +| Insert, merge, update, delete | yes | yes | -- | +| Filter, project, aggregate, join | yes | yes | yes | +| Vector and full-text search | yes | yes | yes | +| Feature engineering, jobs | yes | yes | -- | +| Runs against a local database | yes | yes | **no** | + +Two entries in that table do most of the work of explaining the rest. + +## SQL needs a deployment {#sql-is-remote} + + +SQL is not available on a local database. A connection to a directory or a +bucket has no server to plan and execute a statement, and asking it to raises +`NotImplementedError`. + + +```python +db = lancedb.connect("./.lancedb") # no SQL +db = lancedb.connect("db://my-database") # SQL +``` + +This is the single largest behavioural difference between the embedded and +deployed forms, and the reason the SDKs are the portable choice: SDK code moves +between the two by changing the URI, SQL does not. + +Enterprise Queries are served over an +[Arrow Flight SQL](https://arrow.apache.org/docs/format/FlightSql.html) +endpoint, so any Flight SQL client can reach it and results arrive as Arrow +record batches -- no conversion to Python objects unless you ask for one. + +## SQL from each language {#sql-clients} + +From Python, the connection runs SQL directly: + +```python +reader = db.execute_query("SELECT id, text FROM my_docs LIMIT 10") +table = reader.read_all() +``` + +`execute_query` returns a blocking Arrow reader once results start arriving; it +does not wait for the whole query. `execute_query_async` returns a +[job](/basics/jobs) instead, for a statement you would rather not block on. + + +The TypeScript SDK has no SQL method. Reach the endpoint with a Flight SQL +client instead -- see [connecting a client](/search/sql). This is a gap in the +SDK, not in the endpoint: the same statements work. + + +## Choosing between them {#choosing} + +Reach for **the SDKs** when you are writing data, managing the catalog, running +feature engineering, or building an application that should work the same +embedded and deployed. They are the complete surface. + +Reach for **SQL** when the question is analytical -- joins across tables, +aggregates, window functions -- or when the thing asking is a BI tool, a +notebook, or a person who already thinks in SQL. It is also the only way to +create a [view](/management/view). + +The two compose. A common shape is SDK code to build and maintain tables, and +SQL for the analysis on top. + +## What stays the same {#invariants} + +Whichever you pick: + +- **Types are Arrow types.** A vector column is a `FixedSizeList` + everywhere, not a SQL type mapped onto storage. See + [Functions and types](/sql/functions). +- **Reads are Arrow.** Nothing is serialized into language objects on the way + out unless you convert it. +- **The catalog is one catalog.** A table created from TypeScript is visible to + Python and to SQL immediately -- there is no per-language namespace. diff --git a/docs/web/basics/catalog.mdx b/docs/web/basics/catalog.mdx new file mode 100644 index 000000000..6670a7e7f --- /dev/null +++ b/docs/web/basics/catalog.mdx @@ -0,0 +1,134 @@ +--- +title: "The catalog" +sidebarTitle: "Catalog" +description: "What lives in a LanceDB catalog, how the objects nest, and how you connect to one." +icon: sitemap +keywords: ["catalog", "database", "namespace", "table", "view", "function"] +--- + +A catalog is what LanceDB knows about. Everything you create lives in one and is +addressed through it. + +## The objects {#objects} + +| Object | What it is | +| --- | --- | +| Database | The root. A connection opens exactly one. | +| [Namespace](/namespaces) | A named grouping inside a database. Can nest. | +| [Table](/tables) | Rows and columns, versioned. What most operations act on. | +| [Materialized view](/feature-engineering/materialized-views) | A table whose contents come from a query, computed and stored. | +| [View](/management/view) | A saved query. Computed when read, not stored. | +| [Function](/feature-engineering/functions) | Registered code, applied to produce column values. | + +``` +database +└── namespace + ├── table + ├── materialized view (a table, defined by a query) + ├── view (a query, not stored) + └── function +``` + +A namespace can contain further namespaces, so the hierarchy goes as deep as you +need. A table is reached by its path through it. + +This page is about what the objects *are*. For the operations on them -- +creating, listing, dropping -- see [Management](/management). + +## Connecting {#connecting} + +```python +import lancedb + +db = lancedb.connect("./.lancedb") # a local directory +db = lancedb.connect("s3://bucket/path") # object storage +db = lancedb.connect("db://my-database", api_key="...") # a deployment +``` + +The URI decides what kind of database you get, and that decision reaches further +than it looks. A local or object-store URI opens the data directly from your +process. A `db://` URI talks to a deployment, which is what makes server-side +execution -- [functions](/feature-engineering/functions), remote refreshes, +[SQL](/basics/apis) -- available, and what makes a few local-only features +unavailable. + +`connect_async` returns an `AsyncConnection` with the same surface as +coroutines. + +### What a table handle actually is {#table-handles} + +The URI also decides what `open_table` hands back, which is worth knowing +because the two behave the same but fail differently: + +- **`LanceTable`** -- direct access to storage (a local path, `file://`, `s3://` + and similar). Your process resolves and reads the table itself. +- **`RemoteTable`** -- catalog-backed access through a deployment (`db://`). + Resolution and execution happen on the server. + +![](/static/assets/images/overview/understanding-tables.png) + +Both expose the same table API -- create, open, mutate, query. What differs is +where the work happens, and therefore which operations exist at all. + +### Storage options {#storage-options} + +`storage_options` configures the underlying store -- credentials, endpoints, and +per-table defaults for tables created through this connection: + +```python +db = lancedb.connect( + "s3://bucket/path", + storage_options={"new_table_enable_stable_row_ids": "true"}, +) +``` + + +`new_table_enable_stable_row_ids` is worth setting deliberately. Stable row ids +cannot be enabled after a table exists, and a +[materialized view](/feature-engineering/materialized-views) requires its source +to have them. A table created without them cannot later become a view's source +without being recreated. + + +See [Storage](/storage) for the full set. + +### Read consistency {#consistency} + +`read_consistency_interval` controls how often a connection checks for versions +committed by other writers. The default reads a snapshot taken when the table was +opened, so a long-lived connection will not see other processes' writes until it +reopens. + +Set it when several processes write the same table and readers must observe those +writes: + +```python +from datetime import timedelta + +db = lancedb.connect(uri, read_consistency_interval=timedelta(seconds=5)) +``` + +The cost is a metadata check on that interval. See +[Consistency](/tables/consistency). + +## Views and materialized views are different objects {#views} + +The names are close enough to mislead. Both are defined by a query over other +tables; the difference is when that query runs. + +- A **materialized view** stores its rows. A refresh computes them, and reads are + reads of stored data -- fast, and as current as the last refresh. +- A **view** stores only the query. Every read runs it, so a view is never stale + and never faster than the query it wraps. + +Reach for a materialized view when the query is expensive and the data changes +less often than it is read. Reach for a view when correctness matters more than +latency, or when the query is cheap. + +## What is not a catalog object {#elsewhere} + +**Indexes** belong to a table rather than standing alone, and creating one is a +performance decision -- see [Maintenance](/indexing). + +**Jobs** are not stored data at all; they are handles to work the server is +doing. See [Jobs](/basics/jobs). diff --git a/docs/web/basics/jobs.mdx b/docs/web/basics/jobs.mdx new file mode 100644 index 000000000..501d1f5fe --- /dev/null +++ b/docs/web/basics/jobs.mdx @@ -0,0 +1,181 @@ +--- +title: "Jobs" +sidebarTitle: "Jobs" +description: "The handle every long-running operation returns, and how to wait on, inspect, or cancel one." +icon: hourglass-half +keywords: ["job", "async", "background", "create index", "refresh column"] +--- + +Some operations take longer than a request should. Building an index over a +large table, refreshing a computed column, registering a function -- these run +on the server after the call that started them has returned. + +Each is submitted the same way and returns the same handle. Learn the handle +once and every long-running operation behaves alike. + +## The `_async` convention {#convention} + +An operation that can run in the background has two forms. The plain name +blocks until the work is done and returns its result. The `_async` / +`Async` name submits the work and returns a **job** immediately. + +| Blocking | Returns a job | What runs | +| --- | --- | --- | +| `create_index` | `create_index_async` | index build | +| `refresh_column` | `refresh_column_async` | recompute a [computed column](/feature-engineering/computed-columns) | +| `create_function` | `create_function_async` | register a [function](/feature-engineering/functions) | +| `drop_table` | `drop_table_async` | drop a table | +| `execute_query` | `execute_query_async` | a [SQL](/basics/apis) query | + + +The blocking form is not a different operation -- it submits the same job and +waits. Reach for the `_async` form when you want to do something else +meanwhile, run several at once, or survive the client going away. + + +## Submitting and waiting {#waiting} + + +```python Python icon="python" +job = table.create_index_async("vector") + +print(job.id) # the server's id for this operation +print(job.status()) # "running" | "finished" | "failed" | "cancelled" + +job.wait() # blocks until terminal; raises if it failed +``` + +```typescript TypeScript icon="square-js" +const job = await table.createIndexAsync("vector"); + +console.log(job.id); // the server's id for this operation +console.log(await job.status()); // "running" | "finished" | "failed" | "cancelled" + +await job.wait(); // resolves when terminal; rejects if it failed +``` + + +`wait` is the one call that raises on failure. `status` is a point snapshot and +returns the failed state rather than raising, which is what you want in a +polling loop. + +In Python `wait` takes an optional timeout and returns the operation's typed +result -- a `RefreshColumnJobResult`, a `FunctionVersion`, or `None` for an +operation that has no result: + +```python +from datetime import timedelta + +result = table.refresh_column_async("embedding").wait(timeout=timedelta(minutes=30)) +``` + +`TimeoutError` means the timeout elapsed, not that the job stopped. The job is +still running; the handle just gave up watching. + + +The TypeScript `wait()` resolves to `void`. Read the outcome from `result` +after it resolves. + + +## Inspecting a job {#inspecting} + +Submitting an operation returns only an id, so the detail fields are empty +until the handle fetches the record. `refresh()` does that; `status()` does it +as a side effect. + + +```python Python icon="python" +job.refresh() + +job.state # last observed state, without another round trip +job.job_type # what kind of job, as the server names it +job.creation_ms # when it was created, epoch milliseconds +job.spec # what it was submitted with +job.result # its terminal result, once it succeeds +job.failure # why it failed, when it did +``` + +```typescript TypeScript icon="square-js" +await job.refresh(); + +job.state; // last observed state, without another round trip +job.jobType; // what kind of job, as the server names it +job.creationMs; // when it was created, epoch milliseconds +job.spec; // what it was submitted with +job.result; // its terminal result, once it succeeds +job.failure; // why it failed, when it did +``` + + +Printing the handle renders every field it knows, which is usually faster than +reading them one at a time. + +## Events {#events} + +`result` only tells you anything once the job has finished. Events are written +as it runs, and they outlive the workers that produced them -- so a job that is +still going, or one that never finished, can still account for what it did. + + +```python Python icon="python" +events = job.events(limit=5000, filter="state = 'claim_complete'") +``` + +```typescript TypeScript icon="square-js" +const events = await job.events({ limit: 5000, filter: "state = 'claim_complete'" }); +``` + + +A distributed job records a `claim` / `claim_complete` pair per unit of work, +each carrying `rows_processed`. Summing `rows_processed` over the completed +claims is how you get progress on a job that has no percentage. + +`filter` is a SQL-like expression over `state`, `updated_by`, `emitted_from`, +`emitted_by`, and `claim_entity`. + + +The server returns 1000 event rows by default and 10,000 at most, and +**truncates without telling you**. A job that emits an event per fragment will +silently lose its tail unless you pass `limit`. + + +Events are recorded server-side, so this raises for an operation that ran +in-process. + +## Finding and cancelling jobs {#managing} + +The connection lists and reopens jobs, so a handle lost to a restarted process +is recoverable from its id. + + +```python Python icon="python" +db.list_jobs() # every job the server knows about +job = db.open_job(job_id) # reattach to one +db.cancel_job(job_id) # ask it to stop +``` + +```typescript TypeScript icon="square-js" +await db.listJobs(); // every job the server knows about +const job = await db.openJob(jobId); // reattach to one +await db.cancelJob(jobId); // ask it to stop +``` + + +Cancellation is a request, not a guarantee, and cancelling a job that already +finished does nothing. `JobNotFoundError` means the id does not name a job the +server still has. + + +A job id is opaque. It correlates with server logs and the calls above, but +parsing it or storing it to resume the job later is not supported. + + +## In-process jobs {#in-process} + +A local connection has no server to run anything, so its operations run in your +process. They still return a handle, so the same code works against both, but +one with nothing behind it: `id` is `None`, `status()` reports `"finished"`, +`wait()` returns immediately, `cancel()` does nothing, and `events()` raises. + +This is the difference to expect when moving between a local database and a +deployment -- see [Python, TypeScript, and SQL](/basics/apis). diff --git a/docs/web/basics/lance.mdx b/docs/web/basics/lance.mdx new file mode 100644 index 000000000..b38f63f29 --- /dev/null +++ b/docs/web/basics/lance.mdx @@ -0,0 +1,115 @@ +--- +title: "What about Lance?" +sidebarTitle: "What about Lance" +description: "Lance is the open format underneath. LanceDB is what implements a lakehouse on top of it." +icon: "/static/assets/logo/lance-logo-gray.svg" +keywords: ["lance", "format", "open standard", "lakehouse", "namespace", "catalog"] +--- + +Two names, two things. [Lance](https://lance.org/) is an open format and +specification. LanceDB is an implementation built on it. + +The distinction matters when you are deciding what you are committing to: the +format is a standard other engines read, and your data stays readable by them +whichever LanceDB you run. + +## Lance is the open standard {#standard} + +Lance is an open-source, columnar lakehouse format for multimodal AI. It +specifies three things: + +| | | +| --- | --- | +| **File format** | how bytes are laid out on disk, optimized for random access without giving up scan throughput | +| **Table format** | how files compose into a versioned table -- schema evolution, deletions, indexes | +| **Namespace specification** | how tables are named, grouped, and resolved -- the catalog layer | + +Being a specification rather than a product is the point. Object storage holds +the bytes, and open engines -- PyTorch, Ray, Spark, Trino, DuckDB, Polars -- +read them without going through LanceDB at all. + + + The format specification: design, layout, and how it enables the multimodal lakehouse. + + +## LanceDB embedded implements the storage half {#embedded} + +Connecting to a directory or a bucket gives you a **storage-only lakehouse**. +Your process opens Lance tables directly and does the work itself: it plans and +executes queries, builds indexes, and writes new versions. + +```python +db = lancedb.connect("./.lancedb") +db = lancedb.connect("s3://bucket/path") +``` + +There is no server, which is what makes it embedded -- and also what bounds it. +There is nothing to run work on your behalf, so there are no server-side +[jobs](/basics/jobs), no [SQL endpoint](/basics/apis), and no shared catalog +beyond what the storage layer itself provides. + +## LanceDB Enterprise implements the catalog specification {#enterprise} + +Enterprise A `db://` connection talks to a deployment +that implements the **Lance REST Namespace** -- the REST server defined by the +Lance namespace specification, covering the catalog operations in the +[REST API reference](/api-reference/rest). + +```python +db = lancedb.connect("db://my-database", api_key="...") +``` + +Implementing the specification rather than inventing an interface is what makes +the catalog reachable by anything that speaks it, and what makes +[authorization](/governance/authorization), server-side jobs, and +[SQL](/basics/apis) possible: there is now a server to enforce and to run them. + +The tables are the same Lance tables either way. What changes is who resolves +their names and who does the work. + +## What the format gives you {#capabilities} + +These are properties of Lance, so they hold in both forms: + +| Capability | What it enables | +| --- | --- | +| Multimodal storage | Images, video, audio, text, embeddings, annotations and metadata in one table. | +| First-class blob API | Large binary objects in blob columns, with lazy reads and streaming byte access. | +| Fast random access and scans | Sample, shuffle, and fetch individual rows without giving up sequential throughput. | +| Zero-copy schema evolution | Add, drop, rename, or alter columns without rewriting existing data files. | +| Versioned tables | Reproduce experiments and tie artifacts to the exact table version they used. | +| Hybrid search and indexing | Vector search, full-text search, and scalar filters over the same dataset. | +| Open interoperability | Object storage underneath, open engines on top. | + +Two of them explain behaviour you will meet elsewhere in these docs. + +### Versioning is why old data lingers {#versioning} + +Every write creates a version, and old versions are not immediately destroyed -- +another reader may be part-way through one. A hundred versions are not a hundred +copies of the data, but they do carry a hundred times the metadata, which +eventually slows queries down. + +Deletion works the same way: a deleted row is marked in a deletion file rather +than erased, so the data is recoverable for as long as the version is. + +[Compaction](/maintenance/compaction) is what resolves both. + +### Column evolution is why feature engineering is cheap {#evolution} + +Adding a derived column writes only the new column. Existing data -- the +expensive part, the images and videos -- is untouched. That is what makes +[computed columns](/feature-engineering/computed-columns) a reasonable thing to +do repeatedly on a large table rather than a full rewrite each time. + + + Lance is a separate open source project with its own documentation. + diff --git a/docs/web/catalog/database.mdx b/docs/web/catalog/database.mdx deleted file mode 100644 index f9c78173f..000000000 --- a/docs/web/catalog/database.mdx +++ /dev/null @@ -1,94 +0,0 @@ ---- -title: "Database" -sidebarTitle: "Database" -description: "The root of a catalog. Connecting to one, and the operations it holds." -icon: database ---- - -A database is the root of a catalog. A connection opens exactly one, and every -other object is reached through it. - -## Connecting {#connecting} - -```python -import lancedb - -db = lancedb.connect("./.lancedb") # a local directory -db = lancedb.connect("s3://bucket/path") # object storage -db = lancedb.connect("db://my-database", api_key="...") # Cloud or Enterprise -``` - -The URI decides what kind of database you get, and that decision reaches -further than it looks. A local or object-store URI opens the data directly from -your process. A `db://` URI talks to a deployment, which is what makes -server-side execution — [functions](/feature-engineering/functions), remote refreshes, -[SQL](/sql) — available, and what makes a few local-only features unavailable. - -`connect_async` returns an `AsyncConnection` with the same surface as -coroutines. - -### Storage options {#storage-options} - -`storage_options` configures the underlying store — credentials, endpoints, and -per-table defaults for tables created through this connection: - -```python -db = lancedb.connect( - "s3://bucket/path", - storage_options={"new_table_enable_stable_row_ids": "true"}, -) -``` - - -`new_table_enable_stable_row_ids` is worth setting deliberately. Stable row ids -cannot be enabled after a table exists, and a -[materialized view](/feature-engineering/materialized-views) requires its source to have -them. A table created without them cannot later become a view's source without -being recreated. - - -See [Storage](/storage) for the full set. - -## Operations {#operations} - -| | | -| --- | --- | -| `db.table_names()` | list the tables in the database | -| `db.create_table(name, data)` | create a table — see [Table](/tables) | -| `db.open_table(name)` | open an existing table | -| `db.drop_table(name)` | drop one table | -| `db.drop_database()` | drop everything in the database | - -Each accepts `namespace_path` to act inside a [namespace](/namespaces) -rather than at the root. - -```python -db.table_names() -tbl = db.open_table("documents") -db.drop_table("documents") -``` - - -`drop_database()` removes every table in the database. There is no confirmation -and no undo — table [versioning](/tables/versioning) does not survive the -table being dropped. - - -## Read consistency {#consistency} - -`read_consistency_interval` controls how often a connection checks for versions -committed by other writers. The default reads a snapshot taken when the table -was opened, so a long-lived connection will not see other processes' writes -until it reopens. - -Set it when several processes write the same table and readers must observe -those writes: - -```python -from datetime import timedelta - -db = lancedb.connect(uri, read_consistency_interval=timedelta(seconds=5)) -``` - -The cost is a metadata check on that interval. See -[Consistency](/tables/consistency). diff --git a/docs/web/catalog/index.mdx b/docs/web/catalog/index.mdx deleted file mode 100644 index a993a2bf5..000000000 --- a/docs/web/catalog/index.mdx +++ /dev/null @@ -1,58 +0,0 @@ ---- -title: "Catalog" -sidebarTitle: "Overview" -description: "What lives in a LanceDB catalog: databases, namespaces, tables, views, and functions." -icon: sitemap ---- - -A catalog is what LanceDB knows about. Everything you create lives in one and is -addressed through it — the objects below, arranged in a hierarchy. - -This section covers what each object is and the basic operations on it. Where an -object has depth — how to search a table, how to refresh a view — the page links -to the section that covers it rather than repeating it. - -## The objects {#objects} - -| Object | What it is | -| --- | --- | -| [Database](/catalog/database) | The root. A connection opens one. | -| [Namespace](/namespaces) | A named grouping of tables inside a database. | -| [Table](/tables) | Rows and columns, versioned. The thing most operations act on. | -| [Materialized view](/feature-engineering/materialized-views) | A table whose contents come from a query, computed and stored. | -| [View](/catalog/view) | A saved query. Computed when read, not stored. | -| [Function](/feature-engineering/functions) | Registered code, applied to produce column values. | - -## How they nest {#hierarchy} - -``` -database -└── namespace - ├── table - ├── materialized view (a table, defined by a query) - ├── view (a query, not stored) - └── function -``` - -A namespace can contain further namespaces, so the hierarchy is as deep as your -organisation needs. A table is reached by its path through that hierarchy. - -## Views and materialized views are different objects {#views} - -The names are close enough to mislead. Both are defined by a query over other -tables; the difference is when that query runs. - -- A **materialized view** stores its rows. A refresh computes them, and reads - are reads of stored data — fast, and as current as the last refresh. -- A **view** stores only the query. Every read runs it, so a view is never stale - and never faster than the query it wraps. - -Reach for a materialized view when the query is expensive and the data changes -less often than it is read. Reach for a view when correctness matters more than -latency, or when the query is cheap. - -## What is not here {#elsewhere} - -Indexes belong to a table but are covered under [Maintenance](/indexing), -because creating one is a decision about performance rather than about -structure. diff --git a/docs/web/docs.json b/docs/web/docs.json index 37c874110..bbe60eb53 100644 --- a/docs/web/docs.json +++ b/docs/web/docs.json @@ -30,6 +30,34 @@ { "source": "/tables/update", "destination": "/ingestion/update" + }, + { + "source": "/search/fts-examples", + "destination": "/search/full-text-search" + }, + { + "source": "/search/optimize-queries", + "destination": "/search/explain" + }, + { + "source": "/lance", + "destination": "/basics/lance" + }, + { + "source": "/tables-and-namespaces", + "destination": "/basics/catalog" + }, + { + "source": "/catalog", + "destination": "/basics/catalog" + }, + { + "source": "/catalog/database", + "destination": "/basics/catalog" + }, + { + "source": "/sql", + "destination": "/basics/apis" } ], "navigation": { @@ -45,10 +73,19 @@ ] }, { - "group": "Catalog", + "group": "Basics", "pages": [ - "catalog/index", - "catalog/database", + "basics/catalog", + "basics/apis", + "basics/jobs", + "basics/agentic-skills", + "basics/lance" + ] + }, + { + "group": "Management", + "pages": [ + "management/index", { "group": "Namespace", "pages": [ @@ -64,11 +101,7 @@ "tables/schema", "tables/multimodal" ] - }, - "feature-engineering/materialized-views", - "feature-engineering/functions", - "lance", - "tables-and-namespaces" + } ] }, { @@ -85,16 +118,18 @@ ] }, { - "group": "Search", + "group": "Query", "pages": [ "search/index", + "search/select", "search/vector-search", "search/multivector-search", "search/full-text-search", - "search/fts-examples", "search/hybrid-search", "search/filtering", - "search/optimize-queries", + "search/explain", + "search/explain-analyze", + "sql/functions", { "group": "Reranking", "pages": [ @@ -120,7 +155,9 @@ "group": "Feature engineering", "pages": [ "feature-engineering/index", - "feature-engineering/computed-columns" + "feature-engineering/computed-columns", + "feature-engineering/materialized-views", + "feature-engineering/functions" ] }, { @@ -152,13 +189,6 @@ "performance" ] }, - { - "group": "SQL", - "pages": [ - "sql/index", - "sql/functions" - ] - }, { "group": "Support", "pages": [ diff --git a/docs/web/feature-engineering/computed-columns.mdx b/docs/web/feature-engineering/computed-columns.mdx index 6405698e6..b9f07c4d2 100644 --- a/docs/web/feature-engineering/computed-columns.mdx +++ b/docs/web/feature-engineering/computed-columns.mdx @@ -75,7 +75,7 @@ declared earlier in the same mapping. ## Filling as a job {#async} -Cloud and Enterprise +Enterprise Against a remote database the refresh runs on the server. `refresh_column` raises there; use `refresh_column_async`, which returns a job you can wait on or @@ -95,7 +95,7 @@ a long backfill can be started, left, and checked on later. ## Filling with Python instead of SQL {#functions} -Cloud and Enterprise +Enterprise When the value cannot be expressed in SQL — an embedding, a model call, an image decode — the column is filled by a registered function instead of an expression. diff --git a/docs/web/feature-engineering/functions.mdx b/docs/web/feature-engineering/functions.mdx index 660ef5517..8d5ab604c 100644 --- a/docs/web/feature-engineering/functions.mdx +++ b/docs/web/feature-engineering/functions.mdx @@ -6,7 +6,7 @@ icon: code keywords: ["udf", "functions", "create_function", "embedding", "feature engineering"] --- -Cloud and Enterprise +Enterprise Some columns cannot be written as a SQL expression: an embedding, a model call, an image decode. A function is a Python callable registered with the database, diff --git a/docs/web/feature-engineering/index.mdx b/docs/web/feature-engineering/index.mdx index 00c4c0e5c..527a790ef 100644 --- a/docs/web/feature-engineering/index.mdx +++ b/docs/web/feature-engineering/index.mdx @@ -25,7 +25,7 @@ filled by a registered function — but each is worth understanding on its own. | --- | --- | --- | --- | | [Computed columns](/feature-engineering/computed-columns) | a new column on an existing table | `refresh_column()` | everywhere | | [Materialized views](/feature-engineering/materialized-views) | a new table derived by a query | `refresh()` | local databases | -| [Functions](/feature-engineering/functions) | a computed column filled by Python | a registered function | Cloud and Enterprise | +| [Functions](/feature-engineering/functions) | a computed column filled by Python | a registered function | Enterprise | **Computed columns** are the common case. A column is declared with a SQL expression and starts empty; a refresh gives values to the rows that lack them. diff --git a/docs/web/feature-engineering/materialized-views.mdx b/docs/web/feature-engineering/materialized-views.mdx index 1b27d55b4..534c006fd 100644 --- a/docs/web/feature-engineering/materialized-views.mdx +++ b/docs/web/feature-engineering/materialized-views.mdx @@ -88,7 +88,7 @@ Enterprise documentation if that is where your data lives. The direction is the reverse of [functions](/feature-engineering/functions), -which register only against Cloud and Enterprise and raise locally. Worth +which register only against Enterprise and raise locally. Worth knowing before concluding either is unavailable. ## Working with the view {#working-with-the-view} diff --git a/docs/web/lance.mdx b/docs/web/lance.mdx deleted file mode 100644 index d6db5f17c..000000000 --- a/docs/web/lance.mdx +++ /dev/null @@ -1,99 +0,0 @@ ---- -title: "Lance format" -sidebarTitle: "Lance format" -description: "Open-source lakehouse format for multimodal AI." -icon: "/static/assets/logo/lance-logo-gray.svg" ---- - -[Lance](https://lance.org/) is an open-source, columnar lakehouse format for multimodal AI. -It provides a file format, table format, and lightweight catalog spec, allowing developers -to build a complete open lakehouse on top of object storage. - -Building on top of open foundations and optimizing the format for random access -(without compromising scan performance) enables -high-performance vector search, full-text search, indexing, and feature engineering capabilities. -[LanceDB](/enterprise) builds on these capabilities so teams can work with one multimodal data layer -instead of moving data across separate storage, search, feature, and training systems. - - - Visit the Lance format documentation to learn more about its design, features, and how it enables the multimodal lakehouse. - - -## Capabilities of the Lance format {#capabilities-of-the-lance-format} - -Capability | What it enables ---- | --- -Multimodal storage | Store images, video, audio, text, embeddings, annotations, metadata, features, and more, all in one table. -First-class blob API | Store large binary objects such as images, video, audio, and model artifacts in blob columns with lazy reads and streaming byte access. -Fast random access and scans | Sample, shuffle, and retrieve individual rows efficiently without giving up high-throughput sequential reads. -Flexible data evolution | Add, drop, rename, or alter columns as datasets change, often without rewriting existing data files. -Versioned tables | Reproduce experiments, restore previous states, and tie downstream artifacts to the exact table version they used. -Hybrid search and indexing | Combine vector search, full-text search, and scalar filters on the same dataset with Lance indexes. -Open lakehouse interoperability | Build on object storage and connect Lance tables to open engines such as PyTorch, Ray, Spark, Trino, DuckDB and Polars. - -## Key concepts {#key-concepts} - -The following concepts are core to the Lance format: - - - - **Arrow-native, columnar storage** and **interoperability** with the open lakehouse ecosystem (including other file formats and compute engines). - - - **Zero-copy** data evolution, meaning you can easily add derived columns (like features or embeddings) at a later time, **without full table rewrites**. Only new data is written; expensive existing data (like images/videos) remain untouched. - - - Data is **versioned**, with each insert operation creating a new version of the dataset and an update to the manifest that tracks versions via metadata - - - - -### Data versioning {#data-versioning} - -Data in Lance tables are versioned -- this helps keep LanceDB scalable and consistent. -We do not immediately blow away old versions when creating new ones because other clients might be -in the middle of querying the old version. It's important to retain older versions for as long as they -might be queried. - -Each version contains metadata and just the new/updated data in your transaction. So if you have 100 -versions, they aren't 100 duplicates of the same data. However, they do have 100x the metadata overhead -of a single version, which can result in slower queries. - -### Data compaction {#data-compaction} - -As you insert more data, your dataset will grow and you'll need to perform compaction to maintain query -throughput (i.e., keep latencies down to a minimum). Compaction is the process of merging fragments -together to reduce the amount of metadata that needs to be managed, and to reduce the number of files -that need to be opened while scanning the dataset. - -Running compaction on a Lance dataset will do the following: - -- Remove deleted rows from fragments -- Remove dropped columns from fragments -- Merge small fragments into larger ones - -Compaction focuses on read performance, not immediate disk reclamation. During compaction, Lance writes -new compacted files while older files are still referenced by previous table versions. This means disk -usage can increase temporarily until old versions are cleaned up. - -### Data deletion and recovery {#data-deletion-and-recovery} - -Although Lance allows you to delete rows from a dataset, it does not actually delete the data immediately. -It simply marks the row as deleted in the `DataFile` that represents a fragment. - -For a given version of the dataset, each fragment can have up to one deletion file (if no rows were ever -deleted from that fragment, it will not have a deletion file). This is important to keep in mind because -it means that the data is still there, and can be recovered if needed, as long as that version still -exists based on your backup policy. - - - Lance is a separate open source project. Check out its documentation to learn more. - diff --git a/docs/web/management/index.mdx b/docs/web/management/index.mdx new file mode 100644 index 000000000..3ec4622a7 --- /dev/null +++ b/docs/web/management/index.mdx @@ -0,0 +1,152 @@ +--- +title: "Management" +sidebarTitle: "Overview" +description: "Create, list, and drop the objects in a catalog, from Python, TypeScript, or SQL." +icon: folder-tree +keywords: ["ddl", "create", "drop", "list", "catalog", "management"] +--- + +Every [catalog object](/basics/catalog) is created, listed, and dropped the same +few ways. This section is the reference for those operations; the pages linked +from it cover what each object is for. + +Operations that can take a while have an `_async` form returning a +[job](/basics/jobs). + +## The surface at a glance {#surface} + + +SQL requires a deployment -- a local database has no engine to plan a statement. +See [Python, TypeScript, and SQL](/basics/apis). + + +### Namespaces {#namespaces} + +| | Python | TypeScript | SQL | +| --- | --- | --- | --- | +| create | `create_namespace` | `createNamespace` | -- | +| list | `list_namespaces` | `listNamespaces` | -- | +| describe | `describe_namespace` | `describeNamespace` | -- | +| exists | `namespace_exists` | -- | -- | +| drop | `drop_namespace` | `dropNamespace` | -- | + +See [Namespaces](/namespaces). + +### Tables {#tables} + +| | Python | TypeScript | SQL | +| --- | --- | --- | --- | +| create | `create_table` | `createTable` | `CREATE TABLE` | +| create empty | `create_table(schema=)` | `createEmptyTable` | `CREATE TABLE` | +| open | `open_table` | `openTable` | -- | +| list | `table_names`, `list_tables` | `tableNames`, `listTables` | `SHOW TABLES` | +| exists | `table_exists` | -- | -- | +| rename | `rename_table` | `renameTable` | -- | +| clone | `clone_table` | `cloneTable` | -- | +| drop | `drop_table`, `drop_table_async` | `dropTable`, `dropTableAsync` | `DROP TABLE` | +| drop all | `drop_all_tables` | `dropAllTables` | -- | + +See [Tables](/tables) for creating one and shaping its schema. + +### Materialized views {#materialized-views} + +| | Python | TypeScript | SQL | +| --- | --- | --- | --- | +| create | `create_materialized_view` | `createMaterializedView` | -- | +| open | `open_materialized_view` | `openMaterializedView` | -- | +| list | `list_materialized_views` | `listMaterializedViews` | `SHOW MATERIALIZED VIEWS` | +| refresh | `refresh` | `refresh` | `REFRESH MATERIALIZED VIEW` | +| drop | `drop_table` | `dropTable` | `DROP TABLE` | + + +`list_materialized_views` reads every table's schema to find them, so it costs +one open per table. `SHOW MATERIALIZED VIEWS` asks the server instead. + + +See [Materialized views](/feature-engineering/materialized-views). + +### Views {#views} + +| | Python | TypeScript | SQL | +| --- | --- | --- | --- | +| create | -- | -- | `CREATE VIEW` | +| drop | -- | -- | `DROP VIEW` | + +Enterprise Views exist only in SQL -- there is no SDK +method for them, because the thing a view stores *is* a SQL statement. See +[Views](/management/view). + +### Functions {#functions} + +| | Python | TypeScript | SQL | +| --- | --- | --- | --- | +| create | `create_function`, `create_function_async` | -- | `CREATE FUNCTION` | +| get | `get_function` | -- | -- | +| list | `list_functions` | -- | `SHOW FUNCTIONS` | +| drop | `drop_function` | -- | -- | + +Enterprise A function runs on the server, so there is +no local form. The TypeScript SDK has no function methods; use Python or SQL. +See [Functions](/feature-engineering/functions). + +### Jobs {#jobs} + +| | Python | TypeScript | SQL | +| --- | --- | --- | --- | +| list | `list_jobs` | `listJobs` | `SHOW JOBS` | +| open | `open_job` | `openJob` | -- | +| cancel | `cancel_job` | `cancelJob` | -- | +| errors | `job.events` | `job.events` | `SHOW ERRORS` | + +See [Jobs](/basics/jobs). + +### Indexes {#indexes} + +| | Python | TypeScript | SQL | +| --- | --- | --- | --- | +| create | `create_index`, `create_index_async` | `createIndex`, `createIndexAsync` | `CREATE INDEX` | +| list | `list_indices` | `listIndices` | -- | +| stats | `index_stats` | `indexStats` | -- | +| drop | `drop_index` | `dropIndex` | `DROP INDEX` | + + +SQL `CREATE INDEX` covers one column and takes no user-chosen name -- the server +names it. For a multi-column or named index, use the SDK. + + +Indexes are a performance decision rather than a structural one, so they are +covered under [Maintenance](/indexing). + +## Namespaces qualify everything {#namespace-paths} + +Most connection-level operations take a `namespace_path` (`namespacePath`) to +act inside a namespace rather than at the database root: + + +```python Python icon="python" +db.table_names(namespace_path=["prod", "events"]) +tbl = db.open_table("clicks", namespace_path=["prod", "events"]) +``` + +```typescript TypeScript icon="square-js" +await db.tableNames({ namespacePath: ["prod", "events"] }); +const tbl = await db.openTable("clicks", { namespacePath: ["prod", "events"] }); +``` + +```sql SQL +SELECT * FROM prod.events.clicks +``` + + +In SQL the path is written as dotted identifiers instead. + +## Dropping is not undoable {#dropping} + + +`drop_table` removes the table and every version of it. Table +[versioning](/tables/versioning) does not survive the table being dropped, and +neither `drop_database` nor `drop_all_tables` asks for confirmation. + + +Dropping a **view** removes only the saved query. Dropping a **materialized +view** removes stored rows, because a materialized view is a table. diff --git a/docs/web/performance.mdx b/docs/web/performance.mdx index 0d54f7eb0..5d87eba26 100644 --- a/docs/web/performance.mdx +++ b/docs/web/performance.mdx @@ -221,13 +221,13 @@ print(table.index_stats("vector_idx")) # num_unindexed_rows should be ~0 | `LanceScan` with high `bytes_read` / `iops` | Add a missing index, project columns with `select()`, or check that the dataset has been compacted | | Multiple sequential filters | Reorder filter conditions | -[Optimize Query Performance](/search/optimize-queries) walks through a fully worked before/after example, including how `KNNVectorDistance` and `output_batches` change once indexes are in place. +[Explain Analyze](/search/explain-analyze) walks through a fully worked before/after example, including how `KNNVectorDistance` and `output_batches` change once indexes are in place. ## Where to go next {#where-to-go-next} - - Read execution plans, find the bottleneck. + + Execute a query and read where the time went. Index types, parameters, and tuning in depth. diff --git a/docs/web/search/optimize-queries.mdx b/docs/web/search/explain-analyze.mdx similarity index 57% rename from docs/web/search/optimize-queries.mdx rename to docs/web/search/explain-analyze.mdx index 244e87633..6e4ad7bbe 100644 --- a/docs/web/search/optimize-queries.mdx +++ b/docs/web/search/explain-analyze.mdx @@ -1,29 +1,17 @@ --- -title: "Optimize Query Performance" -sidebarTitle: Query optimization -description: "Analyze and optimize query performance in LanceDB." +title: "Explain analyze a query plan" +sidebarTitle: Explain Analyze +description: "Execute a query and read the runtime metrics attached to every operator." icon: "gauge" --- -LanceDB provides two powerful tools for query analysis and optimization: `explain_plan` and `analyze_plan`. Let's take a better look at how they work: +`analyze_plan` executes the query and returns the physical plan annotated with +runtime metrics -- elapsed time, rows produced, bytes read, index comparisons. +Where [`explain_plan`](/search/explain) tells you what a query intends to do, +this tells you what it cost. -| Method | Purpose | Description | -| :------------- | :----------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `explain_plan` | Query Analysis | Print the resolved query plan to understand how the query will be executed. Helpful for identifying slow queries or unexpected query results. | -| `analyze_plan` | Performance Tuning | Execute the query and return a physical execution plan annotated with runtime metrics including execution time, number of rows processed, and I/O stats. Essential for performance tuning and debugging. | +## What it measures {#analyze-a-query-plan} -## Query Analysis Tools {#query-analysis-tools} - -### Inspect a query plan {#inspect-a-query-plan} - -Reveals the logical query plan before execution, helping you identify potential issues with query structure and index usage. This tool is useful for: - -- Verifying query optimization strategies -- Validating index selection -- Understanding query execution order -- Detecting missing indices - -### Analyze a query plan {#analyze-a-query-plan} Executes the query and provides detailed runtime metrics, including: - Operation duration (`_elapsed_compute_`) @@ -31,126 +19,6 @@ Executes the query and provides detailed runtime metrics, including: - Index effectiveness (`_index_comparisons_`, `_indices_loaded_`) - Resource utilization (`_iops_`, `_requests_`) -Together, these tools offer a comprehensive view of query performance, from planning to execution. Use `explain_plan` to verify your query structure and `analyze_plan` to measure and optimize actual performance. - -Metadata filters are prefiltered by default, which usually shows the filter pushed into the -`LanceScan` or index scan. If you set `prefilter=False`, expect a separate `FilterExec` after -search instead; that can be useful for some expensive filters, but it changes both latency and -the number of rows available after filtering. - -## Reading the Execution Plan {#reading-the-execution-plan} - -To demonstrate query performance analysis, we'll use a table containing 1.2M rows sampled from the [Wikipedia dataset](https://huggingface.co/datasets/wikimedia/wikipedia). Initially, the table has no indices, allowing us to observe the impact of optimization. - -Let's examine a vector search query that: - -- Filters rows where `identifier` is between 0 and 1,000,000 -- Returns the top 100 matches -- Projects specific columns: `chunk_index`, `title`, and `identifier` - - -```python Python icon="python" -# explain_plan -query_explain_plan = ( - table.search(query_embed) - .where("identifier > 0 AND identifier < 1000000") - .select(["chunk_index", "title", "identifier"]) - .limit(100) - .explain_plan(True) -) -``` - -```typescript TypeScript icon="square-js" -// explain_plan -const explainPlan = await table - .search(queryEmbed) - .where("identifier > 0 AND identifier < 1000000") - .select(["chunk_index", "title", "identifier"]) - .limit(100) - .explainPlan(true); -``` - - -### Execution Plan Components {#execution-plan-components} - -The execution plan reveals the sequence of operations performed to execute your query. Let's examine each component: - -``` -ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance] - RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title" - CoalesceBatchesExec: target_batch_size=1024 - GlobalLimitExec: skip=0, fetch=100 - FilterExec: _distance@3 IS NOT NULL - SortExec: TopK(fetch=100), expr=[_distance@3 ASC NULLS LAST], preserve_partitioning=[false] - KNNVectorDistance: metric=l2 - FilterExec: identifier@1 > 0 AND identifier@1 < 1000000 - LanceScan: uri=***, projection=[vector, identifier], row_id=true, row_addr=false, ordered=false -``` - -#### 1\. Base Layer (LanceScan) {#1-base-layer-lancescan} - -- Initial data scan loading only specified columns to minimize I/O -- Unordered scan enabling parallel processing - -``` -LanceScan: -- projection=[vector, identifier] -- row_id=true, row_addr=false, ordered=false -``` - -#### 2\. First Filter {#2-first-filter} - -- Apply requested filter on `identifier` column -- Reduces the number of vectors that need KNN computation - -``` -FilterExec: identifier@1 > 0 AND identifier@1 < 1000000 -``` - -#### 3\. Vector Search {#3-vector-search} - -- Computes L2 (Euclidean) distances between query vector and all vectors that passed the filter - -``` -KNNVectorDistance: metric=l2 -``` - -#### 4\. Results Processing {#4-results-processing} - -- Filters out null distance results -- Sorts by distance and takes top 100 results -- Processes in batches of 1024 for optimal memory usage - -``` -SortExec: TopK(fetch=100) -- expr=[_distance@3 ASC NULLS LAST] -- preserve_partitioning=[false] -FilterExec: _distance@3 IS NOT NULL -GlobalLimitExec: skip=0, fetch=100 -CoalesceBatchesExec: target_batch_size=1024 -``` - -#### 5\. Data Retrieval {#5-data-retrieval} - -- `RemoteTake` is a key component of Lance's I/O cache -- Handles efficient data retrieval from remote storage locations -- Fetches specific rows and columns needed for the final output -- Optimizes network bandwidth by only retrieving required data - -``` -RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title" -``` - -#### 6\. Final Output {#6-final-output} - -- Returns only requested columns and maintains column ordering - -```python -ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance] -``` - -This plan demonstrates a basic search without index optimizations: it performs a full scan and filter before vector search. - ## Performance Analysis {#performance-analysis} Let's use `analyze_plan` to run the query and analyze the query performance, which will help us identify potential bottlenecks: @@ -421,9 +289,11 @@ A well-optimized table should have `num_unindexed_rows ~ 0`. | | Check whether the dataset has been compacted | | Multiple sequential filters | Reorder filter conditions | -!!! note "Regular Performance Analysis" - Regularly analyze your query plans to identify and address performance bottlenecks. - The `analyze_plan` output provides detailed metrics to guide optimization efforts. + +Regularly analyze your query plans to identify and address performance +bottlenecks. The `analyze_plan` output provides detailed metrics to guide +optimization efforts. + ### 3\. Getting Started with Optimization {#3-getting-started-with-optimization} @@ -432,52 +302,9 @@ For vector search performance: - Create ANN index on your vector column(s) as described in the [index guide](/indexing/vector-index/) - If you often filter by metadata, create [scalar indices](/indexing/scalar-index/) on those columns -## Analyzing non-vector queries {#analyzing-non-vector-queries} - -`explain_plan` and `analyze_plan` aren't vector-specific — they're available on every query builder, including FTS and hybrid. The most common reason to look at the plan for a non-vector query is to confirm whether your `where` clause pushed into the scan (good) or ran as a separate `FilterExec` step on top of the search results (often slower, and a hint that the filter column needs a scalar index). - -### FTS queries {#fts-queries} - - -```python Python icon="python" -plan = ( - table.search("puppy", query_type="fts") - .where("category = 'animals'", prefilter=True) - .limit(10) - .explain_plan(True) -) -print(plan) -``` - -```typescript TypeScript icon="square-js" -const plan = await table - .query() - .fullTextSearch("puppy") - .where("category = 'animals'") - .limit(10) - .explainPlan(true); -``` - - -In an indexed FTS plan you should see a `MatchQuery` (or other FTS execution node) reading from the inverted index, with the metadata filter pushed down. If the plan shows a `LanceScan` followed by `FilterExec` over the entire text column, the FTS index either isn't covering the column or the filter isn't using a scalar index — both worth investigating. - ### Hybrid queries {#hybrid-queries} -For hybrid queries, `explain_plan` returns the reranker label followed by the vector and FTS sub-plans, indented for readability: - - -```python Python icon="python" -plan = ( - table.search("flower moon", query_type="hybrid") - .where("category = 'film'", prefilter=True) - .limit(10) - .explain_plan(True) -) -print(plan) -# RRFReranker(...) -# -# -``` - - -`analyze_plan` does the same, but executes both sub-queries and labels them as `Vector Search Plan:` and `FTS Search Plan:` in the output. This is the easiest way to see whether the filter pushed into both halves uniformly, and which half is dominating latency. +For a hybrid query, `analyze_plan` executes both sub-queries and labels them +`Vector Search Plan:` and `FTS Search Plan:` in the output. This is the easiest +way to see whether the filter pushed into both halves uniformly, and which half +is dominating latency. diff --git a/docs/web/search/explain.mdx b/docs/web/search/explain.mdx new file mode 100644 index 000000000..6c73c8c65 --- /dev/null +++ b/docs/web/search/explain.mdx @@ -0,0 +1,197 @@ +--- +title: "Explain a query plan" +sidebarTitle: Explain +description: "Read the plan LanceDB resolved for a query, before running it." +icon: "map" +--- + +`explain_plan` prints the plan LanceDB resolved for a query without executing +it. It answers structural questions -- did the filter push into the scan, is the +vector index being used, which columns are actually read -- and costs nothing to +run, so it is the first thing to reach for when a query behaves unexpectedly. + +To measure a query rather than read it, use +[`analyze_plan`](/search/explain-analyze), which executes it and annotates every +operator with runtime metrics. + +## What it shows {#inspect-a-query-plan} + +Reveals the logical query plan before execution, helping you identify potential issues with query structure and index usage. This tool is useful for: + +- Verifying query optimization strategies +- Validating index selection +- Understanding query execution order +- Detecting missing indices + +Metadata filters are prefiltered by default, which usually shows the filter pushed into the +`LanceScan` or index scan. If you set `prefilter=False`, expect a separate `FilterExec` after +search instead; that can be useful for some expensive filters, but it changes both latency and +the number of rows available after filtering. + +## Reading the Execution Plan {#reading-the-execution-plan} + +To demonstrate query performance analysis, we'll use a table containing 1.2M rows sampled from the [Wikipedia dataset](https://huggingface.co/datasets/wikimedia/wikipedia). Initially, the table has no indices, allowing us to observe the impact of optimization. + +Let's examine a vector search query that: + +- Filters rows where `identifier` is between 0 and 1,000,000 +- Returns the top 100 matches +- Projects specific columns: `chunk_index`, `title`, and `identifier` + + +```python Python icon="python" +# explain_plan +query_explain_plan = ( + table.search(query_embed) + .where("identifier > 0 AND identifier < 1000000") + .select(["chunk_index", "title", "identifier"]) + .limit(100) + .explain_plan(True) +) +``` + +```typescript TypeScript icon="square-js" +// explain_plan +const explainPlan = await table + .search(queryEmbed) + .where("identifier > 0 AND identifier < 1000000") + .select(["chunk_index", "title", "identifier"]) + .limit(100) + .explainPlan(true); +``` + + +### Execution Plan Components {#execution-plan-components} + +The execution plan reveals the sequence of operations performed to execute your query. Let's examine each component: + +``` +ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance] + RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title" + CoalesceBatchesExec: target_batch_size=1024 + GlobalLimitExec: skip=0, fetch=100 + FilterExec: _distance@3 IS NOT NULL + SortExec: TopK(fetch=100), expr=[_distance@3 ASC NULLS LAST], preserve_partitioning=[false] + KNNVectorDistance: metric=l2 + FilterExec: identifier@1 > 0 AND identifier@1 < 1000000 + LanceScan: uri=***, projection=[vector, identifier], row_id=true, row_addr=false, ordered=false +``` + +#### 1\. Base Layer (LanceScan) {#1-base-layer-lancescan} + +- Initial data scan loading only specified columns to minimize I/O +- Unordered scan enabling parallel processing + +``` +LanceScan: +- projection=[vector, identifier] +- row_id=true, row_addr=false, ordered=false +``` + +#### 2\. First Filter {#2-first-filter} + +- Apply requested filter on `identifier` column +- Reduces the number of vectors that need KNN computation + +``` +FilterExec: identifier@1 > 0 AND identifier@1 < 1000000 +``` + +#### 3\. Vector Search {#3-vector-search} + +- Computes L2 (Euclidean) distances between query vector and all vectors that passed the filter + +``` +KNNVectorDistance: metric=l2 +``` + +#### 4\. Results Processing {#4-results-processing} + +- Filters out null distance results +- Sorts by distance and takes top 100 results +- Processes in batches of 1024 for optimal memory usage + +``` +SortExec: TopK(fetch=100) +- expr=[_distance@3 ASC NULLS LAST] +- preserve_partitioning=[false] +FilterExec: _distance@3 IS NOT NULL +GlobalLimitExec: skip=0, fetch=100 +CoalesceBatchesExec: target_batch_size=1024 +``` + +#### 5\. Data Retrieval {#5-data-retrieval} + +- `RemoteTake` is a key component of Lance's I/O cache +- Handles efficient data retrieval from remote storage locations +- Fetches specific rows and columns needed for the final output +- Optimizes network bandwidth by only retrieving required data + +``` +RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title" +``` + +#### 6\. Final Output {#6-final-output} + +- Returns only requested columns and maintains column ordering + +```python +ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance] +``` + +This plan demonstrates a basic search without index optimizations: it performs a full scan and filter before vector search. + +## Non-vector queries {#analyzing-non-vector-queries} + +`explain_plan` is not vector-specific -- it is available on every query builder, +including FTS and hybrid. The most common reason to read the plan for a +non-vector query is to confirm whether your `where` clause pushed into the scan +(good) or ran as a separate `FilterExec` on top of the search results (often +slower, and a hint that the filter column needs a scalar index). + +### FTS queries {#fts-queries} + + +```python Python icon="python" +plan = ( + table.search("puppy", query_type="fts") + .where("category = 'animals'", prefilter=True) + .limit(10) + .explain_plan(True) +) +print(plan) +``` + +```typescript TypeScript icon="square-js" +const plan = await table + .query() + .fullTextSearch("puppy") + .where("category = 'animals'") + .limit(10) + .explainPlan(true); +``` + + +In an indexed FTS plan you should see a `MatchQuery` (or other FTS execution node) reading from the inverted index, with the metadata filter pushed down. If the plan shows a `LanceScan` followed by `FilterExec` over the entire text column, the FTS index either isn't covering the column or the filter isn't using a scalar index — both worth investigating. + +### Hybrid queries {#hybrid-queries} + +For hybrid queries, `explain_plan` returns the reranker label followed by the vector and FTS sub-plans, indented for readability: + + +```python Python icon="python" +plan = ( + table.search("flower moon", query_type="hybrid") + .where("category = 'film'", prefilter=True) + .limit(10) + .explain_plan(True) +) +print(plan) +# RRFReranker(...) +# +# +``` + + +Reading the plan tells you the shape. To find out which half dominates latency, +[analyze it](/search/explain-analyze#hybrid-queries). diff --git a/docs/web/search/fts-examples.mdx b/docs/web/search/fts-examples.mdx deleted file mode 100644 index dff442673..000000000 --- a/docs/web/search/fts-examples.mdx +++ /dev/null @@ -1,561 +0,0 @@ ---- -title: Full-Text Search Examples -sidebarTitle: FTS Examples -description: Worked examples of fuzzy search, boosting, boolean queries, and substring search with LanceDB full-text search. -icon: "book-open" ---- - -These worked examples build on the concepts from the [Full-Text Search guide](/search/full-text-search). They walk through creating sample tables, building FTS indices, and running fuzzy, phrase, boosted, boolean, and substring queries. - -## Fuzzy Search and Boosting Example {#fuzzy-search-and-boosting-example} - -### Generate Data {#generate-data} - -First, let's create a table with sample text data for testing fuzzy search: - - -```python Python icon="python" -import lancedb -import numpy as np -import pandas as pd -import random - -# Connect to LanceDB -db = lancedb.connect( - uri="db://your-project-slug", - api_key="your-api-key", - region="us-east-1" -) - -# Generate 100 rows of random " " text -table_name = "fts-fuzzy-boosting-test" -vectors = [np.random.randn(128) for _ in range(100)] -verbs = ("runs", "hits", "jumps", "drives", "barfs") -adv = ("crazily.", "dutifully.", "foolishly.", "merrily.", "occasionally.") -adj = ("adorable", "clueless", "dirty", "odd", "stupid") - -def sentence(nouns): - return " ".join(random.choice(words) for words in (nouns, verbs, adv, adj)) - -text = [sentence(("puppy", "car")) for _ in range(100)] -text2 = [sentence(("rabbit", "girl", "monkey")) for _ in range(100)] -count = [random.randint(1, 10000) for _ in range(100)] -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb" - -const db = await lancedb.connect({ - uri: "db://your-project-slug", - apiKey: "your-api-key", - region: "us-east-1" -}); - -// Generate 100 rows of random " " text -const tableName = "fts-fuzzy-boosting-test-ts"; -const n = 100; -const verbs = ["runs", "hits", "jumps", "drives", "barfs"]; -const adverbs = ["crazily", "dutifully", "foolishly", "merrily", "occasionally"]; -const adjectives = ["adorable", "clueless", "dirty", "odd", "stupid"]; - -const pick = (words: string[]) => words[Math.floor(Math.random() * words.length)]; -const sentence = (nouns: string[]) => - [nouns, verbs, adverbs, adjectives].map(pick).join(" "); - -const vectors = Array.from({ length: n }, () => - Array.from({ length: 128 }, () => Math.random() * 2 - 1) -); -const text = Array.from({ length: n }, () => sentence(["puppy", "car"])); -const text2 = Array.from({ length: n }, () => sentence(["rabbit", "girl", "monkey"])); -const count = Array.from({ length: n }, () => Math.floor(Math.random() * 10000) + 1); -``` - - -### Create Table {#create-table} - - -```python Python icon="python" -# Create table with sample data -table = db.create_table( - table_name, - data=pd.DataFrame({ - "vector": vectors, - "id": [i % 2 for i in range(100)], - "text": text, - "text2": text2, - "count": count, - }), - mode="overwrite" -) -``` - -```typescript TypeScript icon="square-js" -// Create table with sample data -const data = makeArrowTable( - vectors.map((vector, i) => ({ - vector, - id: i % 2, - text: text[i], - text2: text2[i], - count: count[i], - })) -); - -const table = await db.createTable(tableName, data, { mode: "overwrite" }); -``` - - -### Construct FTS Index {#construct-fts-index} - -Create a full-text search index on the first text column: - - -```python Python icon="python" -# Create FTS index on first text column -table.create_fts_index("text") -wait_for_index(table, "text_idx") -``` - -```typescript TypeScript icon="square-js" -// Create FTS index on first text column -await table.createIndex("text", { config: Index.fts() }); -await waitForIndex(table, "text_idx"); -``` - - -Then, create an index on the second text column: - - -```python Python icon="python" -# Create FTS index on second text column -table.create_fts_index("text2") -wait_for_index(table, "text2_idx") -``` - -```typescript TypeScript icon="square-js" -// Create FTS index on second text column -await table.createIndex("text2", { config: Index.fts() }); -await waitForIndex(table, "text2_idx"); -``` - - -### Basic and Fuzzy Search {#basic-and-fuzzy-search} - -Now we can perform basic, fuzzy, and prefix match searches: - -#### Basic Exact Search {#basic-exact-search} - - -```python Python icon="python" -from lancedb.query import MatchQuery - -# Basic match (exact search) -basic_match_results = ( - table.search(MatchQuery("crazily", "text")) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" -import { MatchQuery } from "@lancedb/lancedb"; - -// Basic match (exact search) -const basicMatchResults = await table.query() - .fullTextSearch(new MatchQuery("crazily", "text")) - .select(["id", "text"]) - .limit(100) - .toArray(); -``` - - -#### Fuzzy Search with Typos {#fuzzy-search-with-typos} - - -```python Python icon="python" -# Fuzzy match (allows typos) -fuzzy_results = ( - table.search(MatchQuery("craziou", "text", fuzziness=2)) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" -// Fuzzy match (allows typos) -const fuzzyResults = await table.query() - .fullTextSearch(new MatchQuery("craziou", "text", { - fuzziness: 2, - })) - .select(["id", "text"]) - .limit(100) - .toArray(); -``` - - -#### Prefix based Match {#prefix-based-match} - -Prefix-based match allows you to search for documents containing words that start with a specific prefix. - - -```python Python icon="python" -# Fuzzy match (allows typos) -fuzzy_results = ( - table.search(MatchQuery("cra", "text", prefix_length=3)) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" -// Fuzzy match (allows typos) -const fuzzyResults = await table.query() - .fullTextSearch(new MatchQuery("cra", "text", { - prefixLength: 3, - })) - .select(["id", "text"]) - .limit(100) - .toArray(); -``` - - -### Phrase Match {#phrase-match} - -Phrase matching enables you to search for exact sequences of words. Unlike regular text search -which matches individual terms independently, phrase matching requires words to appear in the -specified order with no intervening terms. - - -Phrase queries are supported but only for a single column; providing multiple columns with a quoted phrase raises an error. - - -Phrase matching is particularly useful for: - -- Searching for specific multi-word expressions -- Matching exact titles or quotes -- Finding precise word combinations in a specific order - - -```python Python icon="python" -# Exact phrase match -from lancedb.query import PhraseQuery - -phrase_results = ( - table.search(PhraseQuery("puppy runs", "text")) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" -import { PhraseQuery } from "@lancedb/lancedb"; - -// Exact phrase match -const phraseResults = await table.query() - .fullTextSearch(new PhraseQuery("puppy runs", "text")) - .select(["id", "text"]) - .limit(100) - .toArray(); -``` - - -#### Flexible Phrase Match {#flexible-phrase-match} -To provide more flexible phrase matching, LanceDB supports the `slop` parameter. This allows you to match phrases where the terms appear close to each other, even if they are not directly adjacent or in the exact order, as long as they are within the specified `slop` value. - -For example, the phrase query "puppy merrily" would not return any results by default. However, if you set `slop=1`, it will match phrases like "puppy jumps merrily", "puppy runs merrily", and similar variations where one word appears between "puppy" and "merrily". - - -```python Python icon="python" -# Flexible phrase match with slop=1 for 'puppy merrily' -from lancedb.query import PhraseQuery - -phrase_results = ( - table.search(PhraseQuery("puppy merrily", "text", slop=1)) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" -import { PhraseQuery } from "@lancedb/lancedb"; - -// Flexible phrase match with slop=1 for 'puppy runs' -const phraseResults = await table.query() - .fullTextSearch(new PhraseQuery("puppy runs", "text", { slop: 1 })) - .select(["id", "text"]) - .limit(100) - .toArray(); -``` - - -### Search with Boosting {#search-with-boosting} - -Boosting allows you to control the relative importance of different search terms or fields -in your queries. This feature is particularly useful when you need to: - -* Prioritize matches in certain columns -* Promote specific terms while demoting others -* Fine-tune relevance scoring for better search results - -| Parameter | Type | Default | Description | -| -------------- | ----- | -------- | ------------------------------------------------------------------ | -| positive | Query | required | The primary query terms to match and promote in results | -| negative | Query | required | Terms to demote in the search results | -| negative_boost | float | 0.5 | Multiplier for negative matches (lower values = stronger demotion) | - - -```python Python icon="python" -from lancedb.query import MatchQuery, BoostQuery, MultiMatchQuery - -# Boost data with 'runs' in text more than 'puppy' in text -boosting_results = ( - table.search( - BoostQuery( - MatchQuery("runs", "text"), - MatchQuery("puppy", "text"), - negative_boost=0.2, - ), - ) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) - -# Search across both text and text2 -multi_match_results = ( - table.search(MultiMatchQuery("crazily", ["text", "text2"])) - .select(["id", "text", "text2"]) - .limit(100) - .to_pandas() -) - -# Search with field boosting -multi_match_boosting_results = ( - table.search( - MultiMatchQuery("crazily", ["text", "text2"], boosts=[1.0, 2.0]), - ) - .select(["id", "text", "text2"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" -import { MatchQuery, BoostQuery, MultiMatchQuery } from "@lancedb/lancedb"; - -// Boosting Example -const boostingResults = await table.query() - .fullTextSearch(new BoostQuery(new MatchQuery("runs", "text"), new MatchQuery("puppy", "text"), { - negativeBoost: 0.2, - })) - .select(["id", "text"]) - .limit(100) - .toArray(); - -// Search across both text fields -const multiMatchResults = await table.query() - .fullTextSearch(new MultiMatchQuery("crazily", ["text", "text2"])) - .select(["id", "text", "text2"]) - .limit(100) - .toArray(); - -// Search with field boosting -const multiMatchBoostingResults = await table.query() - .fullTextSearch(new MultiMatchQuery("crazily", ["text", "text2"], { - boosts: [1.0, 2.0], - })) - .select(["id", "text", "text2"]) - .limit(100) - .toArray(); -``` - - - -- Use fuzzy search when handling user input that may contain typos or variations -- Apply field boosting to prioritize matches in more important columns -- Combine fuzzy search with boosting for robust and precise search results - -**Recommendations for optimal FTS performance:** - -- Create full-text search indices on text columns that will be frequently searched -- For hybrid search combining text and vectors, see our [hybrid search guide](/search/hybrid-search/) -- For performance benchmarks, check our [benchmark results](/enterprise/benchmarks/) -- For complex queries, use SQL to combine FTS with other filter conditions - - -### Boolean Queries {#boolean-queries} -LanceDB supports boolean logic in full-text search, allowing you to combine multiple queries using `and` and `or` operators. This is useful when you want to match documents that satisfy multiple conditions (intersection) or at least one of several conditions (union). - -#### Combining Two Match Queries {#combining-two-match-queries} - -In Python, you can combine two MatchQuery objects using either the `and` function or the `&` operator (e.g., `MatchQuery("puppy", "text") and MatchQuery("merrily", "text")`); both methods are supported and yield the same result. Similarly, you can use either the `or` function or the `|` operator to perform an or query. - -In TypeScript, boolean queries are constructed using the `BooleanQuery` class with a list of [Occur, subquery] pairs. For example, to perform an AND query: - -```sql SQL icon="code" -BooleanQuery([ -[Occur.Must, new MatchQuery("puppy", "text")], -[Occur.Must, new MatchQuery("merrily", "text")], -]) -``` - -This approach allows you to specify complex boolean logic by combining multiple subqueries with different Occur values (such as `Must`, `Should`, or `MustNot`). - - -**Which queries are allowed?** - -A boolean query must include at least one `SHOULD` or `MUST` clause. Queries that contain only a `MUST_NOT` clause are not allowed. - - - -```python Python icon="python" -from lancedb.query import MatchQuery - -# Example: Find documents containing both "puppy" and "merrily" -and_query = MatchQuery("puppy", "text") & MatchQuery("merrily", "text") -and_results = ( - table.search(and_query) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) - -# Example: Find documents containing either "puppy" or "merrily" -or_query = MatchQuery("puppy", "text") | MatchQuery("merrily", "text") -or_results = ( - table.search(or_query) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" expandable=true -import { MatchQuery, BooleanQuery, Occur } from "@lancedb/lancedb"; - -// Flexible boolean queries with MatchQuery - -// Find documents containing both "puppy" and "merrily" -const mustResults = await table - .search( - new BooleanQuery([ - [Occur.Must, new MatchQuery("puppy", "text")], - [Occur.Must, new MatchQuery("merrily", "text")], - ]), - ) - .select(["id", "text"]) - .limit(100) - .toArray(); - -// Find documents containing either "puppy" or "merrily" -const shouldResults = await table - .search( - new BooleanQuery([ - [Occur.Should, new MatchQuery("puppy", "text")], - [Occur.Should, new MatchQuery("merrily", "text")], - ]), - ) - .select(["id", "text"]) - .limit(100) - .toArray(); -``` - - - -**How to use booleans?** - -- Use `and`/`&`(Python), `Occur.Must`(Typescript) for intersection (documents must match all queries). -- Use `or`/`|`(Python), `Occur.Should`(Typescript) for union (documents must match at least one query). - - -## Substring Search Example {#substring-search-example} - -LanceDB supports searching for substrings in text columns using n-gram tokenization. This is useful for finding partial matches within text content. - -### Setting Up the Table {#setting-up-the-table} - -First, create a table with sample text data and configure n-gram tokenization: - - -```python Python icon="python" -import pyarrow as pa -import lancedb - -db = lancedb.connect(":memory:") - -data = pa.table({"text": ["hello world", "lance database", "lance is cool"]}) -table = db.create_table("test", data=data) -table.create_fts_index("text", base_tokenizer="ngram") -``` - - -### Basic Substring Search {#basic-substring-search} - -With the default n-gram settings (minimum length of 3), you can search for substrings of length 3 or more: - - -```python Python icon="python" -results = table.search("lan", query_type="fts").limit(10).to_list() -assert len(results) == 2 -assert set(r["text"] for r in results) == {"lance database", "lance is cool"} - -results = ( - table.search("nce", query_type="fts").limit(10).to_list() -) # spellchecker:disable-line -assert len(results) == 2 -assert set(r["text"] for r in results) == {"lance database", "lance is cool"} -``` - - -### Handling Short Substrings {#handling-short-substrings} - -By default, the minimum n-gram length is 3, so shorter substrings like "la" won't match: - - -```python Python icon="python" -results = table.search("la", query_type="fts").limit(10).to_list() -assert len(results) == 0 -``` - - -### Customizing N-gram Parameters {#customizing-n-gram-parameters} - -You can customize the n-gram behavior by adjusting the minimum length and using prefix-only matching: - - -```python Python icon="python" -table.create_fts_index( - "text", - base_tokenizer="ngram", - replace=True, - ngram_min_length=2, - prefix_only=True, -) -``` - - -### Testing Custom N-gram Settings {#testing-custom-n-gram-settings} - -With the new settings, you can now search for shorter substrings and use prefix-only matching: - - -```python Python icon="python" -results = table.search("lan", query_type="fts").limit(10).to_list() -assert len(results) == 2 -assert set(r["text"] for r in results) == {"lance database", "lance is cool"} - -results = ( - table.search("nce", query_type="fts").limit(10).to_list() -) # spellchecker:disable-line -assert len(results) == 0 - -results = table.search("la", query_type="fts").limit(10).to_list() -assert len(results) == 2 -assert set(r["text"] for r in results) == {"lance database", "lance is cool"} -``` - diff --git a/docs/web/search/full-text-search.mdx b/docs/web/search/full-text-search.mdx index 50633745a..77b1de7b6 100644 --- a/docs/web/search/full-text-search.mdx +++ b/docs/web/search/full-text-search.mdx @@ -324,7 +324,7 @@ to find similar terms within a specified edit distance. | fuzziness | int | 0 | Maximum edit distance allowed for each term. If not specified, automatically set based on term length: 0 for length ≤ 2, 1 for length ≤ 5, 2 for length > 5 | | max_expansions | int | 50 | Maximum number of terms to consider for fuzzy matching. Higher values may improve recall but increase search time | -For a complete walkthrough that creates a sample table and demonstrates fuzzy search and relevance boosting, see the [fuzzy search example](/search/fts-examples#fuzzy-search-and-boosting-example). +For a complete walkthrough that creates a sample table and demonstrates fuzzy search and relevance boosting, see [the worked example below](#fuzzy-search-and-boosting-example). ### Search for Substring {#search-for-substring} @@ -337,9 +337,558 @@ LanceDB supports searching for substrings in the text column, you can set the `b | prefix_only | bool | false | Whether to only search for prefixes of the n-grams | -## More Examples {#more-examples} +## Fuzzy Search and Boosting Example {#fuzzy-search-and-boosting-example} -For complete worked examples of fuzzy search, prefix matching, phrase matching, boosting, boolean queries, and substring search — including sample data generation and index setup — see [Full-Text Search Examples](/search/fts-examples). +### Generate Data {#generate-data} + +First, let's create a table with sample text data for testing fuzzy search: + + +```python Python icon="python" +import lancedb +import numpy as np +import pandas as pd +import random + +# Connect to LanceDB +db = lancedb.connect( + uri="db://your-project-slug", + api_key="your-api-key", + region="us-east-1" +) + +# Generate 100 rows of random " " text +table_name = "fts-fuzzy-boosting-test" +vectors = [np.random.randn(128) for _ in range(100)] +verbs = ("runs", "hits", "jumps", "drives", "barfs") +adv = ("crazily.", "dutifully.", "foolishly.", "merrily.", "occasionally.") +adj = ("adorable", "clueless", "dirty", "odd", "stupid") + +def sentence(nouns): + return " ".join(random.choice(words) for words in (nouns, verbs, adv, adj)) + +text = [sentence(("puppy", "car")) for _ in range(100)] +text2 = [sentence(("rabbit", "girl", "monkey")) for _ in range(100)] +count = [random.randint(1, 10000) for _ in range(100)] +``` + +```typescript TypeScript icon="square-js" +import * as lancedb from "@lancedb/lancedb" + +const db = await lancedb.connect({ + uri: "db://your-project-slug", + apiKey: "your-api-key", + region: "us-east-1" +}); + +// Generate 100 rows of random " " text +const tableName = "fts-fuzzy-boosting-test-ts"; +const n = 100; +const verbs = ["runs", "hits", "jumps", "drives", "barfs"]; +const adverbs = ["crazily", "dutifully", "foolishly", "merrily", "occasionally"]; +const adjectives = ["adorable", "clueless", "dirty", "odd", "stupid"]; + +const pick = (words: string[]) => words[Math.floor(Math.random() * words.length)]; +const sentence = (nouns: string[]) => + [nouns, verbs, adverbs, adjectives].map(pick).join(" "); + +const vectors = Array.from({ length: n }, () => + Array.from({ length: 128 }, () => Math.random() * 2 - 1) +); +const text = Array.from({ length: n }, () => sentence(["puppy", "car"])); +const text2 = Array.from({ length: n }, () => sentence(["rabbit", "girl", "monkey"])); +const count = Array.from({ length: n }, () => Math.floor(Math.random() * 10000) + 1); +``` + + +### Create Table {#create-table} + + +```python Python icon="python" +# Create table with sample data +table = db.create_table( + table_name, + data=pd.DataFrame({ + "vector": vectors, + "id": [i % 2 for i in range(100)], + "text": text, + "text2": text2, + "count": count, + }), + mode="overwrite" +) +``` + +```typescript TypeScript icon="square-js" +// Create table with sample data +const data = makeArrowTable( + vectors.map((vector, i) => ({ + vector, + id: i % 2, + text: text[i], + text2: text2[i], + count: count[i], + })) +); + +const table = await db.createTable(tableName, data, { mode: "overwrite" }); +``` + + +### Construct the index for these examples {#example-construct-fts-index} + +Create a full-text search index on the first text column: + + +```python Python icon="python" +# Create FTS index on first text column +table.create_fts_index("text") +wait_for_index(table, "text_idx") +``` + +```typescript TypeScript icon="square-js" +// Create FTS index on first text column +await table.createIndex("text", { config: Index.fts() }); +await waitForIndex(table, "text_idx"); +``` + + +Then, create an index on the second text column: + + +```python Python icon="python" +# Create FTS index on second text column +table.create_fts_index("text2") +wait_for_index(table, "text2_idx") +``` + +```typescript TypeScript icon="square-js" +// Create FTS index on second text column +await table.createIndex("text2", { config: Index.fts() }); +await waitForIndex(table, "text2_idx"); +``` + + +### Basic and Fuzzy Search {#basic-and-fuzzy-search} + +Now we can perform basic, fuzzy, and prefix match searches: + +#### Basic Exact Search {#basic-exact-search} + + +```python Python icon="python" +from lancedb.query import MatchQuery + +# Basic match (exact search) +basic_match_results = ( + table.search(MatchQuery("crazily", "text")) + .select(["id", "text"]) + .limit(100) + .to_pandas() +) +``` + +```typescript TypeScript icon="square-js" +import { MatchQuery } from "@lancedb/lancedb"; + +// Basic match (exact search) +const basicMatchResults = await table.query() + .fullTextSearch(new MatchQuery("crazily", "text")) + .select(["id", "text"]) + .limit(100) + .toArray(); +``` + + +#### Fuzzy Search with Typos {#fuzzy-search-with-typos} + + +```python Python icon="python" +# Fuzzy match (allows typos) +fuzzy_results = ( + table.search(MatchQuery("craziou", "text", fuzziness=2)) + .select(["id", "text"]) + .limit(100) + .to_pandas() +) +``` + +```typescript TypeScript icon="square-js" +// Fuzzy match (allows typos) +const fuzzyResults = await table.query() + .fullTextSearch(new MatchQuery("craziou", "text", { + fuzziness: 2, + })) + .select(["id", "text"]) + .limit(100) + .toArray(); +``` + + +#### Prefix based Match {#prefix-based-match} + +Prefix-based match allows you to search for documents containing words that start with a specific prefix. + + +```python Python icon="python" +# Fuzzy match (allows typos) +fuzzy_results = ( + table.search(MatchQuery("cra", "text", prefix_length=3)) + .select(["id", "text"]) + .limit(100) + .to_pandas() +) +``` + +```typescript TypeScript icon="square-js" +// Fuzzy match (allows typos) +const fuzzyResults = await table.query() + .fullTextSearch(new MatchQuery("cra", "text", { + prefixLength: 3, + })) + .select(["id", "text"]) + .limit(100) + .toArray(); +``` + + +### Phrase Match {#phrase-match} + +Phrase matching enables you to search for exact sequences of words. Unlike regular text search +which matches individual terms independently, phrase matching requires words to appear in the +specified order with no intervening terms. + + +Phrase queries are supported but only for a single column; providing multiple columns with a quoted phrase raises an error. + + +Phrase matching is particularly useful for: + +- Searching for specific multi-word expressions +- Matching exact titles or quotes +- Finding precise word combinations in a specific order + + +```python Python icon="python" +# Exact phrase match +from lancedb.query import PhraseQuery + +phrase_results = ( + table.search(PhraseQuery("puppy runs", "text")) + .select(["id", "text"]) + .limit(100) + .to_pandas() +) +``` + +```typescript TypeScript icon="square-js" +import { PhraseQuery } from "@lancedb/lancedb"; + +// Exact phrase match +const phraseResults = await table.query() + .fullTextSearch(new PhraseQuery("puppy runs", "text")) + .select(["id", "text"]) + .limit(100) + .toArray(); +``` + + +#### Flexible Phrase Match {#flexible-phrase-match} +To provide more flexible phrase matching, LanceDB supports the `slop` parameter. This allows you to match phrases where the terms appear close to each other, even if they are not directly adjacent or in the exact order, as long as they are within the specified `slop` value. + +For example, the phrase query "puppy merrily" would not return any results by default. However, if you set `slop=1`, it will match phrases like "puppy jumps merrily", "puppy runs merrily", and similar variations where one word appears between "puppy" and "merrily". + + +```python Python icon="python" +# Flexible phrase match with slop=1 for 'puppy merrily' +from lancedb.query import PhraseQuery + +phrase_results = ( + table.search(PhraseQuery("puppy merrily", "text", slop=1)) + .select(["id", "text"]) + .limit(100) + .to_pandas() +) +``` + +```typescript TypeScript icon="square-js" +import { PhraseQuery } from "@lancedb/lancedb"; + +// Flexible phrase match with slop=1 for 'puppy runs' +const phraseResults = await table.query() + .fullTextSearch(new PhraseQuery("puppy runs", "text", { slop: 1 })) + .select(["id", "text"]) + .limit(100) + .toArray(); +``` + + +### Search with Boosting {#search-with-boosting} + +Boosting allows you to control the relative importance of different search terms or fields +in your queries. This feature is particularly useful when you need to: + +* Prioritize matches in certain columns +* Promote specific terms while demoting others +* Fine-tune relevance scoring for better search results + +| Parameter | Type | Default | Description | +| -------------- | ----- | -------- | ------------------------------------------------------------------ | +| positive | Query | required | The primary query terms to match and promote in results | +| negative | Query | required | Terms to demote in the search results | +| negative_boost | float | 0.5 | Multiplier for negative matches (lower values = stronger demotion) | + + +```python Python icon="python" +from lancedb.query import MatchQuery, BoostQuery, MultiMatchQuery + +# Boost data with 'runs' in text more than 'puppy' in text +boosting_results = ( + table.search( + BoostQuery( + MatchQuery("runs", "text"), + MatchQuery("puppy", "text"), + negative_boost=0.2, + ), + ) + .select(["id", "text"]) + .limit(100) + .to_pandas() +) + +# Search across both text and text2 +multi_match_results = ( + table.search(MultiMatchQuery("crazily", ["text", "text2"])) + .select(["id", "text", "text2"]) + .limit(100) + .to_pandas() +) + +# Search with field boosting +multi_match_boosting_results = ( + table.search( + MultiMatchQuery("crazily", ["text", "text2"], boosts=[1.0, 2.0]), + ) + .select(["id", "text", "text2"]) + .limit(100) + .to_pandas() +) +``` + +```typescript TypeScript icon="square-js" +import { MatchQuery, BoostQuery, MultiMatchQuery } from "@lancedb/lancedb"; + +// Boosting Example +const boostingResults = await table.query() + .fullTextSearch(new BoostQuery(new MatchQuery("runs", "text"), new MatchQuery("puppy", "text"), { + negativeBoost: 0.2, + })) + .select(["id", "text"]) + .limit(100) + .toArray(); + +// Search across both text fields +const multiMatchResults = await table.query() + .fullTextSearch(new MultiMatchQuery("crazily", ["text", "text2"])) + .select(["id", "text", "text2"]) + .limit(100) + .toArray(); + +// Search with field boosting +const multiMatchBoostingResults = await table.query() + .fullTextSearch(new MultiMatchQuery("crazily", ["text", "text2"], { + boosts: [1.0, 2.0], + })) + .select(["id", "text", "text2"]) + .limit(100) + .toArray(); +``` + + + +- Use fuzzy search when handling user input that may contain typos or variations +- Apply field boosting to prioritize matches in more important columns +- Combine fuzzy search with boosting for robust and precise search results + +**Recommendations for optimal FTS performance:** + +- Create full-text search indices on text columns that will be frequently searched +- For hybrid search combining text and vectors, see our [hybrid search guide](/search/hybrid-search/) +- For performance benchmarks, check our [benchmark results](/enterprise/benchmarks/) +- For complex queries, use SQL to combine FTS with other filter conditions + + +### Boolean Queries {#boolean-queries} +LanceDB supports boolean logic in full-text search, allowing you to combine multiple queries using `and` and `or` operators. This is useful when you want to match documents that satisfy multiple conditions (intersection) or at least one of several conditions (union). + +#### Combining Two Match Queries {#combining-two-match-queries} + +In Python, you can combine two MatchQuery objects using either the `and` function or the `&` operator (e.g., `MatchQuery("puppy", "text") and MatchQuery("merrily", "text")`); both methods are supported and yield the same result. Similarly, you can use either the `or` function or the `|` operator to perform an or query. + +In TypeScript, boolean queries are constructed using the `BooleanQuery` class with a list of [Occur, subquery] pairs. For example, to perform an AND query: + +```sql SQL icon="code" +BooleanQuery([ +[Occur.Must, new MatchQuery("puppy", "text")], +[Occur.Must, new MatchQuery("merrily", "text")], +]) +``` + +This approach allows you to specify complex boolean logic by combining multiple subqueries with different Occur values (such as `Must`, `Should`, or `MustNot`). + + +**Which queries are allowed?** + +A boolean query must include at least one `SHOULD` or `MUST` clause. Queries that contain only a `MUST_NOT` clause are not allowed. + + + +```python Python icon="python" +from lancedb.query import MatchQuery + +# Example: Find documents containing both "puppy" and "merrily" +and_query = MatchQuery("puppy", "text") & MatchQuery("merrily", "text") +and_results = ( + table.search(and_query) + .select(["id", "text"]) + .limit(100) + .to_pandas() +) + +# Example: Find documents containing either "puppy" or "merrily" +or_query = MatchQuery("puppy", "text") | MatchQuery("merrily", "text") +or_results = ( + table.search(or_query) + .select(["id", "text"]) + .limit(100) + .to_pandas() +) +``` + +```typescript TypeScript icon="square-js" expandable=true +import { MatchQuery, BooleanQuery, Occur } from "@lancedb/lancedb"; + +// Flexible boolean queries with MatchQuery + +// Find documents containing both "puppy" and "merrily" +const mustResults = await table + .search( + new BooleanQuery([ + [Occur.Must, new MatchQuery("puppy", "text")], + [Occur.Must, new MatchQuery("merrily", "text")], + ]), + ) + .select(["id", "text"]) + .limit(100) + .toArray(); + +// Find documents containing either "puppy" or "merrily" +const shouldResults = await table + .search( + new BooleanQuery([ + [Occur.Should, new MatchQuery("puppy", "text")], + [Occur.Should, new MatchQuery("merrily", "text")], + ]), + ) + .select(["id", "text"]) + .limit(100) + .toArray(); +``` + + + +**How to use booleans?** + +- Use `and`/`&`(Python), `Occur.Must`(Typescript) for intersection (documents must match all queries). +- Use `or`/`|`(Python), `Occur.Should`(Typescript) for union (documents must match at least one query). + + +## Substring Search Example {#substring-search-example} + +LanceDB supports searching for substrings in text columns using n-gram tokenization. This is useful for finding partial matches within text content. + +### Setting Up the Table {#setting-up-the-table} + +First, create a table with sample text data and configure n-gram tokenization: + + +```python Python icon="python" +import pyarrow as pa +import lancedb + +db = lancedb.connect(":memory:") + +data = pa.table({"text": ["hello world", "lance database", "lance is cool"]}) +table = db.create_table("test", data=data) +table.create_fts_index("text", base_tokenizer="ngram") +``` + + +### Basic Substring Search {#basic-substring-search} + +With the default n-gram settings (minimum length of 3), you can search for substrings of length 3 or more: + + +```python Python icon="python" +results = table.search("lan", query_type="fts").limit(10).to_list() +assert len(results) == 2 +assert set(r["text"] for r in results) == {"lance database", "lance is cool"} + +results = ( + table.search("nce", query_type="fts").limit(10).to_list() +) # spellchecker:disable-line +assert len(results) == 2 +assert set(r["text"] for r in results) == {"lance database", "lance is cool"} +``` + + +### Handling Short Substrings {#handling-short-substrings} + +By default, the minimum n-gram length is 3, so shorter substrings like "la" won't match: + + +```python Python icon="python" +results = table.search("la", query_type="fts").limit(10).to_list() +assert len(results) == 0 +``` + + +### Customizing N-gram Parameters {#customizing-n-gram-parameters} + +You can customize the n-gram behavior by adjusting the minimum length and using prefix-only matching: + + +```python Python icon="python" +table.create_fts_index( + "text", + base_tokenizer="ngram", + replace=True, + ngram_min_length=2, + prefix_only=True, +) +``` + + +### Testing Custom N-gram Settings {#testing-custom-n-gram-settings} + +With the new settings, you can now search for shorter substrings and use prefix-only matching: + + +```python Python icon="python" +results = table.search("lan", query_type="fts").limit(10).to_list() +assert len(results) == 2 +assert set(r["text"] for r in results) == {"lance database", "lance is cool"} + +results = ( + table.search("nce", query_type="fts").limit(10).to_list() +) # spellchecker:disable-line +assert len(results) == 0 + +results = table.search("la", query_type="fts").limit(10).to_list() +assert len(results) == 2 +assert set(r["text"] for r in results) == {"lance database", "lance is cool"} +``` + ## Full-Text Search on Array Fields {#full-text-search-on-array-fields} diff --git a/docs/web/search/hybrid-search.mdx b/docs/web/search/hybrid-search.mdx index d4109c653..76ea5d34f 100644 --- a/docs/web/search/hybrid-search.mdx +++ b/docs/web/search/hybrid-search.mdx @@ -341,7 +341,7 @@ await table.query() ``` -The choice gets baked into both sub-queries, so the vector and FTS halves see the filter applied the same way. Use [`explain_plan`](/search/optimize-queries#analyzing-non-vector-queries) on a hybrid query to see whether the filter pushed into the scan or ran as a separate `FilterExec` step. +The choice gets baked into both sub-queries, so the vector and FTS halves see the filter applied the same way. Use [`explain_plan`](/search/explain#analyzing-non-vector-queries) on a hybrid query to see whether the filter pushed into the scan or ran as a separate `FilterExec` step. ## More on Reranking {#more-on-reranking} diff --git a/docs/web/search/select.mdx b/docs/web/search/select.mdx new file mode 100644 index 000000000..35974bb18 --- /dev/null +++ b/docs/web/search/select.mdx @@ -0,0 +1,176 @@ +--- +title: "SELECT syntax" +sidebarTitle: "SELECT" +description: "The analytical query surface: projection, filtering, grouping, joins, and how each maps onto the SDKs." +icon: table-columns +keywords: ["select", "sql", "join", "group by", "window", "cte", "datafusion"] +--- + +Enterprise SQL runs on a deployment. A local database +has no engine to plan a statement -- see +[Python, TypeScript, and SQL](/basics/apis). + +The dialect is [DataFusion's](/sql/functions), so `SELECT` behaves as its +[reference](https://datafusion.apache.org/user-guide/sql/select.html) documents. +This page is the shape of that surface and where the SDKs meet it. + +## The clauses {#clauses} + +```sql +[WITH cte AS (...)] +SELECT [ALL | DISTINCT] select_expr [, ...] +FROM from_item [[JOIN ...] ON ...] +[WHERE predicate] +[GROUP BY expr [, ...]] +[HAVING predicate] +[WINDOW name AS (...)] +[ORDER BY expr [ASC | DESC] [NULLS FIRST | LAST] [, ...]] +[LIMIT n] [OFFSET n] +``` + +Set operations -- `UNION [ALL]`, `INTERSECT`, `EXCEPT` -- combine whole queries, +and subqueries are allowed anywhere an expression or relation is. + +## Projection {#projection} + + +```sql SQL +SELECT id, title, identifier FROM documents +``` + +```python Python icon="python" +tbl.search().select(["id", "title", "identifier"]).to_arrow() +``` + +```typescript TypeScript icon="square-js" +await tbl.query().select(["id", "title", "identifier"]).toArrow(); +``` + + +Projection pushes into the scan in all three -- an unselected column is never +read off storage. On a table of images or video that is the difference between a +fast query and a slow one, so prefer naming columns over `SELECT *`. + +Row metadata is available as [pseudo-columns](/sql/functions#pseudo-columns) +(`_rowid`, `_rowaddr`, `_rowoffset`) but only when named, so they stay out of +`SELECT *`. + +## Filtering {#filtering} + + +```sql SQL +SELECT * FROM documents +WHERE identifier > 0 AND identifier < 1000000 +``` + +```python Python icon="python" +tbl.search().where("identifier > 0 AND identifier < 1000000").to_arrow() +``` + +```typescript TypeScript icon="square-js" +await tbl.query().where("identifier > 0 AND identifier < 1000000").toArrow(); +``` + + +The SDK `where` takes the same expression language, so a predicate moves between +them unchanged. A predicate on an indexed column becomes a +`ScalarIndexQuery` rather than a scan -- confirm with +[`explain_plan`](/search/explain). See [Filtering](/search/filtering) for the +operators and their pushdown behaviour. + +## Grouping and aggregation {#grouping} + +```sql +SELECT category, count(*) AS n, avg(score) AS mean_score +FROM documents +GROUP BY category +HAVING count(*) > 10 +ORDER BY n DESC +``` + +Aggregates, statistical and approximate functions come from DataFusion's +[aggregate function set](https://datafusion.apache.org/user-guide/sql/aggregate_functions.html). + + +The SDK query builders do not aggregate. Grouping, joins, and window functions +are the reason to reach for SQL rather than a query builder -- see +[choosing between them](/basics/apis#choosing). + + +## Joins {#joins} + +```sql +SELECT d.title, c.name +FROM documents AS d +JOIN categories AS c ON c.id = d.category_id +``` + +`INNER`, `LEFT`, `RIGHT`, `FULL`, and `CROSS` joins are supported. Tables in +different [namespaces](/namespaces) are joined by their dotted paths: + +```sql +SELECT * FROM prod.events.clicks AS c +JOIN prod.reference.users AS u ON u.id = c.user_id +``` + +## Window functions {#windows} + +```sql +SELECT + title, + score, + row_number() OVER (PARTITION BY category ORDER BY score DESC) AS rank +FROM documents +``` + +The full set is DataFusion's +[window functions](https://datafusion.apache.org/user-guide/sql/window_functions.html). + +## Common table expressions {#ctes} + +```sql +WITH recent AS ( + SELECT * FROM documents WHERE created_at > now() - INTERVAL '7 days' +) +SELECT category, count(*) FROM recent GROUP BY category +``` + +A CTE that you want to keep is a [view](/management/view); one whose rows you +want stored is a +[materialized view](/feature-engineering/materialized-views). + +## Search inside SELECT {#search} + +Full-text search is a **table function**, not a predicate, because it produces +relevance alongside the rows and `WHERE` has nowhere to put that: + +```sql +SELECT id, text, _score +FROM fts('my_docs', 'puppy') +ORDER BY _score DESC +LIMIT 5 +``` + +Because it is a relation it composes -- join it, aggregate over it, wrap it in a +CTE. For a keyword test that needs no score, the +[`contains_tokens`](/sql/functions#contains-tokens) predicate is +index-accelerated and usable directly in `WHERE`. + + +Vector search has no SQL form. It is a query-builder operation -- see +[Vector search](/search/vector-search). A query that needs both vector +similarity and SQL analytics runs the search in the SDK and the analytics over +its result. + + +## Reading the plan {#plans} + +`EXPLAIN` and `EXPLAIN ANALYZE` work as DataFusion defines them, and produce the +same operators the SDK methods do: + +```sql +EXPLAIN ANALYZE SELECT * FROM documents WHERE identifier < 1000 +``` + +See [Explain](/search/explain) and [Explain Analyze](/search/explain-analyze) +for how to read the output. diff --git a/docs/web/snippets/geneva_dependency_verification.mdx b/docs/web/snippets/geneva_dependency_verification.mdx deleted file mode 100644 index 5afae693a..000000000 --- a/docs/web/snippets/geneva_dependency_verification.mdx +++ /dev/null @@ -1,12 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyCondaClusterInline = "from geneva.cluster.builder import KubeRayClusterBuilder\n\ncluster = (\n KubeRayClusterBuilder.create(\"my-cluster\")\n .ray_init_kwargs({\n \"runtime_env\": {\n \"conda\": {\n \"channels\": [\"conda-forge\"],\n \"dependencies\": [\n \"python=3.10\",\n \"ffmpeg<8\",\n \"torchvision=0.22.1\",\n ],\n },\n \"config\": {\"eager_install\": True},\n }\n })\n .build()\n)\n"; - -export const PyCondaClusterPath = "from geneva.cluster.builder import KubeRayClusterBuilder\n\ncluster = (\n KubeRayClusterBuilder.create(\"my-cluster\")\n .ray_init_kwargs({\n \"runtime_env\": {\"conda\": \"environment.yml\"}\n })\n .build()\n)\n"; - -export const PyEnvVarsViaCluster = "from geneva.cluster.builder import KubeRayClusterBuilder\nimport os\n\ncluster = (\n KubeRayClusterBuilder.create(\"my-cluster\")\n .ray_init_kwargs({\n \"runtime_env\": {\n \"env_vars\": {\n \"AWS_ACCESS_KEY_ID\": os.environ[\"AWS_ACCESS_KEY_ID\"],\n \"AWS_SECRET_ACCESS_KEY\": os.environ[\"AWS_SECRET_ACCESS_KEY\"],\n }\n }\n })\n .build()\n)\n"; - -export const PyPipManifest = "import geneva\nfrom geneva.manifest.builder import PipManifestBuilder\n\nmanifest = (\n PipManifestBuilder.create(\"my-manifest\")\n .pip([\n \"numpy==1.26.4\",\n \"torch==2.0.1\",\n \"attrs==23.2.0\",\n ])\n .build()\n)\n\nconn = geneva.connect(\"s3://my-bucket/my-db\")\nconn.define_manifest(\"my-manifest\", manifest)\nwith conn.context(cluster=\"my-cluster\", manifest=\"my-manifest\"):\n conn.open_table(\"my-table\").backfill(\"my-column\")\n"; - -export const PyQuickFixManifest = "from geneva.manifest.builder import PipManifestBuilder\n\nmanifest = PipManifestBuilder.create(\"fix\").pip([\"numpy==1.26.4\"]).build()\n"; - diff --git a/docs/web/snippets/geneva_profiling_memory.mdx b/docs/web/snippets/geneva_profiling_memory.mdx deleted file mode 100644 index f6addc420..000000000 --- a/docs/web/snippets/geneva_profiling_memory.mdx +++ /dev/null @@ -1,22 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyBoundedCache = "from functools import lru_cache\n\nclass GoodEmbedding:\n def __init__(self):\n self._embed = None\n\n def setup(self):\n model = load_model()\n self._embed = lru_cache(maxsize=1024)(model.embed)\n\n def __call__(self, text: str) -> list[float]:\n if self._embed is None:\n self.setup()\n return self._embed(text)\n"; - -export const PyConfidenceCheck = "def __call__(self, x):\n scratch = bytearray(8 * 1024 * 1024) # 8 MiB\n self._scratches.append(scratch) # <-- deliberate leak\n return ...\n"; - -export const PyLeakyAggregator = "class BadAggregator:\n def __init__(self):\n self.history = []\n\n def __call__(self, batch: pa.RecordBatch) -> pa.Array:\n self.history.append(batch) # holds every batch ever processed\n ...\n"; - -export const PyLeakyCache = "class BadEmbedding:\n def __init__(self):\n self.cache: dict[str, list[float]] = {}\n\n def __call__(self, text: str) -> list[float]:\n if text not in self.cache:\n self.cache[text] = self.model.embed(text)\n return self.cache[text]\n"; - -export const PyLeakyClosure = "class BadDeferred:\n def __init__(self):\n self.work_queue = []\n\n def __call__(self, x: pa.Array) -> pa.Array:\n # Lambda captures `x` by reference — the whole Array stays alive\n self.work_queue.append(lambda: expensive(x))\n ...\n"; - -export const PyLogMemory = "import resource, pyarrow as pa\n\ndef log_memory(seq: int) -> None:\n rss_bytes = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss\n # ru_maxrss is bytes on macOS, KiB on Linux:\n import sys\n if sys.platform != \"darwin\":\n rss_bytes *= 1024\n arrow_live = pa.total_allocated_bytes()\n print(\n f\"seq={seq} \"\n f\"rss_mb={rss_bytes // 1024**2} \"\n f\"arrow_live_mb={arrow_live // 1024**2} \"\n f\"gap_mb={(rss_bytes - arrow_live) // 1024**2}\",\n flush=True,\n )\n"; - -export const PyMemrayTrackerUdf = "import os, pathlib, uuid\nfrom typing import Any\nimport memray\nimport geneva\nimport pyarrow as pa\n\n_MEMRAY_OUT_DIR_ENV = \"MY_UDF_MEMRAY_OUT_DIR\"\n\n\n@geneva.udf(data_type=pa.list_(pa.float32(), 512))\nclass MyEmbedding:\n def __init__(self):\n self.model = None\n self._tracker: Any = None # memray.Tracker, when profiling is on\n\n def setup(self):\n # Open a memray tracker per worker process, if requested. Each\n # worker writes its own .bin file so traces don't collide.\n out_dir = os.environ.get(_MEMRAY_OUT_DIR_ENV)\n if out_dir:\n pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)\n bin_path = pathlib.Path(out_dir) / (\n f\"memray-{os.getpid()}-{uuid.uuid4().hex}.bin\"\n )\n self._tracker = memray.Tracker(\n str(bin_path), native_traces=False, follow_fork=False\n )\n self._tracker.__enter__()\n self.model = load_model()\n\n def __call__(self, text: str) -> list[float]:\n if self.model is None:\n self.setup()\n return self.model.embed(text)\n"; - -export const PyRayClusterProfile = "from geneva.runners.ray._mgr import ray_cluster\n\nwith ray_cluster(\n local=True,\n extra_env={\"MY_UDF_MEMRAY_OUT_DIR\": \"/tmp/my-udf-profile\"},\n):\n table.backfill(\"embedding\", concurrency=1)\n"; - -export const PyStatefulUdfClass = "@geneva.udf(data_type=pa.list_(pa.float32(), 512))\nclass MyEmbedding:\n def __init__(self):\n self.model = None\n\n def setup(self):\n self.model = load_model() # allocated once per actor\n\n def __call__(self, text: str) -> list[float]:\n if self.model is None:\n self.setup()\n return self.model.embed(text)\n"; - -export const PyTorchInferenceMode = "def __call__(self, text: str) -> list[float]:\n with torch.inference_mode(): # <-- prevents autograd graph retention\n return self.model.encode(text)\n"; - diff --git a/docs/web/snippets/geneva_scalar_udtfs.mdx b/docs/web/snippets/geneva_scalar_udtfs.mdx deleted file mode 100644 index a51feb545..000000000 --- a/docs/web/snippets/geneva_scalar_udtfs.mdx +++ /dev/null @@ -1,20 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyAddColumnsScalarUdtf = "@udf(data_type=pa.list_(pa.float32(), 512))\ndef clip_embedding(clip_bytes: bytes) -> list[float]:\n return embed_model.encode(clip_bytes)\n\n# Add an embedding column to the clips table\nclips.add_columns({\"embedding\": clip_embedding})\n\n# Backfill computes embeddings for all existing clips\nclips.backfill(\"embedding\")\n"; - -export const PyChainingUdtfViews = "# videos → clips (1:N)\nclips = db.create_udtf_view(\n \"clips\", source=videos.search(None), udtf=extract_clips\n)\n\n# clips → frames (1:N)\nframes = db.create_udtf_view(\n \"frames\", source=clips.search(None), udtf=extract_frames\n)\n"; - -export const PyCreateScalarUdtfView = "import geneva\n\ndb = geneva.connect(\"/data/mydb\")\nvideos = db.open_table(\"videos\")\n\n# Create the 1:N materialized view\nclips = db.create_udtf_view(\n \"clips\",\n source=videos.search(None).select([\"video_path\", \"metadata\"]),\n udtf=extract_clips,\n)\n\n# Populate — runs the UDTF on every source row\nclips.refresh()\n"; - -export const PyDocumentChunkingFull = "from geneva import connect, chunker, udf\nfrom typing import Iterator, NamedTuple\nimport pyarrow as pa\n\nclass Chunk(NamedTuple):\n chunk_index: int\n chunk_text: str\n\n@chunker\ndef chunk_document(text: str) -> Iterator[Chunk]:\n \"\"\"Split a document into overlapping chunks.\"\"\"\n words = text.split()\n chunk_size = 500\n overlap = 50\n for i, start in enumerate(range(0, len(words), chunk_size - overlap)):\n chunk_words = words[start:start + chunk_size]\n yield Chunk(chunk_index=i, chunk_text=\" \".join(chunk_words))\n\ndb = connect(\"/data/mydb\")\ndocs = db.open_table(\"documents\")\n\n# Create chunked view — inherits doc_id, title, etc. from source\nchunks = db.create_udtf_view(\n \"doc_chunks\",\n source=docs.search(None).select([\"doc_id\", \"title\", \"text\"]),\n udtf=chunk_document,\n)\nchunks.refresh()\n\n# Add embeddings to chunks for semantic search\n@udf(data_type=pa.list_(pa.float32(), 1536))\ndef embed_text(chunk_text: str) -> list[float]:\n return embedding_model.encode(chunk_text)\n\nchunks.add_columns({\"embedding\": embed_text})\nchunks.backfill(\"embedding\") # Backfills embeddings on all existing chunks\n\n# Query — parent columns available alongside chunk columns\nchunks.search(None).select([\"doc_id\", \"title\", \"chunk_text\", \"embedding\"]).to_pandas()\n"; - -export const PyDocumentChunkingUdtf = "from geneva import chunker\nfrom typing import Iterator, NamedTuple\n\nclass Chunk(NamedTuple):\n chunk_index: int\n chunk_text: str\n\n@chunker\ndef chunk_document(text: str) -> Iterator[Chunk]:\n \"\"\"Split a document into overlapping chunks.\"\"\"\n words = text.split()\n chunk_size = 500\n overlap = 50\n for i, start in enumerate(range(0, len(words), chunk_size - overlap)):\n chunk_words = words[start:start + chunk_size]\n yield Chunk(chunk_index=i, chunk_text=\" \".join(chunk_words))\n"; - -export const PyIncrementalRefresh = "# Add new videos to the source table\nvideos.add(new_video_data)\n\n# Incremental refresh — only processes the new videos\nclips.refresh()\n"; - -export const PyScalarUdtfBatch = "@chunker(batch=True, output_schema=clip_schema)\ndef extract_clips(batch: pa.RecordBatch) -> pa.RecordBatch:\n \"\"\"Process rows in batches. Same 1:N semantic per row.\"\"\"\n ...\n"; - -export const PyScalarUdtfIterator = "from geneva import chunker\nfrom typing import Iterator, NamedTuple\n\nclass Clip(NamedTuple):\n clip_start: float\n clip_end: float\n clip_bytes: bytes\n\n@chunker\ndef extract_clips(video_path: str, duration: float) -> Iterator[Clip]:\n \"\"\"Yields multiple clips per video.\"\"\"\n clip_length = 10.0\n for start in range(0, int(duration), int(clip_length)):\n end = min(start + clip_length, duration)\n clip_data = extract_video_segment(video_path, start, end)\n yield Clip(clip_start=start, clip_end=end, clip_bytes=clip_data)\n"; - -export const PyScalarUdtfList = "@chunker\ndef extract_clips(video_path: str, duration: float) -> list[Clip]:\n clips = []\n for start in range(0, int(duration), 10):\n end = min(start + 10, duration)\n clips.append(Clip(clip_start=start, clip_end=end, clip_bytes=b\"...\"))\n return clips\n"; - diff --git a/docs/web/snippets/geneva_udfs_index.mdx b/docs/web/snippets/geneva_udfs_index.mdx deleted file mode 100644 index 923102a3a..000000000 --- a/docs/web/snippets/geneva_udfs_index.mdx +++ /dev/null @@ -1,8 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyRegistrationScalarUdtf = "db = geneva.connect(\"/data/mydb\")\ndb.create_udtf_view(\"my_view\", source=my_source, udtf=my_chunker)\n"; - -export const PyRegistrationUdf = "mock_table.add_columns({\"col\": my_udf})\n"; - -export const PyRegistrationUdtf = "db = geneva.connect(\"/data/mydb\")\ndb.create_udtf_view(\"my_view\", source=my_source, udtf=my_udtf)\n"; - diff --git a/docs/web/sql/functions.mdx b/docs/web/sql/functions.mdx index cf947b014..3a4e341ad 100644 --- a/docs/web/sql/functions.mdx +++ b/docs/web/sql/functions.mdx @@ -115,6 +115,59 @@ A pseudo-column produced by `fts()`, carrying relevance for the row. It behaves like a normal column — order by it, filter on it, project it — and exists only in a relation that came from `fts()`. +### `contains_tokens(column, tokens)` {#contains-tokens} + +A predicate rather than a relation, for when you want keyword matching inside a +`WHERE` clause and do not need relevance: + +```sql +SELECT * FROM my_docs WHERE contains_tokens(text, 'fox jumps dog') +``` + +It is true when the column contains **all** the tokens, which are split on +whitespace and punctuation. Equivalent to an `fts()` match query with fuzziness +off, `AND` semantics, and the simple tokenizer. + +**It is index-accelerated.** When the column has a full-text index that suits +the query, the planner uses it instead of scanning. That is the reason to prefer +it over `LIKE` for keyword tests. + +Use `fts()` when you need `_score` or the fuzzy, phrase, or boost forms; use +`contains_tokens` when you just need a boolean. + +### JSON functions {#json} + +Lance registers a set of JSON accessors over string columns holding JSON: + +| Function | Returns | +| --- | --- | +| `json_get(col, key)` | the value at `key`, still JSON | +| `json_get_string(col, key)` | `Utf8` | +| `json_get_int(col, key)` | `Int64` | +| `json_get_float(col, key)` | `Float64` | +| `json_get_bool(col, key)` | `Boolean` | +| `json_extract(col, path)` | `Utf8`, by path | +| `json_extract_with_type(col, path, type)` | the requested type | +| `json_exists(col, path)` | `Boolean` | +| `json_array_contains(col, value)` | `Boolean` | +| `json_array_length(col)` | length | + +`json_get` returns JSON, so accessors nest for a nested document: + +```sql +SELECT * FROM events +WHERE json_get_string(json_get(json_get(data, 'user'), 'profile'), 'name') = 'Alice' +``` + +**`json_get`, its typed variants, and `json_extract` are index-accelerated** by a +JSON scalar index on the column. The others evaluate per row. + + +Lance also defines geospatial `st_*` functions, but they are behind an optional +build feature that LanceDB does not ship. Calling one returns an error telling +you the feature is missing rather than an unknown-function error. + + ## Reaching a function LanceDB does not name {#escape-hatch} Nothing here wraps or restricts the DataFusion set — a query reaches any diff --git a/docs/web/sql/index.mdx b/docs/web/sql/index.mdx deleted file mode 100644 index c769b52c1..000000000 --- a/docs/web/sql/index.mdx +++ /dev/null @@ -1,72 +0,0 @@ ---- -title: "SQL manual" -sidebarTitle: "Overview" -description: "Analytical SQL over the lakehouse, served through Arrow Flight SQL." -icon: table-list -keywords: ["sql", "flight sql", "query", "analytics", "datafusion", "fts"] ---- - -LanceDB answers analytical queries in SQL, served over an -[Arrow Flight SQL](https://arrow.apache.org/docs/format/FlightSql.html) -endpoint. Any Flight SQL client can reach it, and results come back as Arrow. - -The dialect is [DataFusion's](/sql/functions), so the syntax and functions are -the ones that reference documents. What this manual covers is the parts that are -LanceDB's: how a query reaches the engine, how Lance columns appear to it, and -the extensions a lakehouse over multimodal data needs. - -## Where a query runs {#execution} - -Queries execute on the server. The client sends a statement and receives Arrow -record batches — nothing is converted to pandas or Python objects unless you ask -for it. - -Because results are Arrow the whole way, a query that feeds training or -downstream processing does not pay a serialization cost to get there. - -## The dialect {#dialect} - -Ordinary analytical SQL — `SELECT`, `WHERE`, `GROUP BY`, `ORDER BY`, `LIMIT`, -joins, CTEs and window functions — over Arrow-typed columns. - -Two things differ from a warehouse you may be used to, and both come from the -storage being Lance rather than something SQL-shaped: - -- **Columns are Arrow types**, not a SQL type system mapped onto storage. A - vector column is a `FixedSizeList` and behaves like one. -- **Row metadata is available as columns** — `_rowid`, `_rowaddr` — but only - when asked for, so they stay out of `SELECT *`. - -Both are covered in [Functions and types](/sql/functions). - -## Full-text search {#fts} - -Full-text search is a **table function** rather than a predicate, because it -produces relevance alongside the matching rows and a `WHERE` clause has nowhere -to put that: - -```sql -SELECT id, text, _score -FROM fts('my_docs', 'puppy') -ORDER BY _score DESC -LIMIT 5 -``` - -`fts()` takes a table name and a query — a bare string, or a JSON query object -for fuzzy, phrase, boolean, boost and multi-match forms. `_score` carries -relevance and behaves like any other column. - -Being a relation is what makes it compose: join its result, filter it, aggregate -over it. - -## Reference {#reference} - - - The DataFusion dialect and version, how Lance columns map onto it, and what - LanceDB adds. - - -## What this manual does not cover {#scope} - -Connecting a Flight SQL client — drivers, authentication, the endpoint address — -is deployment-specific and documented with the deployment. diff --git a/docs/web/tables-and-namespaces.mdx b/docs/web/tables-and-namespaces.mdx deleted file mode 100644 index 6899d42d8..000000000 --- a/docs/web/tables-and-namespaces.mdx +++ /dev/null @@ -1,43 +0,0 @@ ---- -title: "Tables and Namespaces" -sidebarTitle: "Tables and Namespaces" -description: "Learn more about the table abstraction and namespaces in LanceDB." -icon: "table" ---- - -Despite its name, LanceDB is not a "database" in the traditional sense. It is a **Multimodal Lakehouse** built on Lance tables plus a catalog abstraction. -As you dive deeper into LanceDB, it helps to separate two ideas: -- A **table** is where your data lives and is queried. -- A **namespace** is how groups of tables are organized and resolved at the catalog level. - -## Understanding tables {#understanding-tables} - -A table is the core data abstraction in LanceDB: a structured dataset with schema, indexes, and versioned updates. -What changes between deployments is how that table is addressed and accessed. - -The mental model below clarifies table types by connection mode: - -- **`LanceTable`**: direct table access (local path, `file://`, `s3://`, and similar object-store paths). This is the common mode in LanceDB OSS. -- **`RemoteTable`**: catalog-backed table access through a server/cluster (`db://...`). This is the mode you will use in LanceDB Enterprise. - -![](/static/assets/images/overview/understanding-tables.png) - -From an application perspective, both expose a familiar table API: create/open tables, mutate rows, and query data. -The main difference is where resolution and execution happen (directly against storage vs through a remote catalog service). - -## Semantic difference between tables and namespaces {#semantic-difference-between-tables-and-namespaces} - -The easiest way to think about this is: -- A **table** answers: "What data do I store and query?" -- A **namespace** answers: "Where does this table name live in my catalog hierarchy?" - -In other words, tables are data objects; namespaces are catalog objects. - -| Concept | Scope | Owns | Typical operations | -| --- | --- | --- | --- | -| Table | Data layer | Schema, rows, indexes, versions | `create_table`, `open_table`, inserts/updates/deletes, search/query | -| Namespace | Catalog layer | Hierarchy of names, table grouping, table name resolution | `create_namespace`, `list_namespaces`, `drop_namespace`, table ops with `namespace` | - -For simple use cases where you have a relatively flat set of tables, you can ignore namespaces and just use table paths directly. -As your application needs evolve and your tables grow in number and complexity, you may move from table-centric thinking -to catalog-centric thinking. Check out the [Namespaces and the Catalog Model](/namespaces) guide to learn more. diff --git a/docs/web/training/vlm-finetuning.mdx b/docs/web/training/vlm-finetuning.mdx index 39abf24ad..c0b38569c 100644 --- a/docs/web/training/vlm-finetuning.mdx +++ b/docs/web/training/vlm-finetuning.mdx @@ -510,11 +510,11 @@ The complete demo implementation with helper scripts and usage instructions is i The runnable Colab workflow: download, explore, benchmark, train, and evaluate. - + Tier 1, Tier 2, and Tier 3 feature definitions. - Geneva-powered feature materialization. + Feature materialization for the tiers above. QLoRA training from cached Lance columns. diff --git a/docs/web/troubleshooting.mdx b/docs/web/troubleshooting.mdx index bd46b830d..2da1dc999 100644 --- a/docs/web/troubleshooting.mdx +++ b/docs/web/troubleshooting.mdx @@ -28,7 +28,7 @@ print the resolved query plan. LanceDB provides two powerful tools for query analysis and optimization: `explain_plan` and `analyze_plan`. -Read the full guide on [Query Optimization](/search/optimize-queries/). +Read the full guides on [Explain](/search/explain) and [Explain Analyze](/search/explain-analyze). ### The Python multiprocessing module {#the-python-multiprocessing-module} Multiprocessing with `fork` is not supported. You should use `spawn` instead.