From 38a62db418498b75a3b412d9a2680df404b16c2e Mon Sep 17 00:00:00 2001 From: SIDDARTHA REDDY <75976672+SIDDARTHAREDDY8@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:36:24 -0400 Subject: [PATCH 01/19] feat(python,rust): allow lancedb.blob() to set blob tier thresholds (#4323) --- python/python/lancedb/schema.py | 55 +++++++++++++++++++++++-- python/python/tests/test_blob.py | 31 ++++++++++++++ rust/lancedb/src/blob.rs | 70 +++++++++++++++++++++++++++++++- 3 files changed, 151 insertions(+), 5 deletions(-) diff --git a/python/python/lancedb/schema.py b/python/python/lancedb/schema.py index 4dce09f0f..5edda5610 100644 --- a/python/python/lancedb/schema.py +++ b/python/python/lancedb/schema.py @@ -5,7 +5,7 @@ """Schema helpers for Lance blob columns.""" import importlib -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional import pyarrow as pa import pyarrow.ipc @@ -181,13 +181,60 @@ def _resolve_blob_type(): return _resolved_blob_type -def blob(name: str, nullable: bool = True) -> pa.Field: +def blob( + name: str, + nullable: bool = True, + *, + inline_size_threshold: Optional[int] = None, + dedicated_size_threshold: Optional[int] = None, + pack_file_size_threshold: Optional[int] = None, +) -> pa.Field: """Create a Lance blob v2 column field. When pylance is installed this is ``lance.blob.BlobType``. + + Parameters + ---------- + name: + Field name. + nullable: + Whether the column accepts null values. + inline_size_threshold: + Maximum payload size in bytes to keep inline in the data file + before using packed blob storage. Zero is allowed. + dedicated_size_threshold: + Maximum payload size in bytes to store in packed blob storage + before using dedicated blob storage. Must be positive. + pack_file_size_threshold: + Maximum size in bytes of a single packed blob sidecar file before + a new one is started. Must be positive. + + The thresholds are forwarded to ``lance.blob_field``, which validates + them and stores them as field metadata, so pylance is required whenever + any threshold is set. When all thresholds are omitted the behavior is + unchanged. """ - blob_type = _resolve_blob_type() - return pa.field(name, blob_type(), nullable=nullable) + if ( + inline_size_threshold is None + and dedicated_size_threshold is None + and pack_file_size_threshold is None + ): + blob_type = _resolve_blob_type() + return pa.field(name, blob_type(), nullable=nullable) + try: + lance = importlib.import_module("lance") + except ModuleNotFoundError as err: + raise ValueError( + "Setting blob tier thresholds requires pylance to be installed" + ) from err + _resolve_blob_type() + return lance.blob_field( + name, + nullable=nullable, + inline_size_threshold=inline_size_threshold, + dedicated_size_threshold=dedicated_size_threshold, + pack_file_size_threshold=pack_file_size_threshold, + ) def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType: diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 2bb5b6bea..27e920e50 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -95,6 +95,37 @@ def test_blob_factory_declares_v2_field(): assert type(field.type) is LanceBlobType +def test_blob_thresholds_land_in_field_metadata(): + field = lancedb.blob( + "payload", + inline_size_threshold=8 * 1024, + dedicated_size_threshold=1 << 20, + pack_file_size_threshold=1 << 26, + ) + assert field.type.extension_name == "lance.blob.v2" + metadata = field.metadata + assert metadata[b"lance-encoding:blob-inline-size-threshold"] == b"8192" + assert metadata[b"lance-encoding:blob-dedicated-size-threshold"] == b"1048576" + assert metadata[b"lance-encoding:blob-pack-file-size-threshold"] == b"67108864" + + +def test_blob_without_thresholds_has_no_threshold_metadata(): + field = lancedb.blob("image") + metadata = field.metadata or {} + assert not any(b"threshold" in key for key in metadata) + + +def test_blob_thresholds_are_validated(): + with pytest.raises(ValueError, match="inline_size_threshold must be non-negative"): + lancedb.blob("payload", inline_size_threshold=-1) + with pytest.raises(ValueError, match="dedicated_size_threshold must be positive"): + lancedb.blob("payload", dedicated_size_threshold=0) + with pytest.raises(ValueError, match="pack_file_size_threshold must be positive"): + lancedb.blob("payload", pack_file_size_threshold=0) + with pytest.raises(TypeError, match="inline_size_threshold must be an int"): + lancedb.blob("payload", inline_size_threshold="8192") + + def test_blob_type_works_without_pylance(): script = _HIDE_LANCE_BLOB + textwrap.dedent( """\ diff --git a/rust/lancedb/src/blob.rs b/rust/lancedb/src/blob.rs index cbc0724c6..b7f16974e 100644 --- a/rust/lancedb/src/blob.rs +++ b/rust/lancedb/src/blob.rs @@ -262,6 +262,37 @@ pub fn blob(name: impl AsRef, nullable: bool) -> Field { lance::blob::blob_field(name.as_ref(), nullable) } +/// Options for creating a blob v2 column field with tier thresholds. +/// +/// Re-exported from lance so the thresholds are built exactly the way +/// [`lance::blob::blob_field_with_options`] builds them. Note that lance does +/// not currently expose a pack file size threshold option, so only the inline +/// and dedicated size thresholds are supported here. +pub use lance::blob::BlobFieldOptions as BlobOptions; + +/// Creates an Arrow field for a Lance blob v2 column with tier thresholds. +/// +/// Same as [`blob`], but forwards `options` to +/// [`lance::blob::blob_field_with_options`], storing the inline and dedicated +/// size thresholds as field metadata. With [`BlobOptions::default()`] this is +/// identical to [`blob`]. +/// +/// ``` +/// use std::num::NonZeroUsize; +/// +/// use lancedb::blob::BlobOptions; +/// +/// let options = BlobOptions { +/// inline_size_threshold: Some(8 * 1024), +/// dedicated_size_threshold: NonZeroUsize::new(1 << 20), +/// }; +/// let field = lancedb::blob::blob_with_options("payload", true, options); +/// assert!(lancedb::blob::is_blob(&field)); +/// ``` +pub fn blob_with_options(name: impl AsRef, nullable: bool, options: BlobOptions) -> Field { + lance::blob::blob_field_with_options(name.as_ref(), nullable, options) +} + /// Returns true if `field` is a blob v2 column. /// /// ``` @@ -501,8 +532,13 @@ pub(crate) async fn take_blob_files_aligned( #[cfg(test)] mod tests { use super::*; + use std::num::NonZeroUsize; + use arrow_schema::DataType; - use lance_arrow::ARROW_EXT_NAME_KEY; + use lance_arrow::{ + ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, + BLOB_INLINE_SIZE_THRESHOLD_META_KEY, + }; fn blob_schema() -> Schema { Schema::new(vec![ @@ -521,6 +557,38 @@ mod tests { assert!(matches!(field.data_type(), DataType::Struct(_))); } + #[test] + fn blob_with_options_sets_threshold_metadata() { + let options = BlobOptions { + inline_size_threshold: Some(8 * 1024), + dedicated_size_threshold: NonZeroUsize::new(1 << 20), + }; + let field = blob_with_options("payload", true, options); + assert!(is_blob(&field)); + assert_eq!( + field + .metadata() + .get(BLOB_INLINE_SIZE_THRESHOLD_META_KEY) + .map(String::as_str), + Some("8192") + ); + assert_eq!( + field + .metadata() + .get(BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY) + .map(String::as_str), + Some("1048576") + ); + } + + #[test] + fn blob_with_default_options_matches_plain_blob() { + let field = blob_with_options("payload", true, BlobOptions::default()); + let plain = blob("payload", true); + assert_eq!(field.metadata(), plain.metadata()); + assert_eq!(field.data_type(), plain.data_type()); + } + #[test] fn has_blob_columns_detects_blob_fields() { assert!(has_blob_columns(&blob_schema())); From 0212cfd3d66e7bdc2592bbd5ea9b2db4338ba70e Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:49:51 -0700 Subject: [PATCH 02/19] fix(python): normalize blob row ID inputs and errors (#4293) --- docs/src/js/classes/Table.md | 4 + docs/src/python/python.md | 2 + nodejs/__test__/table.test.ts | 12 ++ nodejs/lancedb/table.ts | 4 + python/python/lancedb/_blob.py | 31 +++-- python/python/lancedb/_lancedb.pyi | 4 +- python/python/lancedb/remote/table.py | 8 +- python/python/lancedb/table.py | 54 ++++++--- python/python/tests/test_blob.py | 84 ++++++++++++- python/src/table.rs | 26 +++- rust/lancedb/src/blob.rs | 158 +++++++++++++++++++------ rust/lancedb/src/table.rs | 9 +- rust/lancedb/tests/blob_integration.rs | 50 +++++++- 13 files changed, 368 insertions(+), 78 deletions(-) diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 3ac1dd031..4881d62b5 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -527,6 +527,8 @@ table's current checkout. Preserves input order, duplicates, and nulls. Use this for large payloads. See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions. +One missing or deleted row ID rejects the entire batch, including on remote +tables; no partial handles are returned. #### Parameters @@ -556,6 +558,8 @@ Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid). Reads the table's current checkout. IDs from another version can fail after compaction unless stable row ids are enabled. Results keep input order and duplicates. Null blobs are `null`. Empty blobs are empty buffers. +One missing or deleted row ID rejects the entire batch, including on remote +tables; no partial results are returned. Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes. The client splits requests automatically and reads an individual larger blob through the Range route. This method still materializes all bytes in diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 3776283dc..28e5f9a1b 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -342,6 +342,8 @@ pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns still work. Queries return descriptors. Call [`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or [`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes. +One missing or deleted row ID rejects the entire blob-read batch; the APIs do +not return partial results. ::: lancedb.blob diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index d2244b48c..67d5a02d3 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2817,6 +2817,18 @@ describe("when dealing with blob columns", () => { expect(Buffer.from(await files[1]!.read()).toString()).toBe("beta"); }); + it("rejects a whole blob read batch when one row ID was deleted", async () => { + const { table, rowIds } = await openBlobTable(); + await table.delete("id = 2"); + + await expect(table.fetchBlobs("image", rowIds)).rejects.toThrow( + `first missing row ids: [${rowIds[1]}]`, + ); + await expect(table.fetchBlobFiles("image", rowIds)).rejects.toThrow( + `first missing row ids: [${rowIds[1]}]`, + ); + }); + it("adds a LargeBinary Arrow column to a blob table", async () => { const db = await connect(tmpDir.name); const schema = new Schema([new Field("id", new Int64()), blob("image")]); diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index eae718168..dbf6d7ca2 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -536,6 +536,8 @@ export abstract class Table { * Reads the table's current checkout. IDs from another version can fail after * compaction unless stable row ids are enabled. Results keep input order and * duplicates. Null blobs are `null`. Empty blobs are empty buffers. + * One missing or deleted row ID rejects the entire batch, including on remote + * tables; no partial results are returned. * Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes. * The client splits requests automatically and reads an individual larger * blob through the Range route. This method still materializes all bytes in @@ -553,6 +555,8 @@ export abstract class Table { * * Preserves input order, duplicates, and nulls. Use this for large payloads. * See {@link Table.fetchBlobs} for row-ID validity across versions. + * One missing or deleted row ID rejects the entire batch, including on remote + * tables; no partial handles are returned. */ abstract fetchBlobFiles( column: string, diff --git a/python/python/lancedb/_blob.py b/python/python/lancedb/_blob.py index ee78a7312..0dba427cc 100644 --- a/python/python/lancedb/_blob.py +++ b/python/python/lancedb/_blob.py @@ -6,6 +6,7 @@ from __future__ import annotations import io +import operator from collections.abc import Awaitable, Callable, Iterable from typing import TYPE_CHECKING, Optional, Union @@ -432,16 +433,30 @@ def _leaf_struct_column(tbl: pa.Table, path: str) -> pa.StructArray: def _normalize_blob_row_ids( - row_ids: Union[list[int], pa.Table], blob_column: str + row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table], blob_column: str ) -> list[int]: if isinstance(row_ids, pa.Table): - return read_row_ids_from_hits(row_ids, blob_column) - if isinstance(row_ids, (pa.Array, pa.ChunkedArray)): - raise ValueError( - "pass a query table with _rowid, not a column array " - "(use fetch_blobs('image', hits), not fetch_blobs('image', hits['image']))" - ) - return list(row_ids) + values = read_row_ids_from_hits(row_ids, blob_column) + elif isinstance(row_ids, (pa.Array, pa.ChunkedArray)): + if not pa.types.is_integer(row_ids.type): + raise ValueError( + "row id arrays must have an integer type; pass a query table with " + "_rowid when using a blob descriptor column" + ) + values = row_ids.to_pylist() + else: + values = list(row_ids) + + normalized = [] + for value in values: + try: + row_id = operator.index(value) + except TypeError as exc: + raise ValueError("row ids must be integers without nulls") from exc + if row_id < 0: + raise ValueError("row ids must be non-negative") + normalized.append(row_id) + return normalized def _wrap_blob_files(handles: Iterable[object]) -> list[Optional[BlobFile]]: diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 8d0a86e4a..4b0ebf839 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -1,6 +1,6 @@ from datetime import date, datetime, timedelta from decimal import Decimal -from typing import Dict, List, Optional, Tuple, Any, TypedDict, Union, Literal +from typing import Dict, List, Optional, Sequence, Tuple, Any, TypedDict, Union, Literal from uuid import UUID import pyarrow as pa @@ -563,7 +563,7 @@ class Table: async def fetch_blob_ranges( self, column: str, - requests: List[Tuple[int, int, int]], + requests: Sequence[Sequence[int]], ) -> pa.LargeBinaryArray: ... async def fetch_blob_files( self, column: str, row_ids: list[int] diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 7f03f8fa9..1bb238fbd 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -1154,7 +1154,9 @@ class RemoteTable(Table): return LOOP.run(self._table.blob_columns()) def fetch_blobs( - self, column: str, row_ids: Union[list[int], pa.Table] + self, + column: str, + row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table], ) -> pa.LargeBinaryArray: return LOOP.run(self._table.fetch_blobs(column, row_ids)) @@ -1162,7 +1164,9 @@ class RemoteTable(Table): return LOOP.run(self._table.fetch_blob_ranges(column, list(requests))) def fetch_blob_files( - self, column: str, row_ids: Union[list[int], pa.Table] + self, + column: str, + row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table], ) -> "list[Optional[BlobFile]]": return LOOP.run(self._table.fetch_blob_files(column, row_ids)) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 81b376ebd..833d73c88 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -2058,15 +2058,20 @@ class Table(ABC): @abstractmethod def fetch_blobs( - self, column: str, row_ids: Union[list[int], pa.Table] + self, + column: str, + row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table], ) -> pa.LargeBinaryArray: """Materialize full blob bytes for ``column`` at the given rows. The result has the same length and order as ``row_ids``. Null blobs produce null slots; valid empty blobs produce ``b""``. - ``_rowid`` values stay valid after compaction when the table has stable - row ids. + ``row_ids`` may be a list, an integer Arrow array or chunked array, + or a query table with row ids. ``_rowid`` values stay valid after + compaction when the table has stable row ids. + One missing or deleted row ID rejects the entire batch; no partial + results are returned. Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes. The client splits requests automatically and reads an individual @@ -2078,17 +2083,20 @@ class Table(ABC): def fetch_blob_ranges( self, column: str, - requests: Sequence[Tuple[int, int, int]], + requests: Sequence[Sequence[int]], ) -> pa.LargeBinaryArray: """Materialize row-specific byte ranges from a blob v2 column. - Each request is a ``(row_id, offset, length)`` tuple. Requests may be - repeated or reordered, including multiple ranges for the same blob. + Each request is a three-item ``(row_id, offset, length)`` sequence. + Requests may be repeated or reordered, including multiple ranges for + the same blob. The result has the same length and order as ``requests``; null blobs produce null slots and empty ranges on non-null blobs produce ``b""``. ``_rowid`` values stay valid after compaction when the table has stable row ids. + One missing or deleted row ID rejects the entire batch; no partial + results are returned. Row IDs can be obtained from a query with ``with_row_id(True)``. This API is currently supported only by local tables. @@ -2096,18 +2104,22 @@ class Table(ABC): @abstractmethod def fetch_blob_files( - self, column: str, row_ids: Union[list[int], pa.Table] + self, + column: str, + row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table], ) -> "list[Optional[BlobFile]]": """Open lazy, seekable :class:`~lancedb._blob.BlobFile` handles. Prefer this over :meth:`fetch_blobs` for large payloads. ``row_ids`` is - a ``list[int]`` or a query ``pyarrow.Table`` carrying row identity via - ``_rowid`` or a ``_lance_row_id`` field on the blob descriptor. Null - rows are ``None``. Remote tables require LanceDB Cloud server 0.5.0 or - newer. + a ``list[int]``, an integer Arrow array or chunked array, or a query + ``pyarrow.Table`` carrying row identity via ``_rowid`` or a + ``_lance_row_id`` field on the blob descriptor. Null blob values return + ``None``. Remote tables require LanceDB Cloud server 0.5.0 or newer. ``_rowid`` values stay valid after compaction when the table has stable row ids. + One missing or deleted row ID rejects the entire batch; no partial + handles are returned. """ @abstractmethod @@ -3013,19 +3025,23 @@ class LanceTable(Table): return LOOP.run(self._table.blob_columns()) def fetch_blobs( - self, column: str, row_ids: Union[list[int], pa.Table] + self, + column: str, + row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table], ) -> pa.LargeBinaryArray: return LOOP.run(self._table.fetch_blobs(column, row_ids)) def fetch_blob_ranges( self, column: str, - requests: Sequence[Tuple[int, int, int]], + requests: Sequence[Sequence[int]], ) -> pa.LargeBinaryArray: return LOOP.run(self._table.fetch_blob_ranges(column, list(requests))) def fetch_blob_files( - self, column: str, row_ids: Union[list[int], pa.Table] + self, + column: str, + row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table], ) -> "list[Optional[BlobFile]]": return LOOP.run(self._table.fetch_blob_files(column, row_ids)) @@ -7079,7 +7095,9 @@ class AsyncTable: return await self._inner.blob_columns() async def fetch_blobs( - self, column: str, row_ids: Union[list[int], pa.Table] + self, + column: str, + row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table], ) -> pa.LargeBinaryArray: return await self._inner.fetch_blobs( column, _normalize_blob_row_ids(row_ids, column) @@ -7088,12 +7106,14 @@ class AsyncTable: async def fetch_blob_ranges( self, column: str, - requests: Sequence[Tuple[int, int, int]], + requests: Sequence[Sequence[int]], ) -> pa.LargeBinaryArray: return await self._inner.fetch_blob_ranges(column, list(requests)) async def fetch_blob_files( - self, column: str, row_ids: Union[list[int], pa.Table] + self, + column: str, + row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table], ) -> "list[Optional[BlobFile]]": handles = await self._inner.fetch_blob_files( column, _normalize_blob_row_ids(row_ids, column) diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 27e920e50..f852f28f5 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -70,21 +70,25 @@ def _row_ids_by_id(table): def _assert_missing_blob_row_ids(exc_info): message = str(exc_info.value) - assert "row ids" in message + assert "row ids but some do not exist in the table" in message assert "rowaddr" not in message assert "fragment" not in message def _assert_fetch_apis_reject_missing_row_ids(table, row_ids): + expected = f"first missing row ids: [{row_ids[0]}]" with pytest.raises(ValueError) as exc_info: table.fetch_blobs("image", row_ids) _assert_missing_blob_row_ids(exc_info) + assert expected in str(exc_info.value) with pytest.raises(ValueError) as exc_info: table.fetch_blob_files("image", row_ids) _assert_missing_blob_row_ids(exc_info) + assert expected in str(exc_info.value) with pytest.raises(ValueError) as exc_info: table.fetch_blob_ranges("image", [(row_id, 0, 1) for row_id in row_ids]) _assert_missing_blob_row_ids(exc_info) + assert expected in str(exc_info.value) def test_blob_factory_declares_v2_field(): @@ -861,6 +865,38 @@ def test_fetch_blobs_accepts_query_result(): assert {blobs[i].as_py() for i in range(len(blobs))} == {b"gamma"} +def test_fetch_blob_apis_accept_integer_arrow_row_ids(): + table = _blob_table( + "arrow_row_ids", + [{"id": 1, "image": b"alpha"}, {"id": 2, "image": b"beta"}], + ) + by_id = _row_ids_by_id(table) + ids = [by_id[2], by_id[1]] + + for row_ids in ( + pa.array(ids, type=pa.uint64()), + pa.chunked_array([ids[:1], ids[1:]], type=pa.int64()), + ): + assert table.fetch_blobs("image", row_ids).to_pylist() == [b"beta", b"alpha"] + files = table.fetch_blob_files("image", row_ids) + assert [file.read() for file in files] == [b"beta", b"alpha"] + + +def test_fetch_blob_apis_validate_arrow_and_negative_row_ids(): + table = _blob_table("invalid_arrow_row_ids", [{"id": 1, "image": b"alpha"}]) + hits = table.search().with_row_id(True).to_arrow() + + for fetch in (table.fetch_blobs, table.fetch_blob_files): + with pytest.raises(ValueError, match="non-negative"): + fetch("image", [-1]) + with pytest.raises(ValueError, match="non-negative"): + fetch("image", pa.array([-1], type=pa.int64())) + with pytest.raises(ValueError, match="without nulls"): + fetch("image", pa.array([None], type=pa.int64())) + with pytest.raises(ValueError, match="integer type"): + fetch("image", hits["image"]) + + def test_fetch_blobs_after_compact_with_stable_row_ids(tmp_path): db = lancedb.connect( tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"} @@ -992,6 +1028,15 @@ def test_fetch_blob_ranges_aligns_repeated_ranges_and_nulls(): assert ranges.to_pylist() == [b"cde", None, b"ab", b"cde", b""] +def test_fetch_blob_ranges_accepts_three_item_sequences(): + table = _blob_table("range_sequences", [{"id": 1, "image": b"abcdef"}]) + row_id = _row_ids_by_id(table)[1] + + ranges = table.fetch_blob_ranges("image", [[row_id, 1, 2], (row_id, 4, 2)]) + + assert ranges.to_pylist() == [b"bc", b"ef"] + + def test_fetch_blob_ranges_validates_requests(): table = _blob_table("range_validation", [{"id": 1, "image": b"abc"}]) row_id = _row_ids_by_id(table)[1] @@ -1006,6 +1051,30 @@ def test_fetch_blob_ranges_validates_requests(): table.fetch_blob_ranges("image", [(2**64 - 1, 0, 1)]) _assert_missing_blob_row_ids(exc_info) + for index, name in enumerate(("row id", "offset", "length")): + request = [row_id, 0, 1] + request[index] = -1 + with pytest.raises( + ValueError, match=f"blob range {name} must be a non-negative" + ): + table.fetch_blob_ranges("image", [request]) + + with pytest.raises(ValueError, match="three values"): + table.fetch_blob_ranges("image", [[row_id, 0]]) + + +def test_fetch_blob_apis_reject_row_past_fragment_end(): + table = _blob_table( + "past_fragment_end", + [ + {"id": 1, "image": b"a"}, + {"id": 2, "image": b"b"}, + {"id": 3, "image": b"c"}, + ], + ) + _assert_fetch_apis_reject_missing_row_ids(table, [3]) + _assert_fetch_apis_reject_missing_row_ids(table, [3, 0]) + def test_fetch_blob_apis_reject_missing_fragment_row_addr(): table = _blob_table("missing_frag", [{"id": 1, "image": b"x"}]) @@ -1023,6 +1092,19 @@ def test_fetch_blob_apis_reject_deleted_row_ids(): _assert_fetch_apis_reject_missing_row_ids(table, [by_id[2], by_id[1]]) +def test_fetch_blob_apis_reject_deleted_stable_row_ids(tmp_path): + db = lancedb.connect( + tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"} + ) + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = db.create_table("stable_deleted", schema=schema) + table.add([{"id": 1, "image": b"one"}, {"id": 2, "image": b"two"}]) + by_id = _row_ids_by_id(table) + table.delete("id = 2") + + _assert_fetch_apis_reject_missing_row_ids(table, [by_id[2], by_id[1]]) + + def test_fetch_blob_ranges_empty_requests_returns_empty_array(): table = _blob_table("range_empty", [{"id": 1, "image": b"x"}]) assert table.fetch_blob_ranges("image", []).to_pylist() == [] diff --git a/python/src/table.rs b/python/src/table.rs index d82ebbaaf..7c0ec6610 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -1462,14 +1462,30 @@ impl Table { pub fn fetch_blob_ranges( self_: PyRef<'_, Self>, column: String, - requests: Vec<(u64, u64, u64)>, + requests: Vec>, ) -> PyResult> { let inner = self_.inner_ref()?.clone(); + let requests = requests + .into_iter() + .map(|values| { + let [row_id, offset, length] = <[i128; 3]>::try_from(values).map_err(|_| { + PyValueError::new_err("each blob range request must have three values") + })?; + let to_u64 = |value, name| { + u64::try_from(value).map_err(|_| { + PyValueError::new_err(format!( + "blob range {name} must be a non-negative 64-bit integer" + )) + }) + }; + Ok(BlobRangeRequest::new( + to_u64(row_id, "row id")?, + to_u64(offset, "offset")?, + to_u64(length, "length")?, + )) + }) + .collect::>>()?; future_into_py(self_.py(), async move { - let requests = requests - .into_iter() - .map(|(row_id, offset, length)| BlobRangeRequest::new(row_id, offset, length)) - .collect::>(); let blobs: LargeBinaryArray = inner .fetch_blob_ranges(column, requests) .await diff --git a/rust/lancedb/src/blob.rs b/rust/lancedb/src/blob.rs index b7f16974e..178752952 100644 --- a/rust/lancedb/src/blob.rs +++ b/rust/lancedb/src/blob.rs @@ -399,45 +399,115 @@ pub(crate) fn ensure_blob_v2_column( } } -fn ensure_all_row_ids_resolved(column: &str, requested: usize, resolved: usize) -> Result<()> { - if requested == resolved { +fn missing_blob_row_ids_error(column: &str, requested: usize, missing: &[u64]) -> Error { + let detail = if missing.is_empty() { + String::new() + } else { + format!("; first missing row ids: {missing:?}") + }; + Error::InvalidInput { + message: format!( + "blob read for column '{column}' requested {requested} row ids but some \ + do not exist in the table{detail}; pass row ids collected from this table" + ), + } +} + +/// Diagnose only after a failed read. Stable row IDs use Lance's deletion-aware +/// index; physical row IDs are fragment/offset pairs. Limit the error to three +/// distinct IDs so a large failed batch stays readable. +async fn find_missing_blob_row_ids(dataset: &Dataset, row_ids: &[u64]) -> Result> { + let mut missing = Vec::new(); + if let Some(index) = lance::dataset::rowids::get_row_id_index(dataset).await? { + for (row_id, address) in row_ids.iter().zip(index.get_many(row_ids)?) { + if address.is_none() && !missing.contains(row_id) { + missing.push(*row_id); + if missing.len() == 3 { + break; + } + } + } + } else { + let mut fragments = std::collections::HashMap::new(); + for &row_id in row_ids { + let fragment_id = row_id >> 32; + let offset = row_id as u32; + if let std::collections::hash_map::Entry::Vacant(entry) = fragments.entry(fragment_id) { + let fragment = dataset.get_fragment(fragment_id as usize); + let state = if let Some(fragment) = fragment { + Some(( + fragment.physical_rows().await?, + fragment.get_deletion_vector().await?, + )) + } else { + None + }; + entry.insert(state); + } + let is_missing = match fragments.get(&fragment_id) { + Some(Some((physical_rows, deletions))) => { + offset as usize >= *physical_rows + || deletions + .as_ref() + .is_some_and(|deletions| deletions.contains(offset)) + } + Some(None) => true, + None => unreachable!(), + }; + if is_missing && !missing.contains(&row_id) { + missing.push(row_id); + if missing.len() == 3 { + break; + } + } + } + } + Ok(missing) +} + +async fn ensure_all_row_ids_resolved( + dataset: &Dataset, + column: &str, + row_ids: &[u64], + resolved: usize, +) -> Result<()> { + if row_ids.len() == resolved { return Ok(()); } - if resolved < requested { - Err(Error::InvalidInput { - message: format!( - "blob read for column '{column}' requested {requested} row ids but only {resolved} \ - exist in the table; pass row ids collected from this table" - ), - }) + if resolved < row_ids.len() { + let missing = find_missing_blob_row_ids(dataset, row_ids) + .await + .unwrap_or_default(); + Err(missing_blob_row_ids_error(column, row_ids.len(), &missing)) } else { Err(Error::Runtime { message: format!( - "blob read for column '{column}' returned {resolved} results for {requested} row ids" + "blob read for column '{column}' returned {resolved} results for {} row ids", + row_ids.len() ), }) } } -/// Lance take reports a missing physical row address as NotSupported or InvalidInput. -fn map_blob_take_error(column: &str, requested: usize, err: lance::Error) -> Error { - let missing_row_addr = match &err { - lance::Error::NotSupported { source, .. } => { - source.to_string().contains("must not target deleted rows") - } - lance::Error::InvalidInput { source, .. } => source - .to_string() - .contains("belongs to non-existent fragment"), - _ => false, - }; +/// Lance reports missing physical and stable row ids through several error paths +/// without a shared typed error. +async fn map_blob_take_error( + dataset: &Dataset, + column: &str, + row_ids: &[u64], + err: lance::Error, +) -> Error { + let message = err.to_string(); + let missing_row_id = message.contains("must not target deleted rows") + || message.contains("belongs to non-existent fragment") + || (message.contains("Invalid read params ") && message.contains("addressable rows")) + || message.contains("Could not resolve all requested row IDs"); - if missing_row_addr { - Error::InvalidInput { - message: format!( - "blob read for column '{column}' requested {requested} row ids but some \ - do not exist in the table; pass row ids collected from this table" - ), - } + if missing_row_id { + let missing = find_missing_blob_row_ids(dataset, row_ids) + .await + .unwrap_or_default(); + missing_blob_row_ids_error(column, row_ids.len(), &missing) } else { err.into() } @@ -454,18 +524,25 @@ pub(crate) async fn take_blob_ranges_aligned( return Ok(LargeBinaryBuilder::new().finish()); } + let row_ids = requests + .iter() + .map(|request| request.row_id) + .collect::>(); let lance_requests = requests .iter() .map(|request| LanceBlobRangeRequest::new(request.row_id, request.offset, request.length)) .collect::>(); - let payloads = dataset + let payloads = match dataset .read_blob_ranges(column)? .with_row_ids(lance_requests) .preserve_order(true) .execute() .await - .map_err(|err| map_blob_take_error(column, requests.len(), err))?; - ensure_all_row_ids_resolved(column, requests.len(), payloads.len())?; + { + Ok(payloads) => payloads, + Err(err) => return Err(map_blob_take_error(dataset, column, &row_ids, err).await), + }; + ensure_all_row_ids_resolved(dataset, column, &row_ids, payloads.len()).await?; let mut builder = LargeBinaryBuilder::new(); for payload in payloads { @@ -488,14 +565,17 @@ pub(crate) async fn take_blobs_aligned( return Ok(LargeBinaryBuilder::new().finish()); } - let payloads = dataset + let payloads = match dataset .read_blobs(column)? .with_row_ids(row_ids.to_vec()) .preserve_order(true) .execute() .await - .map_err(|err| map_blob_take_error(column, row_ids.len(), err))?; - ensure_all_row_ids_resolved(column, row_ids.len(), payloads.len())?; + { + Ok(payloads) => payloads, + Err(err) => return Err(map_blob_take_error(dataset, column, row_ids, err).await), + }; + ensure_all_row_ids_resolved(dataset, column, row_ids, payloads.len()).await?; let mut builder = LargeBinaryBuilder::new(); for payload in payloads { @@ -518,11 +598,11 @@ pub(crate) async fn take_blob_files_aligned( return Ok(Vec::new()); } - let handles = dataset - .take_blobs(row_ids, column) - .await - .map_err(|err| map_blob_take_error(column, row_ids.len(), err))?; - ensure_all_row_ids_resolved(column, row_ids.len(), handles.len())?; + let handles = match dataset.take_blobs(row_ids, column).await { + Ok(handles) => handles, + Err(err) => return Err(map_blob_take_error(dataset, column, row_ids, err).await), + }; + ensure_all_row_ids_resolved(dataset, column, row_ids, handles.len()).await?; Ok(handles .into_iter() .map(|handle| handle.map(Into::into)) diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 15ca3702b..3a833b293 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -1259,7 +1259,8 @@ impl Table { /// [`Self::fetch_blob_files`] for large selections. /// /// `_rowid` values stay valid after compaction when the table has stable - /// row ids. + /// row ids. One missing or deleted row id rejects the whole request; no + /// partial results are returned. /// /// ``` /// use arrow_array::UInt64Array; @@ -1303,7 +1304,8 @@ impl Table { /// non-null blobs produce empty byte strings. /// /// `_rowid` values stay valid after compaction when the table has stable - /// row ids. + /// row ids. One missing or deleted row id rejects the whole request; no + /// partial results are returned. /// /// ``` /// use lancedb::blob::BlobRangeRequest; @@ -1344,7 +1346,8 @@ impl Table { /// read from disk until a call to [`BlobFile::read`]. /// /// `_rowid` values stay valid after compaction when the table has stable - /// row ids. + /// row ids. One missing or deleted row id rejects the whole request; no + /// partial handles are returned. /// /// ``` /// # use lancedb::Table; diff --git a/rust/lancedb/tests/blob_integration.rs b/rust/lancedb/tests/blob_integration.rs index dd4051657..0c9942db6 100644 --- a/rust/lancedb/tests/blob_integration.rs +++ b/rust/lancedb/tests/blob_integration.rs @@ -501,17 +501,23 @@ async fn collect_id_rowid(table: &Table) -> Result> { fn assert_missing_blob_row_ids(err: &Error) { assert!(matches!(err, Error::InvalidInput { .. }), "got {err:?}"); let message = err.to_string(); - assert!(message.contains("row ids"), "{message}"); + assert!( + message.contains("row ids but some do not exist in the table"), + "{message}" + ); assert!(!message.contains("rowaddr"), "{message}"); assert!(!message.contains("fragment"), "{message}"); } async fn assert_fetch_apis_reject_missing_row_ids(table: &Table, row_ids: &[u64]) -> Result<()> { + let expected = format!("first missing row ids: [{}]", row_ids[0]); let err = table.fetch_blobs("image", row_ids).await.unwrap_err(); assert_missing_blob_row_ids(&err); + assert!(err.to_string().contains(&expected), "{err}"); let err = table.fetch_blob_files("image", row_ids).await.unwrap_err(); assert_missing_blob_row_ids(&err); + assert!(err.to_string().contains(&expected), "{err}"); let err = table .fetch_blob_ranges( @@ -524,6 +530,7 @@ async fn assert_fetch_apis_reject_missing_row_ids(table: &Table, row_ids: &[u64] .await .unwrap_err(); assert_missing_blob_row_ids(&err); + assert!(err.to_string().contains(&expected), "{err}"); Ok(()) } @@ -790,6 +797,30 @@ async fn fetch_blobs_out_of_range_id_errors_without_panic() -> Result<()> { Ok(()) } +#[tokio::test] +async fn fetch_blob_apis_reject_row_past_fragment_end() -> Result<()> { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().to_str().unwrap()).execute().await?; + let table = + create_inline_blob_table(&db, "t", &[1, 2, 3], &[Some(b"a"), Some(b"b"), Some(b"c")]) + .await?; + + assert_fetch_apis_reject_missing_row_ids(&table, &[3]).await?; + assert_fetch_apis_reject_missing_row_ids(&table, &[3, 0]).await +} + +#[tokio::test] +async fn fetch_blobs_identifies_multiple_missing_ids_without_partial_results() -> Result<()> { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().to_str().unwrap()).execute().await?; + let table = create_inline_blob_table(&db, "t", &[1, 2], &[Some(b"a"), Some(b"b")]).await?; + + let err = table.fetch_blobs("image", &[0, 2, 3]).await.unwrap_err(); + assert_missing_blob_row_ids(&err); + assert!(err.to_string().contains("first missing row ids: [2, 3]")); + Ok(()) +} + #[tokio::test] async fn fetch_blob_files_rejects_missing_fragment_row_addr() -> Result<()> { let tmp = tempdir().unwrap(); @@ -831,6 +862,23 @@ async fn fetch_blob_apis_reject_deleted_row_ids() -> Result<()> { assert_fetch_apis_reject_missing_row_ids(&table, &[deleted_row_addr, live_row_addr]).await } +#[tokio::test] +async fn fetch_blob_apis_reject_deleted_stable_row_ids() -> Result<()> { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().to_str().unwrap()) + .storage_option(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true") + .execute() + .await?; + let table = create_inline_blob_table(&db, "t", &[1, 2], &[Some(b"one"), Some(b"two")]).await?; + let pairs = collect_id_rowid(&table).await?; + let deleted_row_id = pairs.iter().find(|(id, _)| *id == 2).unwrap().1; + let live_row_id = pairs.iter().find(|(id, _)| *id == 1).unwrap().1; + + table.delete("id = 2").await?; + + assert_fetch_apis_reject_missing_row_ids(&table, &[deleted_row_id, live_row_id]).await +} + #[tokio::test] async fn fetch_blobs_rejects_non_blob_column() -> Result<()> { let tmp = tempdir().unwrap(); From c6eeb17f625ba6b942792f6d829fa75d917920f4 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:03:15 -0700 Subject: [PATCH 03/19] fix(python): preserve blob v2 columns on table overwrite (#4289) --- python/python/lancedb/table.py | 63 ++++++++++++++++++++++++++++-- python/python/tests/test_blob.py | 67 ++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 3 deletions(-) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 833d73c88..ad8286032 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -576,6 +576,55 @@ def _cast_to_target_schema( return pa.RecordBatchReader.from_batches(reordered_schema, gen()) +def _preserve_blob_v2_fields( + inferred_schema: pa.Schema, existing_schema: pa.Schema +) -> pa.Schema: + """Keep existing blob fields when an overwrite infers a replacement schema.""" + existing_fields = {field.name: field for field in existing_schema} + fields = [ + _preserve_blob_v2_field(field, existing_fields[field.name]) + if field.name in existing_fields + else field + for field in inferred_schema + ] + return pa.schema(fields, metadata=inferred_schema.metadata) + + +def _preserve_blob_v2_field(inferred: pa.Field, existing: pa.Field) -> pa.Field: + if is_blob_v2_field(existing): + return existing + + inferred_type, existing_type = inferred.type, existing.type + if pa.types.is_struct(inferred_type) and pa.types.is_struct(existing_type): + existing_children = {field.name: field for field in existing_type} + children = [ + _preserve_blob_v2_field(field, existing_children[field.name]) + if field.name in existing_children + else field + for field in inferred_type + ] + return inferred.with_type(pa.struct(children)) + + if _is_list_like(inferred_type) and _is_list_like(existing_type): + value_field = _preserve_blob_v2_field( + inferred_type.value_field, existing_type.value_field + ) + if value_field == inferred_type.value_field: + return inferred + value_field = value_field.with_name(existing_type.value_field.name) + if pa.types.is_fixed_size_list(inferred_type): + value_type = pa.list_(value_field, inferred_type.list_size) + elif pa.types.is_large_list(inferred_type): + value_type = pa.large_list(value_field) + else: + value_type = pa.list_(value_field) + return inferred.with_type(value_type) + + if blob_v2_column_paths(pa.schema([existing])): + return existing + return inferred + + def _coerce_blob_write_columns( batch: pa.RecordBatch, target_schema: pa.Schema ) -> pa.RecordBatch: @@ -1738,7 +1787,8 @@ class Table(ABC): - pyarrow.Table or pyarrow.RecordBatch mode: str The mode to use when writing the data. Valid values are - "append" and "overwrite". + "append" and "overwrite". Overwrite infers a replacement schema + but preserves existing blob v2 fields present in the new data. on_bad_vectors: str, default "error" What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill". @@ -3996,7 +4046,8 @@ class LanceTable(Table): The data to insert into the table. mode: str The mode to use when writing the data. Valid values are - "append" and "overwrite". + "append" and "overwrite". Overwrite infers a replacement schema + but preserves existing blob v2 fields present in the new data. on_bad_vectors: str, default "error" What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill", "null". @@ -6014,7 +6065,8 @@ class AsyncTable: - pyarrow.Table or pyarrow.RecordBatch mode: str The mode to use when writing the data. Valid values are - "append" and "overwrite". + "append" and "overwrite". Overwrite infers a replacement schema + but preserves existing blob v2 fields present in the new data. on_bad_vectors: str, default "error" What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill", "null". @@ -6052,6 +6104,11 @@ class AsyncTable: data, _ = sanitize_create_table( data, None, on_bad_vectors=on_bad_vectors, fill_value=fill_value ) + if blob_v2_column_paths(schema): + # Plain bytes cannot be inferred as blob v2. Keep matching blob + # fields while allowing other columns to use the inferred schema. + target_schema = _preserve_blob_v2_fields(data.schema, schema) + data = _cast_to_target_schema(data, target_schema) elif on_bad_vectors != "error" or ( schema.metadata is not None and b"embedding_functions" in schema.metadata ): diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index f852f28f5..6be0a58a5 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -571,6 +571,73 @@ def test_fetch_blobs_round_trip(): assert [blobs[0].as_py(), blobs[1].as_py()] == [b"alpha", b"beta"] +def test_add_overwrite_preserves_blob_v2(tmp_path): + db = lancedb.connect(tmp_path) + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = db.create_table("overwrite_blob", schema=schema) + table.add([{"id": 1, "image": b"old"}]) + + payload = b"x" * 200_000 + table.add([{"id": 2, "image": payload}], mode="overwrite") + + assert table.count_rows() == 1 + assert blob_v2_column_paths(table.schema) == ["image"] + assert table.blob_columns() == ["image"] + assert table.fetch_blobs("image", [_row_ids_by_id(table)[2]]).to_pylist() == [ + payload + ] + assert list(tmp_path.rglob("*.blob")) + + +def test_add_overwrite_with_blob_still_infers_vector_schema(): + db = lancedb.connect("memory:///overwrite_blob_vector") + schema = pa.schema( + [ + pa.field("vector", pa.list_(pa.float32(), 2)), + lancedb.blob("image"), + ] + ) + table = db.create_table("overwrite_blob_vector", schema=schema) + + table.add([{"vector": [1.0, 2.0, 3.0], "image": b"data"}], mode="overwrite") + + assert table.schema.field("vector").type == pa.list_(pa.float32(), 3) + assert table.blob_columns() == ["image"] + + +@pytest.mark.asyncio +async def test_add_overwrite_preserves_blob_v2_async(): + db = await lancedb.connect_async("memory:///overwrite_blob_async") + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = await db.create_table("overwrite_blob_async", schema=schema) + + await table.add([{"id": 1, "image": b"updated"}], mode="overwrite") + + assert blob_v2_column_paths(await table.schema()) == ["image"] + assert await table.blob_columns() == ["image"] + hits = await table.query().with_row_id().to_arrow() + assert (await table.fetch_blobs("image", hits)).to_pylist() == [b"updated"] + + +def test_add_overwrite_preserves_nested_blob_v2(): + db = lancedb.connect("memory:///overwrite_nested_blob") + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("info", pa.struct([lancedb.blob("image")])), + ] + ) + table = db.create_table("overwrite_nested_blob", schema=schema) + + table.add([{"id": 1, "info": {"image": b"nested"}}], mode="overwrite") + + assert blob_v2_column_paths(table.schema) == ["info.image"] + assert table.blob_columns() == ["info.image"] + assert table.fetch_blobs("info.image", [_row_ids_by_id(table)[1]]).to_pylist() == [ + b"nested" + ] + + def test_merge_insert_writes_python_bytes(): table = _blob_table("merge_bytes", [{"id": 1, "image": b"before"}]) result = ( From 6cc4919c3ceb07824cbc331fc7963a78ea962ac5 Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Tue, 13 Oct 2026 19:59:25 -0700 Subject: [PATCH 04/19] feat: add an authorization API (#4363) Add an authorization API to the LanceDB catalog. This is an API that can be used to bind access control lists (ACLs) to individual catalog objects. It can also be used to manage server-side roles, and introspect principals and groups. The new API is implemented in both the Rust, Python, and async Python styles and includes list_acls, add_acl, delete_acl, list_role_bindings, add_role_binding, delete_role_binding, get_principal, and get_group. This API implements the design principle of forwards-compatibility. So unknown privilege, object, and subject types do not cause errors, but are be supported as best as possible in the library. --- Cargo.lock | 1 + docs/src/python/python.md | 165 +++++ nodejs/__test__/catalog.test.ts | 39 +- python/python/lancedb/_lancedb.pyi | 59 ++ python/python/lancedb/authz.py | 588 +++++++++++++++++ python/python/lancedb/catalog.py | 12 + python/python/tests/test_authz.py | 821 +++++++++++++++++++++++ python/python/tests/test_catalog.py | 42 ++ python/src/authz.rs | 280 ++++++++ python/src/catalog.rs | 7 + python/src/lib.rs | 5 + rust/lancedb/Cargo.toml | 1 + rust/lancedb/src/authz/api.rs | 239 +++++++ rust/lancedb/src/authz/mod.rs | 48 ++ rust/lancedb/src/authz/object.rs | 992 ++++++++++++++++++++++++++++ rust/lancedb/src/authz/privilege.rs | 514 ++++++++++++++ rust/lancedb/src/authz/subject.rs | 378 +++++++++++ rust/lancedb/src/catalog.rs | 12 + rust/lancedb/src/lib.rs | 1 + rust/lancedb/src/remote.rs | 1 + rust/lancedb/src/remote/authz.rs | 254 +++++++ rust/lancedb/src/remote/catalog.rs | 4 + rust/lancedb/src/remote/db.rs | 6 + 23 files changed, 4466 insertions(+), 3 deletions(-) create mode 100644 python/python/lancedb/authz.py create mode 100644 python/python/tests/test_authz.py create mode 100644 python/src/authz.rs create mode 100644 rust/lancedb/src/authz/api.rs create mode 100644 rust/lancedb/src/authz/mod.rs create mode 100644 rust/lancedb/src/authz/object.rs create mode 100644 rust/lancedb/src/authz/privilege.rs create mode 100644 rust/lancedb/src/authz/subject.rs create mode 100644 rust/lancedb/src/remote/authz.rs diff --git a/Cargo.lock b/Cargo.lock index ffed35bcc..e6747eb08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5779,6 +5779,7 @@ dependencies = [ "serial_test", "sha2 0.10.9", "snafu 0.8.9", + "strum_macros 0.28.0", "tempfile", "test-log", "tokenizers", diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 28e5f9a1b..200fdbd73 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -48,6 +48,169 @@ are ordinary connections. Dropping a database requires it to be empty. ::: lancedb.catalog.DatabaseNames +## Authorization + +Some catalogs support the authorization API. This is an API that can be used to +bind access control lists (ACLs) to individual catalog objects. + +Each ACL contains a subject, an object, and privilege. + +### Subjects + +There are three kinds of subjects: principals, groups, and roles. + +A principal uniquely identifies each user accessing the system. Each LanceDB +Enterprise API key corresponds to a unique principal, and each OpenID user has +a unique principal. + +All principals have an immutable ID which never changes, and is safe to include +in log files. For example, the API key +`sk_AAAQEAYEAUDAOCAJBIFQYDIOB4IBCEQTCQKRMFYYDENBWHA5DYPQ====` maps to the +principal ID `pk_aaaqeayeaudaocajbifqydiob4`. APIs such as `list_acls` will +return the principal ID, never the API key. + +OpenID principals also typically have a human-readable name associated with +them. For example, a principal might be named `bob@example.com` and might have +a principal ID of `8a004400-9443-4fab-9997-b9372a7ca8fd`. Whenever +possible, APIs will return the principal name for convenience. However, +principal names are mutable and can be changed by the identity provider. + +Here is an example of getting information about an API key: +```python +from lancedb.authz import Subject + +catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key) +principal = catalog.authz.get_principal(Subject.principal_api_key(api_key)) +print(f"Found principal {principal.principal_id}") +``` + +OpenID principals can be members of groups. Like principals, groups also have +IDs and human-readable names. It is not possible to modify group membership via +this API; it is managed by the OpenID identity provider. + +Here is an example of getting information about a group based on its name: +```python +from lancedb.authz import Subject + +catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key) +group = catalog.authz.get_group(Subject.group_name("Engineering")) +print(f"Found group {group.group_id} with name {group.group_name}") +``` + +Roles can contain either principals or groups. Unlike groups, they are managed +by the catalog, not the identity provider. Roles can also contain API key +principals. + +Here is an example of managing the `reader` role: +```python +from lancedb.authz import Subject + +catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key) +catalog.authz.add_role_binding(subject=Subject.principal_name("Alice"), role="reader") +``` +It may take a few milliseconds for changes to role bindings to become visible +to all clients. + +More subject types may be added in the future. + +### Objects + +An object is any resource in the system that we can attach authorization rules to. +The Python builders and Rust client share resource validation and wire encoding. +Methods also accept literal strings such as `table:tenant/db:public$events`; +malformed names for known resource types raise `ValueError` locally before a +request is sent. Unknown resource types are preserved for compatibility with +newer servers. + +Object types include: + +The System object, a singleton which represents the entire system. You can +refer to it as `Object.system()`. + +Database objects which represents specific databases. For example, +`Object.database("analytics")` represents the "analytics" database. + +Table objects which represent specific tables. For example, +`Object.table(database="analytics", name="events")` represents the "events" +table inside the "analytics" database. Tables are in the default namespace (which +is named "public") unless a specific namespace is given. + +Similarly, View objects represent specific views. For example, +`Object.view(database="analytics", name="recent_events")` represents the +"recent_events" view inside the "analytics" database. Views are in the default +namespace (which is named "public") unless a specific namespace is given. + +Secret objects identify named credentials, for example, +`Object.secret(database="analytics", name="api-key")`. Function objects identify +registered functions, for example, +`Object.function(database="analytics", name="caption")`. Both default to the +`public` namespace. Pass `namespace_path=["platform", "ml"]` to select a nested +namespace. These objects identify resources for ACL operations; a secret object +contains the credential's name, not its value. + +More object types may be added in the future. + +### Access control lists + +An access control list attaches a specific set of privileges to a specific +(subject, object) pair. So, for example, we might have an access control list +that gives `bob@example.com` the ability to read the foo table, that looks like +this: + +```python +from lancedb.authz import Object, Subject, Privilege + +catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key) +catalog.authz.add_acl( + object=Object.table(database="analytics", name="foo"), + subject=Subject.principal_name("bob@example.com"), + privilege=Privilege.SELECT, +) +``` + +Reading a table generally requires USAGE on its database and namespace, plus +SELECT on the table. Each object has at most one owner. + +OWNERSHIP is a special privilege which grants all the other privileges. By +default, the principal who created an object owns it. Only one subject can own +an object; granting OWNERSHIP to a new subject removes it from the previous +owner. OWNERSHIP cannot be deleted. + +To delegate secret or function creation, grant `CREATE_SECRET` or +`CREATE_FUNCTION` on the containing namespace. This preserves namespace ownership: + +```python +for privilege in [Privilege.CREATE_SECRET, Privilege.CREATE_FUNCTION]: + catalog.authz.add_acl( + object=Object.namespace(database="analytics", namespace_path=["public"]), + subject=Subject.role("developer"), + privilege=privilege, + ) +``` + +Both privileges also accept their string names in synchronous and asynchronous +ACL methods. + +By default, the `list_acls` function lists all the access control entries in +the system. By adding a function parameter, the output is filtered to just ACLs +matching that parameter. For example, this code lists all the ACLs for the +"reader" role: + +```python +from lancedb.authz import Subject + +catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key) +for entry in catalog.authz.list_acls(subject=Subject.role("reader")): + print(entry) +``` + +Or, when using the async API: +```python +catalog = await lancedb.connect_catalog_async(endpoint, api_key=api_key) +async for entry in catalog.authz.list_acls(subject=Subject.role("reader")): + print(entry) +``` + ## Remote SQL Submit SQL against a remote LanceDB database through the connection. @@ -361,6 +524,8 @@ not return partial results. ## Exceptions +::: lancedb.remote.errors.HttpError + ::: lancedb.exceptions.MissingValueError ::: lancedb.exceptions.MissingColumnError diff --git a/nodejs/__test__/catalog.test.ts b/nodejs/__test__/catalog.test.ts index 945956b5b..f8b14ca6f 100644 --- a/nodejs/__test__/catalog.test.ts +++ b/nodejs/__test__/catalog.test.ts @@ -266,8 +266,41 @@ describe("remote catalog", () => { "limit", ); } - await expect(catalog.connectDatabase("a$b")).rejects.toThrow( - "Invalid database name", - ); }); + + it.each([ + "", + ".", + "..", + "/db", + "db/", + "a//b", + "a/./b", + "a/../b", + "a b", + "a:b", + "a$b", + "a%b", + "a?b", + "a#b", + "a\\b", + "café", + "a\nb", + ])( + "rejects invalid database name %j before sending requests", + async (name) => { + await withCatalog([], async (catalog, requests) => { + for (const method of [ + "createDatabase", + "connectDatabase", + "dropDatabase", + ] as const) { + await expect(catalog[method](name)).rejects.toThrow( + "Invalid database name", + ); + } + expect(requests).toEqual([]); + }); + }, + ); }); diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 4b0ebf839..38b80c14e 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -955,6 +955,8 @@ class PermutationReader: def __init__(self, base_table: Table, permutation_table: Table): ... class Catalog: + @property + def authz(self) -> AuthorizationClient: ... @property def uri(self) -> str: ... async def create_database( @@ -1001,3 +1003,60 @@ class JobListing: def page_token(self) -> Optional[str]: ... def __aiter__(self) -> JobListing: ... async def __anext__(self) -> JobInfo: ... + +class AuthorizationClient: + async def list_acls(self, request: str) -> str: ... + async def add_acl(self, request: str) -> None: ... + async def delete_acl(self, request: str) -> None: ... + async def list_role_bindings(self, request: str) -> str: ... + async def add_role_binding(self, request: str) -> None: ... + async def delete_role_binding(self, request: str) -> None: ... + async def get_principal(self, request: str) -> str: ... + async def get_group(self, request: str) -> str: ... + +class AuthzSubject: + def __init__(self, kind: str, value: str): ... + @staticmethod + def from_canonical(value: str) -> AuthzSubject: ... + @property + def kind(self) -> str: ... + @property + def value(self) -> str: ... + @property + def display_value(self) -> str: ... + def to_wire(self, context: Optional[str] = None) -> str: ... + def __str__(self) -> str: ... + def __repr__(self) -> str: ... + +class AuthzPrivilege: + def __init__(self, value: str): ... + @staticmethod + def known() -> list[str]: ... + def __str__(self) -> str: ... + +class AuthzObject: + def __init__(self, value: str): ... + @staticmethod + def system() -> AuthzObject: ... + @staticmethod + def database(name: str) -> AuthzObject: ... + @staticmethod + def namespace(database: str, namespace_path: Sequence[str]) -> AuthzObject: ... + @staticmethod + def table( + database: str, namespace_path: Sequence[str], name: str + ) -> AuthzObject: ... + @staticmethod + def view( + database: str, namespace_path: Sequence[str], name: str + ) -> AuthzObject: ... + @staticmethod + def secret( + database: str, namespace_path: Sequence[str], name: str + ) -> AuthzObject: ... + @staticmethod + def function( + database: str, namespace_path: Sequence[str], name: str + ) -> AuthzObject: ... + def __str__(self) -> str: ... + def __repr__(self) -> str: ... diff --git a/python/python/lancedb/authz.py b/python/python/lancedb/authz.py new file mode 100644 index 000000000..e62a276cf --- /dev/null +++ b/python/python/lancedb/authz.py @@ -0,0 +1,588 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Deployment-wide authorization through ``catalog.authz``. + +ACLs grant one privilege on an object to a principal, group, or role. Role +bindings grant a role to a principal or group. Identity-provider group membership +is read-only. Name resolution and permission checks are performed by the server. +""" + +from collections.abc import AsyncIterator, Iterator, Sequence +from dataclasses import dataclass, field +from enum import Enum +import json +from typing import Literal, Optional, TypeVar, Union, cast + +from . import _lancedb +from .background_loop import LOOP + +__all__ = [ + "Authorization", + "AsyncAuthorization", + "Subject", + "Object", + "Privilege", + "AccessControlEntry", + "RoleBinding", + "IdentityInfo", + "PrincipalInfo", + "GroupInfo", +] + +_PAGE_SIZE = 100 +_T = TypeVar("_T") +_SubjectKind = Literal[ + "principal_id", + "principal_name", + "group_id", + "group_name", + "role", + "principal_api_key", + "unknown", +] + + +@dataclass(frozen=True) +class Subject: + """An immutable identity selector. Prefer the named factory methods. + + IDs are stable; names must resolve unambiguously on the server. API-key + selectors identify the subject of an operation, not its authenticated caller. + Secret values are omitted from representations. Unknown canonical subject + kinds returned by newer servers use ``kind="unknown"`` and preserve the full + wire string in ``value``; they can be passed back to ACL methods. + """ + + kind: _SubjectKind + _value: str = field(repr=False) + + _inner: _lancedb.AuthzSubject = field(init=False, repr=False, compare=False) + + def __post_init__(self): + object.__setattr__( + self, "_inner", _lancedb.AuthzSubject(self.kind, self._value) + ) + + @classmethod + def principal_id(cls, id: str) -> "Subject": + """Select a principal by canonical identity-provider ID.""" + return cls("principal_id", id) + + @classmethod + def principal_name(cls, name: str) -> "Subject": + """Select a principal by an unambiguous provider name.""" + return cls("principal_name", name) + + @classmethod + def group_id(cls, id: str) -> "Subject": + """Select a group by canonical identity-provider ID.""" + return cls("group_id", id) + + @classmethod + def group_name(cls, name: str) -> "Subject": + """Select a group by an unambiguous provider name.""" + return cls("group_name", name) + + @classmethod + def role(cls, name: str) -> "Subject": + """Select a LanceDB-managed role, without an ``r:`` prefix.""" + return cls("role", name) + + @classmethod + def principal_api_key(cls, api_key: str) -> "Subject": + """Select the principal holding this secret API key.""" + return cls("principal_api_key", api_key) + + def __repr__(self) -> str: + value = self._inner.display_value + return f"Subject.{self.kind}({value!r})" + + @property + def value(self) -> str: + """The selected value, or full wire string for unknown subject kinds. + + API-key values are secrets. + """ + return self._value + + def __reduce__(self): + return type(self), (self.kind, self._value) + + def _wire(self) -> str: + return self._inner.to_wire() + + @classmethod + def _canonical(cls, value: str) -> "Subject": + inner = _lancedb.AuthzSubject.from_canonical(value) + return cls(cast(_SubjectKind, inner.kind), inner.value) + + +@dataclass(frozen=True) +class Object: + """An immutable resource name in the authorization syntax. + + Use the builders to avoid interpreting separators as part of a name. + Methods also accept raw object strings, validated by the same Rust parser. + Unknown resource types are preserved for compatibility with newer servers. + Omitted or empty namespace paths select the database's public namespace. + """ + + _value: str + _inner: _lancedb.AuthzObject = field(init=False, repr=False, compare=False) + + def __post_init__(self): + object.__setattr__(self, "_inner", _lancedb.AuthzObject(self._value)) + + @classmethod + def system(cls) -> "Object": + """The deployment-wide system object.""" + return cls(str(_lancedb.AuthzObject.system())) + + @classmethod + def database(cls, name: str) -> "Object": + """A database; slash-containing names remain one logical database.""" + return cls(str(_lancedb.AuthzObject.database(name))) + + @classmethod + def namespace( + cls, *, database: str, namespace_path: Sequence[str] = ("public",) + ) -> "Object": + """A namespace path, defaulting to the database's public namespace.""" + return cls(str(_lancedb.AuthzObject.namespace(database, namespace_path))) + + @classmethod + def table( + cls, *, database: str, name: str, namespace_path: Sequence[str] = ("public",) + ) -> "Object": + """A table, defaulting to the database's public namespace.""" + return cls(str(_lancedb.AuthzObject.table(database, namespace_path, name))) + + @classmethod + def view( + cls, *, database: str, name: str, namespace_path: Sequence[str] = ("public",) + ) -> "Object": + """A view, defaulting to the database's public namespace.""" + return cls(str(_lancedb.AuthzObject.view(database, namespace_path, name))) + + @classmethod + def secret( + cls, *, database: str, name: str, namespace_path: Sequence[str] = ("public",) + ) -> "Object": + """A secret, defaulting to the database's public namespace.""" + return cls(str(_lancedb.AuthzObject.secret(database, namespace_path, name))) + + @classmethod + def function( + cls, *, database: str, name: str, namespace_path: Sequence[str] = ("public",) + ) -> "Object": + """A function, defaulting to the database's public namespace.""" + return cls(str(_lancedb.AuthzObject.function(database, namespace_path, name))) + + def __str__(self) -> str: + return str(self._inner) + + def __reduce__(self): + return type(self), (self._value,) + + +class Privilege(str, Enum): + """Known privileges, validated and serialized by Rust. + + ACL methods also accept strings such as ``"SELECT"`` and future privilege + names such as ``"PRIVILEGE20"``. Future names are preserved verbatim; + their meaning and validity are determined by the server. + + Grant ``CREATE_SECRET`` or ``CREATE_FUNCTION`` on a namespace to delegate + creation of those resources without transferring namespace ownership. + """ + + def __new__(cls, value: str): + canonical = str(_lancedb.AuthzPrivilege(value)) + member = str.__new__(cls, canonical) + member._value_ = canonical + return member + + OWNERSHIP = "OWNERSHIP" + USAGE = "USAGE" + SELECT = "SELECT" + INSERT = "INSERT" + UPDATE = "UPDATE" + DELETE = "DELETE" + CREATE_DATABASE = "CREATE_DATABASE" + CREATE_NAMESPACE = "CREATE_NAMESPACE" + CREATE_TABLE = "CREATE_TABLE" + CREATE_MATERIALIZED_VIEW = "CREATE_MATERIALIZED_VIEW" + CREATE_VIEW = "CREATE_VIEW" + CREATE_SECRET = "CREATE_SECRET" + CREATE_FUNCTION = "CREATE_FUNCTION" + OPERATE = "OPERATE" + + +@dataclass(frozen=True) +class AccessControlEntry: + """One object/subject/privilege grant; subject contains a canonical ID or role.""" + + object: str + subject: Subject + subject_name: Optional[str] + privilege: str + + +@dataclass(frozen=True) +class RoleBinding: + """A direct principal/group role binding, not an expanded effective role.""" + + subject: Subject + subject_name: Optional[str] + role: str + + +@dataclass(frozen=True) +class IdentityInfo: + """A canonical identity-provider ID and its optional display name.""" + + id: str + name: Optional[str] = None + + +@dataclass(frozen=True) +class PrincipalInfo: + """A principal's groups and effective roles, including group-inherited roles.""" + + principal_id: str + principal_name: Optional[str] + groups: list[IdentityInfo] + roles: list[str] + + +@dataclass(frozen=True) +class GroupInfo: + """A group's provider-managed members and LanceDB-managed direct roles.""" + + group_id: str + group_name: Optional[str] + principals: list[IdentityInfo] + roles: list[str] + + +def _subject(subject: Subject, context: Optional[str] = None) -> str: + if not isinstance(subject, Subject): + raise ValueError("subject must be a Subject selector") + return subject._inner.to_wire(context) + + +def _object(value: Union[str, Object]) -> str: + if not isinstance(value, (str, Object)) or not str(value): + raise ValueError("object must be a nonempty string or Object") + return str(value) if isinstance(value, Object) else str(_lancedb.AuthzObject(value)) + + +def _limit(limit: Optional[int]) -> None: + if limit is not None and ( + isinstance(limit, bool) or not isinstance(limit, int) or limit < 0 + ): + raise ValueError("limit must be a nonnegative integer or None") + + +def _privilege(value: Union[str, Privilege]) -> str: + if not isinstance(value, str): + raise ValueError("privilege must be a string or Privilege") + return str(_lancedb.AuthzPrivilege(value)) + + +def _entry(value: dict) -> AccessControlEntry: + return AccessControlEntry( + value["object"], + Subject._canonical(value["subject_id"]), + value.get("subject_name"), + value["privilege"], + ) + + +def _binding(value: dict) -> RoleBinding: + return RoleBinding( + Subject._canonical(value["subject_id"]), + value.get("subject_name"), + value["role"], + ) + + +class AsyncAuthorization: + """Async authorization returned by ``AsyncCatalog.authz``. + + HTTP failures raise [HttpError][lancedb.remote.errors.HttpError], preserving + status and request ID. Invalid local arguments raise ValueError. Lists are + lazy async iterators: failures can occur during iteration after some results + have been yielded. Iteration does not provide a consistent snapshot. + """ + + def __init__(self, inner: _lancedb.AuthorizationClient): + self._inner = inner + + async def _pages(self, method, request: dict, key: str, limit: Optional[int]): + yielded = 0 + while limit is None or yielded < limit: + request["max_results"] = ( + _PAGE_SIZE if limit is None else min(_PAGE_SIZE, limit - yielded) + ) + response = json.loads(await method(json.dumps(request))) + for value in response[key]: + yield value + yielded += 1 + if limit is not None and yielded >= limit: + return + token = response.get("continuation_token") + if not token: + return + if token == request.get("continuation_token"): + raise RuntimeError( + "Authorization listing returned a repeated continuation token" + ) + request["continuation_token"] = token + + def list_acls( + self, + *, + object: Optional[Union[str, Object]] = None, + subject: Optional[Subject] = None, + limit: Optional[int] = None, + ) -> AsyncIterator[AccessControlEntry]: + """Iterate grants, optionally filtered by exact object and/or subject. + + Requires USAGE on the supplied object, or on system if object is omitted. + ``limit`` bounds yielded entries across all pages; None lists all and zero + makes no requests. Missing matches produce an empty iterator. + """ + _limit(limit) + request = {} + if object is not None: + request["object"] = _object(object) + if subject is not None: + request["subject"] = _subject(subject) + + async def entries(): + async for value in self._pages( + self._inner.list_acls, request, "entries", limit + ): + yield _entry(value) + + return entries() + + async def add_acl( + self, + *, + object: Union[str, Object], + subject: Subject, + privilege: Union[str, Privilege], + ) -> None: + """Grant one privilege; requires OWNERSHIP on the object. + + A duplicate grant raises HTTP 409. Granting OWNERSHIP transfers ownership. + Writes are not automatically retried after ambiguous failures. + """ + await self._inner.add_acl( + json.dumps( + { + "object": _object(object), + "subject": _subject(subject), + "privilege": _privilege(privilege), + } + ) + ) + + async def delete_acl( + self, + *, + object: Union[str, Object], + subject: Subject, + privilege: Union[str, Privilege], + ) -> None: + """Revoke one privilege; requires OWNERSHIP on the object. + + A missing grant raises HTTP 404. OWNERSHIP itself cannot be deleted. + """ + await self._inner.delete_acl( + json.dumps( + { + "object": _object(object), + "subject": _subject(subject), + "privilege": _privilege(privilege), + } + ) + ) + + def list_role_bindings( + self, + *, + subject: Optional[Subject] = None, + role: Optional[str] = None, + limit: Optional[int] = None, + ) -> AsyncIterator[RoleBinding]: + """Iterate direct bindings, optionally filtered by principal/group and role. + + Requires USAGE on system. Limit semantics match ``list_acls``. Principals' + roles inherited through groups are available through ``get_principal``. + """ + _limit(limit) + request = {} + if subject is not None: + request["subject"] = _subject(subject, "role_member") + if role is not None: + request["role"] = role + + async def bindings(): + async for value in self._pages( + self._inner.list_role_bindings, request, "bindings", limit + ): + yield _binding(value) + + return bindings() + + async def add_role_binding(self, *, subject: Subject, role: str) -> None: + """Grant a role idempotently; requires OPERATE on system.""" + await self._inner.add_role_binding( + json.dumps({"subject": _subject(subject, "role_member"), "role": role}) + ) + + async def delete_role_binding(self, *, subject: Subject, role: str) -> None: + """Idempotently revoke a role; requires OPERATE on system.""" + await self._inner.delete_role_binding( + json.dumps({"subject": _subject(subject, "role_member"), "role": role}) + ) + + async def get_principal(self, subject: Subject) -> PrincipalInfo: + """Look up groups and effective roles; requires USAGE on system. + + Select by ID, name, or API key. Missing/ambiguous identities raise HTTP + 404/409; unavailable or unconfigured resolution raises HTTP 503/501. + """ + value = json.loads( + await self._inner.get_principal( + json.dumps({"principal": _subject(subject, "principal")}) + ) + ) + return PrincipalInfo( + value["principal_id"], + value.get("principal_name"), + [IdentityInfo(**group) for group in value["groups"]], + value["roles"], + ) + + async def get_group(self, subject: Subject) -> GroupInfo: + """Look up a group's members and roles by ID/name; requires USAGE on system. + + Membership is read-only and comes from the identity provider. Lookup + errors have the same HTTP semantics as ``get_principal``. + """ + value = json.loads( + await self._inner.get_group( + json.dumps({"group": _subject(subject, "group")}) + ) + ) + return GroupInfo( + value["group_id"], + value.get("group_name"), + [IdentityInfo(**principal) for principal in value["principals"]], + value["roles"], + ) + + +async def _next(iterator: AsyncIterator[_T]) -> _T: + return await iterator.__anext__() + + +def _sync_iter(iterator: AsyncIterator[_T]) -> Iterator[_T]: + try: + while True: + yield LOOP.run(_next(iterator)) + except StopAsyncIteration: + return + finally: + close = getattr(iterator, "aclose", None) + if close is not None: + LOOP.run(close()) + + +class Authorization: + """Synchronous authorization returned by ``Catalog.authz``. + + Shares the async API's permissions, errors, and lazy pagination behavior. + Group membership comes from the identity provider and cannot be edited here. + """ + + def __init__(self, inner: AsyncAuthorization): + self._inner = inner + + def list_acls( + self, + *, + object: Optional[Union[str, Object]] = None, + subject: Optional[Subject] = None, + limit: Optional[int] = None, + ) -> Iterator[AccessControlEntry]: + """Iterate grants. + + See [list_acls][lancedb.authz.AsyncAuthorization.list_acls] for permissions. + """ + return _sync_iter( + self._inner.list_acls(object=object, subject=subject, limit=limit) + ) + + def add_acl( + self, + *, + object: Union[str, Object], + subject: Subject, + privilege: Union[str, Privilege], + ) -> None: + """Grant a privilege; duplicates raise HTTP 409. + + OWNERSHIP transfers ownership. + """ + LOOP.run( + self._inner.add_acl(object=object, subject=subject, privilege=privilege) + ) + + def delete_acl( + self, + *, + object: Union[str, Object], + subject: Subject, + privilege: Union[str, Privilege], + ) -> None: + """Revoke a privilege. Missing grants raise HTTP 404. + + OWNERSHIP cannot be deleted. + """ + LOOP.run( + self._inner.delete_acl(object=object, subject=subject, privilege=privilege) + ) + + def list_role_bindings( + self, + *, + subject: Optional[Subject] = None, + role: Optional[str] = None, + limit: Optional[int] = None, + ) -> Iterator[RoleBinding]: + """Iterate direct role bindings, using the same limit semantics as list_acls.""" + return _sync_iter( + self._inner.list_role_bindings(subject=subject, role=role, limit=limit) + ) + + def add_role_binding(self, *, subject: Subject, role: str) -> None: + """Grant a role idempotently; requires OPERATE on system.""" + LOOP.run(self._inner.add_role_binding(subject=subject, role=role)) + + def delete_role_binding(self, *, subject: Subject, role: str) -> None: + """Idempotently revoke a role; requires OPERATE on system.""" + LOOP.run(self._inner.delete_role_binding(subject=subject, role=role)) + + def get_principal(self, subject: Subject) -> PrincipalInfo: + """Look up groups and effective roles; requires USAGE on system.""" + return LOOP.run(self._inner.get_principal(subject)) + + def get_group(self, subject: Subject) -> GroupInfo: + """Look up a group's members and roles; requires USAGE on system.""" + return LOOP.run(self._inner.get_group(subject)) diff --git a/python/python/lancedb/catalog.py b/python/python/lancedb/catalog.py index 70aab1bbb..666d414a4 100644 --- a/python/python/lancedb/catalog.py +++ b/python/python/lancedb/catalog.py @@ -3,11 +3,13 @@ """Remote catalogs manage databases through a server's root namespace.""" +from functools import cached_property from datetime import timedelta from typing import Any, AsyncIterator, Iterator, Optional, Union from . import _lancedb from .background_loop import LOOP +from .authz import AsyncAuthorization, Authorization from .db import AsyncConnection, DBConnection from .remote import ClientConfig, OAuthConfig from .remote.db import RemoteDBConnection @@ -86,6 +88,11 @@ class AsyncCatalog: def __init__(self, inner: _lancedb.Catalog): self._inner = inner + @cached_property + def authz(self) -> AsyncAuthorization: + """Authorization sharing this catalog's scope and credentials.""" + return AsyncAuthorization(self._inner.authz) + @property def uri(self) -> str: """The catalog's root namespace endpoint.""" @@ -157,6 +164,11 @@ class Catalog: self._sql_host_override = sql_host_override self._oauth_config = oauth_config + @cached_property + def authz(self) -> Authorization: + """Authorization sharing this catalog's scope and credentials.""" + return Authorization(self._inner.authz) + @property def uri(self) -> str: """The catalog's root namespace endpoint.""" diff --git a/python/python/tests/test_authz.py b/python/python/tests/test_authz.py new file mode 100644 index 000000000..b4e145317 --- /dev/null +++ b/python/python/tests/test_authz.py @@ -0,0 +1,821 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import asyncio +from dataclasses import FrozenInstanceError +import json +from unittest.mock import AsyncMock + +import pytest + +import lancedb +from lancedb.authz import ( + AccessControlEntry, + AsyncAuthorization, + Authorization, + GroupInfo, + IdentityInfo, + Object, + PrincipalInfo, + Privilege, + RoleBinding, + Subject, +) +from lancedb.remote import HeaderProvider +from lancedb.remote.errors import HttpError +from test_catalog import catalog_server as _catalog_server + +catalog_server = _catalog_server + + +class DynamicHeaders(HeaderProvider): + def get_headers(self): + return { + "Authorization": "Bearer rotating-token", + "X-LanceDB-Database": "wrong-dynamic-db", + "X-LanceDB-Database-Prefix": "wrong-prefix", + } + + +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_authz_operations_and_scope(catalog_server, asynchronous): + endpoint, requests, responses = catalog_server + responses.extend( + [ + (200, {}), + (200, {}), + (200, {}), + (200, {}), + ( + 200, + { + "entries": [ + { + "subject_id": "g:team-id", + "object": "table:tenant/db:a$b$events", + "privilege": "PRIVILEGE20", + } + ] + }, + ), + ( + 200, + { + "bindings": [ + { + "subject_id": "p:alice-id", + "subject_name": "Alice", + "role": "reader", + } + ] + }, + ), + ( + 200, + { + "principal_id": "alice-id", + "groups": [{"id": "team-id"}], + "roles": ["reader"], + }, + ), + ( + 200, + { + "group_id": "team-id", + "group_name": "Engineering", + "principals": [{"id": "alice-id", "name": "Alice"}], + "roles": ["reader"], + }, + ), + ] + ) + config = { + "header_provider": DynamicHeaders(), + "extra_headers": {"x-lancedb-database": "wrong-static-db"}, + } + obj = Object.table(database="tenant/db", namespace_path=["a", "b"], name="events") + + async def exercise(): + if asynchronous: + catalog = await lancedb.connect_catalog_async( + endpoint, client_config=config + ) + else: + catalog = lancedb.connect_catalog(endpoint, client_config=config) + authz = catalog.authz + assert authz is catalog.authz + assert requests == [] # Constructing handles does not probe a database. + + async def call(method, *args, **kwargs): + value = method(*args, **kwargs) + return await value if asynchronous else value + + await call( + authz.add_acl, + object=obj, + subject=Subject.group_name("Engineering"), + privilege=Privilege.SELECT, + ) + await call( + authz.delete_acl, + object=obj, + subject=Subject.group_id("team-id"), + privilege="SELECT", + ) + await call( + authz.add_role_binding, + subject=Subject.principal_name("Alice"), + role="reader", + ) + await call( + authz.delete_role_binding, + subject=Subject.principal_api_key("subject-secret"), + role="reader", + ) + iterator = authz.list_acls(object=obj, subject=Subject.role("reader")) + entries = ( + [entry async for entry in iterator] if asynchronous else list(iterator) + ) + assert entries == [ + AccessControlEntry( + str(obj), Subject.group_id("team-id"), None, "PRIVILEGE20" + ) + ] + iterator = authz.list_role_bindings( + subject=Subject.group_id("team-id"), role="reader" + ) + bindings = ( + [binding async for binding in iterator] if asynchronous else list(iterator) + ) + assert bindings == [ + RoleBinding(Subject.principal_id("alice-id"), "Alice", "reader") + ] + assert await call( + authz.get_principal, Subject.principal_id("alice-id") + ) == PrincipalInfo("alice-id", None, [IdentityInfo("team-id")], ["reader"]) + assert await call( + authz.get_group, Subject.group_name("Engineering") + ) == GroupInfo( + "team-id", "Engineering", [IdentityInfo("alice-id", "Alice")], ["reader"] + ) + + asyncio.run(exercise()) + assert [path for path, _, _ in requests] == [ + "/admin/authz/acl/add", + "/admin/authz/acl/delete", + "/admin/authz/role_binding/add", + "/admin/authz/role_binding/delete", + "/admin/authz/acl/list", + "/admin/authz/role_binding/list", + "/admin/authz/principal", + "/admin/authz/group", + ] + assert [body for _, _, body in requests] == [ + {"object": str(obj), "subject": "G:Engineering", "privilege": "SELECT"}, + {"object": str(obj), "subject": "g:team-id", "privilege": "SELECT"}, + {"subject": "P:Alice", "role": "reader"}, + {"subject": "a:subject-secret", "role": "reader"}, + {"object": str(obj), "subject": "r:reader", "max_results": 100}, + {"subject": "g:team-id", "role": "reader", "max_results": 100}, + {"principal": "p:alice-id"}, + {"group": "G:Engineering"}, + ] + for _, headers, _ in requests: + headers = {key.lower(): value for key, value in headers.items()} + assert "x-lancedb-database" not in headers + assert "x-lancedb-database-prefix" not in headers + assert headers["authorization"] == "Bearer rotating-token" + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("limit", [None, 0, 1, 3]) +def test_lazy_expanded_acl_pagination(catalog_server, asynchronous, limit): + endpoint, requests, responses = catalog_server + + def entry(privilege): + return { + "object": "database:db", + "subject_id": "r:reader", + "privilege": privilege, + } + + responses.extend( + [ + ( + 200, + { + "entries": [entry("SELECT"), entry("INSERT")], + "continuation_token": "opaque/first", + }, + ), + (200, {"entries": [], "continuation_token": "opaque/second"}), + (200, {"entries": [entry("UPDATE"), entry("DELETE")]}), + ] + ) + + async def exercise(): + catalog = ( + await lancedb.connect_catalog_async(endpoint, api_key="caller-secret") + if asynchronous + else lancedb.connect_catalog(endpoint, api_key="caller-secret") + ) + iterator = catalog.authz.list_acls(subject=Subject.role("reader"), limit=limit) + assert requests == [] + return [entry async for entry in iterator] if asynchronous else list(iterator) + + values = asyncio.run(exercise()) + expected = ["SELECT", "INSERT", "UPDATE", "DELETE"] + assert [entry.privilege for entry in values] == ( + expected if limit is None else expected[:limit] + ) + assert len(requests) == (0 if limit == 0 else 1 if limit == 1 else 3) + for _, headers, body in requests: + assert headers["x-api-key"] == "caller-secret" + assert body["subject"] == "r:reader" + if len(requests) == 3: + assert requests[1][2]["continuation_token"] == "opaque/first" + assert requests[2][2]["continuation_token"] == "opaque/second" + + +@pytest.mark.parametrize("status", [400, 401, 403, 404, 409, 501, 503]) +def test_http_errors_preserve_status_without_replaying_writes(catalog_server, status): + endpoint, requests, responses = catalog_server + responses.append((status, {"error": "authorization failure"})) + authz = lancedb.connect_catalog(endpoint).authz + with pytest.raises(HttpError) as error: + authz.add_acl( + object="system", subject=Subject.role("operator"), privilege="OPERATE" + ) + assert error.value.status_code == status + assert error.value.request_id + assert len(requests) == 1 + + +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_role_pagination_and_late_failure(catalog_server, asynchronous): + endpoint, requests, responses = catalog_server + responses.extend( + [ + ( + 200, + { + "bindings": [{"subject_id": "g:team", "role": "reader"}], + "continuation_token": "next", + }, + ), + (403, {"error": "denied"}), + ] + ) + + async def exercise(): + catalog = ( + await lancedb.connect_catalog_async(endpoint) + if asynchronous + else lancedb.connect_catalog(endpoint) + ) + iterator = catalog.authz.list_role_bindings(role="reader") + first = await anext(iterator) if asynchronous else next(iterator) + assert first.subject == Subject.group_id("team") + assert len(requests) == 1 + with pytest.raises(HttpError) as error: + if asynchronous: + await anext(iterator) + else: + next(iterator) + assert error.value.status_code == 403 + + asyncio.run(exercise()) + assert requests[1][2] == { + "role": "reader", + "max_results": 100, + "continuation_token": "next", + } + + +def test_early_close_does_not_fetch_next_page(catalog_server): + endpoint, requests, responses = catalog_server + responses.append( + ( + 200, + { + "bindings": [{"subject_id": "g:team", "role": "reader"}], + "continuation_token": "next", + }, + ) + ) + iterator = lancedb.connect_catalog(endpoint).authz.list_role_bindings() + next(iterator) + iterator.close() + assert len(requests) == 1 + + +def test_selectors_objects_and_redaction(): + secret = Subject.principal_api_key("do-not-print-this") + assert "do-not-print-this" not in repr(secret) + assert "do-not-print-this" not in str(secret) + with pytest.raises(FrozenInstanceError): + secret.kind = "role" + assert str(Object.system()) == "system" + assert str(Object.database("tenant/db")) == "database:tenant/db" + assert ( + str(Object.namespace(database="db", namespace_path=["a", "b"])) + == "namespace:db:a$b" + ) + assert str(Object.table(database="db", name="events")) == "table:db:public$events" + assert str(Object.view(database="db", name="recent")) == "view:db:public$recent" + assert Subject.principal_id("p:id")._wire() == "p:p:id" + assert Subject.group_id("team-id").value == "team-id" + for namespace in (["a$b"], [""], "public", [".."]): + with pytest.raises(ValueError): + Object.table(database="db", namespace_path=namespace, name="events") + for database in ("", "a//b", "a/../b", "a:b", "a%2Fb", "a$b"): + with pytest.raises(ValueError): + Object.database(database) + + +@pytest.mark.asyncio +async def test_validation_before_io_and_empty_lists(): + inner = AsyncMock() + inner.list_acls.return_value = json.dumps({"entries": []}) + inner.list_role_bindings.return_value = json.dumps({"bindings": []}) + authz = AsyncAuthorization(inner) + for limit in (-1, 1.5, True): + with pytest.raises(ValueError): + authz.list_acls(limit=limit) + with pytest.raises(ValueError): + Authorization(authz).list_role_bindings(limit=limit) + with pytest.raises(ValueError): + await authz.get_group(Subject.principal_id("alice")) + with pytest.raises(ValueError): + await authz.get_principal(Subject.group_id("team")) + for method in (authz.add_role_binding, authz.delete_role_binding): + with pytest.raises(ValueError): + await method(subject=Subject.role("reader"), role="writer") + with pytest.raises(ValueError): + authz.list_role_bindings(subject=Subject.role("reader")) + assert [entry async for entry in authz.list_acls(limit=0)] == [] + assert inner.mock_calls == [] + assert [entry async for entry in authz.list_acls()] == [] + assert [entry async for entry in authz.list_role_bindings()] == [] + + +def test_api_key_subject_is_not_logged(catalog_server): + import os + import subprocess + import sys + + endpoint, requests, responses = catalog_server + responses.append((200, {"principal_id": "public-id", "groups": [], "roles": []})) + result = subprocess.run( + [ + sys.executable, + "-c", + "import lancedb, sys; from lancedb.authz import Subject; " + "lancedb.connect_catalog(sys.argv[1]).authz.get_principal(" + "Subject.principal_api_key('test-subject-secret-never-log'))", + endpoint, + ], + env={**os.environ, "LANCEDB_LOG": "debug"}, + capture_output=True, + text=True, + check=True, + timeout=30, + ) + assert requests[0][2] == {"principal": "a:test-subject-secret-never-log"} + assert "test-subject-secret-never-log" not in result.stdout + result.stderr + + +@pytest.mark.parametrize( + "builder, kwargs, wire", + [ + ("system", {}, "system"), + ("secret", {"database": "db", "name": "api-key"}, "secret:db:public$api-key"), + ( + "function", + {"database": "db", "name": "caption"}, + "function:db:public$caption", + ), + ( + "secret", + {"database": "tenant/db", "namespace_path": ["a", "b"], "name": "api-key"}, + "secret:tenant/db:a$b$api-key", + ), + ( + "function", + {"database": "tenant/db", "namespace_path": ["a", "b"], "name": "caption"}, + "function:tenant/db:a$b$caption", + ), + ("database", {"name": "tenant/db"}, "database:tenant/db"), + ( + "namespace", + {"database": "tenant/db", "namespace_path": ["a", "b"]}, + "namespace:tenant/db:a$b", + ), + ( + "table", + {"database": "tenant/db", "namespace_path": ["a", "b"], "name": "events"}, + "table:tenant/db:a$b$events", + ), + ( + "view", + {"database": "db", "namespace_path": ["public"], "name": "recent"}, + "view:db:public$recent", + ), + ], +) +def test_rust_backed_object_builders(builder, kwargs, wire): + import pickle + from lancedb import _lancedb + + value = getattr(Object, builder)(**kwargs) + assert str(value) == wire + assert str(value._inner) == wire + assert str(_lancedb.AuthzObject(wire)) == wire + assert value == Object(wire) + assert hash(value) == hash(Object(wire)) + assert pickle.loads(pickle.dumps(value)) == value + + +@pytest.mark.parametrize( + "kind, prefix", + [ + ("principal_id", "p:"), + ("principal_name", "P:"), + ("group_id", "g:"), + ("group_name", "G:"), + ("role", "r:"), + ("principal_api_key", "a:"), + ], +) +def test_rust_backed_subject_selectors(kind, prefix): + import pickle + from lancedb import _lancedb + + value = "p:literal@example.com / 名前" + subject = getattr(Subject, kind)(value) + assert subject.kind == kind + assert subject.value == value + assert subject._wire() == prefix + value + assert _lancedb.AuthzSubject(kind, value).to_wire() == prefix + value + assert pickle.loads(pickle.dumps(subject)) == subject + assert hash(subject) == hash(getattr(Subject, kind)(value)) + with pytest.raises(ValueError): + getattr(Subject, kind)("") + if kind == "principal_api_key": + assert value not in repr(subject._inner) + assert value not in str(subject._inner) + if kind in ("principal_id", "group_id", "role"): + assert Subject._canonical(prefix + value) == subject + else: + with pytest.raises(ValueError) as error: + Subject._canonical(prefix + value) + assert value not in str(error.value) + + +@pytest.mark.parametrize( + "wire", + [ + "", + "database:", + "database:a//b", + "database:a/../b", + "database:a%2Fb", + "namespace:db:a$$b", + "table:db:events", + "table:db:public$", + "table:db:public$..", + "table:db:a/b$events", + "view:db:public$café", + "secret:db:public$", + "function:db:public$..", + "secret:db:a$$key", + "function:db:caption", + "secret:db:public$bad/name", + "function:db:public$bad:name", + "system:db", + ], +) +def test_invalid_raw_objects_fail_before_io(catalog_server, wire): + endpoint, requests, _ = catalog_server + authz = lancedb.connect_catalog(endpoint).authz + with pytest.raises(ValueError): + Object(wire) + with pytest.raises(ValueError): + authz.list_acls(object=wire) + with pytest.raises(ValueError): + authz.add_acl(object=wire, subject=Subject.role("reader"), privilege="SELECT") + assert requests == [] + + +@pytest.mark.parametrize( + "wire", + [ + "future:db:resource", + "table_extension:db:resource", + "secret:tenant/db:a$b$key", + "function:tenant/db:a$b$caption", + ], +) +def test_objects_round_trip_through_acl_requests(catalog_server, wire): + endpoint, requests, responses = catalog_server + responses.extend( + [ + (200, {}), + ( + 200, + { + "entries": [ + { + "object": wire, + "subject_id": "r:reader", + "privilege": "SELECT", + } + ] + }, + ), + (200, {}), + ] + ) + authz = lancedb.connect_catalog(endpoint).authz + obj = Object(wire) + assert str(obj) == wire + authz.add_acl(object=obj, subject=Subject.role("reader"), privilege="SELECT") + entries = list(authz.list_acls(object=wire)) + assert entries[0].object == wire + authz.delete_acl( + object=entries[0].object, subject=Subject.role("reader"), privilege="SELECT" + ) + assert len(requests) == 3 + for _, _, body in requests: + assert body["object"] == wire + + +@pytest.mark.asyncio +async def test_native_requests_validate_selector_context_before_io(catalog_server): + endpoint, requests, _ = catalog_server + catalog = await lancedb.connect_catalog_async(endpoint) + native = catalog.authz._inner + for method, request in [ + (native.list_role_bindings, {"subject": "r:reader"}), + (native.add_role_binding, {"subject": "r:reader", "role": "writer"}), + (native.delete_role_binding, {"subject": "r:reader", "role": "writer"}), + (native.get_principal, {"principal": "g:group"}), + (native.get_group, {"group": "a:never-log-this-key"}), + ]: + with pytest.raises(ValueError) as error: + await method(json.dumps(request)) + assert "never-log-this-key" not in str(error.value) + assert requests == [] + + +def test_privilege_values_use_rust_serialization(): + import pickle + from lancedb import _lancedb + + names = [value.value for value in Privilege] + assert names == _lancedb.AuthzPrivilege.known() + for privilege in Privilege: + assert json.loads(json.dumps(privilege)) == privilege.value + assert str(_lancedb.AuthzPrivilege(privilege)) == privilege.value + assert pickle.loads(pickle.dumps(privilege)) is privilege + for index in range(64): + alias = f"PRIVILEGE{index}" + expected = names[index] if index < len(names) else alias + assert str(_lancedb.AuthzPrivilege(alias)) == expected + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize( + "creation_privilege", [None, "CREATE_SECRET", "CREATE_FUNCTION"] +) +def test_privilege_names_and_future_values_round_trip( + catalog_server, asynchronous, creation_privilege +): + endpoint, requests, responses = catalog_server + inputs = [ + Privilege.SELECT, + "SELECT", + "PRIVILEGE2", + "PRIVILEGE20", + "PRIVILEGE63", + "PRIVILEGE64", + "PRIVILEGE256", + "PRIVILEGE18446744073709551616", + ] + expected = ["SELECT", "SELECT", "SELECT", *inputs[3:]] + object_id = "table:db:public$t" + if creation_privilege is not None: + inputs = [Privilege[creation_privilege], creation_privilege] + expected = [creation_privilege, creation_privilege] + object_id = "namespace:db:public" + for privilege in inputs: + responses.extend( + [ + (200, {}), + ( + 200, + { + "entries": [ + { + "object": object_id, + "subject_id": "r:reader", + "privilege": privilege, + } + ] + }, + ), + (200, {}), + ] + ) + + async def exercise(): + catalog = ( + await lancedb.connect_catalog_async(endpoint) + if asynchronous + else lancedb.connect_catalog(endpoint) + ) + authz = catalog.authz + for privilege, canonical in zip(inputs, expected): + added = authz.add_acl( + object=object_id, + subject=Subject.role("reader"), + privilege=privilege, + ) + if asynchronous: + await added + iterator = authz.list_acls() + entries = ( + [entry async for entry in iterator] if asynchronous else list(iterator) + ) + assert entries[0].privilege == canonical + deleted = authz.delete_acl( + object=entries[0].object, + subject=entries[0].subject, + privilege=entries[0].privilege, + ) + if asynchronous: + await deleted + + asyncio.run(exercise()) + assert len(requests) == 3 * len(inputs) + for index, canonical in enumerate(expected): + assert requests[index * 3][2]["object"] == object_id + assert requests[index * 3][2]["privilege"] == canonical + assert requests[index * 3 + 2][2]["object"] == object_id + assert requests[index * 3 + 2][2]["privilege"] == canonical + + +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_invalid_privileges_fail_before_io(catalog_server, asynchronous): + endpoint, requests, _ = catalog_server + invalid = [ + "", + "select", + "SELECT ", + "UNKNOWN", + "PRIVILEGE", + "PRIVILEGE-1", + "PRIVILEGE+1", + "PRIVILEGE1x", + None, + 2, + True, + ["SELECT"], + ] + + async def exercise(): + catalog = ( + await lancedb.connect_catalog_async(endpoint) + if asynchronous + else lancedb.connect_catalog(endpoint) + ) + async_authz = catalog.authz if asynchronous else catalog.authz._inner + native = async_authz._inner + for privilege in invalid: + for method in [catalog.authz.add_acl, catalog.authz.delete_acl]: + with pytest.raises(ValueError, match="privilege"): + result = method( + object="system", + subject=Subject.role("reader"), + privilege=privilege, + ) + if asynchronous: + await result + for method in [ + native.add_acl, + native.delete_acl, + ]: + with pytest.raises(ValueError): + await method( + json.dumps( + { + "object": "system", + "subject": "r:reader", + "privilege": privilege, + } + ) + ) + + asyncio.run(exercise()) + assert requests == [] + + +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_acl_page_preserves_unknown_subjects(catalog_server, asynchronous): + import pickle + + endpoint, requests, responses = catalog_server + responses.extend( + [ + ( + 200, + { + "entries": [ + { + "subject_id": "p:known", + "object": "system", + "privilege": "USAGE", + }, + { + "subject_id": "x:future-subject", + "object": "system", + "privilege": "USAGE", + }, + ] + }, + ), + (200, {}), + ] + ) + + async def exercise(): + catalog = ( + await lancedb.connect_catalog_async(endpoint) + if asynchronous + else lancedb.connect_catalog(endpoint) + ) + iterator = catalog.authz.list_acls() + entries = ( + [entry async for entry in iterator] if asynchronous else list(iterator) + ) + assert len(entries) == 2 + assert entries[0].subject == Subject.principal_id("known") + subject = entries[1].subject + assert subject.kind == "unknown" + assert subject.value == "x:future-subject" + assert subject._wire() == "x:future-subject" + assert pickle.loads(pickle.dumps(subject)) == subject + result = catalog.authz.delete_acl( + object=entries[1].object, subject=subject, privilege=entries[1].privilege + ) + if asynchronous: + await result + + asyncio.run(exercise()) + assert len(requests) == 2 + assert requests[1][2]["subject"] == "x:future-subject" + + +@pytest.mark.parametrize("builder", ["secret", "function"]) +def test_secret_and_function_builders_validate_components(builder): + from lancedb import _lancedb + + for database, namespace, name in [ + ("a//b", ["public"], "valid"), + ("db", ["a$b"], "valid"), + ("db", "public", "valid"), + ("db", ["public"], "bad$name"), + ("db", ["public"], ".."), + ]: + with pytest.raises(ValueError): + getattr(Object, builder)( + database=database, namespace_path=namespace, name=name + ) + with pytest.raises(ValueError): + getattr(_lancedb.AuthzObject, builder)(database, namespace, name) + + +@pytest.mark.parametrize( + "builder", ["namespace", "table", "view", "secret", "function"] +) +def test_empty_and_omitted_namespace_paths_select_public(builder): + from lancedb import _lancedb + + kwargs = {"database": "db"} + native_args = () + wire = "namespace:db:public" + if builder != "namespace": + kwargs["name"] = "item" + native_args = ("item",) + wire = f"{builder}:db:public$item" + + build = getattr(Object, builder) + expected = build(**kwargs, namespace_path=["public"]) + assert str(expected) == wire + assert build(**kwargs) == expected + for namespace in [[], ()]: + assert build(**kwargs, namespace_path=namespace) == expected + assert ( + str(getattr(_lancedb.AuthzObject, builder)("db", namespace, *native_args)) + == wire + ) diff --git a/python/python/tests/test_catalog.py b/python/python/tests/test_catalog.py index e7b7b1ec2..b2036b7a6 100644 --- a/python/python/tests/test_catalog.py +++ b/python/python/tests/test_catalog.py @@ -263,3 +263,45 @@ async def test_database_names_invalid_limit(catalog_server, asynchronous, page_l next(names) assert names.num_page_results() == 0 assert requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_catalog_rejects_invalid_database_names_before_io( + catalog_server, asynchronous +): + endpoint, requests, _ = catalog_server + catalog = ( + await lancedb.connect_catalog_async(endpoint) + if asynchronous + else lancedb.connect_catalog(endpoint) + ) + for name in [ + "", + ".", + "..", + "/db", + "db/", + "a//b", + "a/./b", + "a/../b", + "a b", + "a:b", + "a$b", + "a%b", + "a?b", + "a#b", + "a\\b", + "café", + "a\nb", + ]: + for method in [ + catalog.create_database, + catalog.connect_database, + catalog.drop_database, + ]: + with pytest.raises(ValueError, match="Invalid database name"): + result = method(name) + if asynchronous: + await result + assert requests == [] diff --git a/python/src/authz.rs b/python/src/authz.rs new file mode 100644 index 000000000..6ad3e69b1 --- /dev/null +++ b/python/src/authz.rs @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +use crate::error::PythonErrorExt; +use crate::runtime::future_into_py; +use lancedb::authz::*; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::types::{PyAnyMethods, PyBytes, PyString}; +use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods}; +use std::sync::Arc; + +#[pyclass] +pub struct AuthorizationClient { + pub(crate) inner: Arc, +} + +#[pymethods] +impl AuthorizationClient { + fn list_acls<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult> { + let request: ListAccessControlEntriesRequest = serde_json::from_str(request) + .map_err(|_| PyValueError::new_err("Invalid authorization request"))?; + let inner = self_.inner.clone(); + future_into_py(self_.py(), async move { + let result = inner.list_acls(request).await.infer_error()?; + serde_json::to_string(&result).map_err(|err| PyRuntimeError::new_err(err.to_string())) + }) + } + fn add_acl<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult> { + let request: AccessControlEntryRequest = serde_json::from_str(request) + .map_err(|_| PyValueError::new_err("Invalid authorization request"))?; + let inner = self_.inner.clone(); + future_into_py(self_.py(), async move { + inner.add_acl(request).await.infer_error() + }) + } + fn delete_acl<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult> { + let request: AccessControlEntryRequest = serde_json::from_str(request) + .map_err(|_| PyValueError::new_err("Invalid authorization request"))?; + let inner = self_.inner.clone(); + future_into_py(self_.py(), async move { + inner.delete_acl(request).await.infer_error() + }) + } + fn list_role_bindings<'py>( + self_: PyRef<'py, Self>, + request: &str, + ) -> PyResult> { + let request: ListRoleBindingsRequest = serde_json::from_str(request) + .map_err(|_| PyValueError::new_err("Invalid authorization request"))?; + request.validate().infer_error()?; + let inner = self_.inner.clone(); + future_into_py(self_.py(), async move { + let result = inner.list_role_bindings(request).await.infer_error()?; + serde_json::to_string(&result).map_err(|err| PyRuntimeError::new_err(err.to_string())) + }) + } + fn add_role_binding<'py>( + self_: PyRef<'py, Self>, + request: &str, + ) -> PyResult> { + let request: RoleBindingRequest = serde_json::from_str(request) + .map_err(|_| PyValueError::new_err("Invalid authorization request"))?; + request.validate().infer_error()?; + let inner = self_.inner.clone(); + future_into_py(self_.py(), async move { + inner.add_role_binding(request).await.infer_error() + }) + } + fn delete_role_binding<'py>( + self_: PyRef<'py, Self>, + request: &str, + ) -> PyResult> { + let request: RoleBindingRequest = serde_json::from_str(request) + .map_err(|_| PyValueError::new_err("Invalid authorization request"))?; + request.validate().infer_error()?; + let inner = self_.inner.clone(); + future_into_py(self_.py(), async move { + inner.delete_role_binding(request).await.infer_error() + }) + } + fn get_principal<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult> { + let request: GetPrincipalRequest = serde_json::from_str(request) + .map_err(|_| PyValueError::new_err("Invalid authorization request"))?; + request.validate().infer_error()?; + let inner = self_.inner.clone(); + future_into_py(self_.py(), async move { + let result = inner.get_principal(request).await.infer_error()?; + serde_json::to_string(&result).map_err(|err| PyRuntimeError::new_err(err.to_string())) + }) + } + fn get_group<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult> { + let request: GetGroupRequest = serde_json::from_str(request) + .map_err(|_| PyValueError::new_err("Invalid authorization request"))?; + request.validate().infer_error()?; + let inner = self_.inner.clone(); + future_into_py(self_.py(), async move { + let result = inner.get_group(request).await.infer_error()?; + serde_json::to_string(&result).map_err(|err| PyRuntimeError::new_err(err.to_string())) + }) + } +} + +/// The shared privilege parser and canonical wire names. +#[pyclass(frozen)] +pub struct AuthzPrivilege { + inner: Privilege, +} + +#[pymethods] +impl AuthzPrivilege { + #[new] + fn new(value: &str) -> PyResult { + Ok(Self { + inner: value.parse().map_err(PyValueError::new_err)?, + }) + } + + #[staticmethod] + fn known() -> Vec { + Privilege::KNOWN.iter().map(ToString::to_string).collect() + } + + fn __str__(&self) -> String { + self.inner.to_string() + } +} + +#[pyclass(frozen)] +pub struct AuthzSubject { + inner: Subject, +} + +#[pymethods] +impl AuthzSubject { + #[new] + fn new(kind: &str, value: &Bound<'_, PyAny>) -> PyResult { + let value: String = value + .extract() + .map_err(|_| PyValueError::new_err("Subject value must be a nonempty string"))?; + Ok(Self { + inner: Subject::new(kind.parse().infer_error()?, value).infer_error()?, + }) + } + + #[staticmethod] + fn from_canonical(value: &str) -> PyResult { + Ok(Self { + inner: Subject::from_canonical(value).infer_error()?, + }) + } + + #[getter] + fn kind(&self) -> &str { + self.inner.kind().as_str() + } + #[getter] + fn value(&self) -> &str { + self.inner.value() + } + #[getter] + fn display_value(&self) -> &str { + self.inner.display_value() + } + + #[pyo3(signature = (context=None))] + fn to_wire(&self, context: Option<&str>) -> PyResult { + match context { + None => {} + Some("principal") => self.inner.validate_principal().infer_error()?, + Some("group") => self.inner.validate_group().infer_error()?, + Some("role_member") => self.inner.validate_role_member().infer_error()?, + Some(_) => return Err(PyValueError::new_err("Invalid subject context")), + } + Ok(self.inner.to_wire()) + } + + fn __repr__(&self) -> String { + format!("{:?}", self.inner) + } + fn __str__(&self) -> String { + self.inner.to_string() + } +} + +#[pyclass(frozen)] +pub struct AuthzObject { + inner: Object, +} + +impl AuthzObject { + fn validated(inner: Object) -> PyResult { + inner.validate().map_err(PyValueError::new_err)?; + Ok(Self { inner }) + } +} + +fn namespace_path(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_instance_of::() || value.is_instance_of::() { + return Err(PyValueError::new_err( + "namespace must be a sequence of components", + )); + } + let components = value + .extract() + .map_err(|_| PyValueError::new_err("namespace must be a sequence of string components"))?; + NamespacePath::new(components).map_err(PyValueError::new_err) +} + +#[pymethods] +impl AuthzObject { + #[new] + fn new(value: &str) -> PyResult { + Ok(Self { + inner: value.parse().infer_error()?, + }) + } + + #[staticmethod] + fn system() -> Self { + Self { + inner: Object::System, + } + } + + #[staticmethod] + fn database(name: &str) -> PyResult { + Self::validated(Object::Database(DatabaseObject::new(name.into()))) + } + + #[staticmethod] + fn namespace(database: &str, namespace: &Bound<'_, PyAny>) -> PyResult { + Self::validated(Object::Namespace(NamespaceObject { + database: database.into(), + namespace: namespace_path(namespace)?, + })) + } + + #[staticmethod] + fn table(database: &str, namespace: &Bound<'_, PyAny>, name: &str) -> PyResult { + Self::validated(Object::Table(TableObject { + database: database.into(), + namespace: namespace_path(namespace)?, + table: name.into(), + })) + } + + #[staticmethod] + fn view(database: &str, namespace: &Bound<'_, PyAny>, name: &str) -> PyResult { + Self::validated(Object::View(ViewObject { + database: database.into(), + namespace: namespace_path(namespace)?, + view: name.into(), + })) + } + + #[staticmethod] + fn secret(database: &str, namespace: &Bound<'_, PyAny>, name: &str) -> PyResult { + Self::validated(Object::Secret(SecretObject { + database: database.into(), + namespace: namespace_path(namespace)?, + secret: name.into(), + })) + } + + #[staticmethod] + fn function(database: &str, namespace: &Bound<'_, PyAny>, name: &str) -> PyResult { + Self::validated(Object::Function(FunctionObject { + database: database.into(), + namespace: namespace_path(namespace)?, + function: name.into(), + })) + } + + fn __str__(&self) -> String { + self.inner.to_string() + } + fn __repr__(&self) -> String { + format!("{:?}", self.inner) + } +} diff --git a/python/src/catalog.rs b/python/src/catalog.rs index e7cc56c49..9e375b314 100644 --- a/python/src/catalog.rs +++ b/python/src/catalog.rs @@ -21,6 +21,13 @@ pub struct Catalog { #[pymethods] impl Catalog { + #[getter] + fn authz(&self) -> PyResult { + Ok(crate::authz::AuthorizationClient { + inner: self.inner.authz().infer_error()?, + }) + } + #[getter] fn uri(&self) -> &str { self.inner.uri() diff --git a/python/src/lib.rs b/python/src/lib.rs index 9f88030ca..066e85927 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -22,6 +22,7 @@ use table::{ }; pub mod arrow; +pub mod authz; pub mod catalog; pub mod connection; pub mod error; @@ -70,6 +71,10 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(catalog::connect_catalog, m)?)?; m.add_class::()?; m.add_class::()?; diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 3cc524bfc..ec78fd103 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -66,6 +66,7 @@ serde = { workspace = true } serde_json = { workspace = true } async-openai = { version = "0.20.0", optional = true } serde_with = { version = "3.8.1" } +strum_macros = "0.28" tempfile = { workspace = true } aws-smithy-types = { workspace = true, optional = true } # Default features include the legacy `rustls` feature, which pulls in rustls 0.21. diff --git a/rust/lancedb/src/authz/api.rs b/rust/lancedb/src/authz/api.rs new file mode 100644 index 000000000..38e7d381b --- /dev/null +++ b/rust/lancedb/src/authz/api.rs @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Authorization requests, responses, and the deployment-wide API. + +use super::{Object, Privilege, Subject}; +use crate::Result; +use serde::{Deserialize, Serialize}; + +/// Filters and pagination for a page of ACL grants. Omitted filters match all. +#[derive(Clone, Default, Serialize, Deserialize)] +#[non_exhaustive] +pub struct ListAccessControlEntriesRequest { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub object: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subject: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub continuation_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + /// Maximum stored subject/object pairs, not expanded privilege entries. + /// The server caps this at 100; None uses its default. Must be positive. + pub max_results: Option, +} + +/// One privilege granted to a canonical subject on an object. +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct AccessControlEntry { + #[serde(deserialize_with = "Subject::deserialize_canonical")] + pub subject_id: Subject, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subject_name: Option, + pub object: Object, + pub privilege: Privilege, +} + +/// One page of grants. A stored subject/object pair may expand to several entries. +#[derive(Clone, Default, Serialize, Deserialize, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct ListAccessControlEntriesResponse { + pub entries: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub continuation_token: Option, +} + +/// One grant to add or delete. The subject may be selected by ID, name, or API key. +#[derive(Clone, Serialize, Deserialize)] +#[non_exhaustive] +pub struct AccessControlEntryRequest { + pub subject: Subject, + pub object: Object, + /// A privilege such as [`Privilege::Select`] or [`Privilege::Usage`]. + /// [`Privilege::Unknown`] preserves future privilege names. + pub privilege: Privilege, +} + +/// Filters and pagination for direct principal/group role bindings. +#[derive(Clone, Default, Serialize, Deserialize)] +#[non_exhaustive] +pub struct ListRoleBindingsRequest { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subject: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub continuation_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + /// Maximum bindings on this page; the server caps this at 100. + pub max_results: Option, +} + +/// A direct principal or group membership in a LanceDB-managed role. +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct RoleBinding { + #[serde(deserialize_with = "Subject::deserialize_canonical")] + pub subject_id: Subject, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subject_name: Option, + pub role: String, +} + +/// One page of direct bindings, without expanding group membership. +#[derive(Clone, Default, Serialize, Deserialize, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct ListRoleBindingsResponse { + pub bindings: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub continuation_token: Option, +} + +/// A role to grant to or revoke from a principal or group. Roles cannot nest. +#[derive(Clone, Serialize, Deserialize)] +#[non_exhaustive] +pub struct RoleBindingRequest { + pub subject: Subject, + pub role: String, +} + +/// Select a principal by canonical ID, provider name, or API key. +#[derive(Clone, Serialize, Deserialize)] +#[non_exhaustive] +pub struct GetPrincipalRequest { + pub principal: Subject, +} + +/// Select a group by canonical ID or provider name. +#[derive(Clone, Serialize, Deserialize)] +#[non_exhaustive] +pub struct GetGroupRequest { + pub group: Subject, +} + +/// A canonical identity-provider ID and optional display name. +#[derive(Clone, Default, Serialize, Deserialize, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct IdentityInfo { + pub id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, +} + +/// A principal, its provider-managed groups, and roles held directly or via groups. +#[derive(Clone, Default, Serialize, Deserialize, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct PrincipalInfo { + pub principal_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub principal_name: Option, + pub groups: Vec, + pub roles: Vec, +} + +/// A group, its provider-managed members, and its directly granted roles. +#[derive(Clone, Default, Serialize, Deserialize, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct GroupInfo { + pub group_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub group_name: Option, + pub principals: Vec, + pub roles: Vec, +} + +/// Authorization operations on one deployment, independent of database selection. +/// +/// ACL writes preserve server conflict/not-found errors and must not be replayed +/// after ambiguous transport failures. Role-binding writes are idempotent. +#[async_trait::async_trait] +pub trait Authorization: Send + Sync + std::fmt::Debug + 'static { + /// List a page of ACLs. Requires USAGE on the object, or system if unfiltered. + async fn list_acls( + &self, + request: ListAccessControlEntriesRequest, + ) -> Result; + /// Grant a privilege; requires OWNERSHIP. Duplicate grants return HTTP 409. + /// Granting OWNERSHIP transfers it from the previous owner. + async fn add_acl(&self, request: AccessControlEntryRequest) -> Result<()>; + /// Revoke a privilege; requires OWNERSHIP. Missing grants return HTTP 404. + /// OWNERSHIP cannot be deleted. + async fn delete_acl(&self, request: AccessControlEntryRequest) -> Result<()>; + /// List a page of direct bindings; requires USAGE on system. + async fn list_role_bindings( + &self, + request: ListRoleBindingsRequest, + ) -> Result; + /// Idempotently grant a role; requires OPERATE on system. + async fn add_role_binding(&self, request: RoleBindingRequest) -> Result<()>; + /// Idempotently revoke a role; requires OPERATE on system. + async fn delete_role_binding(&self, request: RoleBindingRequest) -> Result<()>; + /// Inspect a principal and effective roles; requires USAGE on system. + async fn get_principal(&self, request: GetPrincipalRequest) -> Result; + /// Inspect a group and direct roles; requires USAGE on system. + async fn get_group(&self, request: GetGroupRequest) -> Result; +} + +impl AccessControlEntryRequest { + /// Build one grant from validated resource and identity selectors. + pub fn new(object: Object, subject: Subject, privilege: Privilege) -> Self { + Self { + object, + subject, + privilege, + } + } +} + +impl ListRoleBindingsRequest { + /// Reject role subjects, since roles cannot contain other roles. + pub fn validate(&self) -> Result<()> { + if let Some(subject) = &self.subject { + subject.validate_role_member()?; + } + Ok(()) + } +} + +impl RoleBindingRequest { + /// Build a role grant/revocation for a principal or group. + pub fn new(subject: Subject, role: impl Into) -> Result { + subject.validate_role_member()?; + Ok(Self { + subject, + role: role.into(), + }) + } + + /// Validate the member selector, including deserialized requests. + pub fn validate(&self) -> Result<()> { + self.subject.validate_role_member() + } +} + +impl GetPrincipalRequest { + /// Select a principal by ID, name, or API key. + pub fn new(principal: Subject) -> Result { + principal.validate_principal()?; + Ok(Self { principal }) + } + + /// Validate the lookup selector, including deserialized requests. + pub fn validate(&self) -> Result<()> { + self.principal.validate_principal() + } +} + +impl GetGroupRequest { + /// Select a group by ID or name. + pub fn new(group: Subject) -> Result { + group.validate_group()?; + Ok(Self { group }) + } + + /// Validate the lookup selector, including deserialized requests. + pub fn validate(&self) -> Result<()> { + self.group.validate_group() + } +} diff --git a/rust/lancedb/src/authz/mod.rs b/rust/lancedb/src/authz/mod.rs new file mode 100644 index 000000000..3191627e0 --- /dev/null +++ b/rust/lancedb/src/authz/mod.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Deployment-wide authorization, accessed through a catalog. +//! +//! Subjects use `p:ID`, `P:NAME`, `g:ID`, `G:NAME`, `r:ROLE`, or `a:API_KEY`. +//! Objects use literal names such as `table:tenant/db:public$events`. +//! Names are resolved by the server. Request types intentionally do not implement +//! Debug because their subject selectors may contain secret API keys. +//! +//! ```no_run +//! # #[cfg(feature = "remote")] +//! # async fn example() -> lancedb::Result<()> { +//! let catalog = lancedb::connect_catalog("https://example.com") +//! .api_key("secret") +//! .execute().await?; +//! let authz = catalog.authz()?; +//! use lancedb::authz::{AccessControlEntryRequest, Object, TableObject, NamespacePath, Subject, Privilege}; +//! let grant = AccessControlEntryRequest::new( +//! Object::Table(TableObject { +//! database: "analytics".into(), +//! namespace: NamespacePath::from_component("public"), +//! table: "foo".into(), +//! }), +//! Subject::role("reader")?, +//! Privilege::Select, +//! ); +//! authz.add_acl(grant).await?; +//! let page = authz.list_acls(Default::default()).await?; +//! for entry in page.entries { +//! println!("{} {} {}", entry.object, entry.subject_id, entry.privilege); +//! } +//! # Ok(()) +//! # } +//! ``` + +pub mod api; +pub mod object; +pub mod privilege; +pub mod subject; + +pub use api::*; +pub use object::{ + DatabaseObject, FunctionObject, NamespaceObject, NamespacePath, Object, SecretObject, + TableObject, ViewObject, +}; +pub use privilege::*; +pub use subject::{Subject, SubjectKind}; diff --git a/rust/lancedb/src/authz/object.rs b/rust/lancedb/src/authz/object.rs new file mode 100644 index 000000000..fd324b748 --- /dev/null +++ b/rust/lancedb/src/authz/object.rs @@ -0,0 +1,992 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +use crate::utils::{ + validate_database_name, validate_namespace_component, validate_secret_name, + validate_table_name, validate_view_name, +}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _, ser::Error as _}; + +/// The default namespace for authorization resources. +pub const DEFAULT_NAMESPACE: &str = "public"; + +pub const SYSTEM_TYPE_STRING: &str = "system"; +pub const DATABASE_TYPE_STRING: &str = "database"; +pub const NAMESPACE_TYPE_STRING: &str = "namespace"; +pub const TABLE_TYPE_STRING: &str = "table"; +pub const VIEW_TYPE_STRING: &str = "view"; +pub const SECRET_TYPE_STRING: &str = "secret"; +pub const FUNCTION_TYPE_STRING: &str = "function"; +pub const UNKNOWN_TYPE_STRING: &str = "unknown"; + +/// A non-empty path through the namespace hierarchy. +/// +/// The `$` separator is only the canonical wire encoding. Authorization code +/// should use the components and prefix helpers instead of interpreting the +/// encoded form. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct NamespacePath(Vec); + +impl NamespacePath { + /// Construct a path, validating each component without splitting separators. + /// An empty path selects the default `public` namespace. + /// + /// ``` + /// use lancedb::authz::NamespacePath; + /// + /// let path = NamespacePath::new(vec![]).unwrap(); + /// assert_eq!(path.components(), &["public"]); + /// assert_eq!(path, NamespacePath::default()); + /// ``` + pub fn new(components: Vec) -> Result { + if components.is_empty() { + return Ok(Self::default()); + } + for component in &components { + validate_namespace_component(component).map_err(|error| error.to_string())?; + } + Ok(Self(components)) + } + + /// Construct a single-component path. + /// + /// # Panics + /// Panics if the component is invalid. Use [`Self::new`] for untrusted input. + pub fn from_component(component: impl Into) -> Self { + Self::new(vec![component.into()]).expect("namespace components must be valid") + } + + pub fn components(&self) -> &[String] { + &self.0 + } + + pub fn parent(&self) -> Option { + (self.0.len() > 1).then(|| Self(self.0[..self.0.len() - 1].to_vec())) + } + + pub fn prefixes(&self) -> impl Iterator + '_ { + (1..=self.0.len()).map(|length| Self(self.0[..length].to_vec())) + } + + pub fn is_descendant_of(&self, ancestor: &Self) -> bool { + self.0.len() > ancestor.0.len() && self.0.starts_with(&ancestor.0) + } + + pub fn parse(path: &str) -> Result { + Self::new(path.split('$').map(str::to_string).collect()) + } +} + +impl Default for NamespacePath { + /// Select the default `public` namespace. + fn default() -> Self { + Self(vec![DEFAULT_NAMESPACE.to_string()]) + } +} + +impl std::fmt::Display for NamespacePath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0.join("$")) + } +} + +/// An authorization resource, serialized using its canonical wire name. +/// +/// Unknown resource types are preserved so clients can read rules created by +/// newer servers. Known resource types validate names during parsing and serialization. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum Object { + System, + Database(DatabaseObject), + Namespace(NamespaceObject), + Table(TableObject), + View(ViewObject), + Secret(SecretObject), + Function(FunctionObject), + Unknown(String), +} + +impl Object { + /// Validate resource names, including objects recovered from storage keys. + pub fn validate(&self) -> Result<(), String> { + match self { + Self::System => Ok(()), + Self::Database(object) => object.validate(), + Self::Namespace(object) => object.validate(), + Self::Table(object) => object.validate(), + Self::View(object) => object.validate(), + Self::Secret(object) => object.validate(), + Self::Function(object) => object.validate(), + Self::Unknown(value) if value.is_empty() => Err("object must not be empty".into()), + Self::Unknown(_) => Ok(()), + } + } + + /// The database an object belongs to, or None. + pub fn database(&self) -> Option<&str> { + match self { + Self::System => None, + Self::Database(object) => Some(&object.database), + Self::Namespace(object) => Some(&object.database), + Self::Table(object) => Some(&object.database), + Self::View(object) => Some(&object.database), + Self::Secret(object) => Some(&object.database), + Self::Function(object) => Some(&object.database), + Self::Unknown(_) => None, + } + } + + pub fn type_name(&self) -> &'static str { + match self { + Self::System => SYSTEM_TYPE_STRING, + Self::Database(_) => DATABASE_TYPE_STRING, + Self::Namespace(_) => NAMESPACE_TYPE_STRING, + Self::Table(_) => TABLE_TYPE_STRING, + Self::View(_) => VIEW_TYPE_STRING, + Self::Secret(_) => SECRET_TYPE_STRING, + Self::Function(_) => FUNCTION_TYPE_STRING, + Self::Unknown(_) => UNKNOWN_TYPE_STRING, + } + } +} + +impl std::str::FromStr for Object { + type Err = crate::Error; + + fn from_str(value: &str) -> crate::Result { + Self::deserialize(serde::de::value::StrDeserializer::::new(value)) + .map_err(|error| crate::Error::InvalidInput { + message: error.to_string(), + }) + } +} + +impl std::fmt::Display for Object { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::System => f.write_str(SYSTEM_TYPE_STRING), + Self::Database(object) => object.fmt(f), + Self::Namespace(object) => object.fmt(f), + Self::Table(object) => object.fmt(f), + Self::View(object) => object.fmt(f), + Self::Secret(object) => object.fmt(f), + Self::Function(object) => object.fmt(f), + Self::Unknown(s) => f.write_str(s), + } + } +} + +impl Serialize for Object { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.validate().map_err(S::Error::custom)?; + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Object { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let kind = s.split(':').next().unwrap_or_default(); + if s == SYSTEM_TYPE_STRING { + Ok(Self::System) + } else if s.is_empty() || kind == SYSTEM_TYPE_STRING { + Err(D::Error::custom(format!("invalid Object {s:?}"))) + } else if kind == DATABASE_TYPE_STRING { + let database = DatabaseObject::deserialize(serde::de::value::StringDeserializer::< + D::Error, + >::new(s))?; + Ok(Self::Database(database)) + } else if kind == NAMESPACE_TYPE_STRING { + let namespace = NamespaceObject::deserialize(serde::de::value::StringDeserializer::< + D::Error, + >::new(s))?; + Ok(Self::Namespace(namespace)) + } else if kind == TABLE_TYPE_STRING { + let table = + TableObject::deserialize(serde::de::value::StringDeserializer::::new(s))?; + Ok(Self::Table(table)) + } else if kind == VIEW_TYPE_STRING { + let view = + ViewObject::deserialize(serde::de::value::StringDeserializer::::new(s))?; + Ok(Self::View(view)) + } else if kind == SECRET_TYPE_STRING { + let secret = SecretObject::deserialize( + serde::de::value::StringDeserializer::::new(s), + )?; + Ok(Self::Secret(secret)) + } else if kind == FUNCTION_TYPE_STRING { + let function = FunctionObject::deserialize(serde::de::value::StringDeserializer::< + D::Error, + >::new(s))?; + Ok(Self::Function(function)) + } else { + Ok(Self::Unknown(s)) + } + } +} + +/// A database resource, encoded as `database:NAME`. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct DatabaseObject { + pub database: String, +} + +impl DatabaseObject { + pub fn new(database: String) -> Self { + Self { database } + } + + fn validate(&self) -> Result<(), String> { + validate_database_name(&self.database).map_err(|error| error.to_string())?; + Ok(()) + } +} + +impl std::fmt::Display for DatabaseObject { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "database:{}", self.database) + } +} + +impl Serialize for DatabaseObject { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.validate().map_err(S::Error::custom)?; + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for DatabaseObject { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let database = s + .strip_prefix("database:") + .ok_or_else(|| D::Error::custom(format!("invalid DatabaseObject {s:?}")))?; + let object = Self { + database: database.to_string(), + }; + object.validate().map_err(D::Error::custom)?; + Ok(object) + } +} + +/// A namespace resource, encoded as `namespace:DATABASE:PATH`. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct NamespaceObject { + pub database: String, + pub namespace: NamespacePath, +} + +impl NamespaceObject { + fn validate(&self) -> Result<(), String> { + validate_database_name(&self.database).map_err(|error| error.to_string())?; + Ok(()) + } +} + +impl std::fmt::Display for NamespaceObject { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "namespace:{}:{}", self.database, self.namespace) + } +} + +impl Serialize for NamespaceObject { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.validate().map_err(S::Error::custom)?; + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for NamespaceObject { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let invalid = || D::Error::custom(format!("invalid NamespaceObject {s:?}")); + let rest = s.strip_prefix("namespace:").ok_or_else(invalid)?; + let (database, namespace) = rest.split_once(':').ok_or_else(invalid)?; + let object = Self { + database: database.to_string(), + namespace: NamespacePath::parse(namespace).map_err(D::Error::custom)?, + }; + object.validate().map_err(D::Error::custom)?; + Ok(object) + } +} + +/// A table resource, encoded as `table:DATABASE:NAMESPACE$TABLE`. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct TableObject { + pub database: String, + pub namespace: NamespacePath, + pub table: String, +} + +impl TableObject { + fn validate(&self) -> Result<(), String> { + validate_database_name(&self.database).map_err(|error| error.to_string())?; + validate_table_name(&self.table).map_err(|error| error.to_string())?; + Ok(()) + } +} + +impl std::fmt::Display for TableObject { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "table:{}:{}${}", + self.database, self.namespace, self.table + ) + } +} + +impl Serialize for TableObject { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.validate().map_err(S::Error::custom)?; + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for TableObject { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let invalid = || D::Error::custom(format!("invalid TableObject {s:?}")); + let rest = s.strip_prefix("table:").ok_or_else(invalid)?; + let (database, path) = rest.split_once(':').ok_or_else(invalid)?; + let (namespace, table) = path.rsplit_once('$').ok_or_else(invalid)?; + let object = Self { + database: database.to_string(), + namespace: NamespacePath::parse(namespace).map_err(D::Error::custom)?, + table: table.to_string(), + }; + object.validate().map_err(D::Error::custom)?; + Ok(object) + } +} + +/// A view resource, encoded as `view:DATABASE:NAMESPACE$VIEW`. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct ViewObject { + pub database: String, + pub namespace: NamespacePath, + pub view: String, +} + +impl ViewObject { + fn validate(&self) -> Result<(), String> { + validate_database_name(&self.database).map_err(|error| error.to_string())?; + validate_view_name(&self.view).map_err(|error| error.to_string())?; + Ok(()) + } +} + +impl std::fmt::Display for ViewObject { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "view:{}:{}${}", self.database, self.namespace, self.view) + } +} + +impl Serialize for ViewObject { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.validate().map_err(S::Error::custom)?; + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for ViewObject { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let invalid = || D::Error::custom(format!("invalid ViewObject {s:?}")); + let rest = s.strip_prefix("view:").ok_or_else(invalid)?; + let (database, path) = rest.split_once(':').ok_or_else(invalid)?; + let (namespace, view) = path.rsplit_once('$').ok_or_else(invalid)?; + let object = Self { + database: database.to_string(), + namespace: NamespacePath::parse(namespace).map_err(D::Error::custom)?, + view: view.to_string(), + }; + object.validate().map_err(D::Error::custom)?; + Ok(object) + } +} +/// A secret resource, encoded as `secret:DATABASE:NAMESPACE$SECRET`. +/// +/// ``` +/// use lancedb::authz::{SecretObject, NamespacePath, Object}; +/// +/// let object = Object::Secret(SecretObject { +/// database: "analytics".into(), +/// namespace: NamespacePath::from_component("public"), +/// secret: "example".into(), +/// }); +/// assert_eq!(object.to_string(), "secret:analytics:public$example"); +/// ``` +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct SecretObject { + pub database: String, + pub namespace: NamespacePath, + pub secret: String, +} + +impl SecretObject { + fn validate(&self) -> Result<(), String> { + validate_database_name(&self.database).map_err(|error| error.to_string())?; + validate_secret_name(&self.secret).map_err(|error| error.to_string())?; + Ok(()) + } +} + +impl std::fmt::Display for SecretObject { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "secret:{}:{}${}", + self.database, self.namespace, self.secret + ) + } +} + +impl Serialize for SecretObject { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.validate().map_err(S::Error::custom)?; + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for SecretObject { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let invalid = || D::Error::custom(format!("invalid SecretObject {s:?}")); + let rest = s.strip_prefix("secret:").ok_or_else(invalid)?; + let (database, path) = rest.split_once(':').ok_or_else(invalid)?; + let (namespace, secret) = path.rsplit_once('$').ok_or_else(invalid)?; + let object = Self { + database: database.to_string(), + namespace: NamespacePath::parse(namespace).map_err(D::Error::custom)?, + secret: secret.to_string(), + }; + object.validate().map_err(D::Error::custom)?; + Ok(object) + } +} + +/// A function resource, encoded as `function:DATABASE:NAMESPACE$FUNCTION`. +/// +/// ``` +/// use lancedb::authz::{FunctionObject, NamespacePath, Object}; +/// +/// let object = Object::Function(FunctionObject { +/// database: "analytics".into(), +/// namespace: NamespacePath::from_component("public"), +/// function: "example".into(), +/// }); +/// assert_eq!(object.to_string(), "function:analytics:public$example"); +/// ``` +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct FunctionObject { + pub database: String, + pub namespace: NamespacePath, + pub function: String, +} + +impl FunctionObject { + fn validate(&self) -> Result<(), String> { + validate_database_name(&self.database).map_err(|error| error.to_string())?; + validate_table_name(&self.function).map_err(|error| error.to_string())?; + Ok(()) + } +} + +impl std::fmt::Display for FunctionObject { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "function:{}:{}${}", + self.database, self.namespace, self.function + ) + } +} + +impl Serialize for FunctionObject { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.validate().map_err(S::Error::custom)?; + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for FunctionObject { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let invalid = || D::Error::custom(format!("invalid FunctionObject {s:?}")); + let rest = s.strip_prefix("function:").ok_or_else(invalid)?; + let (database, path) = rest.split_once(':').ok_or_else(invalid)?; + let (namespace, function) = path.rsplit_once('$').ok_or_else(invalid)?; + let object = Self { + database: database.to_string(), + namespace: NamespacePath::parse(namespace).map_err(D::Error::custom)?, + function: function.to_string(), + }; + object.validate().map_err(D::Error::custom)?; + Ok(object) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn system_object_round_trip() { + let encoded = serde_json::to_string(&Object::System).unwrap(); + assert_eq!(encoded, r#""system""#); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + Object::System + ); + } + + #[test] + fn object_database_round_trip() { + let object = Object::Database(DatabaseObject { + database: "db".to_string(), + }); + let encoded = serde_json::to_string(&object).unwrap(); + assert_eq!(encoded, r#""database:db""#); + assert_eq!(serde_json::from_str::(&encoded).unwrap(), object); + } + + #[test] + fn object_namespace_round_trip() { + let object = Object::Namespace(NamespaceObject { + database: "db".to_string(), + namespace: NamespacePath::from_component("ns"), + }); + let encoded = serde_json::to_string(&object).unwrap(); + assert_eq!(encoded, r#""namespace:db:ns""#); + assert_eq!(serde_json::from_str::(&encoded).unwrap(), object); + } + + #[test] + fn object_table_round_trip() { + let object = Object::Table(TableObject { + database: "db".to_string(), + namespace: NamespacePath::from_component("ns"), + table: "foo".to_string(), + }); + let encoded = serde_json::to_string(&object).unwrap(); + assert_eq!(encoded, r#""table:db:ns$foo""#); + assert_eq!(serde_json::from_str::(&encoded).unwrap(), object); + } + + #[test] + fn object_view_round_trip() { + let object = Object::View(ViewObject { + database: "db".to_string(), + namespace: NamespacePath::from_component("ns"), + view: "foo".to_string(), + }); + let encoded = serde_json::to_string(&object).unwrap(); + assert_eq!(encoded, r#""view:db:ns$foo""#); + assert_eq!(serde_json::from_str::(&encoded).unwrap(), object); + } + + #[test] + fn secret_objects_round_trip_and_validate() { + let resource = SecretObject { + database: "tenant/db".into(), + namespace: NamespacePath::new(vec!["a".into(), "b".into()]).unwrap(), + secret: "example".into(), + }; + let wire = "secret:tenant/db:a$b$example"; + assert_eq!(serde_json::to_value(&resource).unwrap(), wire); + assert_eq!( + serde_json::from_value::(serde_json::json!(wire)).unwrap(), + resource + ); + let object = Object::Secret(resource.clone()); + assert_eq!(object.database(), Some("tenant/db")); + assert_eq!(object.type_name(), "secret"); + assert_eq!(wire.parse::().unwrap(), object); + assert_eq!(serde_json::to_value(object).unwrap(), wire); + for name in ["", ".", "..", "bad$name", "bad/name", "bad:name"] { + let invalid = SecretObject { + secret: name.into(), + ..resource.clone() + }; + assert!(serde_json::to_value(&invalid).is_err()); + assert!(serde_json::to_value(Object::Secret(invalid)).is_err()); + } + } + + #[test] + fn function_objects_round_trip_and_validate() { + let resource = FunctionObject { + database: "tenant/db".into(), + namespace: NamespacePath::new(vec!["a".into(), "b".into()]).unwrap(), + function: "example".into(), + }; + let wire = "function:tenant/db:a$b$example"; + assert_eq!(serde_json::to_value(&resource).unwrap(), wire); + assert_eq!( + serde_json::from_value::(serde_json::json!(wire)).unwrap(), + resource + ); + let object = Object::Function(resource.clone()); + assert_eq!(object.database(), Some("tenant/db")); + assert_eq!(object.type_name(), "function"); + assert_eq!(wire.parse::().unwrap(), object); + assert_eq!(serde_json::to_value(object).unwrap(), wire); + for name in ["", ".", "..", "bad$name", "bad/name", "bad:name"] { + let invalid = FunctionObject { + function: name.into(), + ..resource.clone() + }; + assert!(serde_json::to_value(&invalid).is_err()); + assert!(serde_json::to_value(Object::Function(invalid)).is_err()); + } + } + + #[test] + fn unknown_objects_round_trip() { + for wire in [ + "future:db:resource", + "table_extension:db:resource", + "garbage", + ] { + let object: Object = wire.parse().unwrap(); + assert_eq!(object, Object::Unknown(wire.into())); + assert_eq!(object.database(), None); + assert_eq!(object.type_name(), UNKNOWN_TYPE_STRING); + assert_eq!(object.to_string(), wire); + assert_eq!(serde_json::to_value(&object).unwrap(), wire); + assert_eq!( + serde_json::from_value::(serde_json::json!(wire)).unwrap(), + object + ); + } + } + + #[test] + fn database_object_round_trip() { + let object = DatabaseObject { + database: "db".to_string(), + }; + let encoded = serde_json::to_string(&object).unwrap(); + assert_eq!(encoded, r#""database:db""#); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + object + ); + } + + #[test] + fn namespace_object_round_trip() { + let object = NamespaceObject { + database: "db".to_string(), + namespace: NamespacePath::from_component("ns"), + }; + let encoded = serde_json::to_string(&object).unwrap(); + assert_eq!(encoded, r#""namespace:db:ns""#); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + object + ); + } + + #[test] + fn namespace_object_rejects_missing_namespace() { + assert!(serde_json::from_str::(r#""namespace:db""#).is_err()); + } + + #[test] + fn table_object_round_trip() { + let object = TableObject { + database: "db".to_string(), + namespace: NamespacePath::from_component("ns"), + table: "foo".to_string(), + }; + let encoded = serde_json::to_string(&object).unwrap(); + assert_eq!(encoded, r#""table:db:ns$foo""#); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + object + ); + } + + #[test] + fn table_object_rejects_missing_prefix() { + assert!(serde_json::from_str::(r#""foo""#).is_err()); + } + + #[test] + fn table_object_rejects_missing_table() { + assert!(serde_json::from_str::(r#""table:db:ns""#).is_err()); + } + + #[test] + fn objects_order_by_variant_then_path_components() { + let database = Object::Database(DatabaseObject::new("z".to_string())); + let namespace = Object::Namespace(NamespaceObject { + database: "a".to_string(), + namespace: NamespacePath::from_component("a"), + }); + assert!(Object::System < database); + assert!(database < namespace); + + let first_table = Object::Table(TableObject { + database: "db".to_string(), + namespace: NamespacePath::from_component("a"), + table: "z".to_string(), + }); + let second_table = Object::Table(TableObject { + database: "db".to_string(), + namespace: NamespacePath::from_component("b"), + table: "a".to_string(), + }); + assert!(first_table < second_table); + } + + #[test] + fn empty_namespace_paths_select_public() { + let namespace = NamespacePath::new(vec![]).unwrap(); + assert_eq!(namespace, NamespacePath::default()); + assert_eq!(namespace.components(), &["public"]); + assert!(namespace.parent().is_none()); + assert_eq!( + namespace.prefixes().collect::>(), + vec![namespace.clone()] + ); + + let database = "db".to_string(); + let objects = [ + Object::Namespace(NamespaceObject { + database: database.clone(), + namespace: namespace.clone(), + }), + Object::Table(TableObject { + database: database.clone(), + namespace: namespace.clone(), + table: "item".into(), + }), + Object::View(ViewObject { + database: database.clone(), + namespace: namespace.clone(), + view: "item".into(), + }), + Object::Secret(SecretObject { + database: database.clone(), + namespace: namespace.clone(), + secret: "item".into(), + }), + Object::Function(FunctionObject { + database, + namespace, + function: "item".into(), + }), + ]; + for (object, wire) in objects.into_iter().zip([ + "namespace:db:public", + "table:db:public$item", + "view:db:public$item", + "secret:db:public$item", + "function:db:public$item", + ]) { + assert_eq!(serde_json::to_value(&object).unwrap(), wire); + assert_eq!(wire.parse::().unwrap(), object); + } + } + + #[test] + fn namespace_path_returns_cumulative_prefixes() { + let path = NamespacePath::new(vec!["ns1".into(), "ns2".into(), "ns3".into()]).unwrap(); + assert_eq!( + path.prefixes() + .map(|prefix| prefix.to_string()) + .collect::>(), + vec!["ns1", "ns1$ns2", "ns1$ns2$ns3"] + ); + assert_eq!(path.parent().unwrap().to_string(), "ns1$ns2"); + } + + #[test] + fn nested_objects_round_trip_without_escaping() { + for wire in [ + "database:tenant/db", + "namespace:tenant/db:a$b", + "table:tenant/db:a$b$c", + "view:tenant/db:a$b$c", + "secret:tenant/db:a$b$c", + "function:tenant/db:a$b$c", + ] { + let object: Object = serde_json::from_value(serde_json::json!(wire)).unwrap(); + assert_eq!(wire.parse::().unwrap(), object); + assert_eq!(object.database(), Some("tenant/db")); + assert_eq!(object.to_string(), wire); + assert_eq!(serde_json::to_value(object).unwrap(), wire); + } + } + + #[test] + fn objects_reject_invalid_names_and_legacy_forms() { + for database in [ + "", + ".", + "..", + "db:bad", + "db%2Fchild", + "db$child", + "/db", + "db/", + "db//child", + "db/../child", + "db name", + ] { + for wire in [ + format!("database:{database}"), + format!("namespace:{database}:ns"), + format!("table:{database}:ns$t"), + format!("view:{database}:ns$v"), + format!("secret:{database}:ns$s"), + format!("function:{database}:ns$f"), + ] { + assert!( + serde_json::from_value::(serde_json::json!(wire)).is_err(), + "{wire}" + ); + } + } + for name in ["", "bad:name", "bad/name", "bad%24name", "bad name"] { + for wire in [ + format!("namespace:db:{name}"), + format!("table:db:{name}$t"), + format!("view:db:{name}$v"), + format!("table:db:ns${name}"), + format!("view:db:ns${name}"), + format!("secret:db:{name}$s"), + format!("function:db:{name}$f"), + format!("secret:db:ns${name}"), + format!("function:db:ns${name}"), + ] { + assert!( + serde_json::from_value::(serde_json::json!(wire)).is_err(), + "{wire}" + ); + } + } + for wire in [ + "", + "system:db", + "database", + "namespace", + "table", + "view", + "namespace:db/ns", + "table:db/ns/t", + "view:db/ns/v", + "table:db:ns", + "view:db:ns", + "namespace:db:a$$b", + "table:db:ns$.", + "view:db:ns$..", + "table:db:a$$t", + "view:db:$v", + "secret", + "function", + "secret:db/ns/s", + "function:db/ns/f", + "secret:db:ns", + "function:db:ns", + "secret:db:a$$s", + "function:db:$f", + "secret:db:ns$.", + "function:db:ns$..", + ] { + assert!( + serde_json::from_value::(serde_json::json!(wire)).is_err(), + "{wire}" + ); + assert!(matches!( + wire.parse::(), + Err(crate::Error::InvalidInput { .. }) + )); + } + } + + #[test] + fn namespace_components_use_namespace_validation() { + for component in ["a$b", "c/d", "e%f", "a:b", "", ".", ".."] { + assert!(NamespacePath::new(vec![component.into()]).is_err()); + } + assert!(NamespacePath::new(vec!["a.b".into(), "c..d".into()]).is_ok()); + } + + #[test] + fn serialization_rejects_invalid_directly_constructed_objects() { + let database = DatabaseObject::new("db:bad".into()); + let namespace = NamespaceObject { + database: "db%2Fbad".into(), + namespace: NamespacePath::from_component("ns"), + }; + let table = TableObject { + database: "db".into(), + namespace: NamespacePath::from_component("ns"), + table: "bad$name".into(), + }; + let view = ViewObject { + database: "db".into(), + namespace: NamespacePath::from_component("ns"), + view: "..".into(), + }; + assert!(serde_json::to_string(&database).is_err()); + assert!(serde_json::to_string(&namespace).is_err()); + assert!(serde_json::to_string(&table).is_err()); + assert!(serde_json::to_string(&view).is_err()); + for object in [ + Object::Database(database), + Object::Namespace(namespace), + Object::Table(table), + Object::View(view), + ] { + assert!(serde_json::to_string(&object).is_err()); + } + } +} diff --git a/rust/lancedb/src/authz/privilege.rs b/rust/lancedb/src/authz/privilege.rs new file mode 100644 index 000000000..66b07643a --- /dev/null +++ b/rust/lancedb/src/authz/privilege.rs @@ -0,0 +1,514 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +use std::{fmt, str::FromStr}; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; + +/// A LanceDB privilege, serialized by name (for example, `SELECT`). +/// Numeric aliases use `PRIVILEGE` followed by decimal digits. Numbers 0 through +/// 13 map to the known privileges in declaration order. Unassigned numbers are +/// preserved verbatim; their meaning and validity are determined by the server. +/// +/// ``` +/// use lancedb::authz::Privilege; +/// +/// assert_eq!("PRIVILEGE0".parse::().unwrap(), Privilege::Ownership); +/// assert_eq!("PRIVILEGE2".parse::().unwrap(), Privilege::Select); +/// +/// let privilege: Privilege = "PRIVILEGE256".parse().unwrap(); +/// assert_eq!(privilege.to_string(), "PRIVILEGE256"); +/// +/// // Delegate creation on a namespace without transferring its ownership. +/// for privilege in [Privilege::CreateSecret, Privilege::CreateFunction] { +/// assert!(privilege.valid_for_namespace()); +/// } +/// ``` +#[derive(Debug, Clone, PartialEq, Eq, strum_macros::Display)] +#[strum(serialize_all = "SCREAMING_SNAKE_CASE")] +#[non_exhaustive] +pub enum Privilege { + /// A special privilege that conveys ownership. + Ownership, + + /// On a database: allows accessing the namespaces inside. + /// On a namespace: allows accessing the tables inside. + /// + /// When looking for Usage on a Database, if there are no ACLs + /// found for that database, we assume it doesn't exist yet and + /// check for CreateDatabase on System. + /// + /// When looking for Usage on a public (aka default) Namespace, + /// if there are no ACLs found for that Namespace, we assume it + /// doesn't exist yet and check for CreateNamespace on the + /// containing database instead. + Usage, + + /// On a table: allows reading rows. + Select, + + /// On a table: allows appending rows. + Insert, + + /// On a table: allows modifying existing rows. + Update, + + /// On a table: allows deleting rows. + Delete, + + /// On the system object: can create a database. + CreateDatabase, + + /// On a database or namespace: can create immediate child namespaces. + CreateNamespace, + + /// On a namespace: can create a table in that namespace. + CreateTable, + + /// On a namespace: can create a materialized view in that namespace. + CreateMaterializedView, + + /// On a namespace: can create a view in that namespace. + CreateView, + + /// On a namespace: can create a secret without taking ownership of the namespace. + CreateSecret, + + /// On a namespace: can create a function without taking ownership of the namespace. + CreateFunction, + + /// On the system object: + /// - can read server configuration. + Operate, + + /// A future privilege name, preserved verbatim. + #[strum(to_string = "{0}")] + Unknown(UnknownPrivilege), +} + +/// An unknown privilege. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnknownPrivilege(String); + +impl fmt::Display for UnknownPrivilege { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl Privilege { + pub const KNOWN: [Self; 14] = [ + Self::Ownership, + Self::Usage, + Self::Select, + Self::Insert, + Self::Update, + Self::Delete, + Self::CreateDatabase, + Self::CreateNamespace, + Self::CreateTable, + Self::CreateMaterializedView, + Self::CreateView, + Self::CreateSecret, + Self::CreateFunction, + Self::Operate, + ]; + + pub fn valid_for_system(&self) -> bool { + matches!( + self, + Self::Operate | Self::CreateDatabase | Self::Usage | Self::Unknown(_) + ) + } + + pub fn valid_for_database(&self) -> bool { + matches!( + self, + Self::Ownership | Self::Usage | Self::CreateNamespace | Self::Unknown(_) + ) + } + + pub fn valid_for_namespace(&self) -> bool { + matches!( + self, + Self::Ownership + | Self::Usage + | Self::CreateNamespace + | Self::CreateTable + | Self::CreateMaterializedView + | Self::CreateView + | Self::CreateSecret + | Self::CreateFunction + | Self::Unknown(_) + ) + } + + pub fn valid_for_table(&self) -> bool { + matches!( + self, + Self::Ownership + | Self::Select + | Self::Insert + | Self::Update + | Self::Delete + | Self::Unknown(_) + ) + } + + pub fn valid_for_view(&self) -> bool { + matches!(self, Self::Ownership | Self::Select | Self::Unknown(_)) + } + + pub fn valid_for_secret(&self) -> bool { + matches!( + self, + Self::Ownership | Self::Usage | Self::Update | Self::Unknown(_) + ) + } + + pub fn valid_for_function(&self) -> bool { + matches!(self, Self::Ownership | Self::Usage | Self::Unknown(_)) + } + + /// Whether this privilege can mutate database contents or schema. + /// Unknown privileges are conservatively treated as writes. + pub fn is_write(&self) -> bool { + !matches!(self, Self::Usage | Self::Select | Self::Operate) + } +} + +impl FromStr for Privilege { + type Err = String; + + fn from_str(name: &str) -> Result { + if let Some(privilege) = Self::KNOWN.into_iter().find(|p| p.to_string() == name) { + return Ok(privilege); + } + if let Some(digits) = name.strip_prefix("PRIVILEGE") + && !digits.is_empty() + && digits.bytes().all(|b| b.is_ascii_digit()) + { + return Ok(match digits.parse::() { + Ok(0) => Self::Ownership, + Ok(1) => Self::Usage, + Ok(2) => Self::Select, + Ok(3) => Self::Insert, + Ok(4) => Self::Update, + Ok(5) => Self::Delete, + Ok(6) => Self::CreateDatabase, + Ok(7) => Self::CreateNamespace, + Ok(8) => Self::CreateTable, + Ok(9) => Self::CreateMaterializedView, + Ok(10) => Self::CreateView, + Ok(11) => Self::CreateSecret, + Ok(12) => Self::CreateFunction, + Ok(13) => Self::Operate, + _ => Self::Unknown(UnknownPrivilege(name.to_owned())), + }); + } + Err(format!("invalid privilege {name:?}")) + } +} + +/// Converts a privilege to its numeric identifier. +/// +/// Unknown privileges return an error if their number exceeds [`u8::MAX`]. +/// +/// ``` +/// use lancedb::authz::Privilege; +/// +/// assert_eq!(u8::try_from(&Privilege::Select).unwrap(), 2); +/// let unknown: Privilege = "PRIVILEGE255".parse().unwrap(); +/// assert_eq!(u8::try_from(&unknown).unwrap(), 255); +/// let too_large: Privilege = "PRIVILEGE256".parse().unwrap(); +/// assert!(u8::try_from(&too_large).is_err()); +/// ``` +impl TryFrom<&Privilege> for u8 { + type Error = std::num::ParseIntError; + + fn try_from(privilege: &Privilege) -> Result { + Ok(match privilege { + Privilege::Ownership => 0, + Privilege::Usage => 1, + Privilege::Select => 2, + Privilege::Insert => 3, + Privilege::Update => 4, + Privilege::Delete => 5, + Privilege::CreateDatabase => 6, + Privilege::CreateNamespace => 7, + Privilege::CreateTable => 8, + Privilege::CreateMaterializedView => 9, + Privilege::CreateView => 10, + Privilege::CreateSecret => 11, + Privilege::CreateFunction => 12, + Privilege::Operate => 13, + Privilege::Unknown(unknown) => unknown.0["PRIVILEGE".len()..].parse()?, + }) + } +} + +/// Converts an owned privilege to its numeric identifier. +/// +/// Like the borrowed conversion, this returns an error for numbers above 255. +/// +/// ``` +/// use lancedb::authz::Privilege; +/// +/// assert_eq!(u8::try_from(Privilege::Ownership).unwrap(), 0); +/// ``` +impl TryFrom for u8 { + type Error = std::num::ParseIntError; + + fn try_from(privilege: Privilege) -> Result { + Self::try_from(&privilege) + } +} + +impl Serialize for Privilege { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for Privilege { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + String::deserialize(deserializer)? + .parse() + .map_err(D::Error::custom) + } +} + +#[cfg(test)] +mod tests { + use super::Privilege; + + #[test] + fn display_matches_serde() { + for privilege in Privilege::KNOWN { + assert_eq!( + serde_json::to_string(&privilege).unwrap(), + format!("\"{privilege}\""), + ); + } + } + + #[test] + fn secret_and_function_creation_privileges() { + for (name, privilege) in [ + ("CREATE_SECRET", Privilege::CreateSecret), + ("CREATE_FUNCTION", Privilege::CreateFunction), + ] { + assert_eq!(name.parse::().unwrap(), privilege); + assert_eq!(privilege.to_string(), name); + assert_eq!(serde_json::to_value(&privilege).unwrap(), name); + assert_eq!( + serde_json::from_value::(serde_json::json!(name)).unwrap(), + privilege + ); + assert!(privilege.valid_for_namespace()); + assert!(!privilege.valid_for_system()); + assert!(!privilege.valid_for_database()); + assert!(!privilege.valid_for_table()); + assert!(!privilege.valid_for_view()); + assert!(privilege.is_write()); + } + } + + #[test] + fn numeric_aliases_resolve_to_known_privileges() { + for (number, privilege) in [ + (0, Privilege::Ownership), + (1, Privilege::Usage), + (2, Privilege::Select), + (3, Privilege::Insert), + (4, Privilege::Update), + (5, Privilege::Delete), + (6, Privilege::CreateDatabase), + (7, Privilege::CreateNamespace), + (8, Privilege::CreateTable), + (9, Privilege::CreateMaterializedView), + (10, Privilege::CreateView), + (11, Privilege::CreateSecret), + (12, Privilege::CreateFunction), + (13, Privilege::Operate), + ] { + assert_eq!(u8::try_from(&privilege).unwrap(), number); + assert_eq!(u8::try_from(privilege.clone()).unwrap(), number); + for name in [ + format!("PRIVILEGE{number}"), + format!("PRIVILEGE{number:03}"), + ] { + let parsed = name.parse::().unwrap(); + assert_eq!(parsed, privilege, "{name}"); + assert_eq!( + serde_json::from_value::(serde_json::json!(name)).unwrap(), + privilege + ); + assert_eq!( + serde_json::to_value(&parsed).unwrap(), + privilege.to_string() + ); + assert_eq!( + privilege.to_string().parse::().unwrap(), + privilege + ); + } + } + } + + #[test] + fn unknown_privileges_convert_to_u8() { + for number in 14..=u8::MAX { + for name in [ + format!("PRIVILEGE{number}"), + format!("PRIVILEGE{number:03}"), + ] { + let privilege: Privilege = name.parse().unwrap(); + assert!(matches!(privilege, Privilege::Unknown(_))); + assert_eq!(u8::try_from(&privilege).unwrap(), number); + assert_eq!(u8::try_from(privilege).unwrap(), number); + } + } + } + + #[test] + fn privileges_outside_u8_range_fail_conversion() { + for name in [ + "PRIVILEGE256", + "PRIVILEGE0256", + "PRIVILEGE18446744073709551616", + ] { + let privilege: Privilege = name.parse().unwrap(); + assert_eq!( + u8::try_from(&privilege).unwrap_err().kind(), + &std::num::IntErrorKind::PosOverflow + ); + assert_eq!( + u8::try_from(privilege).unwrap_err().kind(), + &std::num::IntErrorKind::PosOverflow + ); + } + } + + #[test] + fn future_privilege_names_round_trip() { + for name in [ + "PRIVILEGE14", + "PRIVILEGE20", + "PRIVILEGE64", + "PRIVILEGE256", + "PRIVILEGE18446744073709551616", + "PRIVILEGE020", + ] { + let privilege: Privilege = name.parse().unwrap(); + assert!(matches!(privilege, Privilege::Unknown(_))); + assert_eq!(privilege.to_string(), name); + assert_eq!(serde_json::to_value(&privilege).unwrap(), name); + assert_eq!( + serde_json::from_value::(serde_json::json!(name)).unwrap(), + privilege + ); + } + } + + #[test] + fn rejects_invalid_privileges() { + for name in [ + "", + "select", + " SELECT", + "SELECT ", + "PRIVILEGE", + "PRIVILEGE-1", + "PRIVILEGE+1", + "PRIVILEGE 1", + "PRIVILEGE1x", + "privilege12", + "UNKNOWN", + ] { + assert!(name.parse::().is_err(), "{name}"); + assert!( + serde_json::from_value::(serde_json::json!(name)).is_err(), + "{name}" + ); + } + } + + #[test] + fn unknown_privileges_are_valid_for_all_objects_and_treated_as_writes() { + for name in ["PRIVILEGE14", "PRIVILEGE20", "PRIVILEGE64", "PRIVILEGE256"] { + let privilege: Privilege = name.parse().unwrap(); + assert!(privilege.valid_for_system()); + assert!(privilege.valid_for_database()); + assert!(privilege.valid_for_namespace()); + assert!(privilege.valid_for_table()); + assert!(privilege.valid_for_view()); + assert!(privilege.valid_for_secret()); + assert!(privilege.valid_for_function()); + assert!(privilege.is_write()); + } + assert!(!Privilege::Ownership.valid_for_system()); + assert!(!Privilege::Select.valid_for_database()); + } + + #[test] + fn test_privilege_is_write() { + for privilege in Privilege::KNOWN { + assert_eq!( + privilege.is_write(), + matches!( + privilege, + Privilege::Ownership + | Privilege::Insert + | Privilege::Update + | Privilege::Delete + | Privilege::CreateDatabase + | Privilege::CreateNamespace + | Privilege::CreateTable + | Privilege::CreateMaterializedView + | Privilege::CreateView + | Privilege::CreateSecret + | Privilege::CreateFunction + ), + "unexpected write classification for {privilege}" + ); + } + } + + #[test] + fn create_secret_and_create_function_are_namespace_scoped() { + for privilege in [Privilege::CreateSecret, Privilege::CreateFunction] { + assert!(privilege.valid_for_namespace()); + assert!(!privilege.valid_for_system()); + assert!(!privilege.valid_for_database()); + assert!(!privilege.valid_for_table()); + assert!(!privilege.valid_for_view()); + assert!(!privilege.valid_for_secret()); + assert!(!privilege.valid_for_function()); + } + } + + #[test] + fn secret_and_function_privileges() { + assert_eq!( + Privilege::KNOWN + .into_iter() + .filter(Privilege::valid_for_secret) + .collect::>(), + vec![Privilege::Ownership, Privilege::Usage, Privilege::Update] + ); + assert_eq!( + Privilege::KNOWN + .into_iter() + .filter(Privilege::valid_for_function) + .collect::>(), + vec![Privilege::Ownership, Privilege::Usage] + ); + } +} diff --git a/rust/lancedb/src/authz/subject.rs b/rust/lancedb/src/authz/subject.rs new file mode 100644 index 000000000..2cc607fc4 --- /dev/null +++ b/rust/lancedb/src/authz/subject.rs @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Identity selectors. Provider names resolve on the server; IDs remain literal. + +use std::{fmt, str::FromStr}; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; + +use crate::{Error, Result}; + +/// The kind of identifier supplied to an authorization operation. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum SubjectKind { + PrincipalId, + PrincipalName, + GroupId, + GroupName, + Role, + PrincipalApiKey, + /// An unrecognized canonical subject. Its full wire string is stored as the value. + Unknown, +} + +impl SubjectKind { + /// Stable name used by language bindings. + pub fn as_str(self) -> &'static str { + match self { + Self::PrincipalId => "principal_id", + Self::PrincipalName => "principal_name", + Self::GroupId => "group_id", + Self::GroupName => "group_name", + Self::Role => "role", + Self::PrincipalApiKey => "principal_api_key", + Self::Unknown => "unknown", + } + } + + fn prefix(self) -> &'static str { + match self { + Self::PrincipalId => "p:", + Self::PrincipalName => "P:", + Self::GroupId => "g:", + Self::GroupName => "G:", + Self::Role => "r:", + Self::PrincipalApiKey => "a:", + Self::Unknown => "", + } + } +} + +const KINDS: [SubjectKind; 6] = [ + SubjectKind::PrincipalId, + SubjectKind::PrincipalName, + SubjectKind::GroupId, + SubjectKind::GroupName, + SubjectKind::Role, + SubjectKind::PrincipalApiKey, +]; + +impl FromStr for SubjectKind { + type Err = Error; + + fn from_str(value: &str) -> Result { + if value == "unknown" { + return Ok(Self::Unknown); + } + KINDS + .into_iter() + .find(|kind| kind.as_str() == value) + .ok_or_else(|| invalid("Invalid subject kind")) + } +} + +/// An immutable validated principal, group, or role selector. +/// +/// Unrecognized canonical prefixes are preserved as [`SubjectKind::Unknown`], +/// with the complete wire string in [`Self::value`], so future subject kinds can +/// be listed and sent back to the server. +/// +/// ``` +/// use lancedb::authz::{Subject, SubjectKind}; +/// let subject = Subject::from_canonical("x:future-subject")?; +/// assert_eq!(subject.kind(), SubjectKind::Unknown); +/// assert_eq!(subject.to_wire(), "x:future-subject"); +/// # Ok::<(), lancedb::Error>(()) +/// ``` +/// +/// Debug and Display redact API keys. [`Self::to_wire`] and serialization +/// intentionally expose the key for transport and must not be logged. +/// +/// ``` +/// use lancedb::authz::Subject; +/// let subject = Subject::group_name("Engineering")?; +/// assert_eq!(subject.to_wire(), "G:Engineering"); +/// assert_eq!(subject, "G:Engineering".parse::()?); +/// # Ok::<(), lancedb::Error>(()) +/// ``` +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct Subject { + kind: SubjectKind, + value: String, +} + +fn invalid(message: &str) -> Error { + // Never include the selector: it may contain an API key. + Error::InvalidInput { + message: message.into(), + } +} + +impl Subject { + /// Construct a selector without resolving it against an identity provider. + pub fn new(kind: SubjectKind, value: impl Into) -> Result { + let value = value.into(); + if value.is_empty() { + return Err(invalid("Subject value must be a nonempty string")); + } + if kind == SubjectKind::Unknown + && (!value + .split_once(':') + .is_some_and(|(prefix, value)| !prefix.is_empty() && !value.is_empty()) + || KINDS.iter().any(|kind| value.starts_with(kind.prefix()))) + { + return Err(invalid("Invalid unknown subject selector")); + } + Ok(Self { kind, value }) + } + + /// Select a principal by canonical ID, without stripping any prefix in the ID. + pub fn principal_id(id: impl Into) -> Result { + Self::new(SubjectKind::PrincipalId, id) + } + /// Select a principal by an unambiguous provider name. + pub fn principal_name(name: impl Into) -> Result { + Self::new(SubjectKind::PrincipalName, name) + } + /// Select a group by canonical ID. + pub fn group_id(id: impl Into) -> Result { + Self::new(SubjectKind::GroupId, id) + } + /// Select a group by an unambiguous provider name. + pub fn group_name(name: impl Into) -> Result { + Self::new(SubjectKind::GroupName, name) + } + /// Select a LanceDB-managed role. + pub fn role(name: impl Into) -> Result { + Self::new(SubjectKind::Role, name) + } + /// Select the principal holding this secret API key, not the request's caller. + pub fn principal_api_key(key: impl Into) -> Result { + Self::new(SubjectKind::PrincipalApiKey, key) + } + + /// The selector kind. + pub fn kind(&self) -> SubjectKind { + self.kind + } + /// The selected ID, name, role, or API key. API-key values are secrets. + /// For unknown kinds, this is the full canonical wire string. + pub fn value(&self) -> &str { + &self.value + } + /// The value safe for display; API keys are replaced with ``. + pub fn display_value(&self) -> &str { + if self.kind == SubjectKind::PrincipalApiKey { + "" + } else { + &self.value + } + } + /// Encode the literal wire selector. The result can contain a secret API key. + pub fn to_wire(&self) -> String { + format!("{}{}", self.kind.prefix(), self.value) + } + + /// Parse a returned canonical identity, rejecting names and secret selectors. + pub fn from_canonical(value: &str) -> Result { + let subject: Self = value.parse()?; + if !matches!( + subject.kind, + SubjectKind::PrincipalId + | SubjectKind::GroupId + | SubjectKind::Role + | SubjectKind::Unknown + ) { + return Err(invalid("Server returned a non-canonical subject")); + } + Ok(subject) + } + + /// Deserialize a canonical response subject, rejecting secrets and name selectors. + pub fn deserialize_canonical<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> std::result::Result { + Self::from_canonical(&String::deserialize(deserializer)?).map_err(D::Error::custom) + } + + /// Require a principal selector for a principal lookup. + pub fn validate_principal(&self) -> Result<()> { + if matches!( + self.kind, + SubjectKind::PrincipalId | SubjectKind::PrincipalName | SubjectKind::PrincipalApiKey + ) { + Ok(()) + } else { + Err(invalid("Subject kind is not valid for a principal lookup")) + } + } + /// Require a group selector for a group lookup. + pub fn validate_group(&self) -> Result<()> { + if matches!(self.kind, SubjectKind::GroupId | SubjectKind::GroupName) { + Ok(()) + } else { + Err(invalid("Subject kind is not valid for a group lookup")) + } + } + /// Require a principal/group selector. Roles cannot contain other roles. + pub fn validate_role_member(&self) -> Result<()> { + if self.kind == SubjectKind::Role { + Err(invalid("Roles cannot be members of other roles")) + } else { + Ok(()) + } + } +} + +impl FromStr for Subject { + type Err = Error; + fn from_str(value: &str) -> Result { + for kind in KINDS { + if let Some(value) = value.strip_prefix(kind.prefix()) { + return Self::new(kind, value); + } + } + Self::new(SubjectKind::Unknown, value) + } +} + +impl fmt::Debug for Subject { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Subject") + .field("kind", &self.kind) + .field("value", &self.display_value()) + .finish() + } +} +impl fmt::Display for Subject { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}{}", self.kind.prefix(), self.display_value()) + } +} +impl Serialize for Subject { + fn serialize(&self, serializer: S) -> std::result::Result { + serializer.serialize_str(&self.to_wire()) + } +} +impl<'de> Deserialize<'de> for Subject { + fn deserialize>(deserializer: D) -> std::result::Result { + String::deserialize(deserializer)? + .parse() + .map_err(D::Error::custom) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn selectors_roundtrip_without_normalizing_values() { + for kind in KINDS { + for value in [ + "alice@example.com", + "p:literal-id", + "Engineering / R&D", + "名前", + ] { + let subject = Subject::new(kind, value).unwrap(); + assert_eq!(subject.to_wire().parse::().unwrap(), subject); + let json = serde_json::to_string(&subject).unwrap(); + assert_eq!(serde_json::from_str::(&json).unwrap(), subject); + assert_eq!(subject.value(), value); + assert!(Subject::new(kind, "").is_err()); + } + } + } + + #[test] + fn contexts_and_canonical_ids() { + for kind in KINDS { + let subject = Subject::new(kind, "value").unwrap(); + assert_eq!( + subject.validate_principal().is_ok(), + matches!( + kind, + SubjectKind::PrincipalId + | SubjectKind::PrincipalName + | SubjectKind::PrincipalApiKey + ) + ); + assert_eq!( + subject.validate_group().is_ok(), + matches!(kind, SubjectKind::GroupId | SubjectKind::GroupName) + ); + assert_eq!( + subject.validate_role_member().is_ok(), + kind != SubjectKind::Role + ); + assert_eq!( + Subject::from_canonical(&subject.to_wire()).is_ok(), + matches!( + kind, + SubjectKind::PrincipalId | SubjectKind::GroupId | SubjectKind::Role + ) + ); + } + } + + #[test] + fn unknown_subjects_preserve_canonical_wire_strings() { + for wire in ["x:future-subject", "future:literal:identifier"] { + let subject = Subject::from_canonical(wire).unwrap(); + assert_eq!(subject.kind(), SubjectKind::Unknown); + assert_eq!(subject.value(), wire); + assert_eq!(subject.to_wire(), wire); + assert_eq!(serde_json::to_value(&subject).unwrap(), wire); + assert_eq!( + serde_json::from_value::(serde_json::json!(wire)).unwrap(), + subject + ); + assert!(subject.validate_principal().is_err()); + assert!(subject.validate_group().is_err()); + } + for wire in [ + "", + "no-prefix", + ":value", + "x:", + "p:value", + "P:value", + "g:value", + "G:value", + "r:value", + "a:secret", + ] { + assert!(Subject::new(SubjectKind::Unknown, wire).is_err()); + } + } + + #[test] + fn acl_page_preserves_known_and_unknown_subjects() { + let payload = serde_json::json!({"entries": [ + {"subject_id": "p:known", "object": "system", "privilege": "USAGE"}, + {"subject_id": "x:future-subject", "object": "system", "privilege": "USAGE"} + ]}); + let page: crate::authz::ListAccessControlEntriesResponse = + serde_json::from_value(payload.clone()).unwrap(); + assert_eq!(page.entries.len(), 2); + assert_eq!(page.entries[0].subject_id.kind(), SubjectKind::PrincipalId); + assert_eq!(page.entries[1].subject_id.kind(), SubjectKind::Unknown); + assert_eq!(serde_json::to_value(page).unwrap(), payload); + } + + #[test] + fn secrets_are_only_exposed_by_explicit_serialization() { + let secret = "never-log-this-key"; + let subject = Subject::principal_api_key(secret).unwrap(); + assert!(!format!("{subject:?} {subject}").contains(secret)); + assert!(serde_json::to_string(&subject).unwrap().contains(secret)); + let error = Subject::from_canonical(&subject.to_wire()).unwrap_err(); + assert!(!error.to_string().contains(secret)); + assert!(!format!("{error:?}").contains(secret)); + let error = secret.parse::().unwrap_err(); + assert!(!error.to_string().contains(secret)); + } +} diff --git a/rust/lancedb/src/catalog.rs b/rust/lancedb/src/catalog.rs index 576eb8e88..ad8674b77 100644 --- a/rust/lancedb/src/catalog.rs +++ b/rust/lancedb/src/catalog.rs @@ -193,6 +193,13 @@ impl Stream for DatabaseNames { /// A backend that manages databases. Implementations own database lifecycle semantics. #[async_trait::async_trait] pub trait Catalog: Send + Sync + std::fmt::Debug + 'static { + /// Obtain deployment-wide authorization, if supported by this backend. + fn authz(&self) -> Result> { + Err(crate::Error::NotSupported { + message: "Authorization administration is not supported by this catalog".into(), + }) + } + /// Catalog endpoint or location. fn uri(&self) -> &str; /// Create a database, or open it when `exist_ok` permits. @@ -239,6 +246,11 @@ impl CatalogConnection { pub fn uri(&self) -> &str { self.catalog.uri() } + /// Obtain a handle sharing this catalog's deployment scope and credentials. + pub fn authz(&self) -> Result> { + self.catalog.authz() + } + /// Access the underlying backend. pub fn catalog(&self) -> &Arc { &self.catalog diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index e215f3c6b..7750f66b5 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -173,6 +173,7 @@ #![recursion_limit = "256"] pub mod arrow; +pub mod authz; pub mod blob; pub mod catalog; pub mod connection; diff --git a/rust/lancedb/src/remote.rs b/rust/lancedb/src/remote.rs index bc534c0f4..a6b2f0e62 100644 --- a/rust/lancedb/src/remote.rs +++ b/rust/lancedb/src/remote.rs @@ -6,6 +6,7 @@ //! building client/server applications with LanceDB or as a client for some //! other custom LanceDB service. +mod authz; pub mod catalog; pub(crate) mod client; pub(crate) mod db; diff --git a/rust/lancedb/src/remote/authz.rs b/rust/lancedb/src/remote/authz.rs new file mode 100644 index 000000000..88c2841ce --- /dev/null +++ b/rust/lancedb/src/remote/authz.rs @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +use super::client::{HttpSend, RequestResultExt, RestfulLanceDbClient, Sender}; +use crate::Result; +use crate::authz::*; +use serde::{Serialize, de::DeserializeOwned}; + +pub struct RemoteAuthorization { + pub(crate) client: RestfulLanceDbClient, +} + +impl std::fmt::Debug for RemoteAuthorization { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RemoteAuthorization") + .finish_non_exhaustive() + } +} + +impl RemoteAuthorization { + async fn post( + &self, + route: &str, + request: &T, + ) -> Result<(String, reqwest::Response)> { + // Structured objects can be constructed with invalid names. Serialize + // before building the request so validation errors remain local errors. + let body = serde_json::to_value(request).map_err(|error| crate::Error::InvalidInput { + message: error.to_string(), + })?; + let request = self + .client + .post(&format!("/admin/authz/{route}")) + .json(&body); + // Every subject selector can contain a secret. Never log these bodies, + // and never replay an ACL mutation after an ambiguous response. + let (id, response) = self.client.send_suppressing_body(request).await?; + let response = self.client.check_response(&id, response).await?; + Ok((id, response)) + } + + async fn read( + &self, + route: &str, + request: &T, + ) -> Result { + let (id, response) = self.post(route, request).await?; + response.json().await.err_to_http(id) + } +} + +#[async_trait::async_trait] +impl Authorization for RemoteAuthorization { + async fn list_acls( + &self, + request: ListAccessControlEntriesRequest, + ) -> Result { + self.read("acl/list", &request).await + } + async fn add_acl(&self, request: AccessControlEntryRequest) -> Result<()> { + self.post("acl/add", &request).await?; + Ok(()) + } + async fn delete_acl(&self, request: AccessControlEntryRequest) -> Result<()> { + self.post("acl/delete", &request).await?; + Ok(()) + } + async fn list_role_bindings( + &self, + request: ListRoleBindingsRequest, + ) -> Result { + request.validate()?; + self.read("role_binding/list", &request).await + } + async fn add_role_binding(&self, request: RoleBindingRequest) -> Result<()> { + request.validate()?; + self.post("role_binding/add", &request).await?; + Ok(()) + } + async fn delete_role_binding(&self, request: RoleBindingRequest) -> Result<()> { + request.validate()?; + self.post("role_binding/delete", &request).await?; + Ok(()) + } + async fn get_principal(&self, request: GetPrincipalRequest) -> Result { + request.validate()?; + self.read("principal", &request).await + } + async fn get_group(&self, request: GetGroupRequest) -> Result { + request.validate()?; + self.read("group", &request).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::remote::client::test_utils::client_with_handler; + use serde_json::json; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + #[tokio::test] + async fn list_preserves_wire_identity_and_continuation() { + let authz = RemoteAuthorization { + client: client_with_handler(|request| { + assert_eq!(request.url().path(), "/admin/authz/acl/list"); + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); + assert_eq!( + body, + json!({"object": "table:tenant/db:a$b$events", "subject": "G:Engineering", "max_results": 100, "continuation_token": "opaque/token"}) + ); + http::Response::builder().status(200).body(json!({ + "entries": [{"object": "table:tenant/db:a$b$events", "subject_id": "g:team-id", "privilege": "PRIVILEGE20"}], + "continuation_token": "next/token" + }).to_string()).unwrap() + }), + }; + let result = authz + .list_acls(ListAccessControlEntriesRequest { + object: Some("table:tenant/db:a$b$events".parse().unwrap()), + subject: Some(Subject::group_name("Engineering").unwrap()), + max_results: Some(100), + continuation_token: Some("opaque/token".into()), + }) + .await + .unwrap(); + assert_eq!(result.continuation_token.as_deref(), Some("next/token")); + assert_eq!( + result.entries[0].subject_id, + Subject::group_id("team-id").unwrap() + ); + assert_eq!(result.entries[0].subject_name, None); + assert_eq!( + result.entries[0].privilege, + "PRIVILEGE20".parse::().unwrap() + ); + } + + #[tokio::test] + async fn acl_failure_is_not_retried_or_reclassified() { + for status in [404, 409, 503] { + let calls = Arc::new(AtomicUsize::new(0)); + let count = calls.clone(); + let authz = RemoteAuthorization { + client: client_with_handler(move |request| { + count.fetch_add(1, Ordering::SeqCst); + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()) + .unwrap(); + assert_eq!(body["privilege"], "USAGE"); + http::Response::builder() + .status(status) + .body("failure") + .unwrap() + }), + }; + let result = authz + .add_acl(AccessControlEntryRequest { + object: Object::Database(DatabaseObject::new("db".into())), + subject: Subject::principal_api_key("secret").unwrap(), + privilege: Privilege::Usage, + }) + .await; + let Err(crate::Error::Http { + status_code, + request_id, + .. + }) = result + else { + panic!("expected Http error") + }; + assert_eq!(status_code.unwrap().as_u16(), status); + assert!(!request_id.is_empty()); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + } + + #[tokio::test] + async fn rejects_invalid_objects_before_sending() { + let authz = RemoteAuthorization { + client: client_with_handler(|_| -> http::Response { + panic!("invalid objects must not send a request") + }), + }; + let request = AccessControlEntryRequest::new( + Object::Database(DatabaseObject::new("bad:database".into())), + Subject::principal_api_key("secret").unwrap(), + Privilege::Usage, + ); + for result in [ + authz.add_acl(request.clone()).await, + authz.delete_acl(request.clone()).await, + authz + .list_acls(ListAccessControlEntriesRequest { + object: Some(request.object), + ..Default::default() + }) + .await + .map(|_| ()), + ] { + assert!(matches!(result, Err(crate::Error::InvalidInput { .. }))); + assert!(!result.unwrap_err().to_string().contains("secret")); + } + } + + #[tokio::test] + async fn rejects_incompatible_subjects_before_sending() { + let authz = RemoteAuthorization { + client: client_with_handler(|_| -> http::Response { + panic!("invalid selectors must not send a request") + }), + }; + let request = RoleBindingRequest { + subject: Subject::role("reader").unwrap(), + role: "writer".into(), + }; + assert!(matches!( + authz.add_role_binding(request.clone()).await, + Err(crate::Error::InvalidInput { .. }) + )); + assert!(matches!( + authz.delete_role_binding(request).await, + Err(crate::Error::InvalidInput { .. }) + )); + let request = ListRoleBindingsRequest { + subject: Some(Subject::role("reader").unwrap()), + ..Default::default() + }; + assert!(matches!( + authz.list_role_bindings(request).await, + Err(crate::Error::InvalidInput { .. }) + )); + assert!(matches!( + authz + .get_principal(GetPrincipalRequest { + principal: Subject::group_id("group").unwrap() + }) + .await, + Err(crate::Error::InvalidInput { .. }) + )); + assert!(matches!( + authz + .get_group(GetGroupRequest { + group: Subject::principal_api_key("secret").unwrap() + }) + .await, + Err(crate::Error::InvalidInput { .. }) + )); + } +} diff --git a/rust/lancedb/src/remote/catalog.rs b/rust/lancedb/src/remote/catalog.rs index fa6b185cc..df3782368 100644 --- a/rust/lancedb/src/remote/catalog.rs +++ b/rust/lancedb/src/remote/catalog.rs @@ -176,6 +176,10 @@ impl RemoteCatalog { #[async_trait] impl Catalog for RemoteCatalog { + fn authz(&self) -> Result> { + Ok(self.root.authz()) + } + fn uri(&self) -> &str { &self.endpoint } diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 28582cdc3..a0f4aa953 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -297,6 +297,12 @@ pub struct RemoteHostOverrides { } impl RemoteDatabase { + pub(crate) fn authz(&self) -> Arc { + Arc::new(super::authz::RemoteAuthorization { + client: self.client.clone(), + }) + } + pub(crate) fn try_new( uri: &str, api_key: &str, From d2a42e90568e55244e22c3959222ad0b215a3143 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:44:58 -0700 Subject: [PATCH 05/19] fix: coerce blob items in list columns on add (#4302) --- python/python/tests/test_blob.py | 5 +- .../src/table/datafusion/blob_coerce.rs | 409 +++++++++++++++++- rust/lancedb/src/table/datafusion/cast.rs | 10 +- 3 files changed, 417 insertions(+), 7 deletions(-) diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 6be0a58a5..aef5e5203 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -1066,13 +1066,14 @@ def test_add_list_of_dicts_to_blob_list_column(large_list): table = db.create_table(f"blob_{large_list}_list_add", data=seed) table.add([{"id": 1, "items": [None]}]) + table.add([{"id": 2, "items": [b"a", None, b""]}]) table.add( - [{"id": 2, "items": [b"a", None]}], + [{"id": 3, "items": [b"b", None]}], on_bad_vectors="fill", ) ids = table.search().select(["id"]).to_arrow()["id"].to_pylist() - assert sorted(ids) == [0, 1, 2] + assert sorted(ids) == [0, 1, 2, 3] assert pa.types.is_large_list(table.schema.field("items").type) is large_list diff --git a/rust/lancedb/src/table/datafusion/blob_coerce.rs b/rust/lancedb/src/table/datafusion/blob_coerce.rs index 3b79ec022..80b6cd6fc 100644 --- a/rust/lancedb/src/table/datafusion/blob_coerce.rs +++ b/rust/lancedb/src/table/datafusion/blob_coerce.rs @@ -9,16 +9,22 @@ use std::fmt; use std::hash::{Hash, Hasher}; use std::sync::Arc; -use arrow_array::{Array, BooleanArray, RecordBatch}; +use arrow_array::{ + Array, ArrayRef, BooleanArray, FixedSizeListArray, LargeListArray, ListArray, RecordBatch, +}; +use arrow_buffer::OffsetBuffer; +use arrow_cast::cast; use arrow_schema::{DataType, Field, FieldRef, Fields, Schema}; use arrow_select::nullif::nullif; use datafusion::functions::core::{get_field, named_struct}; -use datafusion_common::ScalarValue; use datafusion_common::config::ConfigOptions; +use datafusion_common::{DataFusionError, ScalarValue}; use datafusion_expr::ColumnarValue; use datafusion_physical_expr::ScalarFunctionExpr; use datafusion_physical_expr::expressions::{CastExpr, Literal}; use datafusion_physical_plan::PhysicalExpr; +use datafusion_physical_plan::expressions::Column; +use lance_arrow::FieldExt; use crate::error::{Error, Result}; @@ -157,6 +163,198 @@ pub(super) fn coerce_blob_expr( Ok((expr, table_field.clone())) } +/// Apply the normal blob coercion to list values while retaining the list's +/// offsets and validity. A child batch lets the existing blob expression handle +/// binary, null, and descriptor inputs exactly as it does for top-level fields. +pub(super) fn coerce_blob_list_expr( + input_expr: Arc, + input_field: &Field, + table_field: &FieldRef, + config: &Arc, +) -> Result, FieldRef)>> { + let (DataType::List(input_item) + | DataType::LargeList(input_item) + | DataType::FixedSizeList(input_item, _)) = input_field.data_type() + else { + return Ok(None); + }; + let (DataType::List(table_item) + | DataType::LargeList(table_item) + | DataType::FixedSizeList(table_item, _)) = table_field.data_type() + else { + return Ok(None); + }; + if !table_item.is_blob_v2() || input_item == table_item { + return Ok(None); + } + + let (item_expr, _) = coerce_blob_expr( + Arc::new(Column::new(input_item.name(), 0)), + input_item, + table_item, + config, + )?; + let expr: Arc = Arc::new(CoerceBlobList { + source: input_expr, + input_item: input_item.clone(), + item_expr, + field: table_field.clone(), + }); + Ok(Some((expr, table_field.clone()))) +} + +#[derive(Debug, Clone)] +struct CoerceBlobList { + source: Arc, + input_item: FieldRef, + item_expr: Arc, + field: FieldRef, +} + +impl fmt::Display for CoerceBlobList { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "coerce_blob_list({})", self.source) + } +} + +impl PartialEq for CoerceBlobList { + fn eq(&self, other: &Self) -> bool { + self.source.eq(&other.source) + && self.input_item == other.input_item + && self.item_expr.eq(&other.item_expr) + && self.field == other.field + } +} + +impl Eq for CoerceBlobList {} + +impl Hash for CoerceBlobList { + fn hash(&self, state: &mut H) { + self.source.hash(state); + self.input_item.hash(state); + self.item_expr.hash(state); + self.field.hash(state); + } +} + +impl PhysicalExpr for CoerceBlobList { + fn return_field(&self, _input_schema: &Schema) -> datafusion_common::Result { + Ok(self.field.clone()) + } + + fn nullable(&self, _input_schema: &Schema) -> datafusion_common::Result { + Ok(self.field.is_nullable()) + } + + fn evaluate(&self, batch: &RecordBatch) -> datafusion_common::Result { + let source = self.source.evaluate(batch)?.into_array(batch.num_rows())?; + // First convert the outer list kind, if needed, while keeping its binary + // item type. PyArrow commonly infers List even for a LargeList column. + let intermediate_type = match self.field.data_type() { + DataType::List(_) => DataType::List(self.input_item.clone()), + DataType::LargeList(_) => DataType::LargeList(self.input_item.clone()), + DataType::FixedSizeList(_, size) => { + DataType::FixedSizeList(self.input_item.clone(), *size) + } + _ => unreachable!("validated list type when building the expression"), + }; + let source = if source.data_type() == &intermediate_type { + source + } else { + cast(source.as_ref(), &intermediate_type)? + }; + let values = match source.data_type() { + DataType::List(_) => { + let list = source.as_any().downcast_ref::().unwrap(); + let first = list.offsets()[0] as usize; + let last = *list.offsets().last().unwrap() as usize; + list.values().slice(first, last - first) + } + DataType::LargeList(_) => { + let list = source.as_any().downcast_ref::().unwrap(); + let first = list.offsets()[0] as usize; + let last = *list.offsets().last().unwrap() as usize; + list.values().slice(first, last - first) + } + DataType::FixedSizeList(_, _) => source + .as_any() + .downcast_ref::() + .unwrap() + .values() + .clone(), + other => { + return Err(DataFusionError::Internal(format!( + "coerce_blob_list expected a list, got {other}" + ))); + } + }; + let item_batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![self.input_item.clone()])), + vec![values.clone()], + )?; + let coerced = self + .item_expr + .evaluate(&item_batch)? + .into_array(values.len())?; + + let rebuilt: ArrayRef = match (source.data_type(), self.field.data_type()) { + (DataType::List(_), DataType::List(item)) => { + let list = source.as_any().downcast_ref::().unwrap(); + Arc::new(ListArray::try_new( + item.clone(), + OffsetBuffer::from_lengths(list.offsets().lengths()), + coerced, + list.nulls().cloned(), + )?) + } + (DataType::LargeList(_), DataType::LargeList(item)) => { + let list = source.as_any().downcast_ref::().unwrap(); + Arc::new(LargeListArray::try_new( + item.clone(), + OffsetBuffer::from_lengths(list.offsets().lengths()), + coerced, + list.nulls().cloned(), + )?) + } + (DataType::FixedSizeList(_, _), DataType::FixedSizeList(item, size)) => { + let list = source + .as_any() + .downcast_ref::() + .unwrap(); + Arc::new(FixedSizeListArray::try_new_with_length( + item.clone(), + *size, + coerced, + list.nulls().cloned(), + list.len(), + )?) + } + _ => unreachable!("validated list types when building the expression"), + }; + Ok(ColumnarValue::Array(rebuilt)) + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.source] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> datafusion_common::Result> { + Ok(Arc::new(Self { + source: children[0].clone(), + input_item: self.input_item.clone(), + item_expr: self.item_expr.clone(), + field: self.field.clone(), + })) + } + + fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + /// Carries the source column's nullity onto the struct built for it. /// /// This is its own expression rather than a `CASE` because the projection @@ -253,9 +451,11 @@ mod tests { use super::*; use crate::blob::blob; use arrow_array::{ - Array, ArrayRef, BinaryArray, BinaryViewArray, Int32Array, Int64Array, LargeBinaryArray, - NullArray, RecordBatch, StringArray, StringViewArray, StructArray, UInt8Array, UInt64Array, + Array, ArrayRef, BinaryArray, BinaryViewArray, FixedSizeListArray, Int32Array, Int64Array, + LargeBinaryArray, LargeListArray, ListArray, NullArray, RecordBatch, StringArray, + StringViewArray, StructArray, UInt8Array, UInt64Array, }; + use arrow_buffer::{NullBuffer, OffsetBuffer}; use arrow_schema::Schema; use datafusion::prelude::SessionContext; use datafusion_catalog::MemTable; @@ -366,6 +566,207 @@ mod tests { ); } + #[tokio::test] + async fn binary_list_items_coerce_to_blobs_without_losing_list_shape() { + for kind in [ + "list", + "large_list", + "large_list_from_list", + "fixed_size_list", + ] { + let input_item = Arc::new(Field::new("item", DataType::Binary, true)); + let table_item = Arc::new(wide_blob_field("item")); + let values: ArrayRef = Arc::new(BinaryArray::from_iter(vec![ + Some(b"a".as_slice()), + None, + Some(b"".as_slice()), + Some(b"ignored".as_slice()), + None, + None, + ])); + let (input_type, table_type, list): (DataType, DataType, ArrayRef) = match kind { + "list" => ( + DataType::List(input_item.clone()), + DataType::List(table_item.clone()), + Arc::new(ListArray::new( + input_item, + OffsetBuffer::new(vec![0, 3, 3].into()), + values, + Some(NullBuffer::from(vec![true, false])), + )), + ), + "large_list" => ( + DataType::LargeList(input_item.clone()), + DataType::LargeList(table_item.clone()), + Arc::new(LargeListArray::new( + input_item, + OffsetBuffer::new(vec![0_i64, 3, 3].into()), + values, + Some(NullBuffer::from(vec![true, false])), + )), + ), + "large_list_from_list" => ( + DataType::List(input_item.clone()), + DataType::LargeList(table_item.clone()), + Arc::new(ListArray::new( + input_item, + OffsetBuffer::new(vec![0, 3, 3].into()), + values, + Some(NullBuffer::from(vec![true, false])), + )), + ), + "fixed_size_list" => ( + DataType::FixedSizeList(input_item.clone(), 3), + DataType::FixedSizeList(table_item.clone(), 3), + Arc::new(FixedSizeListArray::new( + input_item, + 3, + values, + Some(NullBuffer::from(vec![true, false])), + )), + ), + _ => unreachable!(), + }; + let batch = batch_with_image(Field::new("image", input_type, true), list); + let schema = Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("image", table_type, true), + ]); + let coerced = coerce(batch, &schema).await; + let image = coerced.column_by_name("image").unwrap(); + let output_schema = coerced.schema(); + let (DataType::List(item_field) + | DataType::LargeList(item_field) + | DataType::FixedSizeList(item_field, _)) = + output_schema.field_with_name("image").unwrap().data_type() + else { + unreachable!() + }; + assert!(item_field.is_blob_v2(), "{kind} lost blob metadata"); + let (offsets, items): (Vec, &StructArray) = match kind { + "list" => { + let array = image.as_any().downcast_ref::().unwrap(); + assert!(array.is_null(1)); + ( + array.offsets().iter().map(|v| i64::from(*v)).collect(), + array.values().as_any().downcast_ref().unwrap(), + ) + } + "large_list" | "large_list_from_list" => { + let array = image.as_any().downcast_ref::().unwrap(); + assert!(array.is_null(1)); + ( + array.offsets().iter().copied().collect(), + array.values().as_any().downcast_ref().unwrap(), + ) + } + "fixed_size_list" => { + let array = image.as_any().downcast_ref::().unwrap(); + assert!(array.is_null(1)); + ( + vec![0, 3, 6], + array.values().as_any().downcast_ref().unwrap(), + ) + } + _ => unreachable!(), + }; + assert_eq!( + offsets, + if kind == "fixed_size_list" { + vec![0, 3, 6] + } else { + vec![0, 3, 3] + } + ); + assert_eq!(items.len(), if kind == "fixed_size_list" { 6 } else { 3 }); + let data: &LargeBinaryArray = items + .column_by_name("data") + .unwrap() + .as_any() + .downcast_ref() + .unwrap(); + assert_eq!(data.value(0), b"a"); + assert!(items.is_null(1)); + assert_eq!(data.value(2), b""); + assert!(!items.is_null(2)); + } + } + + #[tokio::test] + async fn sliced_binary_lists_only_coerce_visible_items() { + for large in [false, true] { + let input_item = Arc::new(Field::new("item", DataType::Binary, true)); + let table_item = Arc::new(wide_blob_field("item")); + let values: ArrayRef = Arc::new(BinaryArray::from_iter(vec![ + Some(b"before".as_slice()), + Some(b"a".as_slice()), + None, + Some(b"after".as_slice()), + Some(b"tail".as_slice()), + ])); + let (input_type, table_type, list): (DataType, DataType, ArrayRef) = if large { + ( + DataType::LargeList(input_item.clone()), + DataType::LargeList(table_item), + Arc::new( + LargeListArray::new( + input_item, + OffsetBuffer::new(vec![0_i64, 1, 3, 4, 5].into()), + values, + None, + ) + .slice(1, 2), + ), + ) + } else { + ( + DataType::List(input_item.clone()), + DataType::List(table_item), + Arc::new( + ListArray::new( + input_item, + OffsetBuffer::new(vec![0, 1, 3, 4, 5].into()), + values, + None, + ) + .slice(1, 2), + ), + ) + }; + let batch = batch_with_image(Field::new("image", input_type, true), list); + let schema = Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("image", table_type, true), + ]); + let coerced = coerce(batch, &schema).await; + let image = coerced.column_by_name("image").unwrap(); + let (offsets, items): (Vec, &StructArray) = if large { + let list = image.as_any().downcast_ref::().unwrap(); + ( + list.offsets().iter().copied().collect(), + list.values().as_any().downcast_ref().unwrap(), + ) + } else { + let list = image.as_any().downcast_ref::().unwrap(); + ( + list.offsets().iter().map(|v| i64::from(*v)).collect(), + list.values().as_any().downcast_ref().unwrap(), + ) + }; + assert_eq!(offsets, [0, 2, 3]); + assert_eq!(items.len(), 3); + let data: &LargeBinaryArray = items + .column_by_name("data") + .unwrap() + .as_any() + .downcast_ref() + .unwrap(); + assert_eq!(data.value(0), b"a"); + assert!(items.is_null(1)); + assert_eq!(data.value(2), b"after"); + } + } + #[tokio::test] async fn binary_view_coerces_to_declared_blob_struct() { let batch = batch_with_image( diff --git a/rust/lancedb/src/table/datafusion/cast.rs b/rust/lancedb/src/table/datafusion/cast.rs index 2f3f8b868..063572cbe 100644 --- a/rust/lancedb/src/table/datafusion/cast.rs +++ b/rust/lancedb/src/table/datafusion/cast.rs @@ -20,7 +20,7 @@ use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr}; use lance_arrow::json::{ARROW_JSON_EXT_NAME, has_json_fields, is_arrow_json_field, is_json_field}; use lance_arrow::{ARROW_EXT_NAME_KEY, FieldExt}; -use super::blob_coerce::coerce_blob_expr; +use super::blob_coerce::{coerce_blob_expr, coerce_blob_list_expr}; use crate::{Error, Result}; pub fn cast_to_table_schema( @@ -113,6 +113,14 @@ fn build_field_exprs( continue; } + // Blob values in lists need the same coercion as top-level blob columns. + if let Some(coerced) = + coerce_blob_list_expr(input_expr.clone(), input_field, table_field, &config)? + { + result.push(coerced); + continue; + } + // A column whose values are all null infers as `Null` (pyarrow does this for a list of // dicts), so there is no input type to cast from. Emit typed nulls carrying the table // field verbatim: a plain cast would drop the field metadata, and extension columns From 6487503cff00cb6fdebfc494fb04b3780b280592 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:50:31 -0700 Subject: [PATCH 06/19] fix: omit unfetchable list blobs from column discovery (#4312) --- docs/src/js/classes/Table.md | 5 +- docs/src/js/functions/blob.md | 2 + docs/src/python/python.md | 4 + nodejs/__test__/table.test.ts | 14 ++- nodejs/lancedb/blob.ts | 2 + nodejs/lancedb/table.ts | 5 +- python/python/lancedb/table.py | 8 +- python/python/tests/test_blob.py | 10 +- rust/lancedb/src/blob.rs | 115 +++++++++++++++++--- rust/lancedb/src/remote/table.rs | 141 ++++++++++++++++++++++--- rust/lancedb/src/remote/table/blobs.rs | 62 ++++++++++- rust/lancedb/src/table.rs | 14 +-- rust/lancedb/tests/blob_integration.rs | 41 +++++++ 13 files changed, 378 insertions(+), 45 deletions(-) diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 4881d62b5..59c3209c8 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -143,7 +143,8 @@ containing the new version number of the table after altering the columns. abstract blobColumns(): Promise ``` -Blob v2 columns, including nested dotted paths. +Blob v2 columns addressable by row ID, including struct-nested dotted paths. +Blobs inside lists are omitted because a row ID does not identify an element. #### Returns @@ -560,6 +561,8 @@ compaction unless stable row ids are enabled. Results keep input order and duplicates. Null blobs are `null`. Empty blobs are empty buffers. One missing or deleted row ID rejects the entire batch, including on remote tables; no partial results are returned. +Blobs inside lists cannot be fetched by row ID. + Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes. The client splits requests automatically and reads an individual larger blob through the Range route. This method still materializes all bytes in diff --git a/docs/src/js/functions/blob.md b/docs/src/js/functions/blob.md index 20a734cd4..1a65b5133 100644 --- a/docs/src/js/functions/blob.md +++ b/docs/src/js/functions/blob.md @@ -14,6 +14,8 @@ Declares a `lance.blob.v2` column. Query results are descriptors, not payload bytes. Use [Table.fetchBlobs](../classes/Table.md#fetchblobs) or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles) to read bytes. +Blobs inside lists can be written and queried as descriptors, but cannot +currently be fetched because a row ID does not identify a list element. ## Parameters diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 200fdbd73..24372a83d 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -508,6 +508,10 @@ still work. Queries return descriptors. Call One missing or deleted row ID rejects the entire blob-read batch; the APIs do not return partial results. +Blobs inside lists can be written and queried as descriptors. They are omitted +from `blob_columns()` because row ids do not identify individual list elements, +so the fetch APIs cannot read their bytes yet. + ::: lancedb.blob ::: lancedb._blob.BlobFile diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 67d5a02d3..6c770494b 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -3041,7 +3041,7 @@ describe("when dealing with blob columns", () => { expect(bytes[0]!.equals(payload)).toBe(true); }); - it("creates and adds list blob columns", async () => { + it("creates list blobs without advertising unfetchable elements", async () => { const db = await connect(tmpDir.name); const schema = new Schema([ new Field("id", new Int64(), true), @@ -3060,7 +3060,16 @@ describe("when dealing with blob columns", () => { { id: 3n, images: [gamma, null] }, { id: 4n, images: [] }, ]); - expect(await table.blobColumns()).toEqual(["images.image"]); + expect(await table.blobColumns()).toEqual([]); + const [first] = await table.query().withRowId().toArray(); + for (const column of ["images", "images.image"]) { + await expect( + table.fetchBlobs(column, [first._rowid as bigint]), + ).rejects.toThrow(/blobs inside lists cannot be fetched/); + await expect( + table.fetchBlobFiles(column, [first._rowid as bigint]), + ).rejects.toThrow(/blobs inside lists cannot be fetched/); + } const rows = await table.query().toArray(); const byId = new Map(rows.map((row) => [Number(row.id), row])); expect(descriptorSizes(byId.get(1)!.images)).toEqual([ @@ -3106,6 +3115,7 @@ describe("when dealing with blob columns", () => { ], }, ]); + expect(await table.blobColumns()).toEqual([]); const rows = await table.query().toArray(); const byId = new Map(rows.map((row) => [Number(row.id), row])); expect( diff --git a/nodejs/lancedb/blob.ts b/nodejs/lancedb/blob.ts index f3d648f48..7b0276f14 100644 --- a/nodejs/lancedb/blob.ts +++ b/nodejs/lancedb/blob.ts @@ -81,6 +81,8 @@ export type BlobOptions = { * * Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs} * or {@link Table.fetchBlobFiles} to read bytes. + * Blobs inside lists can be written and queried as descriptors, but cannot + * currently be fetched because a row ID does not identify a list element. * * @example * ```ts diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index dbf6d7ca2..630d06f29 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -526,7 +526,8 @@ export abstract class Table { abstract takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery; /** - * Blob v2 columns, including nested dotted paths. + * Blob v2 columns addressable by row ID, including struct-nested dotted paths. + * Blobs inside lists are omitted because a row ID does not identify an element. */ abstract blobColumns(): Promise; @@ -538,6 +539,8 @@ export abstract class Table { * duplicates. Null blobs are `null`. Empty blobs are empty buffers. * One missing or deleted row ID rejects the entire batch, including on remote * tables; no partial results are returned. + * Blobs inside lists cannot be fetched by row ID. + * * Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes. * The client splits requests automatically and reads an individual larger * blob through the Range route. This method still materializes all bytes in diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index ad8286032..9a9d93698 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -2104,7 +2104,11 @@ class Table(ABC): @abstractmethod def blob_columns(self) -> list[str]: - """Names of the blob v2 columns declared on this table.""" + """Names of blob v2 columns that can be fetched by row id. + + Struct-nested blobs use dotted paths. Blobs inside lists are omitted + because a row id does not identify an individual list element. + """ @abstractmethod def fetch_blobs( @@ -2123,6 +2127,8 @@ class Table(ABC): One missing or deleted row ID rejects the entire batch; no partial results are returned. + Blobs inside lists cannot be fetched by row id yet. + Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes. The client splits requests automatically and reads an individual larger blob through the Range route. This method still materializes all diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index aef5e5203..73f9b7663 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -879,12 +879,18 @@ def test_merge_insert_list_blob_dicts(): def test_list_blob_column_queries_as_raw_descriptors(): table = _list_blob_table("list_query") + assert table.blob_columns() == [] hits = table.search().limit(10).to_arrow() element = hits.schema.field("images").type.value_type assert pa.types.is_struct(element) assert "_lance_row_id" not in element.names - with pytest.raises(ValueError, match="expected struct before segment"): - table.fetch_blobs("images.image", [0]) + for column in ("images", "images.image"): + with pytest.raises(ValueError, match="blobs inside lists cannot be fetched"): + table.fetch_blobs(column, [0]) + with pytest.raises(ValueError, match="blobs inside lists cannot be fetched"): + table.fetch_blob_files(column, [0]) + with pytest.raises(ValueError, match="blobs inside lists cannot be fetched"): + table.fetch_blob_ranges(column, [(0, 0, 1)]) def test_row_addressable_paths_exclude_list_children(): diff --git a/rust/lancedb/src/blob.rs b/rust/lancedb/src/blob.rs index 178752952..ca68164e4 100644 --- a/rust/lancedb/src/blob.rs +++ b/rust/lancedb/src/blob.rs @@ -247,8 +247,10 @@ impl BlobRangeRequest { /// `Struct` with the `lance.blob.v2` marker. Same layout Lance /// expects on write. /// -/// A blob column may be top-level or nested inside a struct or list. Nested -/// blobs are addressed by a dotted path (e.g. `info.blob`) in the read APIs. +/// A blob column may be top-level or nested inside a struct or list. Blobs +/// nested inside structs use dotted paths (e.g. `info.blob`) in the read APIs. +/// Blobs inside lists can be written and queried as descriptors, but cannot +/// currently be fetched by row id. /// /// ``` /// use arrow_schema::{DataType, Field, Schema}; @@ -317,7 +319,7 @@ fn field_tree_has_blob_v2(field: &Field) -> bool { } } -/// Collects the dotted paths of blob v2 columns under `field`, into `paths`. +/// Collects the dotted paths of row-addressable blob v2 columns under `field`. fn collect_blob_paths(field: &Field, prefix: &str, paths: &mut Vec) { let path = if prefix.is_empty() { field.name().clone() @@ -328,26 +330,58 @@ fn collect_blob_paths(field: &Field, prefix: &str, paths: &mut Vec) { paths.push(path); return; } - match field.data_type() { - DataType::Struct(children) => { - for child in children { - collect_blob_paths(child, &path, paths); - } + if let DataType::Struct(children) = field.data_type() { + for child in children { + collect_blob_paths(child, &path, paths); } - DataType::List(child) | DataType::LargeList(child) | DataType::FixedSizeList(child, _) => { - collect_blob_paths(child, &path, paths) - } - _ => {} } } +fn is_list_blob_path(field: &Field, prefix: &str, column: &str, inside_list: bool) -> bool { + let path = if prefix.is_empty() { + field.name().clone() + } else { + format!("{prefix}.{}", field.name()) + }; + if field.is_blob_v2() { + return inside_list && column == path; + } + match field.data_type() { + DataType::Struct(children) => children + .iter() + .any(|child| is_list_blob_path(child, &path, column, inside_list)), + DataType::List(child) | DataType::LargeList(child) | DataType::FixedSizeList(child, _) => { + (column == path && field_tree_has_blob_v2(child)) + || is_list_blob_path(child, &path, column, true) + } + _ => false, + } +} + +/// Give list blob paths a useful error before they reach the row-based reader. +pub(crate) fn reject_list_blob_column(schema: &Schema, column: &str) -> Result<()> { + if schema + .fields() + .iter() + .any(|field| is_list_blob_path(field, "", column, false)) + { + return Err(Error::InvalidInput { + message: format!( + "column '{column}' contains a blob inside a list; blobs inside lists cannot be fetched by row id yet" + ), + }); + } + Ok(()) +} + /// Returns true if `schema` declares any blob v2 column, including nested ones. pub(crate) fn has_blob_columns(schema: &Schema) -> bool { schema.fields().iter().any(|f| field_tree_has_blob_v2(f)) } -/// Blob v2 column paths in `schema`, declaration order preserved. Nested blobs -/// are dotted paths (e.g. `info.blob`). +/// Row-addressable blob v2 column paths in `schema`, in declaration order. +/// Struct-nested blobs use dotted paths (e.g. `info.blob`); list-nested blobs +/// are omitted because a row id cannot identify an individual list element. pub(crate) fn blob_column_names(schema: &Schema) -> Vec { let mut paths = Vec::new(); for field in schema.fields() { @@ -382,6 +416,11 @@ pub(crate) fn ensure_blob_v2_column( schema: &lance_core::datatypes::Schema, column: &str, ) -> Result<()> { + // A top-level blob cannot have a list ancestor. Keep the common read path + // from converting the entire schema just to validate that condition. + if column.contains('.') || schema.field(column).is_none_or(|field| !field.is_blob_v2()) { + reject_list_blob_column(&Schema::from(schema), column)?; + } match schema.field(column) { Some(field) if field.is_blob_v2() => Ok(()), Some(field) if field.is_blob() => Err(Error::InvalidInput { @@ -767,6 +806,54 @@ mod tests { assert_eq!(blob_column_names(&schema), vec!["info.blob"]); } + #[test] + fn list_blob_paths_are_not_row_addressable() { + let schema = Schema::new(vec![ + blob("image", true), + Field::new( + "info", + DataType::Struct(vec![blob("thumbnail", true)].into()), + true, + ), + Field::new("images", DataType::List(Arc::new(blob("item", true))), true), + Field::new( + "groups", + DataType::LargeList(Arc::new(Field::new( + "group", + DataType::Struct(vec![blob("photo", true)].into()), + true, + ))), + true, + ), + Field::new( + "frames", + DataType::FixedSizeList(Arc::new(blob("frame", true)), 2), + true, + ), + ]); + assert!(has_blob_columns(&schema)); + assert_eq!(blob_column_names(&schema), vec!["image", "info.thumbnail"]); + + for column in [ + "images", + "images.item", + "groups", + "groups.group.photo", + "frames", + "frames.frame", + ] { + let err = reject_list_blob_column(&schema, column).unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. })); + assert!( + err.to_string() + .contains("blobs inside lists cannot be fetched") + ); + } + for column in ["image", "info.thumbnail", "info"] { + reject_list_blob_column(&schema, column).unwrap(); + } + } + #[test] fn storage_version_noop_without_blob_columns() { let schema = Schema::new(vec![Field::new("id", DataType::Int64, false)]); diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 356fb1e1a..27a1d90fd 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -5251,6 +5251,14 @@ mod tests { assert_eq!(data[0].as_ref().unwrap(), &expected_data); } + fn schema_describe_response(schema: &Schema) -> http::Response { + let json_schema = JsonSchema::try_from(schema).unwrap(); + http::Response::builder() + .status(200) + .body(serde_json::json!({ "version": 1, "schema": json_schema }).to_string()) + .unwrap() + } + fn blob_describe_response() -> http::Response { let schema = Schema::new(vec![ Field::new("id", DataType::Int64, false), @@ -5258,11 +5266,7 @@ mod tests { Field::new("caption", DataType::Utf8, true), crate::blob("thumbnail", true), ]); - let json_schema = JsonSchema::try_from(&schema).unwrap(); - http::Response::builder() - .status(200) - .body(serde_json::json!({ "version": 1, "schema": json_schema }).to_string()) - .unwrap() + schema_describe_response(&schema) } #[rstest] @@ -5281,6 +5285,95 @@ mod tests { assert_eq!(columns, vec!["image".to_string(), "thumbnail".to_string()]); } + #[tokio::test] + async fn test_list_blob_paths_are_not_advertised_or_fetchable() { + let schema = Schema::new(vec![ + crate::blob("thumbnail", true), + Field::new( + "imgs", + DataType::List(Arc::new(crate::blob("item", true))), + true, + ), + ]); + let table = Table::new_with_handler_version( + "my_table", + semver::Version::new(0, 5, 0), + move |request| match request.url().path() { + "/v1/table/my_table/describe/" => schema_describe_response(&schema), + "/v1/table/my_table/fetch_blobs/" | "/v1/table/my_table/query/" => { + http::Response::builder() + .status(400) + .body("Invalid input, column 'imgs' is not a blob column".to_string()) + .unwrap() + } + path if path.starts_with("/v1/table/my_table/blob/") => http::Response::builder() + .status(400) + .body("Invalid input, column 'imgs' is not a blob column".to_string()) + .unwrap(), + path => panic!("unexpected request: {path}"), + }, + ); + + assert_eq!(table.blob_columns().await.unwrap(), vec!["thumbnail"]); + for column in ["imgs", "imgs.item"] { + let err = table.fetch_blobs(column, &[10]).await.unwrap_err(); + assert!( + err.to_string() + .contains("blobs inside lists cannot be fetched") + ); + let err = table.fetch_blob_files(column, &[10]).await.unwrap_err(); + assert!( + err.to_string() + .contains("blobs inside lists cannot be fetched") + ); + } + } + + #[tokio::test] + async fn test_fetch_blob_files_explains_uncached_list_descriptors() { + let list_field = Field::new( + "imgs", + DataType::List(Arc::new(crate::blob("item", true))), + true, + ); + let schema = Schema::new(vec![list_field.clone()]); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + list_field.clone(), + Field::new(lance::dataset::ROW_ID, DataType::UInt64, false), + ])), + vec![ + arrow_array::new_null_array(list_field.data_type(), 1), + Arc::new(arrow_array::UInt64Array::from(vec![10])), + ], + ) + .unwrap(); + let table = Table::new_with_handler_version( + "my_table", + semver::Version::new(0, 5, 0), + move |request| match request.url().path() { + "/v1/table/my_table/describe/" => { + schema_describe_response(&schema).map(|body| body.into_bytes()) + } + "/v1/table/my_table/query/" => http::Response::builder() + .status(200) + .header(CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE) + .header(VERSION_HEADER, "1") + .body(write_ipc_stream_uncompressed(&batch)) + .unwrap(), + path => panic!("unexpected request: {path}"), + }, + ); + + let error = table.fetch_blob_files("imgs", &[10]).await.unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("blobs inside lists cannot be fetched") + ); + } + #[tokio::test] async fn test_fetch_blobs_decodes_null_aligned_bytes() { let mut builder = LargeBinaryBuilder::new(); @@ -5543,14 +5636,28 @@ mod tests { async fn test_fetch_blobs_does_not_split_unrelated_bad_requests() { let requests = Arc::new(AtomicUsize::new(0)); let seen = requests.clone(); - let table = - Table::new_with_handler_version("my_table", semver::Version::new(0, 5, 0), move |_| { - seen.fetch_add(1, Ordering::SeqCst); - http::Response::builder() - .status(400) - .body(b"unknown blob column".to_vec()) - .unwrap() - }); + let table = Table::new_with_handler_version( + "my_table", + semver::Version::new(0, 5, 0), + move |request| match request.url().path() { + "/v1/table/my_table/describe/" => { + schema_describe_response(&Schema::new(vec![Field::new( + "id", + DataType::Int64, + false, + )])) + .map(|body| body.into_bytes()) + } + "/v1/table/my_table/fetch_blobs/" => { + seen.fetch_add(1, Ordering::SeqCst); + http::Response::builder() + .status(400) + .body(b"unknown blob column".to_vec()) + .unwrap() + } + path => panic!("unexpected request: {path}"), + }, + ); assert_fetch_blobs_http_error( table.fetch_blobs("missing", &[10, 20]).await.unwrap_err(), @@ -5713,10 +5820,18 @@ mod tests { #[tokio::test] async fn test_fetch_blobs_sends_a_nested_column_as_a_dotted_path() { let ipc = one_row_blob_ipc_stream("info.blob"); + let schema = Schema::new(vec![Field::new( + "info", + DataType::Struct(vec![crate::blob("blob", true)].into()), + true, + )]); let table = Table::new_with_handler_version( "my_table", semver::Version::new(0, 5, 0), move |request| { + if request.url().path() == "/v1/table/my_table/describe/" { + return schema_describe_response(&schema).map(|body| body.into_bytes()); + } let body = request_body_json(&request); assert_eq!(body["column"], "info.blob"); http::Response::builder() diff --git a/rust/lancedb/src/remote/table/blobs.rs b/rust/lancedb/src/remote/table/blobs.rs index b8506adb3..9db901838 100644 --- a/rust/lancedb/src/remote/table/blobs.rs +++ b/rust/lancedb/src/remote/table/blobs.rs @@ -375,6 +375,34 @@ fn validate_partial_response( } impl RemoteTable { + async fn reject_list_blob_path_if_known(&self, column: &str) -> Result<()> { + // A dotted path needs client-side validation because the server may + // report an internal type mismatch when the path crosses a list. Use + // an already cached schema for top-level list fields as well. + if column.contains('.') || self.schema_cache.try_get().is_some() { + let schema = self.schema().await?; + crate::blob::reject_list_blob_column(schema.as_ref(), column)?; + } + Ok(()) + } + + async fn explain_list_blob_bad_request(&self, column: &str, error: Error) -> Error { + let is_bad_request = match &error { + Error::Http { + status_code: Some(code), + .. + } => *code == StatusCode::BAD_REQUEST || *code == StatusCode::UNPROCESSABLE_ENTITY, + _ => false, + }; + if is_bad_request + && let Ok(schema) = self.schema().await + && let Err(list_error) = crate::blob::reject_list_blob_column(schema.as_ref(), column) + { + return list_error; + } + error + } + /// Blob v2 columns are marked in field metadata, which `describe` returns. Reading /// them from the cached schema needs no route of its own and no version gate. pub(super) async fn blob_columns_impl(&self) -> Result> { @@ -387,6 +415,7 @@ impl RemoteTable { column: &str, row_ids: &[u64], ) -> Result { + self.reject_list_blob_path_if_known(column).await?; // Empty requests do not require blob-route support. if row_ids.is_empty() { return Ok(LargeBinaryArray::from(Vec::>::new())); @@ -466,7 +495,10 @@ impl RemoteTable { let (request_id, response) = self .send_with_freshness(request, true, read_snapshot.freshness) .await?; - let mut stream = self.read_arrow_response(&request_id, response).await?; + let mut stream = match self.read_arrow_response(&request_id, response).await { + Ok(stream) => stream, + Err(error) => return Err(self.explain_list_blob_bad_request(column, error).await), + }; let mut blob_chunks: Vec> = Vec::new(); while let Some(batch) = stream.try_next().await? { @@ -535,6 +567,7 @@ impl RemoteTable { column: &str, row_ids: &[u64], ) -> Result>> { + self.reject_list_blob_path_if_known(column).await?; // Empty requests do not require blob-route support. if row_ids.is_empty() { return Ok(Vec::new()); @@ -542,9 +575,13 @@ impl RemoteTable { self.ensure_blob_files_supported()?; let mut read_snapshot = self.snapshot_read_state().await; - let sizes = self + let sizes = match self .take_blob_descriptor_sizes(column, row_ids, &mut read_snapshot) - .await?; + .await + { + Ok(sizes) => sizes, + Err(error) => return Err(self.explain_list_blob_bad_request(column, error).await), + }; let version = read_snapshot.version; let mut files = Vec::with_capacity(row_ids.len()); @@ -568,7 +605,10 @@ impl RemoteTable { } } } - let probed = probe_blob_files(probe_requesters).await?; + let probed = match probe_blob_files(probe_requesters).await { + Ok(files) => files, + Err(error) => return Err(self.explain_list_blob_bad_request(column, error).await), + }; for (index, file) in probe_indices.into_iter().zip(probed) { files[index] = file; } @@ -589,7 +629,10 @@ impl RemoteTable { .iter() .map(|row_id| self.blob_range_requester(column, *row_id, &read_snapshot)) .collect(); - probe_blob_files(requesters).await + match probe_blob_files(requesters).await { + Ok(files) => Ok(files), + Err(error) => Err(self.explain_list_blob_bad_request(column, error).await), + } } /// Read the blob descriptor of every unique row in `row_ids`. @@ -696,6 +739,15 @@ impl RemoteTable { let values = batch.column_by_name(column).ok_or_else(|| { invalid_response(format!("blob descriptor take is missing '{column}'")) })?; + // An uncached top-level list can reach the descriptor query. Explain + // that unsupported path before the generic descriptor type check. + if matches!( + values.data_type(), + DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _) + ) && let Ok(schema) = self.schema().await + { + crate::blob::reject_list_blob_column(schema.as_ref(), column)?; + } // Only blob v2 columns read back as descriptors. let descriptors = values .as_any() diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 3a833b293..91a28fd12 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -1241,10 +1241,11 @@ impl Table { self.inner.count_rows(filter).await } - /// Names of the blob v2 columns in this table, in declaration order. + /// Names of the row-addressable blob v2 columns, in declaration order. /// - /// Nested blobs use dotted paths (e.g. `info.blob`). Returns - /// [`Error::NotSupported`] on table types without blob support. + /// Blobs nested in structs use dotted paths (e.g. `info.blob`). Blobs + /// inside lists are omitted because row ids cannot identify their elements. + /// Returns [`Error::NotSupported`] on table types without blob support. pub async fn blob_columns(&self) -> Result> { self.inner.blob_columns().await } @@ -1284,9 +1285,10 @@ impl Table { /// # } /// ``` /// - /// Returns [`Error::InvalidInput`] when the column does not exist or is - /// not a blob v2 column, and [`Error::NotSupported`] on table types - /// without blob support. + /// Blobs inside lists cannot be fetched by row id. Returns + /// [`Error::InvalidInput`] for those paths, when the column does not exist, + /// or when it is not a blob v2 column. Returns [`Error::NotSupported`] on + /// table types without blob support. pub async fn fetch_blobs( &self, column: impl AsRef, diff --git a/rust/lancedb/tests/blob_integration.rs b/rust/lancedb/tests/blob_integration.rs index 0c9942db6..063690129 100644 --- a/rust/lancedb/tests/blob_integration.rs +++ b/rust/lancedb/tests/blob_integration.rs @@ -618,6 +618,47 @@ async fn blob_columns_lists_nested_dotted_paths() -> Result<()> { Ok(()) } +#[tokio::test] +async fn blob_columns_excludes_list_elements_and_fetches_explain_limitation() -> Result<()> { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().to_str().unwrap()).execute().await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + blob("thumbnail", true), + Field::new("imgs", DataType::List(Arc::new(blob("item", true))), true), + ])); + let table = db + .create_empty_table("list_blobs", schema) + .execute() + .await?; + + assert_eq!(table.blob_columns().await?, vec!["thumbnail"]); + for column in ["imgs", "imgs.item"] { + let err = table.fetch_blobs(column, &[0]).await.unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. })); + assert!( + err.to_string() + .contains("blobs inside lists cannot be fetched") + ); + + let err = table.fetch_blob_files(column, &[0]).await.unwrap_err(); + assert!( + err.to_string() + .contains("blobs inside lists cannot be fetched") + ); + + let err = table + .fetch_blob_ranges(column, [BlobRangeRequest::new(0, 0, 1)]) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("blobs inside lists cannot be fetched") + ); + } + Ok(()) +} + #[tokio::test] async fn blob_columns_lists_blob_fields_in_order() -> Result<()> { let tmp = tempdir().unwrap(); From 8bb04d0fee9a3a42071b25c312f5453dadb93745 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:04:45 -0700 Subject: [PATCH 07/19] fix: preserve remote projection column order (#4292) --- Cargo.lock | 1 + Cargo.toml | 2 +- rust/lancedb/src/remote/table.rs | 129 +++++++++++++++++++++++++++++-- 3 files changed, 126 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6747eb08..0515319b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9446,6 +9446,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", diff --git a/Cargo.toml b/Cargo.toml index 78ed6bea8..5033b1529 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ oauth2 = { version = "5.0", default-features = false } regex = "1.10" semver = "1.0.25" serde = "1" -serde_json = "1" +serde_json = { version = "1", features = ["preserve_order"] } tempfile = "3.5.0" tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] } tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] } diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 27a1d90fd..6467f6616 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -1057,6 +1057,8 @@ impl RemoteTable { ); } Select::Dynamic(pairs) => { + // Keep the caller's alias order in the JSON object sent to the server. + // serde_json's preserve_order feature is required here. let alias_map = serde_json::Map::from_iter(pairs.iter().map(|(name, expr)| { (name.clone(), serde_json::Value::String(expr.clone())) @@ -1402,24 +1404,29 @@ impl RemoteTable { }); let streams = futures::future::try_join_all(futures); - if let Some(timeout) = options.timeout { + let streams = if let Some(timeout) = options.timeout { let timeout_future = tokio::time::sleep(timeout); tokio::pin!(timeout_future); tokio::pin!(streams); tokio::select! { _ = &mut timeout_future => { - Err(Error::Other { + return Err(Error::Other { message: format!("Query timeout after {} ms", timeout.as_millis()), source: None, }) } result = &mut streams => { - Ok(result?) + result? } } } else { - Ok(streams.await?) - } + streams.await? + }; + + Ok(streams + .into_iter() + .map(|stream| restore_projection_order(stream, &query.base().select)) + .collect()) } fn prepare_query_bodies( @@ -1461,6 +1468,52 @@ impl RemoteTable { } } +fn restore_projection_order( + stream: Pin>, + select: &Select, +) -> Pin> { + let pairs = match select { + Select::Dynamic(pairs) => pairs.iter().map(|(name, _)| name).collect::>(), + Select::Expr(pairs) => pairs.iter().map(|(name, _)| name).collect::>(), + _ => return stream, + }; + + let schema = stream.schema(); + let mut remaining = (0..schema.fields().len()).collect::>(); + let mut indices = Vec::with_capacity(remaining.len()); + for name in pairs { + if let Some(position) = remaining + .iter() + .position(|&index| schema.field(index).name() == name) + { + indices.push(remaining.remove(position)); + } + } + indices.extend(remaining); + if indices + .iter() + .enumerate() + .all(|(index, &field)| index == field) + { + return stream; + } + + // Older servers may sort alias maps even when the request preserves their + // order. Reorder the schema and every batch while keeping server-added fields. + let fields = indices + .iter() + .map(|&index| schema.field(index).clone()) + .collect::>(); + let schema = Arc::new(arrow_schema::Schema::new_with_metadata( + fields, + schema.metadata().clone(), + )); + let batches = stream.map(move |batch| { + batch.and_then(|batch| batch.project(&indices).map_err(DataFusionError::from)) + }); + Box::pin(RecordBatchStreamAdapter::new(schema, batches)) +} + #[derive(Deserialize)] struct TableDescription { version: u64, @@ -6544,6 +6597,72 @@ mod tests { .unwrap(); } + #[rstest] + #[case::sql(Select::dynamic(&[("p", "x"), ("id", "id")]))] + #[case::typed(Select::expr_projection(&[ + ("p", crate::expr::col("x")), + ("id", crate::expr::col("id")), + ]))] + #[tokio::test] + async fn test_query_dynamic_projection_order(#[case] select: Select) { + let table = Table::new_with_handler("my_table", |request| { + assert_eq!(request.url().path(), "/v1/table/my_table/query/"); + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); + let aliases = body["columns"] + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect::>(); + assert_eq!(aliases, ["p", "id"]); + + // Simulate a server that returns aliases in alphabetical order. + let data = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("p", DataType::Int32, false), + ])), + vec![ + Arc::new(Int32Array::from(vec![7])), + Arc::new(Int32Array::from(vec![42])), + ], + ) + .unwrap(); + http::Response::builder() + .status(200) + .header(CONTENT_TYPE, ARROW_FILE_CONTENT_TYPE) + .body(write_ipc_file(&data)) + .unwrap() + }); + + let batches = table + .query() + .select(select) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches.len(), 1); + let schema = batches[0].schema(); + assert_eq!( + schema + .fields() + .iter() + .map(|field| field.name().as_str()) + .collect::>(), + ["p", "id"] + ); + let projected = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(projected.value(0), 42); + } + #[tokio::test] async fn test_query_vector_nested_field_path() { let expected_data = RecordBatch::try_new( From 3336bdfbeca450afc30be5bb7f48be970277f865 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:27:25 -0700 Subject: [PATCH 08/19] fix(remote): bound concurrent blob reads across handles (#4311) --- docs/src/js/interfaces/ClientConfig.md | 12 + nodejs/__test__/remote.test.ts | 9 + nodejs/src/remote.rs | 5 + python/python/lancedb/remote/__init__.py | 5 + python/python/lancedb/remote/db.py | 1 + python/python/tests/test_remote_db.py | 5 + python/src/connection.rs | 2 + rust/lancedb/src/error.rs | 2 +- rust/lancedb/src/remote/client.rs | 62 +++++ rust/lancedb/src/remote/retry.rs | 23 ++ rust/lancedb/src/remote/table.rs | 4 +- rust/lancedb/src/remote/table/blobs.rs | 279 +++++++++++++++-------- rust/lancedb/src/table.rs | 3 + 13 files changed, 317 insertions(+), 95 deletions(-) diff --git a/docs/src/js/interfaces/ClientConfig.md b/docs/src/js/interfaces/ClientConfig.md index 2b4984b8d..d5ce32117 100644 --- a/docs/src/js/interfaces/ClientConfig.md +++ b/docs/src/js/interfaces/ClientConfig.md @@ -8,6 +8,18 @@ ## Properties +### blobRequestConcurrency? + +```ts +optional blobRequestConcurrency: number; +``` + +Maximum number of concurrent blob HTTP requests across all tables and +blob handles on this connection. Defaults to 8. Can also be set with +`LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. Must be greater than zero. + +*** + ### extraHeaders? ```ts diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 6911b24b9..4f70ac1aa 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -480,6 +480,15 @@ describe("remote connection", () => { ); }); + it("rejects a zero blob request concurrency limit", async () => { + await expect( + connect("db://dev", { + apiKey: "fake", + clientConfig: { blobRequestConcurrency: 0 }, + }), + ).rejects.toThrow("blob_request_concurrency"); + }); + it("shows the full error messages on retry errors", async () => { await withMockDatabase( (_req, res) => { diff --git a/nodejs/src/remote.rs b/nodejs/src/remote.rs index c2d671735..d7e2a21cd 100644 --- a/nodejs/src/remote.rs +++ b/nodejs/src/remote.rs @@ -92,6 +92,10 @@ pub struct ClientConfig { pub user_agent: Option, pub retry_config: Option, pub timeout_config: Option, + /// Maximum number of concurrent blob HTTP requests across all tables and + /// blob handles on this connection. Defaults to 8. Can also be set with + /// `LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. Must be greater than zero. + pub blob_request_concurrency: Option, pub extra_headers: Option>, /// The delimiter joining a namespace path and a name into one object /// identifier. `"$"` is the only supported value, and leaving this unset is @@ -428,6 +432,7 @@ impl From for lancedb::remote::ClientConfig { .unwrap_or(concat!("LanceDB-Node-Client/", env!("CARGO_PKG_VERSION")).to_string()), retry_config: config.retry_config.map(Into::into).unwrap_or_default(), timeout_config: config.timeout_config.map(Into::into).unwrap_or_default(), + blob_request_concurrency: config.blob_request_concurrency.map(|value| value as usize), extra_headers: config.extra_headers.unwrap_or_default(), id_delimiter: config.id_delimiter, tls_config: config.tls_config.map(Into::into), diff --git a/python/python/lancedb/remote/__init__.py b/python/python/lancedb/remote/__init__.py index dbcaa1b64..a1c8ebee5 100644 --- a/python/python/lancedb/remote/__init__.py +++ b/python/python/lancedb/remote/__init__.py @@ -170,6 +170,10 @@ class ClientConfig: Configuration for retrying failed requests. timeout_config: Optional[TimeoutConfig] Configuration for request timeouts. + blob_request_concurrency: Optional[int] + Maximum concurrent blob HTTP requests across all handles on this + connection. Defaults to 8. Set a positive integer here or via + `LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. extra_headers: Optional[dict] Additional headers to include in requests. id_delimiter: Optional[str] @@ -193,6 +197,7 @@ class ClientConfig: user_agent: str = f"LanceDB-Python-Client/{__version__}" retry_config: RetryConfig = field(default_factory=RetryConfig) timeout_config: Optional[TimeoutConfig] = field(default_factory=TimeoutConfig) + blob_request_concurrency: Optional[int] = None extra_headers: Optional[dict] = None id_delimiter: Optional[str] = None tls_config: Optional[TlsConfig] = None diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index 0f3d35fc7..ac6b8b99d 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -112,6 +112,7 @@ def _client_config_to_dict(config: ClientConfig) -> dict[str, Any]: "user_agent": config.user_agent, "retry_config": _retry_config_to_dict(config.retry_config), "timeout_config": _timeout_config_to_dict(config.timeout_config), + "blob_request_concurrency": config.blob_request_concurrency, "extra_headers": config.extra_headers, "id_delimiter": config.id_delimiter, "tls_config": _tls_config_to_dict(config.tls_config), diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 3e66ac0e0..dbf64f4bb 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -2728,6 +2728,11 @@ def test_create_client(): assert isinstance(db.client_config, ClientConfig) assert db.client_config.retry_config.retries == 42 + db = lancedb.connect( + **mandatory_args, client_config={"blob_request_concurrency": 3} + ) + assert db.client_config.blob_request_concurrency == 3 + with pytest.warns(DeprecationWarning): db = lancedb.connect(**mandatory_args, connection_timeout=42) assert db.client_config.timeout_config.connect_timeout == timedelta(seconds=42) diff --git a/python/src/connection.rs b/python/src/connection.rs index ec0768eb2..9ac081b9d 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -1220,6 +1220,7 @@ pub struct PyClientConfig { user_agent: String, retry_config: Option, timeout_config: Option, + blob_request_concurrency: Option, extra_headers: Option>, id_delimiter: Option, tls_config: Option, @@ -1305,6 +1306,7 @@ impl From for lancedb::remote::ClientConfig { user_agent: value.user_agent, retry_config: value.retry_config.map(Into::into).unwrap_or_default(), timeout_config: value.timeout_config.map(Into::into).unwrap_or_default(), + blob_request_concurrency: value.blob_request_concurrency, extra_headers: value.extra_headers.unwrap_or_default(), id_delimiter: value.id_delimiter, tls_config: value.tls_config.map(Into::into), diff --git a/rust/lancedb/src/error.rs b/rust/lancedb/src/error.rs index f6a2df9a6..6052b67d8 100644 --- a/rust/lancedb/src/error.rs +++ b/rust/lancedb/src/error.rs @@ -127,7 +127,7 @@ pub enum Error { "Hit retry limit for request_id={request_id} (\ request_failures={request_failures}/{max_request_failures}, \ connect_failures={connect_failures}/{max_connect_failures}, \ - read_failures={read_failures}/{max_read_failures})" + read_failures={read_failures}/{max_read_failures}): {source}" ))] Retry { request_id: String, diff --git a/rust/lancedb/src/remote/client.rs b/rust/lancedb/src/remote/client.rs index 5f24d24c7..783d2ef84 100644 --- a/rust/lancedb/src/remote/client.rs +++ b/rust/lancedb/src/remote/client.rs @@ -8,6 +8,7 @@ use reqwest::{ header::{HeaderMap, HeaderValue}, }; use std::{collections::HashMap, future::Future, str::FromStr, sync::Arc, time::Duration}; +use tokio::sync::Semaphore; use crate::error::{Error, Result}; use crate::remote::db::RemoteOptions; @@ -15,6 +16,7 @@ use crate::remote::retry::{ResolvedRetryConfig, RetryCounter}; use crate::utils::validate_database_name; const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id"); +const DEFAULT_BLOB_REQUEST_CONCURRENCY: usize = 8; pub fn redact_sensitive_headers(headers: &mut HeaderMap) { const SENSITIVE_HEADERS: [&str; 5] = [ @@ -86,6 +88,11 @@ const DEFAULT_MAX_REQUEST_DURATION_DIVISOR: u32 = 2; pub struct ClientConfig { pub timeout_config: TimeoutConfig, pub retry_config: RetryConfig, + /// Maximum concurrent blob HTTP requests for this connection, shared by all + /// tables and blob handles. Defaults to 8. The environment variable + /// `LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY` can also set this limit. + /// The value must be greater than zero. + pub blob_request_concurrency: Option, /// User agent to use for requests. The default provides the library /// name and version. pub user_agent: String, @@ -140,6 +147,7 @@ impl std::fmt::Debug for ClientConfig { f.debug_struct("ClientConfig") .field("timeout_config", &self.timeout_config) .field("retry_config", &self.retry_config) + .field("blob_request_concurrency", &self.blob_request_concurrency) .field("user_agent", &self.user_agent) .field("extra_headers", &self.extra_headers) .field("id_delimiter", &self.id_delimiter) @@ -160,6 +168,7 @@ impl Default for ClientConfig { Self { timeout_config: TimeoutConfig::default(), retry_config: RetryConfig::default(), + blob_request_concurrency: None, user_agent: concat!("LanceDB-Rust-Client/", env!("CARGO_PKG_VERSION")).into(), extra_headers: HashMap::new(), id_delimiter: None, @@ -308,6 +317,7 @@ pub struct RestfulLanceDbClient { client: reqwest::Client, host: String, pub(crate) retry_config: ResolvedRetryConfig, + pub(crate) blob_request_semaphore: Arc, pub(crate) sender: S, pub(crate) header_provider: Option>, /// Connection-level read consistency interval. Drives the @@ -469,6 +479,11 @@ impl ClientConfig { /// mistake is reported where it was made rather than as a confusing /// response later. Public so a caller can ask without connecting. pub fn validate(&self) -> Result<()> { + if self.blob_request_concurrency == Some(0) { + return Err(Error::InvalidInput { + message: "blob_request_concurrency must be greater than zero".into(), + }); + } if let Some(delimiter) = &self.id_delimiter { validate_id_delimiter(delimiter)?; } @@ -504,6 +519,8 @@ impl RestfulLanceDbClient { // Before anything is built from it, so the error names the caller's // configuration rather than a request. client_config.validate()?; + let blob_request_concurrency = + Self::resolve_blob_request_concurrency(client_config.blob_request_concurrency)?; // Get the timeouts let timeout = @@ -603,6 +620,7 @@ impl RestfulLanceDbClient { client, host, retry_config, + blob_request_semaphore: Arc::new(Semaphore::new(blob_request_concurrency)), sender: Sender, header_provider: client_config.header_provider, read_consistency_interval, @@ -611,6 +629,26 @@ impl RestfulLanceDbClient { }) } + fn resolve_blob_request_concurrency(passed: Option) -> Result { + let value = if let Some(value) = passed { + value + } else if let Ok(env) = std::env::var("LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY") { + env.parse::().map_err(|_| Error::InvalidInput { + message: format!( + "LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY must be a positive integer, got '{env}'" + ), + })? + } else { + DEFAULT_BLOB_REQUEST_CONCURRENCY + }; + if value == 0 { + return Err(Error::InvalidInput { + message: "blob request concurrency must be greater than zero".into(), + }); + } + Ok(value) + } + /// Resolve the max bytes per insert request from config, environment, or the /// default. A value of `0` (from either source) disables request splitting. fn resolve_max_bytes_per_request(passed: Option) -> Result> { @@ -1106,6 +1144,7 @@ pub mod test_utils { client: reqwest::Client::new(), host: "http://localhost".to_string(), retry_config: RetryConfig::default().try_into().unwrap(), + blob_request_semaphore: Arc::new(Semaphore::new(DEFAULT_BLOB_REQUEST_CONCURRENCY)), sender: MockSender { f: Arc::new(wrapper), }, @@ -1132,6 +1171,11 @@ pub mod test_utils { client: reqwest::Client::new(), host: "http://localhost".to_string(), retry_config: config.retry_config.try_into().unwrap(), + blob_request_semaphore: Arc::new(Semaphore::new( + config + .blob_request_concurrency + .unwrap_or(DEFAULT_BLOB_REQUEST_CONCURRENCY), + )), sender: MockSender { f: Arc::new(wrapper), }, @@ -1183,6 +1227,21 @@ mod tests { .expect_err("a configured delimiter other than the supported one must be refused"); } + #[test] + fn test_blob_request_concurrency_must_be_positive() { + let error = super::ClientConfig { + blob_request_concurrency: Some(0), + ..Default::default() + } + .validate() + .unwrap_err(); + assert!(error.to_string().contains("blob_request_concurrency")); + assert_eq!( + RestfulLanceDbClient::::resolve_blob_request_concurrency(Some(3)).unwrap(), + 3 + ); + } + use super::*; use serial_test::serial; use std::time::Duration; @@ -1515,6 +1574,7 @@ mod tests { client: reqwest::Client::new(), host: "https://example.com".to_string(), retry_config: RetryConfig::default().try_into().unwrap(), + blob_request_semaphore: Arc::new(Semaphore::new(DEFAULT_BLOB_REQUEST_CONCURRENCY)), sender: Sender, header_provider: Some(Arc::new(provider) as Arc), read_consistency_interval: None, @@ -1553,6 +1613,7 @@ mod tests { client: reqwest::Client::new(), host: "https://example.com".to_string(), retry_config: RetryConfig::default().try_into().unwrap(), + blob_request_semaphore: Arc::new(Semaphore::new(DEFAULT_BLOB_REQUEST_CONCURRENCY)), sender: Sender, header_provider: Some(Arc::new(provider) as Arc), read_consistency_interval: None, @@ -1617,6 +1678,7 @@ mod tests { client: reqwest::Client::new(), host: "https://example.com".to_string(), retry_config: RetryConfig::default().try_into().unwrap(), + blob_request_semaphore: Arc::new(Semaphore::new(DEFAULT_BLOB_REQUEST_CONCURRENCY)), sender: Sender, header_provider: Some(Arc::new(provider) as Arc), read_consistency_interval: None, diff --git a/rust/lancedb/src/remote/retry.rs b/rust/lancedb/src/remote/retry.rs index 1a354eebb..19aa301e4 100644 --- a/rust/lancedb/src/remote/retry.rs +++ b/rust/lancedb/src/remote/retry.rs @@ -198,6 +198,29 @@ mod tests { assert_eq!(counter.connect_failures, 0); } + #[test] + fn retry_limit_message_includes_last_status_error() { + let config = test_config(); + let mut counter = RetryCounter::new(&config, "test".to_string()); + for _ in 0..2 { + counter + .increment_request_failures(crate::Error::Http { + source: "502 Bad Gateway".into(), + request_id: "test".to_string(), + status_code: Some(reqwest::StatusCode::BAD_GATEWAY), + }) + .unwrap(); + } + let error = counter + .increment_request_failures(crate::Error::Http { + source: "last response: 502 Bad Gateway".into(), + request_id: "test".to_string(), + status_code: Some(reqwest::StatusCode::BAD_GATEWAY), + }) + .unwrap_err(); + assert!(error.to_string().contains("last response: 502 Bad Gateway")); + } + #[tokio::test] async fn test_increment_from_error_respects_global_limits() { // If request_failures is already at max, a connect error should still diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 6467f6616..7d7a6301a 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -5660,7 +5660,7 @@ mod tests { .header(VERSION_HEADER, "42") .body(b"0".to_vec()) .unwrap(), - "bytes=0-" => http::Response::builder() + "bytes=0-9" => http::Response::builder() .status(206) .header(reqwest::header::CONTENT_RANGE, "bytes 0-9/10") .body(b"0123456789".to_vec()) @@ -5682,7 +5682,7 @@ mod tests { assert_eq!(blobs.value(4), b"bbb"); let requests = requests.lock().unwrap(); assert!(requests.contains(&"GET bytes=0-0".to_string())); - assert!(requests.contains(&"GET bytes=0-".to_string())); + assert!(requests.contains(&"GET bytes=0-9".to_string())); } #[tokio::test] diff --git a/rust/lancedb/src/remote/table/blobs.rs b/rust/lancedb/src/remote/table/blobs.rs index 9db901838..901616fb2 100644 --- a/rust/lancedb/src/remote/table/blobs.rs +++ b/rust/lancedb/src/remote/table/blobs.rs @@ -11,13 +11,13 @@ use std::time::Duration; use arrow_array::{Array, LargeBinaryArray, StructArray, UInt8Array, UInt64Array}; use arrow_schema::DataType; -use bytes::{Bytes, BytesMut}; +use bytes::Bytes; use datafusion_expr::{col, lit}; use futures::{StreamExt, TryStreamExt}; use lance::dataset::ROW_ID; use lance_core::datatypes::BlobKind; use reqwest::{Response, StatusCode, header}; -use tokio::sync::Mutex; +use tokio::sync::{Mutex, OwnedSemaphorePermit}; use crate::Error; use crate::blob::BlobFile; @@ -57,7 +57,26 @@ trait BlobRangeRequester: Send + Sync + std::fmt::Debug { range_header: &str, mode: RangeRequestMode, version: Option, - ) -> Result<(String, Response)>; + ) -> Result; +} + +/// Keep the connection slot until the response body has been consumed or dropped. +#[derive(Debug)] +struct BlobRangeResponse { + request_id: String, + response: Response, + _permit: Option, +} + +#[cfg(test)] +impl From<(String, Response)> for BlobRangeResponse { + fn from((request_id, response): (String, Response)) -> Self { + Self { + request_id, + response, + _permit: None, + } + } } #[derive(Debug)] @@ -79,7 +98,14 @@ impl BlobRangeRequester for TableBlobRangeRequester { range_header: &str, mode: RangeRequestMode, version: Option, - ) -> Result<(String, Response)> { + ) -> Result { + let permit = self + .client + .blob_request_semaphore + .clone() + .acquire_owned() + .await + .expect("blob request semaphore is never closed"); let freshness_request = freshness_headers_snapshot(&self.freshness, self.read_consistency_interval); let mut request = freshness_request @@ -96,32 +122,32 @@ impl BlobRangeRequester for TableBlobRangeRequester { if response.status() == StatusCode::RANGE_NOT_SATISFIABLE && matches!(mode, RangeRequestMode::SizeProbe) { - return Ok((request_id, response)); + return Ok(BlobRangeResponse { + request_id, + response, + _permit: Some(permit), + }); } let response = self.client.check_response(&request_id, response).await?; freshness_request.observe_headers(&self.freshness, response.headers()); self.parent_freshness_request .observe_headers(&self.parent_freshness, response.headers()); - Ok((request_id, response)) + Ok(BlobRangeResponse { + request_id, + response, + _permit: Some(permit), + }) } } -#[derive(Debug)] -struct SequentialResponse { - response: Response, - request_id: String, - buffered: Bytes, -} - #[derive(Debug, Default)] struct RemoteBlobState { cursor: u64, - sequential_response: Option, } /// Seekable Cloud blob handle over HTTP Range. /// -/// Nonempty handles read the table version returned by their size probe. +/// Handles read the table version used to fetch their descriptor or probe their size. #[derive(Debug)] pub struct RemoteBlobFile { requester: Arc, @@ -145,11 +171,6 @@ impl RemoteBlobFile { /// Close the handle without waiting for an in-flight read. pub(crate) async fn close(&self) -> lance_core::Result<()> { self.closed.store(true, Ordering::Release); - // Drop a retained response when the state lock is immediately available. - // A reader holding the lock drops it instead once it observes the flag. - if let Ok(mut state) = self.state.try_lock() { - state.sequential_response = None; - } Ok(()) } @@ -185,7 +206,11 @@ impl RemoteBlobFile { return Ok(Bytes::new()); } let range_header = format!("bytes={}-{}", range.start, range.end - 1); - let (request_id, response) = self + let BlobRangeResponse { + request_id, + response, + _permit, + } = self .requester .request_range(&range_header, RangeRequestMode::DataRead, self.version) .await @@ -252,62 +277,20 @@ impl RemoteBlobFile { return Ok(Bytes::new()); } - // Remove the retained response from shared state before awaiting. Failed or - // cancelled reads leave the committed cursor unchanged and force the next - // read to open a fresh response. - let mut sequential_response = state.sequential_response.take(); - let mut cursor = state.cursor; - let mut output = BytesMut::with_capacity(target_len); - while output.len() < target_len { - if sequential_response.is_none() { - let range_header = format!("bytes={cursor}-"); - let (request_id, response) = self - .requester - .request_range(&range_header, RangeRequestMode::DataRead, self.version) - .await - .map_err(remote_blob_error)?; - self.ensure_open()?; - validate_partial_response(&response, cursor..self.size, self.size)?; - sequential_response = Some(SequentialResponse { - response, - request_id, - buffered: Bytes::new(), - }); - } - - let needed = target_len - output.len(); - let active = sequential_response.as_mut().unwrap(); - if !active.buffered.is_empty() { - let take = needed.min(active.buffered.len()); - output.extend_from_slice(&active.buffered.split_to(take)); - cursor += take as u64; - continue; - } - let chunk = active - .response - .chunk() - .await - .err_to_http(active.request_id.clone()) - .map_err(remote_blob_error)?; - self.ensure_open()?; - let chunk = chunk.ok_or_else(|| { - remote_blob_error("response ended before the requested blob range") - })?; - active.buffered = chunk; - } + // A partial read must release its HTTP response before returning. Keeping + // it open across calls would hold a connection slot indefinitely and + // prevent other handles from making progress. + let end = state.cursor + target_len as u64; + let bytes = self.read_range(state.cursor..end).await?; self.ensure_open()?; - state.cursor = cursor; - if state.cursor < self.size { - state.sequential_response = sequential_response; - } - Ok(output.freeze()) + state.cursor = end; + Ok(bytes) } pub(crate) async fn seek(&self, new_cursor: u64) -> lance_core::Result<()> { self.ensure_open()?; let mut state = self.state.lock().await; self.ensure_open()?; - state.sequential_response = None; state.cursor = new_cursor; Ok(()) } @@ -869,7 +852,11 @@ fn probe_blob_version(response: &Response, request_id: &str) -> Result { /// /// `204` represents null. `416` with `bytes */0` represents an empty blob. async fn probe_blob_file(requester: Arc) -> Result> { - let (request_id, response) = requester + let BlobRangeResponse { + request_id, + response, + _permit, + } = requester .request_range("bytes=0-0", RangeRequestMode::SizeProbe, None) .await?; match response.status() { @@ -1424,6 +1411,114 @@ mod tests { assert!(requests.lock().unwrap().contains(&"bytes=5-11".to_string())); } + #[tokio::test(flavor = "multi_thread", worker_threads = 8)] + async fn blob_requests_are_bounded_across_handles() { + let in_flight = Arc::new(AtomicUsize::new(0)); + let max_in_flight = Arc::new(AtomicUsize::new(0)); + let table = RemoteTable::new_mock_with_version_and_config( + "my_table".to_string(), + { + let in_flight = in_flight.clone(); + let max_in_flight = max_in_flight.clone(); + move |request| { + let range = request.headers().get(header::RANGE).unwrap(); + if range != "bytes=0-0" { + let now = in_flight.fetch_add(1, Ordering::SeqCst) + 1; + max_in_flight.fetch_max(now, Ordering::SeqCst); + std::thread::sleep(Duration::from_millis(20)); + in_flight.fetch_sub(1, Ordering::SeqCst); + } + range_response(&request, PAYLOAD) + } + }, + Some(Version::new(0, 5, 0)), + crate::remote::ClientConfig { + blob_request_concurrency: Some(3), + ..Default::default() + }, + ); + let files = table + .probe_files("image", &(0..16).collect::>()) + .await + .unwrap(); + let start = Arc::new(tokio::sync::Barrier::new(files.len() + 1)); + let reads = files.into_iter().map(|file| { + let start = start.clone(); + tokio::spawn(async move { + start.wait().await; + file.unwrap().read_range(1..2).await + }) + }); + let reads = reads.collect::>(); + start.wait().await; + for result in futures::future::join_all(reads).await { + assert_eq!(result.unwrap().unwrap(), &PAYLOAD[1..2]); + } + let max = max_in_flight.load(Ordering::SeqCst); + assert!(max > 1, "reads never overlapped"); + assert!( + max <= 3, + "{max} blob requests exceeded the configured limit" + ); + } + + #[tokio::test] + async fn partial_reads_release_slots_for_other_handles() { + let table = RemoteTable::new_mock_with_version_and_config( + "my_table".to_string(), + |request| range_response(&request, PAYLOAD), + Some(Version::new(0, 5, 0)), + crate::remote::ClientConfig { + blob_request_concurrency: Some(1), + ..Default::default() + }, + ); + let files = table.probe_files("image", &[10, 20]).await.unwrap(); + let reads = files + .iter() + .map(|file| file.as_ref().unwrap().read_up_to(1)); + let results = + tokio::time::timeout(Duration::from_secs(1), futures::future::join_all(reads)) + .await + .expect("an idle partial response held the only connection slot"); + for result in results { + assert_eq!(result.unwrap(), &PAYLOAD[..1]); + } + } + + #[tokio::test] + async fn blob_retry_error_includes_last_status() { + let table = RemoteTable::new_mock_with_version_and_config( + "my_table".to_string(), + |request| { + if request.headers().get(header::RANGE).unwrap() == "bytes=0-0" { + range_response(&request, PAYLOAD) + } else { + http::Response::builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .body(Vec::new()) + .unwrap() + } + }, + Some(Version::new(0, 5, 0)), + crate::remote::ClientConfig { + retry_config: crate::remote::RetryConfig { + retries: Some(1), + ..Default::default() + }, + ..Default::default() + }, + ); + let file = table.probe_files("image", &[10]).await.unwrap(); + let error = file[0] + .as_ref() + .unwrap() + .read_range(1..2) + .await + .unwrap_err(); + assert!(error.to_string().contains("503"), "got: {error}"); + } + #[tokio::test] async fn remote_blob_file_reads_the_probed_version_after_table_changes() { let changed = Arc::new(AtomicBool::new(false)); @@ -1480,8 +1575,11 @@ mod tests { [ ("bytes=0-0".to_string(), None), ("bytes=5-8".to_string(), Some("5".to_string())), - ("bytes=0-".to_string(), Some("5".to_string())), - ("bytes=10-".to_string(), Some("5".to_string())), + ("bytes=0-3".to_string(), Some("5".to_string())), + ( + format!("bytes=10-{}", PAYLOAD.len() - 1), + Some("5".to_string()) + ), ] ); } @@ -1527,7 +1625,7 @@ mod tests { } #[tokio::test] - async fn remote_blob_file_reuses_sequential_response_until_seek() { + async fn remote_blob_file_releases_response_after_each_partial_read() { let requests = Arc::new(StdMutex::new(Vec::new())); let table = mock_remote_blob_table(requests.clone()); let file = table @@ -1545,7 +1643,7 @@ mod tests { assert_eq!(file.tell().await.unwrap(), 24); assert_eq!( requests.lock().unwrap().as_slice(), - ["bytes=0-0", "bytes=0-", "bytes=20-"] + ["bytes=0-0", "bytes=0-3", "bytes=4-6", "bytes=20-23"] ); } @@ -1741,11 +1839,11 @@ mod tests { #[tokio::test] async fn remote_blob_file_failed_read_preserves_cursor_and_retries_fresh() { - let sequential_requests = Arc::new(AtomicUsize::new(0)); + let read_requests = Arc::new(AtomicUsize::new(0)); let table = RemoteTable::new_mock( "my_table".to_string(), { - let sequential_requests = sequential_requests.clone(); + let read_requests = read_requests.clone(); move |request| { let range = request .headers() @@ -1757,15 +1855,14 @@ mod tests { if range == "bytes=0-0" { return range_response(&request, PAYLOAD); } - let attempt = sequential_requests.fetch_add(1, Ordering::SeqCst); + let attempt = read_requests.fetch_add(1, Ordering::SeqCst); if attempt == 0 { - // End the response five bytes early to simulate a truncated - // sequential read. + // Return too few bytes for the requested range. return http::Response::builder() .status(StatusCode::PARTIAL_CONTENT) .header( header::CONTENT_RANGE, - format!("bytes 0-{}/{}", PAYLOAD.len() - 1, PAYLOAD.len()), + format!("bytes 0-9/{}", PAYLOAD.len()), ) .body(PAYLOAD[..5].to_vec()) .unwrap(); @@ -1785,9 +1882,7 @@ mod tests { let error = file.read_up_to(10).await.unwrap_err(); assert!( - error - .to_string() - .contains("response ended before the requested blob range"), + error.to_string().contains("returned 5 bytes, expected 10"), "got: {error}" ); assert_eq!(file.tell().await.unwrap(), 0); @@ -1795,7 +1890,7 @@ mod tests { // Retry from the last committed cursor with a fresh request. let retried = file.read_up_to(4).await.unwrap(); assert_eq!(retried, b"0123".as_slice()); - assert_eq!(sequential_requests.load(Ordering::SeqCst), 2); + assert_eq!(read_requests.load(Ordering::SeqCst), 2); } #[tokio::test] @@ -1828,7 +1923,7 @@ mod tests { range_header: &str, _mode: RangeRequestMode, _version: Option, - ) -> Result<(String, Response)> { + ) -> Result { assert_eq!(range_header, "bytes=0-0"); let now = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1; self.max_in_flight.fetch_max(now, Ordering::SeqCst); @@ -1845,7 +1940,7 @@ mod tests { .header(VERSION_HEADER, "5") .body(vec![0u8]) .unwrap(); - Ok((format!("probe-{}", self.index), Response::from(response))) + Ok((format!("probe-{}", self.index), Response::from(response)).into()) } } @@ -2009,7 +2104,7 @@ mod tests { range_header: &str, _mode: RangeRequestMode, _version: Option, - ) -> Result<(String, Response)> { + ) -> Result { let now = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1; self.max_in_flight.fetch_max(now, Ordering::SeqCst); tokio::time::sleep(Duration::from_millis(5)).await; @@ -2029,7 +2124,7 @@ mod tests { ) .body(PAYLOAD[start..=end].to_vec()) .unwrap(); - Ok(("range".to_string(), Response::from(response))) + Ok(("range".to_string(), Response::from(response)).into()) } } @@ -2130,8 +2225,8 @@ mod tests { range_header: &str, _mode: RangeRequestMode, _version: Option, - ) -> Result<(String, Response)> { - if range_header.ends_with('-') { + ) -> Result { + if range_header == "bytes=0-3" { self.started.notify_one(); self.release.wait().await; } @@ -2143,7 +2238,7 @@ mod tests { ) .body(PAYLOAD.to_vec()) .unwrap(); - Ok(("hung".to_string(), Response::from(response))) + Ok(("hung".to_string(), Response::from(response)).into()) } } diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 91a28fd12..ca58b54de 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -1346,6 +1346,9 @@ impl Table { /// /// Same length and order as `row_ids`. Null rows are `None`. Bytes are not /// read from disk until a call to [`BlobFile::read`]. + /// Remote blob requests share a connection-wide concurrency limit. Set + /// `ClientConfig::blob_request_concurrency` or the + /// `LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY` environment variable to tune it. /// /// `_rowid` values stay valid after compaction when the table has stable /// row ids. One missing or deleted row id rejects the whole request; no From a1e4dc9f911ac325dcc3151fc72d6e042ac6ad21 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:41:54 -0700 Subject: [PATCH 09/19] fix(python): align BlobFile seek and read behavior (#4290) --- python/python/lancedb/_blob.py | 42 ++++++++++++++++++++---- python/python/lancedb/_lancedb.pyi | 1 + python/python/tests/test_blob.py | 46 ++++++++++++++++++++++++++ python/python/tests/test_remote_db.py | 23 +++++++++++++ python/src/table.rs | 47 +++++++++++++++++++++------ 5 files changed, 142 insertions(+), 17 deletions(-) diff --git a/python/python/lancedb/_blob.py b/python/python/lancedb/_blob.py index 0dba427cc..d2d044f1b 100644 --- a/python/python/lancedb/_blob.py +++ b/python/python/lancedb/_blob.py @@ -34,14 +34,17 @@ FetchBlobsAsync = Callable[[str, pa.Table], Awaitable[pa.Array | pa.ChunkedArray class BlobFile(io.RawIOBase): """Seekable lazy handle from :meth:`~lancedb.table.Table.fetch_blob_files`. - Bytes load on ``read`` or ``read_range``, not when the handle is opened. - Use :meth:`aread` from async code. + Bytes load on ``read``, ``read1``, ``read_range``, or ``read_ranges``, not + when the handle is opened. Range reads leave the cursor unchanged. Use + :meth:`aread` from async code. """ def __init__(self, inner) -> None: self._inner = inner async def aread(self) -> bytes: + if self.closed: + raise ValueError("read of closed file") return await self._inner.read() def close(self) -> None: @@ -59,13 +62,16 @@ class BlobFile(io.RawIOBase): def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: if whence == io.SEEK_SET: - self._inner.seek(offset) + position = offset elif whence == io.SEEK_CUR: - self._inner.seek(self._inner.tell() + offset) + position = self._inner.tell() + offset elif whence == io.SEEK_END: - self._inner.seek(self._inner.size() + offset) + position = self._inner.size() + offset else: raise ValueError(f"invalid whence: {whence}") + if position < 0: + raise ValueError(f"negative seek value {position}") + self._inner.seek(position) return self._inner.tell() def tell(self) -> int: @@ -75,17 +81,39 @@ class BlobFile(io.RawIOBase): return self._inner.size() def readall(self) -> bytes: + if self.closed: + raise ValueError("read of closed file") return self._inner.read_bytes() - def read(self, size: int = -1) -> bytes: - if size == -1: + def read(self, size: Optional[int] = -1) -> bytes: + if self.closed: + raise ValueError("read of closed file") + if size is None or size < 0: return self._inner.read_bytes() return super().read(size) + def read1(self, size: Optional[int] = -1) -> bytes: + """Read up to ``size`` bytes with one underlying read.""" + if self.closed: + raise ValueError("read of closed file") + if size is None or size < 0: + size = io.DEFAULT_BUFFER_SIZE + return self._inner.read_up_to(size) + def read_range(self, offset: int, length: int) -> bytes: + if self.closed: + raise ValueError("read of closed file") return self._inner.read_range(offset, length) + def read_ranges(self, ranges: list[tuple[int, int]]) -> list[bytes]: + """Read ``(offset, length)`` pairs without moving the cursor.""" + if self.closed: + raise ValueError("read of closed file") + return self._inner.read_ranges(ranges) + def readinto(self, b: WriteableBuffer) -> int: + if self.closed: + raise ValueError("readinto of closed file") view = memoryview(b).cast("B") chunk = self._inner.read_up_to(len(view)) view[: len(chunk)] = chunk diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 38b80c14e..80554baf0 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -311,6 +311,7 @@ class BlobFile: def tell(self) -> int: ... def size(self) -> int: ... def read_range(self, offset: int, length: int) -> bytes: ... + def read_ranges(self, ranges: list[tuple[int, int]]) -> list[bytes]: ... def read_up_to(self, length: int) -> bytes: ... class Job: diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 73f9b7663..a70ed785b 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -1335,6 +1335,9 @@ async def test_async_fetch_blob_files_lazy_read(): handles = await table.fetch_blob_files("image", hits) assert len(handles) == 1 assert await handles[0].aread() == payload + handles[0].close() + with pytest.raises(ValueError, match="read of closed file"): + await handles[0].aread() def test_fetch_blobs_from_query_result_without_row_id_raises(): @@ -1470,6 +1473,49 @@ def test_blob_file_seek_read_and_read_range(): handle.seek(0, 99) +def test_blob_file_matches_file_error_and_range_contract(): + payload = b"abcdef" + table = _blob_table("file_contract", [{"id": 1, "image": payload}]) + row_id = _row_ids_by_id(table)[1] + handle = table.fetch_blob_files("image", [row_id])[0] + + handle.seek(2) + for offset, whence in [(-1, io.SEEK_SET), (-3, io.SEEK_CUR), (-7, io.SEEK_END)]: + with pytest.raises(ValueError, match="negative seek value -1"): + handle.seek(offset, whence) + assert handle.tell() == 2 + + assert handle.read(None) == payload[2:] + handle.seek(2) + assert handle.read1(2) == payload[2:4] + assert handle.read1() == payload[4:] + handle.seek(2) + assert handle.read_ranges([(0, 2), (4, 2), (2, 0)]) == [b"ab", b"ef", b""] + assert handle.tell() == 2 + + with pytest.raises(ValueError): + handle.read_range(5, 2) + with pytest.raises(ValueError): + handle.read_ranges([(0, 1), (5, 2)]) + assert handle.tell() == 2 + + handle.close() + with pytest.raises(ValueError, match="read of closed file"): + handle.read(1) + with pytest.raises(ValueError, match="read of closed file"): + handle.read(None) + with pytest.raises(ValueError, match="read of closed file"): + handle.read1(1) + with pytest.raises(ValueError, match="read of closed file"): + handle.readall() + with pytest.raises(ValueError, match="readinto of closed file"): + handle.readinto(bytearray(1)) + with pytest.raises(ValueError): + handle.read_range(0, 1) + with pytest.raises(ValueError): + handle.read_ranges([]) + + def test_fetch_blob_files_from_query_partial_read(): payload = _identifiable_payload(65536) table = _blob_table("query_partial", [{"id": 1, "image": payload}]) diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index dbf64f4bb..489d9f49b 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -4,6 +4,7 @@ from concurrent.futures import ThreadPoolExecutor import contextlib from datetime import timedelta import http.server +import io import json import multiprocessing as mp import pickle @@ -3503,6 +3504,28 @@ def test_remote_blob_files_are_lazy_seekable_handles(): alpha.read_range(0, 1) +def test_remote_blob_file_matches_file_error_and_range_contract(): + with blob_remote_table() as table: + handle = table.fetch_blob_files("image", [10])[0] + handle.seek(2) + + with pytest.raises(ValueError, match="negative seek value -1"): + handle.seek(-3, io.SEEK_CUR) + assert handle.tell() == 2 + assert handle.read(None) == b"pha" + + handle.seek(2) + assert handle.read1(2) == b"ph" + assert handle.read_ranges([(0, 2), (3, 2)]) == [b"al", b"ha"] + assert handle.tell() == 4 + with pytest.raises(ValueError): + handle.read_range(4, 2) + + handle.close() + with pytest.raises(ValueError, match="read of closed file"): + handle.read(1) + + def test_remote_blob_fetch_accepts_query_table(): hits = pa.table({"_rowid": pa.array([10, 20, 30], type=pa.uint64())}) diff --git a/python/src/table.rs b/python/src/table.rs index 7c0ec6610..23b48e3fd 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -697,6 +697,14 @@ impl PyBlobFile { } } +fn blob_file_error(operation: &str, error: lance_core::Error) -> pyo3::PyErr { + let message = format!("blob {operation} failed: {error}"); + match error { + lance_core::Error::InvalidInput { .. } => PyValueError::new_err(message), + _ => PyRuntimeError::new_err(message), + } +} + #[pymethods] impl PyBlobFile { fn read_bytes(self_: PyRef<'_, Self>) -> PyResult> { @@ -705,7 +713,7 @@ impl PyBlobFile { let py = self_.py(); let bytes = py .detach(move || block_on(async move { inner.read().await })) - .map_err(|e| PyRuntimeError::new_err(format!("blob read failed: {e}")))?; + .map_err(|e| blob_file_error("read", e))?; Ok(PyBytes::new(py, bytes.as_ref()).unbind()) } @@ -713,10 +721,7 @@ impl PyBlobFile { self_.ensure_open()?; let inner = self_.inner.clone(); future_into_py(self_.py(), async move { - let bytes = inner - .read() - .await - .map_err(|e| PyRuntimeError::new_err(format!("blob read failed: {e}")))?; + let bytes = inner.read().await.map_err(|e| blob_file_error("read", e))?; Python::attach(|py| Ok(PyBytes::new(py, bytes.as_ref()).unbind())) }) } @@ -739,7 +744,7 @@ impl PyBlobFile { self_ .py() .detach(move || block_on(async move { inner.close().await })) - .map_err(|e| PyRuntimeError::new_err(format!("blob close failed: {e}"))) + .map_err(|e| blob_file_error("close", e)) } fn is_closed(self_: PyRef<'_, Self>) -> bool { @@ -752,7 +757,7 @@ impl PyBlobFile { self_ .py() .detach(move || block_on(async move { inner.seek(position).await })) - .map_err(|e| PyRuntimeError::new_err(format!("blob seek failed: {e}"))) + .map_err(|e| blob_file_error("seek", e)) } fn tell(self_: PyRef<'_, Self>) -> PyResult { @@ -761,7 +766,7 @@ impl PyBlobFile { self_ .py() .detach(move || block_on(async move { inner.tell().await })) - .map_err(|e| PyRuntimeError::new_err(format!("blob tell failed: {e}"))) + .map_err(|e| blob_file_error("tell", e)) } fn size(self_: PyRef<'_, Self>) -> u64 { @@ -778,17 +783,39 @@ impl PyBlobFile { let py = self_.py(); let bytes = py .detach(move || block_on(async move { inner.read_range(offset..end).await })) - .map_err(|e| PyRuntimeError::new_err(format!("blob read_range failed: {e}")))?; + .map_err(|e| blob_file_error("read_range", e))?; Ok(PyBytes::new(py, bytes.as_ref()).unbind()) } + /// Read blob-local byte ranges without moving the cursor. + fn read_ranges(self_: PyRef<'_, Self>, ranges: Vec<(u64, u64)>) -> PyResult>> { + let ranges = ranges + .into_iter() + .map(|(offset, length)| { + offset + .checked_add(length) + .map(|end| offset..end) + .ok_or_else(|| PyValueError::new_err("offset + length overflowed")) + }) + .collect::>>()?; + let inner = self_.inner.clone(); + let py = self_.py(); + let bytes = py + .detach(move || block_on(async move { inner.read_ranges(&ranges).await })) + .map_err(|e| blob_file_error("read_ranges", e))?; + Ok(bytes + .iter() + .map(|bytes| PyBytes::new(py, bytes.as_ref()).unbind()) + .collect()) + } + fn read_up_to(self_: PyRef<'_, Self>, length: usize) -> PyResult> { self_.ensure_open()?; let inner = self_.inner.clone(); let py = self_.py(); let bytes = py .detach(move || block_on(async move { inner.read_up_to(length).await })) - .map_err(|e| PyRuntimeError::new_err(format!("blob read_up_to failed: {e}")))?; + .map_err(|e| blob_file_error("read_up_to", e))?; Ok(PyBytes::new(py, bytes.as_ref()).unbind()) } } From e485469e9236003e46e7645f52a60e282a47c372 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:53:03 -0700 Subject: [PATCH 10/19] fix(python): coerce URI strings for blob v2 writes (#4294) --- python/python/lancedb/table.py | 46 +++++++++++++++++++++++++------- python/python/tests/test_blob.py | 44 ++++++++++++++++++++++++++++++ python/python/tests/test_util.py | 33 +++++++++++++++++++++++ 3 files changed, 114 insertions(+), 9 deletions(-) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 9a9d93698..bc1bf12d9 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -740,12 +740,7 @@ def _coerce_blob_list_values( def _coerce_value_to_blob(values: pa.Array, target_field: pa.Field) -> pa.Array: - if pa.types.is_null(values.type): - data = pa.nulls(len(values), type=pa.large_binary()) - elif pa.types.is_large_binary(values.type): - data = values - else: - data = values.cast(pa.large_binary()) + value_field = "uri" if _is_string_like(values.type) else "data" length = len(values) storage_type = target_field.type if isinstance(storage_type, pa.ExtensionType): @@ -753,8 +748,11 @@ def _coerce_value_to_blob(values: pa.Array, target_field: pa.Field) -> pa.Array: storage_fields = list(storage_type) children = [] for storage_field in storage_fields: - if storage_field.name == "data": - children.append(data) + if storage_field.name == value_field and not pa.types.is_null(values.type): + if values.type == storage_field.type: + children.append(values) + else: + children.append(values.cast(storage_field.type)) else: children.append(pa.nulls(length, type=storage_field.type)) storage = pa.StructArray.from_arrays( @@ -774,7 +772,19 @@ def _physical_array_and_type(array: pa.Array) -> tuple[pa.Array, pa.DataType]: def _can_coerce_to_blob(data_type: pa.DataType) -> bool: - return _is_binary_like(data_type) or pa.types.is_null(data_type) + return ( + _is_binary_like(data_type) + or _is_string_like(data_type) + or pa.types.is_null(data_type) + ) + + +def _is_string_like(data_type: pa.DataType) -> bool: + return ( + pa.types.is_string(data_type) + or pa.types.is_large_string(data_type) + or pa.types.is_string_view(data_type) + ) def _is_binary_like(data_type: pa.DataType) -> bool: @@ -936,6 +946,24 @@ def _arrow_json_storage_type(input_type: pa.DataType) -> Optional[pa.DataType]: def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field: + if is_blob_v2_field(target_field): + source_type = field.type + if pa.types.is_struct(source_type): + if not any(child.name in ("data", "uri") for child in source_type): + raise ValueError( + f"blob struct input for column '{field.name}' must contain " + "a 'data' or 'uri' child" + ) + elif not _can_coerce_to_blob(source_type) and not isinstance( + source_type, pa.ExtensionType + ): + raise ValueError( + f"cannot coerce column '{field.name}' with type {source_type} " + "into a blob v2 struct. expected binary bytes " + "(Binary, LargeBinary, BinaryView), strings " + "(Utf8, LargeUtf8, Utf8View), or a Struct with a " + "'data' or 'uri' child" + ) # LanceDB exposes stored JSON columns as lance.json (JSONB-backed LargeBinary), but # casting the input to that storage type here merely relabels the raw JSON bytes as # JSONB. Lance must see arrow.json so it can perform the JSONB encoding. diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index a70ed785b..9d91a5db6 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -653,6 +653,50 @@ def test_merge_insert_writes_python_bytes(): assert blobs.to_pylist() == [b"updated", b"inserted"] +@pytest.mark.parametrize("write", ["merge_insert", "create_table"]) +def test_uri_string_blob_write_reaches_external_base_validation(tmp_path, write): + blob_path = tmp_path / "payload.bin" + blob_path.write_bytes(b"payload") + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + rows = pa.table( + {"id": pa.array([1], pa.int64()), "image": pa.array([blob_path.as_uri()])} + ) + db = lancedb.connect(tmp_path) + + # These APIs do not expose add's outside-base opt-in. The URI should reach + # the blob policy check instead of failing at PyArrow's string-to-struct cast. + with pytest.raises(ValueError, match="outside registered external bases"): + if write == "merge_insert": + table = db.create_table("merge_uri", schema=schema) + table.merge_insert("id").when_not_matched_insert_all().execute(rows) + else: + db.create_table("create_uri", data=rows, schema=schema) + + +@pytest.mark.parametrize("write", ["merge_insert", "create_table"]) +@pytest.mark.parametrize( + "source_type, value", + [ + (pa.binary(4), b"abcd"), + (pa.list_(pa.uint8()), [1, 2]), + (pa.int64(), 42), + ], +) +def test_blob_write_rejects_unsupported_input(write, source_type, value): + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + rows = pa.table( + {"id": pa.array([1], pa.int64()), "image": pa.array([value], source_type)} + ) + db = lancedb.connect("memory:///") + + with pytest.raises(ValueError, match="cannot coerce column 'image' with type"): + if write == "merge_insert": + table = db.create_table("merge_invalid", schema=schema) + table.merge_insert("id").when_not_matched_insert_all().execute(rows) + else: + db.create_table("create_invalid", data=rows, schema=schema) + + def test_merge_insert_bytes_after_reopen_without_touching_blob_type(tmp_path): db = lancedb.connect(tmp_path) schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) diff --git a/python/python/tests/test_util.py b/python/python/tests/test_util.py index fe5187604..9deb0c073 100644 --- a/python/python/tests/test_util.py +++ b/python/python/tests/test_util.py @@ -987,6 +987,39 @@ def test_cast_to_target_schema_coerces_binary_to_blob_v2(): ] +@pytest.mark.parametrize( + "string_type", [pa.string(), pa.large_string(), pa.string_view()] +) +def test_cast_to_target_schema_coerces_string_to_blob_v2_uri(string_type): + data = pa.table({"image": pa.array(["file:///payload", None], type=string_type)}) + target = pa.schema([lancedb.blob("image")]) + + output = _cast_to_target_schema(data.to_reader(), target).read_all() + + image = output["image"].chunk(0) + assert type(image.type) is lancedb.BlobType + assert image.storage.to_pylist() == [ + {"data": None, "uri": "file:///payload", "position": None, "size": None}, + None, + ] + + +@pytest.mark.parametrize( + "source_type, value", + [ + (pa.binary(4), b"abcd"), + (pa.list_(pa.uint8()), [1, 2]), + (pa.int64(), 42), + ], +) +def test_cast_to_target_schema_rejects_unsupported_blob_input(source_type, value): + data = pa.table({"image": pa.array([value], type=source_type)}) + target = pa.schema([lancedb.blob("image")]) + + with pytest.raises(ValueError, match="cannot coerce column 'image' with type"): + _cast_to_target_schema(data.to_reader(), target) + + def test_cast_to_target_schema_coerces_binary_to_metadata_blob_struct(): storage = lancedb.blob("image").type.storage_type target = pa.schema( From 52da9ed934a3b403b4831a671c1054781dca02e8 Mon Sep 17 00:00:00 2001 From: Drew Date: Wed, 14 Oct 2026 10:57:19 -0700 Subject: [PATCH 11/19] feat(python): add blob_mode to to_arrow and to_list (#4529) --- docs/web-tests/py/test_multimodal.py | 20 +- docs/web/management/table.mdx | 9 +- docs/web/snippets/multimodal.mdx | 2 +- python/python/lancedb/_blob.py | 215 ++++++++--- python/python/lancedb/query.py | 437 +++++++++++++++++----- python/python/lancedb/remote/table.py | 5 +- python/python/lancedb/table.py | 50 ++- python/python/tests/test_blob.py | 477 +++++++++++++++++++++++++ python/python/tests/test_query.py | 22 +- python/python/tests/test_remote_db.py | 39 ++ python/python/tests/test_wal_hybrid.py | 4 +- 11 files changed, 1113 insertions(+), 167 deletions(-) diff --git a/docs/web-tests/py/test_multimodal.py b/docs/web-tests/py/test_multimodal.py index cb4a647c3..bf7ec1bc8 100644 --- a/docs/web-tests/py/test_multimodal.py +++ b/docs/web-tests/py/test_multimodal.py @@ -215,18 +215,28 @@ async def test_query_to_pandas_kwargs(db_path_factory): .to_pandas(blob_mode="bytes") ) - # Vector / FTS / hybrid queries can't materialize blob columns, - # so omit them from the projection + # Vector / FTS / hybrid queries return "bytes" or "descriptions". + # Lazy BlobFile handles need a plain scan. df_vec = ( tbl.search(query_vector) .limit(10) - .select(["id", "vector"]) - .to_pandas(split_blocks=True, self_destruct=True) + .select(["id", "video"]) + .to_pandas(blob_mode="bytes", split_blocks=True, self_destruct=True) ) + + # to_arrow and to_list take the same blob_mode + hits = tbl.search(query_vector).limit(2).select(["id", "video"]) + arrow_bytes = hits.to_arrow(blob_mode="bytes") + rows = hits.to_list(blob_mode="bytes") # --8<-- [end:query_to_pandas_kwargs] assert len(df_lazy) == 1 assert len(df_bytes) == 1 assert df_bytes["video"].iloc[0] == b"fake_video_bytes_1" assert len(df_vec) == 10 - assert "video" not in df_vec.columns + assert all( + video == f"fake_video_bytes_{i}".encode() + for i, video in zip(df_vec["id"], df_vec["video"]) + ) + assert arrow_bytes["video"].to_pylist() == [row["video"] for row in rows] + assert all(isinstance(row["video"], bytes) for row in rows) diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index 7f84b9131..447b59029 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -456,8 +456,6 @@ When you call `to_pandas()` on a local LanceDB table that contains Blob API colu - `"bytes"`: eagerly materializes each blob as `bytes`. Use this when you need the raw payload in the DataFrame, for example to decode an image or audio clip in-process. - `"descriptions"`: returns blob descriptors (offsets, sizes, and positions) instead of the data itself. Use this when you want to plan I/O without paying the cost of loading every blob. -`"bytes"` and `"descriptions"` require a filesystem-backed Lance dataset and are not supported on in-memory tables. - Extra keyword arguments are forwarded to the underlying PyArrow / Lance pandas conversion, so you can also pass options like `split_blocks` or `self_destruct`: @@ -466,10 +464,13 @@ Extra keyword arguments are forwarded to the underlying PyArrow / Lance pandas c -Query builders also accept `blob_mode` on their `to_pandas()` method: +Query builders also accept `blob_mode` on `to_pandas()`, `to_arrow()`, and `to_list()`: - Plain scans support `"lazy"`, `"bytes"`, and `"descriptions"` with filters, projections, aliases, `limit`, and `offset`. -- Vector, FTS, hybrid, and ordered queries can't materialize blob columns through `to_pandas()`; omit blob columns from the projection for those query shapes. +- Vector, FTS, hybrid, and take queries support `"bytes"` and `"descriptions"`. Lazy `BlobFile` handles need a plain scan. +- In `to_arrow()` and `to_list()`, `"lazy"` returns blob descriptors that carry the row id `fetch_blobs` reads, and `"descriptions"` returns them without it. +- A plain scan's `to_pandas()` without `timeout` reads the local dataset through Lance's scanner, so it also returns `"bytes"` for legacy blob columns and under namespace query pushdown. In-memory tables have no local dataset, so they take the `to_arrow()` route. +- Every other `"bytes"` route raises `ValueError` for legacy blob columns and blobs inside lists, and `NotImplementedError` on remote tables and namespace query pushdown, where the blob fetch could read a different table version than the query. - This works on both sync and async query builders. Extra PyArrow kwargs like `split_blocks` and `self_destruct` are still forwarded. diff --git a/docs/web/snippets/multimodal.mdx b/docs/web/snippets/multimodal.mdx index f41af6629..90e275db7 100644 --- a/docs/web/snippets/multimodal.mdx +++ b/docs/web/snippets/multimodal.mdx @@ -16,7 +16,7 @@ export const PyMultimodalImports = "import lancedb\nimport pyarrow as pa\nimport export const PyProcessResults = "# Convert back to PIL Image\nfor _, row in results.iterrows():\n image_bytes = row['image_blob']\n image = Image.open(io.BytesIO(image_bytes))\n print(f\"Retrieved image: {row['filename']}, Size: {image.size}\")\n # You can now use 'image' with other libraries or display it\n"; -export const PyQueryToPandasKwargs = "# Plain scan query: blob_mode is supported end to end\ndf_lazy = (\n tbl.search()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"lazy\")\n)\n\n# Same call shape works on async query builders\ndf_bytes = await (\n tbl_async.query()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\")\n)\n\n# Vector / FTS / hybrid queries can't materialize blob columns,\n# so omit them from the projection\ndf_vec = (\n tbl.search(query_vector)\n .limit(10)\n .select([\"id\", \"vector\"])\n .to_pandas(split_blocks=True, self_destruct=True)\n)\n"; +export const PyQueryToPandasKwargs = "# Plain scan query: blob_mode is supported end to end\ndf_lazy = (\n tbl.search()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"lazy\")\n)\n\n# Same call shape works on async query builders\ndf_bytes = await (\n tbl_async.query()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\")\n)\n\n# Vector / FTS / hybrid queries return \"bytes\" or \"descriptions\".\n# Lazy BlobFile handles need a plain scan.\ndf_vec = (\n tbl.search(query_vector)\n .limit(10)\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\", split_blocks=True, self_destruct=True)\n)\n\n# to_arrow and to_list take the same blob_mode\nhits = tbl.search(query_vector).limit(2).select([\"id\", \"video\"])\narrow_bytes = hits.to_arrow(blob_mode=\"bytes\")\nrows = hits.to_list(blob_mode=\"bytes\")\n"; export const PySearchData = "# Search for similar images\nquery_vector = np.random.rand(128).astype(np.float32)\nresults = tbl.search(query_vector).limit(1).to_pandas()\n"; diff --git a/python/python/lancedb/_blob.py b/python/python/lancedb/_blob.py index d2d044f1b..1e9110dd0 100644 --- a/python/python/lancedb/_blob.py +++ b/python/python/lancedb/_blob.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Optional, Union import pyarrow as pa from .expr import Expr -from .schema import row_addressable_blob_v2_paths +from .schema import blob_column_paths, row_addressable_blob_v2_paths from .types import BlobMode, QueryProjection, QueryProjectionSpec if TYPE_CHECKING: @@ -27,8 +27,8 @@ BLOB_MODE_TO_HANDLING = { ROW_ID_FIELD_NAME = "_lance_row_id" -FetchBlobsSync = Callable[[str, pa.Table], pa.Array | pa.ChunkedArray] -FetchBlobsAsync = Callable[[str, pa.Table], Awaitable[pa.Array | pa.ChunkedArray]] +FetchBlobsSync = Callable[[str, list[int]], pa.Array | pa.ChunkedArray] +FetchBlobsAsync = Callable[[str, list[int]], Awaitable[pa.Array | pa.ChunkedArray]] class BlobFile(io.RawIOBase): @@ -136,7 +136,7 @@ def projection_includes_blob_column( columns = set(blob_columns) if not columns: return False - if projection is None: + if not projection: return True for output, source in _iter_projection_pairs(projection): if output in columns or any( @@ -150,18 +150,68 @@ def blob_v2_projection_sources( schema: pa.Schema, projection: QueryProjection, ) -> dict[str, str]: - blob_columns = row_addressable_blob_v2_paths(schema) - if not blob_columns: - return {} - if projection is None: + return _project_blob_paths(row_addressable_blob_v2_paths(schema), projection) + + +def reject_unfetchable_blob_columns( + schema: pa.Schema, projection: QueryProjection, blob_sources: dict[str, str] +) -> None: + # Rerankers can drop field metadata, so validate against the source schema. + unfetchable = [ + output + for output in _project_blob_paths(blob_column_paths(schema), projection) + if output not in blob_sources + ] + if unfetchable: + raise ValueError( + f"blob_mode='bytes' cannot fetch blob columns {unfetchable}. Bytes are " + "fetched by row id, which covers blob v2 columns with one blob per " + "row but not legacy blob columns or blobs inside lists. Use " + "blob_mode='descriptions' or remove these columns from the projection." + ) + + +def reject_unmatched_blob_outputs( + output_schema: pa.Schema, blob_sources: dict[str, str] +) -> None: + # Another spelling of a blob column, such as a quoted or recased name, is a + # blob in the query's output schema but maps to no source by name. + unmatched = [ + path for path in blob_column_paths(output_schema) if path not in blob_sources + ] + if unmatched: + raise ValueError( + f"blob_mode='bytes' could not match result columns {unmatched} to a blob " + "column of the table. Select blob columns by their plain names, or " + "use blob_mode='descriptions'." + ) + + +def _project_blob_paths( + blob_columns: list[str], projection: QueryProjection +) -> dict[str, str]: + """Map each projected output path to the blob column path it reads.""" + # An empty projection selects all columns. + if not projection: return {column: column for column in blob_columns} - sources = {} + sources: dict[str, str] = {} + entries: dict[str, str] = {} for output, source in _iter_projection_pairs(projection): for column in blob_columns: if column == source: - sources[output] = column + path = output elif column.startswith(f"{source}."): - sources[f"{output}{column[len(source) :]}"] = column + path = f"{output}{column[len(source) :]}" + else: + continue + if path in sources: + raise ValueError( + f"projection entries {entries[path]!r} and {output!r} both " + f"produce blob column '{path}'. Rename one of them or drop " + "the redundant selection." + ) + sources[path] = column + entries[path] = output return sources @@ -212,11 +262,14 @@ async def replace_v2_blob_columns_with_bytes( blob_sources: dict[str, str], fetch_blobs: FetchBlobsAsync, ) -> pa.Table: - for output_name, source_name in blob_sources.items(): - if not _has_column_path(tbl.schema, output_name): - continue - blobs = await fetch_blobs(source_name, tbl) - tbl = _set_blob_column(tbl, output_name, blobs) + present = _present_blob_sources(tbl, blob_sources) + if not present: + return tbl + row_ids = _hit_row_ids(tbl, present) + for output_name, source_name in present.items(): + tbl = _set_blob_column( + tbl, output_name, await fetch_blobs(source_name, row_ids) + ) return tbl @@ -225,19 +278,48 @@ def replace_v2_blob_columns_with_bytes_sync( blob_sources: dict[str, str], fetch_blobs: FetchBlobsSync, ) -> pa.Table: - for output_name, source_name in blob_sources.items(): - if not _has_column_path(tbl.schema, output_name): - continue - blobs = fetch_blobs(source_name, tbl) - tbl = _set_blob_column(tbl, output_name, blobs) + present = _present_blob_sources(tbl, blob_sources) + if not present: + return tbl + row_ids = _hit_row_ids(tbl, present) + for output_name, source_name in present.items(): + tbl = _set_blob_column(tbl, output_name, fetch_blobs(source_name, row_ids)) return tbl +def _present_blob_sources( + tbl: pa.Table, blob_sources: dict[str, str] +) -> dict[str, str]: + return { + output: source + for output, source in blob_sources.items() + if _has_column_path(tbl.schema, output) + } + + +def _hit_row_ids(tbl: pa.Table, blob_sources: dict[str, str]) -> list[int]: + """Read row ids before replacing columns, since a swapped alias can overwrite + the lookup source.""" + if "_rowid" in tbl.column_names: + return tbl["_rowid"].to_pylist() + for output in blob_sources: + descriptor = _leaf_struct_column(tbl, output) + if pa.types.is_struct(descriptor.type) and ( + ROW_ID_FIELD_NAME in descriptor.type.names + ): + return descriptor.field(ROW_ID_FIELD_NAME).to_pylist() + raise ValueError( + f"query result has no '_rowid' column and no '{ROW_ID_FIELD_NAME}' " + f"field on blob columns {list(blob_sources)}. Call .with_row_id(True) " + "to fetch blob bytes for this query." + ) + + def stash_auto_row_ids(tbl: pa.Table, blob_paths: Iterable[str]) -> pa.Table: if "_rowid" not in tbl.column_names: raise ValueError("query result has no '_rowid' column to hide") - present_paths = [p for p in blob_paths if p.split(".")[0] in tbl.column_names] + present_paths = [p for p in blob_paths if _has_column_path(tbl.schema, p)] if not present_paths: raise ValueError("query result has no blob v2 column to carry a row id") @@ -257,7 +339,7 @@ def read_row_ids_from_hits(hits: pa.Table, blob_column: str) -> list[int]: try: leaf = _leaf_struct_column(hits, blob_column) - if ROW_ID_FIELD_NAME in leaf.type.names: + if pa.types.is_struct(leaf.type) and ROW_ID_FIELD_NAME in leaf.type.names: return leaf.field(ROW_ID_FIELD_NAME).to_pylist() except KeyError: pass @@ -321,31 +403,22 @@ def _iter_projection_pairs( def _has_column_path(schema: pa.Schema, path: str) -> bool: - if schema.get_field_index(path) >= 0: - return True - top_name, *rest = path.split(".") + top_name, rest = _split_column_path(schema, path) index = schema.get_field_index(top_name) - if index < 0: - return False - field = schema.field(index) - for name in rest: - if not pa.types.is_struct(field.type): - return False - index = field.type.get_field_index(name) - if index < 0: - return False - field = field.type.field(index) - return True + return ( + index >= 0 and _walk_struct_fields(schema.field(index).type, rest) is not None + ) def _set_blob_column( tbl: pa.Table, output_name: str, blobs: pa.Array | pa.ChunkedArray ) -> pa.Table: - index = tbl.schema.get_field_index(output_name) - if index >= 0: - return tbl.set_column(index, pa.field(output_name, blobs.type), [blobs]) + top_name, rest = _split_column_path(tbl.schema, output_name) + if not rest: + index = tbl.schema.get_field_index(top_name) + return tbl.set_column(index, pa.field(top_name, blobs.type), [blobs]) - parent_path, leaf_name = output_name.rsplit(".", 1) + leaf_name = rest[-1] if isinstance(blobs, pa.ChunkedArray): blobs = blobs.combine_chunks() @@ -357,9 +430,9 @@ def _set_blob_column( leaf_name, blobs.type, nullable=field.nullable ) return - raise KeyError(f"Blob field '{leaf_name}' not found in struct '{parent_path}'") + raise KeyError(f"Blob field '{leaf_name}' not found in '{output_name}'") - return _transform_struct_column(tbl, parent_path, replace_blob) + return _transform_struct_column(tbl, top_name, rest[:-1], replace_blob) def _embed_row_id_in_column(tbl: pa.Table, path: str, row_ids: pa.Array) -> pa.Table: @@ -367,7 +440,9 @@ def _embed_row_id_in_column(tbl: pa.Table, path: str, row_ids: pa.Array) -> pa.T children.append(row_ids) child_fields.append(pa.field(ROW_ID_FIELD_NAME, pa.uint64(), nullable=False)) - return _transform_struct_column(tbl, path, add_row_id) + return _transform_struct_column( + tbl, *_split_column_path(tbl.schema, path), add_row_id + ) def strip_auto_row_ids(tbl: pa.Table, blob_paths: Iterable[str]) -> pa.Table: @@ -384,16 +459,20 @@ def strip_auto_row_ids(tbl: pa.Table, blob_paths: Iterable[str]) -> pa.Table: return for path in blob_paths: - if path.split(".")[0] not in tbl.column_names: + if not _has_column_path(tbl.schema, path): continue - tbl = _transform_struct_column(tbl, path, drop_row_id) + tbl = _transform_struct_column( + tbl, *_split_column_path(tbl.schema, path), drop_row_id + ) return tbl def _transform_struct_column( - tbl: pa.Table, path: str, leaf_transform: Callable[[list, list], None] + tbl: pa.Table, + top_name: str, + rest: list[str], + leaf_transform: Callable[[list, list], None], ) -> pa.Table: - top_name, *rest = path.split(".") top_index = tbl.schema.get_field_index(top_name) top_field = tbl.schema.field(top_index) top_array = tbl.column(top_name) @@ -451,15 +530,49 @@ def _rebuild_struct( def _leaf_struct_column(tbl: pa.Table, path: str) -> pa.StructArray: - parts = path.split(".") - column = tbl.column(parts[0]) + top_name, rest = _split_column_path(tbl.schema, path) + column = tbl.column(top_name) if isinstance(column, pa.ChunkedArray): column = column.combine_chunks() - for part in parts[1:]: + for part in rest: + if not pa.types.is_struct(column.type): + raise KeyError(f"blob path '{path}' crosses non-struct column") column = column.field(part) return column +def _split_column_path(schema: pa.Schema, path: str) -> tuple[str, list[str]]: + """Prefer a path ending in a struct, since a flat alias can shadow a nested + blob path.""" + parts = path.split(".") + fallback = None + for cut in range(len(parts), 0, -1): + top_name, rest = ".".join(parts[:cut]), parts[cut:] + index = schema.get_field_index(top_name) + if index < 0: + continue + leaf_type = _walk_struct_fields(schema.field(index).type, rest) + if leaf_type is None: + continue + if pa.types.is_struct(leaf_type): + return top_name, rest + fallback = fallback or (top_name, rest) + return fallback or (parts[0], parts[1:]) + + +def _walk_struct_fields( + data_type: pa.DataType, names: list[str] +) -> Optional[pa.DataType]: + for name in names: + if not pa.types.is_struct(data_type): + return None + index = data_type.get_field_index(name) + if index < 0: + return None + data_type = data_type.field(index).type + return data_type + + def _normalize_blob_row_ids( row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table], blob_column: str ) -> list[int]: diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index eece390c2..15a52c5e3 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -12,6 +12,7 @@ from typing import ( TYPE_CHECKING, Any, Dict, + Iterable, List, Literal, Optional, @@ -40,7 +41,7 @@ from .rerankers.base import Reranker from .rerankers.rrf import RRFReranker from .rerankers.util import check_reranker_result from .schema import blob_column_paths, is_blob_like_field, schema_has_blob_field -from .util import _validate_query_vector, flatten_columns +from .util import _validate_query_vector, flatten_columns, get_uri_scheme from . import _wal_hybrid # WAL-PK-FUSION: delete. from ._blob import ( BLOB_MODE_TO_HANDLING, @@ -49,6 +50,8 @@ from ._blob import ( blob_auto_row_id_for_scan, blob_v2_projection_sources, finalize_blob_query_table, + reject_unfetchable_blob_columns, + reject_unmatched_blob_outputs, replace_v2_blob_columns_with_bytes, replace_v2_blob_columns_with_bytes_sync, strip_auto_row_ids, @@ -99,15 +102,45 @@ class _LanceScanner(Protocol): def to_reader(self): ... -def _blob_mode_requires_native_pandas(blob_mode: BlobMode, schema: pa.Schema) -> bool: - return blob_mode in BLOB_MODE_TO_HANDLING and schema_has_blob_field(schema) - - def _unsupported_blob_pandas_error(reason: str) -> RuntimeError: return RuntimeError( "blob columns require Lance native scanner conversion for query " - f"to_pandas(), but {reason}. Use a plain scan query or remove blob " - "columns from the projection." + f"to_pandas(), but {reason}. Use a plain scan query, pass " + "blob_mode='bytes' or blob_mode='descriptions', or remove blob columns " + "from the projection." + ) + + +def _supports_local_blob_fetch( + async_table: "AsyncTable", pushdown_operations: Iterable[str] +) -> bool: + # Remote tables and QueryTable pushdown answer queries on a server, so a + # fetch through this handle could read another version. + return async_table._inner._is_native() and "QueryTable" not in pushdown_operations + + +def _blob_bytes_without_snapshot_error() -> NotImplementedError: + return NotImplementedError( + "blob_mode='bytes' cannot safely materialize blob columns without a " + "stable table snapshot. Remote tables and namespace QueryTable pushdown " + "run the query on a server, so the blob fetch could read another table " + "version. Use blob_mode='descriptions'." + ) + + +def _query_needs_lance_scanner(builder: Any) -> bool: + """Whether the query sets options that to_arrow ignores.""" + to_query_object = getattr(builder, "to_query_object", None) + if to_query_object is None: + return False + try: + query = to_query_object() + except NotImplementedError: + return False + return ( + query.fragments is not None + or query.fragment_ids is not None + or bool(query.with_row_address) ) @@ -151,7 +184,8 @@ def _combine_where( def _projection_to_scanner_kwargs(columns: QueryProjection) -> Dict[str, Any]: - if columns is None: + # An empty projection selects all columns. + if not columns: return {} if isinstance(columns, list): if all(isinstance(column, str) for column in columns): @@ -175,7 +209,8 @@ def _projection_to_scanner_kwargs(columns: QueryProjection) -> Dict[str, Any]: def _query_request_projection(req: "PyQueryRequest") -> QueryProjection: - if req.select_source_columns is not None: + # Treat empty map as a calculated select + if req.select_source_columns: return req.select_source_columns return req.select @@ -1072,44 +1107,42 @@ class LanceQueryBuilder(ABC): The maximum time to wait for the query to complete. If None, wait indefinitely. blob_mode: str, default "lazy" - Controls how blob columns are returned for plain scan queries. - Vector, FTS, hybrid, and other non-native query shapes keep the - existing Arrow conversion path and only support blob descriptions. + Controls how blob columns are returned. ``"lazy"`` returns + :class:`~lancedb._blob.BlobFile` handles and needs a plain scan. + ``"descriptions"`` and ``"bytes"`` work for every query shape. A + plain scan without ``timeout`` reads the local dataset through + Lance's scanner, which also returns legacy blob columns as bytes. + Other shapes use :meth:`to_arrow` with the same ``blob_mode``. **kwargs Forwarded to pyarrow.Table.to_pandas after query execution and optional flattening. """ validate_blob_mode(blob_mode) output_schema = getattr(self, "output_schema", None) - if output_schema is not None: - schema = output_schema() - if _blob_mode_requires_native_pandas(blob_mode, schema): - native_error = None - if (flatten is None or blob_mode == "descriptions") and timeout is None: - try: - df = self._plain_scan_to_pandas( - blob_mode, flatten=flatten, **kwargs - ) - if df is not None: - return df - except Exception as err: - native_error = err - reason = ( + if output_schema is not None and schema_has_blob_field(output_schema()): + if (flatten is None or blob_mode != "lazy") and timeout is None: + df = self._plain_scan_to_pandas(blob_mode, flatten=flatten, **kwargs) + if df is not None: + return df + if blob_mode == "lazy" or _query_needs_lance_scanner(self): + raise _unsupported_blob_pandas_error( "this query shape cannot use Lance native pandas conversion" - if native_error is None - else str(native_error) ) - raise _unsupported_blob_pandas_error(reason) from native_error - tbl = flatten_columns(self.to_arrow(timeout=timeout), flatten) - if _blob_mode_requires_native_pandas(blob_mode, tbl.schema): + tbl = flatten_columns(self._to_arrow_in_mode(timeout, blob_mode), flatten) + if blob_mode == "lazy" and schema_has_blob_field(tbl.schema): raise _unsupported_blob_pandas_error( "this query shape cannot use Lance native pandas conversion" ) return tbl.to_pandas(**kwargs) @abstractmethod - def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table: + def to_arrow( + self, + *, + timeout: Optional[timedelta] = None, + blob_mode: BlobMode = "lazy", + ) -> pa.Table: """ Execute the query and return the results as an [Apache Arrow Table](https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table). @@ -1123,6 +1156,11 @@ class LanceQueryBuilder(ABC): timeout: Optional[timedelta] The maximum time to wait for the query to complete. If None, wait indefinitely. + blob_mode: str, default "lazy" + ``"lazy"`` returns blob v2 descriptors that carry the row id + ``fetch_blobs`` reads. ``"descriptions"`` returns them without it. + ``"bytes"`` returns payload bytes and raises for legacy blob columns + and blobs inside lists. """ raise NotImplementedError @@ -1153,7 +1191,12 @@ class LanceQueryBuilder(ABC): """ raise NotImplementedError - def to_list(self, *, timeout: Optional[timedelta] = None) -> List[dict]: + def to_list( + self, + *, + timeout: Optional[timedelta] = None, + blob_mode: BlobMode = "lazy", + ) -> List[dict]: """ Execute the query and return the results as a list of dictionaries. @@ -1166,8 +1209,19 @@ class LanceQueryBuilder(ABC): timeout: Optional[timedelta] The maximum time to wait for the query to complete. If None, wait indefinitely. + blob_mode: str, default "lazy" + Passed through to :meth:`to_arrow`. """ - return self.to_arrow(timeout=timeout).to_pylist() + validate_blob_mode(blob_mode) + return self._to_arrow_in_mode(timeout, blob_mode).to_pylist() + + def _to_arrow_in_mode( + self, timeout: Optional[timedelta], blob_mode: BlobMode + ) -> pa.Table: + # Older subclasses may not accept blob_mode. Omit it when it has no effect. + if blob_mode == "lazy" or not schema_has_blob_field(self._table.schema): + return self.to_arrow(timeout=timeout) + return self.to_arrow(timeout=timeout, blob_mode=blob_mode) def to_pydantic( self, model: type[T], *, timeout: Optional[timedelta] = None @@ -1340,19 +1394,62 @@ class LanceQueryBuilder(ABC): query.with_row_id = True return query - def _finalize_blob_query_table(self, tbl: pa.Table) -> pa.Table: - blob_auto_row_id = self._blob_auto_row_id_enabled() - blob_paths = ( - blob_v2_projection_sources(self._table.schema, self._columns).keys() - if blob_auto_row_id - else () + def _validated_blob_sources(self, blob_mode: BlobMode) -> dict[str, str]: + """Map projected blob v2 outputs to their source columns. + + Bytes mode is checked here, before the query runs. + """ + validate_blob_mode(blob_mode) + schema = self._table.schema + blob_sources = blob_v2_projection_sources(schema, self._columns) + if blob_mode != "bytes": + return blob_sources + reject_unfetchable_blob_columns(schema, self._columns, blob_sources) + if blob_sources and not _supports_local_blob_fetch( + self._table._table, getattr(self._table, "_pushdown_operations", ()) + ): + raise _blob_bytes_without_snapshot_error() + reject_unmatched_blob_outputs(self._blob_output_schema(), blob_sources) + return blob_sources + + def _blob_output_schema(self) -> pa.Schema: + return self.output_schema() + + def _apply_arrow_blob_mode( + self, tbl: pa.Table, blob_mode: BlobMode, blob_sources: dict[str, str] + ) -> pa.Table: + if blob_mode == "lazy": + return tbl + if blob_mode == "descriptions": + return strip_auto_row_ids(tbl, blob_sources) + return replace_v2_blob_columns_with_bytes_sync( + tbl, blob_sources, self._table.fetch_blobs ) - return finalize_blob_query_table( + + def _finalize_blob_query_table( + self, + tbl: pa.Table, + *, + blob_mode: BlobMode, + blob_sources: dict[str, str], + ) -> pa.Table: + blob_auto_row_id = self._blob_auto_row_id_enabled() + finalized = finalize_blob_query_table( tbl, user_requested_row_id=self._user_requested_row_id(), blob_auto_row_id=blob_auto_row_id, - blob_paths=blob_paths, + blob_paths=blob_sources.keys() if blob_auto_row_id else (), ) + finalized = self._apply_arrow_blob_mode(finalized, blob_mode, blob_sources) + # with_row_id(False) does not reach the scan, which adds _rowid anyway. + # Lazy keeps it because fetch_blobs(hits) reads it there. + if ( + blob_mode != "lazy" + and self._with_row_id is False + and "_rowid" in finalized.column_names + ): + finalized = finalized.drop_columns(["_rowid"]) + return finalized def with_row_address(self, with_row_address: bool = True) -> Self: """Set whether to return row addresses. @@ -1565,6 +1662,11 @@ class LanceQueryBuilder(ABC): if not _query_is_plain_scan(query): return None + # Remote and in-memory tables have no Lance dataset to scan. + if not self._table._table._inner._is_native() or ( + get_uri_scheme(self._table._dataset_path) == "memory" + ): + return None dataset = self._table.to_lance() blob_auto_row_id = self._blob_auto_row_id_enabled() blob_sources = ( @@ -1572,6 +1674,10 @@ class LanceQueryBuilder(ABC): if blob_mode == "bytes" else {} ) + if blob_sources: + reject_unfetchable_blob_columns( + self._table.schema, query.columns, blob_sources + ) scanner = dataset.scanner( **_scanner_kwargs_for_query( query, @@ -1585,7 +1691,7 @@ class LanceQueryBuilder(ABC): blob_mode=blob_mode, blob_sources=blob_sources, fetch_blobs=self._table.fetch_blobs, - strip_auto_row_id=blob_auto_row_id, + strip_auto_row_id=not self._user_requested_row_id(), flatten=flatten, **kwargs, ) @@ -1846,7 +1952,12 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): """ return self._table._output_schema(self.to_query_object()) - def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table: + def to_arrow( + self, + *, + timeout: Optional[timedelta] = None, + blob_mode: BlobMode = "lazy", + ) -> pa.Table: """ Execute the query and return the results as an [Apache Arrow Table](https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table). @@ -1860,9 +1971,17 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): timeout: Optional[timedelta] The maximum time to wait for the query to complete. If None, wait indefinitely. + blob_mode: str, default "lazy" + ``"lazy"`` returns blob v2 descriptors that carry the row id + ``fetch_blobs`` reads. ``"descriptions"`` returns them without it. + ``"bytes"`` returns payload bytes and raises for legacy blob columns + and blobs inside lists. """ + blob_sources = self._validated_blob_sources(blob_mode) return self._finalize_blob_query_table( - self.to_batches(timeout=timeout).read_all() + self.to_batches(timeout=timeout).read_all(), + blob_mode=blob_mode, + blob_sources=blob_sources, ) def to_query_object(self) -> Query: @@ -2135,16 +2254,23 @@ class LanceFtsQueryBuilder(LanceQueryBuilder): """ return self._table._output_schema(self.to_query_object()) - def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table: + def to_arrow( + self, + *, + timeout: Optional[timedelta] = None, + blob_mode: BlobMode = "lazy", + ) -> pa.Table: + blob_sources = self._validated_blob_sources(blob_mode) self._table._ensure_no_legacy_fts_index() query = self._query_for_scan() - results = self._table._execute_query(query, timeout=timeout) - results = results.read_all() + results = self._table._execute_query(query, timeout=timeout).read_all() if self._reranker is not None: results = self._reranker.rerank_fts(self._query, results) check_reranker_result(results) - return self._finalize_blob_query_table(results) + return self._finalize_blob_query_table( + results, blob_mode=blob_mode, blob_sources=blob_sources + ) def to_batches( self, /, batch_size: Optional[int] = None, timeout: Optional[timedelta] = None @@ -2171,9 +2297,17 @@ class LanceFtsQueryBuilder(LanceQueryBuilder): class LanceEmptyQueryBuilder(LanceQueryBuilder): - def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table: + def to_arrow( + self, + *, + timeout: Optional[timedelta] = None, + blob_mode: BlobMode = "lazy", + ) -> pa.Table: + blob_sources = self._validated_blob_sources(blob_mode) return self._finalize_blob_query_table( - self.to_batches(timeout=timeout).read_all() + self.to_batches(timeout=timeout).read_all(), + blob_mode=blob_mode, + blob_sources=blob_sources, ) def to_query_object(self) -> Query: @@ -2306,16 +2440,30 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): def to_query_object(self) -> Query: raise NotImplementedError("to_query_object not yet supported on a hybrid query") - def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table: + def to_arrow( + self, + *, + timeout: Optional[timedelta] = None, + blob_mode: BlobMode = "lazy", + ) -> pa.Table: + blob_sources = self._validated_blob_sources(blob_mode) # WAL-PK-FUSION: without the fallback, this body is `_run_hybrid`'s. return _wal_hybrid.with_pk_fallback( - lambda: self._run_hybrid(timeout=timeout), + lambda: self._run_hybrid( + timeout=timeout, blob_mode=blob_mode, blob_sources=blob_sources + ), self._table, fused=lambda: self._pk_fusion is not None, caller_requested_row_id=self._caller_requested_row_id, ) - def _run_hybrid(self, *, timeout: Optional[timedelta] = None) -> pa.Table: + def _run_hybrid( + self, + *, + timeout: Optional[timedelta] = None, + blob_mode: BlobMode, + blob_sources: dict[str, str], + ) -> pa.Table: self._create_query_builders() fts_query, vector_query = self._fts_query, self._vector_query # WAL-PK-FUSION: without the fallback, both legs always ask for row ids. @@ -2342,18 +2490,33 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): with_row_ids=True, offset=self._offset, ) - return self._finish_hybrid_results(results) + return self._finish_hybrid_results( + results, blob_mode=blob_mode, blob_sources=blob_sources + ) - def _finish_hybrid_results(self, results: pa.Table) -> pa.Table: + def _finish_hybrid_results( + self, + results: pa.Table, + *, + blob_mode: BlobMode, + blob_sources: dict[str, str], + ) -> pa.Table: if self._pk_fusion is not None: # WAL-PK-FUSION: delete. - return self._pk_fusion.strip(results) + return self._apply_arrow_blob_mode( + self._pk_fusion.strip(results), blob_mode, blob_sources + ) if self._user_requested_row_id(): - return results + return self._apply_arrow_blob_mode(results, blob_mode, blob_sources) if self._blob_auto_row_id_enabled(): - return self._finalize_blob_query_table(results) + return self._finalize_blob_query_table( + results, blob_mode=blob_mode, blob_sources=blob_sources + ) if "_rowid" in results.column_names: - return results.drop(["_rowid"]) - return results + # Fetch bytes before dropping _rowid. + return self._apply_arrow_blob_mode(results, blob_mode, blob_sources).drop( + ["_rowid"] + ) + return self._apply_arrow_blob_mode(results, blob_mode, blob_sources) @staticmethod def _combine_hybrid_results( @@ -2804,6 +2967,10 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): ) return "\n".join(results) + def _blob_output_schema(self) -> pa.Schema: + self._create_query_builders() + return self._vector_query.output_schema() + def _create_query_builders(self): """Set up and configure the vector and FTS query builders.""" vector_query, fts_query = self._validate_query( @@ -3064,7 +3231,48 @@ class AsyncQueryBase(object): """ return await self._inner.output_schema() - async def to_arrow(self, timeout: Optional[timedelta] = None) -> pa.Table: + async def _validated_blob_sources(self, blob_mode: BlobMode) -> dict[str, str]: + """Map projected blob v2 outputs to their source columns. + + Bytes mode is checked here, before the query runs. + """ + validate_blob_mode(blob_mode) + # Lazy needs no mapping, and skipping it saves remote tables a request. + if blob_mode == "lazy" or self._table is None: + return {} + schema = await self._table.schema() + projection = _query_request_projection(self._inner.to_query_request()) + blob_sources = blob_v2_projection_sources(schema, projection) + if blob_mode != "bytes": + return blob_sources + reject_unfetchable_blob_columns(schema, projection, blob_sources) + if blob_sources and not _supports_local_blob_fetch( + self._table, self._table._pushdown_operations + ): + raise _blob_bytes_without_snapshot_error() + reject_unmatched_blob_outputs(await self._blob_output_schema(), blob_sources) + return blob_sources + + async def _blob_output_schema(self) -> pa.Schema: + return await self.output_schema() + + async def _apply_arrow_blob_mode( + self, tbl: pa.Table, blob_mode: BlobMode, blob_sources: dict[str, str] + ) -> pa.Table: + if blob_mode == "lazy": + return tbl + if blob_mode == "descriptions": + return strip_auto_row_ids(tbl, blob_sources) + return await replace_v2_blob_columns_with_bytes( + tbl, blob_sources, self._table.fetch_blobs + ) + + async def to_arrow( + self, + timeout: Optional[timedelta] = None, + *, + blob_mode: BlobMode = "lazy", + ) -> pa.Table: """ Execute the query and collect the results into an Apache Arrow Table. @@ -3078,13 +3286,26 @@ class AsyncQueryBase(object): The maximum time to wait for the query to complete. If not specified, no timeout is applied. If the query does not complete within the specified time, an error will be raised. + blob_mode: str, default "lazy" + ``"lazy"`` returns blob v2 descriptors that carry the row id + ``fetch_blobs`` reads. ``"descriptions"`` returns them without it. + ``"bytes"`` returns payload bytes and raises for legacy blob columns + and blobs inside lists. + Remote tables and namespace query pushdown refuse ``"bytes"``. """ + blob_sources = await self._validated_blob_sources(blob_mode) batch_iter = await self.to_batches(timeout=timeout) - return self._finalize_blob_query_table( + tbl = self._finalize_blob_query_table( pa.Table.from_batches(await batch_iter.read_all(), schema=batch_iter.schema) ) + return await self._apply_arrow_blob_mode(tbl, blob_mode, blob_sources) - async def to_list(self, timeout: Optional[timedelta] = None) -> List[dict]: + async def to_list( + self, + timeout: Optional[timedelta] = None, + *, + blob_mode: BlobMode = "lazy", + ) -> List[dict]: """ Execute the query and return the results as a list of dictionaries. @@ -3098,8 +3319,10 @@ class AsyncQueryBase(object): The maximum time to wait for the query to complete. If not specified, no timeout is applied. If the query does not complete within the specified time, an error will be raised. + blob_mode: str, default "lazy" + Passed through to :meth:`to_arrow`. """ - return (await self.to_arrow(timeout=timeout)).to_pylist() + return (await self.to_arrow(timeout=timeout, blob_mode=blob_mode)).to_pylist() async def to_pandas( self, @@ -3141,40 +3364,42 @@ class AsyncQueryBase(object): If not specified, no timeout is applied. If the query does not complete within the specified time, an error will be raised. blob_mode: str, default "lazy" - Controls how blob columns are returned. Remote queries support - "descriptions"; "bytes" and "lazy" are not yet supported. + Controls how blob columns are returned. ``"lazy"`` returns + :class:`~lancedb._blob.BlobFile` handles and needs a plain scan. + ``"descriptions"`` and ``"bytes"`` work for every query shape. A + plain scan without ``timeout`` reads the local dataset through + Lance's scanner, which also returns legacy blob columns as bytes. + Other shapes use :meth:`to_arrow` with the same ``blob_mode``. + Remote queries support ``"descriptions"`` only. **kwargs Forwarded to pyarrow.Table.to_pandas after query execution and optional flattening. """ validate_blob_mode(blob_mode) if self._table is not None and not self._table._inner._is_native(): + await self._validated_blob_sources(blob_mode) return await self._remote_to_pandas( flatten=flatten, timeout=timeout, blob_mode=blob_mode, **kwargs ) - if hasattr(self._inner, "output_schema"): - schema = await self.output_schema() - if _blob_mode_requires_native_pandas(blob_mode, schema): - native_error = None - if (flatten is None or blob_mode == "descriptions") and timeout is None: - try: - df = await self._plain_scan_to_pandas( - blob_mode, flatten=flatten, **kwargs - ) - if df is not None: - return df - except Exception as err: - native_error = err - reason = ( + if hasattr(self._inner, "output_schema") and schema_has_blob_field( + await self.output_schema() + ): + if (flatten is None or blob_mode != "lazy") and timeout is None: + df = await self._plain_scan_to_pandas( + blob_mode, flatten=flatten, **kwargs + ) + if df is not None: + return df + if blob_mode == "lazy" or _query_needs_lance_scanner(self): + raise _unsupported_blob_pandas_error( "this query shape cannot use Lance native pandas conversion" - if native_error is None - else str(native_error) ) - raise _unsupported_blob_pandas_error(reason) from native_error - tbl = flatten_columns(await self.to_arrow(timeout=timeout), flatten) - if _blob_mode_requires_native_pandas(blob_mode, tbl.schema): + tbl = flatten_columns( + await self.to_arrow(timeout=timeout, blob_mode=blob_mode), flatten + ) + if blob_mode == "lazy" and schema_has_blob_field(tbl.schema): raise _unsupported_blob_pandas_error( "this query shape cannot use Lance native pandas conversion" ) @@ -3229,6 +3454,13 @@ class AsyncQueryBase(object): if blob_mode == "bytes" else {} ) + if blob_sources: + reject_unfetchable_blob_columns(schema, query.columns, blob_sources) + # Remote and in-memory tables have no Lance dataset to scan. + if not self._table._inner._is_native() or ( + get_uri_scheme(await self._table.uri()) == "memory" + ): + return None dataset = await self._table.to_lance() scanner = dataset.scanner( **_scanner_kwargs_for_query( @@ -3243,7 +3475,7 @@ class AsyncQueryBase(object): blob_mode=blob_mode, blob_sources=blob_sources, fetch_blobs=self._table.fetch_blobs, - strip_auto_row_id=blob_auto_row_id, + strip_auto_row_id=self._with_row_id is not True, flatten=flatten, **kwargs, ) @@ -4083,6 +4315,10 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase): return self + async def _blob_output_schema(self) -> pa.Schema: + vector_query = AsyncVectorQuery(self._inner.to_vector_query(), self._table) + return await vector_query.output_schema() + async def _create_child_queries( self, ) -> Tuple["AsyncFTSQuery", "AsyncVectorQuery", int, int]: @@ -4444,7 +4680,12 @@ class BaseQueryBuilder(object): return pa.RecordBatchReader.from_batches(async_reader.schema, iter_sync()) - def to_arrow(self, timeout: Optional[timedelta] = None) -> pa.Table: + def to_arrow( + self, + timeout: Optional[timedelta] = None, + *, + blob_mode: BlobMode = "lazy", + ) -> pa.Table: """ Execute the query and collect the results into an Apache Arrow Table. @@ -4458,10 +4699,20 @@ class BaseQueryBuilder(object): The maximum time to wait for the query to complete. If not specified, no timeout is applied. If the query does not complete within the specified time, an error will be raised. + blob_mode: str, default "lazy" + ``"lazy"`` returns blob v2 descriptors that carry the row id + ``fetch_blobs`` reads. ``"descriptions"`` returns them without it. + ``"bytes"`` returns payload bytes and raises for legacy blob columns + and blobs inside lists. """ - return LOOP.run(self._inner.to_arrow(timeout)) + return LOOP.run(self._inner.to_arrow(timeout, blob_mode=blob_mode)) - def to_list(self, timeout: Optional[timedelta] = None) -> List[dict]: + def to_list( + self, + timeout: Optional[timedelta] = None, + *, + blob_mode: BlobMode = "lazy", + ) -> List[dict]: """ Execute the query and return the results as a list of dictionaries. @@ -4475,8 +4726,10 @@ class BaseQueryBuilder(object): The maximum time to wait for the query to complete. If not specified, no timeout is applied. If the query does not complete within the specified time, an error will be raised. + blob_mode: str, default "lazy" + Passed through to :meth:`to_arrow`. """ - return LOOP.run(self._inner.to_list(timeout)) + return LOOP.run(self._inner.to_list(timeout, blob_mode=blob_mode)) def to_pandas( self, @@ -4518,7 +4771,7 @@ class BaseQueryBuilder(object): If not specified, no timeout is applied. If the query does not complete within the specified time, an error will be raised. blob_mode: str, default "lazy" - Controls how blob columns are returned for plain scan queries. + Passed through to :meth:`AsyncQueryBase.to_pandas`. **kwargs Forwarded to pyarrow.Table.to_pandas after query execution and optional flattening. diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 1bb238fbd..53b6c402a 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -57,6 +57,7 @@ from lancedb.table import IndexConfigType, KNOWN_METRICS import pyarrow as pa from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME +from lancedb._blob import validate_blob_mode from lancedb.merge import LanceMergeInsertBuilder from lancedb.embeddings import EmbeddingFunctionRegistry from lancedb.table import _normalize_progress @@ -239,12 +240,14 @@ class RemoteTable(Table): """List all versions of the table""" return LOOP.run(self._table.list_versions()) - def to_arrow(self) -> pa.Table: + def to_arrow(self, *, blob_mode: BlobMode = "lazy") -> pa.Table: """to_arrow() is not supported for remote tables.""" + validate_blob_mode(blob_mode) raise NotImplementedError("to_arrow() is not supported for remote tables.") def to_pandas(self, blob_mode: BlobMode = "lazy", **kwargs): """to_pandas() is not supported for remote tables.""" + validate_blob_mode(blob_mode) raise NotImplementedError("to_pandas() is not supported for remote tables.") def checkout(self, version: Union[int, str]): diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index bc1bf12d9..d39053775 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -1399,9 +1399,17 @@ class Table(ABC): return self.to_arrow().to_pandas(**kwargs) @abstractmethod - def to_arrow(self) -> pa.Table: + def to_arrow(self, *, blob_mode: BlobMode = "lazy") -> pa.Table: """Return the table as a pyarrow Table. + Parameters + ---------- + blob_mode: str, default "lazy" + ``"lazy"`` returns blob v2 descriptors that carry the row id + ``fetch_blobs`` reads. ``"descriptions"`` returns them without it. + ``"bytes"`` returns payload bytes and raises for legacy blob columns + and blobs inside lists. + Returns ------- pa.Table @@ -2961,6 +2969,8 @@ class LanceTable(Table): managed_versioning=managed_versioning, ) ) + # Take queries read the pushdown set from the inner table. + self._table._pushdown_operations = self._pushdown_operations @property def name(self) -> str: @@ -3339,12 +3349,23 @@ class LanceTable(Table): return self.to_lance().to_pandas(blob_mode=blob_mode, **kwargs) - def to_arrow(self) -> pa.Table: + def to_arrow(self, *, blob_mode: BlobMode = "lazy") -> pa.Table: """Return the table as a pyarrow Table. + Parameters + ---------- + blob_mode: str, default "lazy" + ``"lazy"`` returns blob v2 descriptors that carry the row id + ``fetch_blobs`` reads. ``"descriptions"`` returns them without it. + ``"bytes"`` returns payload bytes and raises for legacy blob columns + and blobs inside lists. + Returns ------- pa.Table""" + validate_blob_mode(blob_mode) + if blob_mode != "lazy" and schema_has_blob_field(self.schema): + return self.search().to_arrow(blob_mode=blob_mode) if ( _should_push_down_query_table( self._namespace_client, self._pushdown_operations @@ -5788,22 +5809,37 @@ class AsyncTable: return await self.query().to_pandas(blob_mode=blob_mode, **kwargs) return (await self.to_lance()).to_pandas(blob_mode=blob_mode, **kwargs) - async def to_arrow(self) -> pa.Table: + async def to_arrow(self, *, blob_mode: BlobMode = "lazy") -> pa.Table: """Return the table as a pyarrow Table. + Parameters + ---------- + blob_mode: str, default "lazy" + ``"lazy"`` returns blob v2 descriptors that carry the row id + ``fetch_blobs`` reads. ``"descriptions"`` returns them without it. + ``"bytes"`` returns payload bytes and raises for legacy blob columns + and blobs inside lists. + Returns ------- pa.Table """ + validate_blob_mode(blob_mode) + schema = await self.schema() + fetches_bytes = blob_mode == "bytes" and schema_has_blob_field(schema) if ( - _should_push_down_query_table( + not fetches_bytes + and _should_push_down_query_table( self._namespace_client, self._pushdown_operations ) and not self._route_pushdown_to_rust ): - return (await self._execute_query(Query())).read_all() - - return await self.query().to_arrow() + tbl = (await self._execute_query(Query())).read_all() + if blob_mode == "descriptions": + tbl = strip_auto_row_ids(tbl, row_addressable_blob_v2_paths(schema)) + return tbl + # The query builder refuses bytes under pushdown before running the query. + return await self.query().to_arrow(blob_mode=blob_mode) async def create_index( self, diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 9d91a5db6..71e78643c 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -22,6 +22,7 @@ from lancedb._blob import ( ) from lancedb.expr import col from lancedb.index import FTS +from lancedb.rerankers import Reranker from lancedb.schema import blob_column_paths, blob_v2_column_paths @@ -411,6 +412,158 @@ def test_nested_blob_to_pandas_bytes(tmp_path): ].tolist() == [b"hello", None, None] +def test_to_arrow_and_to_list_blob_mode(tmp_path): + table = _nested_blob_table(lancedb.connect(tmp_path), "arrow_blob_mode") + expected = [ + {"name": "first", "blob": b"hello"}, + {"name": "second", "blob": None}, + None, + ] + + assert table.to_arrow(blob_mode="bytes")["info"].to_pylist() == expected + assert table.search().to_arrow(blob_mode="bytes")["info"].to_pylist() == expected + assert table.search().to_list(blob_mode="bytes") == [ + {"id": 0, "info": expected[0]}, + {"id": 1, "info": expected[1]}, + {"id": 2, "info": expected[2]}, + ] + assert table.search().select({"b": "info.blob"}).to_list(blob_mode="bytes") == [ + {"b": b"hello"}, + {"b": None}, + {"b": None}, + ] + assert table.search().select(["id", "info.blob"]).to_list(blob_mode="bytes") == [ + {"id": 0, "info.blob": b"hello"}, + {"id": 1, "info.blob": None}, + {"id": 2, "info.blob": None}, + ] + assert table.search().select({"x.y": "info"}).to_list(blob_mode="bytes") == [ + {"x.y": expected[0]}, + {"x.y": expected[1]}, + {"x.y": expected[2]}, + ] + assert [ + row["info"] for row in table.search().select([]).to_list(blob_mode="bytes") + ] == expected + assert table.search().select([]).to_pandas(blob_mode="bytes")["info"].tolist() == ( + table.search().select([]).to_arrow(blob_mode="bytes")["info"].to_pylist() + ) + assert "_rowid" not in table.search().select([]).to_arrow().column_names + # A flat non-blob column named like the nested blob path keeps its values. + shadowed = table.search().select({"info": "info", "info.blob": "id"}) + assert [row["info.blob"] for row in shadowed.to_list()] == [0, 1, 2] + assert [row["info"] for row in shadowed.to_list(blob_mode="bytes")] == expected + with pytest.raises(ValueError, match="both produce blob column 'info.blob'"): + table.search().select(["info", "info.blob"]).to_list(blob_mode="bytes") + + default_blob = table.search().select({"b": "info.blob"}).to_list()[0]["b"] + assert isinstance(default_blob, dict) + lazy = table.to_arrow(blob_mode="lazy")["info"].to_pylist() + assert table.to_arrow()["info"].to_pylist() == lazy + lazy_descriptor = lazy[0]["blob"] + assert "_lance_row_id" in lazy_descriptor + lazy_descriptor.pop("_lance_row_id") + descriptions = table.to_arrow(blob_mode="descriptions")["info"].to_pylist() + assert descriptions[0]["blob"] == lazy_descriptor + assert ( + table.search().to_list(blob_mode="descriptions")[0]["info"]["blob"] + == lazy_descriptor + ) + + with pytest.raises(ValueError, match="blob_mode must be one of"): + table.to_arrow(blob_mode="handles") + with pytest.raises(ValueError, match="blob_mode must be one of"): + table.search().to_list(blob_mode="handles") + + +@pytest.mark.asyncio +async def test_async_to_arrow_and_to_list_blob_mode(tmp_path): + db = await lancedb.connect_async(tmp_path) + table = await db.create_table( + "async_arrow_blob_mode", + schema=pa.schema( + [ + pa.field("id", pa.int64()), + pa.field( + "info", + pa.struct([pa.field("name", pa.string()), lancedb.blob("blob")]), + ), + ] + ), + ) + await table.add( + pa.table( + { + "id": pa.array([0, 1]), + "info": pa.StructArray.from_arrays( + [ + pa.array(["first", "second"]), + pa.array([b"hello", None], type=pa.large_binary()), + ], + names=["name", "blob"], + ), + } + ) + ) + expected = [ + {"name": "first", "blob": b"hello"}, + {"name": "second", "blob": None}, + ] + + assert (await table.to_arrow(blob_mode="bytes"))["info"].to_pylist() == expected + assert (await table.query().to_arrow(blob_mode="bytes"))[ + "info" + ].to_pylist() == expected + assert await table.query().to_list(blob_mode="bytes") == [ + {"id": 0, "info": expected[0]}, + {"id": 1, "info": expected[1]}, + ] + default_blob = (await table.query().select({"b": "info.blob"}).to_list())[0]["b"] + assert isinstance(default_blob, dict) + descriptions = await table.to_arrow(blob_mode="descriptions") + assert ( + "_lance_row_id" + not in descriptions.schema.field("info").type.field("blob").type.names + ) + + +def test_vector_query_to_arrow_blob_mode_bytes(tmp_path): + db = lancedb.connect(tmp_path) + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("vector", pa.list_(pa.float32(), 2)), + lancedb.blob("blob"), + ] + ) + table = db.create_table("vector_arrow_blob", schema=schema) + table.add( + [ + {"id": 1, "vector": [1.0, 0.0], "blob": b"one"}, + {"id": 2, "vector": [0.0, 1.0], "blob": b"two"}, + ] + ) + table.add([{"id": 3, "vector": [0.9, 0.1], "blob": b"three"}]) + table.add([{"id": 4, "vector": [0.1, 0.9], "blob": None}]) + + hits = table.search([0.0, 1.0]).limit(4).to_arrow(blob_mode="bytes").to_pylist() + assert [(hit["id"], hit["blob"]) for hit in hits] == [ + (2, b"two"), + (4, None), + (3, b"three"), + (1, b"one"), + ] + arrow = table.search([1.0, 0.0]).limit(1).to_arrow(blob_mode="bytes") + assert arrow["blob"].to_pylist() == [b"one"] + assert table.search([1.0, 0.0]).limit(1).to_list(blob_mode="bytes")[0]["blob"] == ( + b"one" + ) + assert isinstance( + table.search([1.0, 0.0]).limit(1).to_list()[0]["blob"], + dict, + ) + + @pytest.mark.asyncio async def test_nested_blob_to_pandas_bytes_async(tmp_path): db = await lancedb.connect_async(tmp_path) @@ -547,6 +700,19 @@ async def test_async_table_to_pandas_descriptions_mode_omits_row_id(): assert set(descriptor.keys()) == {"kind", "position", "size", "blob_id", "blob_uri"} +@pytest.mark.asyncio +async def test_async_calculated_select_does_not_read_every_blob_column(): + db = await lancedb.connect_async("memory:///calculated_select") + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = await db.create_table("calculated_select", schema=schema) + await table.add([{"id": 1, "image": b"hello"}]) + + hits = await table.query().select({"next_id": col("id") + 1}).to_arrow() + + assert hits.column_names == ["next_id"] + assert hits["next_id"].to_pylist() == [2] + + @pytest.mark.asyncio async def test_async_typed_blob_projection_preserves_source_column(): db = await lancedb.connect_async("memory:///typed_blob_projection") @@ -561,6 +727,185 @@ async def test_async_typed_blob_projection_preserves_source_column(): assert blobs.to_pylist() == [b"alpha"] +@pytest.mark.asyncio +@pytest.mark.parametrize("shape", ["scan", "hybrid"]) +async def test_async_bytes_mode_rejects_blob_spelling_it_cannot_match(shape): + db = await lancedb.connect_async("memory:///") + table = await _hybrid_blob_table_async(db) + query = table.query() + if shape == "hybrid": + query = query.nearest_to([1.0, 0.0]).nearest_to_text("hello") + + with pytest.raises(ValueError, match=r"could not match result columns \['pic'\]"): + await query.select({"id": "id", "pic": "`image`"}).to_arrow(blob_mode="bytes") + + +def test_bytes_mode_keeps_struct_with_blob_descriptor_field_names(): + descriptor_like = pa.struct( + [ + pa.field("kind", pa.uint8()), + pa.field("position", pa.uint64()), + pa.field("size", pa.uint64()), + pa.field("blob_id", pa.uint32()), + pa.field("blob_uri", pa.utf8()), + ] + ) + meta = {"kind": 1, "position": 2, "size": 3, "blob_id": 4, "blob_uri": "u"} + table = lancedb.connect("memory:///").create_table( + "descriptor_like", + schema=pa.schema( + [ + pa.field("id", pa.int64()), + lancedb.blob("image"), + pa.field("meta", descriptor_like), + ] + ), + ) + table.add([{"id": 1, "image": b"one", "meta": meta}]) + + assert table.search().to_list(blob_mode="bytes") == [ + {"id": 1, "image": b"one", "meta": meta} + ] + + +@pytest.mark.parametrize("source", ["`image`", "IMAGE"]) +def test_bytes_mode_rejects_blob_spelling_it_cannot_match(source): + table = _blob_table("blob_spelling", [{"id": 1, "image": b"one"}]) + + with pytest.raises(ValueError, match="could not match result columns"): + table.search().select({"id": "id", "pic": source}).to_arrow(blob_mode="bytes") + + +def test_to_pandas_does_not_drop_fragment_selection_on_fallback(): + table = _blob_table("blob_fragments", [{"id": 1, "image": b"one"}]) + table.add([{"id": 2, "image": b"two"}]) + + query = table.search().fragment_ids([0]) + with pytest.raises(RuntimeError, match="native pandas conversion"): + query.to_pandas(blob_mode="bytes") + with pytest.raises(RuntimeError, match="native pandas conversion"): + table.search().with_row_address(True).to_pandas(blob_mode="descriptions") + + +@pytest.mark.parametrize("blob_mode", ["lazy", "descriptions", "bytes"]) +def test_to_pandas_reports_lance_dataset_failures(tmp_path, blob_mode): + table = lancedb.connect(tmp_path).create_table( + "dataset_failure", + schema=pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]), + ) + table.add([{"id": 1, "image": b"one"}]) + + def reject_credentials(): + raise PermissionError("storage credentials rejected") + + table.to_lance = reject_credentials + with pytest.raises(PermissionError, match="storage credentials rejected"): + table.search().to_pandas(blob_mode=blob_mode) + + +def test_to_pandas_keeps_to_arrow_overrides_without_blob_mode(): + from lancedb.query import LanceEmptyQueryBuilder + + class ToArrowWithoutBlobMode(LanceEmptyQueryBuilder): + def to_arrow(self, *, timeout=None): + return super().to_arrow(timeout=timeout) + + table = lancedb.connect("memory:///").create_table("no_blob", data=[{"id": 1}]) + query = ToArrowWithoutBlobMode(table) + assert query.to_list(blob_mode="bytes") == [{"id": 1}] + assert query.to_pandas(blob_mode="descriptions")["id"].tolist() == [1] + with pytest.raises(ValueError, match="blob_mode must be one of"): + query.to_list(blob_mode="handles") + with pytest.raises(ValueError, match="blob_mode must be one of"): + table.search().to_list(blob_mode="handles") + + +def test_take_queries_honor_blob_mode(): + table = _blob_table( + "take_blob_mode", [{"id": 1, "image": b"one"}, {"id": 2, "image": b"two"}] + ) + + hits = table.take_offsets([1, 0]).to_arrow(blob_mode="bytes") + assert dict(zip(hits["id"].to_pylist(), hits["image"].to_pylist())) == { + 1: b"one", + 2: b"two", + } + assert [ + row["image"] for row in table.take_row_ids([0]).to_list(blob_mode="bytes") + ] == [b"one"] + assert table.take_offsets([0]).to_pandas(blob_mode="bytes")["image"].tolist() == [ + b"one" + ] + description = table.take_row_ids([0]).to_list(blob_mode="descriptions")[0]["image"] + assert "_lance_row_id" not in description + + +def test_bytes_mode_follows_swapped_blob_aliases(): + db = lancedb.connect("memory:///") + table = db.create_table( + "swapped_aliases", + schema=pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("caption", pa.utf8()), + lancedb.blob("image"), + lancedb.blob("thumb"), + ] + ), + ) + table.add([{"id": 0, "caption": "cap", "image": b"img", "thumb": b"th"}]) + + swapped = table.search().select({"id": "id", "image": "thumb", "thumb": "image"}) + assert swapped.to_list(blob_mode="bytes") == [ + {"id": 0, "image": b"th", "thumb": b"img"} + ] + over_text = table.search().select({"id": "id", "image": "caption", "pic": "image"}) + assert over_text.to_list(blob_mode="bytes") == [ + {"id": 0, "image": "cap", "pic": b"img"} + ] + + +def test_bytes_mode_refused_under_namespace_query_pushdown(tmp_path): + db = lancedb.connect_namespace( + "dir", + {"root": str(tmp_path)}, + namespace_client_pushdown_operations=["QueryTable"], + ) + table = db.create_table( + "pushdown_blob", + schema=pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]), + ) + table.add([{"id": 1, "image": b"one"}]) + + with pytest.raises(NotImplementedError, match="stable table snapshot"): + table.to_arrow(blob_mode="bytes") + with pytest.raises(NotImplementedError, match="stable table snapshot"): + table.search().to_list(blob_mode="bytes") + with pytest.raises(NotImplementedError, match="stable table snapshot"): + table.take_row_ids([0]).to_list(blob_mode="bytes") + descriptor = table.to_arrow(blob_mode="descriptions")["image"].to_pylist()[0] + assert "_lance_row_id" not in descriptor + + +@pytest.mark.asyncio +async def test_async_bytes_mode_refused_under_namespace_query_pushdown(tmp_path): + db = lancedb.connect_namespace_async( + "dir", + {"root": str(tmp_path)}, + namespace_client_pushdown_operations=["QueryTable"], + ) + table = await db.create_table( + "pushdown_blob", + schema=pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]), + ) + await table.add([{"id": 1, "image": b"one"}]) + + with pytest.raises(NotImplementedError, match="stable table snapshot"): + await table.to_arrow(blob_mode="bytes") + with pytest.raises(NotImplementedError, match="stable table snapshot"): + await table.query().to_list(blob_mode="bytes") + + def test_fetch_blobs_round_trip(): table = _blob_table( "round_trip", @@ -937,6 +1282,17 @@ def test_list_blob_column_queries_as_raw_descriptors(): table.fetch_blob_ranges(column, [(0, 0, 1)]) +def test_bytes_mode_rejects_blobs_inside_lists(): + table = _list_blob_table("list_blob_bytes") + + with pytest.raises(ValueError, match=r"\['images.image'\].*blobs inside lists"): + table.to_arrow(blob_mode="bytes") + with pytest.raises(ValueError, match="blobs inside lists"): + table.search().select({"pics": "images"}).to_list(blob_mode="bytes") + descriptions = table.to_arrow(blob_mode="descriptions")["images"].to_pylist() + assert descriptions[0][0]["size"] == len(b"before") + + def test_row_addressable_paths_exclude_list_children(): from lancedb.schema import row_addressable_blob_v2_paths @@ -1261,6 +1617,11 @@ def test_fetch_blobs_nested_path(): blobs = table.fetch_blobs("info.blob", [by_id[1], by_id[2]]) assert [blobs[0].as_py(), blobs[1].as_py()] == [b"nested-alpha", b"nested-beta"] + # The source path's parent name now holds a string column in the hits. + hits = table.search().select({"id": "id", "info": "info.name", "b": "info.blob"}) + blobs = table.fetch_blobs("info.blob", hits.to_arrow()) + assert blobs.to_pylist() == [b"nested-alpha", b"nested-beta"] + def test_fetch_blob_files_lazy_read(): payload = b"lazy-read" * 100 @@ -1438,6 +1799,122 @@ def test_blob_v2_hybrid_fetch_blobs(): assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"} +def _blob_query_shape(table, shape): + if shape == "scan": + return table.search().where("id < 3") + if shape == "vector": + return table.search([1.0, 0.0]) + if shape == "fts": + return table.search("hello", query_type="fts") + return table.search(query_type="hybrid").vector([1.0, 0.0]).text("hello") + + +@pytest.mark.parametrize("with_row_id", [None, True, False]) +@pytest.mark.parametrize("blob_mode", ["bytes", "descriptions"]) +@pytest.mark.parametrize("shape", ["scan", "vector", "fts", "hybrid"]) +def test_blob_mode_for_every_query_shape(shape, blob_mode, with_row_id): + table = _hybrid_blob_table(lancedb.connect("memory:///")) + query = _blob_query_shape(table, shape).select(["id", "image"]).limit(2) + if with_row_id is not None: + query = query.with_row_id(with_row_id) + + hits = query.to_arrow(blob_mode=blob_mode) + + assert ("_rowid" in hits.column_names) == (with_row_id is True) + images = {row["id"]: row["image"] for row in hits.to_pylist()} + if blob_mode == "bytes": + assert images == {1: b"alpha", 2: b"beta"} + else: + assert sorted(images) == [1, 2] + assert all("_lance_row_id" not in image for image in images.values()) + df = query.to_pandas(blob_mode=blob_mode) + assert dict(zip(df["id"], df["image"])) == images + + +@pytest.mark.asyncio +@pytest.mark.parametrize("shape", ["scan", "vector", "fts", "hybrid"]) +async def test_async_blob_bytes_for_every_query_shape(shape): + db = await lancedb.connect_async("memory:///") + table = await _hybrid_blob_table_async(db) + query = table.query() + if shape in ("vector", "hybrid"): + query = query.nearest_to([1.0, 0.0]) + if shape in ("fts", "hybrid"): + query = query.nearest_to_text("hello") + if shape == "scan": + query = query.where("id < 3") + query = query.select(["id", "image"]).limit(2) + + hits = await query.to_arrow(blob_mode="bytes") + + assert {row["id"]: row["image"] for row in hits.to_pylist()} == { + 1: b"alpha", + 2: b"beta", + } + df = await query.to_pandas(blob_mode="bytes") + assert dict(zip(df["id"], df["image"])) == {1: b"alpha", 2: b"beta"} + + +class _KeepAllScoresReranker(Reranker): + """Rebuilds the merged table like model rerankers with return_score="all".""" + + def __init__(self): + super().__init__(return_score="all") + + def rerank_hybrid(self, query, vector_results, fts_results): + combined = self._merge_and_keep_scores(vector_results, fts_results) + scores = pa.array([1.0] * len(combined), type=pa.float32()) + return combined.append_column("_relevance_score", scores) + + +def test_bytes_mode_rejects_legacy_blob_after_reranker_drops_metadata(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table( + "legacy_rerank", + pa.table( + { + "id": pa.array([1, 2], pa.int64()), + "text": ["hello alpha", "hello beta"], + "vector": pa.array( + [[1.0, 0.0], [0.9, 0.1]], type=pa.list_(pa.float32(), 2) + ), + "blob": pa.array([b"one", b"two"], pa.large_binary()), + }, + schema=_HYBRID_BLOB_SCHEMA.remove(3).append( + pa.field( + "blob", pa.large_binary(), metadata={"lance-encoding:blob": "true"} + ) + ), + ), + storage_options={"new_table_data_storage_version": "2.1"}, + ) + table.create_index("text", config=FTS(with_position=False)) + query = ( + table.search(query_type="hybrid") + .vector([1.0, 0.0]) + .text("hello") + .rerank(_KeepAllScoresReranker()) + ) + + with pytest.raises(ValueError, match=r"\['blob'\].*legacy blob columns"): + query.to_arrow(blob_mode="bytes") + + +@pytest.mark.parametrize("source", ["`image`", "IMAGE"]) +def test_bytes_mode_rejects_blob_spelling_after_reranker_drops_metadata(source): + table = _hybrid_blob_table(lancedb.connect("memory:///")) + query = ( + table.search(query_type="hybrid") + .vector([1.0, 0.0]) + .text("hello") + .select({"id": "id", "pic": source}) + .rerank(_KeepAllScoresReranker()) + ) + + with pytest.raises(ValueError, match=r"could not match result columns \['pic'\]"): + query.to_arrow(blob_mode="bytes") + + @pytest.mark.asyncio async def test_blob_v2_hybrid_fetch_blobs_async(): db = await lancedb.connect_async("memory:///hybrid_blob_fetch_async") diff --git a/python/python/tests/test_query.py b/python/python/tests/test_query.py index 74a67b909..a05247c5e 100644 --- a/python/python/tests/test_query.py +++ b/python/python/tests/test_query.py @@ -509,18 +509,30 @@ def test_vector_query_to_pandas_blob_mode_requires_native_path(tmp_db): ) -def test_vector_query_to_pandas_blob_descriptions_requires_plain_scan(tmp_db): +def test_vector_query_to_pandas_legacy_blob_descriptions_and_bytes(tmp_db): pytest.importorskip("lance") table = tmp_db.create_table( "test_vector_query_blob_descriptions", _blob_query_data(), storage_options=LEGACY_BLOB_STORAGE_OPTIONS, ) + query = table.search([1.0, 0.0]).select(["blob", "vector"]).limit(1) - with pytest.raises(RuntimeError, match="plain scan query"): - table.search([1.0, 0.0]).select(["blob", "vector"]).limit(1).to_pandas( - blob_mode="descriptions" - ) + df = query.to_pandas(blob_mode="descriptions") + assert df["blob"].tolist() == [{"position": 0, "size": 3}] + + with pytest.raises(ValueError, match="legacy blob columns"): + query.to_pandas(blob_mode="bytes") + with pytest.raises(ValueError, match="legacy blob columns"): + query.to_arrow(blob_mode="bytes") + with pytest.raises(ValueError, match="legacy blob columns"): + table.to_arrow(blob_mode="bytes") + assert table.to_pandas(blob_mode="bytes")["blob"].tolist() == [ + b"one", + b"two", + b"three", + b"four", + ] def test_order_by_plain_query(mem_db): diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 489d9f49b..f43972a22 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -3469,6 +3469,45 @@ async def test_async_remote_blob_pandas_rejects_unpinned_fetch(monkeypatch, blob await table.to_pandas(**pandas_kwargs) +@pytest.mark.parametrize("method", ["to_arrow", "to_list"]) +def test_remote_search_bytes_refused_before_query(monkeypatch, method): + with blob_remote_table() as table: + query = table.search().select(["id", "image"]) + descriptions = getattr(query, method)(blob_mode="descriptions") + images = ( + descriptions["image"].to_pylist() + if method == "to_arrow" + else [row["image"] for row in descriptions] + ) + assert "_lance_row_id" not in images[0] + + def fail_query(*args, **kwargs): + raise AssertionError("bytes must be refused before the query runs") + + monkeypatch.setattr(table, "_execute_query", fail_query) + with pytest.raises(NotImplementedError, match="stable table snapshot"): + getattr(query, method)(blob_mode="bytes") + with pytest.raises(ValueError, match="blob_mode must be one of"): + table.to_pandas(blob_mode="handles") + + +@pytest.mark.asyncio +async def test_async_remote_to_arrow_bytes_refused_before_query(monkeypatch): + with blob_remote_table() as remote_table: + table = remote_table._table + + async def fail_fetch(*args, **kwargs): + raise AssertionError("blob fetch must not use an unpinned remote table") + + monkeypatch.setattr(table, "fetch_blobs", fail_fetch) + query = table.query().select(["id", "image"]) + monkeypatch.setattr(query, "to_batches", fail_fetch) + with pytest.raises(NotImplementedError, match="stable table snapshot"): + await query.to_arrow(blob_mode="bytes") + with pytest.raises(NotImplementedError, match="stable table snapshot"): + await table.to_arrow(blob_mode="bytes") + + @pytest.mark.asyncio @pytest.mark.parametrize("blob_mode", ["bytes", "lazy"]) async def test_async_remote_nonblob_projection_to_pandas(blob_mode): diff --git a/python/python/tests/test_wal_hybrid.py b/python/python/tests/test_wal_hybrid.py index 11a9755e0..9c7efe151 100644 --- a/python/python/tests/test_wal_hybrid.py +++ b/python/python/tests/test_wal_hybrid.py @@ -119,7 +119,9 @@ def test_wal_hybrid_fuses_on_the_key_and_drops_it_again(): limit=10, with_row_ids=True, ) - results = builder._finish_hybrid_results(combined) + results = builder._finish_hybrid_results( + combined, blob_mode="lazy", blob_sources={} + ) assert "_rowid" not in results.column_names, "the surrogate is internal" assert "id" not in results.column_names, "the caller selected only `text`" From eabec507a874d8b7ff643f181a815ff6e0ae5046 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 14 Oct 2026 10:58:12 -0700 Subject: [PATCH 12/19] ci: pin protoc v36.2 with checksums across wheel builds (#4535) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The Python wheel builds used different protoc versions: v24.4 on Linux, v21.12 on Windows, and whatever Homebrew shipped on macOS. None of them verified the download. A new composite action, `.github/actions/setup-protoc`, now holds one pin, v36.2, with a SHA-256 for each platform's zip. - macOS and Windows: the action downloads protoc, checks its hash, and installs it. - Linux: protoc has to be installed inside the manylinux container, so the build reads the version and hash from the action's outputs and checks the hash there. protobuf does not publish checksum files for protoc. The pinned hashes are the digests GitHub recorded for the v36.2 release assets. Older releases such as v24.4 have none, which is why this moves to v36.2. protoc only runs at build time, through `prost-build` in the Lance crates. The Linux protoc binary is statically linked, so it runs in the manylinux 2_17 container. The action is also added to the path triggers of `pypi-publish.yml` and `python.yml`, so a change to the pin runs the wheel builds. Follow-up to https://github.com/lancedb/lancedb/pull/4522#issuecomment-6160848344. The sdist job in that PR can use the same action once this merges. ## Evidence Running the action's install script locally on linux-aarch_64 installs `libprotoc 36.2`, and a wrong hash fails with `protoc checksum mismatch`. `cargo check -p lancedb` with `PROTOC` set to protoc 36.2 builds the Lance protos. The PyPI Publish dry run on this PR covers the Linux, macOS and Windows builds. ## Merge Risk **Door:** Two-way. CI only. **Blast Radius:** Python wheel builds. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .github/actions/setup-protoc/action.yml | 80 +++++++++++++++++++ .../workflows/build_linux_wheel/action.yml | 15 +++- .github/workflows/build_mac_wheel/action.yml | 5 +- .../workflows/build_windows_wheel/action.yml | 10 +-- .github/workflows/pypi-publish.yml | 1 + .github/workflows/python.yml | 1 + 6 files changed, 95 insertions(+), 17 deletions(-) create mode 100644 .github/actions/setup-protoc/action.yml diff --git a/.github/actions/setup-protoc/action.yml b/.github/actions/setup-protoc/action.yml new file mode 100644 index 000000000..a99cf3320 --- /dev/null +++ b/.github/actions/setup-protoc/action.yml @@ -0,0 +1,80 @@ +name: Setup protoc +description: Install the protoc version pinned for wheel builds, and expose it for builds that install protoc themselves + +inputs: + install: + description: "Install protoc on the runner. Set to false when the build runs in a container (e.g. manylinux) and installs protoc there." + required: false + default: "true" + +outputs: + version: + description: "The pinned protoc version, e.g. 36.2" + value: ${{ steps.pin.outputs.version }} + linux-x86_64-sha256: + description: "SHA-256 of the protoc linux-x86_64 zip" + value: ${{ steps.pin.outputs.linux-x86_64-sha256 }} + linux-aarch_64-sha256: + description: "SHA-256 of the protoc linux-aarch_64 zip" + value: ${{ steps.pin.outputs.linux-aarch_64-sha256 }} + +runs: + using: composite + steps: + # protobuf does not publish checksum files for protoc. These are the digests + # GitHub records for the release assets; when bumping, copy them from + # `gh api repos/protocolbuffers/protobuf/releases/tags/v`. + - name: Set protoc pin + id: pin + shell: bash + run: | + { + echo "version=36.2" + echo "linux-x86_64-sha256=121f6c7afe1d4d0e3ea6aab9432038599250134cbf4474cb1167d2c7decd4278" + echo "linux-aarch_64-sha256=8b8f18bd2b30346efbc698dd5a73dd7c805f3ef8380f6dfc95c768f3f1852f6a" + echo "osx-universal_binary-sha256=83d0dc80672f53486f54299bca6177159f9cad1fd54d4d3143eeea53c4423e1b" + echo "win64-sha256=f0c128dc0d8492eceece83bb459a4c0e316764b929ffbf1aa416357fd644edd3" + } >> "$GITHUB_OUTPUT" + - name: Install protoc + if: ${{ inputs.install == 'true' }} + shell: bash + env: + PROTOC_VERSION: ${{ steps.pin.outputs.version }} + RUNNER_OS_NAME: ${{ runner.os }} + RUNNER_ARCH_NAME: ${{ runner.arch }} + LINUX_X86_64_SHA256: ${{ steps.pin.outputs.linux-x86_64-sha256 }} + LINUX_AARCH_64_SHA256: ${{ steps.pin.outputs.linux-aarch_64-sha256 }} + OSX_SHA256: ${{ steps.pin.outputs.osx-universal_binary-sha256 }} + WIN64_SHA256: ${{ steps.pin.outputs.win64-sha256 }} + run: | + set -euo pipefail + case "$RUNNER_OS_NAME/$RUNNER_ARCH_NAME" in + Linux/X64) platform="linux-x86_64"; sha256="$LINUX_X86_64_SHA256" ;; + Linux/ARM64) platform="linux-aarch_64"; sha256="$LINUX_AARCH_64_SHA256" ;; + macOS/*) platform="osx-universal_binary"; sha256="$OSX_SHA256" ;; + Windows/X64) platform="win64"; sha256="$WIN64_SHA256" ;; + *) echo "Unsupported runner: $RUNNER_OS_NAME/$RUNNER_ARCH_NAME" >&2; exit 1 ;; + esac + zip="$RUNNER_TEMP/protoc.zip" + dest="$RUNNER_TEMP/protoc" + curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${platform}.zip" -o "$zip" + # Hash stdin rather than a path: Windows paths contain backslashes, + # which make sha256sum escape its output. + if [ "$RUNNER_OS_NAME" = "macOS" ]; then + actual=$(shasum -a 256 < "$zip" | cut -d' ' -f1) + else + actual=$(sha256sum < "$zip" | cut -d' ' -f1) + fi + if [ "$actual" != "$sha256" ]; then + echo "protoc checksum mismatch: expected $sha256, got $actual" >&2 + exit 1 + fi + if [ "$RUNNER_OS_NAME" = "Windows" ]; then + # Git Bash on Windows runners does not reliably ship unzip. + 7z x -y -o"$dest" "$zip" > /dev/null + else + unzip -q -o "$zip" -d "$dest" + fi + rm "$zip" + echo "$dest/bin" >> "$GITHUB_PATH" + "$dest/bin/protoc" --version diff --git a/.github/workflows/build_linux_wheel/action.yml b/.github/workflows/build_linux_wheel/action.yml index 8fa5b1f79..44ce82ee5 100644 --- a/.github/workflows/build_linux_wheel/action.yml +++ b/.github/workflows/build_linux_wheel/action.yml @@ -29,6 +29,11 @@ inputs: runs: using: "composite" steps: + - name: Get protoc version + id: protoc + uses: ./.github/actions/setup-protoc + with: + install: "false" - name: CONFIRM ARM BUILD shell: bash env: @@ -54,13 +59,14 @@ runs: maturin-version: "1.12.4" command: build working-directory: python - docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" + docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-x86_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" target: x86_64-unknown-linux-gnu manylinux: ${{ inputs.manylinux }} args: ${{ inputs.args }} before-script-linux: | set -e - curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip -o /tmp/protoc.zip + curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip -o /tmp/protoc.zip + echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c - unzip /tmp/protoc.zip -d /usr/local rm /tmp/protoc.zip /usr/local/bin/protoc --version @@ -71,14 +77,15 @@ runs: maturin-version: "1.12.4" command: build working-directory: python - docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" + docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-aarch_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" target: aarch64-unknown-linux-gnu manylinux: ${{ inputs.manylinux }} args: ${{ inputs.args }} before-script-linux: | set -e yum install -y clang - curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip -o /tmp/protoc.zip + curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-aarch_64.zip -o /tmp/protoc.zip + echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c - unzip /tmp/protoc.zip -d /usr/local rm /tmp/protoc.zip /usr/local/bin/protoc --version diff --git a/.github/workflows/build_mac_wheel/action.yml b/.github/workflows/build_mac_wheel/action.yml index 75d942c96..727cd190c 100644 --- a/.github/workflows/build_mac_wheel/action.yml +++ b/.github/workflows/build_mac_wheel/action.yml @@ -12,10 +12,7 @@ inputs: runs: using: "composite" steps: - - name: Install macos dependency - shell: bash - run: | - brew install protobuf + - uses: ./.github/actions/setup-protoc - name: Build wheel uses: PyO3/maturin-action@v1 with: diff --git a/.github/workflows/build_windows_wheel/action.yml b/.github/workflows/build_windows_wheel/action.yml index 18f39818f..954ee7d35 100644 --- a/.github/workflows/build_windows_wheel/action.yml +++ b/.github/workflows/build_windows_wheel/action.yml @@ -12,15 +12,7 @@ inputs: runs: using: "composite" steps: - - name: Install Protoc v21.12 - working-directory: C:\ - run: | - New-Item -Path 'C:\protoc' -ItemType Directory - Set-Location C:\protoc - Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip - 7z x protoc.zip - Add-Content $env:GITHUB_PATH "C:\protoc\bin" - shell: powershell + - uses: ./.github/actions/setup-protoc - name: Build wheel uses: PyO3/maturin-action@v1 with: diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 4f883aec1..d16203979 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -11,6 +11,7 @@ on: - .github/workflows/build_linux_wheel/action.yml - .github/workflows/build_mac_wheel/action.yml - .github/workflows/build_windows_wheel/action.yml + - .github/actions/setup-protoc/action.yml - Cargo.toml # Change in dependency frequently breaks builds - Cargo.lock diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 3594bd826..3608c24ca 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -15,6 +15,7 @@ on: - .github/workflows/build_linux_wheel/** - .github/workflows/build_mac_wheel/** - .github/workflows/build_windows_wheel/** + - .github/actions/setup-protoc/** - .github/workflows/run_tests/** permissions: From 4be1720c51d8f966d24172e06ae1d228bb6b1508 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:23:10 -0700 Subject: [PATCH 13/19] fix(python): support default blob conversion in pandas query results (#4308) --- python/python/lancedb/query.py | 116 +++++++++----- python/python/tests/test_blob.py | 6 +- python/python/tests/test_query.py | 209 +++++++++++++++++++++++++- python/python/tests/test_remote_db.py | 54 ++++++- 4 files changed, 341 insertions(+), 44 deletions(-) diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index 15a52c5e3..9734c9b43 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -128,6 +128,11 @@ def _blob_bytes_without_snapshot_error() -> NotImplementedError: ) +def _is_remote_query_table(table: Any) -> bool: + async_table = getattr(table, "_table", table) + return not async_table._inner._is_native() + + def _query_needs_lance_scanner(builder: Any) -> bool: """Whether the query sets options that to_arrow ignores.""" to_query_object = getattr(builder, "to_query_object", None) @@ -347,6 +352,28 @@ def _finish_plain_scan_pandas( return df +def _finish_remote_pandas( + tbl: pa.Table, + *, + blob_mode: BlobMode, + blob_paths: Iterable[str], + flatten: Optional[Union[int, bool]], + **kwargs, +) -> pd.DataFrame: + # A live remote table can be replaced between query and blob fetch. + # Row ids and version numbers do not identify the producing table. + if not blob_column_paths(tbl.schema): + return flatten_columns(tbl, flatten).to_pandas(**kwargs) + if blob_mode == "descriptions": + tbl = strip_auto_row_ids(tbl, blob_paths) + return flatten_columns(tbl, flatten).to_pandas(**kwargs) + raise NotImplementedError( + f"remote to_pandas(blob_mode={blob_mode!r}) cannot safely materialize " + "blob columns without a stable table snapshot; " + "use blob_mode='descriptions'" + ) + + async def _finish_plain_scan_pandas_async( scanner: _LanceScanner, *, @@ -1107,34 +1134,50 @@ class LanceQueryBuilder(ABC): The maximum time to wait for the query to complete. If None, wait indefinitely. blob_mode: str, default "lazy" - Controls how blob columns are returned. ``"lazy"`` returns - :class:`~lancedb._blob.BlobFile` handles and needs a plain scan. - ``"descriptions"`` and ``"bytes"`` work for every query shape. A - plain scan without ``timeout`` reads the local dataset through - Lance's scanner, which also returns legacy blob columns as bytes. - Other shapes use :meth:`to_arrow` with the same ``blob_mode``. + Controls how blob columns are returned. Eligible local plain scans + use Lance's scanner, which returns lazy blob handles in ``"lazy"`` + mode and also supports legacy blob columns. Other local blob v2 + queries return descriptors in ``"lazy"`` and ``"descriptions"`` + modes, or payloads in ``"bytes"`` mode, through :meth:`to_arrow`. + Remote blob queries support only ``"descriptions"`` until a stable + table snapshot can bind the query and blob fetches. **kwargs Forwarded to pyarrow.Table.to_pandas after query execution and optional flattening. """ validate_blob_mode(blob_mode) + if _is_remote_query_table(self._table): + blob_sources = self._validated_blob_sources(blob_mode) + return _finish_remote_pandas( + self.to_arrow(timeout=timeout), + blob_mode=blob_mode, + blob_paths=blob_sources, + flatten=flatten, + **kwargs, + ) + output_schema = getattr(self, "output_schema", None) if output_schema is not None and schema_has_blob_field(output_schema()): if (flatten is None or blob_mode != "lazy") and timeout is None: df = self._plain_scan_to_pandas(blob_mode, flatten=flatten, **kwargs) if df is not None: return df - if blob_mode == "lazy" or _query_needs_lance_scanner(self): + if _query_needs_lance_scanner(self): raise _unsupported_blob_pandas_error( "this query shape cannot use Lance native pandas conversion" ) - tbl = flatten_columns(self._to_arrow_in_mode(timeout, blob_mode), flatten) - if blob_mode == "lazy" and schema_has_blob_field(tbl.schema): + arrow_mode = "descriptions" if blob_mode == "lazy" else blob_mode + tbl = self._to_arrow_in_mode(timeout, arrow_mode) + if ( + blob_mode == "lazy" + and schema_has_blob_field(tbl.schema) + and not blob_v2_projection_sources(self._table.schema, self._columns) + ): raise _unsupported_blob_pandas_error( - "this query shape cannot use Lance native pandas conversion" + "the query has no fetchable blob v2 source" ) - return tbl.to_pandas(**kwargs) + return flatten_columns(tbl, flatten).to_pandas(**kwargs) @abstractmethod def to_arrow( @@ -3364,13 +3407,13 @@ class AsyncQueryBase(object): If not specified, no timeout is applied. If the query does not complete within the specified time, an error will be raised. blob_mode: str, default "lazy" - Controls how blob columns are returned. ``"lazy"`` returns - :class:`~lancedb._blob.BlobFile` handles and needs a plain scan. - ``"descriptions"`` and ``"bytes"`` work for every query shape. A - plain scan without ``timeout`` reads the local dataset through - Lance's scanner, which also returns legacy blob columns as bytes. - Other shapes use :meth:`to_arrow` with the same ``blob_mode``. - Remote queries support ``"descriptions"`` only. + Controls how blob columns are returned. Eligible local plain scans + use Lance's scanner, which returns lazy blob handles in ``"lazy"`` + mode and also supports legacy blob columns. Other local blob v2 + queries return descriptors in ``"lazy"`` and ``"descriptions"`` + modes, or payloads in ``"bytes"`` mode, through :meth:`to_arrow`. + Remote blob queries support only ``"descriptions"`` until a stable + table snapshot can bind the query and blob fetches. **kwargs Forwarded to pyarrow.Table.to_pandas after query execution and optional flattening. @@ -3391,19 +3434,22 @@ class AsyncQueryBase(object): ) if df is not None: return df - if blob_mode == "lazy" or _query_needs_lance_scanner(self): + if _query_needs_lance_scanner(self): raise _unsupported_blob_pandas_error( "this query shape cannot use Lance native pandas conversion" ) - tbl = flatten_columns( - await self.to_arrow(timeout=timeout, blob_mode=blob_mode), flatten - ) - if blob_mode == "lazy" and schema_has_blob_field(tbl.schema): + arrow_mode = "descriptions" if blob_mode == "lazy" else blob_mode + tbl = await self.to_arrow(timeout=timeout, blob_mode=arrow_mode) + if ( + blob_mode == "lazy" + and schema_has_blob_field(tbl.schema) + and not await self._validated_blob_sources("descriptions") + ): raise _unsupported_blob_pandas_error( - "this query shape cannot use Lance native pandas conversion" + "the query has no fetchable blob v2 source" ) - return tbl.to_pandas(**kwargs) + return flatten_columns(tbl, flatten).to_pandas(**kwargs) async def _remote_to_pandas( self, @@ -3413,21 +3459,13 @@ class AsyncQueryBase(object): blob_mode: BlobMode, **kwargs, ) -> "pd.DataFrame": - # A live remote table can be replaced between query and blob fetch. - # Row ids and version numbers do not identify the producing table. tbl = await self.to_arrow(timeout=timeout) - projected_blob_paths = set(blob_column_paths(tbl.schema)) - if not projected_blob_paths: - return flatten_columns(tbl, flatten).to_pandas(**kwargs) - - if blob_mode == "descriptions": - tbl = strip_auto_row_ids(tbl, self._blob_paths) - return flatten_columns(tbl, flatten).to_pandas(**kwargs) - - raise NotImplementedError( - f"remote to_pandas(blob_mode={blob_mode!r}) cannot safely materialize " - "blob columns without a stable table snapshot; " - "use blob_mode='descriptions'" + return _finish_remote_pandas( + tbl, + blob_mode=blob_mode, + blob_paths=self._blob_paths, + flatten=flatten, + **kwargs, ) async def _plain_scan_to_pandas( diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 71e78643c..8be90115c 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -1632,7 +1632,7 @@ def test_fetch_blob_files_lazy_read(): assert handles[0].read() == payload handles[0].close() assert handles[0].closed - with pytest.raises(RuntimeError, match="already closed"): + with pytest.raises(ValueError, match="read of closed file"): handles[0].read() @@ -1681,8 +1681,8 @@ def test_blob_file_finalizer_and_close_on_runtime_worker(): explicit_handle.close() # Close remains idempotent after the worker call. try: explicit_handle.read() - except RuntimeError as error: - assert "already closed" in str(error) + except ValueError as error: + assert "read of closed file" in str(error) else: raise AssertionError("a closed blob file was readable") print("worker close completed") diff --git a/python/python/tests/test_query.py b/python/python/tests/test_query.py index a05247c5e..c8d0ad159 100644 --- a/python/python/tests/test_query.py +++ b/python/python/tests/test_query.py @@ -111,6 +111,26 @@ async def _create_blob_v2_query_table_async(db, name): return table +def _nested_blob_v2_query_schema(): + return pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("vector", pa.list_(pa.float32(), list_size=2)), + pa.field( + "info", + pa.struct([pa.field("label", pa.string()), lancedb.blob("blob")]), + ), + ] + ) + + +def _nested_blob_v2_query_rows(): + return [ + {"id": 1, "vector": [1.0, 0.0], "info": {"label": "one", "blob": b"one"}}, + {"id": 2, "vector": [2.0, 0.0], "info": {"label": "two", "blob": b"two"}}, + ] + + def _assert_lazy_blob(value, expected: bytes): assert hasattr(value, "readall") assert value.readall() == expected @@ -503,7 +523,7 @@ def test_vector_query_to_pandas_blob_mode_requires_native_path(tmp_db): storage_options=LEGACY_BLOB_STORAGE_OPTIONS, ) - with pytest.raises(RuntimeError, match="Lance native pandas conversion"): + with pytest.raises(RuntimeError, match="no fetchable blob v2 source"): table.search([1.0, 0.0]).select(["blob", "vector"]).limit(1).to_pandas( blob_mode="lazy" ) @@ -535,6 +555,193 @@ def test_vector_query_to_pandas_legacy_blob_descriptions_and_bytes(tmp_db): ] +@pytest.mark.parametrize("blob_mode", ["default", "lazy", "bytes", "descriptions"]) +@pytest.mark.parametrize("with_row_id", [None, True, False]) +def test_vector_query_to_pandas_blob_v2_modes(tmp_db, blob_mode, with_row_id): + pytest.importorskip("lance") + table = _create_blob_v2_query_table(tmp_db, f"test_vector_blob_v2_{blob_mode}") + + query = table.search([1.0, 0.0]).select(["id", "blob", "vector"]).limit(1) + if with_row_id is not None: + query = query.with_row_id(with_row_id) + df = ( + query.to_pandas() + if blob_mode == "default" + else query.to_pandas(blob_mode=blob_mode) + ) + + assert df["id"].tolist() == [1] + assert ("_rowid" in df.columns) == (with_row_id is True) + if blob_mode == "bytes": + assert df["blob"].tolist() == [b"one"] + else: + descriptor = df["blob"].iloc[0] + assert descriptor != b"one" + assert not hasattr(descriptor, "readall") + assert "_lance_row_id" not in descriptor + + +@pytest.mark.parametrize("blob_mode", ["lazy", "bytes", "descriptions"]) +def test_blob_v2_non_scan_query_shapes_to_pandas(tmp_db, blob_mode): + pytest.importorskip("lance") + table = _create_blob_v2_query_table(tmp_db, "test_blob_v2_query_shapes") + table.create_index("tag", config=FTS()) + + queries = [ + table.search("keep", query_type="fts").limit(2), + table.search(query_type="hybrid").vector([1.0, 0.0]).text("keep").limit(2), + table.take_offsets([1, 2]).select(["id", "blob"]), + ] + for query in queries: + df = query.to_pandas(blob_mode=blob_mode) + assert len(df) == 2 + if blob_mode == "bytes": + assert df["blob"].tolist() == [ + {1: b"one", 2: b"two", 3: b"three", 4: b"four"}[row_id] + for row_id in df["id"] + ] + else: + assert all("_lance_row_id" not in blob for blob in df["blob"]) + assert "_rowid" not in df.columns + + +@pytest.mark.parametrize("rename_parent", [False, True]) +def test_vector_query_to_pandas_nested_blob_bytes(tmp_db, rename_parent): + pytest.importorskip("lance") + table = tmp_db.create_table( + f"test_vector_nested_blob_{rename_parent}", + schema=_nested_blob_v2_query_schema(), + ) + table.add(_nested_blob_v2_query_rows()) + + projection = {"id": "id", "renamed": "info"} if rename_parent else ["id", "info"] + df = ( + table.search([1.0, 0.0]) + .select(projection) + .limit(2) + .to_pandas(blob_mode="bytes") + ) + + assert df["id"].tolist() == [1, 2] + parent = "renamed" if rename_parent else "info" + assert df[parent].tolist() == [ + {"label": "one", "blob": b"one"}, + {"label": "two", "blob": b"two"}, + ] + for blob_mode in ("lazy", "descriptions"): + descriptors = ( + table.search([1.0, 0.0]) + .select(projection) + .limit(1) + .to_pandas(blob_mode=blob_mode) + ) + assert "_lance_row_id" not in descriptors[parent].iloc[0]["blob"] + + +@pytest.mark.asyncio +async def test_async_vector_query_to_pandas_nested_blob_bytes(tmp_db_async): + pytest.importorskip("lance") + table = await tmp_db_async.create_table( + "test_async_vector_nested_blob", + schema=_nested_blob_v2_query_schema(), + ) + await table.add(_nested_blob_v2_query_rows()) + + df = await ( + table.vector_search([1.0, 0.0]) + .select(["id", "info"]) + .limit(2) + .to_pandas(blob_mode="bytes") + ) + + assert df["id"].tolist() == [1, 2] + assert df["info"].tolist() == [ + {"label": "one", "blob": b"one"}, + {"label": "two", "blob": b"two"}, + ] + + +def test_blob_v2_vector_projection_and_plain_scan_options(tmp_db, monkeypatch): + pytest.importorskip("lance") + table = _create_blob_v2_query_table(tmp_db, "test_blob_v2_pandas_options") + + df = ( + table.search([1.0, 0.0]) + .select({"payload": "blob", "id": "id"}) + .limit(1) + .to_pandas(blob_mode="bytes") + ) + assert df["payload"].tolist() == [b"one"] + + query = table.search().where("id = 1") + + def fail_to_arrow(*args, **kwargs): + raise AssertionError("the native scanner should handle bytes with flatten") + + monkeypatch.setattr(query, "to_arrow", fail_to_arrow) + assert query.to_pandas(blob_mode="bytes", flatten=True)["blob"].tolist() == [b"one"] + + flattened_lazy = table.search().where("id = 1").to_pandas(flatten=True) + assert flattened_lazy["id"].tolist() == [1] + assert not any("_lance_row_id" in col for col in flattened_lazy.columns) + + timed = ( + table.search() + .where("id = 1") + .to_pandas(blob_mode="bytes", timeout=timedelta(seconds=10)) + ) + assert timed["blob"].tolist() == [b"one"] + + timed_lazy = table.search().where("id = 1").to_pandas(timeout=timedelta(seconds=10)) + assert timed_lazy["id"].tolist() == [1] + + +def test_blob_v2_pandas_preserves_fetch_error(tmp_db, monkeypatch): + pytest.importorskip("lance") + table = _create_blob_v2_query_table(tmp_db, "test_blob_v2_fetch_error") + + def missing_blob(*args, **kwargs): + raise FileNotFoundError("missing obj.bin") + + monkeypatch.setattr(table, "fetch_blobs", missing_blob) + with pytest.raises(FileNotFoundError, match="missing obj.bin"): + table.search().to_pandas(blob_mode="bytes") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("blob_mode", ["lazy", "bytes", "descriptions"]) +@pytest.mark.parametrize("with_row_id", [False, True]) +async def test_async_vector_query_to_pandas_blob_v2_modes( + tmp_db_async, blob_mode, with_row_id +): + pytest.importorskip("lance") + table = await _create_blob_v2_query_table_async( + tmp_db_async, f"test_async_vector_blob_v2_{blob_mode}" + ) + + query = table.vector_search([1.0, 0.0]).limit(1) + if with_row_id: + query = query.with_row_id() + df = await query.to_pandas(blob_mode=blob_mode) + assert df["id"].tolist() == [1] + assert ("_rowid" in df.columns) == with_row_id + if blob_mode == "bytes": + assert df["blob"].tolist() == [b"one"] + else: + assert "_lance_row_id" not in df["blob"].iloc[0] + + +@pytest.mark.asyncio +async def test_async_take_query_to_pandas_blob_v2_bytes(tmp_db_async): + pytest.importorskip("lance") + table = await _create_blob_v2_query_table_async( + tmp_db_async, "test_async_take_blob_v2" + ) + + df = await table.take_offsets([1, 2]).to_pandas(blob_mode="bytes") + assert set(df["blob"]) == {b"two", b"three"} + + def test_order_by_plain_query(mem_db): table = mem_db.create_table( "test_order_by", diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index f43972a22..717490dcd 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -3539,7 +3539,7 @@ def test_remote_blob_files_are_lazy_seekable_handles(): assert empty.read_range(0, 0) == b"" alpha.close() assert alpha.closed - with pytest.raises(RuntimeError, match="already closed"): + with pytest.raises(ValueError, match="read of closed file"): alpha.read_range(0, 1) @@ -3584,6 +3584,56 @@ def test_remote_blob_query_stashes_row_ids_for_fetch(): assert blobs.to_pylist() == [b"alpha", None, b"gamma"] +def test_remote_blob_query_to_pandas(): + with blob_remote_table() as table: + query = table.search().select(["id", "image"]).limit(3) + df = query.to_pandas(blob_mode="descriptions", split_blocks=True) + + assert df["id"].tolist() == [1, 2, 3] + assert "_rowid" not in df.columns + assert df["image"].iloc[1] is None + assert "_lance_row_id" not in df["image"].iloc[0] + + +@pytest.mark.parametrize("blob_mode", ["default", "lazy", "bytes"]) +def test_remote_blob_query_to_pandas_rejects_unpinned_fetch(monkeypatch, blob_mode): + with blob_remote_table() as table: + + def fail_fetch(*args, **kwargs): + raise AssertionError("blob fetch must not use an unpinned remote table") + + monkeypatch.setattr(table, "fetch_blobs", fail_fetch) + query = table.search().select(["id", "image"]).limit(3) + pandas_kwargs = {} if blob_mode == "default" else {"blob_mode": blob_mode} + with pytest.raises(NotImplementedError, match="stable table snapshot"): + query.to_pandas(**pandas_kwargs) + + +@pytest.mark.parametrize("blob_mode", ["lazy", "bytes"]) +def test_remote_nonblob_projection_to_pandas(blob_mode): + with blob_remote_table() as table: + df = table.search().select(["id"]).to_pandas(blob_mode=blob_mode) + + assert df["id"].tolist() == [1, 2, 3] + assert list(df.columns) == ["id"] + + +@pytest.mark.asyncio +async def test_async_remote_blob_query_to_pandas_flatten(): + with blob_remote_table() as table: + df = ( + await table._table.query() + .select(["id", "image"]) + .limit(3) + .to_pandas(blob_mode="descriptions", flatten=True, split_blocks=True) + ) + + assert df["id"].tolist() == [1, 2, 3] + assert df["image.size"].iloc[0] == 5 + assert "image._lance_row_id" not in df.columns + assert "_rowid" not in df.columns + + def test_remote_blob_query_survives_a_server_that_ignores_the_row_id_request(): def handler(request): if request.path == "/v1/table/test/describe/": @@ -3613,6 +3663,8 @@ def test_remote_blob_query_survives_a_server_that_ignores_the_row_id_request(): assert "_lance_row_id" not in hits.schema.field("image").type.names with pytest.raises(ValueError, match="pass a list of row ids"): table.fetch_blobs("image", hits) + with pytest.raises(NotImplementedError, match="stable table snapshot"): + table.search().select(["id", "image"]).to_pandas(blob_mode="bytes") def test_remote_blob_byte_apis_not_supported_on_old_server(): From 9f37f3926756f999a204527b292c94dc6df89f3f Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 14 Oct 2026 17:40:04 -0700 Subject: [PATCH 14/19] fix(python): resolve the single vector column despite a query dimension mismatch (#4526) \`inf_vector_column_query\`'s \`dim\` filter (added in #3423) is meant to disambiguate between multiple vector columns of different sizes. When a schema has exactly one vector column and the query's dimension doesn't match it, the filter excluded that column too, raising "There is no vector column in the data" instead of resolving the column and letting the dimension mismatch surface at query execution. This was latent until #4409 wired client-side vector-column inference into remote search: a mismatched-dimension query against a single-vector-column remote table now fails client-side with the wrong error, before ever reaching the server's own dimension validation. Fall back to the unfiltered candidate set when the dimension filter leaves zero matches, and only raise when that is still ambiguous or genuinely empty. --- python/python/lancedb/util.py | 13 +++++++++ python/python/tests/test_table.py | 20 ++++++++++++++ python/python/tests/test_util.py | 46 ++++++++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/python/python/lancedb/util.py b/python/python/lancedb/util.py index fefc234ce..d4533887b 100644 --- a/python/python/lancedb/util.py +++ b/python/python/lancedb/util.py @@ -236,6 +236,19 @@ def inf_vector_column_query(schema: pa.Schema, dim: Optional[int] = None) -> str vector_col_names = [] for field in schema: vector_col_names.extend(_iter_vector_columns(field, [], dim)) + + if len(vector_col_names) == 0 and dim is not None: + # The query's dimension matched no column. If the schema is + # unambiguous without that filter, the single vector column is still + # the right one to search: a dimension mismatch is a query-execution + # error, not a missing-column error. + unfiltered = [] + for field in schema: + unfiltered.extend(_iter_vector_columns(field, [])) + if len(unfiltered) == 1: + return unfiltered[0] + vector_col_names = unfiltered + if len(vector_col_names) > 1: raise ValueError( "Schema has more than one vector column. " diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index a963a0d19..7ec2003b7 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -4016,6 +4016,26 @@ def test_search_nested_vector_no_candidates(mem_db: DBConnection): table.search([0.0, 1.0]).limit(1).to_arrow() +def test_search_single_vector_column_wrong_query_dim(mem_db: DBConnection): + """A query vector whose dimension doesn't match the (only) vector column + must still resolve that column, surfacing a dimension-mismatch error from + query execution rather than a misleading "no vector column" error.""" + schema = pa.schema( + [ + pa.field("id", pa.uint32(), False), + pa.field("vector_col", pa.list_(pa.float32(), 32), False), + ] + ) + table = mem_db.create_table( + "wrong_query_dim", + data=[{"id": 1, "vector_col": [0.1] * 32}], + schema=schema, + ) + + with pytest.raises(ValueError, match="dim"): + table.search([0.01] * 120).limit(1).to_arrow() + + def test_compact_cleanup(tmp_db: DBConnection): pytest.importorskip("lance") table = tmp_db.create_table( diff --git a/python/python/tests/test_util.py b/python/python/tests/test_util.py index 9deb0c073..741214794 100644 --- a/python/python/tests/test_util.py +++ b/python/python/tests/test_util.py @@ -28,7 +28,13 @@ import pandas as pd import polars as pl import pytest import lancedb -from lancedb.util import flatten_columns, get_uri_scheme, join_uri, value_to_sql +from lancedb.util import ( + flatten_columns, + get_uri_scheme, + inf_vector_column_query, + join_uri, + value_to_sql, +) from utils import exception_output @@ -1251,3 +1257,41 @@ def test_infer_vector_column_raises_clear_error(tmp_path): # Plain vector search on a table with no vector column should raise # a clear ValueError, not a cryptic TypeError. table.search([1.0, 2.0]).to_list() + + +def test_inf_vector_column_query_dim_mismatch_still_resolves(): + """Regression: a query dimension that matches no column must not hide an + otherwise-unambiguous vector column behind "no vector column in the data". + + `dim` exists to disambiguate between *multiple* vector columns of + different sizes. When the schema has exactly one vector column, that + column is still the only sensible one to search; a dimension mismatch is + a query-execution concern, not a missing-column one. This previously + broke remote search (#4409 wired up client-side inference there), which + expects the server to reject the mismatched dimension itself. + """ + schema = pa.schema( + [ + pa.field("id", pa.uint32(), False), + pa.field("vector_col", pa.list_(pa.float32(), 32), False), + pa.field("s", pa.string(), False), + ] + ) + + assert inf_vector_column_query(schema, dim=32) == "vector_col" + assert inf_vector_column_query(schema, dim=120) == "vector_col" + assert inf_vector_column_query(schema) == "vector_col" + + +def test_inf_vector_column_query_dim_mismatch_with_multiple_columns(): + """With more than one vector column, a dimension that matches none of + them is still ambiguous and must raise, not silently pick one.""" + schema = pa.schema( + [ + pa.field("vector1", pa.list_(pa.float32(), 10), False), + pa.field("vector2", pa.list_(pa.float32(), 20), False), + ] + ) + + with pytest.raises(ValueError, match="more than one vector column"): + inf_vector_column_query(schema, dim=99) From ae59de29501b2f72c021fed5d3b0864b8960d8e5 Mon Sep 17 00:00:00 2001 From: LuQQiu Date: Thu, 15 Oct 2026 12:30:28 -0700 Subject: [PATCH 15/19] ci: ignore rustls 0.23.45 advisories until 0.23.46 clears the release age policy (#4551) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The \`deny\` job fails on main and on every open PR (for example #4539) because RustSec published four advisories against rustls 0.23.45: - RUSTSEC-2026-0346: \`Debug\` impls for client session values expose TLS secrets - RUSTSEC-2026-0347: buffered application data sent to the outer server after ECH rejection - RUSTSEC-2026-0348: Brotli certificate decompression can allocate up to 1 GiB before authentication - RUSTSEC-2026-0349: unauthenticated ServerHello can downgrade later connections to classical key exchange All four are fixed in rustls 0.23.46, published 2026-10-14 09:54 UTC. The 48-hour dependency age policy does not allow that upgrade until 2026-10-16 09:54 UTC, so this mirrors lance-format/lance#9985 and ignores the four advisories with that reason. rustls reaches us only through lance-io -> aws-config -> aws-smithy-http-client. It also ignores RUSTSEC-2024-0449 (\`unsafe-libyaml\` unmaintained), reached through lance-tokenizer -> lindera -> serde_yaml_ng. No maintained upgrade exists, so this one stays until lindera moves off serde_yaml_ng. Follow-up once the window passes: drop the four rustls entries and run \`cargo update -p rustls\`. Validation: \`cargo deny --all-features check advisories bans licenses sources\` (the CI invocation) passes locally with cargo-deny 0.20.2: \`advisories ok, bans ok, licenses ok, sources ok\`. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Fable 5.1 --- deny.toml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/deny.toml b/deny.toml index 20510adc2..d112095ec 100644 --- a/deny.toml +++ b/deny.toml @@ -78,6 +78,27 @@ ignore = [ { id = "RUSTSEC-2026-0098", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" }, { id = "RUSTSEC-2026-0099", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" }, + # rustls 0.23.45: four advisories fixed in 0.23.46 (Debug impls leaking + # TLS secrets, ECH rejection buffering, Brotli certificate decompression + # allocation, ServerHello key-exchange downgrade). 0.23.46 was published + # 2026-10-14 and the dependency age policy requires releases to be 48 + # hours old, so the upgrade cannot land yet. Remove these entries and run + # `cargo update -p rustls` once the window has passed. Mirrors lance #9985. + # https://rustsec.org/advisories/RUSTSEC-2026-0346 + # https://rustsec.org/advisories/RUSTSEC-2026-0347 + # https://rustsec.org/advisories/RUSTSEC-2026-0348 + # https://rustsec.org/advisories/RUSTSEC-2026-0349 + { id = "RUSTSEC-2026-0346", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" }, + { id = "RUSTSEC-2026-0347", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" }, + { id = "RUSTSEC-2026-0348", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" }, + { id = "RUSTSEC-2026-0349", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" }, + + # unsafe-libyaml: unmaintained. Transitive via lance-tokenizer → lindera + # → serde_yaml_ng. No maintained upgrade exists; clearing it requires + # lindera to move off serde_yaml_ng. + # https://rustsec.org/advisories/RUSTSEC-2024-0449 + { id = "RUSTSEC-2024-0449", reason = "transitive via lindera (lance-tokenizer); no maintained upgrade available" }, + # rustls-webpki 0.101.7: reachable panic in CRL parsing. Same legacy # rustls 0.21 chain from aws-smithy-http-client as above. The 0.103 line # we actively use is upgraded to 0.103.13 which contains the fix. From 486f6c66bbbc5229d114b5a59150c1d4a6f3cb66 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Thu, 15 Oct 2026 14:50:47 -0700 Subject: [PATCH 16/19] docs: revise getting started and concepts (#4491) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR is part of the documentation revamp. Once the full set of PRs is merged, we’ll publish the combined site to staging for review before updating docs.lancedb.com. The introductory pages are inconsistent, and the old PR split asked reviewers to review some pages twice. This brings the final quickstart, connection guide, architecture and Concepts pages together. It clarifies OSS and Enterprise, simplifies naming rules and includes the navigation and diagram fixes from #4489. Function and Secret details on the Concepts pages are MDX comments until their release, and CONTRIBUTING.md explains how to write docs before a release. Includes #4532. Ingestion, search and integrations are in #4533; data workflows, SQL and operations are in #4534. Each MDX file has one PR owner. Local assembly, Mint validation and link checks pass. The Function pages stay published from `main` until #4534 hides them, so hold publication until the complete stack is integrated. --- CONTRIBUTING.md | 40 +- docs/web-tests/py/test_connection.py | 106 +++- docs/web-tests/py/test_quickstart.py | 397 +++++++----- docs/web-tests/py/test_read_consistency.py | 35 ++ docs/web-tests/rs/Cargo.lock | 345 ++++++---- docs/web-tests/rs/Cargo.toml | 4 +- docs/web-tests/rs/connection.rs | 97 ++- docs/web-tests/rs/quickstart.rs | 106 ++-- docs/web-tests/ts/connection.test.ts | 118 ++-- docs/web-tests/ts/quickstart.test.ts | 395 +++++++----- docs/web/architecture.mdx | 199 +++--- docs/web/basics/agentic-skills.mdx | 9 +- docs/web/basics/catalog.mdx | 139 ++--- docs/web/basics/connect.mdx | 132 ++++ docs/web/basics/guarantees.mdx | 161 ++--- docs/web/basics/jobs.mdx | 243 ++------ docs/web/basics/lakehouse.mdx | 137 ++-- docs/web/basics/lance.mdx | 135 +--- docs/web/basics/limits.mdx | 137 ++-- docs/web/basics/naming.mdx | 69 ++ docs/web/basics/offerings.mdx | 70 +++ docs/web/basics/query.mdx | 113 ++-- docs/web/basics/sdk.mdx | 146 +++-- docs/web/basics/sql.mdx | 196 +++--- docs/web/docs.json | 9 +- docs/web/quickstart.mdx | 590 ++++++------------ docs/web/snippets/connection.mdx | 24 +- docs/web/snippets/quickstart.mdx | 52 +- docs/web/snippets/read_consistency.mdx | 4 + .../images/overview/lancedb-architecture.svg | 60 ++ .../assets/images/overview/lancedb-suite.svg | 19 +- .../overview/training-data-lifecycle.svg | 11 +- docs/web/tables/consistency.mdx | 197 ++---- 33 files changed, 2325 insertions(+), 2170 deletions(-) create mode 100644 docs/web-tests/py/test_read_consistency.py create mode 100644 docs/web/basics/connect.mdx create mode 100644 docs/web/basics/naming.mdx create mode 100644 docs/web/basics/offerings.mdx create mode 100644 docs/web/snippets/read_consistency.mdx create mode 100644 docs/web/static/assets/images/overview/lancedb-architecture.svg diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a13086061..0aabd8fec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -94,7 +94,7 @@ what the capability is, and what the embedded form does instead. A separate private repository supplies the fuller version of each, which replaces the page at the same path when the published site is assembled. -Two things to know before editing: +Four things to know before editing: **Code examples are not written into the pages.** They live in real tests under `docs/web-tests/{py,ts,rs}`, are extracted into `docs/web/snippets/`, and are @@ -116,6 +116,44 @@ Enterprise pages attach their additions to the right section, so they are assigned once and never regenerated. Leave an existing anchor alone even when you reword the heading above it; only new headings need a new one. +**A page can declare where it applies.** Its frontmatter says whether LanceDB +OSS and LanceDB Enterprise support the feature: use only `available` or +`unavailable`, and one sentence that explains its scope. Availability describes +product support, not whether a particular deployment has enabled or configured +the feature. A supported feature stays `available` when it requires setup, a +supported SDK or server version, or has differences between offerings. State +those requirements in the summary or the relevant section, including unsupported +subfeatures. Do not use `varies`, infer support from an unknown status, or mark an +absent capability available. Narrow a page to the capability it actually covers +when necessary. For example: + +```yaml +availability: + oss: unavailable + enterprise: available + summary: >- + Requests to a deployment carry an API key or an OAuth token. Embedded + LanceDB has no server to sign in to. +``` + +The assembled site renders this as a label above the page's content and as a +sidebar tag when only one offering has the feature. Do not also open the page +with a badge of its own, and keep a qualification that only one section needs +in that section's text. A `mint dev` preview of `docs/web` alone renders +neither the labels nor the tags. + +**A feature can be documented before its release.** List a draft page in +`docs/web/.mintignore`, keep it out of the navigation in `docs.json`, and do not +link to it from published pages. Wrap a draft section in an MDX comment, +`{/* ... */}`, whose first line names the release, such as +`Draft for the Feature X release.` Keep draft table rows and list items in a +comment below the table or list, since a comment between rows or items breaks +it. Drafts are hidden from readers but stay in the source, and availability does +not change what is published. At release, remove the comments and the +`.mintignore` entries, and restore the navigation and links. See +[Writing docs before release](https://github.com/lancedb/docs#writing-docs-before-release) +in the documentation repository's README for previews and release checks. + ## Release process For information on the release process, see: [release_process.md](release_process.md) diff --git a/docs/web-tests/py/test_connection.py b/docs/web-tests/py/test_connection.py index 783ac2d8d..f5c1f2b22 100644 --- a/docs/web-tests/py/test_connection.py +++ b/docs/web-tests/py/test_connection.py @@ -4,6 +4,8 @@ from pathlib import Path import shutil +import pytest + def test_connection(): # --8<-- [start:connect] @@ -29,19 +31,16 @@ async def connect_async_example(): def connect_enterprise_quickstart_config(): + # --8<-- [start:connect_enterprise_quickstart] + import os + import lancedb - # --8<-- [start:connect_enterprise_quickstart] - uri = "db://your-database-uri" - api_key = "your-api-key" - region = "us-east-1" - host_override = "https://your-enterprise-endpoint.com" - db = lancedb.connect( - uri=uri, - api_key=api_key, - region=region, - host_override=host_override, + uri=os.environ["LANCEDB_URI"], # db://your-database + api_key=os.environ["LANCEDB_API_KEY"], + region=os.environ.get("LANCEDB_REGION", "us-east-1"), + host_override=os.environ["LANCEDB_HOST_OVERRIDE"], # https://your-endpoint ) # --8<-- [end:connect_enterprise_quickstart] return db @@ -57,17 +56,29 @@ def test_connect_enterprise_quickstart(monkeypatch): return object() monkeypatch.setattr(lancedb, "connect", fake_connect) + monkeypatch.setenv("LANCEDB_URI", "db://your-database") + monkeypatch.setenv("LANCEDB_API_KEY", "your-api-key") + monkeypatch.delenv("LANCEDB_REGION", raising=False) + monkeypatch.setenv("LANCEDB_HOST_OVERRIDE", "https://your-endpoint") - db = connect_enterprise_quickstart_config() - assert db is not None + assert connect_enterprise_quickstart_config() is not None assert captured == { - "uri": "db://your-database-uri", + "uri": "db://your-database", "api_key": "your-api-key", "region": "us-east-1", - "host_override": "https://your-enterprise-endpoint.com", + "host_override": "https://your-endpoint", } +def test_connect_enterprise_quickstart_requires_an_endpoint(monkeypatch): + monkeypatch.setenv("LANCEDB_URI", "db://your-database") + monkeypatch.setenv("LANCEDB_API_KEY", "your-api-key") + monkeypatch.delenv("LANCEDB_HOST_OVERRIDE", raising=False) + + with pytest.raises(KeyError, match="LANCEDB_HOST_OVERRIDE"): + connect_enterprise_quickstart_config() + + def connect_object_storage_config(): # --8<-- [start:connect_object_storage] import lancedb @@ -105,14 +116,26 @@ def namespace_table_ops_example(): mode="create", # use "overwrite" only if you want to replace existing table ) - # Verify - print(db.list_namespaces()) # ['prod'] - print(db.list_namespaces(namespace_path=["prod"])) # ['recommendations', 'search'] - print(db.list_tables(namespace_path=["prod", "search"])) # ['user'] - print(db.list_tables(namespace_path=["prod", "recommendations"])) # ['user'] + # Verify: listings return a response with the names and a page token + print(db.list_namespaces().namespaces) # ['prod'] + print(db.list_namespaces(namespace_path=["prod"]).namespaces) + # ['recommendations', 'search'] + print(db.list_tables(namespace_path=["prod", "search"]).tables) # ['user'] + print(db.list_tables(namespace_path=["prod", "recommendations"]).tables) # ['user'] # --8<-- [end:namespace_table_ops] +def test_namespace_table_ops(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + namespace_table_ops_example() + assert capsys.readouterr().out.splitlines() == [ + "['prod']", + "['recommendations', 'search']", + "['user']", + "['user']", + ] + + def namespace_admin_ops_example(): # --8<-- [start:namespace_admin_ops] import lancedb @@ -124,8 +147,8 @@ def namespace_admin_ops_example(): db.create_namespace(["prod", "search"]) child_namespaces = db.list_namespaces(namespace_path=["prod"]).namespaces - print(f"Child namespaces under {namespace}: {child_namespaces}") - # Child namespaces under ['prod', 'search']: ['search'] + print(f"Child namespaces under ['prod']: {child_namespaces}") + # Child namespaces under ['prod']: ['search'] metadata = db.describe_namespace(["prod", "search"]) print(f"Metadata for namespace {namespace}: {metadata}") @@ -134,7 +157,17 @@ def namespace_admin_ops_example(): db.drop_namespace(["prod", "search"], mode="skip") db.drop_namespace(["prod"], mode="skip") # --8<-- [end:namespace_admin_ops] - return child_namespaces, metadata + return db, child_namespaces, metadata + + +def test_namespace_admin_ops(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + db, child_namespaces, metadata = namespace_admin_ops_example() + assert child_namespaces == ["search"] + assert capsys.readouterr().out.splitlines()[0] == ( + "Child namespaces under ['prod']: ['search']" + ) + assert db.list_namespaces().namespaces == [] async def connect_object_storage_config_async(): # --8<-- [start:connect_object_storage_async] @@ -146,3 +179,32 @@ async def connect_object_storage_config_async(): # --8<-- [end:connect_object_storage_async] return async_db + + +def test_connect_enterprise_placeholders(monkeypatch): + import lancedb + + captured = {} + + def fake_connect(**kwargs): + captured.update(kwargs) + return object() + + monkeypatch.setattr(lancedb, "connect", fake_connect) + # --8<-- [start:connect_enterprise_placeholders] + import lancedb + + db = lancedb.connect( + uri="db://your-database", + api_key="your-api-key", + region="us-east-1", + host_override="https://your-endpoint", + ) + # --8<-- [end:connect_enterprise_placeholders] + assert db is not None + assert captured == { + "uri": "db://your-database", + "api_key": "your-api-key", + "region": "us-east-1", + "host_override": "https://your-endpoint", + } diff --git a/docs/web-tests/py/test_quickstart.py b/docs/web-tests/py/test_quickstart.py index ee01e7317..779a268d4 100644 --- a/docs/web-tests/py/test_quickstart.py +++ b/docs/web-tests/py/test_quickstart.py @@ -1,14 +1,125 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import os +import shutil +import uuid +from pathlib import Path + import lancedb import pytest +import pytest_asyncio + +# What the quickstart's download step leaves in an application's directory. +IMAGE = ( + Path(__file__).resolve().parents[2] + / "web/static/assets/images/quickstart/sir-lancelot.jpg" +) +DEPLOYMENT = ("LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY") +# The connection settings alone never send these tests to a deployment. +OPT_IN = "LANCEDB_DOCS_TEST_DEPLOYMENT" -def test_quickstart(db_path_factory): - uri = db_path_factory("quickstart_db") - db = lancedb.connect(uri) +def deployment_settings(): + if os.environ.get(OPT_IN) != "1": + pytest.skip(f"set {OPT_IN}=1 and {', '.join(DEPLOYMENT)} to use a deployment") + missing = [name for name in DEPLOYMENT if not os.environ.get(name)] + if missing: + pytest.fail(f"{OPT_IN}=1 needs {', '.join(missing)}") + return dict( + uri=os.environ["LANCEDB_URI"], + api_key=os.environ["LANCEDB_API_KEY"], + region=os.environ.get("LANCEDB_REGION", "us-east-1"), + host_override=os.environ["LANCEDB_HOST_OVERRIDE"], + ) + +class OwnedTables: + """A deployment connection on which a test creates only tables of its own. + + Each name gets a suffix unique to the test and is claimed only while no + table has it; the test drops what it claimed, whether or not it passed. + """ + + def __init__(self, db): + self.db = db + self.suffix = uuid.uuid4().hex[:12] + self.claimed = [] + + def claim(self, name, existing, mode): + owned = f"{name}_{self.suffix}" + if owned in self.claimed: + return owned, mode + if owned in existing: + raise RuntimeError(f"{owned} already exists and is not this test's") + self.claimed.append(owned) + return owned, "create" + + def table_names(self): + names, token = [], None + while True: + page = self.db.list_tables(page_token=token) + names += page.tables + if not (token := page.page_token): + return names + + def create_table(self, name, *args, mode="create", **kwargs): + owned, mode = self.claim(name, self.table_names(), mode) + return self.db.create_table(owned, *args, mode=mode, **kwargs) + + def drop_claimed(self): + existing = self.table_names() + for name in self.claimed: + if name in existing: + self.db.drop_table(name) + + +class AsyncOwnedTables(OwnedTables): + async def table_names(self): + names, token = [], None + while True: + page = await self.db.list_tables(page_token=token) + names += page.tables + if not (token := page.page_token): + return names + + async def create_table(self, name, *args, mode="create", **kwargs): + owned, mode = self.claim(name, await self.table_names(), mode) + return await self.db.create_table(owned, *args, mode=mode, **kwargs) + + async def drop_claimed(self): + existing = await self.table_names() + for name in self.claimed: + if name in existing: + await self.db.drop_table(name) + + +@pytest.fixture +def deployment(): + owned = OwnedTables(lancedb.connect(**deployment_settings())) + yield owned + owned.drop_claimed() + + +@pytest.fixture(params=["local", "enterprise"]) +def db(request, db_path_factory): + if request.param == "local": + yield lancedb.connect(db_path_factory("quickstart_db")) + else: + yield request.getfixturevalue("deployment") + + +@pytest_asyncio.fixture(params=["local", "enterprise"]) +async def async_db(request, db_path_factory): + if request.param == "local": + yield await lancedb.connect_async(db_path_factory("quickstart_async_db")) + return + owned = AsyncOwnedTables(await lancedb.connect_async(**deployment_settings())) + yield owned + await owned.drop_claimed() + + +def create_characters(db): # --8<-- [start:quickstart_data] data = [ { @@ -41,14 +152,29 @@ def test_quickstart(db_path_factory): # --8<-- [start:quickstart_create_table] table = db.create_table("characters", data=data, mode="overwrite") # --8<-- [end:quickstart_create_table] - assert len(table) == 3 - # Drop the table to test create without overwrite - db.drop_table("characters") + assert table.count_rows() == 3 + + # --8<-- [start:quickstart_add_data] + table.add( + [ + { + "id": "4", + "name": "Morgana", + "role": "Sorceress", + "description": "Powerful sorceress of Avalon.", + "stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4}, + "vector": [0.3, 0.9, 0.6, 0.8], + } + ] + ) + # --8<-- [end:quickstart_add_data] + assert table.count_rows() == 4 + return table + + +def test_quickstart(db): + table = create_characters(db) - # --8<-- [start:quickstart_create_table_no_overwrite] - table = db.create_table("characters", data=data) - # --8<-- [end:quickstart_create_table_no_overwrite] - assert len(table) == 3 # --8<-- [start:quickstart_vector_search_1] # Search for examples similar to a "wise magical advisor" query_vector = [0.2, 0.8, 0.4, 0.9] @@ -62,7 +188,14 @@ def test_quickstart(db_path_factory): ) print(result) # --8<-- [end:quickstart_vector_search_1] - assert result.head(1)["name"][0] == "Merlin" + assert result["name"].to_list() == ["Merlin", "Morgana"] + + # --8<-- [start:quickstart_output_pandas] + # Ensure you run `pip install pandas` beforehand + result = table.search(query_vector).limit(2).to_pandas() + print(result) + # --8<-- [end:quickstart_output_pandas] + assert result["name"].to_list() == ["Merlin", "Morgana"] # --8<-- [start:quickstart_curate_with_metadata] curated = ( @@ -74,131 +207,15 @@ def test_quickstart(db_path_factory): ) print(curated) # --8<-- [end:quickstart_curate_with_metadata] - assert curated.head(1)["name"][0] == "Merlin" + assert curated["name"].to_list() == ["Merlin", "Morgana"] - # --8<-- [start:quickstart_output_pandas] - # Ensure you run `pip install pandas` beforehand - result = table.search(query_vector).limit(2).to_pandas() - print(result) - # --8<-- [end:quickstart_output_pandas] - assert result.iloc[0]["name"] == "Merlin" - - # --8<-- [start:quickstart_add_feature] - table.add_columns( - { - "power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)" - } - ) - # --8<-- [end:quickstart_add_feature] - assert "power_score" in table.schema.names - - # --8<-- [start:quickstart_query_feature] - features = table.search().select(["name", "role", "power_score"]).to_polars() - print(features) - # --8<-- [end:quickstart_query_feature] - assert "power_score" in features.columns - - # --8<-- [start:quickstart_multimodal_bytes] - from pathlib import Path - - image_path = Path("docs/web/static/assets/images/quickstart/sir-lancelot.jpg") - image_bytes = image_path.read_bytes() - - multimodal_table = db.create_table( - "character_images", - data=[ - { - "id": "lancelot", - "description": "Portrait of Sir Lancelot", - "image": image_bytes, - "vector": [0.9, 0.1, 0.5, 0.4], - } - ], - mode="overwrite", - ) - # --8<-- [end:quickstart_multimodal_bytes] - assert len(multimodal_table) == 1 - - # --8<-- [start:quickstart_open_table] - table = db.open_table("characters") - # --8<-- [end:quickstart_open_table] - - # --8<-- [start:quickstart_add_data] - more_data = [ - { - "id": "4", - "name": "Morgana", - "role": "Sorceress", - "description": "Powerful sorceress of Avalon.", - "stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4}, - "vector": [0.3, 0.9, 0.6, 0.8], - "power_score": 3.75, - }, - ] - - # Add data to table - table.add(more_data) - # --8<-- [end:quickstart_add_data] - assert len(table) == 4 - - # --8<-- [start:quickstart_vector_search_2] - # Search for examples similar to a "powerful sorceress" - query_vector = [0.3, 0.9, 0.6, 0.8] - - results = table.search(query_vector).limit(2).to_polars() - print(results) - # --8<-- [end:quickstart_vector_search_2] - assert results.head(1)["name"][0] == "Morgana" - - # --8<-- [start:quickstart_computed_feature] - table.add_columns(computed={"combat_score": "(stats.magic + stats.strength) / 2.0"}) - table.refresh_column("combat_score") - # --8<-- [end:quickstart_computed_feature] - scores = ( - table.search().select(["combat_score"]).to_arrow()["combat_score"].to_pylist() - ) - assert scores == [2.5, 3.5, 3.0, 3.5] - - # --8<-- [start:quickstart_training] - import torch - from lancedb.permutation import Permutation - - dataset = Permutation.identity(table).select_columns( - ["power_score", "combat_score"] - ) - dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True) - # --8<-- [end:quickstart_training] - batches = list(dataloader) - assert len(batches) == 1 - assert set(batches[0]) == {"power_score", "combat_score"} - assert all(isinstance(value, torch.Tensor) for value in batches[0].values()) - assert sorted(batches[0]["combat_score"].tolist()) == sorted(scores) - - # --8<-- [start:quickstart_training_projection] - dataset = Permutation.identity(table).select_columns(["power_score"]) - dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024) - # --8<-- [end:quickstart_training_projection] - batch = next(iter(dataloader)) - assert set(batch) == {"power_score"} - assert batch["power_score"].tolist() == [3.5, 4.0, 3.0, 3.75] + curated = table.search(query_vector).where("stats.magic < 4").limit(2).to_list() + assert [row["name"] for row in curated] == ["King Arthur", "Sir Lancelot"] @pytest.mark.asyncio -async def test_quickstart_async_api(db_path_factory): - db_uri = db_path_factory("quickstart_async_db") - import lancedb - async_db = await lancedb.connect_async(db_uri) - - # --8<-- [start:quickstart_data_async] +async def test_quickstart_async_api(async_db): data = [ - { - "id": "1", - "name": "King Arthur", - "role": "King", - "description": "Leader of Camelot and wielder of Excalibur.", - "stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4}, - "vector": [0.7, 0.1, 0.9, 0.7], - }, { "id": "2", "name": "Merlin", @@ -206,17 +223,8 @@ async def test_quickstart_async_api(db_path_factory): "description": "Advisor and prophet with deep magical knowledge.", "stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5}, "vector": [0.2, 0.9, 0.4, 0.9], - }, - { - "id": "3", - "name": "Sir Lancelot", - "role": "Knight", - "description": "Legendary knight known for courage and combat skill.", - "stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3}, - "vector": [0.9, 0.1, 0.5, 0.4], - }, + } ] - # --8<-- [end:quickstart_data_async] # --8<-- [start:quickstart_create_table_async] async_table = await async_db.create_table( @@ -236,5 +244,110 @@ async def test_quickstart_async_api(db_path_factory): ).select(["name", "role", "description", "_distance"]).limit(2).to_polars() print(async_result) # --8<-- [end:quickstart_vector_search_1_async] + assert async_result["name"].to_list() == ["Merlin"] - assert async_result.head(1)["name"][0] == "Merlin" + +def test_quickstart_derived_feature(db): + table = create_characters(db) + + # --8<-- [start:quickstart_add_feature] + table.add_columns( + { + "power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)" + } + ) + # --8<-- [end:quickstart_add_feature] + + # --8<-- [start:quickstart_query_feature] + features = table.search().select(["name", "role", "power_score"]).to_polars() + print(features) + # --8<-- [end:quickstart_query_feature] + assert sorted(zip(features["name"], features["power_score"])) == [ + ("King Arthur", 3.5), + ("Merlin", 4.0), + ("Morgana", 3.75), + ("Sir Lancelot", 3.0), + ] + + +def description_lengths(table): + rows = table.search().select(["description", "description_length"]).to_list() + return {row["description"]: row["description_length"] for row in rows} + + +def test_quickstart_computed_column(db_path_factory): + table = create_characters(lancedb.connect(db_path_factory("computed"))) + + # --8<-- [start:quickstart_computed_feature] + table.add_columns(computed={"description_length": "length(description)"}) + table.refresh_column("description_length") + # --8<-- [end:quickstart_computed_feature] + lengths = description_lengths(table) + assert lengths == {text: len(text) for text in lengths} + + +def test_quickstart_computed_column_on_a_deployment(deployment): + table = create_characters(deployment) + + # --8<-- [start:quickstart_computed_feature_enterprise] + table.add_columns(computed={"description_length": "length(description)"}) + job = table.refresh_column_async("description_length") + job.wait() + # --8<-- [end:quickstart_computed_feature_enterprise] + lengths = description_lengths(table) + assert len(lengths) == 4 + assert lengths == {text: len(text) for text in lengths} + with pytest.raises(Exception): + table.refresh_column("description_length") + + +def test_quickstart_multimodal(db, tmp_path, monkeypatch): + app = tmp_path / "app" + (app / "data").mkdir(parents=True) + shutil.copy(IMAGE, app / "data/sir-lancelot.jpg") + monkeypatch.chdir(app) + + # --8<-- [start:quickstart_multimodal_bytes] + from pathlib import Path + + image_bytes = Path("data/sir-lancelot.jpg").read_bytes() + + multimodal_table = db.create_table( + "character_images", + data=[ + { + "id": "lancelot", + "description": "Portrait of Sir Lancelot", + "image": image_bytes, + "vector": [0.9, 0.1, 0.5, 0.4], + } + ], + mode="overwrite", + ) + # --8<-- [end:quickstart_multimodal_bytes] + assert multimodal_table.count_rows() == 1 + stored = multimodal_table.search().select(["image"]).to_list()[0]["image"] + assert stored == IMAGE.read_bytes() + + +def test_quickstart_training(db): + table = create_characters(db) + + # --8<-- [start:quickstart_training] + import torch + from lancedb.permutation import Permutation + + table.add_columns( + { + "magic": "cast(stats.magic as float)", + "strength": "cast(stats.strength as float)", + } + ) + dataset = Permutation.identity(table).select_columns(["magic", "strength"]) + dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True) + # --8<-- [end:quickstart_training] + batches = list(dataloader) + assert len(batches) == 1 + assert set(batches[0]) == {"magic", "strength"} + assert all(isinstance(value, torch.Tensor) for value in batches[0].values()) + assert sorted(batches[0]["magic"].tolist()) == [1.0, 1.0, 5.0, 5.0] diff --git a/docs/web-tests/py/test_read_consistency.py b/docs/web-tests/py/test_read_consistency.py new file mode 100644 index 000000000..886bb5676 --- /dev/null +++ b/docs/web-tests/py/test_read_consistency.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + + +def test_read_consistency(): + # --8<-- [start:read_consistency] + from datetime import timedelta + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-consistency-") as uri: + writer = lancedb.connect(uri) + table = writer.create_table("documents", [{"id": 1, "text": "first"}]) + + default_reader = lancedb.connect(uri).open_table("documents") + latest_reader = lancedb.connect( + uri, read_consistency_interval=timedelta(0) + ).open_table("documents") + + table.add([{"id": 2, "text": "second"}]) + + default_count = default_reader.count_rows() + latest_count = latest_reader.count_rows() + print(f"Default reader: {default_count} rows") # 1 + print(f"Zero-interval reader: {latest_count} rows") # 2 + + default_reader.checkout_latest() + refreshed_count = default_reader.count_rows() + print(f"After explicit refresh: {refreshed_count} rows") # 2 + # --8<-- [end:read_consistency] + + assert default_count == 1 + assert latest_count == 2 + assert refreshed_count == 2 diff --git a/docs/web-tests/rs/Cargo.lock b/docs/web-tests/rs/Cargo.lock index 599633c5e..915e15e58 100644 --- a/docs/web-tests/rs/Cargo.lock +++ b/docs/web-tests/rs/Cargo.lock @@ -452,6 +452,61 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.76" @@ -845,6 +900,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -1769,26 +1834,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "deepsize" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c" -dependencies = [ - "deepsize_derive", -] - -[[package]] -name = "deepsize_derive" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "deranged" version = "0.5.5" @@ -2416,6 +2461,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humantime" version = "2.3.0" @@ -2436,6 +2487,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "pin-utils", @@ -2480,9 +2532,11 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -2938,15 +2992,15 @@ dependencies = [ "half", "humantime", "itertools 0.13.0", - "lance-arrow 8.0.0", - "lance-core 8.0.0", + "lance-arrow", + "lance-core", "lance-datafusion", "lance-encoding", "lance-file", "lance-index", "lance-io", "lance-linalg", - "lance-namespace 8.0.0", + "lance-namespace", "lance-select", "lance-table", "lance-tokenizer", @@ -2974,28 +3028,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "lance-arrow" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "253f4a0a70580c985b91e65e9ca6cad644825a4078de28d8efbacf3ffbd7ecdc" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ipc", - "arrow-ord", - "arrow-schema", - "arrow-select", - "bytes", - "futures", - "getrandom 0.2.16", - "half", - "jsonb", - "num-traits", - "rand 0.9.5", -] - [[package]] name = "lance-arrow" version = "8.0.0" @@ -3056,41 +3088,6 @@ dependencies = [ "seq-macro", ] -[[package]] -name = "lance-core" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13f84020da5a484e2f07dd1796e09785ed7cd889857ebc4cb77e32ef214ee594" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", - "async-trait", - "byteorder", - "bytes", - "deepsize", - "futures", - "itertools 0.13.0", - "lance-arrow 7.0.0", - "libc", - "log", - "moka", - "num_cpus", - "object_store", - "pin-project", - "prost", - "rand 0.9.5", - "roaring", - "serde_json", - "snafu 0.9.1", - "tempfile", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", - "url", -] - [[package]] name = "lance-core" version = "8.0.0" @@ -3108,7 +3105,7 @@ dependencies = [ "datafusion-sql", "futures", "itertools 0.13.0", - "lance-arrow 8.0.0", + "lance-arrow", "lance-derive", "libc", "libm", @@ -3152,8 +3149,8 @@ dependencies = [ "datafusion-physical-expr", "futures", "jsonb", - "lance-arrow 8.0.0", - "lance-core 8.0.0", + "lance-arrow", + "lance-core", "lance-datagen", "log", "pin-project", @@ -3214,9 +3211,9 @@ dependencies = [ "hex", "hyperloglogplus", "itertools 0.13.0", - "lance-arrow 8.0.0", + "lance-arrow", "lance-bitpacking", - "lance-core 8.0.0", + "lance-core", "log", "lz4", "num-traits", @@ -3248,8 +3245,8 @@ dependencies = [ "bytes", "datafusion-common", "futures", - "lance-arrow 8.0.0", - "lance-core 8.0.0", + "lance-arrow", + "lance-core", "lance-encoding", "lance-io", "log", @@ -3294,9 +3291,9 @@ dependencies = [ "itertools 0.13.0", "jieba-rs", "jsonb", - "lance-arrow 8.0.0", + "lance-arrow", "lance-arrow-stats", - "lance-core 8.0.0", + "lance-core", "lance-datafusion", "lance-datagen", "lance-encoding", @@ -3351,9 +3348,9 @@ dependencies = [ "futures", "http", "io-uring", - "lance-arrow 8.0.0", - "lance-core 8.0.0", - "lance-namespace 8.0.0", + "lance-arrow", + "lance-core", + "lance-namespace", "log", "moka", "object_store", @@ -3379,26 +3376,12 @@ dependencies = [ "arrow-schema", "cc", "half", - "lance-arrow 8.0.0", - "lance-core 8.0.0", + "lance-arrow", + "lance-core", "num-traits", "rand 0.9.5", ] -[[package]] -name = "lance-namespace" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "014e8332ca0615506342e0d3af608639864b68396973be14239f09c9f21f1fc2" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "lance-core 7.0.0", - "lance-namespace-reqwest-client 0.7.7", - "snafu 0.9.1", -] - [[package]] name = "lance-namespace" version = "8.0.0" @@ -3408,8 +3391,8 @@ dependencies = [ "arrow", "async-trait", "bytes", - "lance-core 8.0.0", - "lance-namespace-reqwest-client 0.8.6", + "lance-core", + "lance-namespace-reqwest-client", "snafu 0.9.1", ] @@ -3423,42 +3406,33 @@ dependencies = [ "arrow-ipc", "arrow-schema", "async-trait", + "axum", "bytes", "datafusion-common", "datafusion-physical-plan", "futures", "lance", - "lance-core 8.0.0", + "lance-core", "lance-index", "lance-io", "lance-linalg", - "lance-namespace 8.0.0", + "lance-namespace", "lance-table", "log", "object_store", "rand 0.9.5", + "reqwest", "roaring", + "serde", "serde_json", "time", "tokio", + "tower", + "tower-http 0.5.2", "url", "uuid", ] -[[package]] -name = "lance-namespace-reqwest-client" -version = "0.7.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6369eee4682fb11edf538388b43c61ce288b8302fe89bb40944d7daa7faaae99" -dependencies = [ - "reqwest", - "serde", - "serde_json", - "serde_repr", - "serde_with", - "url", -] - [[package]] name = "lance-namespace-reqwest-client" version = "0.8.6" @@ -3485,7 +3459,7 @@ dependencies = [ "byteorder", "bytes", "itertools 0.13.0", - "lance-core 8.0.0", + "lance-core", "roaring", "tracing", ] @@ -3506,8 +3480,8 @@ dependencies = [ "bytes", "chrono", "futures", - "lance-arrow 8.0.0", - "lance-core 8.0.0", + "lance-arrow", + "lance-core", "lance-file", "lance-io", "lance-select", @@ -3538,7 +3512,7 @@ dependencies = [ "arrow-array", "arrow-schema", "criterion", - "lance-arrow 8.0.0", + "lance-arrow", "num-traits", "pprof", "rand 0.9.5", @@ -3589,9 +3563,10 @@ dependencies = [ "datafusion-sql", "futures", "half", + "http", "lance", - "lance-arrow 8.0.0", - "lance-core 8.0.0", + "lance-arrow", + "lance-core", "lance-datafusion", "lance-datagen", "lance-encoding", @@ -3599,7 +3574,7 @@ dependencies = [ "lance-index", "lance-io", "lance-linalg", - "lance-namespace 8.0.0", + "lance-namespace", "lance-namespace-impls", "lance-table", "lance-testing", @@ -3613,6 +3588,7 @@ dependencies = [ "polars-arrow", "rand 0.9.5", "regex", + "reqwest", "semver", "serde", "serde_json", @@ -3621,6 +3597,7 @@ dependencies = [ "tempfile", "tokio", "url", + "urlencoding", "uuid", ] @@ -3867,6 +3844,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "matrixmultiply" version = "0.3.10" @@ -5391,7 +5374,7 @@ dependencies = [ "tokio-rustls", "tokio-util", "tower", - "tower-http", + "tower-http 0.6.8", "tower-service", "url", "wasm-bindgen", @@ -5472,7 +5455,7 @@ dependencies = [ "arrow-schema", "futures", "futures-util", - "lance-namespace 7.0.0", + "lance-namespace", "lancedb", "polars", "polars-arrow", @@ -5659,7 +5642,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ "bitflags 2.10.0", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -5730,6 +5713,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_repr" version = "0.1.20" @@ -6203,6 +6197,27 @@ dependencies = [ "windows 0.52.0", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -6444,6 +6459,24 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.10.0", + "bytes", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", ] [[package]] @@ -6487,6 +6520,7 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -6641,6 +6675,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8-ranges" version = "1.0.5" @@ -6927,8 +6967,8 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link 0.1.3", - "windows-result", - "windows-strings", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] @@ -6986,6 +7026,17 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + [[package]] name = "windows-result" version = "0.3.4" @@ -6995,6 +7046,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-strings" version = "0.4.2" @@ -7004,6 +7064,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-sys" version = "0.52.0" diff --git a/docs/web-tests/rs/Cargo.toml b/docs/web-tests/rs/Cargo.toml index c8a4cc7ca..6b68f5fff 100644 --- a/docs/web-tests/rs/Cargo.toml +++ b/docs/web-tests/rs/Cargo.toml @@ -9,8 +9,8 @@ arrow-json = "58.3.0" arrow-schema = "58.3.0" futures = "0.3.31" futures-util = "0.3.31" -lance-namespace = "7.0.0" -lancedb = { version = "0.31.0", features = ["polars"] } +lance-namespace = "8.0.0" +lancedb = { version = "0.31.0", features = ["polars", "remote"] } polars = ">=0.37,<0.40.0" polars-arrow = ">=0.37,<0.40.0" serde = { version = "1", features = ["derive"] } diff --git a/docs/web-tests/rs/connection.rs b/docs/web-tests/rs/connection.rs index 4f7212ef5..06fbf879f 100644 --- a/docs/web-tests/rs/connection.rs +++ b/docs/web-tests/rs/connection.rs @@ -3,48 +3,69 @@ use std::path::PathBuf; -use lancedb::connect; +use lancedb::{Connection, connect}; -// --8<-- [start:connect] -async fn connect_example(uri: &str) { - let db = connect(uri).execute().await.unwrap(); - let _ = db; +async fn connect_local() -> lancedb::Result { + // --8<-- [start:connect] + let db = connect("ex_lancedb").execute().await?; + // --8<-- [end:connect] + Ok(db) } -// --8<-- [end:connect] #[tokio::main] -async fn main() { - let temp_dir = tempfile::tempdir().unwrap(); - let uri = temp_dir.path().join("ex_lancedb"); - connect_example(uri.to_str().unwrap()).await; - - // Keep the enterprise snippet in this file, but don't run it in CI. - let _ = connect_enterprise_quickstart_config(); - let _ = connect_object_storage_config(); -} - -fn connect_enterprise_quickstart_config() -> (String, String, String, String) { - // --8<-- [start:connect_enterprise_quickstart] - let uri = "db://your-database-uri"; - let api_key = "your-api-key"; - let region = "us-east-1"; - let host_override = "https://your-enterprise-endpoint.com"; - // --8<-- [end:connect_enterprise_quickstart] - ( - uri.to_string(), - api_key.to_string(), - region.to_string(), - host_override.to_string(), +async fn main() -> Result<(), Box> { + let temp_dir = tempfile::tempdir()?; + std::env::set_current_dir(temp_dir.path())?; + let db = connect_local().await?; + db.create_empty_table( + "probe", + std::sync::Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new( + "id", + arrow_schema::DataType::Int64, + false, + )])), ) + .execute() + .await?; + let reopened = connect(temp_dir.path().join("ex_lancedb").to_str().unwrap()) + .execute() + .await?; + assert_eq!(reopened.table_names().execute().await?, vec!["probe"]); + + // Connects to a deployment only when asked, and only lists its tables. + if std::env::var("LANCEDB_DOCS_TEST_DEPLOYMENT").as_deref() == Ok("1") { + let db = connect_enterprise_quickstart().await?; + println!("{:?}", db.table_names().execute().await?); + } + Ok(()) } -fn connect_object_storage_config() -> &'static str { +async fn connect_enterprise_quickstart() -> Result> { + // --8<-- [start:connect_enterprise_quickstart] + let uri = std::env::var("LANCEDB_URI")?; // db://your-database + let api_key = std::env::var("LANCEDB_API_KEY")?; + let region = std::env::var("LANCEDB_REGION").unwrap_or_else(|_| "us-east-1".to_string()); + let host_override = std::env::var("LANCEDB_HOST_OVERRIDE")?; // https://your-endpoint + + let db = connect(&uri) + .api_key(&api_key) + .region(®ion) + .host_override(&host_override) + .execute() + .await?; + // --8<-- [end:connect_enterprise_quickstart] + Ok(db) +} + +// Compiled as documentation, never called by the example runner. +#[allow(dead_code)] +async fn connect_object_storage() -> lancedb::Result { // --8<-- [start:connect_object_storage] let uri = "s3://your-bucket/path"; // You can also use "gs://your-bucket/path" or "az://your-container/path". + let db = connect(uri).execute().await?; // --8<-- [end:connect_object_storage] - - uri + Ok(db) } async fn namespace_table_ops_example(uri: &str) -> lancedb::Result<()> { @@ -132,3 +153,17 @@ async fn namespace_admin_ops_example() -> lancedb::Result<()> { fn repo_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..") } + +// Compiled as documentation, never called by the example runner. +#[allow(dead_code)] +async fn connect_enterprise_placeholders() -> Result> { + // --8<-- [start:connect_enterprise_placeholders] + let db = connect("db://your-database") + .api_key("your-api-key") + .region("us-east-1") + .host_override("https://your-endpoint") + .execute() + .await?; + // --8<-- [end:connect_enterprise_placeholders] + Ok(db) +} diff --git a/docs/web-tests/rs/quickstart.rs b/docs/web-tests/rs/quickstart.rs index fd26b012c..f6ea7e0f9 100644 --- a/docs/web-tests/rs/quickstart.rs +++ b/docs/web-tests/rs/quickstart.rs @@ -1,23 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -use std::sync::Arc; - -use arrow_array::types::Float32Type; -use arrow_array::{ - FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator, StructArray, -}; -use arrow_schema::{DataType, Field, FieldRef, Schema}; -use lancedb::arrow::IntoPolars; -use lancedb::database::CreateTableMode; -use lancedb::query::{ExecutableQuery, QueryBase, Select}; use lancedb::{connect, table::NewColumnTransform}; -use polars::prelude::DataFrame; -use serde::{Deserialize, Serialize}; // --8<-- [start:quickstart_define_struct] +use std::sync::Arc; + +use lancedb::arrow::arrow_array::types::Float32Type; +use lancedb::arrow::arrow_array::{ + FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator, + RecordBatchReader, StructArray, +}; +use lancedb::arrow::arrow_schema::{DataType, Field, FieldRef, Fields, Schema}; + // Define structs representing the data schema -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone)] struct Stats { strength: i8, magic: i8, @@ -25,7 +22,7 @@ struct Stats { wisdom: i8, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone)] struct Character { id: String, name: String, @@ -43,7 +40,7 @@ fn characters_schema() -> Arc { Field::new("description", DataType::LargeUtf8, false), Field::new( "stats", - DataType::Struct(arrow_schema::Fields::from(vec![ + DataType::Struct(Fields::from(vec![ Arc::new(Field::new("strength", DataType::Int8, false)), Arc::new(Field::new("magic", DataType::Int8, false)), Arc::new(Field::new("leadership", DataType::Int8, false)), @@ -58,11 +55,11 @@ fn characters_schema() -> Arc { ), ])) } -// --8<-- [end:quickstart_define_struct] -type BatchIter = Box; - -fn characters_to_reader(schema: Arc, rows: &[Character]) -> BatchIter { +fn characters_to_reader( + schema: Arc, + rows: &[Character], +) -> Box { let ids = LargeStringArray::from_iter_values(rows.iter().map(|row| row.id.as_str())); let names = LargeStringArray::from_iter_values(rows.iter().map(|row| row.name.as_str())); let roles = LargeStringArray::from_iter_values(rows.iter().map(|row| row.role.as_str())); @@ -114,6 +111,7 @@ fn characters_to_reader(schema: Arc, rows: &[Character]) -> BatchIter { schema, )) } +// --8<-- [end:quickstart_define_struct] #[tokio::main] async fn main() { @@ -166,6 +164,8 @@ async fn main() { // --8<-- [end:quickstart_data] // --8<-- [start:quickstart_create_table] + use lancedb::database::CreateTableMode; + let schema = characters_schema(); let table = db .create_table("characters", characters_to_reader(schema.clone(), &data)) @@ -175,22 +175,37 @@ async fn main() { .unwrap(); // --8<-- [end:quickstart_create_table] assert_eq!(table.count_rows(None).await.unwrap(), 3); - db.drop_table("characters", &[]).await.unwrap(); - // --8<-- [start:quickstart_create_table_no_overwrite] - let table = db - .create_table("characters", characters_to_reader(schema.clone(), &data)) + // --8<-- [start:quickstart_add_data] + let more_data = vec![Character { + id: "4".to_string(), + name: "Morgana".to_string(), + role: "Sorceress".to_string(), + description: "Powerful sorceress of Avalon.".to_string(), + stats: Stats { + strength: 2, + magic: 5, + leadership: 4, + wisdom: 4, + }, + vector: [0.3, 0.9, 0.6, 0.8], + }]; + table + .add(characters_to_reader(schema.clone(), &more_data)) .execute() .await .unwrap(); - // --8<-- [end:quickstart_create_table_no_overwrite] - assert_eq!(table.count_rows(None).await.unwrap(), 3); + // --8<-- [end:quickstart_add_data] + assert_eq!(table.count_rows(None).await.unwrap(), 4); // --8<-- [start:quickstart_vector_search_1] + use lancedb::arrow::IntoPolars; + use lancedb::query::{ExecutableQuery, QueryBase, Select}; + // Search for examples similar to a "wise magical advisor" let query_vector = [0.2, 0.8, 0.4, 0.9]; - let result: DataFrame = table + let result = table .query() .nearest_to(&query_vector) .unwrap() @@ -211,9 +226,10 @@ async fn main() { // --8<-- [end:quickstart_vector_search_1] let name_col = result.column("name").unwrap().str().unwrap(); assert_eq!(name_col.get(0).unwrap(), "Merlin"); + assert_eq!(name_col.get(1).unwrap(), "Morgana"); // --8<-- [start:quickstart_curate_with_metadata] - let curated: DataFrame = table + let curated = table .query() .nearest_to(&query_vector) .unwrap() @@ -235,9 +251,10 @@ async fn main() { // --8<-- [end:quickstart_curate_with_metadata] let curated_name_col = curated.column("name").unwrap().str().unwrap(); assert_eq!(curated_name_col.get(0).unwrap(), "Merlin"); + assert_eq!(curated_name_col.get(1).unwrap(), "Morgana"); // --8<-- [start:quickstart_output_array] - let result: DataFrame = table + let result = table .query() .nearest_to(&query_vector) .unwrap() @@ -274,7 +291,7 @@ async fn main() { // --8<-- [end:quickstart_add_feature] // --8<-- [start:quickstart_query_feature] - let features: DataFrame = table + let features = table .query() .select(Select::Columns(vec![ "name".to_string(), @@ -289,19 +306,23 @@ async fn main() { .unwrap(); println!("{features:?}"); // --8<-- [end:quickstart_query_feature] - assert!(features.column("power_score").is_ok()); + assert_eq!(features.column("power_score").unwrap().len(), 4); + + // What the quickstart's download step leaves in an application's directory. + let app = tempfile::tempdir().unwrap(); + std::fs::create_dir(app.path().join("data")).unwrap(); + std::fs::copy( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../web/static/assets/images/quickstart/sir-lancelot.jpg"), + app.path().join("data/sir-lancelot.jpg"), + ) + .unwrap(); + std::env::set_current_dir(app.path()).unwrap(); // --8<-- [start:quickstart_multimodal_bytes] - use std::sync::Arc; + use lancedb::arrow::arrow_array::BinaryArray; - use arrow_array::{ - BinaryArray, FixedSizeListArray, LargeStringArray, RecordBatch, RecordBatchIterator, - }; - use arrow_schema::{DataType, Field, Schema}; - - let image_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../web/static/assets/images/quickstart/sir-lancelot.jpg"); - let image_bytes = std::fs::read(image_path).unwrap(); + let image_bytes = std::fs::read("data/sir-lancelot.jpg").unwrap(); let image_schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::LargeUtf8, false), @@ -333,9 +354,10 @@ async fn main() { ], ) .unwrap(); - let image_reader: Box = Box::new( - RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), image_schema), - ); + let image_reader: Box = Box::new(RecordBatchIterator::new( + vec![Ok(image_batch)].into_iter(), + image_schema, + )); let multimodal_table = db .create_table("character_images", image_reader) .mode(CreateTableMode::Overwrite) diff --git a/docs/web-tests/ts/connection.test.ts b/docs/web-tests/ts/connection.test.ts index f522654ec..891501d4c 100644 --- a/docs/web-tests/ts/connection.test.ts +++ b/docs/web-tests/ts/connection.test.ts @@ -1,65 +1,89 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, jest, test } from "@jest/globals"; -import * as path from "node:path"; -import { withTempDirectory } from "./util.ts"; -// --8<-- [start:connect] +import { afterEach, describe, expect, jest, test } from "@jest/globals"; import * as lancedb from "@lancedb/lancedb"; - -async function connectExample(uri: string) { - const db = await lancedb.connect(uri); - return db; -} -// --8<-- [end:connect] +import { withTempDirectory } from "./util.ts"; test("connect to a local database", async () => { await withTempDirectory(async (tempDir) => { - const uri = path.join(tempDir, "ex_lancedb"); - const db = await connectExample(uri); - expect(db).toBeDefined(); + const cwd = process.cwd(); + process.chdir(tempDir); + try { + // --8<-- [start:connect] + const db = await lancedb.connect("ex_lancedb"); + // --8<-- [end:connect] + await db.createTable("probe", [{ id: 1 }]); + expect((await db.listTables()).tables).toEqual(["probe"]); + } finally { + process.chdir(cwd); + } + const reopened = await lancedb.connect(`${tempDir}/ex_lancedb`); + expect((await reopened.listTables()).tables).toEqual(["probe"]); }); }); async function connectEnterpriseQuickstart() { // --8<-- [start:connect_enterprise_quickstart] - const uri = "db://your-database-uri"; - const apiKey = "your-api-key"; - const region = "us-east-1"; - const hostOverride = "https://your-enterprise-endpoint.com"; + const setting = (name: string) => { + const value = process.env[name]; + if (!value) throw new Error(`Set ${name}`); + return value; + }; - const db = await lancedb.connect(uri, { - apiKey, - region, - hostOverride, + const db = await lancedb.connect(setting("LANCEDB_URI"), { + apiKey: setting("LANCEDB_API_KEY"), + region: process.env.LANCEDB_REGION ?? "us-east-1", + hostOverride: setting("LANCEDB_HOST_OVERRIDE"), }); // --8<-- [end:connect_enterprise_quickstart] return db; } -test("enterprise quickstart connect uses placeholder config", async () => { - const mockDb = { __mock: true } as unknown as Awaited< - ReturnType - >; - const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb); - - const db = await connectEnterpriseQuickstart(); - expect(db).toBe(mockDb); - expect(spy).toHaveBeenCalledWith("db://your-database-uri", { - apiKey: "your-api-key", - region: "us-east-1", - hostOverride: "https://your-enterprise-endpoint.com", +describe("enterprise quickstart connect", () => { + const saved = { ...process.env }; + afterEach(() => { + process.env = { ...saved }; + }); + + test("reads the connection from the environment", async () => { + const mockDb = { __mock: true } as unknown as Awaited< + ReturnType + >; + const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb); + process.env.LANCEDB_URI = "db://your-database"; + process.env.LANCEDB_API_KEY = "your-api-key"; + delete process.env.LANCEDB_REGION; + process.env.LANCEDB_HOST_OVERRIDE = "https://your-endpoint"; + + const db = await connectEnterpriseQuickstart(); + expect(db).toBe(mockDb); + expect(spy).toHaveBeenCalledWith("db://your-database", { + apiKey: "your-api-key", + region: "us-east-1", + hostOverride: "https://your-endpoint", + }); + spy.mockRestore(); + }); + + test("requires an endpoint", async () => { + process.env.LANCEDB_URI = "db://your-database"; + process.env.LANCEDB_API_KEY = "your-api-key"; + delete process.env.LANCEDB_HOST_OVERRIDE; + + await expect(connectEnterpriseQuickstart()).rejects.toThrow( + "Set LANCEDB_HOST_OVERRIDE", + ); }); - spy.mockRestore(); }); -// --8<-- [start:connect_object_storage] async function connectObjectStorageExample() { + // --8<-- [start:connect_object_storage] const uri = "s3://your-bucket/path"; // You can also use "gs://your-bucket/path" or "az://your-container/path". const db = await lancedb.connect(uri); + // --8<-- [end:connect_object_storage] return db; } -// --8<-- [end:connect_object_storage] async function namespaceTableOpsExample() { // --8<-- [start:namespace_table_ops] @@ -119,3 +143,25 @@ void [ namespaceTableOpsExample, namespaceAdminOpsExample, ]; + +test("enterprise placeholders use an explicit endpoint", async () => { + const mockDb = {} as Awaited>; + const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb); + try { + // --8<-- [start:connect_enterprise_placeholders] + const db = await lancedb.connect("db://your-database", { + apiKey: "your-api-key", + region: "us-east-1", + hostOverride: "https://your-endpoint", + }); + // --8<-- [end:connect_enterprise_placeholders] + expect(db).toBe(mockDb); + expect(spy).toHaveBeenCalledWith("db://your-database", { + apiKey: "your-api-key", + region: "us-east-1", + hostOverride: "https://your-endpoint", + }); + } finally { + spy.mockRestore(); + } +}); diff --git a/docs/web-tests/ts/quickstart.test.ts b/docs/web-tests/ts/quickstart.test.ts index 55ae9d56c..e32a14f86 100644 --- a/docs/web-tests/ts/quickstart.test.ts +++ b/docs/web-tests/ts/quickstart.test.ts @@ -1,176 +1,263 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; +import { describe, expect, test } from "@jest/globals"; +import { randomUUID } from "node:crypto"; +import * as fs from "node:fs"; +import * as path from "node:path"; import * as lancedb from "@lancedb/lancedb"; import { withTempDirectory } from "./util.ts"; -test("quickstart example (async)", async () => { +// What the quickstart's download step leaves in an application's directory. +const IMAGE = path.join( + __dirname, + "../../web/static/assets/images/quickstart/sir-lancelot.jpg", +); +const deployment = ["LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY"]; +// The connection settings alone never send these tests to a deployment. +const optIn = "LANCEDB_DOCS_TEST_DEPLOYMENT"; +const connections = + process.env[optIn] === "1" ? ["local", "enterprise"] : ["local"]; + +async function tableNames(db: lancedb.Connection) { + const names: string[] = []; + let pageToken: string | undefined; + do { + const page = await db.listTables({ pageToken }); + names.push(...page.tables); + pageToken = page.pageToken; + } while (pageToken); + return names; +} + +// A deployment connection on which a test creates only tables of its own: +// each name gets a suffix unique to the test and is claimed only while no +// table has it; the test drops what it claimed, whether or not it passed. +async function withOwnedTables( + db: lancedb.Connection, + fn: (db: lancedb.Connection) => Promise, +) { + const suffix = randomUUID().replaceAll("-", "").slice(0, 12); + const claimed: string[] = []; + const createTable = async ( + name: string, + data: Parameters[1], + options: Partial = {}, + ) => { + const owned = `${name}_${suffix}`; + let mode = options.mode; + if (!claimed.includes(owned)) { + if ((await tableNames(db)).includes(owned)) { + throw new Error(`${owned} already exists and is not this test's`); + } + claimed.push(owned); + mode = "create"; + } + return db.createTable(owned, data, { ...options, mode }); + }; + try { + await fn( + new Proxy(db, { + get: (target, property, receiver) => + property === "createTable" + ? createTable + : Reflect.get(target, property, receiver), + }), + ); + } finally { + const existing = await tableNames(db); + for (const name of claimed) { + if (existing.includes(name)) { + await db.dropTable(name); + } + } + } +} + +async function withDatabase( + connection: string, + fn: (db: lancedb.Connection) => Promise, +) { + if (connection === "enterprise") { + const missing = deployment.filter((name) => !process.env[name]); + if (missing.length) { + throw new Error(`${optIn}=1 needs ${missing.join(", ")}`); + } + const db = await lancedb.connect(process.env.LANCEDB_URI!, { + apiKey: process.env.LANCEDB_API_KEY!, + region: process.env.LANCEDB_REGION ?? "us-east-1", + hostOverride: process.env.LANCEDB_HOST_OVERRIDE!, + }); + await withOwnedTables(db, fn); + return; + } await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); + await fn(await lancedb.connect(databaseDir)); + }); +} - // --8<-- [start:quickstart_data] - const data = [ - { - id: "1", - name: "King Arthur", - role: "King", - description: "Leader of Camelot and wielder of Excalibur.", - stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 }, - vector: [0.7, 0.1, 0.9, 0.7], - }, - { - id: "2", - name: "Merlin", - role: "Wizard", - description: "Advisor and prophet with deep magical knowledge.", - stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 }, - vector: [0.2, 0.9, 0.4, 0.9], - }, - { - id: "3", - name: "Sir Lancelot", - role: "Knight", - description: "Legendary knight known for courage and combat skill.", - stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 }, - vector: [0.9, 0.1, 0.5, 0.4], - }, - ]; - // --8<-- [end:quickstart_data] +async function createCharacters(db: lancedb.Connection) { + // --8<-- [start:quickstart_data] + const data = [ + { + id: "1", + name: "King Arthur", + role: "King", + description: "Leader of Camelot and wielder of Excalibur.", + stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 }, + vector: [0.7, 0.1, 0.9, 0.7], + }, + { + id: "2", + name: "Merlin", + role: "Wizard", + description: "Advisor and prophet with deep magical knowledge.", + stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 }, + vector: [0.2, 0.9, 0.4, 0.9], + }, + { + id: "3", + name: "Sir Lancelot", + role: "Knight", + description: "Legendary knight known for courage and combat skill.", + stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 }, + vector: [0.9, 0.1, 0.5, 0.4], + }, + ]; + // --8<-- [end:quickstart_data] - // --8<-- [start:quickstart_create_table] - let table = await db.createTable("characters", data, { mode: "overwrite" }); - // --8<-- [end:quickstart_create_table] - expect(await table.countRows()).toBe(3); - await db.dropTable("characters"); + // --8<-- [start:quickstart_create_table] + const table = await db.createTable("characters", data, { mode: "overwrite" }); + // --8<-- [end:quickstart_create_table] + expect(await table.countRows()).toBe(3); - // --8<-- [start:quickstart_create_table_no_overwrite] - table = await db.createTable("characters", data); - // --8<-- [end:quickstart_create_table_no_overwrite] - expect(await table.countRows()).toBe(3); + // --8<-- [start:quickstart_add_data] + await table.add([ + { + id: "4", + name: "Morgana", + role: "Sorceress", + description: "Powerful sorceress of Avalon.", + stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 }, + vector: [0.3, 0.9, 0.6, 0.8], + }, + ]); + // --8<-- [end:quickstart_add_data] + expect(await table.countRows()).toBe(4); + return table; +} - // --8<-- [start:quickstart_vector_search_1] - // Search for examples similar to a "wise magical advisor" - let queryVector = [0.2, 0.8, 0.4, 0.9]; +describe.each(connections)("quickstart (%s)", (connection) => { + test("create, add and search", async () => { + await withDatabase(connection, async (db) => { + const table = await createCharacters(db); - let result = await table - .search(queryVector) - .select(["name", "role", "description", "_distance"]) - .limit(2) - .toArray(); - console.table(result); - // --8<-- [end:quickstart_vector_search_1] - expect(result[0].name).toBe("Merlin"); + // --8<-- [start:quickstart_vector_search_1] + // Search for examples similar to a "wise magical advisor" + const queryVector = [0.2, 0.8, 0.4, 0.9]; - // --8<-- [start:quickstart_curate_with_metadata] - const curated = await table - .search(queryVector) - .where("stats.magic >= 4") - .select(["name", "role", "description", "_distance"]) - .limit(2) - .toArray(); - console.table(curated); - // --8<-- [end:quickstart_curate_with_metadata] - expect(curated[0].name).toBe("Merlin"); + const result = await table + .search(queryVector) + .select(["name", "role", "description", "_distance"]) + .limit(2) + .toArray(); + console.table(result); + // --8<-- [end:quickstart_vector_search_1] + expect(result.map((row) => row.name)).toEqual(["Merlin", "Morgana"]); - // --8<-- [start:quickstart_output_array] - result = await table.search(queryVector).limit(2).toArray(); - console.table(result); - // --8<-- [end:quickstart_output_array] - expect(result[0].name).toBe("Merlin"); + // --8<-- [start:quickstart_curate_with_metadata] + const curated = await table + .search(queryVector) + .where("stats.magic >= 4") + .select(["name", "role", "description", "_distance"]) + .limit(2) + .toArray(); + console.table(curated); + // --8<-- [end:quickstart_curate_with_metadata] + expect(curated.map((row) => row.name)).toEqual(["Merlin", "Morgana"]); + }); + }); - // --8<-- [start:quickstart_add_feature] - await table.addColumns([ - { - name: "power_score", - valueSql: - "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)", - }, - ]); - // --8<-- [end:quickstart_add_feature] - const schemaWithFeature = await table.schema(); - expect(schemaWithFeature.fields.some((f) => f.name === "power_score")).toBe( - true, - ); + test("a derived feature", async () => { + await withDatabase(connection, async (db) => { + const table = await createCharacters(db); - // --8<-- [start:quickstart_query_feature] - const features = await table - .query() - .select(["name", "role", "power_score"]) - .toArray(); - console.table(features); - // --8<-- [end:quickstart_query_feature] - expect(features[0]).toHaveProperty("power_score"); - - // --8<-- [start:quickstart_multimodal_bytes] - const arrow = await import("apache-arrow"); - const path = await import("node:path"); - const { readFile } = await import("node:fs/promises"); - - const imagePath = path.join( - __dirname, - "../../web/static/assets/images/quickstart/sir-lancelot.jpg", - ); - const imageBytes = await readFile(imagePath); - const imageSchema = new arrow.Schema([ - new arrow.Field("id", new arrow.Utf8()), - new arrow.Field("description", new arrow.Utf8()), - new arrow.Field("image", new arrow.Binary()), - new arrow.Field( - "vector", - new arrow.FixedSizeList( - 4, - new arrow.Field("item", new arrow.Float32(), true), - ), - ), - ]); - const imageData = lancedb.makeArrowTable( - [ + // --8<-- [start:quickstart_add_feature] + await table.addColumns([ { - id: "lancelot", - description: "Portrait of Sir Lancelot", - image: imageBytes, - vector: [0.9, 0.1, 0.5, 0.4], + name: "power_score", + valueSql: + "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)", }, - ], - { schema: imageSchema }, - ); - const multimodalTable = await db.createTable( - "character_images", - imageData, - { mode: "overwrite" }, - ); - // --8<-- [end:quickstart_multimodal_bytes] - expect(await multimodalTable.countRows()).toBe(1); + ]); + // --8<-- [end:quickstart_add_feature] - // --8<-- [start:quickstart_open_table] - table = await db.openTable("characters"); - // --8<-- [end:quickstart_open_table] + // --8<-- [start:quickstart_query_feature] + const features = await table + .query() + .select(["name", "role", "power_score"]) + .toArray(); + console.table(features); + // --8<-- [end:quickstart_query_feature] + expect( + features.map((row) => [row.name, row.power_score]).sort(), + ).toEqual([ + ["King Arthur", 3.5], + ["Merlin", 4], + ["Morgana", 3.75], + ["Sir Lancelot", 3], + ]); + }); + }); - // --8<-- [start:quickstart_add_data] - const moreData = [ - { - id: "4", - name: "Morgana", - role: "Sorceress", - description: "Powerful sorceress of Avalon.", - stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 }, - vector: [0.3, 0.9, 0.6, 0.8], - power_score: 3.75, - }, - ]; + test("multimodal data", async () => { + await withDatabase(connection, async (db) => { + await withTempDirectory(async (app) => { + fs.mkdirSync(path.join(app, "data")); + fs.copyFileSync(IMAGE, path.join(app, "data/sir-lancelot.jpg")); + const cwd = process.cwd(); + process.chdir(app); + try { + // --8<-- [start:quickstart_multimodal_bytes] + const arrow = await import("apache-arrow"); + const { readFile } = await import("node:fs/promises"); - // Add data to table - await table.add(moreData); - // --8<-- [end:quickstart_add_data] - expect(await table.countRows()).toBe(4); - - // --8<-- [start:quickstart_vector_search_2] - // Search for examples similar to a "powerful sorceress" - queryVector = [0.3, 0.9, 0.6, 0.8]; - - const results = await table.search(queryVector).limit(2).toArray(); - console.table(results); - // --8<-- [end:quickstart_vector_search_2] - expect(results[0].name).toBe("Morgana"); + const imageBytes = await readFile("data/sir-lancelot.jpg"); + const imageSchema = new arrow.Schema([ + new arrow.Field("id", new arrow.Utf8()), + new arrow.Field("description", new arrow.Utf8()), + new arrow.Field("image", new arrow.Binary()), + new arrow.Field( + "vector", + new arrow.FixedSizeList( + 4, + new arrow.Field("item", new arrow.Float32(), true), + ), + ), + ]); + const imageData = lancedb.makeArrowTable( + [ + { + id: "lancelot", + description: "Portrait of Sir Lancelot", + image: imageBytes, + vector: [0.9, 0.1, 0.5, 0.4], + }, + ], + { schema: imageSchema }, + ); + const multimodalTable = await db.createTable( + "character_images", + imageData, + { mode: "overwrite" }, + ); + // --8<-- [end:quickstart_multimodal_bytes] + expect(await multimodalTable.countRows()).toBe(1); + const [row] = await multimodalTable.query().select(["image"]).toArray(); + expect(Buffer.from(row.image)).toEqual(fs.readFileSync(IMAGE)); + } finally { + process.chdir(cwd); + } + }); + }); }); }); diff --git a/docs/web/architecture.mdx b/docs/web/architecture.mdx index 4ee1bc0e7..997bb95b0 100644 --- a/docs/web/architecture.mdx +++ b/docs/web/architecture.mdx @@ -1,153 +1,124 @@ --- title: "Architecture" sidebarTitle: "Architecture" -description: "How LanceDB is put together, and what changes between running it embedded and running it distributed." +description: "How LanceDB Enterprise separates query serving, background work, and storage, and how it compares with embedded LanceDB OSS." icon: cubes keywords: ["architecture", "embedded", "distributed", "object storage", "compute", "deployment"] --- -LanceDB has one storage layer and two ways to put compute on top of it. Which -one you run decides where queries execute, what runs on your behalf, and what -you have to operate — and almost nothing else. The data is the same -[Lance](/basics/lance) tables either way. +LanceDB OSS is an embedded library for working with Lance tables. LanceDB +Enterprise is a multimodal lakehouse for data curation, feature engineering, +search, and training data access. Its architecture separates query serving, +read execution, caching, and background work. Both use the open +[Lance](/basics/lance) format. -## One storage layer {#storage} +Teams can curate datasets, derive features, search multimodal records, and +read the resulting tables into training frameworks. Enterprise separates the +resources that serve shared data from those that build indexes and run +background work, so teams can size them for these different workloads. -Everything durable lives in object storage: table data, manifests, indexes. No -machine owns a dataset, and no server has to be running for the data to exist. +![LanceDB OSS runs the application and embedded library in one process, using local disk or object storage. Enterprise applications connect to query serving, with separate read execution and caching, indexing and background workers, and durable object storage. Both use the Lance format, but they do not share a storage location automatically.](/static/assets/images/overview/lancedb-architecture.svg) -``` - ┌──────────────────────────────────────────┐ - │ object storage │ - │ table data · manifests · indexes │ - └──────────────────────────────────────────┘ -``` +The diagram shows the main compute and storage boundaries, not every request +path. The control plane governs the Enterprise service. -That is the whole persistence story, and it is why the two deployment shapes can -differ so much above it while staying compatible below it. A table written -embedded is opened by a deployment, and the reverse, without conversion. +## A shared format, different storage choices {#storage} -## Embedded {#embedded} +Lance stores table data, versions, and indexes. OSS can read and write tables on +local disk or in object storage. Enterprise keeps durable table data and index +artifacts in object storage, independently of the machines serving requests. +Its caches accelerate reads; they are not the durable source of truth. -``` - your process - ┌────────────────────────────────┐ - │ application │ - │ LanceDB library │ ← plans, executes, indexes, compacts - └────────────────────────────────┘ - │ - ┌─────────▼─────────┐ - │ object storage │ - └───────────────────┘ -``` +Sharing a format does not mean that an Enterprise deployment automatically +opens an existing OSS database. Plan how to move or connect your data and +configure access for your deployment. -`lancedb.connect("./data")` or `connect("s3://bucket/path")` gives you a -**library**, not a client. Your process opens the tables, plans the queries, -runs the scans, builds the indexes, and writes the new versions. There is no -server in the picture at all. +## OSS: an embedded library inside your application {#embedded} -**What follows from that.** Nothing runs unless you call it — no background -compaction, no server-side [jobs](/basics/jobs), no -[SQL endpoint](/basics/sql). Scale is your process's scale: a query gets the -cores and memory your process has. Access control is whatever your object store -enforces, since there is no other gatekeeper. +With an embedded connection, the LanceDB library plans and executes queries, +builds indexes, and updates tables inside your process. There is no separate +database server to deploy. -**What it is good at.** Development, notebooks, single-node pipelines, embedding -LanceDB inside an application, and anything where "no service to operate" is -worth more than elasticity. Comfortably handles millions of vectors on one -node -- disk-based indexes keep it performant without extra infrastructure. +This fits local retrieval applications, notebooks, development, and +single-node data pipelines. Your application controls the compute resources and +when to run [table maintenance](/maintenance/compaction). You also manage access +to the local files or object storage that hold your tables. -## Distributed {#distributed} +## Enterprise: a multimodal lakehouse for shared workloads {#distributed} -Enterprise +Applications connect to remote tables through a LanceDB SDK. The service handles +requests across a fleet, with components sized for different kinds of work: -``` - your process - ┌──────────────┐ - │ application │ - │ LanceDB SDK │ ← a thin client - └──────┬───────┘ - │ db:// (REST / Flight SQL) - ┌──────▼──────────────────────────────────────┐ - │ control plane config · identity · policy │ - ├─────────────────────────────────────────────┤ - │ data plane query serving · writes │ - │ indexers indexing · compaction │ - └──────┬──────────────────────────────────────┘ - │ - ┌──────▼────────────┐ - │ object storage │ - └───────────────────┘ -``` +- **Query serving** receives requests, resolves tables, and plans operations. + Add serving capacity as more applications and users depend on your data. +- **Read execution and caching** use separately provisioned resources. Executors + cache table data to reduce repeated reads from object storage. +- **Indexing and background workers** build indexes and perform maintenance + outside the query request path. Size this work separately from query serving + as your data grows. -`connect("db://...")` gives you a **client**. Planning and execution move to a -cluster, and the work splits across tiers that scale independently: a control -plane governing configuration, identity and policy; a data plane serving reads -and writes; and indexers building indexes and compacting outside the request -path. +The control plane manages configuration, identity, policy, and cluster +lifecycle. It governs the service rather than executing table queries. +[Authorization](/governance/authorization) lets teams control access to catalog +objects when enabled and configured by the administrator. Deployment settings govern background +[compaction](/maintenance/compaction) and +[cleanup](/maintenance/cleanup), including retention. -**What follows from that.** Compute and storage are separated, so a query fleet -scales for interactive traffic without also scaling indexing capacity, and heavy -background work does not steal resources from user queries. Because a server -exists, so do the things that need one: [SQL](/sql), -[functions](/management/function), server-side jobs, and -[authorization](/governance/authorization) per object rather than per bucket. +These boundaries support different parts of an AI data workflow. +[SQL curation](/curation) requires a deployment with SQL enabled. +[Feature engineering](/feature-engineering) can produce derived columns and +tables; computed columns and materialized views require a deployment that +enables them. [Geneva](/geneva) runs distributed Python feature backfills while +its capabilities continue to transition to LanceDB APIs. +[Training data loaders](/training/data-loading) read tables into model training +frameworks in both offerings. LanceDB provides data access for training; the +training framework runs the model. -**What it is good at.** Many concurrent queries, continuous ingestion and -indexing, multi-team access with real permissions, and workloads too large for -one machine -- customers regularly operate on billions of rows and tens of -terabytes. +Size and monitor each tier for your workload. Read the +[Enterprise architecture](/enterprise/architecture) for the component details +and request flows. -## What changes, precisely {#differences} +## Application changes for Enterprise {#differences} -| | Embedded | Distributed | -| --- | --- | --- | -| What you connect to | a library in your process | a cluster | -| Where queries plan and execute | your process | the data plane | -| Table handle | `LanceTable` | `RemoteTable` | -| Indexing and compaction | when you call it | dedicated indexers | -| [Jobs](/basics/jobs) | run in-process, no server record | real jobs: id, status, events, cancel | -| [SQL](/sql) | unavailable | Flight SQL endpoint | -| [Functions](/management/function) | unavailable | run on workers | -| Access control | object-store permissions | [per-object grants](/governance/authorization) | -| Scale bound | your process | the cluster | -| What you operate | nothing | a deployment | +Core operations remain familiar: create a table, add data, and run a search. +With OSS, the application executes those operations. With Enterprise, the SDK +sends requests to the service, and the service manages execution and configured +background maintenance. -Both expose the same table API — create, open, mutate, query — so application -code moves between them by changing the URI. What does not move is code that -depends on a row in the right-hand column. +To use Enterprise, configure the cluster endpoint, database URI and credentials, +and make your data available in the Enterprise database. Review SDK methods +that behave differently on remote tables, including result retrieval and +background jobs. [Moving to LanceDB Enterprise](/basics/sdk#moving-to-enterprise) +describes these application changes. If you plan to use materialized views, check the +[stable row ID requirements](/basics/limits#schema) before creating source tables. -## Choosing, and changing your mind {#choosing} +## Choose for the workload you need to run {#choosing} -Start embedded. It has no operational cost and the API is the same one you will -keep using. +Choose OSS when you want an embedded library and can manage its resources and +maintenance in your application. -Move when you hit something in the right-hand column — usually concurrent -serving, ingestion that outgrows one machine, or a permissions requirement. -Because the storage layer is identical, moving is a connection-string change -plus a data migration, not a rewrite. +Choose Enterprise when multiple applications or teams need a shared multimodal +lakehouse for curation, feature engineering, search, and training data access. +It also fits workloads whose query and indexing demand grow at different rates, +or that need configured service-level access controls and background operations. +You can begin with Enterprise. - -The one thing to decide early is **stable row ids**: the SDK can give them to a -table only when it creates one on a local path or an object store, and a -deployment may require them on a -[materialized view](/management/materialized-view) source. See -[Limits](/basics/limits#schema). - +[Talk to us about your workload](https://www.lancedb.com/contact) to plan capacity, +deployment options, and the path from your existing data to Enterprise. ## Where to go next {#next} + + Query serving, execution, and background work in detail. + + + Connect an application to your deployment. + What a write promises, and what a reader sees. - The open format underneath both shapes. - - - The distributed tiers in detail. - - - The bounds you can actually hit. + The open format underneath both offerings. diff --git a/docs/web/basics/agentic-skills.mdx b/docs/web/basics/agentic-skills.mdx index 319e770b4..bda6e1f65 100644 --- a/docs/web/basics/agentic-skills.mdx +++ b/docs/web/basics/agentic-skills.mdx @@ -269,10 +269,11 @@ where appropriate. Choose [LanceDB Enterprise](/enterprise) when the resulting table becomes shared production infrastructure, and the workload needs distributed capacity, -private deployment, or platform-managed operations. The underlying data format and table -API stay the same, so the pipeline does not need to be redesigned. The agent -instead connects to a remote `db://` table and lets the cluster handle -maintenance and background work. +private deployment, or platform-managed operations. The data format and the core +table API stay the same, so the pipeline's table and search code carries over: +the agent connects to a remote `db://` table, leaves maintenance to the +deployment instead of calling `optimize()`, and submits long-running work as +jobs. See [Moving to LanceDB Enterprise](/basics/sdk#moving-to-enterprise). ## What agents are good at here {#workflows} diff --git a/docs/web/basics/catalog.mdx b/docs/web/basics/catalog.mdx index 133599f16..c6950b9de 100644 --- a/docs/web/basics/catalog.mdx +++ b/docs/web/basics/catalog.mdx @@ -1,107 +1,66 @@ --- title: "Lance Catalog" sidebarTitle: "Lance Catalog" -description: "The naming layer: what objects exist, how they nest, and who may reach them." +description: "Organize, find and control access to your data in LanceDB." icon: sitemap -keywords: ["catalog", "namespace", "hierarchy", "governance", "database", "table", "view", "function"] +keywords: ["catalog", "namespace", "hierarchy", "governance", "database", "table", "view"] --- -A catalog answers one question: **what exists, and what is it called.** It holds -names, the hierarchy they sit in, and the rules about who may reach them. It -does not store rows, run queries, or build indexes -- that is the -[lakehouse](/basics/lakehouse) on top of it. +## Catalog overview {#catalog} -Keeping the two apart is what lets a Lance table be reached by something that is -not LanceDB. The catalog is a specification -- the -[Lance Namespace](/basics/lance#standard) -- so any engine that implements it -resolves the same names to the same tables. +A catalog keeps track of named data objects, such as tables and views, so +applications can discover and access them. In LanceDB, it organizes those +objects into databases and namespaces. -## The objects {#objects} +## Databases, namespaces and tables {#objects} -| Object | What it is | +A database contains namespaces that group related tables and views. Namespaces +can also contain other namespaces, allowing teams to organize data by project, +environment or purpose. + +For example, a search application can keep production and staging data in +separate namespaces within the same database: + +```mermaid +flowchart TD + database["ai_data · database"] --> prod["prod · namespace"] + database --> staging["staging · namespace"] + prod --> prod_docs["documents · table"] + staging --> staging_docs["documents · table"] +``` + + + +| Object | What it represents | | --- | --- | -| [Database](/management/database) | The root. A connection opens exactly one. | -| [Namespace](/management/namespace) | A named grouping inside a database. Can nest. | -| [Table](/management/table) | Rows and columns, versioned. What most operations act on. | -| [Materialized view](/management/materialized-view) | A table whose contents come from a query, computed and stored. | -| [View](/management/view) | A saved query. Computed when read, not stored. | -| [Function](/management/function) | Registered code, applied to produce column values. | +| [Database](/management/database) | A container for namespaces and their data objects. | +| [Namespace](/management/namespace) | A named group of tables, views or nested namespaces. | +| [Table](/management/table) | Data stored as rows and columns. | +| [View](/management/view) | A saved query, evaluated when read. Available in Enterprise. | +| [Materialized view](/management/materialized-view) | The stored results of a query, updated by a refresh. | -``` -database -└── namespace - ├── table - ├── materialized view (a table, defined by a query) - ├── view (a query, not stored) - └── function -``` +{/* +Draft for the Functions release. Add this row to the table above: +| [Function](/management/function) | Registered code used to compute column values. Available in Enterprise. | +*/} -A namespace can contain further namespaces, so the hierarchy goes as deep as -your organisation needs. A table is reached by its path through it: -`prod.events.clicks`. + -This page is about what the objects *are*. For the operations on them, see -[Management](/management). +## Organization and access control {#governance} -## Resolution {#resolution} +- **Separate projects and environments.** Keep production and staging tables + in distinct namespaces, even when they use the same table names. +- **Find shared datasets.** Give teams a consistent place to discover and + access the data they work with. +- **Control access in Enterprise.** Use permissions on databases, namespaces + and tables to manage who can access shared data and what they can do with it. -A name is resolved by walking the path, and the walk is the point: reaching a -table means traversing the namespaces above it. That is what makes the hierarchy -more than decoration -- it is the unit of both organisation and access. + -A flat set of tables needs none of this. Namespaces start earning their keep when -several teams, environments or domains share one catalog and you want -`prod.search.documents` and `staging.search.documents` to be unambiguous. +## Learn more {#next} -## Governance {#governance} - -Enterprise - -Because the catalog knows the hierarchy, it is the layer that can enforce -access. A grant names **one privilege** on **one object** for **one subject**, -and it inherits down the path: reaching a table requires `USAGE` on the -namespace containing it and on the database containing that. - -That is the practical reason to put a boundary in the hierarchy -- a namespace -per team gives you something to grant on. See -[Authorization](/governance/authorization). - -Embedded LanceDB has no such layer. Access is whatever the object store enforces -on the bucket, which has no concept of a table. - -## Views and materialized views are different objects {#views} - -The names are close enough to mislead. Both are catalog objects defined by a -query over other tables; the difference is when that query runs. - -- A **materialized view** stores its rows, and a refresh brings them up to - date. Reads are reads of stored data -- fast, and as current as the last - refresh. -- A **view** stores only the query. Every read runs it, so a view is never stale - and never faster than the query it wraps. - -Reach for a materialized view when the query is expensive and the data changes -less often than it is read. Reach for a view when correctness matters more than -latency, or when the query is cheap. - -## What the catalog does not hold {#elsewhere} - -**Indexes** belong to a table rather than standing alone. Creating one is a -performance decision, not a structural one -- see [Indexing](/indexing). - -**Jobs** are not stored data at all; they are handles to work in progress. See -[Jobs](/basics/jobs). - -**Rows** are in the [Lance](/basics/lance) files the catalog points at. The -catalog knows a table exists and where it lives; it does not know what is in it. - -## Next {#next} - - - - What operates on these objects: search, compute, maintenance. - - - Creating, listing and dropping each one. - - +- [Databases](/management/database): connect to and manage a database. +- [Namespaces](/management/namespace): create groups and address tables within them. +- [Tables](/management/table): create, inspect and manage your data. +- [Views](/management/view) and [materialized views](/management/materialized-view): work with query-defined data. +- [Authorization](/governance/authorization): configure Enterprise access permissions. diff --git a/docs/web/basics/connect.mdx b/docs/web/basics/connect.mdx new file mode 100644 index 000000000..ac7061bbc --- /dev/null +++ b/docs/web/basics/connect.mdx @@ -0,0 +1,132 @@ +--- +title: "Connect to LanceDB" +sidebarTitle: "Connect" +description: "Use the embedded LanceDB OSS library with tables in a local directory or object storage, or connect to LanceDB Enterprise." +icon: plug +keywords: ["connect", "uri", "embedded", "object storage", "s3", "enterprise", "db://"] +--- + +import { + PyConnect, + PyConnectAsync, + PyConnectEnterprisePlaceholders, + PyConnectEnterpriseQuickstart, + PyConnectObjectStorage, + PyConnectObjectStorageAsync, + RsConnect, + TsConnect, + TsConnectEnterprisePlaceholders, + TsConnectEnterpriseQuickstart, + TsConnectObjectStorage, +} from '/snippets/connection.mdx'; + +The URI you pass to `connect` decides where the work runs. The same Python, +TypeScript and Rust SDKs open all three kinds; +[OSS and Enterprise](/basics/offerings) describes how the two offerings differ. + +| URI | Where queries run | +| --- | --- | +| A local path, such as `./data` | OSS: in your process | +| `s3://`, `gs://` or `az://` | OSS: in your process, reading object storage | +| `db://` | LanceDB Enterprise: queries run on your cluster | + +## A local directory {#local} + +Embedded in your application, LanceDB reads and writes the tables in a +directory on disk. + + + + {PyConnect} + + + + {PyConnectAsync} + + + + { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } + {TsConnect} + + + + { "use lancedb::connect;\n" } + { "\n" } + {RsConnect} + + + +## Object storage {#object-storage} + +With an object storage URI the library is still embedded: your process runs +every query and reads and writes the bucket directly, with the credentials from +its environment or its storage options. [Configuring storage](/deployment/storage) +covers credentials, endpoints and provider options. + + + + {PyConnectObjectStorage} + + + + {PyConnectObjectStorageAsync} + + + + { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } + {TsConnectObjectStorage} + + + +In Rust, pass the same URI to `connect`. + +## LanceDB Enterprise {#enterprise} + +A `db://` URI connects to LanceDB Enterprise, where your cluster runs queries +and background work. Get the database URI, API key, region and endpoint from +your administrator, then replace the placeholders below. Use your cluster's region +in place of `us-east-1`. In application code, load credentials from environment +variables or a secrets manager. + + + + {PyConnectEnterprisePlaceholders} + + + + { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } + {TsConnectEnterprisePlaceholders} + + + +In Rust, enable the crate's `remote` feature; the connection builder then takes +the same settings through `api_key`, `region` and `host_override`. To use OAuth +instead of an API key, see +[Authentication](/enterprise/authentication); for every connection option, see +[Database](/management/database#deployment). + + + + +Set these values in the shell that starts your application. Replace the URI, API +key, region and endpoint with your Enterprise connection settings. In production, a secrets +manager can supply these environment variables for you. + +```shell +export LANCEDB_URI="db://your-database" +export LANCEDB_API_KEY="your-api-key" +export LANCEDB_REGION="us-east-1" +export LANCEDB_HOST_OVERRIDE="https://your-endpoint" +``` + + + + {PyConnectEnterpriseQuickstart} + + + { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } + {TsConnectEnterpriseQuickstart} + + + + diff --git a/docs/web/basics/guarantees.mdx b/docs/web/basics/guarantees.mdx index 47c7c8d7c..840ea2d50 100644 --- a/docs/web/basics/guarantees.mdx +++ b/docs/web/basics/guarantees.mdx @@ -1,134 +1,91 @@ --- title: "Guarantees" sidebarTitle: "Guarantees" -description: "What a write promises when it returns, what a reader sees, and what survives a failure." +description: "Understand table commits, concurrent writes and the boundaries of recovery." icon: shield-check -keywords: ["guarantees", "consistency", "atomicity", "durability", "concurrency", "acid"] +keywords: ["guarantees", "consistency", "atomicity", "durability", "concurrency"] --- -What the system promises, and — just as usefully — what it does not. Most -surprises in practice come from assuming one of the guarantees below is -stronger or weaker than it is. +## Atomic table writes {#writes} -## Writes {#writes} +A table transaction commits its changes atomically: readers see the committed +table version, not a partially applied transaction. For example, a committed +append adds its rows together. -**A write is atomic.** Every write commits a new table version. Either the -version exists and holds all of the change, or it does not exist and holds none -of it. There is no state in which a reader sees half an `add()`. +This boundary is one table transaction. A workflow that updates several tables +or runs a background job can contain multiple commits. Do not assume the whole +workflow is atomic. -**A write returns only after the commit.** When the call returns, the version -is durable in object storage. Nothing is buffered in your process waiting to be -flushed. + -**Every write advances the version**, including one that affects zero rows. A -delete whose predicate matches nothing still commits. Version numbers count -commits, not changes. +Schema changes are also part of table history. Changing a column's type can +rewrite data; adding a derived column can write the new column independently. -**Concurrent writes are resolved optimistically.** A transaction records the -change and the version it was based on. At commit time, a conflicting -concurrent operation is detected rather than silently overwritten, and the -commit is retried against the newer version. Operations that do not collide — -appending while another process appends — both succeed. +## Consistent reads {#reads} - -`merge_insert` reports its retry attempts in the result. A high count means -contention on the same keys, not an error. - +A read of a table uses a consistent snapshot. A consistent snapshot can still +be older than another writer's latest commit. [Consistency](/tables/consistency) +explains how OSS and Enterprise choose the version a reader sees. -## Reads {#reads} +For an experiment that must keep using the same data, select a specific retained +table version rather than relying on a connection's default freshness. -**A read sees a consistent snapshot.** A query reads one table version -throughout; it never sees a mixture of two. +## Concurrent writes {#concurrency} -**Which version, by default, is the one open at the time the table was opened.** -This is the guarantee most often assumed to be stronger than it is: +Lance uses optimistic concurrency control. Writers prepare changes, then check +compatibility with intervening commits. Compatible changes can be committed; +conflicts may be retried or returned as errors. - -A long-lived connection **does not observe other processes' writes** until it -reopens the table or its `read_consistency_interval` elapses. The default is -right for a batch job reading a stable snapshot and wrong for a service -expected to see fresh data. - +Applications must handle failed writes. Concurrent appends are compatible, but +that does not promise success through storage failures or exhausted retries. +Do not rely on a particular ordering between independent writers. -Set `read_consistency_interval` when several processes write the same table and -readers must observe those writes. The cost is a metadata check on that -interval. See [Consistency](/tables/consistency). +The [Lance transaction specification](https://lance.org/format/table/transaction/) +describes commit behavior and compatibility between operations. -**Your own writes are always visible to you.** The interval governs other -writers, not yourself. +## Storage durability {#durability} -## Durability and history {#durability} +A committed table version is stored in the table’s configured storage. LanceDB +OSS uses your local filesystem or object store; LanceDB Enterprise uses the +cluster’s configured storage. -**Old versions are retained, not overwritten.** A new version does not destroy -the one before it, which is what makes [time travel](/management/table#versioning) work -and what lets a reader finish a scan that started on an older version. +Table version history does not replace backups. Configure storage redundancy +and backups to meet your recovery requirements. -**Deletion is a mark, not an erase.** A deleted row is recorded in a deletion -file. The data is recoverable for as long as the version that contains it -exists. +## Background job completion {#jobs} - -Retention is finite. [Compaction](/maintenance/compaction) prunes versions -older than the retention window — **7 days by default** — and a tag or branch -pointing at a pruned version no longer resolves. If you rely on reading old -versions, widen the window before running it, not after. - +In LanceDB Enterprise, operations such as index builds and column refreshes +can run as background jobs. Receiving a job handle confirms submission. Wait +for the job to succeed before relying on its result. -**Dropping a table ends its history.** Versioning does not survive -`drop_table`; there is no undrop. +Some operations save results in batches. If a job fails or is cancelled, +batches already committed can remain in the table. Each operation’s guide +explains whether it can leave partial results. -## Jobs {#jobs} +## Writes after a client timeout {#timeouts} -**A refresh recomputes only what changed.** It gives values to rows that lack -them and recomputes rows whose inputs changed since they were computed; it -leaves the rest alone. Two consequences, both relied on elsewhere: +If a client times out while waiting for a write, the write may already have +committed. Retrying an append can therefore insert the same records again. -- **Retrying is safe.** Re-running a refresh fills what is still empty or stale. - It is not guaranteed to cost only the rows that did not complete: a retry can - recompute values that were already written. -- **A filled value catches up at the next refresh.** If a column derives from - `price` and a row's `price` later changes, the derived value keeps its old - value until the next refresh recomputes it. +Check whether the intended records were written before retrying. For an +operation submitted as a background job, check its status using the job handle. -**What a failure keeps depends on where the refresh ran.** `refresh_column` in -your process commits the column once, so a failure keeps none of its values. A -refresh job on a deployment commits finished batches as it goes, so some values -can remain. +## Version history and recovery {#history} -**Cancellation is a request.** `cancel()` asks a job to stop; it does not -guarantee it stops before doing more work, and cancelling a finished job does -nothing. +Retained table versions let you inspect earlier data or restore a table to a +previous version. Cleanup can remove those versions and their underlying files, +so configure retention to cover the history you need. -**A job id is opaque.** It correlates with server logs and the jobs API. -Parsing it, or storing it to resume the job later, is not supported. +Dropping a table is different from deleting records. Table version history +cannot undo `drop_table`. -See [Jobs](/basics/jobs). + -## Schema {#schema} +## Learn more {#next} -**Schema changes are versioned like data changes.** Adding, renaming, retyping -or dropping a column commits a version, so a schema change can be travelled back -through. - -**A computed column's inputs are protected.** While a declaration references a -column, that column cannot be renamed, retyped, or dropped — the declaration is -part of the schema and will not be invalidated out from under itself. - -**Most schema changes do not rewrite data.** Adding a derived column writes only -the new column. Type changes are the exception and can be substantial. - -## What is not guaranteed {#not-guaranteed} - -Worth stating plainly, because each is occasionally assumed: - -| | | -| --- | --- | -| Multi-table transactions | A commit covers one table. There is no cross-table atomic write. | -| Read-your-writes across processes | Governed by `read_consistency_interval`, not automatic. | -| Unenforced primary keys | `SET UNENFORCED PRIMARY KEY` records intent; it does not enforce uniqueness. | -| Ordering of concurrent appends | Two appends both succeed; their relative row order is not defined. | -| Index freshness | An index does not cover rows added after it was built until [optimized](/maintenance/compaction). | -| Automatic compaction | Nothing runs in the background. `optimize()` runs when you call it. | - -The last two are the most common source of "why is this query slow" — see -[Limits](/basics/limits) for the bounds that pair with them. +- [Consistency](/tables/consistency): control when reads observe updates. +- [Table versioning](/management/table#versioning): inspect and restore history. +- [Cleanup](/maintenance/cleanup): configure retention and understand protected versions. +- [Jobs](/basics/jobs): distinguish submission, completion and cancellation. +- [Column refresh](/feature-engineering/refresh-column): understand partial progress. +- [Architecture](/architecture): understand storage and background maintenance. diff --git a/docs/web/basics/jobs.mdx b/docs/web/basics/jobs.mdx index 652935ea2..0577cd120 100644 --- a/docs/web/basics/jobs.mdx +++ b/docs/web/basics/jobs.mdx @@ -1,196 +1,75 @@ --- title: "Jobs" sidebarTitle: "Job" -description: "Work that outlives the call that started it: submitting, waiting, inspecting, cancelling." +description: "Track background work from submission to completion." icon: hourglass-half -keywords: ["job", "async", "background", "events", "cancel", "refresh"] +keywords: ["job", "background", "progress", "events", "cancel"] +availability: + oss: available + enterprise: available + summary: >- + Job handles track operations in OSS and Enterprise. Enterprise also + supports listing server jobs, reopening them by ID and reading their events. --- -Some operations take longer than a request should. Building an index over a -large table, refreshing a computed column, registering a function -- these run -on the server after the call that started them has returned. +## Job handles {#convention} -Each is submitted the same way and returns the same handle. +A job handle lets you track an operation, such as an index build or a computed +column refresh. Use it to check progress, wait for completion and inspect +failures. - -A [query](/basics/query) also has an `_async` form, but its handle streams rows -rather than waiting for an outcome. Both answer the same four questions -- -which one is this, how is it going, give me the result, stop it -- and differ -only in the third. - +Methods such as Python's `create_index_async` and TypeScript's +`createIndexAsync` return job handles. In OSS, the work runs in your application +process; in Enterprise, it runs on server workers. A returned handle can +represent an operation that has already finished, so check its status before +waiting or requesting cancellation. See the operation's API reference for when +the method returns. -## The `_async` convention {#convention} +## Submit and wait for a job {#waiting} -An operation that can run in the background has two forms. The plain name blocks -until the work is done. The `_async` / `Async` name submits the work and returns -a **job** immediately. +1. **Submit the operation.** Keep the returned job handle. For an Enterprise + job, also keep its ID if another process needs to look it up later. +2. **Check progress.** Read the current status while the operation runs. +3. **Wait for completion.** Check that the job succeeded before using its + result. If it failed or was cancelled, inspect the failure details and any + results it committed before stopping. -| Blocking | Returns a job | SQL | +In Python, `job.status()` reports the current state and `job.wait()` waits for +completion, raising if the job fails or is cancelled. If `job.wait()` times out, the client stops waiting. +The timeout does not cancel the server job. + + + + +## Inspect and cancel jobs {#managing} + +| Task | What to do | +| --- | --- | +| Inspect progress | Check status and refresh the handle to retrieve details. Enterprise job events can help explain ongoing work or failures. | +| Reattach from another process | Open an existing server job with its ID, such as Python's `db.open_job(job_id)`. The server must still retain the job record. | +| Diagnose a failure | Inspect failure details and relevant events. Enterprise SQL also provides `SHOW ERRORS`. | +| Request cancellation | Call the job's cancellation method. Cancellation may take time, and work already committed is not rolled back. | + +Reopening a job retrieves the existing operation; it does not restart failed +work. Treat the ID as an opaque identifier and pass it unchanged to the jobs API. + +Event responses have [row limits](/basics/limits#jobs). An explicit limit can +request more than the default, but it does not guarantee a complete event +history. + +## OSS and Enterprise execution {#in-process} + +| | OSS | Enterprise | | --- | --- | --- | -| `create_index` | `create_index_async` | `CREATE INDEX` | -| `refresh_column` | `refresh_column_async` | `REFRESH COLUMN` | -| `create_function` | `create_function_async` | `CREATE FUNCTION` | -| `drop_table` | `drop_table_async` | `DROP TABLE` | +| Execution | In your application process | On server workers | +| Job identity | No server ID | A server ID for looking up the job | +| Event history | No server event history | Available through the jobs API | +| Reattaching after a client restart | No server job to reopen | Reopen by ID while the record is retained | - -The blocking form is not a different operation -- it submits the same job and -waits. Reach for the `_async` form when you want to do something else meanwhile, -run several at once, or survive the client going away. - +## Learn more {#next} -## Submitting and waiting {#waiting} - - -```python Python icon="python" -from lancedb.index import IvfPq - -job = table.create_index_async("vector", config=IvfPq()) - -print(job.id) # the server's id for this operation -print(job.status()) # "running" | "finished" | "failed" | "cancelled" - -job.wait() # blocks until terminal; raises if it failed -``` - -```typescript TypeScript icon="square-js" -const job = await table.createIndexAsync("vector"); - -console.log(job.id); // the server's id for this operation -console.log(await job.status()); // "running" | "finished" | "failed" | "cancelled" - -await job.wait(); // resolves when terminal; rejects if it failed -``` - - -`wait` is the one call that raises on failure. `status` is a point snapshot and -returns the failed state rather than raising, which is what you want in a -polling loop. - -In Python `wait` takes an optional timeout and returns the operation's typed -result -- a `RefreshColumnJobResult`, a `FunctionVersion`, or `None` for an -operation that has no result: - -```python -from datetime import timedelta - -result = table.refresh_column_async("embedding").wait(timeout=timedelta(minutes=30)) -``` - -`TimeoutError` means the timeout elapsed, not that the job stopped. The job is -still running; the handle just gave up watching. - - -The TypeScript `wait()` resolves to `void`. Read the outcome from `result` after -it resolves. - - -## Inspecting a job {#inspecting} - -Submitting an operation returns only an id, so the detail fields are empty until -the handle fetches the record. `refresh()` does that; `status()` does it as a -side effect. - - -```python Python icon="python" -job.refresh() - -job.state # last observed state, without another round trip -job.job_type # what kind of job, as the server names it -job.creation_ms # when it was created, epoch milliseconds -job.spec # what it was submitted with -job.result # its terminal result, once it succeeds -job.failure # why it failed, when it did -``` - -```typescript TypeScript icon="square-js" -await job.refresh(); - -job.state; // last observed state, without another round trip -job.jobType; // what kind of job, as the server names it -job.creationMs; // when it was created, epoch milliseconds -job.spec; // what it was submitted with -job.result; // its terminal result, once it succeeds -job.failure; // why it failed, when it did -``` - - -Printing the handle renders every field it knows, which is usually faster than -reading them one at a time. - -## Progress, from events {#events} - -`result` only tells you anything once the job has finished. Events are written -as it runs, and they outlive the workers that produced them -- so a job that is -still going, or one that never finished, can still account for what it did. - - -```python Python icon="python" -events = job.events(limit=5000, filter="state = 'claim_complete'") -``` - -```typescript TypeScript icon="square-js" -const events = await job.events({ limit: 5000, filter: "state = 'claim_complete'" }); -``` - -```sql SQL -SHOW ERRORS ON my_table FOR JOB 'job-id' LIMIT 100 -``` - - -A distributed job records a `claim` / `claim_complete` pair per unit of work, -each carrying `rows_processed`. Summing `rows_processed` over the completed -claims is how you get progress on a job that has no percentage. - -`filter` is a SQL-like expression over `state`, `updated_by`, `emitted_from`, -`emitted_by`, and `claim_entity`. - - -The server returns 1000 event rows by default and 10,000 at most, and -**truncates without telling you**. A job that emits an event per fragment will -silently lose its tail unless you pass `limit`. - - -Events are recorded server-side, so this raises for an operation that ran -in-process. - -## Finding and cancelling jobs {#managing} - -The connection lists and reopens jobs, so a handle lost to a restarted process -is recoverable from its id. - - -```python Python icon="python" -db.list_jobs() # every job the server knows about -job = db.open_job(job_id) # reattach to one -db.cancel_job(job_id) # ask it to stop -``` - -```typescript TypeScript icon="square-js" -await db.listJobs(); -const job = await db.openJob(jobId); -await db.cancelJob(jobId); -``` - -```sql SQL -SHOW JOBS -``` - - -Cancellation is a request, not a guarantee, and cancelling a job that already -finished does nothing. `JobNotFoundError` means the id does not name a job the -server still has. - - -A job id is opaque. It correlates with server logs and the calls above, but -parsing it or storing it to resume the job later is not supported. - - -## In-process jobs {#in-process} - -A local database has no server to run anything, so its operations run in your -process. They still return a handle, so the same code works against both -- but -one with nothing behind it: `id` is `None`, `status()` reports `"finished"`, -`wait()` returns immediately, `cancel()` does nothing, and `events()` raises. - -This is the difference to expect when moving between embedded and deployed -- -see [SDKs](/basics/sdk). +- [Build and repair indexes](/indexing/reindexing): submit index work. +- [Column refresh](/feature-engineering/refresh-column): populate derived values. +- [Background jobs](/maintenance/background-jobs): understand Enterprise maintenance. +- [SQL job commands](/sql/ddl/job): list jobs and inspect errors. +- [SDKs](/basics/sdk): find the job API for your client. diff --git a/docs/web/basics/lakehouse.mdx b/docs/web/basics/lakehouse.mdx index 7dd631ac3..04e07afaf 100644 --- a/docs/web/basics/lakehouse.mdx +++ b/docs/web/basics/lakehouse.mdx @@ -1,102 +1,75 @@ --- -title: "Lance Lakehouse" -sidebarTitle: "Lance Lakehouse" -description: "What operates on the catalog's objects: storage, search, computation, and maintenance." +title: "LanceDB Lakehouse" +sidebarTitle: "LanceDB Lakehouse" +description: "Bring multimodal data, search and feature engineering together for AI workloads." icon: layer-group -keywords: ["lakehouse", "multimodal", "search", "compute", "maintenance", "feature engineering"] +keywords: ["lakehouse", "multimodal", "search", "compute", "training", "feature engineering"] --- -The [catalog](/basics/catalog) says what exists. The **lakehouse** is everything -that acts on it: storing the rows, searching them, computing new ones, and -keeping the whole thing fast as it grows. +## Multimodal storage and processing {#multimodal} -The distinction matters because the two layers have different lifetimes. A -catalog entry is a name and a pointer. A lakehouse operation reads and writes -the [Lance](/basics/lance) files that name points at -- and those operations are -where almost all of the cost, and all of the tuning, lives. +LanceDB brings storage, search and data processing together for AI workloads. +You can store multimodal data, derive features, search your records and prepare +training datasets using tables built on the open [Lance format](/basics/lance). -## Multimodal is the point {#multimodal} +The format provides the storage foundation. The [catalog](/basics/catalog) +organizes your tables. LanceDB provides the tools to work with that data. -A lakehouse table holds images, video, audio, text, embeddings, annotations and -the features you derive from them **in one table**, side by side. +## Example: a video search workflow {#capabilities} -That is the difference from assembling a warehouse plus a vector store plus a -feature store. There is one copy of the data, one set of versions, and no -pipeline keeping three systems agreeing about the same rows. A search can filter -on a label, rank by a vector, and return the blob, without leaving the table. +Consider a team building a video search application: -| | | -| --- | --- | -| **Columns** | vectors, blobs, and columns [computed](/feature-engineering/add-computed-column) from other columns | -| **Versions** | every write commits one, so reads can [travel back](/management/table#versioning) | -| **Indexes** | vector, full-text and scalar, built over those columns | -| **Branches** | isolated lines of change you can [merge or discard](/tables/branching) | + + + -## The four capabilities {#capabilities} +1. **Ingest the clips.** Store video clips with metadata such as timestamps, + source and labels. +2. **Add features.** Generate captions and embeddings with your models, and + store them alongside each clip. +3. **Use the dataset.** Search for relevant clips using embeddings and metadata + filters, or select a dataset to read into a training framework. -Everything the lakehouse does falls into one of four groups, and the rest of -these docs is organized the same way. +```mermaid +flowchart TD + clips["Video clips and metadata"] --> table["Store in a Lance table"] + table --> features["Add captions and embeddings"] + features --> search["Search for relevant clips"] + features --> training["Select data for training"] +``` -### Ingest {#ingest} +The clips, metadata and derived features stay associated in the same table, +so retrieval and training workflows can use the same dataset. Your training +framework runs the model; LanceDB provides access to the training data. -Getting rows in, and changing the ones that are there: append, merge, update, -delete. Every write commits a version rather than mutating in place, which is -what makes the other three safe to run against a moving table. See -[Data Ingestion](/ingestion). +## Shared compute and maintenance in Enterprise {#where} -### Search {#search} +LanceDB OSS runs as an embedded library in your application. LanceDB Enterprise +provides shared infrastructure for applications and teams working with larger +multimodal workloads. -Vector, full-text and hybrid retrieval over the same rows, with filters that -push into the scan rather than running after it. The search is a lakehouse -operation, not a separate service holding a second copy of your embeddings. See -[Semantic Search](/search). + -### Compute {#compute} +- **Scale resources for different workloads.** Provision query serving and + background workers independently as search traffic and indexing demand grow. +- **Share data across teams.** Give applications access to shared tables, with + Enterprise permissions controlling who can read or change them. +- **Manage ongoing maintenance.** Configure background indexing, compaction + and cleanup as your datasets grow. -Deriving new columns and tables from what is already stored: -[computed columns](/feature-engineering/add-computed-column), -[functions](/management/function) that run Python where the data is, and -[materialized views](/management/materialized-view) that persist a query's -result. +See [Architecture](/architecture) for how Enterprise separates storage, query +serving and background work. -The reason this belongs in the lakehouse rather than in a pipeline outside it is -the format. Adding a derived column writes only that column -- the images and -videos are never rewritten -- so deriving a feature is an ordinary operation -rather than a migration. See [Feature Engineering](/feature-engineering) and -[Data Curation](/curation). +## Learn more {#next} -### Maintenance {#maintenance} +- [Data ingestion](/ingestion): load and update your data. +- [Search](/search): retrieve records with vector, full-text and hybrid search. +- [Feature engineering](/feature-engineering): derive columns and datasets. +- [Training data access](/training/data-loading): read datasets into training workflows. +- [Enterprise](/enterprise): explore deployment options for shared workloads. -The work that keeps the first three fast: building and repairing -[indexes](/indexing), and [compaction](/maintenance/compaction) to merge small -files, prune old versions and fold new rows into existing indexes. - - -None of it runs on its own. There is no background compactor -- `optimize()` -runs when you call it, and an index does not cover rows added after it was -built until you do. See [Guarantees](/basics/guarantees#not-guaranteed). - - -## Where the work happens {#where} - -The same lakehouse runs in two shapes, and which one you use decides where these -capabilities execute rather than whether you have them: - -- **Embedded** -- your process does all four itself, directly against object - storage. Nothing runs unless you call it. -- **Distributed** -- a cluster does them, with indexing and compaction on - workers separate from the ones serving queries, plus - [SQL](/sql) and server-side [jobs](/basics/jobs). - -See [Architecture](/architecture) for the comparison in full. - -## Next {#next} - - - - The objects these operations act on. - - - The open format underneath, and why column evolution is cheap. - - +{/* +Draft for the Functions release. Add this item to the list above, after +Feature engineering: +- [Functions](/management/function): run Python to compute column values. +*/} diff --git a/docs/web/basics/lance.mdx b/docs/web/basics/lance.mdx index 6552c3735..f2dd0ecfa 100644 --- a/docs/web/basics/lance.mdx +++ b/docs/web/basics/lance.mdx @@ -1,122 +1,45 @@ --- title: "Lance format" sidebarTitle: "Lance Format" -description: "Lance is the open format underneath. LanceDB is what implements a lakehouse on top of it." +description: "The open format for multimodal data behind LanceDB." icon: "/static/assets/logo/lance-logo-gray.svg" -keywords: ["lance", "format", "open standard", "lakehouse", "namespace", "catalog"] +keywords: ["lance", "format", "multimodal", "versioning", "column evolution"] --- -Two names, two things. [Lance](https://lance.org/) is an open format and -specification. LanceDB is an implementation built on it. +## Lance format overview {#standard} -The distinction matters when you are deciding what you are committing to: the -format is a standard other engines read, and your data stays readable by them -whichever LanceDB you run. +[Lance](https://lance.org/) is an open-source, columnar format designed for +multimodal AI. It defines how data is stored and organized into versioned tables. -## Lance is the open standard {#standard} + + -Lance is an open-source, columnar lakehouse format for multimodal AI. It -specifies three things: +LanceDB OSS and LanceDB Enterprise both use Lance as their storage foundation. +The format is developed as an independent open-source project, with its own +specification and tools. -| | | -| --- | --- | -| **File format** | how bytes are laid out on disk, optimized for random access without giving up scan throughput | -| **Table format** | how files compose into a versioned table -- schema evolution, deletions, indexes | -| **Namespace specification** | how tables are named, grouped, and resolved -- the catalog layer | +## Multimodal data {#storage} -Being a specification rather than a product is the point. Object storage holds -the bytes, and open engines -- PyTorch, Ray, Spark, Trino, DuckDB, Polars -- -read them without going through LanceDB at all. +A Lance table can hold images, video, audio, text, embeddings and structured +metadata together. For example, a video dataset can keep each clip alongside +its caption, labels and embeddings, so applications can search and work with +those fields together. -Lance exists because columnar formats derived from Parquet were not designed -for random access on vectors: [benchmarks](https://lancedb.com/blog/benchmarking-random-access-in-lance/) -show Lance up to 1000x faster than Parquet for that access pattern. +## Data access, versioning and column updates {#capabilities} - - The format specification: design, layout, and how it enables the multimodal lakehouse. - +- **Efficient data access.** Read individual rows for retrieval and training, + or scan columns for analytics. +- **Versioned data.** Read an earlier table version to reproduce an experiment + or inspect previous results, while that version is retained. +- **Independent column updates.** Add embeddings or other derived features + without rewriting the existing image, video or text columns. -## LanceDB embedded implements the storage half {#embedded} +## Learn more {#next} -Connecting to a directory or a bucket gives you a **storage-only lakehouse**. -Your process opens Lance tables directly and does the work itself: it plans and -executes queries, builds indexes, and writes new versions. + + -```python -db = lancedb.connect("./.lancedb") -db = lancedb.connect("s3://bucket/path") -``` - -There is no server, which is what makes it embedded -- and also what bounds it. -There is nothing to run work on your behalf, so there are no server-side -[jobs](/basics/jobs), no [SQL endpoint](/basics/sql), and no shared catalog -beyond what the storage layer itself provides. - -LanceDB OSS is Apache 2.0 licensed. LanceDB Enterprise is the commercial, -deployment-only product that implements the catalog specification below. - -## LanceDB Enterprise implements the catalog specification {#enterprise} - -Enterprise A `db://` connection talks to a deployment -that implements the **Lance REST Namespace** -- the REST server defined by the -Lance namespace specification, covering the catalog operations in the -[REST API reference](/api-reference/rest). - -```python -db = lancedb.connect("db://my-database", api_key="...") -``` - -Implementing the specification rather than inventing an interface is what makes -the catalog reachable by anything that speaks it, and what makes -[authorization](/governance/authorization), server-side jobs, and -[SQL](/basics/sql) possible: there is now a server to enforce and to run them. - -The tables are the same Lance tables either way. What changes is who resolves -their names and who does the work. - -## What the format gives you {#capabilities} - -These are properties of Lance, so they hold in both forms: - -| Capability | What it enables | -| --- | --- | -| Multimodal storage | Images, video, audio, text, embeddings, annotations and metadata in one table. | -| First-class blob API | Large binary objects in blob columns, with lazy reads and streaming byte access. | -| Fast random access and scans | Sample, shuffle, and fetch individual rows without giving up sequential throughput. | -| Zero-copy schema evolution | Add, drop, rename, or alter columns without rewriting existing data files. | -| Versioned tables | Reproduce experiments and tie artifacts to the exact table version they used. | -| Hybrid search and indexing | Vector search, full-text search, and scalar filters over the same dataset. | -| Open interoperability | Object storage underneath, open engines on top. | - -Two of them explain behaviour you will meet elsewhere in these docs. - -### Versioning is why old data lingers {#versioning} - -Every write creates a version, and old versions are not immediately destroyed -- -another reader may be part-way through one. A hundred versions are not a hundred -copies of the data, but they do carry a hundred times the metadata, which -eventually slows queries down. - -Deletion works the same way: a deleted row is marked in a deletion file rather -than erased, so the data is recoverable for as long as the version is. - -[Compaction](/maintenance/compaction) is what resolves both. - -### Column evolution is why feature engineering is cheap {#evolution} - -Adding a derived column writes only the new column. Existing data -- the -expensive part, the images and videos -- is untouched. That is what makes -[computed columns](/feature-engineering/add-computed-column) a reasonable thing to -do repeatedly on a large table rather than a full rewrite each time. - - - Lance is a separate open source project with its own documentation. - +- [Lance format specification](https://lance.org/format): how the format works. +- [Table versioning](/management/table#versioning): read and manage table versions. +- [Computed columns](/feature-engineering/add-computed-column): add derived features to your data. +- [Architecture](/architecture): how LanceDB OSS and Enterprise build on Lance. diff --git a/docs/web/basics/limits.mdx b/docs/web/basics/limits.mdx index f135de8d4..0d4f1b028 100644 --- a/docs/web/basics/limits.mdx +++ b/docs/web/basics/limits.mdx @@ -1,106 +1,71 @@ --- title: "Limits" sidebarTitle: "Limits" -description: "The bounds you can actually hit, in one place." +description: "Find fixed constraints, configurable bounds and what happens when you reach them." icon: ruler keywords: ["limits", "maximum", "quota", "constraints", "bounds"] --- -Limits worth designing around, collected so you do not have to find them by -hitting them. Each one is enforced -- where a value is refused rather than -clamped, that is stated, because the two fail very differently. + -This page covers bounds that are the same for everyone. Deployment-specific -quotas -- storage, throughput, concurrency -- come from your deployment. +This page lists limits for job events, SQL error reports, sampling and +full-text indexing. Each table identifies the affected API and what happens +when a request reaches its limit. SQL and server job limits apply to Enterprise. -## Names {#names} + -| Object | Maximum | | +[Naming Rules](/basics/naming) is the reference for name lengths, characters and +namespace depth. Client and server versions can differ in which rules they +enforce. + + + + +## Fixed API limits {#fixed-limits} + +| Operation | Scope | Limit and behavior | | --- | --- | --- | -| Database | 64 characters | | -| Namespace | 64 characters | | -| Table | 128 characters | | -| Branch | 64 characters | | +| Job event retrieval | Enterprise | At most 10,000 rows per response. Results can be truncated without a truncation indicator. | +| `SHOW ERRORS ... LIMIT` | Enterprise SQL | Accepts 1 through 100,000. Values outside the range are rejected. | +| `SHOW ERRORS` without `FOR JOB` | Enterprise SQL | Inspects up to the 32 most recent failed jobs. Use `FOR JOB` to inspect a specific job. | +| Percentage sampling | Enterprise SQL | `TABLESAMPLE` accepts percentages from 0 through 100. Values outside the range are rejected. | +| Row-count sampling | Enterprise SQL | `TABLESAMPLE (... ROWS)` requires a whole number. | -A name must not be empty, and must not contain `$` or `:` -- both are reserved -as path delimiters, so a name holding one could not be addressed unambiguously. -Length is counted in characters, not bytes. +See [SQL job commands](/sql/ddl/job#errors) and +[Sampling](/sql/dql/sampling) for syntax and restrictions on supported table types. -## Jobs {#jobs} + -| | | +Data-type and index constraints belong to the operation you choose. Check +[Data types](/sql/data-types) and [Vector indexes](/indexing/vector-index) +before choosing a schema or index. For materialized views, check +[stable row ID requirements](/feature-engineering/materialized-views#stable-row-ids) +before creating source tables. + +## Configurable limits {#not-quotas} + +| Setting | What it controls | | --- | --- | -| `job.events()` default | 1,000 rows | -| `job.events()` maximum | 10,000 rows | -| `SHOW ERRORS ... LIMIT` | between 1 and 100,000 | -| Failed jobs scanned by one `SHOW ERRORS` | 32 | +| Job event `limit` | Requests more or fewer event rows. The default is 1,000; the 10,000-row maximum still applies. | +| FTS `max_token_length` | The maximum indexed token length. The default is 40; longer tokens are excluded from the index. | +| Enterprise capacity and quotas | Storage, request rate and concurrency are governed by your deployment's resources and configuration. Confirm these with your administrator or LanceDB support. | - -`job.events()` **truncates silently**. A job that emits an event per fragment -will lose its tail unless you pass `limit` explicitly. This is the limit on this -page most likely to mislead you, because nothing in the result says it was cut. - +## Handling rejected or incomplete results {#behavior} -`SHOW ERRORS` without `FOR JOB` scans the most recent failed jobs and stops at -32 of them. Narrow with `FOR JOB` when you need a specific one rather than a -survey. +- **Rejected requests:** correct the input before retrying, such as a sampling + percentage outside the accepted range. +- **Truncated responses:** do not assume that a returned event list is the full + history. Increasing `limit` cannot exceed the maximum; use filters to narrow + the events you inspect. +- **Excluded data:** an FTS token-length setting can affect which words are + searchable. Choose it for your text and indexing requirements. -See [Jobs](/basics/jobs). + -## Schema and types {#schema} +## Learn more {#next} -| | | -| --- | --- | -| `map` with `keys_sorted = true` | rejected | -| `IVF_RQ` vector dimensions | must be divisible by 8 | -| FTS `max_token_length` | 40 by default; longer tokens are dropped | - -Through the SDK, a table gets **stable row ids** only at creation, from the -`new_table_enable_stable_row_ids` storage option on a local path or an object -store; no SDK call adds them later, so decide early. A local -[materialized view](/feature-engineering/materialized-views#stable-row-ids) does -not need them. On a deployment, whether a view's source needs them and how a -table gets them depend on the deployment; see -[Against a deployment](/feature-engineering/materialized-views#availability). - -See [Data types](/sql/data-types). - -## SQL {#sql} - -Enterprise - -| | | -| --- | --- | -| `TABLESAMPLE` percentage | 0 to 100, **refused** outside -- not clamped | -| `TABLESAMPLE (n ROWS)` | whole numbers only | -| `TABLESAMPLE BUCKET` | not supported | -| `TABLESAMPLE` on a WAL-backed table | not supported | -| `CREATE INDEX` | one column, server-chosen name | -| `REFRESH MATERIALIZED VIEW ... CASCADE` | depends on the deployment's version; see [REFRESH](/sql/dml/refresh#refresh-materialized-view) | - -A sampling percentage over 100 is an error rather than a full scan, because a -query asking for 150 percent has a mistake in it and reading it as 100 would -hide the mistake. See [Sampling](/sql/dql/sampling). - -## What is unavailable where {#availability} - -Not quotas, but the same class of surprise -- things that exist in one place and -not another. - -| | | -| --- | --- | -| SQL on a local database | unavailable; `execute_query` raises | -| SQL methods in the TypeScript SDK | none; use a Flight SQL client | -| Function methods in the TypeScript SDK | none; use Python or SQL | -| Vector search in SQL | no SQL form; use a query builder | -| Geospatial `st_*` functions | behind a build feature LanceDB does not ship | -| `tableExists` / `namespaceExists` in TypeScript | not present | -| Server-side jobs on a local database | run in-process; `events()` raises | - -See [SQL](/basics/sql) for why the first two hold. - -## Nothing here is a quota {#not-quotas} - -None of the above is a billing or capacity limit. Rows per table, table count, -storage, and request rate are bounded by your deployment rather than by the -format -- ask your deployment, not this page. +- [Naming Rules](/basics/naming): choose portable names and paths. +- [Jobs](/basics/jobs): track work and inspect events. +- [Full-text indexes](/indexing/fts-index): configure tokenization. +- [SQL](/basics/sql) and [SDKs](/basics/sdk): check interface support. +- [Enterprise support](/deployment/support): discuss deployment capacity. diff --git a/docs/web/basics/naming.mdx b/docs/web/basics/naming.mdx new file mode 100644 index 000000000..af61c2b6c --- /dev/null +++ b/docs/web/basics/naming.mdx @@ -0,0 +1,69 @@ +--- +title: "Naming rules" +sidebarTitle: "Naming Rules" +description: "Choose portable names for your data objects and namespace paths." +icon: signature +keywords: ["names", "naming", "table name", "namespace", "identifier", "limits"] +--- + +## Naming rules {#rules} + + + +Databases, namespaces, tables, views and branches all have names. Choose names +that describe their purpose, such as `documents`, `prod` or `training_2026`. +Consistent names make data easier to find and application code easier to read. + +Use the following rules for names that work across LanceDB OSS and Enterprise. + +| Object | Characters | Length | +| --- | --- | --- | +| Table, materialized view, view | ASCII letters and digits, `-`, `_`, `.` | 1 to 128 characters | +| Namespace component | ASCII letters and digits, `-`, `_`, `.` | 1 to 64 characters; at most 10 components in a path | +| Database | ASCII letters and digits, `-`, `_`, `.`, with `/` between components | 1 to 64 characters for the whole name | +| Branch | Avoid `$` and `:`; use ASCII letters, digits, `-` and `_` for portability | Keep to 1 to 64 characters | + +- Names and individual namespace or database components must not be empty. +- Table, view, namespace and database components cannot be `.` or `..`. +- Avoid `$` and `:` because they separate parts of qualified identifiers. +- A namespace path beginning with `public` cannot contain another component. + +{/* +Draft for the Functions release. Add these rows to the end of the table above: +| Secret | ASCII letters and digits, `-`, `_`, `.` | 1 to 128 characters | +| Function | ASCII letters and digits, `_` | 1 to 128 characters | +Add this item to the list above, after the rule for `.` and `..`: +- Secret names cannot be `.` or `..`. +*/} + +## Namespace paths {#paths} + +A namespace path is a list of names. `["prod", "search"]` identifies the +`search` namespace within `prod`. A table named `documents` belongs to that +namespace; its table name remains `documents`. + +Use the SDK's namespace-path argument to supply the list. A period inside an +SDK name is a character in that name, not a separator that creates another +namespace. SQL uses its own qualified-name syntax; see +[SQL namespace paths](/sql/dql/select#namespaces). + +The empty path `[]` identifies the root namespace. + +## Examples {#examples} + +| Name or path | Guidance | +| --- | --- | +| `documents`, `user-events.v2`, `train_2026` | Portable table names | +| `["prod", "search"]` | A two-component namespace path | +| `my table` | Use `my_table` instead of a space | +| `orders$2026` | Use `orders_2026`; `$` is reserved | +| `résumés` | Use an ASCII name such as `resumes` | +| `["public", "search"]` | Invalid nesting under `public`; use a different parent namespace | + + + +## Learn more {#next} + +- [Namespaces](/management/namespace): create namespaces and address tables. +- [Branches](/tables/branching): name and manage separate lines of table history. +- [Limits](/basics/limits): check other constraints before designing your schema. diff --git a/docs/web/basics/offerings.mdx b/docs/web/basics/offerings.mdx new file mode 100644 index 000000000..c66b77e78 --- /dev/null +++ b/docs/web/basics/offerings.mdx @@ -0,0 +1,70 @@ +--- +title: "LanceDB OSS or Enterprise?" +sidebarTitle: "OSS and Enterprise" +description: "Start with an embedded library, or use the LanceDB Enterprise multimodal lakehouse for shared AI workloads." +icon: compass +keywords: ["oss", "enterprise", "embedded", "deployment", "availability", "comparison"] +--- + +LanceDB gives you two ways to work with multimodal data, both built on the open +[Lance](/basics/lance) format. + +**LanceDB OSS** is an embedded library. Run it inside your application +with tables on local disk or in object storage. You control the application, +its resources, and table maintenance, without a separate database service. +[Try the quickstart](/quickstart) to create a table and run your first search. + +**LanceDB Enterprise** is a multimodal lakehouse for data curation, feature +engineering, search, and training data access. It adds query-serving capacity, +data caching, and background operations for teams working with shared +multimodal datasets. +[Connect to Enterprise](/basics/connect#enterprise) when your cluster is ready. + + + + +## Why choose Enterprise? {#why-enterprise} + +### Serve growing workloads {#serving} + +Add query-serving capacity as more applications and users depend on your data. +Enterprise spreads query traffic across a fleet and caches frequently read +table data near compute, reducing repeated reads from object storage. Query +serving and background work use separate resources, so you can size each for +your workload. See [Enterprise architecture](/enterprise/architecture). + +### Reduce operational work {#operations} + +Let the service handle routine table maintenance instead of scheduling it in +application code. Enterprise handles background [compaction](/maintenance/compaction) and can +[clean up old table versions](/maintenance/cleanup) according to your retention +policy. Configure these maintenance tasks for your Enterprise cluster so each +application does not have to schedule them. + +### Fit your deployment requirements {#deployment} + +Choose a managed deployment or bring LanceDB into your own cloud account. +Enterprise integrates with OpenTelemetry for metrics and traces, helping your +team monitor the service alongside your applications. Talk to us about your +cloud, networking, and operational requirements to plan the right deployment. +The [Enterprise overview](/enterprise) explains the available options. + +### Connect your data workflows {#workflows} + +Use the same tables for retrieval, dataset preparation, and training data access. +With its [SQL endpoint](/basics/sql) enabled, Enterprise supports analytical +[curation](/curation), including joins, grouping, and sampling. +[Feature engineering](/feature-engineering) adds derived columns and tables; +computed columns and materialized views require the corresponding features to +be enabled for your Enterprise cluster. +[Geneva](/geneva) runs Python feature backfills as distributed jobs. +[Training data loaders](/training/data-loading), available in both offerings, +read the resulting datasets for model training. + +## Plan your Enterprise deployment {#get-started} + +[Talk to us about your workload](https://www.lancedb.com/contact) to discuss your +data, query traffic, pipelines, and deployment requirements. + +For a closer look at how the service runs, read the +[Enterprise architecture](/enterprise/architecture). diff --git a/docs/web/basics/query.mdx b/docs/web/basics/query.mdx index 449c9b498..624617918 100644 --- a/docs/web/basics/query.mdx +++ b/docs/web/basics/query.mdx @@ -1,94 +1,55 @@ --- title: "Queries" sidebarTitle: "Query" -description: "Running a query, and reading its results before it finishes." +description: "Choose how to query your data and read the results." icon: magnifying-glass -keywords: ["query", "async", "streaming", "arrow", "reader", "execute_query"] +keywords: ["query", "search", "SQL", "async", "streaming", "results"] --- -A query returns rows. How you get them depends on two choices that are easy to -conflate: whether **your thread** blocks, and whether you take the results all -at once or stream them as they arrive. +## Querying tables {#queries} -## Blocking or not is a language concern {#async-client} +A query selects data from your tables. You can filter rows, choose columns, +search for similar content or combine those operations. For example, a video +application might find clips similar to a sample and return only those recorded +in the last month. -A concern of your language, not of LanceDB. The work is identical either way. +## SDK searches and SQL queries {#searches} - -```python Python icon="python" -db = lancedb.connect("db://my-db", api_key="...") # blocking -tbl = db.open_table("documents") +| Interface | When to use it | +| --- | --- | +| **SDK query builders** | Build vector, full-text and hybrid searches, apply filters and choose the fields to return. Available in OSS and Enterprise. | +| **Enterprise SQL** | Write SQL queries for filtering, aggregation, joins and other data curation tasks. Connect through Python or a Flight SQL client. | -db = await lancedb.connect_async("db://my-db", api_key="...") # coroutines -tbl = await db.open_table("documents") -``` +Both interfaces work with LanceDB tables. Choose the one that supports your +operation and fits your application. See [SQL](/basics/sql) for connection +instructions and SQL search support, and [Search](/search) for the query builders. -```typescript TypeScript icon="square-js" -// TypeScript is async throughout; there is no blocking form. -const db = await lancedb.connect("db://my-db", { apiKey: "..." }); -const tbl = await db.openTable("documents"); -``` - +## Collect or stream results {#results} -Python offers both. `AsyncConnection` exposes the same surface as coroutines, so -choosing between them is about the program you are writing -- a service wants -the async form, a script does not care. +- **Collect the results** when you want a list, Arrow table or DataFrame that + fits in memory. Use a result limit and select only the columns you need. +- **Read batches** when you want to process a large result incrementally. + Reading batches avoids loading the entire result into application memory. + It does not guarantee that the query can return its first batch immediately. -## Reading before the query finishes {#queries} +### Asynchronous clients {#async-client} -A query is not a job, but it has the same shape of choice: wait for everything, -or take a handle and start reading. +An async client lets your application do other work while a request is waiting. +That is separate from whether you collect results or read them in batches. +Python offers synchronous and asynchronous clients; TypeScript uses async APIs. +Check the SDK reference for the methods available in your client. - -```python Python icon="python" -reader = db.execute_query("SELECT id, text FROM my_docs LIMIT 1000") -table = reader.read_all() +### Query handles and background jobs {#jobs} -query = db.execute_query_async("SELECT id, text FROM my_docs") -``` +An Enterprise SQL query handle lets you inspect query execution and request +cancellation. Background operations such as index builds return a +[job](/basics/jobs) handle instead. Use the API for the returned handle to +check status and retrieve results. -```sql SQL -SELECT id, text FROM my_docs LIMIT 1000 -``` - +## Learn more {#next} -`execute_query` returns a blocking Arrow reader **once results start arriving** --- it does not wait for the whole query to finish. Results stream as record -batches, so a large result set does not have to fit in memory before you see its -first row. - -`execute_query_async` returns the query handle instead: - -```python -query = db.execute_query_async("SELECT id, text FROM my_docs") - -query.id # identifies it on the server -query.describe() # what it is doing, without consuming results -reader = query.reader() # stream the rows when you want them -query.cancel() # stop it -``` - -`describe_query(id)` reports on one from elsewhere, the way `open_job(id)` does -for a job. - - -SQL requires a deployment. See [SQL](/basics/sql#sql-is-remote). - - -Query-builder searches -- vector, full-text, hybrid -- are ordinary calls that -return results. They are fast enough not to need a handle. See -[Query](/search). - -## Searches are ordinary calls {#searches} - -Query-builder searches -- [vector](/search/vector-search), -[full-text](/search/full-text-search), [hybrid](/search/hybrid-search) -- return -results directly. They are fast enough not to need a handle, and they have no -SQL form. See [Semantic Search](/search). - -## When the work outlives the call {#jobs} - -Some operations take longer than a query should: building an index, refreshing a -computed column, registering a function. Those return a **job** rather than -rows, and a job is a different handle with a different shape. See -[Jobs](/basics/jobs). +- [Quickstart](/quickstart): run a complete create, insert and search example. +- [Search](/search): choose vector, full-text or hybrid retrieval. +- [Filtering](/search/filtering): narrow results with predicates. +- [SQL](/basics/sql): connect a client and run SQL queries. +- [SDKs](/basics/sdk): choose a language and find its API reference. diff --git a/docs/web/basics/sdk.mdx b/docs/web/basics/sdk.mdx index 3f8170e11..fc386bc7a 100644 --- a/docs/web/basics/sdk.mdx +++ b/docs/web/basics/sdk.mdx @@ -1,99 +1,117 @@ --- title: "SDKs" sidebarTitle: "SDKs" -description: "The language clients — Python, TypeScript, Rust — and what stays true across all of them." +description: "Choose a LanceDB SDK and find the API reference for your language." icon: code-compare keywords: ["python", "typescript", "rust", "java", "sdk", "rest", "api reference"] +availability: + oss: available + enterprise: available + summary: >- + Python, TypeScript and Rust support both LanceDB OSS and LanceDB Enterprise. + Java provides a client for the Enterprise REST API. --- -A language SDK is the primary way to work with LanceDB: create and manage -tables, write and query data, build indexes, run feature engineering. The same -table API works whether you're connected to an embedded database or an -Enterprise deployment -- application code moves between the two by changing -the connection URI. +Use a LanceDB SDK to create tables, add data, build indexes and run searches +from your application. Python, TypeScript and Rust work with both the embedded +OSS library and LanceDB Enterprise. -## Officially supported SDKs {#supported-sdks} +The [quickstart](/quickstart) includes installation instructions and complete +examples in all three languages. For connection settings, see +[Connect to LanceDB](/basics/connect). -Python, TypeScript and Rust are supported against both embedded and deployed -LanceDB. +## Official SDKs {#supported-sdks} -| Reference | Description | -|:--------------|-------------------| -| [Python](https://lancedb.github.io/lancedb/python/python/) | Full-featured Python client with pandas and numpy integration | -| [TypeScript](https://lancedb.github.io/lancedb/js/) | A TypeScript wrapper around the Rust library, built with `napi-rs` | -| [Rust](https://docs.rs/lancedb/latest/lancedb/index.html) | Native Rust library with persistent storage and high performance | +| SDK | Use it with | API reference | +| --- | --- | --- | +| Python | Python applications, notebooks, pandas and PyArrow | [Python reference](https://lancedb.github.io/lancedb/python/python/) | +| TypeScript | Node.js and TypeScript applications | [TypeScript reference](https://lancedb.github.io/lancedb/js/) | +| Rust | Rust applications using the native library | [Rust reference](https://docs.rs/lancedb/latest/lancedb/index.html) | +| Java | Applications using the Enterprise REST API | [Java reference](https://lancedb.github.io/lancedb/java/java/) | -Not every SDK covers the same surface, though: +Guides show the available language examples together. Use the API reference for +method signatures and options; SDKs do not expose every feature through the same +methods. In Rust, enable the `remote` feature to connect to Enterprise. -| | Python | TypeScript | -| --- | :---: | :---: | -| Create and manage catalog objects | yes | yes | -| Insert, merge, update, delete | yes | yes | -| Filter, project, aggregate, join | yes | yes | -| Vector search | yes | yes | -| Full-text search | yes | yes | -| Feature engineering, jobs | yes | partly | -| Runs against a local database | yes | yes | - -TypeScript has no methods for [functions](/management/function). Reach the -gap with SQL instead -- see [SQL](/basics/sql). +For SQL joins, aggregations and window functions, use +[SQL](/basics/sql). Python provides a SQL client, and applications in other +languages can use a Flight SQL client. TypeScript's table query API does not +provide a SQL execution method. ## REST API {#rest} -Enterprise A deployment implements the -[Lance REST Namespace](/basics/lance#enterprise), so the catalog is reachable -over HTTP by anything that speaks it -- see the -[REST API reference](/api-reference/rest). - -| Reference | Description | -|:--------------|-------------------| -| [Java](https://lancedb.github.io/lancedb/java/java/) | REST API Enterprise SDK in Java | +LanceDB Enterprise also exposes an HTTP API for catalog and table operations. +Use it when integrating a language or tool without a LanceDB SDK. The +[REST API reference](/api-reference/rest) describes its endpoints and request +formats. For SQL queries, connect through [Flight SQL](/basics/sql). ## Community SDKs {#community-sdks} -Contributed by the community rather than supported by LanceDB, so feature parity -and release cadence vary. +These clients are maintained by the community. Check each project's supported +features, installation instructions and releases before choosing one. -| Reference | Description | -|:--------------|-------------------| -| [Go](https://pkg.go.dev/github.com/lancedb/lancedb-go/pkg/lancedb) | Community-contributed Go SDK | -| [Ruby](https://github.com/scientist-labs/lancelot) | Community-contributed Ruby bindings | -| [Swift](https://github.com/RyanLisse/LanceDbSwiftKit) | Community-contributed Swift SDK | -| [R](https://github.com/CathalByrneGit/lancedb) | Community-contributed R package | -| [Flutter](https://github.com/Alexcn/flutter_lancedb) | Community-contributed Flutter bindings | - -For any other language, [SQL](/basics/sql) reaches the same tables without a -LanceDB-specific SDK at all. +| Language | Project | +| --- | --- | +| Go | [lancedb-go](https://pkg.go.dev/github.com/lancedb/lancedb-go/pkg/lancedb) | +| Ruby | [Lancelot](https://github.com/scientist-labs/lancelot) | +| Swift | [LanceDbSwiftKit](https://github.com/RyanLisse/LanceDbSwiftKit) | +| R | [lancedb](https://github.com/CathalByrneGit/lancedb) | +| Flutter | [flutter_lancedb](https://github.com/Alexcn/flutter_lancedb) | ## Python multiprocessing {#multiprocessing} -Do not use `fork` as the start method with Python's `multiprocessing` module. -Lance is multi-threaded internally, and `fork` with alive threads is unsafe -- -use `spawn` instead. See -[this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555) -for why. +Use `spawn` when starting Python worker processes with `multiprocessing`. +LanceDB uses threads internally, and forking a process with active threads can +cause deadlocks. Create a connection inside each worker instead of sharing an +open connection from the parent process. -## What stays the same {#invariants} +For PyTorch worker setup and connection factories, see +[PyTorch integration](/training/torch). -Whichever you pick: +## Moving to LanceDB Enterprise {#moving-to-enterprise} -- **Types are Arrow types.** A vector column is a `FixedSizeList` - everywhere, not a SQL type mapped onto storage. See - [Functions and types](/sql/functions). -- **Reads are Arrow.** Nothing is serialized into language objects on the way out - unless you convert it. -- **The catalog is one catalog.** A table created from TypeScript is visible to - Python and to SQL immediately -- there is no per-language namespace. +The core table workflow stays familiar: connect, create or open a table, write +records, and query them. Enterprise runs those operations on remote compute and +manages background maintenance. -## What to read next {#next} +- **Configure the connection.** Replace the local path with your database URI, + and supply the endpoint, region and credentials provided by your administrator. + [Connect to Enterprise](/basics/connect#enterprise) shows these settings. +- **Retrieve query results.** Python's remote tables do not support whole-table + `to_arrow()` or `to_pandas()` calls. Use a query and select the columns and + number of rows you need, for example `table.search().limit(10).to_arrow()`. +- **Handle table listings and namespaces.** Remote table listings are paginated. + Use the namespace and pagination options described in + [Namespaces](/management/namespace). +- **Use server-side jobs.** Index builds and feature refreshes can run as + [jobs](/basics/jobs). Each operation's guide explains how to submit work and + track completion. +- **Let Enterprise manage maintenance.** Use + [background maintenance](/maintenance/background-jobs) rather than calling + the embedded table's `optimize()` method. + +For SQL connection details, see [SQL](/basics/sql). For the operating model and +deployment options, see [LanceDB Enterprise](/enterprise). + +## Shared data across languages {#invariants} + +LanceDB uses Arrow schemas across its SDKs. A fixed-size vector column stores +vectors of a consistent dimension; the [data types reference](/sql/data-types) +describes the supported element types and other column types. + +Applications in different languages can work with the same tables. Connect to +the same database and namespace, use the appropriate credentials, and account +for [read consistency](/tables/consistency) when another process writes data. + +## Related guides {#next} - The same tables, reached by any Flight SQL client -- and the one way in for a language with no SDK. + Query Enterprise tables with SQL from Python or a Flight SQL client. - Give a coding agent a current reference to this same API surface. + Use the LanceDB plugin to help a coding agent work with your data. diff --git a/docs/web/basics/sql.mdx b/docs/web/basics/sql.mdx index 6b74486eb..1c4ad46cd 100644 --- a/docs/web/basics/sql.mdx +++ b/docs/web/basics/sql.mdx @@ -1,125 +1,159 @@ --- title: "SQL" sidebarTitle: "SQL" -description: "The same tables, reached with SQL — the universal access path, and where every operation is formally documented." +description: "Query LanceDB Enterprise with SQL from Python or an Arrow Flight SQL client." icon: table-list keywords: ["sql", "flight sql", "datafusion", "reference", "dialect"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL through Arrow Flight SQL. Use the SDK table APIs for embedded OSS tables. See [Connection settings](#sql-is-remote) to connect. --- -Enterprise SQL runs on a deployment. A local -database has no engine to plan a statement, so `execute_query` raises there. +Use SQL in LanceDB Enterprise to filter records, join tables, calculate +aggregates, and prepare datasets. SQL queries work with the same Lance tables +you create and update through the SDKs, without exporting the data to another +system. -SQL is a second way to reach the same tables the [SDKs](/basics/sdk) do -- -not a different product, a different interface onto one engine. A table -written from Python is queried from SQL without an export step, and a table -created from SQL is visible to every SDK immediately. +You can run SQL through the Python SDK or an Arrow Flight SQL client. This +guide covers connection settings, a first query, and how SQL fits alongside +the SDK table APIs. -## Any language, via Flight SQL {#flight-sql} +## Connect a SQL client {#flight-sql} -SQL is served over an +Enterprise exposes an [Arrow Flight SQL](https://arrow.apache.org/docs/format/FlightSql.html) -endpoint, so any Flight SQL client reaches it -- which makes SQL the way in -for a language LanceDB has no SDK for. You don't need a LanceDB-specific -client at all, just a Flight SQL driver for your language. +endpoint. A compatible client sends SQL statements to this endpoint and +receives results as Arrow record batches. -The TypeScript SDK itself has no SQL method, for the same reason a Go or C# -application would reach for SQL first: connect a Flight SQL client instead of -waiting on a native binding. See [SDKs](/basics/sdk) for the officially -supported and community clients, and reach for Flight SQL whenever yours -isn't among them. +- **Python:** use `execute_query()` on an Enterprise connection, as shown below. +- **TypeScript:** use a separate Flight SQL client. The LanceDB TypeScript SDK + does not expose a SQL execution method. +- **Other languages and tools:** use a Flight SQL client or a tool with Flight + SQL integration. A LanceDB-specific SDK is not required to submit SQL. -## Every SDK operation, in SQL {#sdk-parity} +The [SQL reference](/sql#connecting) shows a query from Python. See +[SDKs](/basics/sdk) for the LanceDB language clients. -Broadly, a public SDK method has a corresponding SQL statement -- creating a -table is `CREATE TABLE`, inserting rows is `INSERT INTO`, building an index is -`CREATE INDEX`. The two describe the same operations, so whichever one your -code speaks, the other is available too. +### Connection settings {#sql-is-remote} -| | SDK | SQL | -| --- | :---: | :---: | -| Create and manage catalog objects | yes | partly | -| Insert, merge, update, delete | yes | yes | -| Filter, project, aggregate, join | yes | yes | -| Vector search | yes | version-dependent | -| Full-text search | yes | yes | -| Feature engineering, jobs | yes | yes | -| Runs against a local database | yes | **no** | +Get the following values from your Enterprise administrator: -## Vector search depends on the deployment {#no-vector-sql} +- Database URI, such as `db://your-database`. +- API key and region. +- API endpoint for `host_override`. +- Flight SQL endpoint for `sql_host_override`, including its scheme and port. -The SDK query builder provides vector search across embedded and deployed -databases. SQL support depends on the deployment version and supported query -shape. Some versions recognize `ORDER BY distance(vector, query_vector) LIMIT k` -over a table scan and plan vector search, including indexed and flat search. -Confirm support on your deployment and use SQL -[`EXPLAIN`](/sql/dql/explain) to check the chosen plan; this is not a guarantee -that every deployed release supports that form. +The Flight SQL endpoint must be enabled and reachable from your client. Pass +it explicitly as `sql_host_override` in Python, as in the example below. The +API and SQL endpoints may use different hosts or ports. -Full-text search is available through the [`fts()`](/sql/functions#fts) table -function and the [`contains_tokens`](/sql/scalar-functions#contains-tokens) -predicate. +See [Connect to Enterprise](/basics/connect#enterprise) for other connection options. -The reverse gap also exists: a [view](/management/view) is a saved SQL query, -so SQL is the only way to create one -- there's nothing for an SDK method to -call. +SQL runs in Enterprise. Connections to a local directory or an object-storage +bucket use the embedded OSS library and cannot execute SQL statements. Use +SDK table operations to query those tables. -## SQL needs a deployment {#sql-is-remote} +## Run a query in Python {#sql-from-python} -A connection to a directory or a bucket has no server to plan and execute a -statement, and asking it to raises `NotImplementedError`. This is the largest -behavioral difference between the embedded and deployed forms, and the reason -the SDKs are the portable choice: SDK code moves between the two by changing -the URI, SQL does not. +Install the SDK with `pip install lancedb`. Replace the connection placeholders +with your Enterprise settings, including the region and both endpoint URLs. +This example reads an existing table named `my_docs` with `id` and `text` +columns. Change the query to match your table. ```python -db = lancedb.connect("./.lancedb") # no SQL -db = lancedb.connect("db://my-database") # SQL -``` +import lancedb -## Running SQL from Python {#sql-from-python} +db = lancedb.connect( + "db://your-database", + api_key="your-api-key", + region="us-east-1", + host_override="https://your-api-endpoint", + sql_host_override="https://your-sql-endpoint:443", +) -```python reader = db.execute_query("SELECT id, text FROM my_docs LIMIT 10") table = reader.read_all() +print(table.to_pylist()) ``` -`execute_query` returns a blocking Arrow reader once results start arriving; it -does not wait for the whole query. `execute_query_async` returns a handle -instead -- see [Queries](/basics/query). +In application code, load credentials from environment variables or a secrets +manager. The [connection guide](/basics/connect#enterprise) shows that setup. -## Choosing between them {#choosing} +`execute_query()` returns a blocking Arrow reader when results start arriving. +The example uses `read_all()` to collect the small result in memory. For larger +results, consume the reader in batches. To submit a query and check its status +separately, use `execute_query_async()`. See [Queries](/basics/query) for query +handles, streaming, and cancellation. -Reach for **the SDKs** when you are writing data, managing the catalog, running -vector search, or building an application that should work the same embedded and -deployed. +## Choose SQL or SDK table operations {#choosing} -Reach for **SQL** when the question is analytical -- joins across tables, -aggregates, window functions -- when the thing asking is a BI tool, a -notebook, or a person who already thinks in SQL, or when it's the only client -available for your language. +Use **SQL** for joins, aggregations, window functions, and analytical queries +written in SQL. Use **SDK table operations** to build applications around table +management, ingestion, filtering, and search, including applications that run +with the embedded OSS library. -The two compose. A common shape is SDK code to build and maintain tables, and -SQL for the analysis on top. +You can combine them. For example, create and populate a table with an SDK, +then query it with SQL to prepare a training dataset. In Python, both workflows +can use the same Enterprise connection. -## The complete reference {#reference} +### Common SQL operations {#sdk-parity} -This page is the concept; [SQL Reference](/sql) is the specification. It has -one page per statement, syntax first -- and since every SDK operation has a -SQL form, those pages end up defining the precise behavior of the SDK -operations too, not just the SQL dialect. +Several tasks are available through both interfaces, but the methods and SQL +statements do not have identical coverage. SDK support also differs by +language. These SQL reference pages describe the supported statements: + +| Task | SQL reference | +| --- | --- | +| Create a table | [CREATE TABLE](/sql/ddl/table) | +| Insert or change rows | [INSERT](/sql/dml/insert), [UPDATE](/sql/dml/update), [DELETE](/sql/dml/delete) | +| Build an index | [CREATE INDEX](/sql/ddl/index) | +| Join and aggregate data | [SELECT](/sql/dql/select) | +| Save a query as a view | [CREATE VIEW](/sql/ddl/view) | + +[Computed columns](/feature-engineering/add-computed-column) store SQL +expressions whose values are populated by a refresh. {/* Draft for the Functions release: +They are separate from registered Python Functions. */} +See the computed-column and +[materialized-view](/management/materialized-view) guides for their SDK and SQL +operations and Enterprise setup requirements. + +## Vector and full-text search {#no-vector-sql} + +SQL vector search support depends on the Enterprise server version and query +form. Versions that support `ORDER BY distance(vector, query_vector) LIMIT k` +over a table scan can plan indexed or flat vector search. Check that your +server version supports this form, then use +[`EXPLAIN`](/sql/dql/explain) to inspect the query plan. + +The [SDK vector search API](/search/vector-search) is also available in +Enterprise and OSS. Use it when you prefer the SDK query builder or need to +search embedded tables. + +For full-text search in SQL, use the [`fts()`](/sql/functions#fts) table +function or the [`contains_tokens`](/sql/scalar-functions#contains-tokens) +predicate. + +## SQL syntax and API references {#reference} + +The [SQL reference](/sql) documents statement syntax, supported expressions, +and data types. For SDK methods and their arguments, use the +[Python](https://lancedb.github.io/lancedb/python/python/), +[TypeScript](https://lancedb.github.io/lancedb/js/), or +[Rust](https://docs.rs/lancedb/latest/lancedb/index.html) API reference. - DDL by object, then DML, then DQL -- the detailed technical documentation - that defines the behavior of every operation and SDK API. + Statement syntax, expressions, and data types for LanceDB SQL. -## What to read next {#next} +## Next steps {#next} - - The language clients, and what stays true across all of them. + + Read results, track long-running queries, and cancel work. - - Give a coding agent a current reference to this same API surface. + + Compare language clients and find their API references. diff --git a/docs/web/docs.json b/docs/web/docs.json index 4c002b0a5..38c2bc6cd 100644 --- a/docs/web/docs.json +++ b/docs/web/docs.json @@ -163,10 +163,6 @@ "source": "/storage/monitoring", "destination": "/deployment/monitoring" }, - { - "source": "/enterprise/deployment", - "destination": "/enterprise/deployment/index" - }, { "source": "/search/sql", "destination": "/sql" @@ -253,6 +249,8 @@ "group": "Getting Started", "pages": [ "index", + "basics/offerings", + "basics/connect", "quickstart", "architecture", "basics/sdk", @@ -270,7 +268,8 @@ "basics/jobs", "tables/consistency", "basics/guarantees", - "basics/limits" + "basics/limits", + "basics/naming" ] }, { diff --git a/docs/web/quickstart.mdx b/docs/web/quickstart.mdx index 36001c97e..788dd6585 100644 --- a/docs/web/quickstart.mdx +++ b/docs/web/quickstart.mdx @@ -1,96 +1,55 @@ --- title: Quickstart sidebarTitle: "Quickstart" -description: "Get started with LanceDB in minutes." +description: "Install LanceDB, create a table, and run your first vector search." icon: rocket --- -import { - PyConnect, - PyConnectAsync, - PyConnectEnterpriseQuickstart, - PyConnectObjectStorage, - PyConnectObjectStorageAsync, - RsConnect, - RsConnectEnterpriseQuickstart, - RsConnectObjectStorage, - TsConnect, - TsConnectEnterpriseQuickstart, - TsConnectObjectStorage, -} from '/snippets/connection.mdx'; -import { - PyQuickstartData, - PyQuickstartComputedFeature, - PyQuickstartTraining, - PyQuickstartTrainingProjection, - PyQuickstartCreateTable, - PyQuickstartCreateTableAsync, - PyQuickstartAddFeature, - PyQuickstartCurateWithMetadata, - PyQuickstartMultimodalBytes, - PyQuickstartQueryFeature, - PyQuickstartVectorSearch1, - PyQuickstartVectorSearch1Async, - PyQuickstartOutputPandas, - RsQuickstartAddFeature, - RsQuickstartCurateWithMetadata, - RsQuickstartCreateTable, - RsQuickstartData, - RsQuickstartDefineStruct, - RsQuickstartMultimodalBytes, - RsQuickstartQueryFeature, - RsQuickstartVectorSearch1, - TsQuickstartAddFeature, - TsQuickstartCurateWithMetadata, - TsQuickstartCreateTable, - TsQuickstartData, - TsQuickstartMultimodalBytes, - TsQuickstartQueryFeature, - TsQuickstartVectorSearch1, -} from '/snippets/quickstart.mdx'; +import { PyConnect, TsConnect, RsConnect, PyConnectEnterprisePlaceholders, TsConnectEnterprisePlaceholders, RsConnectEnterprisePlaceholders } from '/snippets/connection.mdx'; +import { PyQuickstartData, PyQuickstartCreateTable, PyQuickstartAddData, PyQuickstartVectorSearch1, TsQuickstartData, TsQuickstartCreateTable, TsQuickstartAddData, TsQuickstartVectorSearch1, RsQuickstartDefineStruct, RsQuickstartData, RsQuickstartCreateTable, RsQuickstartAddData, RsQuickstartVectorSearch1 } from '/snippets/quickstart.mdx'; -As described in [the landing page](/), LanceDB provides one data layer for -curation, feature engineering, search and retrieval, and model training. Whether you are preparing -training data, building a RAG or agentic retrieval system, reviewing examples, or adding model-generated -features, you'll work with the same underlying table and search primitives. +This quickstart uses a local database and works without an account or API key. -Let's get started in just a few steps! +Follow the steps below, or jump to the [complete examples](#code) to copy a runnable program. ## 1\. Install LanceDB {#1-install-lancedb} -Install LanceDB in your client SDK. +Choose your language. Python uses Polars to display the search results. TypeScript needs Node.js 22.18 or later to run the `.mts` example directly. -```bash pip icon="terminal" -pip install lancedb +```bash Python (pip) icon="python" +pip install lancedb polars ``` -```bash uv icon="terminal" -uv add lancedb - -# Or, in an existing virtual environment: -uv pip install lancedb +```bash Python (uv) icon="python" +uv venv +uv pip install lancedb polars ``` -```bash TypeScript icon=js +```bash TypeScript icon="square-js" npm install @lancedb/lancedb ``` -```bash Rust icon=Rust -cargo add lancedb +```bash Rust icon="rust" +cargo init --name lancedb_quickstart +cargo add lancedb@0.31.0 --features polars,remote +cargo add tokio --features macros,rt-multi-thread ``` +Run the Rust commands in a new, empty directory. The `polars` feature provides DataFrame output; `remote` also lets this example connect to Enterprise. + + ### Python pre-release builds {#python-pre-release-builds} To pick up the latest features and bug fixes before the next stable release, install a pre-release from LanceDB's Fury index. -```bash pip icon="terminal" +```bash Python (pip) icon="terminal" pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb ``` -```bash uv icon="terminal" +```bash Python (uv) icon="terminal" uv venv uv pip install --prerelease allow --index https://pypi.fury.io/lancedb/ lancedb @@ -105,436 +64,237 @@ for more than 6 months after release. For real-world workloads, we recommend you as far as possible. - +### Older x86_64 CPUs {#older-x86-cpus} The default `lancedb` wheel targets `x86-64-haswell` (AVX2 + FMA + F16C). On older x86_64 CPUs without AVX2 (Intel Sandy Bridge, Ivy Bridge, Westmere; AMD Bulldozer, Piledriver, Steamroller), `import lancedb` crashes with `Illegal instruction`. Install `lancedb-compat` instead. It exposes the same API (`import lancedb` still works), is built at the -`x86-64-v2` baseline, and uses runtime SIMD dispatch to still leverage AVX2, FMA, or AVX-512 when available: +`x86-64-v2` baseline, and uses runtime SIMD dispatch to still use AVX2, FMA, or AVX-512 when available: -```bash pip icon="terminal" +```bash Python (pip) icon="terminal" pip install lancedb-compat ``` The two packages share the same `lancedb/` namespace and conflict at install time, so pick one. To switch, uninstall the other first (`pip uninstall lancedb && pip install lancedb-compat`). + + -## 2\. Connect to a LanceDB database {#2-connect-to-a-lancedb-database} +## 2\. Connect to a database {#2-connect-to-a-lancedb-database} -LanceDB supports several URI patterns to connect to a database. +### Use a local directory {#connect-via-local-directory-path} -- A local filesystem path (when using it as an embedded library) -- A `db://...` URI (when using LanceDB Enterprise) -- An object storage URI: `s3://...`, `gs://...`, or `az://...` (when connecting directly from the client SDK) +LanceDB creates an `ex_lancedb` directory to store your tables. No server is needed. -### Connect via local directory path {#connect-via-local-directory-path} - -The simplest way to begin is to use LanceDB as an embedded library. Import LanceDB in your -client SDK of choice and point to a local directory path. - - - + + {PyConnect} - - - {PyConnectAsync} - - + {"import * as lancedb from \"@lancedb/lancedb\";\n\n"} {TsConnect} - - { "use lancedb::connect;\n" } - { "\n" } + {"use lancedb::connect;\n\n"} {RsConnect} - + +The [complete examples](#code) include the imports and entry points for each language. For Python's async API, see [Connecting to a database](/basics/connect). -### Connect via object storage URIs {#connect-via-object-storage-uris} - -You can also connect directly to object storage from the client SDK: - - - - {PyConnectObjectStorage} - - - - {PyConnectObjectStorageAsync} - - - - {TsConnectObjectStorage} - - - - {RsConnectObjectStorage} - - - -For credentials, endpoints, and provider-specific options, see -[Configuring storage](/deployment/storage). - + ### Connect to LanceDB Enterprise {#connect-to-lancedb-enterprise} -If you're using LanceDB Enterprise, you can connect to the remote database using the -`db://` URI along with the API key, region, and cluster endpoint you received from the -LanceDB team. Pass the cluster endpoint via `host_override` so the client routes -requests to your deployment. +Use the database URI, API key, and endpoint provided by your administrator or the LanceDB team. Replace the placeholders below with those details. Use the deployment's region if provided; otherwise keep `us-east-1`. - + - { "import lancedb\n\n" } - {PyConnectEnterpriseQuickstart} + {PyConnectEnterprisePlaceholders} - - { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } - {TsConnectEnterpriseQuickstart} + {"import * as lancedb from \"@lancedb/lancedb\";\n\n"} + {TsConnectEnterprisePlaceholders} - - { "use lancedb::connect;\n\n" } - {RsConnectEnterpriseQuickstart} + {"use lancedb::connect;\n\n"} + {RsConnectEnterprisePlaceholders} - + -`host_override` is the full URL of your cluster endpoint, including the scheme -(`https://`) and a port if your deployment listens on a non-default one -(e.g. `https://your-enterprise-endpoint.com:443`). If you don't have the -endpoint, [contact the LanceDB team](https://www.lancedb.com/contact). +In application code, load credentials from environment variables or a secrets manager. See [connection settings](/basics/connect#enterprise) for an example. -To learn more about `RemoteTable` semantics and how Enterprise differs operationally from -embedded LanceDB, see the [Enterprise overview](/enterprise). - -## 3\. Create a new table {#3-create-a-new-table} - -Let's create a small table of characters from the kingdom of Camelot. Each row stores source text, -metadata, structured fields, and a vector embedding in the same LanceDB table. - - -The embeddings we use in this example are synthetic and for demonstration purposes only. In a real AI -data workflow, you would generate them from text, images, audio, or video using an embedding model of choice. - - -Each row has source text, metadata, structured fields, and a vector: - -```json -{ - "id": "2", - "name": "Merlin", - "role": "Wizard", - "description": "Advisor and prophet with deep magical knowledge.", - "stats": { "strength": 2, "magic": 5, "leadership": 4, "wisdom": 5 }, - "vector": [0.2, 0.9, 0.4, 0.9] -} -``` - -The full raw records are included below: - - - - - {PyQuickstartData} - - - - {TsQuickstartData} - - - - {RsQuickstartDefineStruct} - {RsQuickstartData} - - +Need a deployment? [Contact the LanceDB team](https://www.lancedb.com/contact). For differences between local and server execution, see [Moving to LanceDB Enterprise](/basics/sdk#moving-to-enterprise). -You can now create a LanceDB table from those records. The code below creates a LanceDB table -with the appropriate schema and ingests the data. + +### Connect via object storage URIs {#connect-via-object-storage-uris} - - +Use an `s3://`, `gs://`, or `az://` URI in place of the local path. See [Configuring storage](/deployment/storage) for credentials and provider-specific options. + + +## 3\. Create a table and add data {#3-create-a-new-table} + +Each row describes a character and stores a vector alongside their name, description, and stats. + + +These vectors are made up for this example. In your application, use an embedding model to generate vectors from your data. + + + + + + {PyQuickstartData} + + + {TsQuickstartData} + + + {RsQuickstartDefineStruct} + {RsQuickstartData} + + + + +Create a `characters` table from those records: + + + {PyQuickstartCreateTable} - - - {PyQuickstartCreateTableAsync} - - {TsQuickstartCreateTable} - {RsQuickstartCreateTable} - + -## 4\. Semantic search {#4-semantic-search} +This example uses overwrite mode so you can run it again. It replaces any existing `characters` table in the database you connect to. -Search is a useful capability for all kinds of AI data pipelines. Below, we do a vector similarity -search for samples similar to a "_wise magical advisor_" (transforming the natural language query to -an embedding), and project only the columns needed by the next step. +Add one more character: -Search (which requires random access) is a ubiquitous access pattern that appears in many workloads: -whether you're building a RAG or recommendation system, serving agent memory, or curating a training -dataset. + + + {PyQuickstartAddData} + + + {TsQuickstartAddData} + + + {RsQuickstartAddData} + + - - +## 4\. Search the table {#4-semantic-search} + +Find the two vectors closest to the query vector. This example uses a fixed vector; it does not call an embedding model. + + + {PyQuickstartVectorSearch1} - - - {PyQuickstartVectorSearch1Async} - - {TsQuickstartVectorSearch1} - {RsQuickstartVectorSearch1} - + -The example for Python above shows how to convert results to a Polars DataFrame. -Depending on your language, you can collect query results as a list/array of objects or DataFrames -to be used downstream in your application. +Merlin and Morgana come back first. Python and Rust display a Polars DataFrame; TypeScript displays an array of results. The `_distance` column measures how far each result is from your query vector, with smaller values indicating a closer match. - - Use the `to_pandas()` method to convert query results into a Pandas DataFrame. - - {PyQuickstartOutputPandas} - +## Complete examples {#code} + +Each example below creates the local database, adds four characters, and prints the two closest matches. Use the dependencies from step 1. + + +Save this as `quickstart.py` and run `python quickstart.py`. If you installed with uv, run `uv run quickstart.py`. + + + + {PyConnect} + {PyQuickstartData} + {PyQuickstartCreateTable} + {PyQuickstartAddData} + {PyQuickstartVectorSearch1} + + -## 5\. Curation {#5-curation} + +Save this as `quickstart.mts` and run `node quickstart.mts` from the directory where you installed LanceDB. -Searching for relevant results can be more useful when combined with metadata filters. -In this tiny example, we filter to examples with high `magic` stats. - - - - {PyQuickstartCurateWithMetadata} + + + {"import * as lancedb from \"@lancedb/lancedb\";\n\n"} + {TsConnect} + {TsQuickstartData} + {TsQuickstartCreateTable} + {TsQuickstartAddData} + {TsQuickstartVectorSearch1} + + - - {TsQuickstartCurateWithMetadata} + +Replace `src/main.rs` with this program, then run `cargo run` from your project directory. The schema and Arrow conversion helpers are included. + + + + {"use lancedb::connect;\n\n"} + {RsQuickstartDefineStruct} + {"\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n"} + {RsConnect} + {RsQuickstartData} + {RsQuickstartCreateTable} + {RsQuickstartAddData} + {RsQuickstartVectorSearch1} + {"\nOk(())\n}\n"} - - - {RsQuickstartCurateWithMetadata} - - - -When working with large datasets, it's common to use the same pattern to filter on quality labels, -train/eval splits, numeric fields, categorical values, timestamp windows, or generated tags and labels. - -## 6\. Add a derived feature {#6-add-a-derived-feature} - -Feature engineering is the process of cleaning up your data and creating new signals that -help your model learn, make better predictions, or your agent retrieve more useful information. -In the example below, we add a `power_score` column from the structured `stats` fields. -Lance supports data evolution, so you can add new columns without rewriting the entire table. - - - - {PyQuickstartAddFeature} - - - - {TsQuickstartAddFeature} - - - - { "use lancedb::table::NewColumnTransform;\n\n" } - {RsQuickstartAddFeature} - - - -Next, you can query a compact view of the new feature: - - - - {PyQuickstartQueryFeature} - - - - {TsQuickstartQueryFeature} - - - - {RsQuickstartQueryFeature} - - - -| name | role | power_score | -| --- | --- | --- | -| King Arthur | King | 3.5 | -| Merlin | Wizard | 4.0 | -| Sir Lancelot | Knight | 3.0 | - -The same workflow is used for data preparation tasks when adding derived features, cached model signals, review scores, or dataset -quality indicators. - -### Declare now, compute later {#computed-columns} - -The call above computes every row as part of the call, which is fine for three -rows. On a table large enough that you would rather choose *when* the work -happens, declare a column instead and fill it separately. The stored `power_score` -column stays available; declare a separate -`combat_score` column from magic and strength, then refresh it: - - -{PyQuickstartComputedFeature} - - -A refresh gives values to rows that have none and recomputes only rows whose -inputs changed, so appending more characters and refreshing again fills just -the new ones -- it does not recompute the rows that are already current. That -is what makes a backfill safe to retry and cheap to repeat as data arrives. - -On a deployment the same refresh runs server-side, on a cluster, as a -[job](/basics/jobs) you can walk away from: - -```python -job = table.refresh_column_async("combat_score") -job.wait() -``` - -See [Computed columns](/feature-engineering/add-computed-column) for expressions, -and [Functions](/management/function) for filling a column with Python --- an embedding model, a captioner, anything you can register. - -## 7\. Store multimodal data {#7-store-multimodal-data} - -Multimodal data is a first-class citizen in LanceDB. Binary data (image, audio, video, etc.) is -stored as blobs or inline Arrow binary types in a LanceDB column, and they benefit from the same -table operations and data versioning semantics as other data types. All the data is governed -in the same table, so you can search, filter, and retrieve multimodal records together with structured -fields, metadata, and embeddings. - -In this example, the -[`lancedb/magical_kingdom`](https://huggingface.co/datasets/lancedb/magical_kingdom) dataset stores -character images, descriptions, structured stats, image embeddings, and text embeddings together. - -Say we downloaded the image for Sir Lancelot from that dataset locally. You can read the image bytes -in your client SDK and store them in a LanceDB column. The image bytes can be used for downstream tasks -like retrieval, evaluation, or training. - -
- Sir Lancelot from the lancedb/magical_kingdom dataset -
- -These snippets load the local image file and store the bytes in an `image` column: - - - - {PyQuickstartMultimodalBytes} - - - - {TsQuickstartMultimodalBytes} - - - - {RsQuickstartMultimodalBytes} - - - -For more examples, see [Data types](/management/table#multimodal). - -## 8\. Train on the result {#8-train} - -The point of the previous steps is this one. The table you searched, filtered, -and added features to is directly usable as a PyTorch -[Dataset](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.Dataset) --- there is no export step, and no second copy of the data to keep in sync. - -Install PyTorch with `pip install torch`. Select the numeric feature columns -before constructing the loader; the full table also contains strings, -structured stats, and vectors. - - -{PyQuickstartTraining} - - -```python -for batch in dataloader: - ... # train with batch["power_score"] and batch["combat_score"] -``` - -Because the loader reads from the table, the feature columns you just -materialized are read straight off storage rather than recomputed every epoch. -That is the whole argument for doing feature engineering in the lakehouse: an -expensive transform runs once, is versioned with the data, and every subsequent -epoch is a read. - -Projection matters here. If a training step only needs `power_score`, select -that column so the descriptions, stats, and vectors are excluded: - - -{PyQuickstartTrainingProjection} - - -See [Training](/training/data-loading) for shuffling, multi-worker loading, and reading from -a remote table. - -## Code {#code} - -See the full code for these examples (including helper functions) in the -`quickstart` file for the appropriate client language in the -[files provided in the repo](https://github.com/lancedb/docs/tree/main/tests). - + + ## Next steps {#next-steps} -You've gone from a table, through search and curation, to derived features and a -training loop -- in one table, with no export step between any two stages. That -arc is the argument for a multimodal lakehouse, and every stage of it scales -past this toy example without changing shape. -Continue to the guides to build on it with schema options, versioning, indexing, -full-text and hybrid search, reranking, and server-side feature engineering. +You've created a table, added rows, and searched it. Continue with the guides below. + +### Filter results {#5-curation} + +Combine vector search with metadata filters, such as `stats.magic >= 4`. See [SQL filtering](/search/filtering). + +### Extend your table {#extensions} + +#### Add a derived feature {#6-add-a-derived-feature} + +Add columns calculated from existing fields. See [Adding columns](/management/table#schema-add-new-columns). + +#### Computed columns {#computed-columns} + +Declare a column and choose when to fill or refresh it. See [Computed columns](/feature-engineering/add-computed-column). + +#### Store multimodal data {#7-store-multimodal-data} + +Store images, audio, and other binary data alongside vectors and metadata. See [Multimodal data types](/management/table#multimodal). + +#### Train on your data {#8-train} + +Read table data into PyTorch for training. See [Data loading](/training/data-loading). - - Build on this quickstart with table creation, updates, and schema tips. + +Create, update, and manage your tables. - - Learn how to build Retrieval-Augmented Generation (RAG) applications using LanceDB. + +Use retrieved context in a language model application. - - Create vector, full-text, and scalar indexes to speed up queries on larger datasets. + +Speed up queries on larger datasets. - - Use LanceDB for projected, shuffled, random-access reads in training workflows. + +Generate vectors from your own data. diff --git a/docs/web/snippets/connection.mdx b/docs/web/snippets/connection.mdx index c7d8a187d..c145c1fb6 100644 --- a/docs/web/snippets/connection.mdx +++ b/docs/web/snippets/connection.mdx @@ -4,31 +4,37 @@ export const PyConnect = "import lancedb\n\nuri = \"ex_lancedb\"\ndb = lancedb.c export const PyConnectAsync = "import lancedb\n\nuri = \"ex_lancedb\"\nasync_db = await lancedb.connect_async(uri)\n"; -export const PyConnectEnterpriseQuickstart = "uri = \"db://your-database-uri\"\napi_key = \"your-api-key\"\nregion = \"us-east-1\"\nhost_override = \"https://your-enterprise-endpoint.com\"\n\ndb = lancedb.connect(\n uri=uri,\n api_key=api_key,\n region=region,\n host_override=host_override,\n)\n"; +export const PyConnectEnterprisePlaceholders = "import lancedb\n\ndb = lancedb.connect(\n uri=\"db://your-database\",\n api_key=\"your-api-key\",\n region=\"us-east-1\",\n host_override=\"https://your-endpoint\",\n)\n"; + +export const PyConnectEnterpriseQuickstart = "import os\n\nimport lancedb\n\ndb = lancedb.connect(\n uri=os.environ[\"LANCEDB_URI\"], # db://your-database\n api_key=os.environ[\"LANCEDB_API_KEY\"],\n region=os.environ.get(\"LANCEDB_REGION\", \"us-east-1\"),\n host_override=os.environ[\"LANCEDB_HOST_OVERRIDE\"], # https://your-endpoint\n)\n"; export const PyConnectObjectStorage = "import lancedb\n\nuri = \"s3://your-bucket/path\"\n# You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\ndb = lancedb.connect(uri)\n"; export const PyConnectObjectStorageAsync = "import lancedb\n\nuri = \"s3://your-bucket/path\"\n# You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\nasync_db = await lancedb.connect_async(uri)\n"; -export const PyNamespaceAdminOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\nnamespace = [\"prod\", \"search\"]\n\ndb.create_namespace([\"prod\"])\ndb.create_namespace([\"prod\", \"search\"])\n\nchild_namespaces = db.list_namespaces(namespace_path=[\"prod\"]).namespaces\nprint(f\"Child namespaces under {namespace}: {child_namespaces}\")\n# Child namespaces under ['prod', 'search']: ['search']\n\nmetadata = db.describe_namespace([\"prod\", \"search\"])\nprint(f\"Metadata for namespace {namespace}: {metadata}\")\n# Metadata for namespace ['prod', 'search']: properties=None\n\ndb.drop_namespace([\"prod\", \"search\"], mode=\"skip\")\ndb.drop_namespace([\"prod\"], mode=\"skip\")\n"; +export const PyNamespaceAdminOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\nnamespace = [\"prod\", \"search\"]\n\ndb.create_namespace([\"prod\"])\ndb.create_namespace([\"prod\", \"search\"])\n\nchild_namespaces = db.list_namespaces(namespace_path=[\"prod\"]).namespaces\nprint(f\"Child namespaces under ['prod']: {child_namespaces}\")\n# Child namespaces under ['prod']: ['search']\n\nmetadata = db.describe_namespace([\"prod\", \"search\"])\nprint(f\"Metadata for namespace {namespace}: {metadata}\")\n# Metadata for namespace ['prod', 'search']: properties=None\n\ndb.drop_namespace([\"prod\", \"search\"], mode=\"skip\")\ndb.drop_namespace([\"prod\"], mode=\"skip\")\n"; -export const PyNamespaceTableOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\n\n# Create namespace tree: prod/search\ndb.create_namespace([\"prod\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"search\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"recommendations\"], mode=\"exist_ok\")\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 1, \"vector\": [0.1, 0.2], \"name\": \"alice\"}],\n namespace_path=[\"prod\", \"search\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 2, \"vector\": [0.3, 0.4], \"name\": \"bob\"}],\n namespace_path=[\"prod\", \"recommendations\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\n# Verify\nprint(db.list_namespaces()) # ['prod']\nprint(db.list_namespaces(namespace_path=[\"prod\"])) # ['recommendations', 'search']\nprint(db.list_tables(namespace_path=[\"prod\", \"search\"])) # ['user']\nprint(db.list_tables(namespace_path=[\"prod\", \"recommendations\"])) # ['user']\n"; +export const PyNamespaceTableOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\n\n# Create namespace tree: prod/search\ndb.create_namespace([\"prod\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"search\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"recommendations\"], mode=\"exist_ok\")\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 1, \"vector\": [0.1, 0.2], \"name\": \"alice\"}],\n namespace_path=[\"prod\", \"search\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 2, \"vector\": [0.3, 0.4], \"name\": \"bob\"}],\n namespace_path=[\"prod\", \"recommendations\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\n# Verify: listings return a response with the names and a page token\nprint(db.list_namespaces().namespaces) # ['prod']\nprint(db.list_namespaces(namespace_path=[\"prod\"]).namespaces)\n# ['recommendations', 'search']\nprint(db.list_tables(namespace_path=[\"prod\", \"search\"]).tables) # ['user']\nprint(db.list_tables(namespace_path=[\"prod\", \"recommendations\"]).tables) # ['user']\n"; -export const TsConnect = "import * as lancedb from \"@lancedb/lancedb\";\n\nasync function connectExample(uri: string) {\n const db = await lancedb.connect(uri);\n return db;\n}\n"; +export const TsConnect = "const db = await lancedb.connect(\"ex_lancedb\");\n"; -export const TsConnectEnterpriseQuickstart = "const uri = \"db://your-database-uri\";\nconst apiKey = \"your-api-key\";\nconst region = \"us-east-1\";\nconst hostOverride = \"https://your-enterprise-endpoint.com\";\n\nconst db = await lancedb.connect(uri, {\n apiKey,\n region,\n hostOverride,\n});\n"; +export const TsConnectEnterprisePlaceholders = "const db = await lancedb.connect(\"db://your-database\", {\n apiKey: \"your-api-key\",\n region: \"us-east-1\",\n hostOverride: \"https://your-endpoint\",\n});\n"; -export const TsConnectObjectStorage = "async function connectObjectStorageExample() {\n const uri = \"s3://your-bucket/path\";\n // You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\n const db = await lancedb.connect(uri);\n return db;\n}\n"; +export const TsConnectEnterpriseQuickstart = "const setting = (name: string) => {\n const value = process.env[name];\n if (!value) throw new Error(`Set ${name}`);\n return value;\n};\n\nconst db = await lancedb.connect(setting(\"LANCEDB_URI\"), {\n apiKey: setting(\"LANCEDB_API_KEY\"),\n region: process.env.LANCEDB_REGION ?? \"us-east-1\",\n hostOverride: setting(\"LANCEDB_HOST_OVERRIDE\"),\n});\n"; + +export const TsConnectObjectStorage = "const uri = \"s3://your-bucket/path\";\n// You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\nconst db = await lancedb.connect(uri);\n"; export const TsNamespaceAdminOps = "const db = await lancedb.connectNamespace(\"dir\", { root: \"./local_lancedb\" });\nconst namespace = [\"prod\", \"search\"];\n\nawait db.createNamespace([\"prod\"]);\nawait db.createNamespace([\"prod\", \"search\"]);\n\nconst childNamespaces = (await db.listNamespaces([\"prod\"])).namespaces;\nconsole.log(`Child namespaces under ${JSON.stringify(namespace)}:`, childNamespaces);\n// Child namespaces under [\"prod\",\"search\"]: [ 'search' ]\n\nconst metadata = await db.describeNamespace([\"prod\", \"search\"]);\nconsole.log(`Metadata for namespace ${JSON.stringify(namespace)}:`, metadata);\n\nawait db.dropNamespace([\"prod\", \"search\"], { mode: \"skip\" });\nawait db.dropNamespace([\"prod\"], { mode: \"skip\" });\n"; export const TsNamespaceTableOps = "const db = await lancedb.connectNamespace(\"dir\", { root: \"./local_lancedb\" });\n\n// Create namespace tree: prod/search and prod/recommendations\nawait db.createNamespace([\"prod\"], { mode: \"exist_ok\" });\nawait db.createNamespace([\"prod\", \"search\"], { mode: \"exist_ok\" });\nawait db.createNamespace([\"prod\", \"recommendations\"], { mode: \"exist_ok\" });\n\nawait db.createTable(\n \"user\",\n [{ id: 1, vector: [0.1, 0.2], name: \"alice\" }],\n [\"prod\", \"search\"],\n { mode: \"create\" }, // use \"overwrite\" only if you want to replace existing table\n);\n\nawait db.createTable(\n \"user\",\n [{ id: 2, vector: [0.3, 0.4], name: \"bob\" }],\n [\"prod\", \"recommendations\"],\n { mode: \"create\" },\n);\n\n// Verify\nconsole.log((await db.listNamespaces()).namespaces); // [\"prod\"]\nconsole.log((await db.listNamespaces([\"prod\"])).namespaces); // [\"recommendations\", \"search\"]\nconsole.log(await db.tableNames([\"prod\", \"search\"])); // [\"user\"]\nconsole.log(await db.tableNames([\"prod\", \"recommendations\"])); // [\"user\"]\n"; -export const RsConnect = "async fn connect_example(uri: &str) {\n let db = connect(uri).execute().await.unwrap();\n let _ = db;\n}\n"; +export const RsConnect = "let db = connect(\"ex_lancedb\").execute().await?;\n"; -export const RsConnectEnterpriseQuickstart = "let uri = \"db://your-database-uri\";\nlet api_key = \"your-api-key\";\nlet region = \"us-east-1\";\nlet host_override = \"https://your-enterprise-endpoint.com\";\n"; +export const RsConnectEnterprisePlaceholders = "let db = connect(\"db://your-database\")\n .api_key(\"your-api-key\")\n .region(\"us-east-1\")\n .host_override(\"https://your-endpoint\")\n .execute()\n .await?;\n"; -export const RsConnectObjectStorage = "let uri = \"s3://your-bucket/path\";\n// You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\n"; +export const RsConnectEnterpriseQuickstart = "let uri = std::env::var(\"LANCEDB_URI\")?; // db://your-database\nlet api_key = std::env::var(\"LANCEDB_API_KEY\")?;\nlet region = std::env::var(\"LANCEDB_REGION\").unwrap_or_else(|_| \"us-east-1\".to_string());\nlet host_override = std::env::var(\"LANCEDB_HOST_OVERRIDE\")?; // https://your-endpoint\n\nlet db = connect(&uri)\n .api_key(&api_key)\n .region(®ion)\n .host_override(&host_override)\n .execute()\n .await?;\n"; + +export const RsConnectObjectStorage = "let uri = \"s3://your-bucket/path\";\n// You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\nlet db = connect(uri).execute().await?;\n"; export const RsNamespaceAdminOps = "let mut properties = std::collections::HashMap::new();\nproperties.insert(\"root\".to_string(), \"./local_lancedb\".to_string());\nlet db = lancedb::connect_namespace(\"dir\", properties).execute().await?;\nlet namespace = vec![\"prod\".to_string(), \"search\".to_string()];\n\ndb.create_namespace(lance_namespace::models::CreateNamespaceRequest {\n id: Some(vec![\"prod\".to_string()]),\n ..Default::default()\n})\n.await?;\ndb.create_namespace(lance_namespace::models::CreateNamespaceRequest {\n id: Some(namespace.clone()),\n ..Default::default()\n})\n.await?;\n\nlet child_namespaces = db\n .list_namespaces(lance_namespace::models::ListNamespacesRequest {\n id: Some(vec![\"prod\".to_string()]),\n ..Default::default()\n })\n .await?;\nprintln!(\n \"Child namespaces under {:?}: {:?}\",\n namespace, child_namespaces\n);\n// Child namespaces under [\"prod\", \"search\"]: [\"search\"]\n\ndb.drop_namespace(lance_namespace::models::DropNamespaceRequest {\n id: Some(namespace.clone()),\n ..Default::default()\n})\n.await?;\ndb.drop_namespace(lance_namespace::models::DropNamespaceRequest {\n id: Some(vec![\"prod\".to_string()]),\n ..Default::default()\n})\n.await?;\n"; diff --git a/docs/web/snippets/quickstart.mdx b/docs/web/snippets/quickstart.mdx index 7a4622979..c7cf98e32 100644 --- a/docs/web/snippets/quickstart.mdx +++ b/docs/web/snippets/quickstart.mdx @@ -1,82 +1,66 @@ {/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} -export const PyQuickstartAddData = "more_data = [\n {\n \"id\": \"4\",\n \"name\": \"Morgana\",\n \"role\": \"Sorceress\",\n \"description\": \"Powerful sorceress of Avalon.\",\n \"stats\": {\"strength\": 2, \"magic\": 5, \"leadership\": 4, \"wisdom\": 4},\n \"vector\": [0.3, 0.9, 0.6, 0.8],\n \"power_score\": 3.75,\n },\n]\n\n# Add data to table\ntable.add(more_data)\n"; +export const PyQuickstartAddData = "table.add(\n [\n {\n \"id\": \"4\",\n \"name\": \"Morgana\",\n \"role\": \"Sorceress\",\n \"description\": \"Powerful sorceress of Avalon.\",\n \"stats\": {\"strength\": 2, \"magic\": 5, \"leadership\": 4, \"wisdom\": 4},\n \"vector\": [0.3, 0.9, 0.6, 0.8],\n }\n ]\n)\n"; export const PyQuickstartAddFeature = "table.add_columns(\n {\n \"power_score\": \"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)\"\n }\n)\n"; -export const PyQuickstartComputedFeature = "table.add_columns(computed={\"combat_score\": \"(stats.magic + stats.strength) / 2.0\"})\ntable.refresh_column(\"combat_score\")\n"; +export const PyQuickstartComputedFeature = "table.add_columns(computed={\"description_length\": \"length(description)\"})\ntable.refresh_column(\"description_length\")\n"; + +export const PyQuickstartComputedFeatureEnterprise = "table.add_columns(computed={\"description_length\": \"length(description)\"})\njob = table.refresh_column_async(\"description_length\")\njob.wait()\n"; export const PyQuickstartCreateTable = "table = db.create_table(\"characters\", data=data, mode=\"overwrite\")\n"; export const PyQuickstartCreateTableAsync = "async_table = await async_db.create_table(\n \"characters\",\n data=data,\n mode=\"overwrite\",\n)\n"; -export const PyQuickstartCreateTableNoOverwrite = "table = db.create_table(\"characters\", data=data)\n"; - export const PyQuickstartCurateWithMetadata = "curated = (\n table.search(query_vector)\n .where(\"stats.magic >= 4\")\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .to_polars()\n)\nprint(curated)\n"; export const PyQuickstartData = "data = [\n {\n \"id\": \"1\",\n \"name\": \"King Arthur\",\n \"role\": \"King\",\n \"description\": \"Leader of Camelot and wielder of Excalibur.\",\n \"stats\": {\"strength\": 4, \"magic\": 1, \"leadership\": 5, \"wisdom\": 4},\n \"vector\": [0.7, 0.1, 0.9, 0.7],\n },\n {\n \"id\": \"2\",\n \"name\": \"Merlin\",\n \"role\": \"Wizard\",\n \"description\": \"Advisor and prophet with deep magical knowledge.\",\n \"stats\": {\"strength\": 2, \"magic\": 5, \"leadership\": 4, \"wisdom\": 5},\n \"vector\": [0.2, 0.9, 0.4, 0.9],\n },\n {\n \"id\": \"3\",\n \"name\": \"Sir Lancelot\",\n \"role\": \"Knight\",\n \"description\": \"Legendary knight known for courage and combat skill.\",\n \"stats\": {\"strength\": 5, \"magic\": 1, \"leadership\": 3, \"wisdom\": 3},\n \"vector\": [0.9, 0.1, 0.5, 0.4],\n },\n]\n"; -export const PyQuickstartDataAsync = "data = [\n {\n \"id\": \"1\",\n \"name\": \"King Arthur\",\n \"role\": \"King\",\n \"description\": \"Leader of Camelot and wielder of Excalibur.\",\n \"stats\": {\"strength\": 4, \"magic\": 1, \"leadership\": 5, \"wisdom\": 4},\n \"vector\": [0.7, 0.1, 0.9, 0.7],\n },\n {\n \"id\": \"2\",\n \"name\": \"Merlin\",\n \"role\": \"Wizard\",\n \"description\": \"Advisor and prophet with deep magical knowledge.\",\n \"stats\": {\"strength\": 2, \"magic\": 5, \"leadership\": 4, \"wisdom\": 5},\n \"vector\": [0.2, 0.9, 0.4, 0.9],\n },\n {\n \"id\": \"3\",\n \"name\": \"Sir Lancelot\",\n \"role\": \"Knight\",\n \"description\": \"Legendary knight known for courage and combat skill.\",\n \"stats\": {\"strength\": 5, \"magic\": 1, \"leadership\": 3, \"wisdom\": 3},\n \"vector\": [0.9, 0.1, 0.5, 0.4],\n },\n]\n"; - -export const PyQuickstartMultimodalBytes = "from pathlib import Path\n\nimage_path = Path(\"docs/web/static/assets/images/quickstart/sir-lancelot.jpg\")\nimage_bytes = image_path.read_bytes()\n\nmultimodal_table = db.create_table(\n \"character_images\",\n data=[\n {\n \"id\": \"lancelot\",\n \"description\": \"Portrait of Sir Lancelot\",\n \"image\": image_bytes,\n \"vector\": [0.9, 0.1, 0.5, 0.4],\n }\n ],\n mode=\"overwrite\",\n)\n"; - -export const PyQuickstartOpenTable = "table = db.open_table(\"characters\")\n"; +export const PyQuickstartMultimodalBytes = "from pathlib import Path\n\nimage_bytes = Path(\"data/sir-lancelot.jpg\").read_bytes()\n\nmultimodal_table = db.create_table(\n \"character_images\",\n data=[\n {\n \"id\": \"lancelot\",\n \"description\": \"Portrait of Sir Lancelot\",\n \"image\": image_bytes,\n \"vector\": [0.9, 0.1, 0.5, 0.4],\n }\n ],\n mode=\"overwrite\",\n)\n"; export const PyQuickstartOutputPandas = "# Ensure you run `pip install pandas` beforehand\nresult = table.search(query_vector).limit(2).to_pandas()\nprint(result)\n"; export const PyQuickstartQueryFeature = "features = table.search().select([\"name\", \"role\", \"power_score\"]).to_polars()\nprint(features)\n"; -export const PyQuickstartTraining = "import torch\nfrom lancedb.permutation import Permutation\n\ndataset = Permutation.identity(table).select_columns(\n [\"power_score\", \"combat_score\"]\n)\ndataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True)\n"; - -export const PyQuickstartTrainingProjection = "dataset = Permutation.identity(table).select_columns([\"power_score\"])\ndataloader = torch.utils.data.DataLoader(dataset, batch_size=1024)\n"; +export const PyQuickstartTraining = "import torch\nfrom lancedb.permutation import Permutation\n\ntable.add_columns(\n {\n \"magic\": \"cast(stats.magic as float)\",\n \"strength\": \"cast(stats.strength as float)\",\n }\n)\ndataset = Permutation.identity(table).select_columns([\"magic\", \"strength\"])\ndataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True)\n"; export const PyQuickstartVectorSearch1 = "# Search for examples similar to a \"wise magical advisor\"\nquery_vector = [0.2, 0.8, 0.4, 0.9]\n\n# Ensure you run `pip install polars` beforehand\nresult = (\n table.search(query_vector)\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .to_polars()\n)\nprint(result)\n"; export const PyQuickstartVectorSearch1Async = "# Search for examples similar to a \"wise magical advisor\"\nquery_vector = [0.2, 0.8, 0.4, 0.9]\n\n# Ensure you run `pip install polars` beforehand\nasync_result = await (\n await async_table.search(query_vector)\n).select([\"name\", \"role\", \"description\", \"_distance\"]).limit(2).to_polars()\nprint(async_result)\n"; -export const PyQuickstartVectorSearch2 = "# Search for examples similar to a \"powerful sorceress\"\nquery_vector = [0.3, 0.9, 0.6, 0.8]\n\nresults = table.search(query_vector).limit(2).to_polars()\nprint(results)\n"; - -export const TsQuickstartAddData = "const moreData = [\n {\n id: \"4\",\n name: \"Morgana\",\n role: \"Sorceress\",\n description: \"Powerful sorceress of Avalon.\",\n stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 },\n vector: [0.3, 0.9, 0.6, 0.8],\n power_score: 3.75,\n },\n];\n\n// Add data to table\nawait table.add(moreData);\n"; +export const TsQuickstartAddData = "await table.add([\n {\n id: \"4\",\n name: \"Morgana\",\n role: \"Sorceress\",\n description: \"Powerful sorceress of Avalon.\",\n stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 },\n vector: [0.3, 0.9, 0.6, 0.8],\n },\n]);\n"; export const TsQuickstartAddFeature = "await table.addColumns([\n {\n name: \"power_score\",\n valueSql:\n \"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)\",\n },\n]);\n"; -export const TsQuickstartCreateTable = "let table = await db.createTable(\"characters\", data, { mode: \"overwrite\" });\n"; - -export const TsQuickstartCreateTableNoOverwrite = "table = await db.createTable(\"characters\", data);\n"; +export const TsQuickstartCreateTable = "const table = await db.createTable(\"characters\", data, { mode: \"overwrite\" });\n"; export const TsQuickstartCurateWithMetadata = "const curated = await table\n .search(queryVector)\n .where(\"stats.magic >= 4\")\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .toArray();\nconsole.table(curated);\n"; export const TsQuickstartData = "const data = [\n {\n id: \"1\",\n name: \"King Arthur\",\n role: \"King\",\n description: \"Leader of Camelot and wielder of Excalibur.\",\n stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 },\n vector: [0.7, 0.1, 0.9, 0.7],\n },\n {\n id: \"2\",\n name: \"Merlin\",\n role: \"Wizard\",\n description: \"Advisor and prophet with deep magical knowledge.\",\n stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 },\n vector: [0.2, 0.9, 0.4, 0.9],\n },\n {\n id: \"3\",\n name: \"Sir Lancelot\",\n role: \"Knight\",\n description: \"Legendary knight known for courage and combat skill.\",\n stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 },\n vector: [0.9, 0.1, 0.5, 0.4],\n },\n];\n"; -export const TsQuickstartMultimodalBytes = "const arrow = await import(\"apache-arrow\");\nconst path = await import(\"node:path\");\nconst { readFile } = await import(\"node:fs/promises\");\n\nconst imagePath = path.join(\n __dirname,\n \"../../web/static/assets/images/quickstart/sir-lancelot.jpg\",\n);\nconst imageBytes = await readFile(imagePath);\nconst imageSchema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Utf8()),\n new arrow.Field(\"description\", new arrow.Utf8()),\n new arrow.Field(\"image\", new arrow.Binary()),\n new arrow.Field(\n \"vector\",\n new arrow.FixedSizeList(\n 4,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n]);\nconst imageData = lancedb.makeArrowTable(\n [\n {\n id: \"lancelot\",\n description: \"Portrait of Sir Lancelot\",\n image: imageBytes,\n vector: [0.9, 0.1, 0.5, 0.4],\n },\n ],\n { schema: imageSchema },\n);\nconst multimodalTable = await db.createTable(\n \"character_images\",\n imageData,\n { mode: \"overwrite\" },\n);\n"; - -export const TsQuickstartOpenTable = "table = await db.openTable(\"characters\");\n"; - -export const TsQuickstartOutputArray = "result = await table.search(queryVector).limit(2).toArray();\nconsole.table(result);\n"; +export const TsQuickstartMultimodalBytes = "const arrow = await import(\"apache-arrow\");\nconst { readFile } = await import(\"node:fs/promises\");\n\nconst imageBytes = await readFile(\"data/sir-lancelot.jpg\");\nconst imageSchema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Utf8()),\n new arrow.Field(\"description\", new arrow.Utf8()),\n new arrow.Field(\"image\", new arrow.Binary()),\n new arrow.Field(\n \"vector\",\n new arrow.FixedSizeList(\n 4,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n]);\nconst imageData = lancedb.makeArrowTable(\n [\n {\n id: \"lancelot\",\n description: \"Portrait of Sir Lancelot\",\n image: imageBytes,\n vector: [0.9, 0.1, 0.5, 0.4],\n },\n ],\n { schema: imageSchema },\n);\nconst multimodalTable = await db.createTable(\n \"character_images\",\n imageData,\n { mode: \"overwrite\" },\n);\n"; export const TsQuickstartQueryFeature = "const features = await table\n .query()\n .select([\"name\", \"role\", \"power_score\"])\n .toArray();\nconsole.table(features);\n"; -export const TsQuickstartVectorSearch1 = "// Search for examples similar to a \"wise magical advisor\"\nlet queryVector = [0.2, 0.8, 0.4, 0.9];\n\nlet result = await table\n .search(queryVector)\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .toArray();\nconsole.table(result);\n"; +export const TsQuickstartVectorSearch1 = "// Search for examples similar to a \"wise magical advisor\"\nconst queryVector = [0.2, 0.8, 0.4, 0.9];\n\nconst result = await table\n .search(queryVector)\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .toArray();\nconsole.table(result);\n"; -export const TsQuickstartVectorSearch2 = "// Search for examples similar to a \"powerful sorceress\"\nqueryVector = [0.3, 0.9, 0.6, 0.8];\n\nconst results = await table.search(queryVector).limit(2).toArray();\nconsole.table(results);\n"; +export const RsQuickstartAddData = "let more_data = vec![Character {\n id: \"4\".to_string(),\n name: \"Morgana\".to_string(),\n role: \"Sorceress\".to_string(),\n description: \"Powerful sorceress of Avalon.\".to_string(),\n stats: Stats {\n strength: 2,\n magic: 5,\n leadership: 4,\n wisdom: 4,\n },\n vector: [0.3, 0.9, 0.6, 0.8],\n}];\ntable\n .add(characters_to_reader(schema.clone(), &more_data))\n .execute()\n .await\n .unwrap();\n"; export const RsQuickstartAddFeature = "table\n .add_columns(\n NewColumnTransform::SqlExpressions(vec![(\n \"power_score\".to_string(),\n \"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)\"\n .to_string(),\n )]),\n None,\n )\n .await\n .unwrap();\n"; -export const RsQuickstartCreateTable = "let schema = characters_schema();\nlet table = db\n .create_table(\"characters\", characters_to_reader(schema.clone(), &data))\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; +export const RsQuickstartCreateTable = "use lancedb::database::CreateTableMode;\n\nlet schema = characters_schema();\nlet table = db\n .create_table(\"characters\", characters_to_reader(schema.clone(), &data))\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; -export const RsQuickstartCreateTableNoOverwrite = "let table = db\n .create_table(\"characters\", characters_to_reader(schema.clone(), &data))\n .execute()\n .await\n .unwrap();\n"; - -export const RsQuickstartCurateWithMetadata = "let curated: DataFrame = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .only_if(\"stats.magic >= 4\")\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{curated:?}\");\n"; +export const RsQuickstartCurateWithMetadata = "let curated = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .only_if(\"stats.magic >= 4\")\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{curated:?}\");\n"; export const RsQuickstartData = "let data = vec![\n Character {\n id: \"1\".to_string(),\n name: \"King Arthur\".to_string(),\n role: \"King\".to_string(),\n description: \"Leader of Camelot and wielder of Excalibur.\".to_string(),\n stats: Stats {\n strength: 4,\n magic: 1,\n leadership: 5,\n wisdom: 4,\n },\n vector: [0.7, 0.1, 0.9, 0.7],\n },\n Character {\n id: \"2\".to_string(),\n name: \"Merlin\".to_string(),\n role: \"Wizard\".to_string(),\n description: \"Advisor and prophet with deep magical knowledge.\".to_string(),\n stats: Stats {\n strength: 2,\n magic: 5,\n leadership: 4,\n wisdom: 5,\n },\n vector: [0.2, 0.9, 0.4, 0.9],\n },\n Character {\n id: \"3\".to_string(),\n name: \"Sir Lancelot\".to_string(),\n role: \"Knight\".to_string(),\n description: \"Legendary knight known for courage and combat skill.\".to_string(),\n stats: Stats {\n strength: 5,\n magic: 1,\n leadership: 3,\n wisdom: 3,\n },\n vector: [0.9, 0.1, 0.5, 0.4],\n },\n];\n"; -export const RsQuickstartDefineStruct = "// Define structs representing the data schema\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct Stats {\n strength: i8,\n magic: i8,\n leadership: i8,\n wisdom: i8,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct Character {\n id: String,\n name: String,\n role: String,\n description: String,\n stats: Stats,\n vector: [f32; 4],\n}\n\nfn characters_schema() -> Arc {\n Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::LargeUtf8, false),\n Field::new(\"name\", DataType::LargeUtf8, false),\n Field::new(\"role\", DataType::LargeUtf8, false),\n Field::new(\"description\", DataType::LargeUtf8, false),\n Field::new(\n \"stats\",\n DataType::Struct(arrow_schema::Fields::from(vec![\n Arc::new(Field::new(\"strength\", DataType::Int8, false)),\n Arc::new(Field::new(\"magic\", DataType::Int8, false)),\n Arc::new(Field::new(\"leadership\", DataType::Int8, false)),\n Arc::new(Field::new(\"wisdom\", DataType::Int8, false)),\n ])),\n false,\n ),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n ]))\n}\n"; +export const RsQuickstartDefineStruct = "use std::sync::Arc;\n\nuse lancedb::arrow::arrow_array::types::Float32Type;\nuse lancedb::arrow::arrow_array::{\n FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator,\n RecordBatchReader, StructArray,\n};\nuse lancedb::arrow::arrow_schema::{DataType, Field, FieldRef, Fields, Schema};\n\n// Define structs representing the data schema\n#[derive(Debug, Clone)]\nstruct Stats {\n strength: i8,\n magic: i8,\n leadership: i8,\n wisdom: i8,\n}\n\n#[derive(Debug, Clone)]\nstruct Character {\n id: String,\n name: String,\n role: String,\n description: String,\n stats: Stats,\n vector: [f32; 4],\n}\n\nfn characters_schema() -> Arc {\n Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::LargeUtf8, false),\n Field::new(\"name\", DataType::LargeUtf8, false),\n Field::new(\"role\", DataType::LargeUtf8, false),\n Field::new(\"description\", DataType::LargeUtf8, false),\n Field::new(\n \"stats\",\n DataType::Struct(Fields::from(vec![\n Arc::new(Field::new(\"strength\", DataType::Int8, false)),\n Arc::new(Field::new(\"magic\", DataType::Int8, false)),\n Arc::new(Field::new(\"leadership\", DataType::Int8, false)),\n Arc::new(Field::new(\"wisdom\", DataType::Int8, false)),\n ])),\n false,\n ),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n ]))\n}\n\nfn characters_to_reader(\n schema: Arc,\n rows: &[Character],\n) -> Box {\n let ids = LargeStringArray::from_iter_values(rows.iter().map(|row| row.id.as_str()));\n let names = LargeStringArray::from_iter_values(rows.iter().map(|row| row.name.as_str()));\n let roles = LargeStringArray::from_iter_values(rows.iter().map(|row| row.role.as_str()));\n let descriptions =\n LargeStringArray::from_iter_values(rows.iter().map(|row| row.description.as_str()));\n\n let strength = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.strength));\n let magic = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.magic));\n let leadership = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.leadership));\n let wisdom = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.wisdom));\n let stats_fields: Vec = vec![\n Arc::new(Field::new(\"strength\", DataType::Int8, false)),\n Arc::new(Field::new(\"magic\", DataType::Int8, false)),\n Arc::new(Field::new(\"leadership\", DataType::Int8, false)),\n Arc::new(Field::new(\"wisdom\", DataType::Int8, false)),\n ];\n let stats = StructArray::new(\n stats_fields.into(),\n vec![\n Arc::new(strength),\n Arc::new(magic),\n Arc::new(leadership),\n Arc::new(wisdom),\n ],\n None,\n );\n\n let vectors = FixedSizeListArray::from_iter_primitive::(\n rows.iter()\n .map(|row| Some(row.vector.iter().copied().map(Some).collect::>())),\n 4,\n );\n\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(ids),\n Arc::new(names),\n Arc::new(roles),\n Arc::new(descriptions),\n Arc::new(stats),\n Arc::new(vectors),\n ],\n )\n .unwrap();\n\n Box::new(RecordBatchIterator::new(\n vec![Ok(batch)].into_iter(),\n schema,\n ))\n}\n"; -export const RsQuickstartMultimodalBytes = "use std::sync::Arc;\n\nuse arrow_array::{\n BinaryArray, FixedSizeListArray, LargeStringArray, RecordBatch, RecordBatchIterator,\n};\nuse arrow_schema::{DataType, Field, Schema};\n\nlet image_path = std::path::Path::new(env!(\"CARGO_MANIFEST_DIR\"))\n .join(\"../../web/static/assets/images/quickstart/sir-lancelot.jpg\");\nlet image_bytes = std::fs::read(image_path).unwrap();\n\nlet image_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::LargeUtf8, false),\n Field::new(\"description\", DataType::LargeUtf8, false),\n Field::new(\"image\", DataType::Binary, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n]));\nlet image_vectors = [[0.9_f32, 0.1, 0.5, 0.4]];\nlet image_batch = RecordBatch::try_new(\n image_schema.clone(),\n vec![\n Arc::new(LargeStringArray::from_iter_values([\"lancelot\"])),\n Arc::new(LargeStringArray::from_iter_values([\n \"Portrait of Sir Lancelot\",\n ])),\n Arc::new(BinaryArray::from_iter_values([image_bytes.as_slice()])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n image_vectors\n .iter()\n .map(|vector| Some(vector.iter().copied().map(Some).collect::>())),\n 4,\n ),\n ),\n ],\n)\n.unwrap();\nlet image_reader: Box = Box::new(\n RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), image_schema),\n);\nlet multimodal_table = db\n .create_table(\"character_images\", image_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; +export const RsQuickstartMultimodalBytes = "use lancedb::arrow::arrow_array::BinaryArray;\n\nlet image_bytes = std::fs::read(\"data/sir-lancelot.jpg\").unwrap();\n\nlet image_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::LargeUtf8, false),\n Field::new(\"description\", DataType::LargeUtf8, false),\n Field::new(\"image\", DataType::Binary, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n]));\nlet image_vectors = [[0.9_f32, 0.1, 0.5, 0.4]];\nlet image_batch = RecordBatch::try_new(\n image_schema.clone(),\n vec![\n Arc::new(LargeStringArray::from_iter_values([\"lancelot\"])),\n Arc::new(LargeStringArray::from_iter_values([\n \"Portrait of Sir Lancelot\",\n ])),\n Arc::new(BinaryArray::from_iter_values([image_bytes.as_slice()])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n image_vectors\n .iter()\n .map(|vector| Some(vector.iter().copied().map(Some).collect::>())),\n 4,\n ),\n ),\n ],\n)\n.unwrap();\nlet image_reader: Box = Box::new(RecordBatchIterator::new(\n vec![Ok(image_batch)].into_iter(),\n image_schema,\n));\nlet multimodal_table = db\n .create_table(\"character_images\", image_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; -export const RsQuickstartOutputArray = "let result: DataFrame = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{result:?}\");\n"; +export const RsQuickstartOutputArray = "let result = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{result:?}\");\n"; -export const RsQuickstartQueryFeature = "let features: DataFrame = table\n .query()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"power_score\".to_string(),\n ]))\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{features:?}\");\n"; +export const RsQuickstartQueryFeature = "let features = table\n .query()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"power_score\".to_string(),\n ]))\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{features:?}\");\n"; -export const RsQuickstartVectorSearch1 = "// Search for examples similar to a \"wise magical advisor\"\nlet query_vector = [0.2, 0.8, 0.4, 0.9];\n\nlet result: DataFrame = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{result:?}\");\n"; +export const RsQuickstartVectorSearch1 = "use lancedb::arrow::IntoPolars;\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\n\n// Search for examples similar to a \"wise magical advisor\"\nlet query_vector = [0.2, 0.8, 0.4, 0.9];\n\nlet result = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{result:?}\");\n"; diff --git a/docs/web/snippets/read_consistency.mdx b/docs/web/snippets/read_consistency.mdx new file mode 100644 index 000000000..f99521a4d --- /dev/null +++ b/docs/web/snippets/read_consistency.mdx @@ -0,0 +1,4 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PyReadConsistency = "from datetime import timedelta\nfrom tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-consistency-\") as uri:\n writer = lancedb.connect(uri)\n table = writer.create_table(\"documents\", [{\"id\": 1, \"text\": \"first\"}])\n\n default_reader = lancedb.connect(uri).open_table(\"documents\")\n latest_reader = lancedb.connect(\n uri, read_consistency_interval=timedelta(0)\n ).open_table(\"documents\")\n\n table.add([{\"id\": 2, \"text\": \"second\"}])\n\n default_count = default_reader.count_rows()\n latest_count = latest_reader.count_rows()\n print(f\"Default reader: {default_count} rows\") # 1\n print(f\"Zero-interval reader: {latest_count} rows\") # 2\n\n default_reader.checkout_latest()\n refreshed_count = default_reader.count_rows()\n print(f\"After explicit refresh: {refreshed_count} rows\") # 2\n"; + diff --git a/docs/web/static/assets/images/overview/lancedb-architecture.svg b/docs/web/static/assets/images/overview/lancedb-architecture.svg new file mode 100644 index 000000000..8fe3ff3c9 --- /dev/null +++ b/docs/web/static/assets/images/overview/lancedb-architecture.svg @@ -0,0 +1,60 @@ + + LanceDB OSS and Enterprise architecture + OSS runs the application and embedded library in one process with local disk or object storage. Enterprise separates query serving, read execution and caching, and indexing and background workers. The control plane governs the service. Enterprise persists tables in object storage. Both offerings use the Lance format, without automatically sharing storage. + + + + + + + + LanceDB OSS + Embedded library + + Application + + LanceDB library + Queries + Indexing + Maintenance + + + Local disk or object storage + + + LanceDB Enterprise + Distributed multimodal + lakehouse + + Applications + + LanceDB SDK + + + Query serving + + + + Read + execution + + caching + + Indexing + + background + work + + + + Durable object storage + + Control plane + Manages the service + + + Shared foundation: Lance format + diff --git a/docs/web/static/assets/images/overview/lancedb-suite.svg b/docs/web/static/assets/images/overview/lancedb-suite.svg index 92a9335b8..0f860935d 100644 --- a/docs/web/static/assets/images/overview/lancedb-suite.svg +++ b/docs/web/static/assets/images/overview/lancedb-suite.svg @@ -2,9 +2,6 @@ LanceDB suite LanceDB OSS supports search. LanceDB Enterprise is a multimodal lakehouse for curation, feature engineering, search and retrieval, and training. Both are built on the Lance open lakehouse format for multimodal AI. - - - @@ -28,21 +25,21 @@ - - + + - + - + - + - + - + - + diff --git a/docs/web/static/assets/images/overview/training-data-lifecycle.svg b/docs/web/static/assets/images/overview/training-data-lifecycle.svg index a533c70e9..4998ee2e3 100644 --- a/docs/web/static/assets/images/overview/training-data-lifecycle.svg +++ b/docs/web/static/assets/images/overview/training-data-lifecycle.svg @@ -2,22 +2,19 @@ Training data lifecycle Four pillars of the LanceDB training data lifecycle: Curation, Feature Engineering, Search and Retrieval, and Training. - - - - + - + - + - + diff --git a/docs/web/tables/consistency.mdx b/docs/web/tables/consistency.mdx index dcf1a9ac6..388655ff8 100644 --- a/docs/web/tables/consistency.mdx +++ b/docs/web/tables/consistency.mdx @@ -1,161 +1,90 @@ --- -title: Consistency +title: "Consistency" sidebarTitle: "Consistency" -description: Learn about consistency settings and versioning in LanceDB tables. +description: "Understand when readers see changes and choose the freshness your application needs." icon: "water" +availability: + oss: available + enterprise: available + summary: >- + Configure read freshness in OSS and Enterprise. OSS checks for table updates; + Enterprise uses the connection setting to tighten its configured freshness bound. --- -import { - PyConsistencyStrong as ConsistencyStrong, - TsConsistencyStrong as TsConsistencyStrong, - RsConsistencyStrong as RsConsistencyStrong, - PyConsistencyEventual as ConsistencyEventual, - TsConsistencyEventual as TsConsistencyEventual, - RsConsistencyEventual as RsConsistencyEventual, - PyConsistencyCheckoutLatest as ConsistencyCheckoutLatest, - TsConsistencyCheckoutLatest as TsConsistencyCheckoutLatest, - RsConsistencyCheckoutLatest as RsConsistencyCheckoutLatest, - RsUpdateMakeUsersReader as RsConsistencyMakeUsersReader, -} from '/snippets/tables.mdx'; -You can set `read_consistency_interval` on the connection to control how often reads check for updates from other writers. +import { PyReadConsistency } from '/snippets/read_consistency.mdx'; -There are three possible settings for `read_consistency_interval`: +## Table snapshots and freshness {#consistency} -1. **Unset (default)**: no automatic cross-process refresh checks. -2. **Zero seconds**: check for updates on every read (strongest freshness). -3. **Non-zero interval**: check for updates after the interval elapses (eventual refresh). +Each table read uses one committed version of the data. **Freshness** describes +whether that version includes recent writes. For example, after another +application appends records, an already-open table handle can still read the +version from before the append. -The value you set depends on your application's consistency needs and performance requirements. -For example, a real-time dashboard might require strong consistency, while a batch analytics job might be -fine with eventual consistency. Stronger consistency is not free — the smaller the interval, the more -often each read pays the cost of refreshing against object storage, which raises per-read latency and cost. +For example, a dashboard may need the latest updates, while an experiment may +need to keep reading a specific table version. Use read freshness settings for the dashboard and an explicit table version +for the experiment. -This setting works for both local ([LanceTable](/management/database#uris)) and remote -tables. It only affects read operations — -write operations are always consistent. +## Visibility of writes {#visibility} - -**Consistency in Remote Tables** +In **OSS**, a table handle sees its own committed writes. By default, it does +not automatically check for changes made through other handles or processes. +Use a read consistency interval or `checkout_latest()` to pick up those changes. -For remote tables (`db://` connections), `read_consistency_interval` is also -respected by the client. The interval is sent to the server as a freshness bound on each read: +In **Enterprise**, readers use the service's configured freshness policy. +A connection can request fresher data. When a read must include a completed +write, use a zero interval or call `checkout_latest()` after the write, with a +client and server that support these freshness controls. -- **Unset (default)**: no freshness header is sent; reads use the server's cached view of the table. -- **Zero seconds**: every read asks the server for the latest committed version. -- **Non-zero interval**: reads accept data at least as fresh as `now - interval`. +## Configure read freshness {#configure-consistency-parameters} -In addition, after any write or after a `checkout_latest` / `restore` on a table handle, subsequent -reads on that same handle carry a freshness floor so you read your own writes without extra -configuration. The floor is the later of the configured interval and the moment of the last -write or refresh, and it is shared across handles to the same table on the same connection. +Set `read_consistency_interval` when connecting. This controls when reads +pick up newer table versions; it does not change how writes commit. -Each remote table handle also tracks the highest dataset version it has observed in a read -response and sends it back with every subsequent read, so successive reads on the same handle -never observe an older version even when a load balancer routes them to query nodes with -differently-cached views. `checkout_latest` resets this watermark. +| Setting | OSS | Enterprise | +| --- | --- | --- | +| **Unset** | Keep the open table version until explicitly refreshed or changed by that handle. | Use the service's configured freshness policy. | +| **Zero** | Check for the latest committed version on every read. | Request the latest committed version on every read. | +| **Non-zero interval** | Check for updates on a read once the interval since the last check has elapsed. | Request data no older than the interval allows; the service may return fresher data. | -Stronger consistency is not free — the smaller the interval, the more often each read pays the cost -of refreshing against storage, which raises per-read latency and cost. +`checkout_latest()` explicitly refreshes an OSS table handle. On Enterprise it +asks subsequent reads on that handle to reflect data at least as recent as the +refresh request. To read a fixed historical version instead, use +[table versioning](/management/table#versioning). -In Enterprise deployments, the server-side default freshness is still -controlled by the cluster-level `weak_read_consistency_interval_seconds` parameter; the client setting -tightens that bound on a per-connection basis. + - -30-60 seconds is a reasonable starting point for -`weak_read_consistency_interval_seconds`: it cuts unnecessary object-store -checks while keeping data fresh enough for most applications. This setting -only affects reads -- writes always remain strongly consistent. - - +Install the Python SDK with `pip install lancedb`, then save this as +`consistency.py` and run `python consistency.py`. It creates a temporary local +database and removes it when finished. No Enterprise connection is needed. -## Configure Consistency Parameters {#configure-consistency-parameters} + +{PyReadConsistency} + -To set strong consistency, set the interval to 0: +The default reader sees one row until refreshed. The reader with a zero +interval sees the new row on its next read. - - - {ConsistencyStrong} - + - - {TsConsistencyStrong} - +## Choose a freshness interval {#choosing} - - {RsConsistencyStrong} - - +- **Reads that must include recent writes:** use a zero interval. Checking more + frequently can increase read latency and storage requests. +- **Dashboards with an acceptable delay:** choose an interval within that delay + budget, then measure the effect on latency and cost. +- **Reproducible experiments:** read an explicit table version and retain it for + as long as the experiment needs it. - - - - {RsConsistencyMakeUsersReader} - - - +Choose the interval based on the delay your application can tolerate between +a write completing and a reader seeing it. -For eventual consistency, use a non-zero interval: +## Learn more {#next} - - - {ConsistencyEventual} - +- [Connect](/basics/connect): configure your client. +- [Table versioning](/management/table#versioning): read and restore retained versions. +- [Guarantees](/basics/guarantees): understand commit and failure boundaries. - - {TsConsistencyEventual} - + - - {RsConsistencyEventual} - - - - -With the default unset interval, tables do not auto-refresh from other writers. -To manually check for updates, use `checkout_latest` / `checkoutLatest`: - - - - {ConsistencyCheckoutLatest} - - - - {TsConsistencyCheckoutLatest} - - - - {RsConsistencyCheckoutLatest} - - - -For reproducible reads, you can also pin a table to a specific snapshot with `checkout(...)` or -a tag, restore a table to a prior version, then return to the live table with -`checkout_latest` / `checkoutLatest`. See -[Versioning](/management/table#versioning) for the full version and tag workflow. - -## Handle bad vectors {#handle-bad-vectors} - - -This section is currently specific to the Python SDK. - - -In LanceDB Python, you can use the `on_bad_vectors` parameter to choose how -invalid vector values are handled. Invalid vectors are vectors that are not valid -because: - -1. They are the wrong dimension -2. They contain NaN values -3. They are null but are on a non-nullable field - -By default, LanceDB will raise an error if it encounters a bad vector. You can -also choose one of the following options: - -* `drop`: Ignore rows with bad vectors -* `fill`: Replace bad values (NaNs) or missing values (too few dimensions) with - the fill value specified in the `fill_value` parameter. An input like - `[1.0, NaN, 3.0]` will be replaced with `[1.0, 0.0, 3.0]` if `fill_value=0.0`. -* `null`: Replace bad vectors with null (only works if the column is nullable). - A bad vector `[1.0, NaN, 3.0]` will be replaced with `null` if the column is - nullable. If the vector column is non-nullable, then bad vectors will cause an - error +For invalid vector handling during ingestion, see +[Create a table](/ingestion/create-table#from-a-custom-schema). From d47183953c7a5be5cfbdfcc2bfa73a2490aa6aff Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Thu, 15 Oct 2026 14:53:48 -0700 Subject: [PATCH 17/19] chore: update lance dependency to v14.0.0-beta.18 (#4552) Updates the Rust workspace dependencies and Java lance-core to [Lance v14.0.0-beta.18](https://github.com/lance-format/lance/releases/tag/v14.0.0-beta.18). Mirrors Lance's pinned object_store patch to support the multipart write mode API required by this release. Validation: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and `git diff --check` passed. --------- Co-authored-by: Lu Qiu Co-authored-by: Claude Fable 5.1 Co-authored-by: Yang Cen --- Cargo.lock | 96 ++++++++++++++++----------------- Cargo.toml | 32 ++++++----- deny.toml | 2 + java/pom.xml | 2 +- rust/lancedb/src/query.rs | 14 +++-- rust/lancedb/src/table/merge.rs | 61 +++++++++++++++++++-- 6 files changed, 137 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0515319b2..a2e317c22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arc-swap", "arrow", @@ -5133,8 +5133,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-buffer", @@ -5156,7 +5156,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-buffer", @@ -5170,7 +5170,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-schema", @@ -5179,8 +5179,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrayref", "crunchy", @@ -5190,8 +5190,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-buffer", @@ -5228,8 +5228,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "arrow-array", @@ -5260,8 +5260,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "arrow-array", @@ -5278,8 +5278,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "proc-macro2", "quote", @@ -5288,8 +5288,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-arith", "arrow-array", @@ -5322,8 +5322,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-arith", "arrow-array", @@ -5355,8 +5355,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "datafusion", "geo-traits", @@ -5370,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arc-swap", "arrow", @@ -5442,8 +5442,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-schema", @@ -5466,8 +5466,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "arrow-array", @@ -5494,6 +5494,7 @@ dependencies = [ "path_abs", "pin-project", "prost", + "quick-xml 0.41.0", "rand 0.9.5", "reqsign-core", "reqsign-file-read-tokio", @@ -5509,8 +5510,8 @@ dependencies = [ [[package]] name = "lance-jsonb" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "byteorder", "ethnum", @@ -5527,8 +5528,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-schema", @@ -5542,8 +5543,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "async-trait", @@ -5557,8 +5558,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arc-swap", "arrow", @@ -5612,8 +5613,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-buffer", @@ -5627,8 +5628,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "arrow-array", @@ -5668,8 +5669,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-schema", @@ -5682,8 +5683,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "frostem", "icu_segmenter", @@ -6819,8 +6820,7 @@ dependencies = [ [[package]] name = "object_store" version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" +source = "git+https://github.com/apache/arrow-rs-object-store.git?rev=c10758c207c0103355f940f0c736fb8ecb157b04#c10758c207c0103355f940f0c736fb8ecb157b04" dependencies = [ "async-trait", "aws-lc-rs", diff --git a/Cargo.toml b/Cargo.toml index 5033b1529..c6b9e84fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow @@ -79,6 +79,10 @@ tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] } uuid = { version = "1.7.0", features = ["v4", "v7"] } chrono = { version = "0.4", default-features = false, features = ["clock"] } +[patch.crates-io] +# Lance v14.0.0-beta.18 requires multipart write modes from this upstream revision. +object_store = { git = "https://github.com/apache/arrow-rs-object-store.git", rev = "c10758c207c0103355f940f0c736fb8ecb157b04" } + [profile.ci] debug = "line-tables-only" inherits = "dev" diff --git a/deny.toml b/deny.toml index d112095ec..d5f42dbff 100644 --- a/deny.toml +++ b/deny.toml @@ -218,4 +218,6 @@ allow-registry = ["https://github.com/rust-lang/crates.io-index"] # releases are cut to crates.io. Allow that specific host. allow-git = [ "https://github.com/lance-format/lance", + # Pinned by Lance for the multipart write mode API (see [patch.crates-io] in Cargo.toml). + "https://github.com/apache/arrow-rs-object-store", ] diff --git a/java/pom.xml b/java/pom.xml index 33446040b..465554706 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 14.0.0-beta.15 + 14.0.0-beta.18 2.18.11 false 2.30.0 diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index 07db69bb3..d66932847 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -3806,8 +3806,12 @@ mod tests { assert_eq!(results[0].num_columns(), 1); } + #[rstest::rstest] + #[case::single_row(1)] + #[case::two_rows(2)] + #[case::three_rows(3)] #[tokio::test] - async fn test_take_offsets_preserves_duplicate_multiplicity() { + async fn test_take_offsets_preserves_duplicate_multiplicity(#[case] max_batch_length: u32) { let tmp_dir = tempdir().unwrap(); let table = make_test_table(&tmp_dir).await; @@ -3815,7 +3819,7 @@ mod tests { .take_offsets(vec![5, 1, 5, 17]) .select(Select::Columns(vec!["id".to_string()])) .execute_with_options(QueryExecutionOptions { - max_batch_length: 2, + max_batch_length, ..Default::default() }) .await @@ -3824,7 +3828,11 @@ mod tests { .await .unwrap(); - assert_eq!(results.len(), 2); + assert!( + results + .iter() + .all(|batch| batch.num_rows() <= max_batch_length as usize) + ); assert!(results.iter().all(|batch| batch.num_columns() == 1)); let mut ids = results .iter() diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index fbe82a198..c74980c5e 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -2035,11 +2035,13 @@ mod lsm_tests { ); } - /// Claims the base table's bitmap index and keeps it as a B-tree, whose - /// flush rows are in the shape the bitmap trainer takes. + /// Claims the base table's bitmap index and keeps it as a B-tree. #[derive(Debug)] struct BitmapAsBTree; + #[derive(Debug)] + struct BitmapAsBTreeIndex(lance::dataset::mem_wal::index::BTreeMemIndex); + fn btree_plugin() -> Arc { lance::dataset::mem_wal::MemIndexRegistry::default() .plugin_for_details_url("/lance.table.BTreeIndexDetails") @@ -2059,7 +2061,8 @@ mod lsm_tests { lance_index::IndexType::Bitmap } fn training_criteria(&self) -> lance_index::scalar::registry::TrainingCriteria { - btree_plugin().training_criteria() + use lance_index::scalar::registry::{TrainingCriteria, TrainingOrdering}; + TrainingCriteria::new(TrainingOrdering::Values).with_row_id() } fn validate( &self, @@ -2071,7 +2074,57 @@ mod lsm_tests { &self, ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>, ) -> lance_core::Result> { - btree_plugin().create(ctx) + let (column, field_id) = ctx.single_column()?; + Ok(Arc::new(BitmapAsBTreeIndex( + lance::dataset::mem_wal::index::BTreeMemIndex::new(field_id, column.to_string()), + ))) + } + } + + #[async_trait::async_trait] + impl lance::dataset::mem_wal::index::MemIndex for BitmapAsBTreeIndex { + fn columns(&self) -> &[String] { + self.0.columns() + } + fn can_answer(&self, query: &dyn lance::dataset::mem_wal::index::MemQuery) -> bool { + self.0.can_answer(query) + } + fn insert(&self, batch: &RecordBatch, row_offset: u64) -> lance_core::Result<()> { + self.0.insert(batch, row_offset) + } + fn resident_bytes(&self) -> usize { + self.0.resident_bytes() + } + fn search( + &self, + query: &dyn lance::dataset::mem_wal::index::MemQuery, + ctx: &lance::dataset::mem_wal::index::SearchContext, + ) -> lance_core::Result> { + self.0.search(query, ctx) + } + async fn flush( + &self, + ctx: &lance::dataset::mem_wal::index::FlushContext<'_>, + ) -> lance_core::Result { + // BTree now trains on row addresses, while Bitmap still takes row IDs. + // These coincide in the flushed generation, which uses physical IDs. + assert!(!ctx.generation()?.dataset.manifest().uses_stable_row_ids()); + let batches = self + .0 + .to_training_batches(ctx.batch_size)? + .into_iter() + .map(|batch| { + assert_eq!(batch.schema().field(1).name(), lance_core::ROW_ADDR); + let schema = Arc::new(Schema::new(vec![ + batch.schema().field(0).clone(), + Field::new(lance_core::ROW_ID, DataType::UInt64, false), + ])); + RecordBatch::try_new(schema, batch.columns().to_vec()) + }) + .collect::, _>>()?; + Ok(lance::dataset::mem_wal::index::FlushOutcome::from_batches( + batches, + )) } } From 758f738b8ce401e8ac30edd94afb3330a231fd1b Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Thu, 15 Oct 2026 14:54:08 -0700 Subject: [PATCH 18/19] docs: complete ingestion, search and integration examples (#4533) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR is part of the documentation revamp. Once the full set of PRs is merged, we’ll publish the combined site to staging for review before updating docs.lancedb.com. Ingestion, search and integration examples are incomplete or inconsistent across languages, and some guidance misstates SDK behavior. This brings the guides and their example sources together. It includes complete Python, TypeScript and Rust workflows, isolated provider and reranker examples, the Superlinked corrections, and guidance on retries that can duplicate writes. Stacked on #4491. Each MDX file has one PR owner. Rebase onto main after #4491 is squash-merged. Local assembly, Mint validation and link checks pass. Example code is unchanged from the previously validated package. Publication stays held through integration. --- docs/web-tests/py/provider_examples.py | 355 +++++++++ docs/web-tests/py/test_ingestion_examples.py | 739 ++++++++++++++++++ docs/web-tests/py/test_integrations.py | 75 +- docs/web-tests/py/test_provider_examples.py | 160 ++++ docs/web-tests/py/test_reranker_examples.py | 96 +++ docs/web-tests/py/test_search_examples.py | 228 ++++++ docs/web-tests/rs/Cargo.lock | 1 + docs/web-tests/rs/Cargo.toml | 33 + docs/web-tests/rs/ingestion_delete.rs | 72 ++ docs/web-tests/rs/ingestion_examples.rs | 97 +++ docs/web-tests/rs/ingestion_progress.rs | 64 ++ docs/web-tests/rs/search_filter_search.rs | 92 +++ docs/web-tests/rs/search_hybrid_search.rs | 99 +++ docs/web-tests/rs/search_query_plan.rs | 81 ++ docs/web-tests/rs/search_text_search.rs | 85 ++ docs/web-tests/rs/search_vector_search.rs | 93 +++ docs/web-tests/ts/ingestion_examples.test.ts | 117 +++ docs/web-tests/ts/search_examples.test.ts | 222 ++++++ docs/web/indexing/fts-index.mdx | 9 +- docs/web/indexing/index.mdx | 36 +- docs/web/indexing/reindexing.mdx | 52 +- docs/web/indexing/scalar-index.mdx | 23 +- docs/web/indexing/vector-index.mdx | 49 +- docs/web/ingestion/create-table.mdx | 178 +++-- docs/web/ingestion/delete.mdx | 139 ++-- docs/web/ingestion/index.mdx | 198 ++--- docs/web/ingestion/insert.mdx | 342 ++++---- docs/web/ingestion/merge-insert.mdx | 136 +++- docs/web/ingestion/update.mdx | 151 ++-- docs/web/integrations/ai/hermes-agent.mdx | 2 +- docs/web/integrations/ai/huggingface.mdx | 6 + docs/web/integrations/ai/langchain.mdx | 3 + docs/web/integrations/data/duckdb.mdx | 2 +- .../integrations/data/pandas_and_pyarrow.mdx | 6 + docs/web/integrations/data/polars_arrow.mdx | 9 +- docs/web/integrations/data/pydantic.mdx | 7 + docs/web/integrations/embedding/cohere.mdx | 4 + docs/web/integrations/embedding/ibm.mdx | 8 +- docs/web/integrations/embedding/jina.mdx | 4 +- docs/web/integrations/embedding/openai.mdx | 8 +- .../integrations/embedding/superlinked.mdx | 31 +- docs/web/integrations/embedding/voyageai.mdx | 4 + docs/web/integrations/lerobotdataset.mdx | 9 +- docs/web/integrations/reranking/cohere.mdx | 4 +- docs/web/integrations/reranking/jina.mdx | 4 +- docs/web/integrations/reranking/openai.mdx | 5 +- docs/web/integrations/reranking/voyageai.mdx | 5 +- docs/web/integrations/reranking/watsonx.mdx | 4 +- docs/web/reranking/index.mdx | 368 +++++---- docs/web/search/explain.mdx | 183 +++-- docs/web/search/filtering.mdx | 193 +++-- docs/web/search/full-text-search.mdx | 493 +++++++----- docs/web/search/hybrid-search.mdx | 210 +++-- docs/web/search/index.mdx | 152 ++-- docs/web/search/vector-search.mdx | 183 +++-- docs/web/snippets/ingestion_delete.mdx | 4 + docs/web/snippets/ingestion_examples.mdx | 66 ++ docs/web/snippets/ingestion_progress.mdx | 4 + docs/web/snippets/integrations.mdx | 16 +- docs/web/snippets/provider_examples.mdx | 18 + docs/web/snippets/search_examples.mdx | 32 + docs/web/snippets/search_filter_search.mdx | 6 + docs/web/snippets/search_hybrid_search.mdx | 6 + docs/web/snippets/search_query_plan.mdx | 6 + docs/web/snippets/search_text_search.mdx | 6 + docs/web/snippets/search_vector_search.mdx | 6 + .../static/code/ingestion-feedback-rust.zip | Bin 0 -> 38831 bytes docs/web/static/code/ingestion-rust.zip | Bin 0 -> 38092 bytes docs/web/static/code/search-rust.zip | Bin 0 -> 42197 bytes 69 files changed, 4734 insertions(+), 1365 deletions(-) create mode 100644 docs/web-tests/py/provider_examples.py create mode 100644 docs/web-tests/py/test_ingestion_examples.py create mode 100644 docs/web-tests/py/test_provider_examples.py create mode 100644 docs/web-tests/py/test_reranker_examples.py create mode 100644 docs/web-tests/py/test_search_examples.py create mode 100644 docs/web-tests/rs/ingestion_delete.rs create mode 100644 docs/web-tests/rs/ingestion_examples.rs create mode 100644 docs/web-tests/rs/ingestion_progress.rs create mode 100644 docs/web-tests/rs/search_filter_search.rs create mode 100644 docs/web-tests/rs/search_hybrid_search.rs create mode 100644 docs/web-tests/rs/search_query_plan.rs create mode 100644 docs/web-tests/rs/search_text_search.rs create mode 100644 docs/web-tests/rs/search_vector_search.rs create mode 100644 docs/web-tests/ts/ingestion_examples.test.ts create mode 100644 docs/web-tests/ts/search_examples.test.ts create mode 100644 docs/web/snippets/ingestion_delete.mdx create mode 100644 docs/web/snippets/ingestion_examples.mdx create mode 100644 docs/web/snippets/ingestion_progress.mdx create mode 100644 docs/web/snippets/provider_examples.mdx create mode 100644 docs/web/snippets/search_examples.mdx create mode 100644 docs/web/snippets/search_filter_search.mdx create mode 100644 docs/web/snippets/search_hybrid_search.mdx create mode 100644 docs/web/snippets/search_query_plan.mdx create mode 100644 docs/web/snippets/search_text_search.mdx create mode 100644 docs/web/snippets/search_vector_search.mdx create mode 100644 docs/web/static/code/ingestion-feedback-rust.zip create mode 100644 docs/web/static/code/ingestion-rust.zip create mode 100644 docs/web/static/code/search-rust.zip diff --git a/docs/web-tests/py/provider_examples.py b/docs/web-tests/py/provider_examples.py new file mode 100644 index 000000000..2ba24fe96 --- /dev/null +++ b/docs/web-tests/py/provider_examples.py @@ -0,0 +1,355 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Reader examples with explicit placeholders. + +These functions are documentation source. test_provider_examples.py runs the +reranker examples offline, with stand-ins for the model and the provider, to +check that each one writes only to its own new directory. Corresponding opt-in +integration tests remain in test_integrations.py and retain real credential +loading. Replace placeholders before running these examples. +""" + + +def embedding_ibm_usage(): + # --8<-- [start:embedding_ibm_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import EmbeddingFunctionRegistry + from lancedb.pydantic import LanceModel, Vector + + registry = EmbeddingFunctionRegistry.get_instance() + registry.set_var("watsonx-api-key", "your-api-key") + watsonx_embed = registry.get("watsonx").create( + name="ibm/slate-125m-english-rtrvr", + api_key="$var:watsonx-api-key", + project_id="your-project-id", + ) + + class TextModel(LanceModel): + text: str = watsonx_embed.SourceField() + vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-demo")) + tbl = db.create_table("watsonx_test", schema=TextModel, mode="overwrite") + tbl.add(data) + + rs = tbl.search("hello").limit(1).to_pandas() + print(rs.head()) + # --8<-- [end:embedding_ibm_usage] + + +def embedding_jina_text(): + # --8<-- [start:embedding_jina_text] + import os + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import EmbeddingFunctionRegistry + from lancedb.pydantic import LanceModel, Vector + + os.environ["JINA_API_KEY"] = "your-api-key" + + jina_embed = ( + EmbeddingFunctionRegistry.get_instance() + .get("jina") + .create(name="jina-embeddings-v2-base-en") + ) + + class TextModel(LanceModel): + text: str = jina_embed.SourceField() + vector: Vector(jina_embed.ndims()) = jina_embed.VectorField() + + data = [{"text": "hello world"}, {"text": "goodbye world"}] + + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-text")) + tbl = db.create_table("test", schema=TextModel, mode="overwrite") + + tbl.add(data) + # --8<-- [end:embedding_jina_text] + + +def embedding_jina_multimodal(): + # --8<-- [start:embedding_jina_multimodal] + import os + import tempfile + from pathlib import Path + + import lancedb + import pandas as pd + import requests + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + + os.environ["JINA_API_KEY"] = "your-api-key" + + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-images")) + func = get_registry().get("jina").create() + + class Images(LanceModel): + label: str + image_uri: str = func.SourceField() + image_bytes: bytes = func.SourceField() + vector: Vector(func.ndims()) = func.VectorField() + vec_from_bytes: Vector(func.ndims()) = func.VectorField() + + table = db.create_table("images", schema=Images) + labels = ["cat", "cat", "dog", "dog", "horse", "horse"] + uris = [ + "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg", + "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg", + "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg", + "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg", + "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg", + "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg", + ] + image_bytes = [requests.get(uri).content for uri in uris] + table.add( + pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes}) + ) + # --8<-- [end:embedding_jina_multimodal] + + +def reranking_cohere_usage(): + # --8<-- [start:reranking_cohere_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import CohereReranker + + embedder = get_registry().get("sentence-transformers").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "cohere-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + reranker = CohereReranker(api_key="your-api-key") + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_cohere_usage] + + +def reranking_jina_usage(): + # --8<-- [start:reranking_jina_usage] + import os + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import JinaReranker + + os.environ["JINA_API_KEY"] = "your-api-key" + embedder = get_registry().get("jina").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + reranker = JinaReranker(api_key="your-api-key") + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_jina_usage] + + +def reranking_watsonx_usage(): + # --8<-- [start:reranking_watsonx_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import WatsonxReranker + + embedder = get_registry().get("sentence-transformers").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + + # Supply your API key and project ID (or use space_id instead of project_id). + reranker = WatsonxReranker( + api_key="your-api-key", + project_id="your-project-id", + ) + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_watsonx_usage] + + +def reranking_openai_usage(): + # --8<-- [start:reranking_openai_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import OpenaiReranker + + embedder = get_registry().get("sentence-transformers").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "openai-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + reranker = OpenaiReranker(api_key="your-api-key") + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_openai_usage] + + +def reranking_voyageai_usage(): + # --8<-- [start:reranking_voyageai_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import VoyageAIReranker + + embedder = get_registry().get("sentence-transformers").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "voyageai-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + reranker = VoyageAIReranker(api_key="your-api-key", model_name="rerank-2") + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_voyageai_usage] diff --git a/docs/web-tests/py/test_ingestion_examples.py b/docs/web-tests/py/test_ingestion_examples.py new file mode 100644 index 000000000..0dedc1458 --- /dev/null +++ b/docs/web-tests/py/test_ingestion_examples.py @@ -0,0 +1,739 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + + +def test_create_table_from_dicts(): + # --8<-- [start:create_table_from_dicts] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + data = [ + {"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}, + {"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1}, + ] + db.create_table("test_table", data, mode="overwrite") + tbl = db["test_table"] + tbl.head() + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_from_dicts] + assert tbl.count_rows() == 2 + + +def test_create_table_conflict_handling(): + # --8<-- [start:create_table_conflict_handling] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}] + db.create_table("conflict_table", data) + # Idempotent open: reuse the existing table if it exists. + # The provided data is ignored; the schema is validated against the + # existing table and a mismatch raises an error. + tbl = db.create_table("conflict_table", data, exist_ok=True) + + # Overwrite: replace the contents with the provided data. + tbl = db.create_table("conflict_table", data, mode="overwrite") + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_conflict_handling] + + +def test_create_table_custom_schema(): + # --8<-- [start:create_table_custom_schema] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + custom_schema = pa.schema( + [ + pa.field("vector", pa.list_(pa.float32(), 4)), + pa.field("lat", pa.float32()), + pa.field("long", pa.float32()), + ] + ) + + data = [ + {"vector": [1.1, 1.2, 1.3, 1.4], "lat": 45.5, "long": -122.7}, + {"vector": [0.2, 1.8, 0.4, 3.6], "lat": 40.1, "long": -74.1}, + ] + tbl = db.create_table( + "my_table_custom_schema", data, schema=custom_schema, mode="overwrite" + ) + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_custom_schema] + assert tbl.count_rows() == 2 + + +def test_create_table_from_pandas(): + # --8<-- [start:create_table_from_pandas] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pandas as pd + + data = pd.DataFrame( + { + "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]], + "lat": [45.5, 40.1], + "long": [-122.7, -74.1], + } + ) + db.create_table("my_table_pandas", data, mode="overwrite") + db["my_table_pandas"].head() + # --8<-- [end:create_table_from_pandas] + + +def test_create_table_from_polars(): + # --8<-- [start:create_table_from_polars] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import polars as pl + + data = pl.DataFrame( + { + "vector": [[3.1, 4.1], [5.9, 26.5]], + "item": ["foo", "bar"], + "price": [10.0, 20.0], + } + ) + tbl = db.create_table("my_table_pl", data, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_from_polars] + assert tbl.count_rows() == 2 + + +def test_create_table_from_arrow(): + # --8<-- [start:create_table_from_arrow] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import numpy as np + import pyarrow as pa + + dim = 16 + total = 2 + schema = pa.schema( + [ + pa.field("vector", pa.list_(pa.float16(), dim)), + pa.field("text", pa.string()), + ] + ) + data = pa.Table.from_arrays( + [ + pa.array( + [np.random.randn(dim).astype(np.float16) for _ in range(total)], + pa.list_(pa.float16(), dim), + ), + pa.array(["foo", "bar"]), + ], + ["vector", "text"], + ) + tbl = db.create_table("f16_tbl", data, schema=schema, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_from_arrow] + assert tbl.count_rows() == 2 + + +def test_create_table_from_pydantic(): + # --8<-- [start:create_table_from_pydantic] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + from lancedb.pydantic import LanceModel, Vector + + class Content(LanceModel): + movie_id: int + vector: Vector(128) + genres: str + title: str + imdb_id: int + + @property + def imdb_url(self) -> str: + return f"https://www.imdb.com/title/tt{self.imdb_id}" + + tbl = db.create_table("movielens_small", schema=Content, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_from_pydantic] + assert tbl.count_rows() == 0 + + +def test_create_table_nested_schema(): + # --8<-- [start:create_table_nested_schema] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + from lancedb.pydantic import LanceModel, Vector + from pydantic import BaseModel + + class Document(BaseModel): + content: str + source: str + + class NestedSchema(LanceModel): + id: str + vector: Vector(1536) + document: Document + + tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_nested_schema] + assert tbl.count_rows() == 0 + + +def test_create_empty_table(): + # --8<-- [start:create_empty_table] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + schema = pa.schema( + [ + pa.field("vector", pa.list_(pa.float32(), 2)), + pa.field("item", pa.string()), + pa.field("price", pa.float32()), + ] + ) + tbl = db.create_table("test_empty_table", schema=schema, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_empty_table] + assert tbl.count_rows() == 0 + + +def test_create_empty_table_pydantic(): + # --8<-- [start:create_empty_table_pydantic] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + from lancedb.pydantic import LanceModel, Vector + + class Item(LanceModel): + vector: Vector(2) + item: str + price: float + + tbl = db.create_table( + "test_empty_table_new", schema=Item.to_arrow_schema(), mode="overwrite" + ) + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_empty_table_pydantic] + assert tbl.count_rows() == 0 + + +def test_open_existing_table(): + # --8<-- [start:open_existing_table] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + # Create a table first + data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}] + db.create_table("test_table", data, mode="overwrite") + + # List table names + print(db.list_tables().tables) + + # Open existing table + tbl = db.open_table("test_table") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:open_existing_table] + assert tbl.count_rows() == 1 + + +def test_drop_table(): + # --8<-- [start:drop_table] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + # Create a table first + data = [{"vector": [1.1, 1.2], "lat": 45.5}] + db.create_table("my_table", data, mode="overwrite") + + # Drop the table + db.drop_table("my_table") + # --8<-- [end:drop_table] + + +def test_add_from_dataset(): + # --8<-- [start:add_from_dataset] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + from pathlib import Path + + import pyarrow as pa + import pyarrow.parquet as pq + + data_path = str(Path(uri) / "input.parquet") + pq.write_table(pa.table({"id": [1, 2], "name": ["Alice", "Bob"]}), data_path) + import pyarrow.dataset as ds + + dataset = ds.dataset(data_path, format="parquet") + table = db.create_table("my_table", schema=dataset.schema, mode="overwrite") + table.add(dataset) + + print("Rows:", table.count_rows()) + # --8<-- [end:add_from_dataset] + assert table.count_rows() == 2 + + +def test_create_table_from_iterator(): + # --8<-- [start:create_table_from_iterator] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + schema = pa.schema( + [ + pa.field("vector", pa.list_(pa.float32(), 4)), + pa.field("item", pa.utf8()), + pa.field("price", pa.float32()), + ] + ) + + def make_batches(): + for i in range(5): + yield pa.RecordBatch.from_arrays( + [ + pa.array( + [[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]], + pa.list_(pa.float32(), 4), + ), + pa.array(["foo", "bar"]), + pa.array([10.0, 20.0]), + ], + ["vector", "item", "price"], + ) + + table = db.create_table("batched_table", schema=schema) + table.add(make_batches()) + + print("Rows:", table.count_rows()) + # --8<-- [end:create_table_from_iterator] + assert table.count_rows() == 10 + + +def test_update_operation(): + # --8<-- [start:update_operation] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + table.update(where="id = 2", values={"name": "Bobby"}) + + print("Rows:", table.count_rows()) + # --8<-- [end:update_operation] + assert table.count_rows() == 2 + assert table.search().where("id = 2").to_list()[0]["name"] == "Bobby" + + +def test_update_using_sql(): + # --8<-- [start:update_using_sql] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + table.update(where="id = 2", values_sql={"login_count": "login_count + 1"}) + + print("Rows:", table.count_rows()) + # --8<-- [end:update_using_sql] + assert table.count_rows() == 2 + assert table.search().where("id = 2").to_list()[0]["login_count"] == 21 + + +def test_merge_matched_update_only(): + # --8<-- [start:merge_matched_update_only] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + "login_count": [21, 5], + } + ) + + (table.merge_insert("id").when_matched_update_all().execute(incoming_users)) + + print("Rows:", table.count_rows()) + # --8<-- [end:merge_matched_update_only] + assert table.count_rows() == 2 + + +def test_insert_if_not_exists(): + # --8<-- [start:insert_if_not_exists] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + "login_count": [21, 5], + } + ) + + (table.merge_insert("id").when_not_matched_insert_all().execute(incoming_users)) + + print("Rows:", table.count_rows()) + # --8<-- [end:insert_if_not_exists] + assert table.count_rows() == 3 + + +def test_merge_update_insert(): + # --8<-- [start:merge_update_insert] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + "login_count": [21, 5], + } + ) + + ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(incoming_users) + ) + + print("Rows:", table.count_rows()) + # --8<-- [end:merge_update_insert] + assert table.count_rows() == 3 + assert table.search().where("id = 2").to_list()[0]["login_count"] == 21 + + +def test_merge_delete_missing_by_source(): + # --8<-- [start:merge_delete_missing_by_source] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2, 3], + "name": ["Alice", "Bob", "Charlie"], + "login_count": [10, 20, 5], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + "login_count": [21, 5], + } + ) + + ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .when_not_matched_by_source_delete() + .execute(incoming_users) + ) + + print("Rows:", table.count_rows()) + # --8<-- [end:merge_delete_missing_by_source] + assert table.count_rows() == 2 + assert table.count_rows("id = 1") == 0 + + +def test_merge_partial_columns(): + # --8<-- [start:merge_partial_columns] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + } + ) + + ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(incoming_users) + ) + + print("Rows:", table.count_rows()) + # --8<-- [end:merge_partial_columns] + assert table.count_rows() == 3 + assert table.search().where("id = 3").to_list()[0]["login_count"] is None + + +def test_write_workflow(): + # --8<-- [start:write_workflow] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "users", + [ + {"id": 1, "name": "Alice", "login_count": 10}, + {"id": 2, "name": "Bob", "login_count": 20}, + ], + ) + table.add([{"id": 3, "name": "Charlie", "login_count": 5}]) + table.update(where="id = 2", values={"name": "Bobby"}) + table.update(where="id = 2", values_sql={"login_count": "login_count + 1"}) + ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute( + [ + {"id": 3, "name": "Charlie", "login_count": 6}, + {"id": 4, "name": "Dana", "login_count": 1}, + ] + ) + ) + print("Rows selected for deletion:", table.search().where("id = 4").to_list()) + table.delete("id = 4") + rows = sorted(table.search().to_list(), key=lambda row: row["id"]) + print(rows) + # --8<-- [end:write_workflow] + + assert rows == [ + {"id": 1, "name": "Alice", "login_count": 10}, + {"id": 2, "name": "Bobby", "login_count": 21}, + {"id": 3, "name": "Charlie", "login_count": 6}, + ] + + +def test_append_rows(): + # --8<-- [start:append_rows] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + table = db.create_table("users", [{"id": 1, "name": "Alice"}]) + table.add([{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}]) + rows = sorted(table.search().to_list(), key=lambda row: row["id"]) + print(rows) + # --8<-- [end:append_rows] + + assert [row["id"] for row in rows] == [1, 2, 3] + + +def test_delete_rows(): + # --8<-- [start:delete_rows] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "users", + [ + {"id": 1, "name": "Alice"}, + {"id": 2, "name": "Bob"}, + {"id": 3, "name": "Charlie"}, + ], + ) + predicate = "id = 3" + print("Selected:", table.search().where(predicate).to_list()) + table.delete(predicate) + rows = sorted(table.search().to_list(), key=lambda row: row["id"]) + print("Remaining:", rows) + # --8<-- [end:delete_rows] + + assert rows == [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}] + + +def test_add_progress(): + # --8<-- [start:add_progress] + from tempfile import TemporaryDirectory + + import lancedb + + def report_progress(progress): + print(f"Rows written: {progress['output_rows']}") + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + table = db.create_table("users", [{"id": 1, "name": "Alice"}]) + table.add( + [{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}], + progress=report_progress, + ) + rows = sorted(table.search().to_list(), key=lambda row: row["id"]) + print("After insertion:", rows) + # --8<-- [end:add_progress] + + assert rows == [ + {"id": 1, "name": "Alice"}, + {"id": 2, "name": "Bob"}, + {"id": 3, "name": "Charlie"}, + ] diff --git a/docs/web-tests/py/test_integrations.py b/docs/web-tests/py/test_integrations.py index ff07dfae0..577847504 100644 --- a/docs/web-tests/py/test_integrations.py +++ b/docs/web-tests/py/test_integrations.py @@ -740,13 +740,16 @@ def test_reranking_answerdotai_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_answerdotai_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import AnswerdotaiRerankers embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "answerdotai-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -756,18 +759,20 @@ def test_reranking_answerdotai_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = AnswerdotaiRerankers() # Run vector search with a reranker result = tbl.search("hello").rerank(reranker=reranker).to_list() + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + # Run FTS search with a reranker result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) result = ( tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() ) @@ -819,13 +824,16 @@ def test_reranking_colbert_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_colbert_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import ColbertReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "colbert-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -835,18 +843,20 @@ def test_reranking_colbert_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = ColbertReranker() # Run vector search with a reranker result = tbl.search("hello").rerank(reranker=reranker).to_list() + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + # Run FTS search with a reranker result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) result = ( tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() ) @@ -857,13 +867,16 @@ def test_reranking_cross_encoder_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_cross_encoder_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import CrossEncoderReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "cross-encoder-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -873,18 +886,20 @@ def test_reranking_cross_encoder_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = CrossEncoderReranker() # Run vector search with a reranker result = tbl.search("hello").rerank(reranker=reranker).to_list() + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + # Run FTS search with a reranker result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) result = ( tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() ) @@ -936,13 +951,16 @@ def test_reranking_linear_combination_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_linear_combination_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import LinearCombinationReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "linear-combination-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -952,7 +970,7 @@ def test_reranking_linear_combination_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = LinearCombinationReranker() @@ -1007,13 +1025,16 @@ def test_reranking_rrf_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_rrf_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import RRFReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "rrf-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -1023,7 +1044,7 @@ def test_reranking_rrf_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = RRFReranker() @@ -1039,13 +1060,16 @@ def test_reranking_mrr_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_mrr_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import MRRReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "mrr-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -1055,7 +1079,7 @@ def test_reranking_mrr_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = MRRReranker(weight_vector=0.7, weight_fts=0.3) @@ -1390,9 +1414,12 @@ def test_frameworks_pydantic_examples() -> None: {"text": "goodbye world", "vector": [0.0, 1.0]}, ] ) - results = table.search("hello world").limit(1).to_pydantic(LanceDocs) + results = table.search([1.0, 0.0]).limit(1).to_pydantic(LanceDocs) print(results[0].text) + # hello world # --8<-- [end:frameworks_pydantic_base_example] + assert isinstance(results[0], LanceDocs) + assert results[0].text == "hello world" # Platform integrations @@ -1686,7 +1713,7 @@ def test_platforms_pandas_examples() -> None: # --8<-- [end:platforms_pandas_vector_search] # --8<-- [start:platforms_pandas_async_example] - async def run_pandas_async_example() -> None: + async def run_pandas_async_example() -> pd.DataFrame: async_db = await lancedb.connect_async( str(Path(tempfile.mkdtemp()) / "pandas-async") ) @@ -1699,16 +1726,16 @@ def test_platforms_pandas_examples() -> None: async_table = await async_db.create_table( "creatures_async", data=async_df, mode="overwrite" ) - async_results = await ( - async_table.search([0.6, 0.4, 0.8]) - .select(["text", "_distance"]) - .limit(1) - .to_pandas() - ) + # An async search returns its query builder from an await + query = await async_table.search([0.6, 0.4, 0.8]) + async_results = await query.select(["text", "_distance"]).limit(1).to_pandas() print(async_results) + return async_results - asyncio.run(run_pandas_async_example()) + async_results = asyncio.run(run_pandas_async_example()) # --8<-- [end:platforms_pandas_async_example] + assert async_results["text"].tolist() == ["sage"] + assert async_results["_distance"].tolist() == [0.0] def test_platforms_polars_examples() -> None: diff --git a/docs/web-tests/py/test_provider_examples.py b/docs/web-tests/py/test_provider_examples.py new file mode 100644 index 000000000..e1836b6ff --- /dev/null +++ b/docs/web-tests/py/test_provider_examples.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""The provider reranker examples write only to a directory they create. + +Each example runs offline. A stand-in embedding function answers for the model +the example asks the registry for, a stand-in reranker replaces the provider +client, and outbound connections are refused, so no model is downloaded and no +provider is called. Sentinel tables in the owned home directory, where these +examples used to write, and in the temporary directory must be left unchanged. +""" + +import importlib.util +import socket +import tempfile +from pathlib import Path +from types import SimpleNamespace +from typing import ClassVar + +import lancedb +import lancedb.embeddings +import lancedb.rerankers +import pyarrow as pa +import pytest +from lancedb.embeddings import TextEmbeddingFunction, register +from lancedb.rerankers import Reranker + +EXAMPLES = Path(__file__).with_name("provider_examples.py") +RERANKERS = { + "reranking_cohere_usage": "CohereReranker", + "reranking_jina_usage": "JinaReranker", + "reranking_watsonx_usage": "WatsonxReranker", + "reranking_openai_usage": "OpenaiReranker", + "reranking_voyageai_usage": "VoyageAIReranker", +} + + +@register("docs-offline-text") +class OfflineEmbedding(TextEmbeddingFunction): + """Deterministic four-dimensional vectors, computed locally.""" + + def ndims(self): + return 4 + + def generate_embeddings(self, texts): + return [[float(len(text)), 1.0, 0.0, 0.0] for text in texts] + + +class OfflineReranker(Reranker): + """Scores every row 1.0 and records which searches it reranked.""" + + instances: ClassVar[list["OfflineReranker"]] = [] + + def __init__(self, **kwargs): + super().__init__() + self.kwargs = kwargs + self.searches = [] + OfflineReranker.instances.append(self) + + def _score(self, results, search): + self.searches.append(search) + drop = [c for c in ("_distance", "_score") if c in results.column_names] + results = results.drop_columns(drop) + scores = pa.array([1.0] * results.num_rows, pa.float32()) + return results.append_column("_relevance_score", scores) + + def rerank_vector(self, query, vector_results): + return self._score(vector_results, "vector") + + def rerank_fts(self, query, fts_results): + return self._score(fts_results, "fts") + + def rerank_hybrid(self, query, vector_results, fts_results): + merged = self.merge_results(vector_results, fts_results) + return self._score(merged, "hybrid") + + +def snapshot(connect, uri): + db = connect(str(uri)) + tables = {} + for name in db.list_tables().tables: + table = db.open_table(name) + tables[name] = (table.version, table.schema, table.to_arrow().to_pylist()) + return tables + + +def load_examples(): + spec = importlib.util.spec_from_file_location("provider_examples", EXAMPLES) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def owned(tmp_path, monkeypatch): + home, temp_root = tmp_path / "home", tmp_path / "tmp" + home.mkdir() + temp_root.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setattr(tempfile, "tempdir", str(temp_root)) + monkeypatch.chdir(tmp_path) + # Anything the examples resolve from "~" must land in the owned home. + assert Path("~/.lancedb").expanduser().resolve().is_relative_to(home.resolve()) + + def refuse(*args, **kwargs): + raise OSError("network access is disabled in this test") + + monkeypatch.setattr(socket.socket, "connect", refuse) + monkeypatch.setattr(socket, "create_connection", refuse) + + real_connect = lancedb.connect + sentinels = [home / ".lancedb", temp_root / "existing"] + for uri in sentinels: + row = {"text": "keep me", "vector": [0.5, 0.5, 0.5, 0.5]} + real_connect(str(uri)).create_table("test", [row]) + before = [snapshot(real_connect, uri) for uri in sentinels] + + registry = lancedb.embeddings.get_registry() + monkeypatch.setattr(registry, "get", lambda name: OfflineEmbedding) + for name in RERANKERS.values(): + monkeypatch.setattr(lancedb.rerankers, name, OfflineReranker) + monkeypatch.setenv("JINA_API_KEY", "replaced-by-the-example") + OfflineReranker.instances.clear() + + connected = [] + + def recording_connect(uri, *args, **kwargs): + connected.append(Path(uri).expanduser().resolve()) + return real_connect(uri, *args, **kwargs) + + monkeypatch.setattr(lancedb, "connect", recording_connect) + return SimpleNamespace( + home=home, + temp_root=temp_root, + sentinels=sentinels, + before=before, + connect=real_connect, + connected=connected, + ) + + +@pytest.mark.parametrize("example", sorted(RERANKERS)) +def test_reranker_example_writes_only_to_its_own_directory(owned, example): + existing = set(owned.temp_root.iterdir()) + + getattr(load_examples(), example)() + + after = [snapshot(owned.connect, uri) for uri in owned.sentinels] + assert after == owned.before + created = set(owned.temp_root.iterdir()) - existing + assert len(created) == 1 + (own_directory,) = created + assert len(owned.connected) == 1 + assert owned.connected[0].is_relative_to(own_directory.resolve()) + assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"] + + (reranker,) = OfflineReranker.instances + assert reranker.searches == ["vector", "fts", "hybrid"] + assert reranker.kwargs["api_key"] == "your-api-key" + assert reranker.kwargs.get("project_id", "your-project-id") == "your-project-id" diff --git a/docs/web-tests/py/test_reranker_examples.py b/docs/web-tests/py/test_reranker_examples.py new file mode 100644 index 000000000..71505d423 --- /dev/null +++ b/docs/web-tests/py/test_reranker_examples.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""The published reranker examples in test_integrations.py write only to a directory they create. + +Each example runs as readers see it: its marked region is extracted the way the snippet +generator extracts it, saved as a script and run offline. A stand-in embedding function answers for the +sentence-transformers model, stand-in rerankers replace the model-backed AnswerDotAI, ColBERT +and cross-encoder rerankers, and outbound connections are refused. The RRF, linear-combination +and MRR rerankers need no model and run unchanged. Sentinel tables in the owned home directory, +where these examples used to write, and in the temporary directory must be left unchanged. +""" + +import importlib.util +import runpy +import sys +from pathlib import Path + +import lancedb.rerankers +import pytest +import test_provider_examples as provider + +owned = provider.owned +SOURCE = Path(__file__).with_name("test_integrations.py") +GENERATOR = Path(__file__).parents[1] / "mdx_snippets_gen.py" +MODEL_RERANKERS = { + "reranking_answerdotai_usage": "AnswerdotaiRerankers", + "reranking_colbert_usage": "ColbertReranker", + "reranking_cross_encoder_usage": "CrossEncoderReranker", +} +LOCAL_RERANKERS = [ + "reranking_linear_combination_usage", + "reranking_mrr_usage", + "reranking_rrf_usage", +] + + +def load_generator(): + if "mdx_snippets_gen" not in sys.modules: + spec = importlib.util.spec_from_file_location("mdx_snippets_gen", GENERATOR) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module # its dataclasses resolve annotations by module + spec.loader.exec_module(module) + return sys.modules["mdx_snippets_gen"] + + +def reader_code(name): + """The example exactly as the snippet generator publishes it.""" + generator = load_generator() + start, end = generator.markers_for_lang("py") + lines = SOURCE.read_text().splitlines(keepends=True) + snippet = generator.parse_snippets(lines, start, end)[name] + return generator.snippet_text(generator.dedent(snippet)) + + +def run_example(owned, tmp_path, name): + script = tmp_path / "scripts" / f"{name}.py" + script.parent.mkdir(exist_ok=True) + script.write_text(reader_code(name)) + existing = set(owned.temp_root.iterdir()) + namespace = runpy.run_path(str(script)) + + after = [provider.snapshot(owned.connect, uri) for uri in owned.sentinels] + assert after == owned.before + created = set(owned.temp_root.iterdir()) - existing + assert len(created) == 1 + (own_directory,) = created + assert len(owned.connected) == 1 + assert owned.connected[0].is_relative_to(own_directory.resolve()) + assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"] + return namespace + + +@pytest.mark.parametrize("name", sorted(MODEL_RERANKERS)) +def test_model_reranker_example_writes_only_to_its_own_directory( + owned, monkeypatch, tmp_path, name +): + monkeypatch.setattr( + lancedb.rerankers, MODEL_RERANKERS[name], provider.OfflineReranker + ) + + namespace = run_example(owned, tmp_path, name) + + (reranker,) = provider.OfflineReranker.instances + assert reranker.searches == ["vector", "fts", "hybrid"] + assert namespace["result"] + + +@pytest.mark.parametrize("name", LOCAL_RERANKERS) +def test_local_reranker_example_writes_only_to_its_own_directory(owned, tmp_path, name): + namespace = run_example(owned, tmp_path, name) + + assert namespace["result"] + assert all("_relevance_score" in row for row in namespace["result"]) + if name == "reranking_mrr_usage": + assert namespace["combined"].num_rows > 0 diff --git a/docs/web-tests/py/test_search_examples.py b/docs/web-tests/py/test_search_examples.py new file mode 100644 index 000000000..241a0f771 --- /dev/null +++ b/docs/web-tests/py/test_search_examples.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + + +def test_vector_search(): + # --8<-- [start:vector_search] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + "vector": [1.0, 0.0], + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + "vector": [0.9, 0.1], + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + "vector": [0.0, 1.0], + }, + ], + ) + rows = ( + table.search([1.0, 0.0]) + .distance_type("l2") + .select(["id", "text", "_distance"]) + .limit(2) + .to_list() + ) + print(rows) + # --8<-- [end:vector_search] + + assert [row["id"] for row in rows] == [1, 2] + assert rows[0]["_distance"] == 0.0 + + +def test_text_search(): + # --8<-- [start:text_search] + from tempfile import TemporaryDirectory + + import lancedb + from lancedb.index import FTS + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + }, + ], + ) + table.create_index("text", config=FTS()) + rows = ( + table.search("puppy", query_type="fts", fts_columns="text") + .select(["id", "text", "_score"]) + .limit(2) + .to_list() + ) + print(rows) + # --8<-- [end:text_search] + + assert [row["id"] for row in rows] == [1] + assert rows[0]["_score"] > 0 + + +def test_hybrid_search(): + # --8<-- [start:hybrid_search] + from tempfile import TemporaryDirectory + + import lancedb + from lancedb.index import FTS + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + "vector": [1.0, 0.0], + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + "vector": [0.9, 0.1], + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + "vector": [0.0, 1.0], + }, + ], + ) + from lancedb.rerankers import RRFReranker + + table.create_index("text", config=FTS()) + rows = ( + table.search( + query_type="hybrid", vector_column_name="vector", fts_columns="text" + ) + .vector([1.0, 0.0]) + .text("puppy") + .rerank(RRFReranker()) + .limit(2) + .to_list() + ) + print(rows) + # --8<-- [end:hybrid_search] + + assert [row["id"] for row in rows] == [1, 2] + assert rows[0]["_relevance_score"] > rows[1]["_relevance_score"] + + +def test_filter_search(): + # --8<-- [start:filter_search] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + "vector": [1.0, 0.0], + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + "vector": [0.9, 0.1], + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + "vector": [0.0, 1.0], + }, + ], + ) + rows = ( + table.search([0.0, 1.0]) + .where("category = 'animals'", prefilter=True) + .select(["id", "text", "category", "_distance"]) + .limit(2) + .to_list() + ) + print(rows) + # --8<-- [end:filter_search] + + assert {row["id"] for row in rows} == {1, 2} + assert all(row["category"] == "animals" for row in rows) + + +def test_query_plan(): + # --8<-- [start:query_plan] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + "vector": [1.0, 0.0], + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + "vector": [0.9, 0.1], + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + "vector": [0.0, 1.0], + }, + ], + ) + query = table.search([1.0, 0.0]).select(["id", "text", "_distance"]).limit(2) + print("Plan:", query.explain_plan()) + print("Measured execution:", query.analyze_plan()) + rows = query.to_list() + print(rows) + # --8<-- [end:query_plan] + + assert [row["id"] for row in rows] == [1, 2] diff --git a/docs/web-tests/rs/Cargo.lock b/docs/web-tests/rs/Cargo.lock index 915e15e58..447bae7f6 100644 --- a/docs/web-tests/rs/Cargo.lock +++ b/docs/web-tests/rs/Cargo.lock @@ -5455,6 +5455,7 @@ dependencies = [ "arrow-schema", "futures", "futures-util", + "lance-index", "lance-namespace", "lancedb", "polars", diff --git a/docs/web-tests/rs/Cargo.toml b/docs/web-tests/rs/Cargo.toml index 6b68f5fff..8bc594c3c 100644 --- a/docs/web-tests/rs/Cargo.toml +++ b/docs/web-tests/rs/Cargo.toml @@ -10,6 +10,7 @@ arrow-schema = "58.3.0" futures = "0.3.31" futures-util = "0.3.31" lance-namespace = "8.0.0" +lance-index = "=8.0.0" lancedb = { version = "0.31.0", features = ["polars", "remote"] } polars = ">=0.37,<0.40.0" polars-arrow = ">=0.37,<0.40.0" @@ -41,3 +42,35 @@ path = "multimodal.rs" [[example]] name = "search" path = "search.rs" + +[[example]] +name = "ingestion_examples" +path = "ingestion_examples.rs" + +[[example]] +name = "ingestion_delete" +path = "ingestion_delete.rs" + +[[example]] +name = "ingestion_progress" +path = "ingestion_progress.rs" + +[[example]] +name = "search_vector_search" +path = "search_vector_search.rs" + +[[example]] +name = "search_text_search" +path = "search_text_search.rs" + +[[example]] +name = "search_hybrid_search" +path = "search_hybrid_search.rs" + +[[example]] +name = "search_filter_search" +path = "search_filter_search.rs" + +[[example]] +name = "search_query_plan" +path = "search_query_plan.rs" diff --git a/docs/web-tests/rs/ingestion_delete.rs b/docs/web-tests/rs/ingestion_delete.rs new file mode 100644 index 000000000..f23b04f2d --- /dev/null +++ b/docs/web-tests/rs/ingestion_delete.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example] +use std::sync::Arc; + +use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase}; + +fn make_users_reader(rows: &[(i64, &str)]) -> Box { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))), + Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))), + ], + ) + .unwrap(); + Box::new(RecordBatchIterator::new( + vec![Ok(batch)].into_iter(), + schema, + )) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let table = db + .create_table( + "users", + make_users_reader(&[(1, "Alice"), (2, "Bob"), (3, "Charlie")]), + ) + .execute() + .await?; + let predicate = "id = 3"; + let selected: Vec = table + .query() + .only_if(predicate) + .execute() + .await? + .try_collect() + .await?; + println!("Selected: {selected:?}"); + table.delete(predicate).await?; + let rows: Vec = table.query().execute().await?.try_collect().await?; + println!("Remaining: {rows:?}"); + assert_eq!(table.count_rows(None).await?, 2); + assert_eq!( + table + .count_rows(Some("id = 1 AND name = 'Alice'".into())) + .await?, + 1 + ); + assert_eq!( + table + .count_rows(Some("id = 2 AND name = 'Bob'".into())) + .await?, + 1 + ); + assert_eq!(table.count_rows(Some("id = 3".into())).await?, 0); + Ok(()) +} +// --8<-- [end:example] diff --git a/docs/web-tests/rs/ingestion_examples.rs b/docs/web-tests/rs/ingestion_examples.rs new file mode 100644 index 000000000..84748c1b9 --- /dev/null +++ b/docs/web-tests/rs/ingestion_examples.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:write_workflow] +use std::sync::Arc; + +use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase}; + +fn make_users_reader(rows: &[(i64, &str, i64)]) -> Box { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), + Field::new("login_count", DataType::Int64, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))), + Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))), + Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.2))), + ], + ) + .unwrap(); + Box::new(RecordBatchIterator::new( + vec![Ok(batch)].into_iter(), + schema, + )) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let table = db + .create_table( + "users", + make_users_reader(&[(1, "Alice", 10), (2, "Bob", 20)]), + ) + .execute() + .await?; + table + .add(make_users_reader(&[(3, "Charlie", 5)])) + .execute() + .await?; + table + .update() + .only_if("id = 2") + .column("name", "'Bobby'") + .execute() + .await?; + table + .update() + .only_if("id = 2") + .column("login_count", "login_count + 1") + .execute() + .await?; + let mut merge = table.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all(); + merge + .execute(make_users_reader(&[(3, "Charlie", 6), (4, "Dana", 1)])) + .await?; + let selected: Vec = table + .query() + .only_if("id = 4") + .execute() + .await? + .try_collect() + .await?; + println!("Rows selected for deletion: {selected:?}"); + table.delete("id = 4").await?; + let rows: Vec = table.query().execute().await?.try_collect().await?; + println!("Remaining rows: {rows:?}"); + assert_eq!(table.count_rows(None).await?, 3); + assert_eq!( + table + .count_rows(Some( + "id = 2 AND name = 'Bobby' AND login_count = 21".into() + )) + .await?, + 1 + ); + assert_eq!( + table + .count_rows(Some("id = 3 AND login_count = 6".into())) + .await?, + 1 + ); + Ok(()) +} +// --8<-- [end:write_workflow] diff --git a/docs/web-tests/rs/ingestion_progress.rs b/docs/web-tests/rs/ingestion_progress.rs new file mode 100644 index 000000000..afeb4c2ad --- /dev/null +++ b/docs/web-tests/rs/ingestion_progress.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example] +use std::sync::Arc; + +use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::ExecutableQuery; + +fn make_users_reader(rows: &[(i64, &str)]) -> Box { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))), + Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))), + ], + ) + .unwrap(); + Box::new(RecordBatchIterator::new( + vec![Ok(batch)].into_iter(), + schema, + )) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let table = db + .create_table("users", make_users_reader(&[(1, "Alice")])) + .execute() + .await?; + table + .add(make_users_reader(&[(2, "Bob"), (3, "Charlie")])) + .progress(|progress| println!("Rows written: {}", progress.output_rows())) + .execute() + .await?; + let rows: Vec = table.query().execute().await?.try_collect().await?; + println!("After insertion: {rows:?}"); + assert_eq!(table.count_rows(None).await?, 3); + assert_eq!( + table + .count_rows(Some("id = 1 AND name = 'Alice'".into())) + .await?, + 1 + ); + assert_eq!( + table + .count_rows(Some("id = 2 AND name = 'Bob'".into())) + .await?, + 1 + ); + assert_eq!(table.count_rows(Some("id = 3".into())).await?, 1); + Ok(()) +} +// --8<-- [end:example] diff --git a/docs/web-tests/rs/search_filter_search.rs b/docs/web-tests/rs/search_filter_search.rs new file mode 100644 index 000000000..0417b5b15 --- /dev/null +++ b/docs/web-tests/rs/search_filter_search.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + Arc::new( + FixedSizeListArray::from_iter_primitive::( + [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] + .into_iter() + .map(|v| Some(v.into_iter().map(Some).collect::>())), + 2, + ), + ), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + let query = table + .vector_search(&[0.0_f32, 1.0])? + .only_if("category = 'animals'") + .select(Select::columns(&["id", "text", "category", "_distance"])) + .limit(2); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!( + ids.into_iter().collect::>(), + std::collections::BTreeSet::from([1, 2]) + ); + assert!(rows.iter().all(|b| { + b.column_by_name("category") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .all(|v| v == Some("animals")) + })); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/rs/search_hybrid_search.rs b/docs/web-tests/rs/search_hybrid_search.rs new file mode 100644 index 000000000..6a57c259c --- /dev/null +++ b/docs/web-tests/rs/search_hybrid_search.rs @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lance_index::scalar::FullTextSearchQuery; +use lancedb::index::{Index, scalar::FtsIndexBuilder}; +use lancedb::query::{ExecutableQuery, QueryBase}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + Arc::new( + FixedSizeListArray::from_iter_primitive::( + [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] + .into_iter() + .map(|v| Some(v.into_iter().map(Some).collect::>())), + 2, + ), + ), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + table + .create_index(&["text"], Index::FTS(FtsIndexBuilder::default())) + .execute() + .await?; + let query = table + .vector_search(&[1.0_f32, 0.0])? + .full_text_search(FullTextSearchQuery::new("puppy".into()).with_column("text".into())?) + .rerank(Arc::new(lancedb::rerankers::rrf::RRFReranker::default())) + .limit(2); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(ids, vec![1, 2]); + let scores: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("_relevance_score") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert!(scores[0] > scores[1]); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/rs/search_query_plan.rs b/docs/web-tests/rs/search_query_plan.rs new file mode 100644 index 000000000..97290fff1 --- /dev/null +++ b/docs/web-tests/rs/search_query_plan.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + Arc::new( + FixedSizeListArray::from_iter_primitive::( + [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] + .into_iter() + .map(|v| Some(v.into_iter().map(Some).collect::>())), + 2, + ), + ), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + let query = table + .vector_search(&[1.0_f32, 0.0])? + .select(Select::columns(&["id", "text", "_distance"])) + .limit(2); + println!("Plan: {}", query.explain_plan(false).await?); + println!("Measured execution: {}", query.analyze_plan().await?); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(ids, vec![1, 2]); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/rs/search_text_search.rs b/docs/web-tests/rs/search_text_search.rs new file mode 100644 index 000000000..ebb4b0292 --- /dev/null +++ b/docs/web-tests/rs/search_text_search.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lance_index::scalar::FullTextSearchQuery; +use lancedb::index::{Index, scalar::FtsIndexBuilder}; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + table + .create_index(&["text"], Index::FTS(FtsIndexBuilder::default())) + .execute() + .await?; + let query = table + .query() + .full_text_search(FullTextSearchQuery::new("puppy".into()).with_column("text".into())?) + .select(Select::columns(&["id", "text", "_score"])) + .limit(2); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(ids, vec![1]); + let scores: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("_score") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert!(scores[0] > 0.0); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/rs/search_vector_search.rs b/docs/web-tests/rs/search_vector_search.rs new file mode 100644 index 000000000..f2807ebef --- /dev/null +++ b/docs/web-tests/rs/search_vector_search.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + Arc::new( + FixedSizeListArray::from_iter_primitive::( + [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] + .into_iter() + .map(|v| Some(v.into_iter().map(Some).collect::>())), + 2, + ), + ), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + let query = table + .vector_search(&[1.0_f32, 0.0])? + .distance_type(lancedb::DistanceType::L2) + .select(Select::columns(&["id", "text", "_distance"])) + .limit(2); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(ids, vec![1, 2]); + let scores: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("_distance") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(scores[0], 0.0); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/ts/ingestion_examples.test.ts b/docs/web-tests/ts/ingestion_examples.test.ts new file mode 100644 index 000000000..4c6b88412 --- /dev/null +++ b/docs/web-tests/ts/ingestion_examples.test.ts @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors +import { expect, test } from "@jest/globals"; + +test("complete ingestion workflow", async () => { + // --8<-- [start:write_workflow_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + + const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("users", [ + { id: 1, name: "Alice", login_count: 10 }, + { id: 2, name: "Bob", login_count: 20 }, + ]); + await table.add([{ id: 3, name: "Charlie", login_count: 5 }]); + await table.update({ where: "id = 2", values: { name: "Bobby" } }); + await table.update({ + where: "id = 2", + valuesSql: { login_count: "login_count + 1" }, + }); + await table + .mergeInsert("id") + .whenMatchedUpdateAll() + .whenNotMatchedInsertAll() + .execute([ + { id: 3, name: "Charlie", login_count: 6 }, + { id: 4, name: "Dana", login_count: 1 }, + ]); + console.log( + "Rows selected for deletion:", + await table.query().where("id = 4").toArray(), + ); + await table.delete("id = 4"); + const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id); + console.log(rows); + // --8<-- [end:write_workflow_body] + expect(rows.map((row) => [row.id, row.name, row.login_count])).toEqual([ + [1, "Alice", 10], + [2, "Bobby", 21], + [3, "Charlie", 6], + ]); + // --8<-- [start:write_workflow_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:write_workflow_tail] +}); + +test("complete delete_rows", async () => { + // --8<-- [start:delete_rows_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("users", [ + { id: 1, name: "Alice" }, + { id: 2, name: "Bob" }, + { id: 3, name: "Charlie" }, + ]); + const predicate = "id = 3"; + console.log("Selected:", await table.query().where(predicate).toArray()); + await table.delete(predicate); + const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id); + console.log("Remaining:", rows); + // --8<-- [end:delete_rows_body] + expect(rows.map((row) => [row.id, row.name])).toEqual([ + [1, "Alice"], + [2, "Bob"], + ]); + // --8<-- [start:delete_rows_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:delete_rows_tail] +}); + +test("complete add_progress", async () => { + // --8<-- [start:add_progress_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("users", [{ id: 1, name: "Alice" }]); + await table.add( + [ + { id: 2, name: "Bob" }, + { id: 3, name: "Charlie" }, + ], + { + progress: (progress) => + console.log(`Rows written: ${progress.outputRows}`), + }, + ); + const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id); + console.log("After insertion:", rows); + // --8<-- [end:add_progress_body] + expect(rows.map((row) => [row.id, row.name])).toEqual([ + [1, "Alice"], + [2, "Bob"], + [3, "Charlie"], + ]); + // --8<-- [start:add_progress_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:add_progress_tail] +}); diff --git a/docs/web-tests/ts/search_examples.test.ts b/docs/web-tests/ts/search_examples.test.ts new file mode 100644 index 000000000..be4c388ee --- /dev/null +++ b/docs/web-tests/ts/search_examples.test.ts @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { expect, test } from "@jest/globals"; + +test("complete vector_search", async () => { + // --8<-- [start:vector_search_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { + id: 1, + text: "A puppy playing in the park", + category: "animals", + vector: [1.0, 0.0], + }, + { + id: 2, + text: "A dog resting at home", + category: "animals", + vector: [0.9, 0.1], + }, + { + id: 3, + text: "A guide to growing tomatoes", + category: "gardening", + vector: [0.0, 1.0], + }, + ]); + const rows = await table + .vectorSearch([1.0, 0.0]) + .distanceType("l2") + .select(["id", "text", "_distance"]) + .limit(2) + .toArray(); + console.log(rows); + // --8<-- [end:vector_search_body] + expect(rows.map((row) => row.id)).toEqual([1, 2]); + expect(rows[0]._distance).toBe(0); + // --8<-- [start:vector_search_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:vector_search_tail] +}); + +test("complete text_search", async () => { + // --8<-- [start:text_search_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { id: 1, text: "A puppy playing in the park", category: "animals" }, + { id: 2, text: "A dog resting at home", category: "animals" }, + { id: 3, text: "A guide to growing tomatoes", category: "gardening" }, + ]); + await table.createIndex("text", { config: lancedb.Index.fts() }); + const rows = await table + .query() + .fullTextSearch("puppy", { columns: ["text"] }) + .select(["id", "text", "_score"]) + .limit(2) + .toArray(); + console.log(rows); + // --8<-- [end:text_search_body] + expect(rows.map((row) => row.id)).toEqual([1]); + expect(rows[0]._score).toBeGreaterThan(0); + // --8<-- [start:text_search_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:text_search_tail] +}); + +test("complete hybrid_search", async () => { + // --8<-- [start:hybrid_search_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { + id: 1, + text: "A puppy playing in the park", + category: "animals", + vector: [1.0, 0.0], + }, + { + id: 2, + text: "A dog resting at home", + category: "animals", + vector: [0.9, 0.1], + }, + { + id: 3, + text: "A guide to growing tomatoes", + category: "gardening", + vector: [0.0, 1.0], + }, + ]); + await table.createIndex("text", { config: lancedb.Index.fts() }); + const reranker = await lancedb.rerankers.RRFReranker.create(); + const rows = await table + .vectorSearch([1.0, 0.0]) + .fullTextSearch("puppy", { columns: ["text"] }) + .rerank(reranker) + .limit(2) + .toArray(); + console.log(rows); + // --8<-- [end:hybrid_search_body] + expect(rows.map((row) => row.id)).toEqual([1, 2]); + expect(rows[0]._relevance_score).toBeGreaterThan(rows[1]._relevance_score); + // --8<-- [start:hybrid_search_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:hybrid_search_tail] +}); + +test("complete filter_search", async () => { + // --8<-- [start:filter_search_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { + id: 1, + text: "A puppy playing in the park", + category: "animals", + vector: [1.0, 0.0], + }, + { + id: 2, + text: "A dog resting at home", + category: "animals", + vector: [0.9, 0.1], + }, + { + id: 3, + text: "A guide to growing tomatoes", + category: "gardening", + vector: [0.0, 1.0], + }, + ]); + const rows = await table + .vectorSearch([0.0, 1.0]) + .where("category = 'animals'") + .select(["id", "text", "category", "_distance"]) + .limit(2) + .toArray(); + console.log(rows); + // --8<-- [end:filter_search_body] + expect(rows.map((row) => row.id).sort()).toEqual([1, 2]); + expect(rows.every((row) => row.category === "animals")).toBe(true); + // --8<-- [start:filter_search_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:filter_search_tail] +}); + +test("complete query_plan", async () => { + // --8<-- [start:query_plan_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { + id: 1, + text: "A puppy playing in the park", + category: "animals", + vector: [1.0, 0.0], + }, + { + id: 2, + text: "A dog resting at home", + category: "animals", + vector: [0.9, 0.1], + }, + { + id: 3, + text: "A guide to growing tomatoes", + category: "gardening", + vector: [0.0, 1.0], + }, + ]); + const query = table + .vectorSearch([1.0, 0.0]) + .select(["id", "text", "_distance"]) + .limit(2); + console.log("Plan:", await query.explainPlan()); + console.log("Measured execution:", await query.analyzePlan()); + const rows = await query.toArray(); + console.log(rows); + // --8<-- [end:query_plan_body] + expect(rows.map((row) => row.id)).toEqual([1, 2]); + // --8<-- [start:query_plan_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:query_plan_tail] +}); diff --git a/docs/web/indexing/fts-index.mdx b/docs/web/indexing/fts-index.mdx index 4e18d5a86..832b67315 100644 --- a/docs/web/indexing/fts-index.mdx +++ b/docs/web/indexing/fts-index.mdx @@ -3,6 +3,13 @@ title: "Full-Text Search (FTS) Index" sidebarTitle: "FTS Index" description: "Create and tune BM25-based full-text search indexes in LanceDB." icon: "book" +availability: + oss: available + enterprise: available + summary: >- + Full-text indexes are built in the embedded library and in LanceDB Enterprise. + The embedded library builds the index before the call returns; a + deployment builds it in the background. --- import { PyFtsIndexAsync as FtsIndexAsync, PyFtsIndexCreate as FtsIndexCreate, PyFtsIndexNested as FtsIndexNested, PyFtsIndexWait as FtsIndexWait } from '/snippets/indexing.mdx'; @@ -114,7 +121,7 @@ await async_table.create_index("payload.text", config=FTS(with_position=True)) - `jieba/*` is for Chinese word segmentation with Jieba. - `lindera/*` loads a compiled Lindera dictionary, such as `lindera/ipadic` for Japanese or `lindera/ko-dic` for Korean. -Model-backed tokenizers such as `jieba/default`, `lindera/ipadic`, and `lindera/ko-dic` require tokenizer model files in Lance's language model home. Lance looks under the default platform data directory for `lance/language_models`, or you can set `LANCE_LANGUAGE_MODEL_HOME` to point to another model root. For example, `jieba/default` is resolved under `/jieba/default/...`. +Model-backed tokenizers such as `jieba/default`, `lindera/ipadic`, and `lindera/ko-dic` require tokenizer model files in Lance's language model home. Lance looks under the default platform data directory for `lance/language_models`, or you can set `LANCE_LANGUAGE_MODEL_HOME` to point to another model root. For example, `jieba/default` is resolved under `/jieba/default/...`. For a table in LanceDB Enterprise, the files must be present where the deployment builds and queries the index, so ask its administrator whether they are installed. `language` is used by token filters, not by the base tokenizer. Stemming supports Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. Built-in stop-word removal supports Danish, Dutch, English, Finnish, French, German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish, and Swedish. For other stemming languages, set `remove_stop_words=False` or pass `custom_stop_words`. diff --git a/docs/web/indexing/index.mdx b/docs/web/indexing/index.mdx index b40fb7918..78844527f 100644 --- a/docs/web/indexing/index.mdx +++ b/docs/web/indexing/index.mdx @@ -3,20 +3,26 @@ title: "Indexing Data" sidebarTitle: "Overview" description: "Optimize search performance with LanceDB using vector indexes, full-text search, scalar indexes, and more." icon: "list" +availability: + oss: available + enterprise: available + summary: >- + Indexes are built in the embedded library and in LanceDB Enterprise. Enterprise builds them in the background, and builds fewer vector index types than the + embedded library. --- -An **index** is a data structure that facilitates efficient scans and lookups on an embedded dataset. -LanceDB provides a comprehensive suite of indexes to optimize performance across different use cases and data types: +Indexes speed up searches and filters by organizing data for specific query +patterns. LanceDB supports three main index families: - **Vector Index**: Efficiently searches for similar vectors across high-dimensional data (e.g. images, audio, or text embeddings) - **Full-Text Search Index**: Enables fast keyword-based searches by indexing words and phrases -- **Scalar Index**: Accelerates filtering and sorting of structured numeric or categorical data +- **Scalar Index**: Accelerates filtering of structured numeric or categorical data ## Supported Indexes | Index | Use Case | Description | | :--------- | :------- | :---------- | -| `IVF` (Vector) | Large-scale vector search with configurable accuracy/speed trade-offs. | Inverted File Index—a partition-based approximate nearest neighbor algorithm that groups similar vectors into partitions for efficient search.
**Quantizations**: `None/Flat` `PQ` `SQ` `RQ`| +| `IVF` (Vector) | Large-scale vector search with configurable accuracy/speed trade-offs. | Inverted File Index: a partition-based approximate nearest neighbor algorithm that groups similar vectors into partitions for efficient search.
**Quantizations**: `None/Flat` `PQ` `SQ` `RQ`| | `IVF_HNSW` (Vector) | Large-scale vector search requiring both high recall and efficient partitioning. Combines the scalability of IVF with the search quality of HNSW. | Hybrid index combining IVF partitioning with HNSW graphs in each partition. Provides improved search quality over pure IVF while maintaining scalability.
**Quantizations**: `None/Flat` `SQ` `PQ`| | `FTS` (Full-text search) | String columns (e.g., title, description, content) requiring keyword-based search with BM25 ranking. | Full-text search index using BM25 ranking algorithm. Tokenizes text with configurable tokenization, stemming, stop word removal, and language-specific processing. | | `BTree` (Scalar) | Numeric, temporal, and string columns with mostly distinct values. Best for selective equality, inequality, and range predicates. | Sorted index storing sorted copies of scalar columns with block headers in a btree cache. Header entries map to blocks of rows (4096 rows per block) for efficient disk reads. | @@ -24,6 +30,10 @@ LanceDB provides a comprehensive suite of indexes to optimize performance across | `LabelList` (Scalar) | List columns (e.g., tags, categories, keywords) requiring `array_contains_all` or `array_contains_any` filters. | Scalar index for `List` and `LargeList` columns of primitive values, using an underlying bitmap index structure to enable fast array membership lookups. | | `FM` (Scalar) | String or binary columns that need raw substring search. | FM-Index over `Utf8`, `LargeUtf8`, `Binary`, or `LargeBinary` data for filters such as `contains(path, 'needle')`. Use FTS instead for tokenized word search and BM25 ranking. | +A deployment builds the vector indexes `IVF_FLAT`, `IVF_PQ`, `IVF_RQ` and +`IVF_HNSW_SQ`. `IVF_SQ`, `IVF_HNSW_FLAT` and `IVF_HNSW_PQ` are built only by the +embedded library. + ## Quantization LanceDB also supports several [quantization](/indexing/vector-index) methods, used by vector indexes to compress vectors and reduce storage requirements: @@ -32,26 +42,30 @@ LanceDB also supports several [quantization](/indexing/vector-index) methods, us | `PQ` (Product Quantization) | Default choice for most vector search scenarios. Use when you need to balance index size and recall. | Divides vectors into subvectors and quantizes each subvector independently. Provides a good balance between compression ratio and search accuracy. | | `RQ` (RabitQ Quantization) | Use when you need maximum compression or have specific per-dimension requirements. | Per-dimension quantization using a RabitQ codebook. Provides fine-grained control over compression per dimension. For `IVF_RQ`, vector dimensions must be divisible by `8`. | | `SQ` (Scalar Quantization) | Use when you need faster indexing or when vector dimensions have consistent value ranges. | Quantizes each dimension independently. Simpler than PQ but typically provides less compression. | -| `None/Flat` | Use for binary vectors (with `hamming` distance) or when you need maximum recall and have sufficient storage. | No quantization—stores raw vectors. Provides the highest accuracy but requires more storage and memory. | +| `None/Flat` | Use for binary vectors (with `hamming` distance) or when you need maximum recall and have sufficient storage. | No quantization; stores raw vectors. Provides the highest accuracy but requires more storage and memory. | ## Performance {#performance} #### Vector indexes {#vector-indexes} -If vector search latency climbs with table size (i.e., queries that ran in milliseconds on a small table take seconds as it grows to millions of rows), the cause is the default brute-force scan over every vector. That works fine below ~100K vectors, but past that you should build a dedicated vector index. Pick the type by your data shape: +If vector search latency climbs with table size (i.e., queries that ran in milliseconds on a small table take seconds as it grows to millions of rows), check whether the query is scanning every vector without an index. The point at which an index helps depends on vector dimensions, hardware and +your latency target. Measure your workload before choosing an index. Pick the type by your data shape: | Index | When to use | |-------|-------------| -| `IVF_PQ` | General-purpose default; what Enterprise builds automatically. Higher accuracy than RQ at small dimensions (≤ 256). | +| `IVF_PQ` | General-purpose default, and what a deployment builds when it is set to index a column automatically. Higher accuracy than RQ at small dimensions (≤ 256). | | `IVF_RQ` | Maximum compression on high-dim vectors, faster builds than PQ. | | `IVF_HNSW_SQ` | Best recall/latency for unfiltered search; higher latency variance under selective filters. | | `IVF_FLAT` | Required for binary vectors with `hamming`. | -The distance metric is fixed once the index is built. Pick the distance metric based on how the embedding model was trained: `cosine` (unnormalized), `dot` (already-normalized, best performance), `l2` (general-purpose, default), `hamming` (binary). If a query explicitly requests a different metric, the index cannot be used and the query falls back to a brute-force scan; LanceDB Enterprise may reject that scan on a large table. For parameter tuning, see [Vector Indexing](/indexing/vector-index). +The distance metric is fixed once the index is built. Pick the distance metric based on how the embedding model was trained: `cosine` (unnormalized), `dot` (already-normalized, best performance), `l2` (general-purpose, default), `hamming` (binary). If a query explicitly requests a different metric, the index cannot be used and the query falls back to a brute-force scan; a deployment can be configured to reject that scan on a large table. For parameter tuning, see [Vector Indexing](/indexing/vector-index). #### Scalar indexes {#scalar-indexes} -If filtered queries slow down as the table grows — even when the filter is selective — the cause is a full column scan: without a scalar index, LanceDB evaluates the `where(...)` predicate on every row, and the same applies to `merge_insert` join keys. The best practice is to build a scalar index on every column you filter or join on, picking the type by the column's shape: +Scalar indexes can reduce the rows scanned for selective filters and speed up +`merge_insert` key lookups. Start with columns used frequently in those operations, +then use [query plans](/search/explain) to check whether the index helps. Choose +the index type based on the column's values: | Index | Best for | |-------|----------| @@ -63,4 +77,6 @@ See [Scalar Indexing](/indexing/scalar-index). #### Full-text search {#full-text-search} -If your full-text index is much larger than expected, or takes longer than expected to build, the cause is usually phrase-query flags being enabled when they aren't needed: `with_position=True` and `remove_stop_words=False` both significantly inflate index size and build time. The best practice is to keep the defaults for most workloads, and only enable those flags when you actually need to search for phrases. See [FTS Indexing](/indexing/fts-index) configuration options for the full set of options. +Token positions and retained stop words increase the information stored in a +full-text index. Keep the defaults for basic keyword search; enable +`with_position=True` and retain the needed stop words for phrase queries. See [FTS Indexing](/indexing/fts-index) configuration options for the full set of options. diff --git a/docs/web/indexing/reindexing.mdx b/docs/web/indexing/reindexing.mdx index dd50f62fe..fdbfe3b5f 100644 --- a/docs/web/indexing/reindexing.mdx +++ b/docs/web/indexing/reindexing.mdx @@ -4,6 +4,14 @@ sidebarTitle: "Build & Repair" description: "Learn how to keep your indexes up-to-date in LanceDB using incremental indexing, including best practices for adding new records without full reindexing." icon: "refresh" keywords: ["reindexing", "re-index", "incremental indexing", "reindex"] +availability: + oss: available + enterprise: available + summary: >- + Indexes are kept up to date in the embedded library and in LanceDB Enterprise. In + the embedded library you add new rows to them with `optimize()`; a + deployment does it in the background, and `optimize()` is not available + there. --- import { @@ -20,16 +28,19 @@ As data is being added and a reindex operation is running, LanceDB will combine Rather than dropping an existing index entirely and reindexing from scratch, LanceDB supports **incremental indexing**. -**Enterprise job types:** building an index runs as `create_ivf_pq`, `create_btree`, -`create_fts`, and one more per index algorithm -- the type names the algorithm, -not a separate feature. Folding new rows into an existing one runs as the -matching `reindex_*` type instead. +**Enterprise job types:** an index build runs as `reindex_` followed by the +algorithm, such as `reindex_ivf_pq`, `reindex_btree` or `reindex_fts`, and so +does folding new rows into an existing index. A create-only build, one that must +not replace an existing index, runs as `create_` followed by the algorithm +instead; Python's `create_scalar_index` and `create_fts_index` ask for that by +default. The name says which algorithm is building, not a separate feature. ## Incremental Reindexing {#incremental-reindexing} -You can manually trigger an incremental indexing operation on updated data -using the `optimize()` method on a table. +In the embedded library, you trigger an incremental indexing operation on +updated data with the `optimize()` method on a table. A deployment does this +for you and does not support `optimize()`, as described below. Table optimization performs three maintenance operations: @@ -45,10 +56,12 @@ Table optimization performs three maintenance operations: Enterprise -LanceDB Enterprise support incremental reindexing through an automated background process. When new data is added to a table, the system automatically triggers a new index build. As the dataset grows, indexes are asynchronously updated in the background. +LanceDB Enterprise supports incremental reindexing through an automated background process. Once enough new rows have been added to a table, the system automatically adds them to its indexes. As the dataset grows, indexes are asynchronously updated in the background. -- While indexes are being rebuilt, queries use brute force methods on unindexed rows, which may temporarily increase latency. To avoid this, set `fast_search=True` to search only indexed data. +- While indexes are being rebuilt, queries use brute force methods on unindexed rows, which may temporarily increase latency. If you can omit unindexed rows, set `fast_search=True` to search only indexed data. + This can return incomplete results while indexing catches up. - Use `index_stats()` to view the number of unindexed rows. This will be zero when indexes are fully up-to-date. If you call `wait_for_index(...)`, it polls the same status and can time out while continuous writes keep adding unindexed rows. +- Rows below the deployment's threshold stay unindexed until more arrive, so the count can stay above zero without further writes. `FM` indexes are not updated in the background. The benefit of using LanceDB Enterprise is that it automates the reindexing process and operates continuously in the background, minimizing the impact on latency under high loads. @@ -71,9 +84,9 @@ If you need to reclaim space more aggressively in OSS, use a shorter retention w ## Building on GPUs {#gpu} -With LanceDB's GPU-powered vector indexing you can index very large datasets in far less time -than you could with the default CPU-based indexing. In our tests, LanceDB -is capable of indexing billions of rows in under four hours on a 1-8 GPU cluster. +GPU acceleration can reduce the time needed to build vector indexes on large +datasets. Actual build time depends on the index configuration, vector +dimensions, storage throughput and available GPU resources. **Automatic GPU indexing** @@ -82,9 +95,8 @@ is capable of indexing billions of rows in under four hours on a 1-8 GPU cluster Automatic GPU Indexing is currently only available in [LanceDB Enterprise](/enterprise/). Please [contact us](https://www.lancedb.com/contact) to enable this feature for your deployment. -The vector index is created when you call `create_index`. The backend will use GPU resources -to build either the IVF or HNSW indexes. The system automatically selects the optimal GPU -configuration based on your data size and available hardware. +The vector index is created when you call `create_index`. The backend uses GPU resources +to build `IVF_PQ` indexes, with the number of GPUs set by the deployment's configuration. This process is also asynchronous by default, but you can use `wait_for_index` to convert it into a synchronous process by waiting until the index is built. @@ -93,9 +105,9 @@ into a synchronous process by waiting until the index is built. `wait_for_index(...)` waits for the index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`. It can time out if the table is receiving continuous writes while the build is trying to catch up. -GPU acceleration changes how the vector index is built, not the lifecycle after the build. Rows -appended later still need `optimize()` before they are part of the index, and `fast_search()` only -searches indexed rows. +GPU acceleration changes how the vector index is built, not the lifecycle after the build. In the +embedded library, rows appended later still need `optimize()` before they are part of the index; a +deployment adds them in the background. `fast_search()` only searches indexed rows. ### Manual GPU indexing in LanceDB OSS {#gpu-manual-gpu-indexing-in-lancedb-oss} @@ -125,9 +137,9 @@ to enable GPU training on your device. ### Performance considerations {#gpu-performance-considerations} - GPU memory usage scales with `num_partitions` and vector dimensions -- For optimal performance, ensure GPU memory exceeds dataset size -- Batch size is automatically tuned based on available GPU memory -- Indexing speed improves with larger batch sizes +- Size GPU memory for the training batches and index configuration +- Monitor memory use while choosing training batch sizes +- Measure build time and recall before changing training settings ### Troubleshooting {#gpu-troubleshooting} diff --git a/docs/web/indexing/scalar-index.mdx b/docs/web/indexing/scalar-index.mdx index e8dcf6bb7..9f1f9a176 100644 --- a/docs/web/indexing/scalar-index.mdx +++ b/docs/web/indexing/scalar-index.mdx @@ -3,6 +3,13 @@ title: "Scalar Indexes" sidebarTitle: "Scalar Index" description: "Learn how to use scalar indexes in LanceDB for efficient metadata filtering and query optimization." icon: "tree" +availability: + oss: available + enterprise: available + summary: >- + Scalar indexes are built in the embedded library and in LanceDB Enterprise. A + deployment builds them in the background and keeps most of them up to date + itself; in the embedded library you update them with `optimize()`. --- import { @@ -19,7 +26,9 @@ import { PyScalarIndexNestedFields as ScalarIndexNestedFields, } from '/snippets/indexing.mdx'; -Scalar indexes organize data by scalar attributes (e.g., numbers, categories) and enable fast filtering of vector data. They accelerate retrieval of scalar data associated with vectors, thus enhancing query performance. +Scalar indexes accelerate filters on values such as numbers, categories and +strings. Use them for selective table queries or metadata filters combined +with vector and full-text search. LanceDB supports four types of scalar indexes: @@ -29,7 +38,7 @@ LanceDB supports four types of scalar indexes: - `FM`: FM-Index over string or binary columns that accelerates substring search via `contains(col, 'needle')`. -Scalar indices serve as a foundational optimization layer, accelerating filtering across diverse search workloads. They can be combined with: +Use scalar indexes with: - Vector search (prefilter results using metadata) - Full-text search (combining keyword matching with structured filters) @@ -70,11 +79,11 @@ If you are using LanceDB Enterprise, the `create_scalar_index` API returns immed -`wait_for_index(...)` waits until the named scalar indexes exist and `index_stats(...)` reports `num_unindexed_rows == 0`. If a table is receiving steady writes, that fully indexed state may not stabilize before the timeout. +`wait_for_index(...)` waits until the named scalar indexes exist and `index_stats(...)` reports `num_unindexed_rows == 0`. If a table is receiving steady writes, that fully indexed state may not stabilize before the timeout. In LanceDB Enterprise, rows added after the index was built are indexed only once enough have accumulated, so a small remainder can also keep the wait from finishing. ### 3. Update the Index -Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index. +Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. In the embedded library, call `optimize`, which will trigger an update to the existing scalar index. @@ -83,7 +92,7 @@ Updating the table data (adding, deleting, or modifying records) requires that y -New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Enterprise automates the optimize process, minimizing the impact on search speed. +New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Enterprise does not support `optimize()`; instead it adds new rows to `BTREE`, `BITMAP` and `LABEL_LIST` indexes in the background once enough have accumulated. It does not update `FM` indexes automatically. ### 4. Run Indexed Searches @@ -125,7 +134,7 @@ Nested paths follow Lance field-path semantics: dot-separate each struct field f The `FM` index is a scalar index built over string or binary columns that accelerates substring lookups expressed as `contains(col, 'needle')`. Unlike the tokenized [FTS index](/indexing/fts-index), which matches whole words after -tokenization, the FM-Index matches arbitrary substrings of the raw bytes — so it +tokenization, the FM-Index matches arbitrary substrings of the raw bytes. It works well for URLs, file paths, identifiers, log lines, or any column where you search for a fragment rather than a word. @@ -143,6 +152,8 @@ or language-aware tokenization. Build an FM-Index with the async `create_index` API by passing the `Fm` config in Python or `Index.fm()` in TypeScript. In Rust, use `Index::Fm(FmIndexBuilder::default())`. +A deployment builds FM-Indexes too, but does not add new rows to them in the background as it +does for the other scalar indexes. ```python Python icon="python" diff --git a/docs/web/indexing/vector-index.mdx b/docs/web/indexing/vector-index.mdx index b10da3f6f..e9768efa6 100644 --- a/docs/web/indexing/vector-index.mdx +++ b/docs/web/indexing/vector-index.mdx @@ -3,6 +3,14 @@ title: "Vector Indexes" sidebarTitle: "Vector Index" description: "Build and manage LanceDB vector indexes." icon: "arrow-up-right-dots" +availability: + oss: available + enterprise: available + summary: >- + Vector indexes are built in the embedded library and in LanceDB Enterprise. The + embedded library builds an index before `create_index()` returns; a + deployment builds it in the background, keeps it up to date, and builds + only `IVF_FLAT`, `IVF_PQ`, `IVF_RQ` and `IVF_HNSW_SQ`. --- import { @@ -53,7 +61,7 @@ Lance with `index_type`. | Maximum compression | `IVF_RQ` | IVF indexing with RaBitQ quantization | Around `1/32` of raw size | `IvfRq` | | Higher accuracy at small dimensions (`dimension <= 256`) | `IVF_PQ` | IVF indexing with product quantization | Usually `1/64` to `1/16` of raw size (depends on `num_sub_vectors`) | `IvfPq` | | Best recall/latency trade-off | `IVF_HNSW_SQ` | IVF-HNSW indexing with scalar | Typically a little larger than `1/4` of raw size | `IvfHnswSq` | -| Highest recall / no quantization | `IVF_HNSW_FLAT` | IVF-HNSW indexing with no quantization | Around raw vector size plus HNSW graph overhead | `IvfHnswFlat` | +| Highest recall / no quantization | `IVF_HNSW_FLAT` | IVF-HNSW indexing with no quantization; embedded library only | Around raw vector size plus HNSW graph overhead | `IvfHnswFlat` | If your vector search frequently includes metadata filters (`where(...)`), use `IVF_RQ` or `IVF_PQ`. In filtered workloads, HNSW-backed IVF indexes such as `IVF_HNSW_FLAT` and `IVF_HNSW_SQ` can show higher latency variance. @@ -114,7 +122,7 @@ your destination. This iterative series of road networks mimics the layered grap In LanceDB, HNSW is not exposed as a top-level vector index. Instead, it's available as a substructure which further indexes the selected vectors inside each IVF partition. This combines the scalability of IVF with the high recall of HNSW. -LanceDB supports IVF-HNSW-based quantized indexes `IVF_HNSW_FLAT`, `IVF_HNSW_PQ`, and `IVF_HNSW_SQ`. +LanceDB supports IVF-HNSW-based quantized indexes `IVF_HNSW_FLAT`, `IVF_HNSW_PQ`, and `IVF_HNSW_SQ`; a deployment builds only `IVF_HNSW_SQ`. @@ -141,7 +149,7 @@ Learn how to configure, build, and search LanceDB vector indexes, including buil | `limit` | Number of results to return (the `k` in `k-ANN`). | | `nprobes` | Sets both `minimum_nprobes` and `maximum_nprobes` to the same value in Lance. Leave unset to use Lance's adaptive defaults. At very high scales you will likely need to experiment to find your preferred recall / latency balance. | | `minimum_nprobes` | Minimum number of partitions scanned. Leave unset to use Lance's default. | -| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results — useful for narrow filters. Leave unset, or set to `0`, to remove the cap. | +| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results, which is useful for narrow filters. Leave unset, or set to `0`, to remove the cap. | | `ef` | HNSW search-time exploration factor. Start around `1.5 * k` and increase up to `10 * k` for higher recall. | | `refine_factor` | Reads and reranks additional candidates in memory to recover recall lost to quantization. Start between `5` and `50` (a factor of `25` reads and reranks the top `25 * k` candidates) and measure the latency cost, typically a few milliseconds, against the recall gained. | | `distance_range(lower_bound, upper_bound)` | Return only rows whose distance falls within `[lower_bound, upper_bound)`. Either bound is optional. Useful for near-duplicate detection or "close-enough" matching. | @@ -152,8 +160,8 @@ The `distance_type` can be specified at search time in addition to build time. I you do this, set it to the same value used to build the vector index. If the search metric does not match the index metric, LanceDB cannot use the index and falls back to a costly flat search. On LanceDB Cloud, the brute-force KNN safety limit -can reject that search on a large table instead of executing it. LanceDB -Enterprise doesn't have the limit. +can reject that search on a large table instead of executing it. An Enterprise +deployment applies the same limit only when it is configured to. When the search metric is omitted, current SDKs use the index metric if an index is available. Some older SDK versions sent `l2` when it was omitted, so explicitly @@ -173,6 +181,8 @@ older client. **Filtered queries and adaptive `nprobes`.** When a `where(...)` filter is active, LanceDB initially scans `minimum_nprobes` partitions and uses a wider scan if too few rows are found. Set `minimum_nprobes == maximum_nprobes` or call `nprobes(n)` to instead fix the partition count. +Whether a deployment widens the scan this way depends on its version; one that does not scans exactly +`minimum_nprobes` partitions. Here is an example of a vector search exercising several of the above parameters. @@ -211,7 +221,7 @@ Multivector indexing currently requires `distance_type="cosine"`. Use `bypass_ve #### Async API and Config Objects Create vector indexes asynchronously with `await table.create_index("vector", config=...)`. The `config` object -admits the same build-time parameters described above — pass an instance of a Python config class from the table in [Choosing the Right Index](#choosing-the-right-index): +admits the same build-time parameters described above. Pass an instance of a Python config class from the table in [Choosing the Right Index](#choosing-the-right-index): @@ -234,7 +244,7 @@ Specify `vector_column_name` if you have multiple vector columns or non-default TypeScript currently doesn't support `IvfSq` (IVF with Scalar Quantization). -For a vector field nested inside a struct, pass its full dotted path as `vector_column_name` (e.g. `image.embedding`) — the same path is used at query time and is what `list_indices()` reports under `columns`. See [Selecting the vector column](/search/vector-search#selecting-the-vector-column) for the full path syntax. +For a vector field nested inside a struct, pass its full dotted path as `vector_column_name` (e.g. `image.embedding`). The same path is used at query time and is what `list_indices()` reports under `columns`. See [Selecting the vector column](/search/vector-search#selecting-the-vector-column) for the full path syntax. @@ -265,17 +275,18 @@ The snippet below builds and queries an `IVF_HNSW_SQ` index. Enterprise-only In LanceDB Enterprise, vector indexes are managed **automatically**. The system asynchronously updates and optimizes indexes as a background process: - Automatically manages indexing parameters and storage -- Infers vector columns from the schema -- GPU-based indexing is available on request -- [contact support](/deployment/support) to enable it for your deployment. A few billion vectors index in under 4 hours on GPU. +- Can build an `IVF_PQ` index automatically on a vector column the deployment is configured to index +- GPU-based `IVF_PQ` builds are available on request -- [contact support](/deployment/support) to enable them for your deployment. Build time depends on dataset size, dimensions and GPU resources. Open-Source LanceDB OSS users can manually create vector indexes by calling `table.create_index()`. See the above sections for guidance on manually tuning index parameters as data changes. -`create_index()` returns immediately, but the vector index builds asynchronously. -To wait until all data is indexed, specify the `wait_timeout` parameter, or call `wait_for_index(...)` afterward — -it waits for the named index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`. +In LanceDB Enterprise, `create_index()` returns immediately and the vector index builds asynchronously. +To wait until all data is indexed, specify the `wait_timeout` parameter, or call `wait_for_index(...)` afterward. +It waits for the named index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`. +In the embedded library the index is built before `create_index()` returns. @@ -287,11 +298,12 @@ fallback and searches only indexed rows. **Operational checks** -After appends or other writes, use `optimize()` to fold new rows into existing indexes. +In the embedded library, use `optimize()` after appends or other writes to fold new rows into existing indexes. +A deployment does this in the background, and `optimize()` is not available there. #### Check Index Status -Vector index creation runs in the background and may take some time to complete. +In LanceDB Enterprise, vector index creation runs in the background and may take some time to complete. While it is ongoing, you can check its status through the API or the **LanceDB Enterprise UI**. In the LanceDB Enterprise UI, navigate to your table page - the "Index" column reflects each column's index status: it is blank when no index exists, shows an "in progress" label while the index is being built, and shows the index type once the build completes. @@ -318,7 +330,7 @@ Each entry returned by `list_indices()` also carries detailed per-index metadata -These fields are populated for local and embedded tables. On LanceDB Enterprise remote tables they are returned as `None` / `undefined` until the server response surfaces them. +These fields are populated for local tables. In LanceDB Enterprise, `list_indices()` returns them except `num_unindexed_rows`; use `index_stats()` for index coverage. #### Custom Index Names @@ -343,9 +355,10 @@ compactly store an index. #### Product Quantization (PQ) To visualize PQ, assume a vector dataset has $d$ dimensions, with a **chunk** of a vector denoting a contiguous block of entries. Imagine that each vector has - $m$ disjoint chunks of $d/m$ entries each, where the first chunk represents entries $0$ through $d-1$, the second contains entries $d$ through $2d - 1$, and so on. + $m$ disjoint chunks of $d/m$ entries each, where the first chunk contains entries $0$ through $d/m-1$, the second contains +entries $d/m$ through $2d/m-1$, and so on. -Now, for each $i$, let $S_i$ represent the set containing chunk $i$ of each vector (entries $(i-1)d$ to $di - 1$). +Now, for each $i$, let $S_i$ represent the set containing chunk $i$ of each vector (entries $(i-1)d/m$ to $id/m-1$). For each $S_i$ independently, a small set of **centroids** is computed corresponding to an approximate solution to the $k$-means clustering problem. For each original vector in the dataset, every chunk is associated to its nearest centroid; thus, the vector itself is associated with the concatenation of $m$ centroids. @@ -397,7 +410,7 @@ among all integral multiples $\{j \times 10^5: -128 \leq j \leq 127 \}$. | Parameter | Description | | :--- | :--- | -| `num_bits` | Bits per dimension for quantization. Only applies to `IVF_PQ`/`IVF_HNSW_PQ` (default `8`) and `IVF_RQ` (default `1`, RaBitQ) — not used by `IVF_FLAT`/`IVF_SQ`/`IVF_HNSW_FLAT`/`IVF_HNSW_SQ`. Higher values improve accuracy at the cost of additional storage. | +| `num_bits` | Bits per dimension for quantization. Only applies to `IVF_PQ`/`IVF_HNSW_PQ` (default `8`) and `IVF_RQ` (default `1`, RaBitQ); not used by `IVF_FLAT`/`IVF_SQ`/`IVF_HNSW_FLAT`/`IVF_HNSW_SQ`. Higher values improve accuracy at the cost of additional storage. | `max_iterations` and `sample_rate` also affect quantizer training, but since they apply to every IVF/HNSW index type (not just quantized ones), they're documented as general [Build-time Parameters](/indexing/vector-index#build-time-parameters) instead. All three are passed as keyword arguments to `create_index`, alongside `index_type`: diff --git a/docs/web/ingestion/create-table.mdx b/docs/web/ingestion/create-table.mdx index fce23ac65..7d4363be7 100644 --- a/docs/web/ingestion/create-table.mdx +++ b/docs/web/ingestion/create-table.mdx @@ -1,9 +1,14 @@ --- title: "Creating a table" sidebarTitle: "Create Table" -description: "Start here: turn the data you already have into a LanceDB table." +description: "Create a table from records or a schema and choose input formats." icon: table keywords: ["create table", "ingest", "arrow", "pandas", "polars", "pydantic", "schema"] +availability: + oss: available + enterprise: available + summary: >- + Create and open tables in OSS and Enterprise. --- import { @@ -11,65 +16,60 @@ import { RsConnect, } from '/snippets/connection.mdx'; import { - PyCreateTableFromDicts as CreateTableFromDicts, TsCreateTableFromDicts as TsCreateTableFromDicts, RsCreateTableFromDicts as RsCreateTableFromDicts, - PyCreateTableConflictHandling as CreateTableConflictHandling, TsCreateTableConflictHandling as TsCreateTableConflictHandling, RsCreateTableConflictHandling as RsCreateTableConflictHandling, - PyCreateTableFromPandas as CreateTableFromPandas, - PyCreateTableCustomSchema as CreateTableCustomSchema, TsCreateTableCustomSchema as TsCreateTableCustomSchema, RsCreateTableCustomSchema as RsCreateTableCustomSchema, - PyCreateTableFromPolars as CreateTableFromPolars, - PyCreateTableFromArrow as CreateTableFromArrow, TsCreateTableFromArrow as TsCreateTableFromArrow, RsCreateTableFromArrow as RsCreateTableFromArrow, - PyCreateTableFromPydantic as CreateTableFromPydantic, - PyCreateTableNestedSchema as CreateTableNestedSchema, - PyOpenExistingTable as OpenExistingTable, TsOpenExistingTable as TsOpenExistingTable, RsOpenExistingTable as RsOpenExistingTable, - PyCreateEmptyTable as CreateEmptyTable, TsCreateEmptyTable as TsCreateEmptyTable, RsCreateEmptyTable as RsCreateEmptyTable, - PyCreateEmptyTablePydantic as CreateEmptyTablePydantic, - PyDropTable as DropTable, TsDropTable as TsDropTable, RsDropTable as RsDropTable, PyTablesBasicConnect as TablesBasicConnect, PyTablesDocumentModel as TablesDocumentModel, PyTablesTzValidator as TablesTzValidator, } from '/snippets/tables.mdx'; +import { + PyCreateTableFromDicts as CreateTableFromDicts, + PyCreateTableConflictHandling as CreateTableConflictHandling, + PyCreateTableFromPandas as CreateTableFromPandas, + PyCreateTableCustomSchema as CreateTableCustomSchema, + PyCreateTableFromPolars as CreateTableFromPolars, + PyCreateTableFromArrow as CreateTableFromArrow, + PyCreateTableFromPydantic as CreateTableFromPydantic, + PyCreateTableNestedSchema as CreateTableNestedSchema, + PyOpenExistingTable as OpenExistingTable, + PyCreateEmptyTable as CreateEmptyTable, + PyCreateEmptyTablePydantic as CreateEmptyTablePydantic, + PyDropTable as DropTable, +} from '/snippets/ingestion_examples.mdx'; -A table is created from data you already have -- a list of objects, an Arrow -table, a pandas or polars frame, Pydantic models -- or from a schema alone when -the rows come later. +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; + +Create a table from existing records, or define an empty table before data +arrives. Define a schema to specify column types, allow or disallow missing +values, and set the size of your vectors. ## Create a table with data {#create-a-table-with-data} -Initialize a LanceDB connection and create a table +The Python input examples below are complete programs. Install `lancedb` and +any additional libraries imported by the example, save one as a Python file, +and run it. Each uses its own temporary local database. - - - {TablesBasicConnect} - - - - {TsConnect} - - - - {RsConnect} - - - -Depending on the SDK, LanceDB can ingest arrays of records, Arrow tables or record batches, and Arrow batch iterators or readers. Let's take a look at some of the common patterns. +The TypeScript and Rust blocks illustrate the corresponding input formats. +For a complete connection-to-result program in either language, expand +[the write workflow](#complete-example). Table names follow the +[naming rules](/basics/naming). ### From list of objects {#from-list-of-objects} You can provide a list of objects to create a table. The Python and TypeScript SDKs -support lists/arrays of dictionaries, while the Rust SDK supports lists of structs. +support lists/arrays of records. The Rust example converts structs to Arrow batches. In Python, pass a list or other batch-like object; a single bare `dict` or single `LanceModel` is rejected. @@ -87,37 +87,42 @@ In Python, pass a list or other batch-like object; a single bare `dict` or singl -### Handle existing tables {#handle-existing-tables} +### If the table already exists {#handle-existing-tables} -By default, `create_table` raises an error if a table with the same name already exists. -You can change this behavior with two parameters that resolve the conflict in different ways: +By default, creating a table fails if its name is already in use. Choose whether +to open that table or overwrite it. -- **Idempotent open**: return the existing table without modifying it. Use when your - code may run more than once (notebooks, retries, init scripts) and you want to reuse - the table on subsequent runs. The provided data is ignored, but the schema is - validated against the existing table and a mismatch raises an error. -- **Overwrite**: drop the existing table and create a new one with the provided data. - Use this for test fixtures or when you intentionally want to replace prior contents. - This permanently discards the old table's data. +#### Open the existing table + +Use `exist_ok=True` in Python or `existOk: true` in TypeScript to return the +existing table without changing its contents. This is useful for setup code +that may run more than once. Any data supplied to the create call is ignored +when the table already exists. + +Inspect the returned table's schema if your code depends on specific columns +or types. Opening an existing table does not change its schema to match the +data supplied in the create call. + +#### Overwrite the table + +Use `mode="overwrite"` in Python or `mode: "overwrite"` in TypeScript to replace +the existing table with the supplied data and schema. Choose this only when +you intend to replace its previous contents. + +The following Python program demonstrates both choices: - {CreateTableConflictHandling} - - {TsCreateTableConflictHandling} - - - {RsCreateTableConflictHandling} - - + `exist_ok` / `existOk` does not append the provided data to an existing table. Use [`table.add()`](/ingestion/insert) for that. If you need to ensure a table exists *and* -contains specific rows, prefer the [empty-table-then-add pattern](#create-empty-table). +contains specific rows, choose an explicit [merge policy](/ingestion/merge-insert). Creating an empty table +and repeatedly appending the same batch does not deduplicate it. ### From a custom schema {#from-a-custom-schema} @@ -138,7 +143,7 @@ You can define a custom Arrow schema for the table. This is useful when you want -An explicit schema is also where you control nullability. If later writes omit a +An explicit schema also determines which columns accept missing values (`null`). If later writes omit a non-nullable column, or provide actual nulls for it, ingestion fails; nullable columns can be omitted or written with null values. Without an explicit schema, Python infers list-like vector values as fixed-size `float32` vector fields from the observed dimension. @@ -186,10 +191,8 @@ The **`vector`** column needs to be a [Vector](/integrations/data/pydantic#vecto ### From a Polars DataFrame {#from-a-polars-dataframe} Python Only -LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library -written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow -under the hood. A deeper integration between LanceDB Tables and Polars DataFrames -is on the way. +Pass a Polars DataFrame as input. Install `polars` for this example; the +conversion preserves the table structure through Arrow. @@ -263,12 +266,14 @@ for a `created_at` field. -When you run this code it, should raise the `ValidationError`. +The example catches a `ValidationError` for the timestamp with a different +timezone. A validator runs when the Pydantic model is constructed; declaring a +model as the table schema does not make every later write run that validator. -Datasets too large to pass to `create_table` at once are covered separately -- -see [Insert data](/ingestion/insert#large-datasets), which explains why -`table.add()` parallelizes where `create_table` does not. +Datasets too large to pass to `create_table` at once are covered separately; +see [Insert data](/ingestion/insert#large-datasets) for batches, file-backed +input and write parallelism. ## Create empty table {#create-empty-table} @@ -292,9 +297,8 @@ An empty table can be initialized via an Arrow schema. -Alternatively, you can also use Pydantic to specify the schema for the empty table. Note that we do not -directly import `pydantic` but instead use `lancedb.pydantic` which is a subclass of `pydantic.BaseModel` -that has been extended to support LanceDB specific types like `Vector`. +Alternatively, define the schema with `LanceModel`, a Pydantic model class +provided by `lancedb.pydantic`. Its `Vector` type declares a fixed vector dimension. @@ -342,6 +346,46 @@ Use the `drop_table()` method on the database to remove a table. -This permanently removes the table and is not recoverable, unlike deleting rows. -By default, if the table does not exist an exception is raised. To suppress this, -you can pass in `ignore_missing=True`. +Dropping a table removes the table object. Deleting rows keeps the table and +changes its contents; retained versions and cleanup are described in +[Cleanup](/maintenance/cleanup). +On a local database, dropping a table that does not exist raises an error; in +Python, pass `ignore_missing=True` to suppress it. A deployment may report +success instead, depending on how it stores its catalog, so do not rely on the +error to tell you a table was missing. Python's deployment connection does not +take `ignore_missing`. + + + +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/ingestion/delete.mdx b/docs/web/ingestion/delete.mdx index 0e2a4fdfa..b7b1f9432 100644 --- a/docs/web/ingestion/delete.mdx +++ b/docs/web/ingestion/delete.mdx @@ -1,77 +1,110 @@ --- title: "Delete rows" sidebarTitle: "Delete" -description: "Remove rows by predicate, and reclaim their space." +description: "Select and delete records, verify the remaining rows, and understand storage retention." icon: trash +availability: + oss: available + enterprise: available + summary: >- + Delete rows through the SDKs in OSS and Enterprise. Storage reclamation follows compaction and retention policy. --- -These examples use the table set up in -[Ingesting data](/ingestion), and the connection made there. +import { RsExample as RsDeleteRows } from '/snippets/ingestion_delete.mdx'; +import { TsDeleteRowsBody, TsDeleteRowsTail } from '/snippets/ingestion_examples.mdx'; +import { PyDeleteRows } from '/snippets/ingestion_examples.mdx'; +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; -import { - PyDeleteOperation as DeleteOperation, - PyUpdateOptimizeCleanup as UpdateOptimizeCleanup, - TsDeleteOperation, - TsUpdateOptimizeCleanup, - RsDeleteOperation, - RsUpdateOptimizeCleanup, -} from '/snippets/tables.mdx'; +Deleting rows removes matching records from the current table contents. It +keeps the table and schema so you can continue reading and writing. To remove +the table itself, see [Drop a table](/ingestion/create-table#drop-a-table). - -In SQL this is [`DELETE`](/sql/dml/delete). - +## Inspect the rows you intend to delete {#select-rows} -## Delete rows {#delete-rows} +Use the same predicate for inspection and deletion. The example below creates +Alice, Bob and Charlie, prints Charlie's row, and deletes it by `id = 3`. +Inspection and deletion are separate operations; concurrent writes can change +which rows match between them. -Delete operations **soft delete** rows that match a given condition. -The underlying data is not immediately removed, but is marked -for deletion (in the [deletion files](https://lance.org/format/table/#deletion-files) at the Lance format level) and excluded from query results. +## Delete matching records {#delete-rows} + +Each program creates Alice, Bob and Charlie in a temporary local database, +deletes Charlie, and prints the remaining rows: **Alice and Bob**. The temporary +database is removed when the program finishes. + +- **Python:** install `lancedb`, save the code as `delete_rows.py`, and run + `python delete_rows.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `delete_rows.mts`, + and run `node delete_rows.mts` with Node.js 22.18 or later. +- **Rust:** download the [Rust examples](/static/code/ingestion-feedback-rust.zip), + extract the project, and run `cargo run --locked --bin delete`. - - {DeleteOperation} - - - - {TsDeleteOperation} - - - - {RsDeleteOperation} - + +{PyDeleteRows} + + +{TsDeleteRowsBody} +{TsDeleteRowsTail} + + +{RsDeleteRows} + -Expected table contents: +Enterprise also supports [`DELETE`](/sql/dml/delete). -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bob | 20 | +## Verify that the rows are absent {#verify} +Query the same predicate after deletion and check the remaining row count. +A reader still using an older snapshot may see the earlier contents; see +[Consistency](/tables/consistency). - -**Deleting rows removes them from the index** +Deleted rows are excluded from current query results, including indexed +searches. Deletion does not imply immediate physical erasure of every stored +copy of the data. -When rows are deleted, those rows are also excluded from the index segments, so indexed queries will not return them either. If ALL the rows are deleted (i.e., the table is emptied), ensure that you recreate the index after ingesting new data. - +## Understand storage reclamation {#storage} -To permanently remove deleted rows, you can optimize the table, which will run compaction and cleans up the soft-deleted rows, which frees up storage space. +Compaction rewrites the live data layout; cleanup removes eligible retained +versions and files. Retained history can continue to reference data that no +longer appears in the latest table version. -- In LanceDB OSS, compaction and cleanup are manual. Run `table.optimize()` regularly to free up disk space. -- In LanceDB Enterprise, files aren't cleaned up by default. You can configure automatic compaction and cleanup behavior at cluster setup time to suit your organization's retention policy. +In OSS, `optimize()` provides maintenance operations. In Enterprise, managed +maintenance and retention follow the service configuration. Choose retention +according to the history you need, rather than shortening it as part of a +routine delete example. See [Compaction](/maintenance/compaction) and +[Cleanup](/maintenance/cleanup) for the separate maintenance workflow. -By default, table cleanup removes data up to 7 days ago. If you need to reclaim space from deleted rows more aggressively, manually call `table.optimize()` use a shorter retention window as follows: +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. - - {UpdateOptimizeCleanup} - - - - {TsUpdateOptimizeCleanup} - - - - {RsUpdateOptimizeCleanup} - + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/ingestion/index.mdx b/docs/web/ingestion/index.mdx index 2a8de6759..c37b46a62 100644 --- a/docs/web/ingestion/index.mdx +++ b/docs/web/ingestion/index.mdx @@ -1,125 +1,97 @@ --- title: "Ingesting data" sidebarTitle: "Overview" -description: "Getting data into a table: creating one from your data, and the four write operations that follow." +description: "Choose how to create a table, append records or change existing data." icon: inbox keywords: ["ingest", "create table", "add", "insert", "arrow", "pandas", "polars", "pydantic"] +availability: + oss: available + enterprise: available + summary: >- + Create tables and write records through the SDKs in OSS and Enterprise. Enterprise also provides SQL write statements. --- -import { - PyUpdateConnectEnterprise as UpdateConnectEnterprise, - PyUpdateConnectLocal as UpdateConnectLocal, - PyUpdateExampleTableSetup as UpdateExampleTableSetup, - TsUpdateConnectEnterprise, - TsUpdateConnectLocal, - TsUpdateExampleTableSetup, - RsUpdateConnectEnterprise, - RsUpdateConnectLocal, - RsUpdateExampleTableSetup, - RsUpdateMakeUsersReader, -} from '/snippets/tables.mdx'; +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; -In LanceDB, tables store records with a defined schema that specifies column -names and types. Across the SDKs, you can create tables from row-oriented data -and Apache Arrow data structures. The Python SDK additionally supports PyArrow -schemas for explicit schema control, and `LanceModel` for Pydantic-based -validation. +Ingestion brings data into a LanceDB table. Start by choosing a schema and +creating the table, then choose a write operation according to how incoming +records relate to data already stored. -Getting data in has two halves: creating a table from data you already have, and -writing to a table that exists. This page covers the first and introduces the -second. +## Choose a write operation {#choose-a-write-method} - -Every operation here has a SQL form -- see [DML](/sql#dml). - - -Start by [creating a table](/ingestion/create-table); the rest of this -section writes to one that exists. - -## Choose a write method {#choose-a-write-method} - -| Family | Method | Use this when... | -| --------------- | --------------- | ---------------- | -| [`add`](/ingestion/insert) | `add(data)` | You are appending new rows and do not need to match against existing ones. | -| `update` | `update(where=..., values=...)` | You want to edit rows that already exist, using a SQL filter. | -| `merge_insert` | `.when_matched_update_all()` | You have incoming rows and want to update keys that already exist in the table. | -| `merge_insert` | `.when_not_matched_insert_all()` | You have incoming rows and want to insert keys that do not exist yet. | -| `merge_insert` | `.when_matched_update_all()` + `.when_not_matched_insert_all()` | You want both behaviors together (often called **upsert**: update existing keys **and** insert missing keys in the same operation). | -| `merge_insert` | `.when_not_matched_by_source_delete(...)` | You want to remove target rows that are missing from the incoming source set. | - -Write operations return status metadata. For example, `update` reports updated row count and -committed `version`, `delete` reports deleted row count and `version`, and `merge_insert` reports -inserted, updated, deleted, retry-attempt, and `version` fields. Writes require a mutable table -handle; after checking out an older version for reads, call `checkout_latest` / `checkoutLatest` -before modifying data. -The committed `version` advances even for writes that affect zero rows, such as a delete predicate -that matches nothing. - -## Connect to LanceDB {#connect-to-lancedb} - -Connect to your local LanceDB instance: - - - - {UpdateConnectLocal} - - - - {TsUpdateConnectLocal} - - - - {RsUpdateConnectLocal} - - - -Or, connect to LanceDB Enterprise: - - - - {UpdateConnectEnterprise} - - - - {TsUpdateConnectEnterprise} - - - - {RsUpdateConnectEnterprise} - - - -In the Rust snippets, a `make_users_reader` helper is used to build Arrow input data. - - - - {RsUpdateMakeUsersReader} - - - -## Create the example table {#create-the-example-table} - -We'll start by creating a simple table with `id`, `name`, and `login_count` columns. All examples below use the same table. - - - {UpdateExampleTableSetup} - - - - {TsUpdateExampleTableSetup} - - - - {RsUpdateExampleTableSetup} - - - -Expected table contents: - -| id | name | login_count | +| Task | Operation | What happens | | --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bob | 20 | +| [Create a table](/ingestion/create-table) | `create_table` / `createTable` | Create a table from data or an explicit schema. | +| [Append records](/ingestion/insert) | `add` | Add rows without matching existing keys. | +| [Upsert by key](/ingestion/merge-insert) | `merge_insert` / `mergeInsert` | Choose whether matching rows are updated and new keys inserted. | +| [Update selected rows](/ingestion/update) | `update` | Change values in rows selected by a predicate. | +| [Delete selected rows](/ingestion/delete) | `delete` | Remove matching rows from the current table contents. | -The example above shows a PyArrow schema. You can just as well create the table using other -table creation patterns (Pandas, Polars, Pydantic, iterators, etc.) -- see [Create a table](/ingestion/create-table#create-a-table-with-data). +Appending the same input twice can create duplicate records. Use merge insert +with a stable key when the incoming data should update existing records. + +## Connect and prepare the data {#connect-to-lancedb} + +Use the [connection guide](/basics/connect) to connect to OSS or Enterprise. +The SDK write operations in this section work with both. Enterprise also +supports [SQL write statements](/sql#dml); an SDK filter expression does not +require a separate SQL connection. + +Choose the column types, required fields and vector dimensions before a large +load. SDKs support different input representations, including row objects and +Arrow batches. The [table creation guide](/ingestion/create-table) covers +those choices. + +## Try a complete write workflow {#create-the-example-table} + +This example starts with Alice and Bob, appends Charlie, updates Bob's name +and login count, then merges a batch containing Charlie and Dana. Finally, it +selects Dana's row for inspection and deletes it. + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + + +## Verify the result {#verify} + +After a write completes, inspect the affected keys and fields and check the row +count. SDK return values differ by operation and language; use the relevant +API reference when relying on returned counts or version metadata. + +Readers with older snapshots may not see another handle's writes immediately. +See [Consistency](/tables/consistency). A table checked out at a historical +version must return to the latest version before it can be modified. + +## Load larger datasets {#large-loads} + +Use batches or file-backed input when the dataset is too large to hold in +memory. The [insert guide](/ingestion/insert#large-datasets) covers input +formats, progress reporting and write parallelism. Keep retry behavior in mind: +repeating an append is not the same as retrying a key-based upsert. diff --git a/docs/web/ingestion/insert.mdx b/docs/web/ingestion/insert.mdx index 2d411eb3b..f2ee0d7db 100644 --- a/docs/web/ingestion/insert.mdx +++ b/docs/web/ingestion/insert.mdx @@ -1,217 +1,237 @@ --- title: "Insert data" sidebarTitle: "Insert" -description: "Append rows to a table, including datasets too large to hold in memory." +description: "Append records, load file-backed data or batches, and verify ingestion." icon: plus +availability: + oss: available + enterprise: available + summary: >- + Append records through the SDKs in OSS and Enterprise. For large Enterprise writes, see [large upload settings](#bulk-ingests-into-a-remote-table). --- -Rows are appended with `table.add()`. It accepts the same shapes -[table creation](/ingestion) does — lists of objects, Arrow tables, pandas -and polars frames, Pydantic models — and commits a new table version. - -These examples use the table set up in [Ingesting data](/ingestion). - +import { + TsCreateTableFromIterator as TsCreateTableFromIterator, + RsCreateTableFromIterator as RsCreateTableFromIterator, +} from '/snippets/tables.mdx'; import { PyAddFromDataset as AddFromDataset, PyCreateTableFromIterator as CreateTableFromIterator, - TsCreateTableFromIterator as TsCreateTableFromIterator, - RsCreateTableFromIterator as RsCreateTableFromIterator, - TsAddProgress, -} from '/snippets/tables.mdx'; +} from '/snippets/ingestion_examples.mdx'; - -In SQL this is [`INSERT`](/sql/dml/insert), including `INSERT ... SELECT` to move rows server-side. - +import { RsExample as RsAddProgress } from '/snippets/ingestion_progress.mdx'; +import { PyAddProgress, TsAddProgressBody, TsAddProgressTail } from '/snippets/ingestion_examples.mdx'; +import { PyAppendRows } from '/snippets/ingestion_examples.mdx'; +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; -## Ingesting large datasets {#large-datasets} +Use `add()` to append records to an existing table. It does not compare incoming +keys with stored keys. Choose [merge insert](/ingestion/merge-insert) when the +same record may arrive again and should update an existing row. -When ingesting large datasets, use `table.add()` on an existing table rather than -passing all data to `create_table()`. The `add()` method auto-parallelizes large -writes, while `create_table(name, data)` does not. +## Append a first batch {#append} - -For best performance with large datasets, create an empty table first and then call -`table.add()`. This enables automatic write parallelism for materialized data sources. - +Install the Python SDK with `pip install lancedb`. This complete example starts +with Alice, appends Bob and Charlie, and prints all three records. -### From files (Parquet, CSV, etc.) {#from-files-parquet-csv-etc} -Python Only + +{PyAppendRows} + -For file-based data, pass a `pyarrow.dataset.Dataset` to `table.add()`. This streams -data from disk without loading the entire dataset into memory. +The program uses temporary local storage. For a complete TypeScript or Rust +example, see [Complete write workflow](#complete-example). +Enterprise SQL offers [`INSERT`](/sql/dml/insert), including `INSERT ... SELECT`. + +## Prepare the input {#prepare} + +Match the table's schema, including required fields and vector dimensions. +Supported input formats differ by SDK; see +[Create a table](/ingestion/create-table). Inspect conversion errors before +retrying, and verify the final row count and representative values. + +## Choose input for a large load {#large-datasets} + +Avoid building one large in-memory list when records already live in files +or arrive as batches. Create a table with the intended schema, then append +through the appropriate input path. + + +### Read from files {#from-files-parquet-csv-etc} + +Python accepts a `pyarrow.dataset.Dataset`. This example writes a small Parquet +file locally, creates an empty table with its schema, and appends the two records. +For your own data, point the dataset reader at the input files instead. + + +{AddFromDataset} + + +A file-backed source can be scanned without first loading the whole dataset +into application memory. This Python input API is distinct from converting +files to Arrow batches yourself in another language. + + +### Produce batches during ingestion {#from-iterators-custom-batch-generation} + +Use batches when processing the input as it arrives. The Python example creates +an empty table and appends ten records produced by a batch iterator. - - {AddFromDataset} - + +{CreateTableFromIterator} + + +{TsCreateTableFromIterator} + + +{RsCreateTableFromIterator} + - -`pa.dataset()` input is currently Python-only. TypeScript and Rust support for -file-based dataset ingestion is tracked in -[lancedb#3173](https://github.com/lancedb/lancedb/issues/3173). - +The language APIs differ: the TypeScript example appends each constructed Arrow +batch, while the Rust example supplies an Arrow reader when creating the table. +These are input-format examples, not equivalent parallel-write benchmarks. -### From iterators (custom batch generation) {#from-iterators-custom-batch-generation} +### Control write parallelism {#write-parallelism} -When you need custom batch logic — generating embeddings on the fly, transforming -rows from an external source, etc. — use an iterator of `RecordBatch` objects. +Python's `add()` can estimate parallelism for materialized inputs. A streaming +reader may not expose its total size or support replay. Use `write_parallelism` +when controlling the number of concurrent partitions; increasing it can also +increase memory use. TypeScript does not expose that manual option. + +Measure with realistic row sizes and storage. Avoid one write per row when a +batch will do, and avoid a large worker count for a small input. + +## Track completion and progress {#tracking-ingestion-progress} + +Python, TypeScript and Rust support progress callbacks when adding data. +These examples start with Alice, append Bob and Charlie, and print the number +of rows written as progress updates arrive. After the write succeeds, they +query the table to show all three records. + +Each program uses a temporary local database and removes it when finished. + +- **Python:** install `lancedb`, save the code as `insert_progress.py`, and run + `python insert_progress.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `insert_progress.mts`, + and run `node insert_progress.mts` with Node.js 22.18 or later. +- **Rust:** download the [Rust examples](/static/code/ingestion-feedback-rust.zip), + extract the project, and run `cargo run --locked --bin progress`. - - {CreateTableFromIterator} - - - - {TsCreateTableFromIterator} - - - - {RsCreateTableFromIterator} - + +{PyAddProgress} + + +{TsAddProgressBody} +{TsAddProgressTail} + + +{RsAddProgress} + -Use this pattern when: +For a progress bar in Python, install `tqdm` and pass `progress=True` instead +of a callback. -- Your source data already arrives in Arrow batches, readers, datasets, or streams. -- Materializing the entire ingest as one giant in-memory list or array would be too expensive. -- You want to control chunk size explicitly during ingestion. +### Read progress updates -Python can also consume iterators of other supported types like Pandas DataFrames or Python lists. +Updates include row and byte counts, elapsed time, and parallel write activity. +The field names differ by language: -### Write parallelism {#write-parallelism} +| Meaning | Python dictionary key | TypeScript property | Rust method | +|:--------|:----------------------|:--------------------|:------------| +| Rows written so far | `output_rows` | `outputRows` | `output_rows()` | +| Bytes written so far | `output_bytes` | `outputBytes` | `output_bytes()` | +| Expected row count, when known | `total_rows` | `totalRows` | `total_rows()` | +| Elapsed time | `elapsed_seconds` | `elapsedSeconds` | `elapsed()` | +| Active write tasks | `active_tasks` | `activeTasks` | `active_tasks()` | +| Total write tasks | `total_tasks` | `totalTasks` | `total_tasks()` | +| Final update | `done` | `done` | `done()` | - -For materialized data (`pa.Table`, `pd.DataFrame`, `pa.dataset()`), LanceDB -automatically parallelizes large writes — no configuration needed. Auto-parallelism -targets approximately 1M rows or 2GB per write partition. +Python and TypeScript report elapsed time in seconds; Rust returns a `Duration`. +The expected row count may be unknown for streaming input. On the final update, +it falls back to the number of rows written. -For streaming sources (iterators, `RecordBatchReader`), LanceDB cannot determine -total size upfront. Python exposes `write_parallelism` on both synchronous and -asynchronous `table.add()` calls; see the [iterator example](#iterator-ingestion-for-data-you-transform-on-the-fly). -TypeScript does not expose a manual write-parallelism option. - +Keep callbacks short and use the write operation's result to determine success. +The final `done` update means progress reporting has ended; it can also occur +after a write fails or is cancelled. Input errors before writing starts may +produce no updates. -### Tracking ingestion progress {#tracking-ingestion-progress} -TypeScript example +TypeScript delivers callbacks asynchronously. Slow callbacks can queue up, +and the `add()` promise may resolve before every callback has run. Exceptions +from a TypeScript callback are logged and do not abort the write. -In TypeScript, for long-running writes, pass a `progress` callback to `table.add()` to surface -per-batch progress in your UI, logs, or metrics pipeline. Progress updates are dispatched -asynchronously to the JavaScript event loop, including a final `done: true` -update when an active write finishes. +## Verify writes and handle retries {#performance} -Each invocation receives a `WriteProgress` object: +Wait for the write operation's result, then check the expected records. +Progress callbacks describe activity; they do not establish that a write +succeeded. If a request fails after the server may have committed it, blindly +repeating an append can duplicate rows. The client's automatic retries can do +the same: on a remote table, the Python and TypeScript 0.40.0 clients resend an +append after a retryable status such as `504`, even when the server had already +committed it, and `add()` still returns success. Reconcile using your record +IDs, or choose an upsert policy designed for replay. -| Field | Description | -|:------|:------------| -| `outputRows` | Rows written so far. | -| `outputBytes` | Bytes written so far. | -| `totalRows` | Expected total rows when the input source reports one. Always set on the final callback. | -| `elapsedSeconds` | Wall-clock seconds since the write started. | -| `activeTasks` | Parallel write tasks currently in flight. | -| `totalTasks` | Total parallel write tasks (the write parallelism). | -| `done` | `true` only for the final callback. | - - - - {TsAddProgress} - - - -A few things to know before you wire this up: - -- Asynchronous delivery: slow callbacks queue updates without back-pressuring the writer. Keep the callback cheap to avoid accumulating queued updates. -- Completion: an active write emits a final `done: true` update when it finishes, including execution errors or cancellation. Errors before writing starts, such as invalid input, may produce no progress callbacks. Use the `table.add()` promise to determine whether the write succeeded; it does not wait for JavaScript to process every queued callback. -- Errors swallowed: anything your callback throws is logged with `console.warn` and won't abort the write, so keep the callback side-effect-only and don't rely on it for control flow. -- Row totals: `totalRows` is only populated when the input source can report it up front (for example, a materialized `arrow.Table`). For streaming sources it stays `undefined` until the final callback, where it falls back to the actual rows written. - -## Performance {#performance} - -If ingestion is taking longer than expected on a large dataset, the cause is almost always how `add()` is called: each call commits a new version and a new fragment, so a per-row loop pays that per-call overhead at every row. The best practice is to pick the ingestion mode that matches your data shape — bulk ingestion when the data is already materialized, or iterator ingestion when it's streamed or computed on the fly. - - -**Why `merge_insert()` is significantly slower than `add()`** - -A merge has to scan existing data to find matches on the join key (or look them up via a scalar index, if one exists), and then delete-and-reinsert any updated rows in a single transaction; `add()` simply appends new fragments. - -Use `add()` for pure appends, and reach for `merge_insert()` only when you need upsert or conditional-insert logic. When you do use it, build a scalar index on the join column first — otherwise the match step falls back to a full column scan, which is the dominant cost at scale. - - -#### Bulk ingestion: for data you already have {#bulk-ingestion-for-data-you-already-have} - -For materialized inputs (Arrow Tables, DataFrames) and file-backed sources (`pyarrow.dataset(...)`), LanceDB auto-parallelizes the write across workers, estimating the partition count from the data size — more partitions means more concurrent writes and higher throughput, up to the CPU core count. - -```python Python icon="python" -table.add(arrow_table) # in-memory -table.add(df) # pandas -table.add(ds.dataset("data/", format="parquet")) # streams from disk, still parallelized -``` - -Pass `progress=True` to watch it happen: LanceDB shows a live tqdm bar with rows written, throughput in MB/s, and active worker count. - -```python Python icon="python" -table.add(ds.dataset("data/", format="parquet"), progress=True) -``` - -```text -71%|███████▏ | 710000/1000000 [00:12<00:05, 58.8kit/s, 42.3 MB/s | 8/8 workers] -``` - -For larger-than-memory data, prefer scanning a file-backed dataset (`ds.dataset(...)`) over a hand-built `pyarrow.RecordBatchReader`: a `Dataset` can be counted and rescanned, so LanceDB knows the row count upfront (better auto-parallelism) and can retry a failed write from the start — a reader can only be consumed once, so neither is possible. Reach for the iterator path below only when the data genuinely can't be backed by files, e.g. to apply custom data transformations as you ingest. - -For very large initial loads, create the table empty first; passing data directly to `create_table(name, data)` skips the auto-parallel path. - -#### Iterator ingestion: for data you transform on the fly {#iterator-ingestion-for-data-you-transform-on-the-fly} - -If each row needs work before it can be written — applying a custom transformation as you ingest, for example — the data doesn't exist as a file you can point `ds.dataset(...)` at. The best practice is to pass an iterator of `pyarrow.RecordBatch` instead; LanceDB consumes one batch at a time as you produce them. - -```python Python icon="python" -def stream(): - for raw in source: - vectors = model.encode(raw["text"]) - yield pa.RecordBatch.from_pydict({**raw, "vector": vectors}) - -table.add(stream()) -``` - -Use decently large chunks of several thousand rows or more, rather than yielding single-row batches. - - -**Set `write_parallelism` yourself for large inputs** - -A reader can't be counted or rescanned the way a `Dataset` can (see above), so LanceDB can't auto-size parallelism for it — set `write_parallelism` explicitly for large inputs: - -```python Python icon="python" -table.add(stream(), write_parallelism=4) -``` - -Each partition becomes its own fragment, so don't over-allocate on a small input — budget one unit of parallelism per ~100K rows or ~1 GB of data as a rule of thumb. - - -#### Bulk ingests into a remote table {#bulk-ingests-into-a-remote-table} +## Configure large uploads {#bulk-ingests-into-a-remote-table} Enterprise This section applies only to remote tables (`db://` connections). Embedded (local) writes are unaffected. -When you write to a remote LanceDB Enterprise table, the client splits each write partition into one or more HTTP `insert` parts and uploads them under a single upload id. Each part is streamed rather than buffered, so peak memory stays bounded. But each part still has to finish within the client read timeout while the server writes it to object storage, so on very large ingests an oversized part can run past that timeout and surface as: +When you write to a remote LanceDB Enterprise table, the client sends the write as one HTTP `insert` request unless it splits the data into more than one partition, which it does for a large input it can measure or when you set [`write_parallelism`](#write-parallelism). A partitioned write goes up as a multipart write: the client cuts each partition into one or more `insert` parts and uploads them under a single upload id. Each part is streamed. Its size and duration still affect resource use and +timeouts: it has to finish within the client read timeout while the server writes it to object storage, so on very large ingests an oversized part can run past that timeout and surface as: ```text lancedb.remote.errors.HttpError: operation timed out ``` -Two environment variables control how parts are cut. Both are picked up automatically by the Python and TypeScript clients, and are the only way to tune this from those SDKs: +Two environment variables control how a multipart write's parts are cut. Both are picked up automatically by the Python and TypeScript clients, for clients that support these settings: | Variable | Default | What it controls | |----------|---------|------------------| | `LANCE_CLIENT_MAX_BYTES_PER_REQUEST` | 8 GiB | Maximum size of a single insert part, in LZ4-compressed Arrow IPC bytes. Set to `0` to disable splitting and send one request per partition. | | `LANCE_CLIENT_MAX_REQUEST_DURATION` | Half the read timeout | Maximum wall-clock time, in whole seconds, that any one insert part may stay open. Set to `0` to disable the time-based cut. | -A part is cut when it hits either limit, whichever comes first. Lower the byte budget when large ingests hit the read timeout; lower the duration when uploads are slow or throttled and the byte budget isn't the limiting factor. +A part is cut after the input batch that reaches either limit, so cuts fall between batches. Lower the byte budget, with smaller input batches, when large ingests hit the read timeout; lower the duration when uploads are slow or throttled and the byte budget isn't the limiting factor. ```bash export LANCE_CLIENT_MAX_BYTES_PER_REQUEST=1073741824 # 1 GiB export LANCE_CLIENT_MAX_REQUEST_DURATION=120 # 2 minutes ``` -Splitting into more parts does not change the final table. The server stages every part under the shared upload id and merges them atomically when the write completes. +Splitting into more parts does not change the final table. The server stages the parts under their shared upload ID before committing +the write. This upload protocol is separate from repeating an application-level +`add()` after a timeout. + +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/ingestion/merge-insert.mdx b/docs/web/ingestion/merge-insert.mdx index 8bfb6084a..6fa59fd9c 100644 --- a/docs/web/ingestion/merge-insert.mdx +++ b/docs/web/ingestion/merge-insert.mdx @@ -1,19 +1,16 @@ --- title: "Merge insert" sidebarTitle: "Merge Insert" -description: "Reconcile incoming rows against a key: update what matches, insert what does not." +description: "Match incoming records by key and choose which rows to update, insert or remove." icon: code-merge +availability: + oss: available + enterprise: available + summary: >- + Merge insert is available through the SDKs in OSS and Enterprise. Use it for key-based updates and inserts. --- -These examples use the table set up in -[Ingesting data](/ingestion), and the connection made there. - import { - PyMergeMatchedUpdateOnly as MergeMatchedUpdateOnly, - PyInsertIfNotExists as InsertIfNotExists, - PyMergeUpdateInsert as MergeUpdateInsert, - PyMergeDeleteMissingBySource as MergeDeleteMissingBySource, - PyMergePartialColumns as MergePartialColumns, TsMergeMatchedUpdateOnly, TsInsertIfNotExists, TsMergeUpdateInsert, @@ -25,15 +22,24 @@ import { RsMergeDeleteMissingBySource, RsMergePartialColumns, } from '/snippets/tables.mdx'; +import { + PyMergeMatchedUpdateOnly as MergeMatchedUpdateOnly, + PyInsertIfNotExists as InsertIfNotExists, + PyMergeUpdateInsert as MergeUpdateInsert, + PyMergeDeleteMissingBySource as MergeDeleteMissingBySource, + PyMergePartialColumns as MergePartialColumns, +} from '/snippets/ingestion_examples.mdx'; - -Merge insert has **no SQL form** -- it needs a source relation and a per-match policy that one statement cannot express. See [DML](/sql/dml/insert#next). - +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; + +Use merge insert when incoming records should be compared with existing rows. +Combine update and insert policies for an **upsert**. Use [append](/ingestion/insert) +when every incoming row should simply be added. ## Merge incoming rows by key {#merge-incoming-rows-by-key} Merging is different from updating because it involves comparing incoming rows to existing rows by key, and then choosing what to do based on whether the key exists in the target table or not. -The `merge_insert(""..."")` method lets you do this. +Use `merge_insert("id")` in Python or the corresponding SDK builder. In merge operations, rows are split into three groups: @@ -45,37 +51,28 @@ Conditional merge clauses can compare old and new values. Use the `target.` pref existing table row and `source.` for the incoming row, for example `target.last_update < source.last_update`. - -**Use scalar indexes to speed up merge insert** +Choose a key that identifies the same record in the source and target. Check +for duplicate and null keys before merging. A primary-key declaration does not +enforce uniqueness on ordinary writes, and append operations do not deduplicate +records for you. -The merge insert command performs a join between the input data and the target table `on` the key you provide. This requires scanning that entire column, which can be expensive for large tables. To speed up this operation, create a scalar index on the join column, which will allow LanceDB to find matches without scanning the whole table. +A scalar index on the key can help matching. Keep it current and measure the +actual workload rather than assuming every merge uses an index. Enterprise may +reject a merge when the unindexed scan exceeds its configured limit; inspect +that error and the index state before retrying. -Read more about scalar indices in the [Scalar Index](/indexing/scalar-index/) guide. - +## Choose a merge policy {#policies} - -If you see this HTTP 400 error from `merge_insert`: `Bad request: Merge insert cannot be performed because the number of unindexed rows exceeds the maximum of 10000`. Verify that the scalar index on the join column is up to date before retrying. - +| Source and target relationship | Available action | +| --- | --- | +| Key appears in both | Update matching target rows, optionally with a condition. | +| Key appears only in the source | Insert it, or leave it out. | +| Key appears only in the target | Keep it, or explicitly delete it under a synchronization policy. | - -**Rust: build merge predicates with DataFusion expressions** - -In the Rust SDK, you can pass a `datafusion_expr::Expr` directly instead of a SQL string by using -`when_matched_update_all_expr` and `when_not_matched_by_source_delete_expr` on `MergeInsertBuilder`. -This is useful when you are constructing predicates programmatically and want to avoid building a SQL string. - -These methods are only supported on local tables. Calling them against a remote table returns a -`NotSupported` error — use the SQL string variants (`when_matched_update_all` / -`when_not_matched_by_source_delete`) for remote tables. - - -Like the create table and add APIs, the merge insert API will automatically compute embeddings based on the [embedding registry](/embedding/index#embedding-registry) if the table has an embedding definition in its schema. - -During `merge_insert`, if the input data doesn't contain the source column (i.e., the original field used to generate embeddings, such as text for a text embedding model or `image_uri` for an image model), or if a vector value is already provided, LanceDB skips embedding generation for that row. Embeddings are only auto-generated when that source field is present in the incoming data, **and** the vector field is empty. - -Primary keys in LanceDB are metadata used by operations such as `merge_insert`; they are not -enforced as uniqueness constraints on ordinary writes. Keep using `merge_insert` or your own -deduplication logic when you need key-based upsert semantics. +The following examples are independent. Each starts from the table shown in +its code, so the expected results do not depend on running the previous example. +Python blocks include their own temporary connection. For a full TypeScript or +Rust program, see [Complete write workflow](#complete-example). ### Update matched rows only {#update-matched-rows-only} @@ -160,7 +157,10 @@ Expected table contents: ### Delete target rows that are missing from source {#delete-target-rows-that-are-missing-from-source} -Use `when_not_matched_by_source_delete()` when you want to remove any target row that does not appear in the incoming source data. +Use `when_not_matched_by_source_delete()` only when the incoming data is the +complete set for the target scope you intend to synchronize. An incremental +batch is usually not a complete source: deleting everything absent from it +could remove unrelated target records. Inspect the unmatched target keys first. @@ -183,7 +183,8 @@ Expected table contents: | 2 | Bobby | 21 | | 3 | Charlie | 5 | -In the example above, LanceDB matches rows by `id`. Rows with `id=2` and `id=3` exist in both the table and incoming data, so they are updated. Row `id=1` exists only in the target, so it is deleted. +Here, IDs 2 and 3 are updated and ID 1 is deleted. This policy synchronizes +the target to the supplied source set. ### Use partial columns in merge updates {#use-partial-columns-in-merge-updates} @@ -212,3 +213,54 @@ Expected table contents: | 3 | Charlie | null | Note that in the example above, when `merge_insert` creates a new row, any missing columns are written as `null`. If a missing column is non-nullable in your schema, the insert will fail. + +## Check the merged records {#verify} + +Inspect the keys that should have changed, confirm that unrelated rows remain, +and check inserted or deleted counts when the SDK returns them. Repeating a +merge is only safe when its keys, conditions and update values produce the +intended result on retry. + +### Embeddings and expression predicates {#advanced} + +When a table uses an [embedding definition](/embedding), behavior depends on +which source and vector fields the incoming records provide. Do not assume a +partial metadata update recomputes embeddings. Inspect the resulting vector +when changing the input used to generate it. + +Rust's `when_matched_update_all_expr` and +`when_not_matched_by_source_delete_expr` accept DataFusion expressions for +local tables. Use the string predicate variants for Enterprise tables. + +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/ingestion/update.mdx b/docs/web/ingestion/update.mdx index e77587ea8..d44a2561e 100644 --- a/docs/web/ingestion/update.mdx +++ b/docs/web/ingestion/update.mdx @@ -1,82 +1,123 @@ --- title: "Update rows" sidebarTitle: "Update" -description: "Change values in rows that already exist, by predicate or SQL expression." +description: "Select existing rows, change values or expressions, and verify the changes." icon: pen +availability: + oss: available + enterprise: available + summary: >- + Update rows through the SDKs in OSS and Enterprise. Enterprise also supports the SQL UPDATE statement. --- -These examples use the table set up in -[Ingesting data](/ingestion), and the connection made there. - import { - PyUpdateOperation as UpdateOperation, - PyUpdateUsingSql as UpdateUsingSql, TsUpdateOperation, TsUpdateUsingSql, RsUpdateOperation, RsUpdateUsingSql, } from '/snippets/tables.mdx'; +import { + PyUpdateOperation as UpdateOperation, + PyUpdateUsingSql as UpdateUsingSql, +} from '/snippets/ingestion_examples.mdx'; - -In SQL this is [`UPDATE`](/sql/dml/update). - +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; -## Update rows {#update-rows} +Use `update` to change rows selected by a predicate. Use +[merge insert](/ingestion/merge-insert) instead when matching an incoming dataset +against existing keys. -Use `update` when you already know which target rows to modify and you do not need to compare against an incoming dataset. +## Select the rows to change {#select-rows} + +Check the predicate with a [filtered query](/search/filtering) before applying +the update. These examples target the row with `id = 2`. Review the scope +carefully when omitting a predicate, since that can update the entire table. + +Each example below starts from Alice with 10 logins and Bob with 20. The +Python programs create and remove their own temporary database. TypeScript +and Rust operation blocks use `db`; a complete program is provided below. + +## Assign a new value {#update-rows} - - {UpdateOperation} - - - - {TsUpdateOperation} - - - - {RsUpdateOperation} - + +{UpdateOperation} + + +{TsUpdateOperation} + + +{RsUpdateOperation} + -Expected table contents: +The result contains Alice and **Bobby**, with their login counts unchanged. +The Python `values` and TypeScript `values` arguments take literal values. +Rust's `column` argument is an expression, so the string value includes SQL quotes. -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bobby | 20 | - - -Updating nested columns is not yet supported. - - -## Update rows with SQL expressions {#update-rows-with-sql-expressions} - -Use `values_sql` when you want to use SQL-like expressions to update rows. This is useful for operations like incrementing a counter, or setting a column value based on another column. +## Calculate a value from the existing row {#update-rows-with-sql-expressions} - - {UpdateUsingSql} - - - - {TsUpdateUsingSql} - - - - {RsUpdateUsingSql} - + +{UpdateUsingSql} + + +{TsUpdateUsingSql} + + +{RsUpdateUsingSql} + -Expected table contents: +In this separate example, Bob keeps his name and his login count becomes **21**. +Python's `values_sql` and TypeScript's `valuesSql` accept expressions. +These SDK expressions work with embedded OSS tables; they do not require an +Enterprise SQL endpoint. Enterprise also offers [`UPDATE`](/sql/dml/update). -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bob | 21 | +## Verify the affected data {#verify} - -See the [SQL queries](/sql#connecting) page for more information on the supported SQL syntax. - +Read the selected rows again and inspect the changed fields. For another +connection or table handle, account for its +[read consistency](/tables/consistency) settings. -When rows are updated, they are moved out of any existing index. The row will still show up in search queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider triggering an index rebuild afterwards. +Nested-field updates are not supported by these operations. For schema changes, +see [schema evolution](/management/table#schema). Treat derived values separately: +changing an input field is not a guarantee that every stored feature has been +recomputed. See [Refresh a column](/feature-engineering/refresh-column). + +Updates can change index coverage and query cost. After a large update, +inspect the workload and follow [index maintenance](/indexing/reindexing) +guidance for the backend you use. + +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/integrations/ai/hermes-agent.mdx b/docs/web/integrations/ai/hermes-agent.mdx index a5521a9d8..621b53129 100644 --- a/docs/web/integrations/ai/hermes-agent.mdx +++ b/docs/web/integrations/ai/hermes-agent.mdx @@ -29,7 +29,7 @@ prompt, plus lexical (keyword) search over past sessions. Both are useful, but k misses paraphrases of what you originally typed — the exact thing you need when recalling a fact you phrased differently months ago. -LanceDB is an embedded retrieval library, which makes it a natural fit here: +LanceDB OSS is an embedded retrieval library, which makes it a natural fit here: - **No server to stand up** — it reads and writes a table on local disk, so the plugin ships as a dependency rather than a service to operate. diff --git a/docs/web/integrations/ai/huggingface.mdx b/docs/web/integrations/ai/huggingface.mdx index 62ac7e8cd..0a38c208b 100644 --- a/docs/web/integrations/ai/huggingface.mdx +++ b/docs/web/integrations/ai/huggingface.mdx @@ -2,6 +2,12 @@ title: "Hugging Face Hub" sidebarTitle: "Hugging Face" description: "Use LanceDB directly on Lance datasets hosted on the Hugging Face Hub for multimodal search and retrieval." +availability: + oss: available + enterprise: unavailable + summary: >- + The embedded library opens Lance datasets on the Hugging Face Hub in place + through `hf://` URIs. A deployment does not read `hf://` datasets. --- [Hugging Face Hub](https://huggingface.co/datasets?format=format:lance&sort=trending) is a popular platform for sharing machine learning datasets, models, and other resources. diff --git a/docs/web/integrations/ai/langchain.mdx b/docs/web/integrations/ai/langchain.mdx index 363331093..14ed628a8 100644 --- a/docs/web/integrations/ai/langchain.mdx +++ b/docs/web/integrations/ai/langchain.mdx @@ -80,6 +80,9 @@ It returns list of ids of the added texts. {PyFrameworksLangchainAddTexts} +With LanceDB Enterprise, `tbl.to_pandas()` raises in the synchronous Python client; +read the rows with a query instead, such as `tbl.search().limit(100).to_pandas()`. + ------ diff --git a/docs/web/integrations/data/duckdb.mdx b/docs/web/integrations/data/duckdb.mdx index 704a47c60..23c4891f9 100644 --- a/docs/web/integrations/data/duckdb.mdx +++ b/docs/web/integrations/data/duckdb.mdx @@ -6,7 +6,7 @@ description: "Learn how to use the DuckDB-Lance extension to query Lance tables LanceDB integrates with [DuckDB](https://duckdb.org/) through the [Lance extension](https://github.com/lance-format/lance-duckdb) for DuckDB. In this page, we'll show how LanceDB manages table lifecycle, and DuckDB provides SQL analytics (including joins) and search over those tables. -Note that earlier versions of LanceDB used to recommend converting Lance tables to Arrow tables via `table.to_arrow()`. Although this method is still available (because DuckDB [natively scans Arrow tables](https://duckdb.org/2021/12/03/duck-arrow)), it is no longer the recommended workflow for working with Lance tables in DuckDB. This page shows how to use the Lance extension with namespace-attached LanceDB tables, allowing you to pushdown SQL queries directly to the Lance layer. +Note that earlier versions of LanceDB used to recommend converting Lance tables to Arrow tables via `table.to_arrow()`. Although this method is still available for local tables (because DuckDB [natively scans Arrow tables](https://duckdb.org/2021/12/03/duck-arrow)), it is no longer the recommended workflow for working with Lance tables in DuckDB. This page shows how to use the Lance extension with namespace-attached LanceDB tables, allowing you to pushdown SQL queries directly to the Lance layer. ## Install {#install} diff --git a/docs/web/integrations/data/pandas_and_pyarrow.mdx b/docs/web/integrations/data/pandas_and_pyarrow.mdx index 170ccf772..c646e7f3b 100644 --- a/docs/web/integrations/data/pandas_and_pyarrow.mdx +++ b/docs/web/integrations/data/pandas_and_pyarrow.mdx @@ -1,6 +1,12 @@ --- title: "Pandas and PyArrow" sidebarTitle: "Pandas & PyArrow" +availability: + oss: available + enterprise: available + summary: >- + DataFrames go into `create_table` and come back from query results the same + way for local and deployment tables. --- diff --git a/docs/web/integrations/data/polars_arrow.mdx b/docs/web/integrations/data/polars_arrow.mdx index 4cb85737e..3acfcfea5 100644 --- a/docs/web/integrations/data/polars_arrow.mdx +++ b/docs/web/integrations/data/polars_arrow.mdx @@ -1,6 +1,13 @@ --- title: "Polars" sidebarTitle: "Polars & Arrow" +availability: + oss: available + enterprise: available + summary: >- + Polars data goes into `create_table`, and query results come back as Polars + DataFrames, for local and deployment tables alike; a LazyFrame over a whole + table is local only. --- @@ -36,7 +43,7 @@ Run vector search and keep the results as a Polars `DataFrame` for further proce ## Work with LazyFrames {#work-with-lazyframes} -When you want to operate on the entire table (potentially larger than RAM), convert to a Polars `LazyFrame` so you can chain transformations without loading everything at once. +When you want to operate on the entire table (potentially larger than RAM), convert to a Polars `LazyFrame` so you can chain transformations without loading everything at once. This works for local tables only; with LanceDB Enterprise, `search(...).to_polars()` returns an eager DataFrame of query results. {PyPlatformsPolarsLazyframe} diff --git a/docs/web/integrations/data/pydantic.mdx b/docs/web/integrations/data/pydantic.mdx index b6f80aca3..3ac660774 100644 --- a/docs/web/integrations/data/pydantic.mdx +++ b/docs/web/integrations/data/pydantic.mdx @@ -1,6 +1,13 @@ --- title: "Pydantic" sidebarTitle: "Pydantic" +availability: + oss: available + enterprise: available + summary: >- + The SDK converts `LanceModel` schemas to Arrow, and query rows back to + models, in your process, so they work the same with local and deployment + tables. --- diff --git a/docs/web/integrations/embedding/cohere.mdx b/docs/web/integrations/embedding/cohere.mdx index 8051462a9..b9a8e3378 100644 --- a/docs/web/integrations/embedding/cohere.mdx +++ b/docs/web/integrations/embedding/cohere.mdx @@ -8,6 +8,10 @@ import { PyEmbeddingCohereUsage } from '/snippets/integrations.mdx'; Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification. You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API. +```shell +export COHERE_API_KEY="your-api-key" +``` + Supported models are: - embed-english-v3.0 diff --git a/docs/web/integrations/embedding/ibm.mdx b/docs/web/integrations/embedding/ibm.mdx index cec6c32bf..6dd3d57e3 100644 --- a/docs/web/integrations/embedding/ibm.mdx +++ b/docs/web/integrations/embedding/ibm.mdx @@ -3,7 +3,7 @@ title: IBM watsonx sidebarTitle: IBM WatsonX --- -import { PyEmbeddingIbmUsage } from '/snippets/integrations.mdx'; +import { PyEmbeddingIbmUsage } from '/snippets/provider_examples.mdx'; Generate text embeddings using IBM's watsonx.ai platform. @@ -54,9 +54,13 @@ Optionally set environment variables (if not passing credentials to `create` dir export WATSONX_API_KEY="YOUR_WATSONX_API_KEY" # Provide exactly one of the following: export WATSONX_PROJECT_ID="YOUR_WATSONX_PROJECT_ID" -export WATSONX_SPACE_ID="YOUR_WATSONX_SPACE_ID" +# export WATSONX_SPACE_ID="YOUR_WATSONX_SPACE_ID" ``` +Replace the credential placeholders with your provider credentials. The registry +variable keeps the API key out of stored embedding configuration. In application +code, load its value from environment variables or a secrets manager. + {PyEmbeddingIbmUsage} diff --git a/docs/web/integrations/embedding/jina.mdx b/docs/web/integrations/embedding/jina.mdx index 70457180d..588500d36 100644 --- a/docs/web/integrations/embedding/jina.mdx +++ b/docs/web/integrations/embedding/jina.mdx @@ -6,7 +6,7 @@ sidebarTitle: Jina import { PyEmbeddingJinaText, PyEmbeddingJinaMultimodal, -} from '/snippets/integrations.mdx'; +} from '/snippets/provider_examples.mdx'; ## Text Embedding Models {#text-embedding-models} @@ -23,6 +23,8 @@ Supported parameters (to be passed in `create` method) are: Usage Example: +Replace `your-api-key` with your Jina API key. This provider reads it from `JINA_API_KEY`; the example sets it explicitly. In application code, supply it through your environment or a secrets manager. + {PyEmbeddingJinaText} diff --git a/docs/web/integrations/embedding/openai.mdx b/docs/web/integrations/embedding/openai.mdx index cdd60d7cd..fbfa62d9d 100644 --- a/docs/web/integrations/embedding/openai.mdx +++ b/docs/web/integrations/embedding/openai.mdx @@ -10,9 +10,15 @@ LanceDB registers the OpenAI embeddings function in the registry by default, as | Parameter | Type | Default Value | Description | |---|---|---|---| | `name` | `str` | `"text-embedding-ada-002"` | The name of the model. | -| `dim` | `int` | Model default | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it | +| `dim` | `int` | Model default | Requested output dimensions for models that support shortening embeddings. Use `func.ndims()` when defining the vector field. | | `use_azure` | bool | `False` | Set true to use Azure OpenAI SDK | +Set your OpenAI API key before running the example: + +```shell +export OPENAI_API_KEY="your-api-key" +``` + {PyEmbeddingOpenaiBasic} diff --git a/docs/web/integrations/embedding/superlinked.mdx b/docs/web/integrations/embedding/superlinked.mdx index f35796230..bb2768d74 100644 --- a/docs/web/integrations/embedding/superlinked.mdx +++ b/docs/web/integrations/embedding/superlinked.mdx @@ -36,6 +36,9 @@ Supported parameters on `.create()`: ## Usage {#usage} ```py Python icon=Python +import tempfile +from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector @@ -47,22 +50,23 @@ sie = get_registry().get("sie").create( ) class Documents(LanceModel): + id: int text: str = sie.SourceField() vector: Vector(sie.ndims()) = sie.VectorField() -db = lancedb.connect("~/.lancedb") -table = db.create_table("docs", schema=Documents, mode="overwrite") +db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "superlinked-demo")) +table = db.create_table("docs", schema=Documents) table.add([ - {"text": "Machine learning is a subset of AI."}, - {"text": "Neural networks use multiple layers."}, - {"text": "Python is popular for ML development."}, + {"id": 1, "text": "Machine learning is a subset of AI."}, + {"id": 2, "text": "Neural networks use multiple layers."}, + {"id": 3, "text": "Python is popular for ML development."}, ]) results = table.search("What is deep learning?").limit(3).to_list() ``` -LanceDB handles embedding generation for both inserts and queries automatically, based on the `SourceField` / `VectorField` declarations on the schema. +LanceDB handles embedding generation for both inserts and queries automatically, based on the `SourceField` / `VectorField` declarations on the schema. The `id` column gives each document a stable key, which [entity extraction](#entity-extraction) uses to merge its results back. ## Hybrid search with reranker {#hybrid-search-with-reranker} @@ -88,7 +92,7 @@ for r in results: The reranker also works with pure vector or pure FTS search via `.rerank()`. ## ColBERT and multivector {#colbert-and-multivector} -`SIEMultiVectorEmbeddingFunction` (registered as `"sie-multivector"`) works with LanceDB's native `MultiVector` type and MaxSim scoring for ColBERT and ColPali models: +`SIEMultiVectorEmbeddingFunction` (registered as `"sie-multivector"`) works with LanceDB's native `MultiVector` type and MaxSim scoring for ColBERT and ColPali models. In LanceDB Python 0.40.0, a multivector that LanceDB generates on insert is stored as null unless the document's token count happens to equal the per-token dimension, so this example computes the document multivectors with `compute_source_embeddings()` and passes them in the `vector` column. LanceDB still generates the query multivector automatically: ```py Python icon=Python from lancedb.pydantic import MultiVector @@ -102,8 +106,12 @@ class ColBERTDocs(LanceModel): text: str = sie_colbert.SourceField() vector: MultiVector(sie_colbert.ndims()) = sie_colbert.VectorField() -table = db.create_table("colbert_docs", schema=ColBERTDocs, mode="overwrite") -table.add([{"text": "Machine learning is a subset of AI."}]) +table = db.create_table("colbert_docs", schema=ColBERTDocs) + +# Compute the document multivectors and store them with their text +texts = ["Machine learning is a subset of AI."] +vectors = sie_colbert.compute_source_embeddings(texts) +table.add([{"text": text, "vector": vector} for text, vector in zip(texts, vectors)]) # MaxSim search - query and document multivectors compared token-by-token results = table.search("What is ML?").limit(5).to_list() @@ -111,7 +119,7 @@ results = table.search("What is ML?").limit(5).to_list() ## Entity extraction {#entity-extraction} -`SIEExtractor` adds entity extraction to LanceDB's data-enrichment workflows. Extract entities from a text column and merge the results back as a structured Arrow column - enabling filtered search on extracted entities: +`SIEExtractor` adds entity extraction to LanceDB's data-enrichment workflows. Extract entities from a text column and merge the results back as a structured Arrow column - enabling filtered search on extracted entities. This example enriches the `docs` table from [Usage](#usage), matching rows on its `id` column: ```py Python icon=Python from sie_lancedb import SIEExtractor @@ -121,8 +129,9 @@ extractor = SIEExtractor( model="urchade/gliner_multi-v2.1", ) +docs = db.open_table("docs") extractor.enrich_table( - table, + docs, source_column="text", target_column="entities", labels=["person", "technology", "organization"], diff --git a/docs/web/integrations/embedding/voyageai.mdx b/docs/web/integrations/embedding/voyageai.mdx index 54d155d1b..8babfc214 100644 --- a/docs/web/integrations/embedding/voyageai.mdx +++ b/docs/web/integrations/embedding/voyageai.mdx @@ -11,6 +11,10 @@ Voyage AI provides cutting-edge embedding and rerankers. Using voyageai API requires voyageai package, which can be installed using `pip install voyageai`. Voyage AI embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification. You also need to set the `VOYAGE_API_KEY` environment variable to use the VoyageAI API. +```shell +export VOYAGE_API_KEY="your-api-key" +``` + Supported models are: - voyage-4-large (best retrieval quality, 1024 default dimensions, supports 256/512/1024/2048) diff --git a/docs/web/integrations/lerobotdataset.mdx b/docs/web/integrations/lerobotdataset.mdx index 08cf8bd15..cfb66b69b 100644 --- a/docs/web/integrations/lerobotdataset.mdx +++ b/docs/web/integrations/lerobotdataset.mdx @@ -143,9 +143,12 @@ class EmbedFromBlob: # SigLIP2 loads once per worker **Curation** is one query across all of it. Semantic search with a predicate on the derived score, no join, no second system: ```python -tbl.search(vec("a gripper closing on an object"), vector_column_name="emb_siglip2") \ - .where("jerk_score > 1.2875") # the roughest 1% of frames - .limit(4) +results = ( + tbl.search(vec("a gripper closing on an object"), vector_column_name="emb_siglip2") + .where("jerk_score > 1.2875") # the roughest 1% of frames + .limit(4) + .to_arrow() +) ``` **Mining** is the same query pointed at the rare cases. `tbl.search().where("jerk_score > 1.2875 AND success = false")` pulls the rough, failed frames out of 27.6M rows in milliseconds, ready to become a fine-tuning set or a labeling queue. diff --git a/docs/web/integrations/reranking/cohere.mdx b/docs/web/integrations/reranking/cohere.mdx index e484f3ee0..26b98a7c8 100644 --- a/docs/web/integrations/reranking/cohere.mdx +++ b/docs/web/integrations/reranking/cohere.mdx @@ -5,7 +5,7 @@ description: Integrate Cohere's powerful reranking API with LanceDB for enhanced --- -import { PyRerankingCohereUsage } from '/snippets/integrations.mdx'; +import { PyRerankingCohereUsage } from '/snippets/provider_examples.mdx'; # Cohere Reranker @@ -18,6 +18,8 @@ This reranker uses the [Cohere](https://cohere.ai/) API to rerank the search res pip install cohere ``` +Replace the credential placeholders with your provider credentials. In application code, load credentials from environment variables or a secrets manager. + {PyRerankingCohereUsage} diff --git a/docs/web/integrations/reranking/jina.mdx b/docs/web/integrations/reranking/jina.mdx index 4624149d2..450a18ecf 100644 --- a/docs/web/integrations/reranking/jina.mdx +++ b/docs/web/integrations/reranking/jina.mdx @@ -5,7 +5,7 @@ description: Integrate Jina's multilingual reranking API with LanceDB for improv --- -import { PyRerankingJinaUsage } from '/snippets/integrations.mdx'; +import { PyRerankingJinaUsage } from '/snippets/provider_examples.mdx'; # Jina Reranker @@ -15,6 +15,8 @@ This reranker uses the [Jina](https://jina.ai/reranker/) API to rerank the searc > **Note:** Supported query types – Hybrid, Vector, and FTS. +Replace the credential placeholders with your provider credentials. In application code, load credentials from environment variables or a secrets manager. + {PyRerankingJinaUsage} diff --git a/docs/web/integrations/reranking/openai.mdx b/docs/web/integrations/reranking/openai.mdx index 413f95e67..22d2af2cc 100644 --- a/docs/web/integrations/reranking/openai.mdx +++ b/docs/web/integrations/reranking/openai.mdx @@ -5,7 +5,7 @@ description: Explore experimental search reranking using OpenAI's GPT models in --- -import { PyRerankingOpenaiUsage } from '/snippets/integrations.mdx'; +import { PyRerankingOpenaiUsage } from '/snippets/provider_examples.mdx'; # OpenAI Reranker (Experimental) @@ -14,6 +14,9 @@ This reranker uses OpenAI chat model to rerank the search results. You can use t > **Warning:** This reranker is experimental. OpenAI does not have a dedicated reranking model, so it uses a chat model under the hood. +Replace `your-api-key` with your provider API key. In application code, load +credentials from environment variables or a secrets manager. + {PyRerankingOpenaiUsage} diff --git a/docs/web/integrations/reranking/voyageai.mdx b/docs/web/integrations/reranking/voyageai.mdx index 62e9a16da..80a666f56 100644 --- a/docs/web/integrations/reranking/voyageai.mdx +++ b/docs/web/integrations/reranking/voyageai.mdx @@ -5,7 +5,7 @@ description: Integrate VoyageAI's cutting-edge reranking models with LanceDB. Fe --- -import { PyRerankingVoyageaiUsage } from '/snippets/integrations.mdx'; +import { PyRerankingVoyageaiUsage } from '/snippets/provider_examples.mdx'; # VoyageAI Reranker @@ -17,6 +17,9 @@ This reranker uses the [VoyageAI](https://docs.voyageai.com/docs/) API to rerank > **Note:** Supported query types – Hybrid, Vector, and FTS. +Replace `your-api-key` with your provider API key. In application code, load +credentials from environment variables or a secrets manager. + {PyRerankingVoyageaiUsage} diff --git a/docs/web/integrations/reranking/watsonx.mdx b/docs/web/integrations/reranking/watsonx.mdx index 775af4f15..5e4c6322c 100644 --- a/docs/web/integrations/reranking/watsonx.mdx +++ b/docs/web/integrations/reranking/watsonx.mdx @@ -5,7 +5,7 @@ description: Rerank LanceDB search results with the IBM watsonx.ai text rerank A --- -import { PyRerankingWatsonxUsage } from '/snippets/integrations.mdx'; +import { PyRerankingWatsonxUsage } from '/snippets/provider_examples.mdx'; # Watsonx Reranker @@ -17,6 +17,8 @@ This reranker uses the [IBM watsonx.ai](https://cloud.ibm.com/docs/apis/watsonx- pip install ibm-watsonx-ai ``` +Replace the credential placeholders with your provider credentials. In application code, load credentials from environment variables or a secrets manager. + {PyRerankingWatsonxUsage} diff --git a/docs/web/reranking/index.mdx b/docs/web/reranking/index.mdx index e17b43552..26db30442 100644 --- a/docs/web/reranking/index.mdx +++ b/docs/web/reranking/index.mdx @@ -1,96 +1,100 @@ --- title: "Reranking Search Results" sidebarTitle: "Reranking" -description: "Use a reranker to improve search relevance." +description: "Choose a reranker, apply it to search candidates and evaluate relevance and latency." icon: "sort-amount-down" keywords: ["re-ranking", "reranker", "rerank"] +availability: + oss: available + enterprise: available + summary: >- + Reranking is available with OSS and Enterprise. --- import { PyRerankingLinearCombinationUsage, PyRerankingRrfUsage, - PyRerankingCohereUsage, PyRerankingMrrUsage, PyRerankingCrossEncoderUsage, } from '/snippets/integrations.mdx'; +import { PyRerankingCohereUsage } from '/snippets/provider_examples.mdx'; -Reranking is the process of re-ordering search results to improve relevance, often using a -different model than the one used for the initial search. LanceDB has built-in support for reranking -with models from Cohere, Sentence-Transformers, and more. +import { PyHybridSearch, TsHybridSearchBody, TsHybridSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsHybridSearchBody, RsExampleTail as RsHybridSearchTail } from '/snippets/search_hybrid_search.mdx'; -### Quickstart {#quickstart} +Reranking changes the order of records returned by an initial search. It can +combine several ranked lists or use a model to score each candidate against +the query. It cannot recover records that the initial searches did not retrieve. -To use a reranker, you run a search and pass the results to the `rerank()` method. The examples below -move from the simplest, model-free rerankers to a model-based one. Each is a complete, runnable script. +## Choose a reranker -**1. Linear combination (simplest).** `LinearCombinationReranker` normalizes the vector and full-text -scores and blends them with a single `weight` (default `0.7`, favoring the vector score). It runs no -model, and reranks [hybrid search](/search/hybrid-search) results. +- **Rank fusion:** start with RRF for hybrid search. It combines vector and + full-text rankings without a model or a provider API call. +- **Model-based reranking:** try a cross encoder or hosted reranking model + when you need a different relevance judgment over the retrieved text. + Compare the relevance gain with added latency and provider cost. +- **Custom ranking:** implement the reranker interface when your application + needs its own scoring policy. + + + + + +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. + +- **Python:** Install with `pip install lancedb`, save as `rerank_search.py`, then run `python rerank_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `rerank_search.mts`, then run `node rerank_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin hybrid_search` from the extracted directory. - - {PyRerankingLinearCombinationUsage} - + +{PyHybridSearch} + + +{TsHybridSearchBody + "\n" + TsHybridSearchTail} + + +{RsHybridSearchBody + "\n" + RsHybridSearchTail} + -**2. Reciprocal Rank Fusion.** `RRFReranker` fuses results by rank position instead of raw score, so it -sidesteps having to make vector and full-text scores comparable. It loads no model either, and it's the -default reranker for hybrid search. +The result includes `_relevance_score` in descending order. RRF is already the +hybrid-search default; passing it explicitly shows where to substitute another +reranker. These scores express a ranking, not a probability of correctness. - - - {PyRerankingRrfUsage} - - + -**3. Cohere (model-based).** For higher relevance, a model-based reranker scores each result against the -query with a trained model. `CohereReranker` works with vector, full-text, or hybrid search, and needs -the `cohere` package plus either `COHERE_API_KEY` in the environment or an `api_key` argument. +## Supported rerankers {#supported-rerankers} - - - {PyRerankingCohereUsage} - - +Use rank fusion to combine search results without a model, or a model-based +reranker to score each document against the query. Reranking runs in your +application after retrieval, including when the table is in Enterprise. -Reach for the model-free rerankers (`LinearCombinationReranker`, `RRFReranker`) when cost and latency -matter most; reach for a model-based one like `CohereReranker` or `CrossEncoderReranker` when you need -higher relevance and can afford to score every query and document pair with a model. +| Reranker | Method | SDK | +|:---------|:-------|:----| +| `RRFReranker` | Reciprocal rank fusion | Python, TypeScript, Rust | +| `MRRReranker` | Weighted reciprocal rank fusion | Python | +| `LinearCombinationReranker` (deprecated) | Blend normalized scores | Python | +| `CrossEncoderReranker` | Score query-document pairs with a local cross encoder | Python | +| `ColbertReranker`, `AnswerdotaiRerankers` | Local model reranking | Python | +| Provider rerankers | Call a hosted model, such as Cohere, Jina or Voyage AI | Python | -### Supported Rerankers {#supported-rerankers} - -LanceDB supports the following rerankers out of the box. The first three are score-based and run no -model; the rest are model-based. The built-in rerankers are documented in this section; the hosted -providers that need an API key live under [integrations](/integrations/reranking). - -| Reranker | Default model | -| --------------------------- | -------------------------------------- | -| `RRFReranker` | None (reciprocal rank fusion) | -| `LinearCombinationReranker` | None (weighted score blend) | -| `MRRReranker` | None (weighted reciprocal rank) | -| `CohereReranker` | `rerank-english-v3.0` | -| `CrossEncoderReranker` | `cross-encoder/ms-marco-TinyBERT-L-6` | -| `ColbertReranker` | `colbert-ir/colbertv2.0` | -| `AnswerdotaiRerankers` | `answerdotai/answerai-colbert-small-v1`| -| `JinaReranker` | `jina-reranker-v2-base-multilingual` | -| `OpenaiReranker` | `gpt-4-turbo-preview` | -| `VoyageAIReranker` | No default (model name required) | -| `WatsonxReranker` | `cross-encoder/ms-marco-minilm-l-12-v2`| - -The model-based rerankers need their provider package installed, and the hosted ones -(`CohereReranker`, `JinaReranker`, `OpenaiReranker`, `VoyageAIReranker`, `WatsonxReranker`) also need an API key, passed -as an `api_key` argument or set in the provider-specific environment variable. +Local model rerankers need their model runtime packages. Hosted rerankers need +provider credentials and make API calls. See [reranking integrations](/integrations/reranking) +for provider setup and model selection; specify a model supported by your provider. Rerankers add `_relevance_score` and return rows ordered by descending relevance. Python rerankers -accept `return_score="relevance"` or `return_score="all"`; use `"all"` when you want to keep the +accept a `return_score` option; use `"all"` where supported to keep the original vector distance or FTS score columns for debugging. Model-based rerankers read from `column="text"` by default, so either return that column in the search results or pass a different column. -Use `refine_factor` on vector or hybrid queries when you're reranking approximate IVF-PQ results and -want a larger candidate pool before the final ranking step. A value of `3` asks LanceDB to fetch -`limit * 3` candidates, refine them with the full vectors, and keep the requested `limit`. Higher -values can improve recall, but they also increase query latency. +`refine_factor` expands the vector index's internal candidate pool and rescores +those candidates using full vectors before keeping the requested limit. It +can improve the candidates available for reranking, but it does not multiply +the final result count. Compare retrieval recall and total query latency. **SDK coverage differs across languages** @@ -106,7 +110,9 @@ yourself. ### Multi-vector reranking {#multi-vector-reranking} -Most rerankers support reranking based on multiple vectors. To rerank based on multiple vectors, you can pass a list of vectors to the `rerank` method. Here's an example of how to rerank based on multiple vector columns using the `CrossEncoderReranker`: +Use `rerank_multivector()` to combine result sets from several vector queries. +The example below assumes `table` has three vector columns with embedding +metadata and uses a cross encoder to rerank their candidates. ```python Python icon="python" @@ -132,15 +138,24 @@ input result sets is missing the `_rowid` column. Therefore, it's recommended to - `RRFReranker.rerank_multivector(...)` always requires `_rowid` on its inputs, regardless of the `deduplicate` flag. -## Creating Custom Rerankers {#creating-custom-rerankers} +## Implement a custom ranking policy {#creating-custom-rerankers} -LanceDB also allows you to create custom rerankers by extending the base `Reranker` class. The custom reranker -should implement the `rerank` method that takes a list of search results and returns a reranked list of -search results. This is covered in more detail in the [creating custom rerankers](/reranking) section. +Python custom rerankers implement `rerank_hybrid()` and can optionally implement +`rerank_vector()` and `rerank_fts()`. TypeScript and Rust expose hybrid reranking +interfaces. See [the interface requirements](#custom) before implementing one. + +## Algorithm options + +The following sections describe the Python constructor options and examples. +Provider and local-model examples require their additional packages and model +or credential setup. They are separate programs, not steps to concatenate. ## Reciprocal rank fusion {#rrf} -## Reciprocal Rank Fusion Reranker + + + + **Reciprocal Rank Fusion (RRF)** is a model-free way to merge several ranked result lists into a single ordering. Rather than comparing raw similarity scores (which aren't directly comparable @@ -162,10 +177,10 @@ why it's the default reranker for LanceDB hybrid search. The implementation foll -### Accepted Arguments {#rrf-accepted-arguments} +### Python arguments {#rrf-accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | -| `K` | `int` | `60` | A constant used in the RRF formula (default is 60). Experiments indicate that k = 60 was near-optimal, but that the choice is not critical. | +| `K` | `int` | `60` | A constant used in the RRF formula (default is 60). Larger values reduce the difference in contribution between nearby rank positions. | | `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score`. If "all", will return all scores from the vector and FTS search along with the relevance score. | `K` must be greater than `0`. In TypeScript, construct the built-in reranker with @@ -185,18 +200,24 @@ You can specify the type of scores you want the reranker to return. The followin #### Hybrid Search {#rrf-hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Returned rows only have the `_relevance_score` column. | -| `all` | ✅ Supported | Returned rows have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | +| `relevance` | ✅ Supported | Keeps `_relevance_score` as the score column alongside the selected data columns. | +| `all` | ✅ Supported | Returned rows have vector(`_distance`) and FTS(`_score`) along with Hybrid Search score(`_relevance_score`). | + + + ## Linear combination {#linear-combination} -## Linear Combination Reranker + -> **Note:** This reranker is deprecated. Use the `RRFReranker` if you need a score-based reranker. + + + +> **Note:** This reranker is deprecated. Use `RRFReranker` to combine vector and full-text rankings. The Linear Combination Reranker combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified, and defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search. -> **Note:** Supported query type – Hybrid search only. +> **Note:** Supported query type: Hybrid search only. @@ -205,7 +226,7 @@ The Linear Combination Reranker combines the results of semantic and full-text s -### Accepted Arguments {#linear-combination-accepted-arguments} +### Python arguments {#linear-combination-accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `weight` | `float` | `0.7` | The weight to use for the semantic search score. The weight for the full-text search score is `1 - weight`. | @@ -222,16 +243,22 @@ You can specify the type of scores you want the reranker to return. The followin #### Hybrid Search {#linear-combination-hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column | -| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_distance`) | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns | +| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`_score`) along with Hybrid Search score(`_relevance_score`) | + + + ## MRR {#mrr} -## MRR Reranker + + + + This reranker uses the Mean Reciprocal Rank (MRR) algorithm to combine and rerank search results from vector and full-text search. You can use this reranker by passing `MRRReranker()` to the `rerank()` method. The MRR algorithm calculates the average of reciprocal ranks across different search results, providing a balanced way to merge results from multiple ranking systems. -> **Note:** Supported query types – Hybrid and Multivector search. +> **Note:** Supported query types: Hybrid and Multivector search. @@ -239,7 +266,7 @@ This reranker uses the Mean Reciprocal Rank (MRR) algorithm to combine and reran -### Accepted Arguments {#mrr-accepted-arguments} +### Python arguments {#mrr-accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `weight_vector` | `float` | `0.5` | Weight for vector search results (0.0 to 1.0). | @@ -258,21 +285,27 @@ You can specify the type of scores you want the reranker to return. The followin #### Hybrid Search {#mrr-hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | +| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`_score`) along with Hybrid Search score(`_relevance_score`). | #### Multivector Search {#mrr-multivector-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | | `all` | ✅ Supported | Results have vector distances from all searches along with `_relevance_score`. | + + + ## Cross encoder {#cross-encoder} -## Cross Encoder Reranker + + + + This reranker uses Cross Encoder models from sentence-transformers to rerank the search results. You can use this reranker by passing `CrossEncoderReranker()` to the `rerank()` method. -> **Note:** Supported query types – Hybrid, Vector, and FTS. +> **Note:** Supported query types: Hybrid, Vector, and FTS. @@ -281,7 +314,7 @@ This reranker uses Cross Encoder models from sentence-transformers to rerank the -### Accepted Arguments {#cross-encoder-accepted-arguments} +### Python arguments {#cross-encoder-accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `model_name` | `str` | `"cross-encoder/ms-marco-TinyBERT-L-6"` | The name of the reranker model to use.| @@ -299,20 +332,23 @@ You can specify the type of scores you want the reranker to return. The followin #### Hybrid Search {#cross-encoder-hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | +| `all` | ✅ Supported | Keeps `_distance`, `_score` and `_relevance_score` alongside the data columns. | #### Vector Search {#cross-encoder-vector-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | | `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | #### FTS Search {#cross-encoder-fts-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | +| `all` | ✅ Supported | Results have FTS(`_score`) along with Hybrid Search score(`_relevance_score`). | + + + ## Building a custom reranker {#custom} @@ -321,11 +357,9 @@ minimum, you need to implement `rerank_hybrid()`, which is the logic that combin full-text search results. Beyond that, you can optionally implement `rerank_vector()` and `rerank_fts()` if you want your reranker to also handle pure vector or pure full-text searches. -Decide up front which surfaces — hybrid, pure vector, or pure full-text — your reranker should -cover, and only override the ones you need. The base class leaves `rerank_vector()` and -`rerank_fts()` unimplemented, so calling `.rerank(...)` on a single-modality search you haven't -overridden raises `NotImplementedError` rather than silently returning unsorted results. That's a -useful guard, but worth knowing about before you wire up a query path you didn't plan for. +Implement the hooks your query paths call. The base class raises +`NotImplementedError` for vector-only or FTS-only reranking unless you +implement those methods. The Python base class exposes hybrid, vector-only, and FTS-only rerank hooks. TypeScript and Rust currently expose the custom reranker interface for hybrid reranking. In Rust, a custom reranker must @@ -350,55 +384,28 @@ For vector-only reranking in Python, pass a text query to `.rerank(..., query_st The vector query itself may be numeric, but `rerank_vector(query, vector_results)` still receives a string query so your reranker can score each candidate against the user's text intent. -Below, we show the pseudocode of a custom reranker that combines the results of semantic and full-text -search using a linear combination of the scores: +The method contract is: - -```python Python icon="python" -from lancedb.rerankers import Reranker -import pyarrow as pa +| Method | Input | Required output | +|:-------|:------|:----------------| +| `rerank_hybrid` | Query text, vector results and FTS results | A merged Arrow table ordered by descending `_relevance_score` | +| `rerank_vector` | Query text and vector results | An Arrow table ordered by descending `_relevance_score` | +| `rerank_fts` | Query text and FTS results | An Arrow table ordered by descending `_relevance_score` | -class MyReranker(Reranker): - def __init__(self, param1, param2, ..., return_score="relevance"): - super().__init__(return_score) - self.param1 = param1 - self.param2 = param2 +Implement only the single-search hooks your application uses. The base class +raises `NotImplementedError` for those hooks until they are implemented. - def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table): - # Use the built-in merging function - combined_result = self.merge_results(vector_results, fts_results) +### Extend an existing reranker {#custom-example} - # Do something with the combined results - # ... - - # Return the combined results - return combined_result - - def rerank_vector(self, query: str, vector_results: pa.Table): - # Do something with the vector results - # ... - - # Return the vector results - return vector_results - - def rerank_fts(self, query: str, fts_results: pa.Table): - # Do something with the FTS results - # ... - - # Return the FTS results - return fts_results -``` - - -### Example {#custom-example} - -As an example, let's build custom reranker that enhances the Cohere Reranker by accepting a filter -query, and accepts any other `CohereReranker` params as `kwargs`. +This class definition extends `CohereReranker` to exclude text patterns after +reranking. It requires the Cohere package and credentials when instantiated. +It can return fewer results because exclusions happen after retrieval; use a +query predicate instead when a metadata filter expresses the same restriction. ```python Python icon="python" from typing import List, Union -import pandas as pd +import pyarrow as pa from lancedb.rerankers import CohereReranker class ModifiedCohereReranker(CohereReranker): @@ -433,58 +440,42 @@ class ModifiedCohereReranker(CohereReranker): ``` - -Under the hood, `vector_results` and `fts_results` are PyArrow tables. You can learn more about -PyArrow tables [here](https://arrow.apache.org/docs/python). The advantage of PyArrow tables is their -interoperability -- you can easily convert them to Pandas/Polars DataFrames, `PyDict`, `PyList`, etc. +The reranker receives and returns PyArrow tables. Use `to_pandas()` for a +DataFrame transformation and `pa.Table.from_pandas()` to convert the result +back, preserving `_relevance_score` and the desired row order. -The benefits are also bidirectional -- just as you can easily convert PyArrow tables _to_ Pandas -DataFrames using the `to_pandas()` method -- you can perform DataFrame transformations -and just as easily convert the DataFrame back to PyArrow tables using `pa.Table.from_pandas()` method -as shown in the example above. - -## Evaluating hybrid search {#eval} +## Evaluate relevance, latency and cost {#eval} -Hybrid search is an often misused and/or misunderstood term. In this section, we're using -the definition of "hybrid search" to mean using a combination of keyword-based and vector search. -Because the vector search operates in a dense embedding space and keyword-based search operate -in a sparse embedding space, their relevance scores cannot be directly compared. -Combining results from multiple searches thus requires a reranking step. - -Before evaluating hybrid search, build an FTS index on the text column and use a table with embedding -metadata or explicit query vectors for the vector side. Otherwise the evaluation is measuring setup -fallbacks or errors rather than the reranker. +Compare rerankers on the same candidate inputs so you can distinguish ranking +changes from retrieval changes. For hybrid retrieval, prepare the text index +and query vectors before collecting those candidates. ### Reranking strategies {#eval-reranking-strategies} -There are two common approaches for reranking search results from multiple sources. +Compare these approaches: -- **Score-based**: Calculate final relevance scores from the individual search algorithm scores. Examples: Reciprocal Rank Fusion (the default in LanceDB), mean reciprocal rank fusion, and weighted linear combination of semantic and keyword-based search scores. +- **Rank fusion:** RRF and MRR combine rank positions from multiple result lists. +- **Score blending:** linear combination mixes normalized vector and text scores. +- **Model-based reranking:** a model scores each query-document pair. -- **Relevance-based**: Discards the existing scores and calculates the relevance of each search result-query pair. Example: Cross Encoder models - - -If you call `.rerank()` on a hybrid query without passing a reranker, LanceDB defaults to -`RRFReranker()` — a score-based reranker that uses Reciprocal Rank Fusion. This is the -score-based path most readers encounter first; `LinearCombinationReranker` is an alternative -score-based strategy you opt into explicitly. - +Hybrid search uses RRF by default. You do not need to call `rerank()` to select +it. Pass a reranker when you want a different algorithm or configuration. By default, rerankers return `_relevance_score`. Pass `return_score="all"` when a reranker supports it, and you also need the original vector or FTS scores for debugging. Evaluation code can rely on returned rows being ordered by descending `_relevance_score`. Empty reranked result sets still include the `_relevance_score` column. -The hybrid `rerank(...)` method also accepts a `normalize` argument that controls how the raw +The synchronous Python hybrid `rerank(...)` method also accepts a `normalize` argument that controls how the raw vector and FTS scores are made comparable before reranking: -- `normalize="score"` (the default) — normalizes the raw vector and FTS scores directly. -- `normalize="rank"` — converts each result list to ranks first, then normalizes. +- `normalize="score"` (the default); normalizes the raw vector and FTS scores directly. +- `normalize="rank"`; converts each result list to ranks first, then normalizes. -This choice materially affects score-based rerankers (such as `LinearCombinationReranker`), so -when you evaluate score-based strategies, treat `normalize` as a tunable hyperparameter -alongside the reranker itself. +Normalization matters for algorithms that use raw scores, such as linear +combination. RRF uses rank positions instead. Compare normalization settings +only when the chosen reranker uses those scores. For score-based evaluation, the main built-in knobs are: @@ -492,17 +483,18 @@ For score-based evaluation, the main built-in knobs are: - `MRRReranker(weight_vector=0.5, weight_fts=0.5)`: weighted reciprocal rank fusion where the weights must sum to `1.0`. - `LinearCombinationReranker(weight=0.7, fill=1.0)`: blends vector and FTS scores directly; `fill` controls how strongly to penalize a result missing from one side. -Even though there are many more strategies for reranking, there are no "universally best" -ones that work well for all cases, because reranking quality is dataset and application specific. -Evaluating whether a reranking strategy is a good fit is also a challenge. In the next -section, we discuss an example evaluation of different reranking strategies on a sample dataset. +A result on one dataset does not establish which reranker will work best on +another. Include representative queries and measure both retrieval quality +and ranking quality. -### Example evaluation {#eval-example-evaluation} +### Historical evaluation example {#eval-example-evaluation} -The table below shows our evaluation results from an experiment comparing multiple rerankers on +The tables below preserve an earlier experiment comparing multiple rerankers on ~800 hybrid search queries. This is a modified version of an evaluation script by [LlamaIndex](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures -hit-rate \@ top-k. +hit-rate \@ top-k. The recorded results do not establish performance for +current models, SDK releases or your dataset; treat them as an example of an +evaluation format. #### Using OpenAI `text-embedding-ada-002` {#eval-using-openai-text-embedding-ada-002} @@ -531,14 +523,14 @@ Vector Search baseline: **0.59** -### Conclusion {#eval-conclusion} +### Compare results on your workload {#eval-conclusion} -The results show that the reranking methods can significantly improve the search relevance. However, -the improvement we saw was not consistent across all rerankers. In reality, the choice of reranker -likely depends on the dataset and the application. +Build a set of representative queries with known relevant records. Compare +retrieval alone with each reranker using the same candidate inputs and result +limit. Record a relevance metric such as recall or nDCG alongside latency and +provider cost. Include queries with no useful matches and queries containing +exact names or identifiers. -It's also important to note that the reranking methods are not a -replacement for the search methods they supplement. They are complementary and it's likely that you'd -have to tune them together to get the best results. The latency vs. recall tradeoff is also an -important factor to consider when choosing the reranker. Hopefully this evaluation -gives you a starting point for your own experiments with hybrid search in LanceDB! +If the relevant records never reach the reranker, adjust retrieval first. +Then change one ranking setting at a time and retain the configuration that +meets your application's quality and latency requirements. diff --git a/docs/web/search/explain.mdx b/docs/web/search/explain.mdx index b2fc12c04..383745e5b 100644 --- a/docs/web/search/explain.mdx +++ b/docs/web/search/explain.mdx @@ -1,40 +1,72 @@ --- title: "Explain a query plan" sidebarTitle: "Query Plans" -description: "Read the plan LanceDB resolved for a query, and measure it with analyze." +description: "Inspect query execution, identify expensive work and compare changes." icon: "map" +availability: + oss: available + enterprise: available + summary: >- + Query plans are available in OSS and Enterprise. --- -`explain_plan` prints the plan LanceDB resolved for a query without executing -it. It answers structural questions -- did the filter push into the scan, is the -vector index being used, which columns are actually read -- and costs nothing to -run, so it is the first thing to reach for when a query behaves unexpectedly. +import { PyQueryPlan, TsQueryPlanBody, TsQueryPlanTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsQueryPlanBody, RsExampleTail as RsQueryPlanTail } from '/snippets/search_query_plan.mdx'; -To measure a query rather than read it, use -[`analyze_plan`](/search/explain), which executes it and annotates every -operator with runtime metrics. +Use query plans to understand how LanceDB reads data, applies filters and uses +indexes. `explain_plan()` shows the planned operations. `analyze_plan()` runs +the query and adds measurements, including execution time and rows processed. - -The SQL equivalent is [`EXPLAIN`](/sql/dql/explain). - +## Inspect a plan or measure execution {#inspect-a-query-plan} -## What it shows {#inspect-a-query-plan} +| Operation | Python / Rust | TypeScript | +|:----------|:--------------|:-----------| +| Inspect the plan without executing the query | `explain_plan()` | `explainPlan()` | +| Execute the query and report runtime measurements | `analyze_plan()` | `analyzePlan()` | -Reveals the logical query plan before execution, helping you identify potential issues with query structure and index usage. This tool is useful for: +Use the first to inspect query structure and the second to investigate work +actually performed. Operator names and available measurements can differ +between OSS and Enterprise. SQL provides [EXPLAIN and EXPLAIN ANALYZE](/sql/dql/explain). -- Verifying query optimization strategies -- Validating index selection -- Understanding query execution order -- Detecting missing indices + -Metadata filters are prefiltered by default, which usually shows the filter pushed into the -`LanceScan` or index scan. If you set `prefilter=False`, expect a separate `FilterExec` after -search instead; that can be useful for some expensive filters, but it changes both latency and -the number of rows available after filtering. + -## Reading the Execution Plan {#reading-the-execution-plan} +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. -To demonstrate query performance analysis, we'll use a table containing 1.2M rows sampled from the [Wikipedia dataset](https://huggingface.co/datasets/wikimedia/wikipedia). Initially, the table has no indices, allowing us to observe the impact of optimization. +- **Python:** Install with `pip install lancedb`, save as `query_plan.py`, then run `python query_plan.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `query_plan.mts`, then run `node query_plan.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin query_plan` from the extracted directory. + + + +{PyQueryPlan} + + +{TsQueryPlanBody + "\n" + TsQueryPlanTail} + + +{RsQueryPlanBody + "\n" + RsQueryPlanTail} + + + +This tiny table demonstrates the API, not a performance benchmark. The plan +uses a scan because the example has no vector index. The following larger +example illustrates how to read scans, filters and index operations. + + + +## Read scans, filters and result retrieval {#reading-the-execution-plan} + +The recorded example below uses 1.2 million rows sampled from the +[Wikipedia dataset](https://huggingface.co/datasets/wikimedia/wikipedia). +The counts describe that run. The table initially has no indexes, so the plan +shows a scan. This Enterprise plan includes `RemoteTake` and hides storage paths +as `uri=***`. A local plan shows the dataset path instead, and fetches the result +rows with Lance's `Take` or, depending on the library version, a second +`LanceRead` whose `source` is `stream(_rowid)`. Let's examine a vector search query that: @@ -67,6 +99,9 @@ const explainPlan = await table ### Execution Plan Components {#execution-plan-components} + + + The execution plan reveals the sequence of operations performed to execute your query. Let's examine each component: ``` @@ -103,7 +138,7 @@ FilterExec: identifier@1 > 0 AND identifier@1 < 1000000 #### 3\. Vector Search {#3-vector-search} -- Computes L2 (Euclidean) distances between query vector and all vectors that passed the filter +- Computes squared L2 distances between query vector and all vectors that passed the filter ``` KNNVectorDistance: metric=l2 @@ -113,7 +148,7 @@ KNNVectorDistance: metric=l2 - Filters out null distance results - Sorts by distance and takes top 100 results -- Processes in batches of 1024 for optimal memory usage +- Coalesces output into batches targeting 1,024 rows in this plan ``` SortExec: TopK(fetch=100) @@ -126,7 +161,7 @@ CoalesceBatchesExec: target_batch_size=1024 #### 5\. Data Retrieval {#5-data-retrieval} -- `RemoteTake` is a key component of Lance's I/O cache +- `RemoteTake` is the deployment's operator for this step; a local plan shows Lance's `Take` or a `LanceRead` over the selected row IDs, depending on the library version - Handles efficient data retrieval from remote storage locations - Fetches specific rows and columns needed for the final output - Optimizes network bandwidth by only retrieving required data @@ -139,19 +174,21 @@ RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title" - Returns only requested columns and maintains column ordering -```python +```text ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance] ``` This plan demonstrates a basic search without index optimizations: it performs a full scan and filter before vector search. + + + ## Non-vector queries {#analyzing-non-vector-queries} -`explain_plan` is not vector-specific -- it is available on every query builder, -including FTS and hybrid. The most common reason to read the plan for a -non-vector query is to confirm whether your `where` clause pushed into the scan -(good) or ran as a separate `FilterExec` on top of the search results (often -slower, and a hint that the filter column needs a scalar index). +Query plans also cover filters and full-text search. Python can show both +sub-plans of a hybrid query. Check where a filter runs and how many rows flow +through each operator; a separate `FilterExec` does not by itself establish +that the plan is inefficient or that an index is missing. ### FTS queries {#fts-queries} @@ -176,7 +213,10 @@ const plan = await table ``` -In an indexed FTS plan you should see a `MatchQuery` (or other FTS execution node) reading from the inverted index, with the metadata filter pushed down. If the plan shows a `LanceScan` followed by `FilterExec` over the entire text column, the FTS index either isn't covering the column or the filter isn't using a scalar index — both worth investigating. +Look for the text-index operation and any scan of unindexed rows. A scan can +be needed to include records written after the index was built. Use index +coverage and execution measurements to determine whether that extra work +matters. Check filter placement separately from text-index use. ### Hybrid queries {#hybrid-queries} @@ -197,21 +237,19 @@ print(plan) ``` -Reading the plan tells you the shape. `analyze_plan` executes both sub-queries -and labels them `Vector Search Plan:` and `FTS Search Plan:`, which is how you -see whether the filter pushed into both halves uniformly and which half -dominates latency. +`analyze_plan` executes both sub-queries and labels them `Vector Search Plan:` +and `FTS Search Plan:`. Compare their filter placement and execution measurements. `analyze_plan` executes the query and returns the physical plan annotated with -runtime metrics -- elapsed time, rows produced, bytes read, index comparisons. +runtime metrics such as elapsed time, rows produced, bytes read and index comparisons. Where [`explain_plan`](#inspect-a-query-plan) tells you what a query intends to do, -this tells you what it cost. +analysis shows the work performed. The SQL equivalent is [`EXPLAIN ANALYZE`](/sql/dql/explain). -## What it measures {#analyze-a-query-plan} +## Read execution measurements {#analyze-a-query-plan} Executes the query and provides detailed runtime metrics, including: @@ -220,7 +258,7 @@ Executes the query and provides detailed runtime metrics, including: - Index effectiveness (`_index_comparisons_`, `_indices_loaded_`) - Resource utilization (`_iops_`, `_requests_`) -## Performance Analysis {#performance-analysis} +## Measure the example query {#performance-analysis} Let's use `analyze_plan` to run the query and analyze the query performance, which will help us identify potential bottlenecks: @@ -249,6 +287,9 @@ const analyzePlan = await table ### Performance Metrics Analysis {#performance-metrics-analysis} + + + ``` ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance], metrics=[output_rows=100, elapsed_compute=1.424µs] RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title", metrics=[output_rows=100, elapsed_compute=175.53097ms, output_batches=1, remote_takes=100] @@ -284,11 +325,14 @@ ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier - Remote take operation for 100 results - Final projection of required columns + + + ### Distributed metrics on remote tables {#distributed-metrics-on-remote-tables} Enterprise -When you call `analyze_plan` against a LanceDB Enterprise table, the query runs across a pool of workers. By default the returned plan aggregates each operator's metrics into a single value, matching the local single-node output. Pass a `distributed_metrics` mode when you need to see how work was split across workers: +When you call `analyze_plan` against a LanceDB Enterprise deployment that distributes its queries across a pool of workers, the query runs on those workers. By default the returned plan aggregates each operator's metrics into a single value, matching the local single-node output. Pass a `distributed_metrics` mode when you need to see how work was split across workers: | Mode | What it shows | When to use it | | :------------ | :---------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------ | @@ -296,7 +340,9 @@ When you call `analyze_plan` against a LanceDB Enterprise table, the query runs | `per_worker` | Metrics reported separately for each worker that participated in the query. | Diagnosing stragglers or uneven work distribution across the cluster. | | `full` | Both the aggregate summary and the per-worker breakdown. | Deep investigations where you need the totals *and* the per-worker detail in a single output. | -The parameter only affects remote plans — local queries always return the aggregate output regardless of what you pass. +Python exposes this option through `distributed_metrics`. In TypeScript, pass +the same mode to `analyzePlan()` as its `distributedMetrics` argument. Local +Python queries ignore the distributed mode and return local measurements. ```python Python icon="python" @@ -310,30 +356,26 @@ plan = ( print(plan) ``` -```typescript TypeScript icon="square-js" -// Show a per-worker breakdown for a remote query -const plan = await table - .search(queryEmbed) - .where("identifier > 0 AND identifier < 1000000") - .limit(100) - .analyzePlan("per_worker"); -``` + -Use `per_worker` or `full` sparingly — the plan output grows with the size of your worker pool. Stick with the default `aggregate` mode for routine tuning. +Use `per_worker` or `full` sparingly; the plan output grows with the size of your worker pool. Stick with the default `aggregate` mode for routine tuning. -### Key Observations {#key-observations} +### Identify expensive operations {#key-observations} -- Vector search is the primary bottleneck (1,099,508 vector comparisons) -- Significant I/O overhead (1.86GB data read) +- The query computes distances for 1,099,508 vectors; inspect timings to assess its contribution to latency +- The scan reads 1.86 GB in this recorded run - Full table scan due to lack of indices -- Substantial optimization potential through proper index implementation +- Compare this scan with an indexed plan using the same query -## Optimized Query Execution {#optimized-query-execution} +## Compare the query after adding indexes {#optimized-query-execution} -After creating vector and scalar indices, the execution plan shows: + + + +The recorded query after adding vector and scalar indexes produced this plan: ``` ProjectionExec: expr=[chunk_index@3 as chunk_index, title@4 as title, identifier@2 as identifier, _distance@0 as _distance] @@ -442,7 +484,7 @@ ANNSubIndex: - Before: L2 calculations on 1,099,508 vectors - After: - - 99.8% reduction in vector comparisons + - About 97.6% fewer vector comparisons (25,893 versus 1,099,508) - Decreased output batches from 1,076 to 20 #### 3\. Data Retrieval Optimization {#3-data-retrieval-optimization} @@ -455,7 +497,10 @@ RemoteTake: - RemoteTake operation remains consistent -## Performance Optimization Guide {#performance-optimization-guide} + + + +## Choose a change to measure {#performance-optimization-guide} ### 1\. Index Implementation {#1-index-implementation} @@ -470,13 +515,14 @@ RemoteTake: | Data Type | Recommended Index | Use Case | | ----------- | ------------------ | ---------------------------------------- | | Vector | IVF_PQ/IVF_HNSW_SQ/IVF_HNSW_FLAT | Approximate nearest neighbor search | -| Scalar | B-Tree | Range queries and sorting | +| Scalar | B-Tree | Equality and range filters | | Categorical | Bitmap | Multi-value filters and set operations | | `List` | Label_list | Multi-label classification and filtering | Use `table.index_stats()` to monitor index coverage. -A well-optimized table should have `num_unindexed_rows ~ 0`. +Unindexed rows can add scan work. Use their count alongside query measurements +to decide when index maintenance is needed. ### 2\. Query Plan Optimization {#2-query-plan-optimization} @@ -485,10 +531,10 @@ A well-optimized table should have `num_unindexed_rows ~ 0`. | Plan Pattern | Optimization | | ------------------------------------------ | -------------------------------------------- | -| LanceScan with high _bytes_read_ or _iops_ | Add missing index | +| Scan with high bytes read or I/O count | Check index coverage and whether the query can use an index | | | Use `select()` to limit returned columns | | | Check whether the dataset has been compacted | -| Multiple sequential filters | Reorder filter conditions | +| Multiple filters | Check their placement and row counts; compare the plan after changing the query | Regularly analyze your query plans to identify and address performance @@ -498,7 +544,8 @@ optimization efforts. ### 3\. Getting Started with Optimization {#3-getting-started-with-optimization} -For vector search performance: - -- Create ANN index on your vector column(s) as described in the [index guide](/indexing/vector-index/) -- If you often filter by metadata, create [scalar indices](/indexing/scalar-index/) on those columns +Keep the query, data and result limit fixed while changing one factor. Compare +rows processed, bytes read, timings and returned results. For ANN changes, +measure recall too: doing less work is useful only if retrieval quality meets +your application's needs. See [vector indexes](/indexing/vector-index) and +[scalar indexes](/indexing/scalar-index) for index choices. diff --git a/docs/web/search/filtering.mdx b/docs/web/search/filtering.mdx index d5bce618b..50d37acc0 100644 --- a/docs/web/search/filtering.mdx +++ b/docs/web/search/filtering.mdx @@ -1,28 +1,103 @@ --- title: "Metadata Filtering in LanceDB" sidebarTitle: Filtering -description: Filter search results in LanceDB based on metadata fields. +description: "Use predicates to select rows and control when filters apply to search." icon: "filter" +availability: + oss: available + enterprise: available + summary: >- + Metadata filtering is available in OSS and Enterprise. --- -LanceDB supports filtering features of query results based on metadata fields. -While joint vector and metadata search at scale presents a significant challenge, -LanceDB achieves sub-100ms latency at thousands of QPS, enabling efficient vector search -with filtering capabilities even on datasets containing billions of records. +import { PyFilterSearch, TsFilterSearchBody, TsFilterSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsFilterSearchBody, RsExampleTail as RsFilterSearchTail } from '/snippets/search_filter_search.mdx'; -**Pre-filtering** means LanceDB applies the metadata `where(...)` condition before running vector search, so the search only considers rows that already match the filter. **Post-filtering** means LanceDB runs vector search first and only then filters the returned candidates. Pre-filtering is enabled by default. In practice, pre-filtering is better when the filter is part of the result contract; post-filtering can be lower-latency for expensive or non-indexable filters, but it can return fewer than `limit` rows, or even zero, if the nearest neighbors do not pass the filter. +A filter restricts results to rows that satisfy a condition. Use it on its +own to select records, or combine it with vector, full-text or hybrid search. +For example, search for similar documents only within a particular category. -On hybrid queries, the same `where(...)` filter is applied to both the vector and full-text halves of the query. The prefilter or postfilter choice controls whether that happens before each subquery scores candidates or after the subquery top-k is produced. + - -The same predicates work in SQL -- see [SELECT](/sql/dql/select#filtering) and [Operators](/sql/operators). - + -## Chaining `where` clauses {#chaining-where-clauses} +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. -In more recent LanceDB SDK versions (see the callout box below for the exact version numbers), you can call `where(...)` (Python and TypeScript) or `only_if(...)` (Rust) more than once on the same query builder. Each additional filter is combined with the previous one using logical `AND`, so `where("a > 0").where("b < 10")` is equivalent to `where("(a > 0) AND (b < 10)")`. +- **Python:** Install with `pip install lancedb`, save as `filtered_search.py`, then run `python filtered_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `filtered_search.mts`, then run `node filtered_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin filter_search` from the extracted directory. -For a fixed predicate, writing one `where(...)` clause with `AND` is just as valid and often clearer. Chaining is mainly useful when code composes filters incrementally, such as applying a shared base predicate in a helper and then adding a per-call predicate at the query site. + + +{PyFilterSearch} + + +{TsFilterSearchBody + "\n" + TsFilterSearchTail} + + +{RsFilterSearchBody + "\n" + RsFilterSearchTail} + + + +The query vector is closest to the gardening record, but the filter excludes +that category. The results contain only the two animal records. + + + +## Prefiltering and postfiltering {#choose-when-the-filter-runs} + +- **Prefiltering**, the default, limits the rows eligible for retrieval before + search chooses its candidates. +- **Postfiltering** removes nonmatching rows after candidates are selected. + It can return fewer than `limit` rows, including zero, even when matching + records exist elsewhere in the table. + +Both modes exclude rows that fail the predicate. They differ in which matching +rows the search can find. On hybrid queries, the filter applies to both the +vector and full-text searches. + +SDK filters use SQL expressions without requiring a SQL connection. For +Enterprise SQL queries, see [SELECT](/sql/dql/select#filtering). + +## Filtering examples by language {#example-metadata-filtering} + +The following excerpts use a connected `db` handle. Create this sample table +before running the queries beneath it; each query uses the same four records. + + +```python Python icon="python" +data = [ + {"vector": [3.1, 4.1], "item": "foo", "price": 10.0}, + {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}, + {"vector": [10.2, 100.8], "item": "baz", "price": 30.0}, + {"vector": [1.4, 9.5], "item": "fred", "price": 40.0}, +] +table = db.create_table("metadata_filter_example", data=data, mode="overwrite") +``` + +```typescript TypeScript icon="square-js" +const data = [ + { vector: [3.1, 4.1], item: "foo", price: 10.0 }, + { vector: [5.9, 26.5], item: "bar", price: 20.0 }, + { vector: [10.2, 100.8], item: "baz", price: 30.0 }, + { vector: [1.4, 9.5], item: "fred", price: 40.0 }, +]; + +const tableName = "metadata_filter_example"; +const table = await db.createTable(tableName, data, { + mode: "overwrite", +}); +``` + + +## Combine predicates {#chaining-where-clauses} + +Repeated filter calls combine predicates with logical `AND`. For example, `where("a > 0").where("b < 10")` is equivalent to `where("(a > 0) AND (b < 10)")`. + +Use a single expression for a fixed filter. Chain calls when your application +builds the filter incrementally. ```python Python icon="python" @@ -51,36 +126,6 @@ const result = await table In Python SDK versions before `0.34.0` and Rust/TypeScript SDK versions before `0.31.0`, a second `where(...)` or `only_if(...)` call replaced the first filter, so only the last predicate was applied. If your code needs to run on those older versions, write a single predicate with `AND` instead of chaining calls. When upgrading to the latest SDKs, review any existing chained filters and drop earlier calls you no longer want to apply. -## Example: Metadata Filtering {#example-metadata-filtering} - -To illustrate filtering capabilities, let's try four data points with combinations of vectors and metadata: - - -```python Python icon="python" -data = [ - {"vector": [3.1, 4.1], "item": "foo", "price": 10.0}, - {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}, - {"vector": [10.2, 100.8], "item": "baz", "price": 30.0}, - {"vector": [1.4, 9.5], "item": "fred", "price": 40.0}, -] -table = db.create_table("metadata_filter_example", data=data, mode="overwrite") -``` - -```typescript TypeScript icon="square-js" -const data = [ - { vector: [3.1, 4.1], item: "foo", price: 10.0 }, - { vector: [5.9, 26.5], item: "bar", price: 20.0 }, - { vector: [10.2, 100.8], item: "baz", price: 30.0 }, - { vector: [1.4, 9.5], item: "fred", price: 40.0 }, -]; - -const tableName = "metadata_filter_example"; -const table = await db.createTable(tableName, data, { - mode: "overwrite", -}); -``` - - ### Filtering Without Vector Search {#filtering-without-vector-search} You can always filter your data without search. This is useful when you need to query based on metadata: @@ -104,9 +149,8 @@ const filteredResult = await table ``` - -If your table is large, this could potentially return a very large amount of data. Please be sure to use a `limit` clause unless you're sure you want to return the whole result set. - +Set `limit()` explicitly to control how many matching records your application +receives. ### Pre-Filtering with Vector Search {#pre-filtering-with-vector-search} @@ -124,6 +168,7 @@ filtered_result = ( const results = await table .search([100, 102]) .where("(item IN ('foo', 'bar')) AND (price > 15.0)") + .limit(3) .toArray(); ``` @@ -141,7 +186,7 @@ post_filtered_result = ( ``` ```typescript TypeScript icon="square-js" -const postFilteredResult = await (table.search([100, 102]) as VectorQuery) +const postFilteredResult = await table.vectorSearch([100, 102]) .where("(item IN ('foo', 'bar')) AND (price > 15.0)") .postfilter() .limit(3) @@ -149,14 +194,13 @@ const postFilteredResult = await (table.search([100, 102]) as VectorQuery) ``` - -When querying large tables, omitting a `limit` clause may overwhelm resources and return excessive data. It's always recommended -to be mindful of the potential impact on performance and costs when working with really large tables. - +Postfiltering can return fewer than the requested number of records because +the predicate removes candidates after search. -## Filtering with SQL {#filtering-with-sql} +## Write SQL predicates {#filtering-with-sql} -Because it's built on top of DataFusion, LanceDB embraces the utilization of standard SQL expressions as predicates for filtering operations. SQL can be used during vector search, update, and deletion operations. +Pass a SQL expression as the filter string. These expressions also select rows +for SDK update and delete operations; they are not standalone SQL statements. LanceDB supports a growing list of SQL expressions: @@ -178,7 +222,7 @@ For example, the following filter string is acceptable: ```python Python icon="python" -tbl.search([100, 102]).where( +table.search([100, 102]).where( "(item IN ('foo', 'baz')) AND (price > 20.0)" ).to_arrow() ``` @@ -202,9 +246,10 @@ path must be wrapped in backticks. AND `nested with space`.`inner with space` < 2 ``` - -Field names containing periods (.) are NOT supported. - + +Periods separate nested fields. LanceDB does not allow a literal period in a +top-level column name; backticks do not remove that restriction. + ### Dates, Timestamps, Decimals {#dates-timestamps-decimals} @@ -231,7 +276,8 @@ parameter. Microsecond precision (6) is the default. ## Apache Arrow Mapping {#apache-arrow-mapping} -LanceDB internally stores data in [Apache Arrow](https://arrow.apache.org/) format. +LanceDB uses Arrow types for schemas and in-memory data. On disk, tables use +the [Lance format](/basics/lance). The mapping from SQL types to Arrow types is: | SQL type | Arrow type | @@ -245,26 +291,27 @@ The mapping from SQL types to Arrow types is: | `double` | `Float64` | | `decimal(precision, scale)` | `Decimal128` | | `date` | `Date32` | -| `timestamp` | `Timestamp` [^1] | +| `timestamp` | `Timestamp` | | `string` | `Utf8` | | `binary` | `Binary` | -## Best Practices {#best-practices} +## Speed up frequently used filters {#best-practices} -**Scalar Indexes**: We strongly recommend creating scalar indices on columns used for filtering, whether combined with a search operation or applied independently (e.g., for updates or deletions). +Consider a [scalar index](/indexing/scalar-index) on frequently filtered columns. +Use the query plan to check whether a predicate uses that index. Functions or +casts applied to a column can change which optimizations are available. +For list columns, a label-list index supports membership filters such as +`array_has_any` and `array_has_all`. -For best performance with large tables or high query volumes: +## Understand missing matches {#limitations} -- Build a scalar index on frequently filtered columns -- Use exact column names in filters (e.g., `user_id` instead of `USER_ID`) -- Avoid complex transformations in filter expressions (keep them simple) -- When running concurrent queries, use connection pooling for better throughput +A restrictive predicate can legitimately leave fewer rows than the requested +limit. Approximate vector search can also miss qualifying neighbors, and +postfiltering can discard candidates without retrieving replacements. -For a column of type LIST(T), you can use `LABEL_LIST` to create a scalar index. Then you should leverage DataFusion's [array functions](https://datafusion.apache.org/user-guide/sql/scalar_functions.html#array-functions) like `array_has_any` or `array_has_all` for optimized filtering. - -## Limitations {#limitations} - -Both **pre-filtering** and **post-filtering** can yield false positives. For pre-filtering, if the filter is too selective, it might eliminate relevant items that the vector search would have otherwise identified as a good match. In this case, increasing `nprobes` parameter will help reduce such false positives. It is recommended to call `bypass_vector_index()` if you know that the filter is highly selective. - -Similarly, a highly selective post-filter can lead to false positives. Increasing both `nprobes` and `refine_factor` can mitigate this issue. When deciding between pre-filtering and post-filtering, pre-filtering is generally the safer choice if you're uncertain. +For a recall comparison, run the same query and predicate while bypassing the +vector index on a suitably sized test table. Then compare the indexed query's +results and latency while changing probe or refinement settings. Increasing +those settings does not guarantee a full result set. See +[vector search](/search/vector-search) and [query plans](/search/explain). diff --git a/docs/web/search/full-text-search.mdx b/docs/web/search/full-text-search.mdx index 0008183dc..0dc0096f7 100644 --- a/docs/web/search/full-text-search.mdx +++ b/docs/web/search/full-text-search.mdx @@ -1,8 +1,13 @@ --- title: Full-Text Search (FTS) sidebarTitle: Full-Text Search (FTS) -description: Learn how to implement full-text search in LanceDB using BM25 for keyword-based retrieval. +description: "Find records by words, phrases or partial matches using a full-text index." icon: "book" +availability: + oss: available + enterprise: available + summary: >- + Full-text search is available in OSS and Enterprise. --- import { @@ -12,17 +17,51 @@ import { } from '/snippets/search.mdx' import { PyFtsBooleanQueries } from '/snippets/doc_function_examples.mdx' -LanceDB provides support for Full-Text Search via Lance, allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. +import { PyTextSearch, TsTextSearchBody, TsTextSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsTextSearchBody, RsExampleTail as RsTextSearchTail } from '/snippets/search_text_search.mdx'; - -In SQL, full-text search is the [`fts()`](/sql/functions#fts) table function, or the [`contains_tokens`](/sql/scalar-functions#contains-tokens) predicate when you do not need a score. - +Full-text search finds documents by matching words and ranks them with BM25. +Use it to search for names, product codes, technical terms or phrases in your +text columns. Create an FTS index on the column before running a query. -## Basic Usage {#basic-usage} +You can also combine keyword matches with vector similarity using +[hybrid search](/search/hybrid-search). -Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search, the FTS index must be created before you can search via keywords. + -### Table Setup {#table-setup} + + +Each example creates a temporary local table, runs the query and removes the +table when finished. No embedding model is needed for a text query. + +- **Python:** Install with `pip install lancedb`, save as `text_search.py`, then run `python text_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `text_search.mts`, then run `node text_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin text_search` from the extracted directory. + + + +{PyTextSearch} + + +{TsTextSearchBody + "\n" + TsTextSearchTail} + + +{RsTextSearchBody + "\n" + RsTextSearchTail} + + + +Each example searches the indexed `text` column and returns the puppy record. +Higher `_score` values rank first; they are relevance scores, not probabilities. + + + +## Basic usage {#basic-usage} + +The steps below create a table, index its `text` column and search for a word. +For a standalone program with imports and sample data, expand the complete +example above. + +### Create the sample table {#table-setup} First, open or create the table you want to search: @@ -66,19 +105,16 @@ let tbl = db ``` -### Construct FTS Index {#construct-fts-index} +### Build the text index {#construct-fts-index} Create a full-text search index on your text column: - -In Python, this page shows the synchronous `create_fts_index(...)` form. For the -asynchronous equivalent (`await table.create_index("text", config=FTS(...))`), see -[FTS index](/indexing/fts-index). - +For asynchronous Python index creation, use +`await table.create_index("text", config=FTS())`. See [FTS index](/indexing/fts-index). ```python Python icon="python" -table.create_fts_index("text") +table.create_index("text", config=FTS()) ``` ```typescript TypeScript icon="square-js" @@ -95,17 +131,24 @@ tbl ``` -### Full-text Search {#full-text-search} +OSS builds the index before the call returns. Enterprise builds it in the +background. Wait for completion with Python’s `table.wait_for_index(["text_idx"])` +or TypeScript’s `await tbl.waitForIndex(["text_idx"], 300)` before querying it. +Use your actual index name if you chose a different one. + +### Run the query {#full-text-search} Perform full-text search and retrieve results: ```python Python icon="python" -results = table.search("puppy") +results = ( + table.search("puppy", query_type="fts", fts_columns="text") .limit(10) - .select(["text"]) + .select(["text", "_score"]) .to_list() -# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}] +) +# Returns the puppy record with its BM25 score. ``` ```typescript TypeScript icon="square-js" @@ -127,19 +170,26 @@ let results = tbl ``` -The search is conducted on all indexed columns by default, so it's useful when there are multiple indexed columns. - -If you want to specify which columns to search use `fts_columns="text"` +Specify the text column with `fts_columns="text"` in synchronous Python, +especially when the table contains multiple text indexes. -LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead. +A string query can also invoke an embedding function when the table has +embedding metadata. Use `query_type="fts"` in Python when you specifically +want full-text search. If a table has more than one FTS index, specify the indexed text column in the query. In Python you can use `fts_columns` or the query builder's `nearest_to_text(..., columns=...)`; in TypeScript, use `query().nearestToText(..., columns)`. The newer Lance-native FTS does not accept legacy Tantivy-only index parameters. +BM25 scores depend on the indexed corpus and its term statistics. Compare +rankings on the dataset you will search; treat scores as relative relevance +values rather than probabilities. + ### Keeping the index up to date {#keeping-the-index-up-to-date} -Rows you add after building an FTS index aren't part of the index until you optimize the table. Until then, queries fall back to a flat scan over the unindexed fragments to keep results complete, which slows them down as the unindexed tail grows. Call `table.optimize()` to fold new rows into the existing index — it's the same operation used for vector indexes: +In OSS, new rows can remain outside the FTS index until you optimize the table. +Search includes those rows by scanning them, but that work can increase query +latency. Call `optimize()` to update the index: @@ -157,13 +207,35 @@ tbl.optimize(OptimizeAction::All).await?; ``` -A useful rule of thumb is to call `optimize()` after roughly 100,000 row changes or 20 data-modification operations, whichever comes first. For tables with continuous ingest, schedule it on a cadence that keeps `num_unindexed_rows` (from `table.index_stats(...)`) close to zero. If you want to skip the flat scan over unindexed rows entirely — for example, on a hot read path where stale results are acceptable — call `.fast_search()` on the query so the search returns only indexed results. +Choose an OSS maintenance schedule based on write volume and query latency. +Inspect index coverage with `table.index_stats(...)` and optimize when the +unindexed data adds too much work. `.fast_search()` skips unindexed rows, so +use it only when omitting recent records is acceptable. -## Advanced Usage {#advanced-usage} +Enterprise manages index updates automatically. You do not call `optimize()` +on an Enterprise table. + +## Choose the matching behavior + +| You need | Use | +|:---------|:----| +| Words in any order | A terms query | +| Words in a particular sequence | A [phrase query](#phrase-vs-terms-queries) and an index with token positions | +| Tolerance for typing errors | [Fuzzy search](#fuzzy-search) | +| Parts of words | An [n-gram index](#search-for-substring) | +| Required, optional or excluded terms | [Boolean queries](#boolean-queries) | + +Use a metadata predicate to restrict qualifying records. See +[filtering options](#filtering-options) for prefiltering and postfiltering. +Enterprise SQL provides [`fts()`](/sql/functions#fts) for scored text retrieval. + +## Configure tokenization and matching {#advanced-usage} ### Tokenize Table Data {#tokenize-table-data} -By default, the text is tokenized by splitting on punctuation and whitespaces, and would filter out words that are longer than 40 characters. All words are converted to lowercase. +The default tokenizer splits text at whitespace and punctuation, lowercases +tokens and removes tokens longer than 40 characters. Stemming, stop-word +removal and accent folding are also enabled by default. Stemming is useful for improving search results by reducing words to their root form, e.g. "running" to "run". LanceDB supports stemming for Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. You should set the `base_tokenizer` parameter rather than `tokenizer_name` because you cannot customize the tokenizer if `tokenizer_name` is specified. @@ -177,7 +249,8 @@ table.create_fts_index("text", language="English", replace=True) ``` -The tokenizer is customizable, you can specify how the tokenizer splits the text, and how it filters out words, etc. +Configure tokenization when you create the index. Rebuild the index to change +how existing text is tokenized. **Default index parameters:** - `base_tokenizer`: `"simple"` @@ -188,7 +261,7 @@ The tokenizer is customizable, you can specify how the tokenizer splits the text - `stem`: true - `remove_stop_words`: true - `ascii_folding`: true -- `custom_stop_words`: `None` — pass a `list[str]` to drop additional words beyond the language defaults. Requires `remove_stop_words=True`. +- `custom_stop_words`: `None`; pass a `list[str]` to drop additional words beyond the language defaults. Requires `remove_stop_words=True`. For multilingual use cases, use `base_tokenizer="icu"` for unicode-aware word segmentation on mixed-language text. ICU stands for [International Components for Unicode](https://icu.unicode.org/). The ICU tokenizer uses bundled ICU4X segmenter data, so it does not require external tokenizer model files. It is a good default when documents mix languages or include scripts where the simple tokenizer would keep an unspaced span as one large token. @@ -215,11 +288,15 @@ export LANCE_LANGUAGE_MODEL_HOME=/path/to/lance/language_models For example, `jieba/default` is resolved under `/jieba/default/...`, `lindera/ipadic` under `/lindera/ipadic/...`, and `lindera/ko-dic` under `/lindera/ko-dic/...`. +Model files must be installed where indexing and queries run: on your machine +for OSS, or on the Enterprise cluster for a remote table. + Built-in stop-word removal supports Danish, Dutch, English, Finnish, French, German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish, and Swedish. If you use another stemming language, such as Arabic, Greek, Romanian, Tamil, or Turkish, set `remove_stop_words=False` or pass `custom_stop_words`. -For example, for language with accents, you can specify the tokenizer to use `ascii_folding` to remove accents, e.g. 'é' to 'e': +Use `ascii_folding=True` to match accented and unaccented forms, such as +“é” and “e”: ```python Python icon="python" @@ -235,9 +312,9 @@ table.create_fts_index( ### Filtering Options {#filtering-options} -LanceDB full text search supports to filter the search results by a condition, both pre-filtering and post-filtering are supported. - -This can be invoked via the familiar `where` syntax. +Use `where(...)` to restrict text results to rows that match a predicate. +Prefiltering applies the restriction before retrieval; postfiltering removes +nonmatching candidates afterward and can leave fewer results. With pre-filtering: @@ -308,15 +385,17 @@ Lance-based FTS doesn't support queries using boolean operators `OR`, `AND` in t For full-text search you can specify either a **phrase** query like `"the old man and the sea"`, or a **terms** search query like `old man sea`. -To search for a phrase, the index must be created with `with_position=True` and `remove_stop_words=False`: +Enable token positions to search for phrases. Set `remove_stop_words=False` if you also need to match common words such as “the” and “and”: ```python Python icon="python" -table.create_fts_index("text", with_position=True, replace=True) +from lancedb.index import FTS + +table.create_index("text", config=FTS(with_position=True, remove_stop_words=False), replace=True) ``` -This will allow you to search for phrases, but it will also significantly increase the index size and indexing time. +Storing token positions increases the index size and adds indexing work. ### Fuzzy Search {#fuzzy-search} @@ -326,7 +405,7 @@ to find similar terms within a specified edit distance. | Parameter | Type | Default | Description | | -------------- | ---- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -| fuzziness | int | 0 | Maximum edit distance allowed for each term. If not specified, automatically set based on term length: 0 for length ≤ 2, 1 for length ≤ 5, 2 for length > 5 | +| fuzziness | int | 0 | Maximum edit distance allowed for each term. The default, `0`, requires an exact match; set `1` or `2` to allow typos. | | max_expansions | int | 50 | Maximum number of terms to consider for fuzzy matching. Higher values may improve recall but increase search time | For a complete walkthrough that creates a sample table and demonstrates fuzzy search and relevance boosting, see [the worked example below](#fuzzy-search-and-boosting-example). @@ -344,65 +423,41 @@ LanceDB supports searching for substrings in the text column, you can set the `b ## Fuzzy Search and Boosting Example {#fuzzy-search-and-boosting-example} + + + ### Generate Data {#generate-data} -First, let's create a table with sample text data for testing fuzzy search: +Run the following steps in order. They use a temporary local database with four records, so the same data is available for each query. ```python Python icon="python" +from tempfile import TemporaryDirectory + import lancedb -import numpy as np import pandas as pd -import random +from lancedb.index import FTS +from lancedb.query import MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery -# Connect to LanceDB -db = lancedb.connect( - uri="db://your-project-slug", - api_key="your-api-key", - region="us-east-1" -) - -# Generate 100 rows of random " " text -table_name = "fts-fuzzy-boosting-test" -vectors = [np.random.randn(128) for _ in range(100)] -verbs = ("runs", "hits", "jumps", "drives", "barfs") -adv = ("crazily.", "dutifully.", "foolishly.", "merrily.", "occasionally.") -adj = ("adorable", "clueless", "dirty", "odd", "stupid") - -def sentence(nouns): - return " ".join(random.choice(words) for words in (nouns, verbs, adv, adj)) - -text = [sentence(("puppy", "car")) for _ in range(100)] -text2 = [sentence(("rabbit", "girl", "monkey")) for _ in range(100)] -count = [random.randint(1, 10000) for _ in range(100)] +directory = TemporaryDirectory() +db = lancedb.connect(directory.name) +table_name = "fts_examples" +text = ["puppy runs merrily", "car drives crazily", "puppy jumps crazily", "car runs slowly"] +text2 = ["rabbit jumps crazily", "girl runs merrily", "monkey jumps slowly", "rabbit runs merrily"] ``` ```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb" +import * as lancedb from "@lancedb/lancedb"; +import { Index, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, Occur } from "@lancedb/lancedb"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; -const db = await lancedb.connect({ - uri: "db://your-project-slug", - apiKey: "your-api-key", - region: "us-east-1" -}); - -// Generate 100 rows of random " " text -const tableName = "fts-fuzzy-boosting-test-ts"; -const n = 100; -const verbs = ["runs", "hits", "jumps", "drives", "barfs"]; -const adverbs = ["crazily", "dutifully", "foolishly", "merrily", "occasionally"]; -const adjectives = ["adorable", "clueless", "dirty", "odd", "stupid"]; - -const pick = (words: string[]) => words[Math.floor(Math.random() * words.length)]; -const sentence = (nouns: string[]) => - [nouns, verbs, adverbs, adjectives].map(pick).join(" "); - -const vectors = Array.from({ length: n }, () => - Array.from({ length: 128 }, () => Math.random() * 2 - 1) -); -const text = Array.from({ length: n }, () => sentence(["puppy", "car"])); -const text2 = Array.from({ length: n }, () => sentence(["rabbit", "girl", "monkey"])); -const count = Array.from({ length: n }, () => Math.floor(Math.random() * 10000) + 1); +const directory = await mkdtemp(join(tmpdir(), "lancedb-fts-")); +const db = await lancedb.connect(directory); +const tableName = "fts_examples"; +const text = ["puppy runs merrily", "car drives crazily", "puppy jumps crazily", "car runs slowly"]; +const text2 = ["rabbit jumps crazily", "girl runs merrily", "monkey jumps slowly", "rabbit runs merrily"]; ``` @@ -414,11 +469,9 @@ const count = Array.from({ length: n }, () => Math.floor(Math.random() * 10000) table = db.create_table( table_name, data=pd.DataFrame({ - "vector": vectors, - "id": [i % 2 for i in range(100)], + "id": range(len(text)), "text": text, "text2": text2, - "count": count, }), mode="overwrite" ) @@ -426,15 +479,7 @@ table = db.create_table( ```typescript TypeScript icon="square-js" // Create table with sample data -const data = makeArrowTable( - vectors.map((vector, i) => ({ - vector, - id: i % 2, - text: text[i], - text2: text2[i], - count: count[i], - })) -); +const data = text.map((value, i) => ({ id: i, text: value, text2: text2[i] })); const table = await db.createTable(tableName, data, { mode: "overwrite" }); ``` @@ -447,14 +492,14 @@ Create a full-text search index on the first text column: ```python Python icon="python" # Create FTS index on first text column -table.create_fts_index("text") -wait_for_index(table, "text_idx") +table.create_index("text", config=FTS(with_position=True, remove_stop_words=False)) +table.wait_for_index(["text_idx"]) ``` ```typescript TypeScript icon="square-js" // Create FTS index on first text column -await table.createIndex("text", { config: Index.fts() }); -await waitForIndex(table, "text_idx"); +await table.createIndex("text", { config: Index.fts({ withPosition: true, removeStopWords: false }) }); +await table.waitForIndex(["text_idx"], 300); ``` @@ -463,14 +508,14 @@ Then, create an index on the second text column: ```python Python icon="python" # Create FTS index on second text column -table.create_fts_index("text2") -wait_for_index(table, "text2_idx") +table.create_index("text2", config=FTS(with_position=True, remove_stop_words=False)) +table.wait_for_index(["text2_idx"]) ``` ```typescript TypeScript icon="square-js" // Create FTS index on second text column -await table.createIndex("text2", { config: Index.fts() }); -await waitForIndex(table, "text2_idx"); +await table.createIndex("text2", { config: Index.fts({ withPosition: true, removeStopWords: false }) }); +await table.waitForIndex(["text2_idx"], 300); ``` @@ -487,19 +532,17 @@ from lancedb.query import MatchQuery # Basic match (exact search) basic_match_results = ( table.search(MatchQuery("crazily", "text")) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" -import { MatchQuery } from "@lancedb/lancedb"; - // Basic match (exact search) const basicMatchResults = await table.query() .fullTextSearch(new MatchQuery("crazily", "text")) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` @@ -512,9 +555,9 @@ const basicMatchResults = await table.query() # Fuzzy match (allows typos) fuzzy_results = ( table.search(MatchQuery("craziou", "text", fuzziness=2)) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` @@ -524,34 +567,35 @@ const fuzzyResults = await table.query() .fullTextSearch(new MatchQuery("craziou", "text", { fuzziness: 2, })) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` -#### Prefix based Match {#prefix-based-match} +#### Require an exact prefix in fuzzy matching {#prefix-based-match} -Prefix-based match allows you to search for documents containing words that start with a specific prefix. +`prefix_length` keeps the first characters unchanged when matching a misspelled term. It does not turn a short query into a prefix-completion search. ```python Python icon="python" # Fuzzy match (allows typos) -fuzzy_results = ( - table.search(MatchQuery("cra", "text", prefix_length=3)) - .select(["id", "text"]) +prefix_results = ( + table.search(MatchQuery("craziou", "text", fuzziness=2, prefix_length=3)) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" // Fuzzy match (allows typos) -const fuzzyResults = await table.query() - .fullTextSearch(new MatchQuery("cra", "text", { +const prefixResults = await table.query() + .fullTextSearch(new MatchQuery("craziou", "text", { + fuzziness: 2, prefixLength: 3, })) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` @@ -580,26 +624,24 @@ from lancedb.query import PhraseQuery phrase_results = ( table.search(PhraseQuery("puppy runs", "text")) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" -import { PhraseQuery } from "@lancedb/lancedb"; - // Exact phrase match const phraseResults = await table.query() .fullTextSearch(new PhraseQuery("puppy runs", "text")) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` #### Flexible Phrase Match {#flexible-phrase-match} -To provide more flexible phrase matching, LanceDB supports the `slop` parameter. This allows you to match phrases where the terms appear close to each other, even if they are not directly adjacent or in the exact order, as long as they are within the specified `slop` value. +To provide more flexible phrase matching, LanceDB supports the `slop` parameter. This allows you to match phrases where the terms appear close to each other, with gaps between them. The `slop` value limits the extra positions allowed in the phrase. For example, the phrase query "puppy merrily" would not return any results by default. However, if you set `slop=1`, it will match phrases like "puppy jumps merrily", "puppy runs merrily", and similar variations where one word appears between "puppy" and "merrily". @@ -608,21 +650,19 @@ For example, the phrase query "puppy merrily" would not return any results by de # Flexible phrase match with slop=1 for 'puppy merrily' from lancedb.query import PhraseQuery -phrase_results = ( +slop_results = ( table.search(PhraseQuery("puppy merrily", "text", slop=1)) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" -import { PhraseQuery } from "@lancedb/lancedb"; - -// Flexible phrase match with slop=1 for 'puppy runs' -const phraseResults = await table.query() - .fullTextSearch(new PhraseQuery("puppy runs", "text", { slop: 1 })) - .select(["id", "text"]) +// Flexible phrase match with slop=1 for 'puppy merrily' +const slopResults = await table.query() + .fullTextSearch(new PhraseQuery("puppy merrily", "text", { slop: 1 })) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` @@ -656,17 +696,17 @@ boosting_results = ( negative_boost=0.2, ), ) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) # Search across both text and text2 multi_match_results = ( table.search(MultiMatchQuery("crazily", ["text", "text2"])) - .select(["id", "text", "text2"]) + .select(["id", "text", "text2", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) # Search with field boosting @@ -674,28 +714,26 @@ multi_match_boosting_results = ( table.search( MultiMatchQuery("crazily", ["text", "text2"], boosts=[1.0, 2.0]), ) - .select(["id", "text", "text2"]) + .select(["id", "text", "text2", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" -import { MatchQuery, BoostQuery, MultiMatchQuery } from "@lancedb/lancedb"; - // Boosting Example const boostingResults = await table.query() .fullTextSearch(new BoostQuery(new MatchQuery("runs", "text"), new MatchQuery("puppy", "text"), { negativeBoost: 0.2, })) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); // Search across both text fields const multiMatchResults = await table.query() .fullTextSearch(new MultiMatchQuery("crazily", ["text", "text2"])) - .select(["id", "text", "text2"]) + .select(["id", "text", "text2", "_score"]) .limit(100) .toArray(); @@ -704,24 +742,12 @@ const multiMatchBoostingResults = await table.query() .fullTextSearch(new MultiMatchQuery("crazily", ["text", "text2"], { boosts: [1.0, 2.0], })) - .select(["id", "text", "text2"]) + .select(["id", "text", "text2", "_score"]) .limit(100) .toArray(); ``` - -- Use fuzzy search when handling user input that may contain typos or variations -- Apply field boosting to prioritize matches in more important columns -- Combine fuzzy search with boosting for robust and precise search results - -**Recommendations for optimal FTS performance:** - -- Create full-text search indices on text columns that will be frequently searched -- For hybrid search combining text and vectors, see our [hybrid search guide](/search/hybrid-search/) -- For performance benchmarks, check our [benchmark results](/enterprise/benchmarks/) -- For complex queries, use SQL to combine FTS with other filter conditions - ### Boolean Queries {#boolean-queries} LanceDB supports boolean logic in full-text search, allowing you to combine multiple queries using `and` and `or` operators. This is useful when you want to match documents that satisfy multiple conditions (intersection) or at least one of several conditions (union). @@ -734,16 +760,7 @@ In Python, combine two `MatchQuery` objects with `&` for intersection or `|` for {PyFtsBooleanQueries} -In TypeScript, boolean queries are constructed using the `BooleanQuery` class with a list of [Occur, subquery] pairs. For example, to perform an AND query: - -```sql SQL icon="code" -BooleanQuery([ -[Occur.Must, new MatchQuery("puppy", "text")], -[Occur.Must, new MatchQuery("merrily", "text")], -]) -``` - -This approach allows you to specify complex boolean logic by combining multiple subqueries with different Occur values (such as `Must`, `Should`, or `MustNot`). +In TypeScript, pass `[Occur, subquery]` pairs to `BooleanQuery`. Use `Must` for required matches, `Should` for alternatives, and `MustNot` to exclude matches. **Which queries are allowed?** @@ -759,24 +776,22 @@ from lancedb.query import MatchQuery and_query = MatchQuery("puppy", "text") & MatchQuery("merrily", "text") and_results = ( table.search(and_query) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) # Example: Find documents containing either "puppy" or "merrily" or_query = MatchQuery("puppy", "text") | MatchQuery("merrily", "text") or_results = ( table.search(or_query) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" expandable=true -import { MatchQuery, BooleanQuery, Occur } from "@lancedb/lancedb"; - // Flexible boolean queries with MatchQuery // Find documents containing both "puppy" and "merrily" @@ -787,7 +802,7 @@ const mustResults = await table [Occur.Must, new MatchQuery("merrily", "text")], ]), ) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); @@ -799,7 +814,7 @@ const shouldResults = await table [Occur.Should, new MatchQuery("merrily", "text")], ]), ) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` @@ -808,12 +823,33 @@ const shouldResults = await table **How to use booleans?** -- Use `and`/`&`(Python), `Occur.Must`(Typescript) for intersection (documents must match all queries). -- Use `or`/`|`(Python), `Occur.Should`(Typescript) for union (documents must match at least one query). +- Use `&` (Python), `Occur.Must`(Typescript) for intersection (documents must match all queries). +- Use `|` (Python), `Occur.Should`(Typescript) for union (documents must match at least one query). + +### Clean up the example + +Remove the temporary database when you have finished running the queries. + + +```python Python icon="python" +directory.cleanup() +``` +```typescript TypeScript icon="square-js" +table.close(); +db.close(); +await rm(directory, { recursive: true, force: true }); +``` + + + + ## Substring Search Example {#substring-search-example} + + + LanceDB supports searching for substrings in text columns using n-gram tokenization. This is useful for finding partial matches within text content. ### Setting Up the Table {#setting-up-the-table} @@ -825,11 +861,15 @@ First, create a table with sample text data and configure n-gram tokenization: import pyarrow as pa import lancedb -db = lancedb.connect(":memory:") +from tempfile import TemporaryDirectory +from lancedb.index import FTS + +directory = TemporaryDirectory() +db = lancedb.connect(directory.name) data = pa.table({"text": ["hello world", "lance database", "lance is cool"]}) table = db.create_table("test", data=data) -table.create_fts_index("text", base_tokenizer="ngram") +table.create_index("text", config=FTS(base_tokenizer="ngram")) ``` @@ -868,12 +908,10 @@ You can customize the n-gram behavior by adjusting the minimum length and using ```python Python icon="python" -table.create_fts_index( +table.create_index( "text", - base_tokenizer="ngram", + config=FTS(base_tokenizer="ngram", ngram_min_length=2, prefix_only=True), replace=True, - ngram_min_length=2, - prefix_only=True, ) ``` @@ -899,34 +937,47 @@ assert set(r["text"] for r in results) == {"lance database", "lance is cool"} ``` + +```python Python icon="python" +directory.cleanup() +``` + + + ## Full-Text Search on Array Fields {#full-text-search-on-array-fields} + + + LanceDB supports full-text search on string array columns, enabling efficient keyword-based search across multiple values within a single field (e.g., tags, keywords). ### Setting Up the Connection {#setting-up-the-connection} -Connect to your LanceDB instance: +This example uses a temporary local database. Run the steps in order. ```python Python icon="python" -import lancedb +from tempfile import TemporaryDirectory -# Connect to LanceDB -db = lancedb.connect( - uri="db://your-project-slug", - api_key="your-api-key", - region="us-east-1" -) +import lancedb +import pyarrow as pa +from lancedb.index import FTS +from lancedb.query import MatchQuery, PhraseQuery + +directory = TemporaryDirectory() +db = lancedb.connect(directory.name) ``` -```typescript TypeScript icon="square-js" expandable=true -import * as lancedb from "@lancedb/lancedb" +```typescript TypeScript icon="square-js" +import * as lancedb from "@lancedb/lancedb"; +import { Index, MatchQuery, PhraseQuery, makeArrowTable } from "@lancedb/lancedb"; +import { Schema, Field, List, Utf8 } from "apache-arrow"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; -const db = await lancedb.connect({ - uri: "db://your-project-slug", - apiKey: "your-api-key", - region: "us-east-1" -}); +const directory = await mkdtemp(join(tmpdir(), "lancedb-array-fts-")); +const db = await lancedb.connect(directory); ``` @@ -1048,25 +1099,25 @@ console.log(`Created table: ${tableName}`); ### Building the Full-Text Search Index {#building-the-full-text-search-index} -Create an FTS index on the tags column to enable efficient text search: +Store token positions for phrase queries. This example disables stemming so fuzzy matching compares the original word forms: ```python Python icon="python" # Create FTS index -table.create_fts_index("tags") -wait_for_index(table, "tags_idx") +table.create_index("tags", config=FTS(with_position=True, stem=False)) +table.wait_for_index(["tags_idx"]) ``` ```typescript TypeScript icon="square-js" // Create FTS index console.log("Creating FTS index on 'tags' column..."); await table.createIndex("tags", { - config: Index.fts() + config: Index.fts({ withPosition: true, stem: false }) }); // Wait for index const ftsIndexName = "tags_idx"; -await waitForIndex(table, ftsIndexName); +await table.waitForIndex([ftsIndexName], 300); ``` @@ -1080,24 +1131,27 @@ Search for terms with typos using fuzzy matching: print("\nSearching for 'learning' in tags with a typo:") result = ( table.search(MatchQuery("learnin", column="tags", fuzziness=1)) - .select(['id', 'tags', 'description']) + .select(["id", "tags", "description", "_score"]) .to_arrow() ) ``` -```typescript TypeScript icon="square-js"> +```typescript TypeScript icon="square-js" // Search examples console.log("\nSearching for 'learning' in tags with a typo:"); const fuzzyResults = await table.query() .fullTextSearch(new MatchQuery("learnin", "tags", { - fuzziness: 2, + fuzziness: 1, })) - .select(["id", "tags", "description"]) + .select(["id", "tags", "description", "_score"]) .toArray(); console.log(fuzzyResults); ``` +A string-array query can return a row more than once when several elements match. +Deduplicate by your record ID if your application needs one result per row. + ### Performing Phrase Search {#performing-phrase-search} Search for exact phrases within the array fields: @@ -1107,7 +1161,7 @@ Search for exact phrases within the array fields: print("\nSearching for 'machine learning' in tags:") result = ( table.search(PhraseQuery("machine learning", column="tags")) - .select(['id', 'tags', 'description']) + .select(["id", "tags", "description", "_score"]) .to_arrow() ) ``` @@ -1116,8 +1170,25 @@ result = ( console.log("\nSearching for 'machine learning' in tags:"); const phraseResults = await table.query() .fullTextSearch(new PhraseQuery("machine learning", "tags")) - .select(["id", "tags", "description"]) + .select(["id", "tags", "description", "_score"]) .toArray(); console.log(phraseResults); ``` + +### Clean up the array example + +Remove the temporary database when you have finished running the queries. + + +```python Python icon="python" +directory.cleanup() +``` +```typescript TypeScript icon="square-js" +table.close(); +db.close(); +await rm(directory, { recursive: true, force: true }); +``` + + + diff --git a/docs/web/search/hybrid-search.mdx b/docs/web/search/hybrid-search.mdx index 4a2d2dead..c3e453145 100644 --- a/docs/web/search/hybrid-search.mdx +++ b/docs/web/search/hybrid-search.mdx @@ -1,26 +1,77 @@ --- title: Hybrid Search sidebarTitle: "Hybrid Search" -description: Learn how to perform hybrid search in LanceDB by combining vector and full-text search techniques with reranking. +description: "Combine vector similarity and keyword matches in a single ranked result set." icon: "search" +availability: + oss: available + enterprise: available + summary: >- + Hybrid search is available in OSS and Enterprise. --- -In certain cases, you may want to retrieve documents that are semantically similar to a given query, -but also prioritize specific keywords. This is an example of **hybrid search**, a query method that combines -multiple search techniques. +import { PyConnectEnterprisePlaceholders, TsConnectEnterprisePlaceholders } from '/snippets/connection.mdx'; -For detailed examples, look at this [Python Notebook](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/saas_examples/python_notebook/Hybrid_search.ipynb) or the [**TypeScript Example**](https://github.com/lancedb/vectordb-recipes/tree/main/examples/saas_examples/ts_example/hybrid-search) +import { PyHybridSearch, TsHybridSearchBody, TsHybridSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsHybridSearchBody, RsExampleTail as RsHybridSearchTail } from '/snippets/search_hybrid_search.mdx'; -## Example: Hybrid Search {#example-hybrid-search} +Hybrid search combines vector similarity with keyword matching. It is useful +when a query needs both: for example, finding products with a particular model +number whose descriptions match what the customer is looking for. + +LanceDB runs a vector search and a full-text search, then merges their results +with a reranker. Reciprocal Rank Fusion (RRF) is the default. It combines the +positions of records in the two ranked lists. + +Before you query, store vectors and create an FTS index on the text column. + + + + + +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. + +- **Python:** Install with `pip install lancedb`, save as `hybrid_search.py`, then run `python hybrid_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `hybrid_search.mts`, then run `node hybrid_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin hybrid_search` from the extracted directory. + + + +{PyHybridSearch} + + +{TsHybridSearchBody + "\n" + TsHybridSearchTail} + + +{RsHybridSearchBody + "\n" + RsHybridSearchTail} + + + +The puppy record ranks first because both searches place it first. The query +returns `_relevance_score`, with higher values first. This small example shows +the mechanics; evaluate retrieval quality with embeddings and queries from +your own application. + + + +## Generate query embeddings automatically {#example-hybrid-search} ### 1\. Setup {#1-setup} -Import the necessary libraries and dependencies for working with LanceDB, OpenAI embeddings, and reranking. +The following walkthrough uses Sentence Transformers in Python and OpenAI in +TypeScript. Install the corresponding provider packages before starting. +Python downloads a local model; TypeScript calls OpenAI. Install +`lancedb sentence-transformers pandas` for Python, or +`@lancedb/lancedb apache-arrow openai` for TypeScript. Set your OpenAI API key +as described in the [OpenAI embedding setup](/integrations/embedding/openai). +Run the steps for your chosen language in order, choosing one connection option. ```python Python icon="python" -import os import lancedb -import openai +from lancedb.index import FTS +from lancedb.rerankers import RRFReranker from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector ``` @@ -33,7 +84,7 @@ import { Utf8 } from "apache-arrow"; ### 2\. Connect to LanceDB {#2-connect-to-lancedb} -Establish a connection to your LanceDB instance, with different options for Enterprise setups or open source. +Choose a local OSS connection or use your Enterprise connection details. OSS @@ -44,9 +95,6 @@ db = lancedb.connect(uri) ``` ```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; -import * as arrow from "apache-arrow"; - const databaseDir = "data/sample-lancedb"; const db = await lancedb.connect(databaseDir); ``` @@ -54,41 +102,21 @@ const db = await lancedb.connect(databaseDir); Enterprise -For LanceDB Enterprise, set the `db://` URI, region and the host override to your private cloud endpoint: +Replace the placeholders with your Enterprise database URI, API key, region and +endpoint. In application code, load credentials from environment variables or a +secrets manager. -```python Python icon="python" -host_override = os.environ.get("LANCEDB_HOST_OVERRIDE") - -db = lancedb.connect( - uri=uri, - api_key=api_key, - region=region, - host_override=host_override -) -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; -import * as arrow from "apache-arrow"; - -const uri = "db://my-lancedb-instance/my-database"; -const apiKey = process.env.LANCEDB_API_KEY; -const region = process.env.LANCEDB_REGION; -const hostOverride = process.env.LANCEDB_HOST_OVERRIDE; - -const db = await lancedb.connect(uri, { - apiKey, - region - hostOverride, -}); -``` + + {PyConnectEnterprisePlaceholders} + + + {TsConnectEnterprisePlaceholders} + - - ### 3\. Configure Embedding Model {#3-configure-embedding-model} -Set up the any embedding model that will convert text into vector representations for semantic search. +Configure the model used for both document and query embeddings. ```python Python icon="python" @@ -108,6 +136,7 @@ Define the data structure for your documents, including both the text content an ```python Python icon="python" class Documents(LanceModel): + category: str text: str = embeddings.SourceField() vector: Vector(embeddings.ndims()) = embeddings.VectorField() @@ -117,6 +146,7 @@ table = db.create_table(table_name, schema=Documents, mode="overwrite") ```typescript TypeScript icon="square-js" const documentSchema = lancedb.embedding.LanceSchema({ + category: new Utf8(), text: embedFunc.sourceField(new Utf8()), vector: embedFunc.vectorField(), }); @@ -134,20 +164,20 @@ Insert sample documents into your table, which will be used for both semantic an ```python Python icon="python" data = [ - {"text": "rebel spaceships striking from a hidden base"}, - {"text": "have won their first victory against the evil Galactic Empire"}, - {"text": "during the battle rebel spies managed to steal secret plans"}, - {"text": "to the Empire's ultimate weapon the Death Star"}, + {"text": "rebel spaceships striking from a hidden base", "category": "film"}, + {"text": "have won their first victory against the evil Galactic Empire", "category": "film"}, + {"text": "during the battle rebel spies managed to steal secret plans", "category": "film"}, + {"text": "to the Empire's ultimate weapon the Death Star", "category": "film"}, ] table.add(data=data) ``` ```typescript TypeScript icon="square-js" const data = [ - { text: "rebel spaceships striking from a hidden base" }, - { text: "have won their first victory against the evil Galactic Empire" }, - { text: "during the battle rebel spies managed to steal secret plans" }, - { text: "to the Empire's ultimate weapon the Death Star" }, + { text: "rebel spaceships striking from a hidden base", category: "film" }, + { text: "have won their first victory against the evil Galactic Empire", category: "film" }, + { text: "during the battle rebel spies managed to steal secret plans", category: "film" }, + { text: "to the Empire's ultimate weapon the Death Star", category: "film" }, ]; await table.add(data); console.log(`Created table: ${tableName} with ${data.length} rows`); @@ -159,8 +189,8 @@ Create a full-text search index on the text column to enable keyword-based searc ```python Python icon="python" -table.create_fts_index("text") -wait_for_index(table, "text_idx") +table.create_index("text", config=FTS()) +table.wait_for_index(["text_idx"]) ``` ```typescript TypeScript icon="square-js" @@ -168,12 +198,13 @@ console.log("Creating full-text search index..."); await table.createIndex("text", { config: lancedb.Index.fts(), }); -await waitForIndex(table as any, "text_idx"); +await table.waitForIndex(["text_idx"], 300); ``` ### 7\. Set Reranker [Optional] {#7-set-reranker-optional} -Initialize the reranker that will combine and rank results from both semantic and keyword search. By default, lancedb uses RRF reranker, but you can choose other rerankers like `Cohere`, `CrossEncoder`, or others lister in integrations section. +RRF is the default. The examples below configure it explicitly. Other +rerankers and their language support are described in [Reranking](/reranking). ```python Python icon="python" @@ -192,7 +223,7 @@ Perform a hybrid search query that combines semantic similarity with keyword mat ```python Python icon="python" results = ( table.search( - "flower moon", + "rebel spaceships", query_type="hybrid", vector_column_name="vector", fts_columns="text", @@ -208,10 +239,10 @@ print(results) ```typescript TypeScript icon="square-js" console.log("Performing hybrid search..."); -const queryVector = await embedFunc.computeQueryEmbeddings("full moon in May"); +const queryVector = await embedFunc.computeQueryEmbeddings("rebel spaceships"); const hybridResults = await table .query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .rerank(reranker) .select(["text"]) @@ -223,13 +254,16 @@ console.log(hybridResults); ``` -### 9\. Hybrid Search - Explicit Vector and Text Query pattern {#9-hybrid-search-explicit-vector-and-text-query-pattern} -You can also pass the vector and text query explicitly. This is useful if you're not using the embedding API or if you're using a separate embedder service. +### 9\. Supply a vector and text separately {#9-hybrid-search-explicit-vector-and-text-query-pattern} +You can supply the vector and text separately. The vector must use the same +model and dimensions as the stored embeddings. This example reuses the model +configured above; the complete example at the top also demonstrates this pattern +in TypeScript and Rust. ```python Python icon="python" -vector_query = [0.1, 0.2, 0.3, 0.4, 0.5] -text_query = "flower moon" +text_query = "rebel spaceships" +vector_query = embeddings.compute_query_embeddings(text_query)[0] ( table.search(query_type="hybrid") .vector(vector_query) @@ -240,13 +274,15 @@ text_query = "flower moon" ``` -## Query controls {#query-controls} +## Control the candidate results {#query-controls} -Hybrid queries inherit the same builder API as vector and FTS queries, so the same knobs for filtering, distance bounds, and row identity apply. These compose with `.rerank(...)` and the explicit `.vector()` / `.text()` form shown above. +Use a result limit, metadata filters and vector distance bounds to control +which candidates are available to the reranker. The examples below show each +option on the query builder. -Always set `.limit(...)` on production hybrid queries. LanceDB's default search limit is 10, but an -explicit cap gives you a clear top-k contract to tune before reranking. +Set `limit()` to the number of results your application needs. The default is 10. +A larger candidate set can help a reranker find relevant records, but adds work. ### Returning row IDs {#returning-row-ids} @@ -256,7 +292,7 @@ Pass `with_row_id(True)` (Python) or `withRowId()` (TypeScript) to include the i ```python Python icon="python" results = ( - table.search("flower moon", query_type="hybrid") + table.search("rebel spaceships", query_type="hybrid") .with_row_id(True) .limit(10) .to_pandas() @@ -267,7 +303,7 @@ results = ( ```typescript TypeScript icon="square-js" const results = await table .query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .withRowId() .limit(10) @@ -282,7 +318,7 @@ const results = await table ```python Python icon="python" results = ( - table.search("flower moon", query_type="hybrid") + table.search("rebel spaceships", query_type="hybrid") .distance_range(lower_bound=0.0, upper_bound=0.4) .limit(10) .to_pandas() @@ -292,7 +328,7 @@ results = ( ```typescript TypeScript icon="square-js" const results = await table .query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .distanceRange(0.0, 0.4) .limit(10) @@ -304,18 +340,21 @@ Either bound can be omitted to leave that side unbounded. ### Prefilter vs. postfilter {#prefilter-vs-postfilter} -When the query carries a metadata filter via `where(...)`, you can choose whether the filter runs before or after the vector and FTS sub-queries. **Prefiltering** (the default) applies `where` to the candidate set before scoring, which is usually what you want — it shrinks the working set and benefits from any scalar indexes on the filter columns. **Postfiltering** runs the filter on the already-ranked top-k from each sub-query; this can be faster when the filter is non-selective or unindexed, but it may return fewer than `limit` rows because some of the top-k may be filtered out. +A hybrid query applies its metadata filter to both the vector and full-text +searches. Prefiltering, the default, restricts eligible rows before retrieval. +Postfiltering removes candidates after retrieval and can leave fewer results +than requested. See [Filtering](/search/filtering). ```python Python icon="python" # Prefilter (default): filter applied before scoring -table.search("flower moon", query_type="hybrid") \ +table.search("rebel spaceships", query_type="hybrid") \ .where("category = 'film'", prefilter=True) \ .limit(10) \ .to_pandas() # Postfilter: filter applied after the sub-queries return top-k -table.search("flower moon", query_type="hybrid") \ +table.search("rebel spaceships", query_type="hybrid") \ .where("category = 'film'", prefilter=False) \ .limit(10) \ .to_pandas() @@ -324,7 +363,7 @@ table.search("flower moon", query_type="hybrid") \ ```typescript TypeScript icon="square-js" // Prefilter (default): just call .where(...) await table.query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .where("category = 'film'") .limit(10) @@ -332,7 +371,7 @@ await table.query() // Postfilter: chain .postfilter() after .where(...) await table.query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .where("category = 'film'") .postfilter() @@ -341,15 +380,20 @@ await table.query() ``` -The choice gets baked into both sub-queries, so the vector and FTS halves see the filter applied the same way. Use [`explain_plan`](/search/explain#analyzing-non-vector-queries) on a hybrid query to see whether the filter pushed into the scan or ran as a separate `FilterExec` step. +In Python, use [`explain_plan`](/search/explain#analyzing-non-vector-queries) on a hybrid query to see whether the filter pushed into the scan or ran as a separate `FilterExec` step. -## More on Reranking {#more-on-reranking} +## Choose and evaluate a reranker {#more-on-reranking} -You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB comes with [**built-in rerankers**](https://docs.lancedb.com/reranking) and you can implement your own **custom reranker** as well. +Start with RRF, then compare alternatives on representative queries. Review +which relevant rows reach the candidate set before evaluating how a reranker +orders them. Record latency as well as relevance: increasing candidate counts +or applying a model-based reranker adds work. -By default, LanceDB uses `RRFReranker()`, which uses reciprocal rank fusion score, to combine and rerank the results of semantic and full-text search. You can customize the hyperparameters as needed or write your own custom reranker. Here's how you can use any of the available rerankers: +Reranking runs in the application for both OSS and Enterprise connections. +See [Reranking](/reranking) for model-free fusion, model-based rerankers, +custom implementations and score interpretation. -| Argument | Type | Default | Description | -|:---------|:-----|:--------|:------------| -| `normalize` | `str` | `"score"` | The method to normalize the scores. Can be `rank` or `score`. If `rank`, the scores are converted to ranks and then normalized. If `score`, the scores are normalized directly. | -| `reranker` | `Reranker` | `RRF()` | The reranker to use. If not specified, the default reranker is used. | +In synchronous Python, `.rerank(...)` accepts a reranker and a `normalize` +option (`"score"` or `"rank"`). Normalization matters for methods that blend +raw scores; RRF uses rank positions. These are Python query options, not a +shared signature across every SDK. diff --git a/docs/web/search/index.mdx b/docs/web/search/index.mdx index 886751331..9d439b204 100644 --- a/docs/web/search/index.mdx +++ b/docs/web/search/index.mdx @@ -1,88 +1,118 @@ --- title: Search sidebarTitle: Overview -description: "Comprehensive guide to all search capabilities in LanceDB including vector search, full-text search, hybrid search, and more." +description: "Choose vector, full-text or hybrid search and return the records your application needs." icon: "list" +availability: + oss: available + enterprise: available + summary: >- + Vector, full-text and hybrid search are available in OSS and Enterprise. --- -| Feature | Description | -|:---------------|:------------| -| [Vector Search](/search/vector-search/) | Semantic similarity search with multiple distance metrics | -| [Multivector Search](/search/vector-search) | Search using multiple vector embeddings per document | -| [Full-Text Search](/search/full-text-search/) | Keyword-based search with BM25 and pre-filtering | -| [Hybrid Search](/search/hybrid-search/) | Combines vector and full-text search with reranking | -| [Filtering](/search/filtering/) | Filter results based on metadata fields | -| [SQL Queries](/sql) | SQL query capabilities for data exploration and analytics | +import { PyVectorSearch, TsVectorSearchBody, TsVectorSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsVectorSearchBody, RsExampleTail as RsVectorSearchTail } from '/snippets/search_vector_search.mdx'; + +LanceDB supports vector, full-text and hybrid search. Use vector search to find +similar content, full-text search to match words, or hybrid search to combine +both. Filters narrow the results to records that match your criteria. + +## Choose a search method + +| You want to find | Use | Prepare | +|:----------------|:----|:--------| +| Items similar to a query, such as related passages or images | [Vector search](/search/vector-search) | Stored vectors and a compatible query vector | +| Documents represented by several embeddings | [Multivector search](/search/vector-search#multivector) | A multivector column and compatible query vectors | +| Records containing particular words or phrases | [Full-text search](/search/full-text-search) | An FTS index on the text column | +| Relevant meaning alongside keyword matches | [Hybrid search](/search/hybrid-search) | Vectors and an FTS index | +| Rows matching a condition, without similarity ranking | [Filtering](/search/filtering) | A predicate over table columns | + +Vector search can run without an index. An approximate nearest-neighbor index +reduces the work needed for larger tables, with a tradeoff between recall and +latency. Full-text search needs an FTS index before you query it. ## Before you search {#before-you-search} -- Vector search can run without an ANN index as an exhaustive scan. That's useful while prototyping, but build a vector index before relying on low-latency searches over larger tables. -- Full-text and hybrid text search require an FTS index on the text column you query. If a table has multiple FTS indexes, specify the target column. FTS also supports phrase, boolean, boosted, multi-match, and fuzzy query forms when you need more than plain terms. -- Phrase FTS queries require an index created with token positions enabled. -- Multivector search currently uses cosine similarity and accepts either one query vector or a matrix of query vectors; every query vector must match the inner dimension of the multivector column. -- Set an explicit `.limit(...)` for production queries. The default top-k is 10 for search builders, - but spelling it out makes latency and result-count assumptions visible. Query builders also - support controls such as prefilter/postfilter, distance ranges, row-id inclusion, offset - pagination, and Arrow/Pandas/list result materialization. +For vector search, generate stored and query vectors with the same embedding +model and compatible settings. Their dimensions must match. Keep the fields +you want to return, such as document text or an item ID, in the same table. -## Performance {#performance} +For full-text search, choose the text column and build its index. Enterprise +builds indexes in the background, so wait for the index to be ready before +querying it. Phrase queries also need token positions in the index. See +[FTS indexes](/indexing/fts-index). -Three knobs materially affect query latency, memory use, and recall. Be deliberate about each one on every query: + - - - If queries return more data than you need or take longer than expected, the cause is usually projecting more columns than necessary, or letting the result count run unbounded. The best practice is to always pass both `select()` and `limit()`: + - ```python Python icon="python" - table.search(emb).select(["id", "title"]).limit(20) - ``` - - - If a query with `prefilter=False` returns fewer than `limit` results — sometimes zero — the cause is that post-filter applies the predicate after the top-k is selected, so only candidates that already passed the search are filtered. Pre-filter is the default and guarantees every result satisfies the predicate. Switch to `prefilter=False` only when fewer-than-`limit` results are acceptable. See [Filtering](/search/filtering). - - - If recall is lower than expected, the cause is that the search-time knobs limit how many candidates the index considers. Adjust based on your index type: +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. - - Quantized indexes (PQ, RQ, SQ): `refine_factor` pulls extra candidates and re-scores them on full vectors. - - HNSW-backed indexes: `ef` at search time. Start at `1.5 × k`, raise toward `10 × k` if recall is short. - - IVF candidate breadth: `nprobes` is auto-tuned; override only when a selective pre-filter leaves too few neighbors. - - +- **Python:** Install with `pip install lancedb`, save as `search.py`, then run `python search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `search.mts`, then run `node search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin vector_search` from the extracted directory. -For hybrid search, the default `RRFReranker()` combines vector and FTS results into a single ranking via reciprocal rank fusion. See [Hybrid Search](/search/hybrid-search). + + +{PyVectorSearch} + + +{TsVectorSearchBody + "\n" + TsVectorSearchTail} + + +{RsVectorSearchBody + "\n" + RsVectorSearchTail} + + -#### Avoid materializing the whole table {#avoid-materializing-the-whole-table} +This query returns the puppy and dog records, ordered by vector distance. +It searches the three-row table directly, without an ANN index or an embedding +service. For connection options, see [Connect](/basics/connect). -If you need every row in the table (for training, export, or migration), calling `to_pandas()` or `to_arrow()` will run you out of memory on any non-trivial dataset — both materialize the full table at once. The best practice is to iterate via `table.search(...)` or `table.query(...)`, which work the same way in both LanceDB OSS and Enterprise. + -In OSS, you can also stream batches through the underlying Lance dataset directly — useful when you need filter pushdown or fragment-level parallelism via `ds.scanner(...)`: +## Read the results -```python Python icon="python" -ds = table.to_lance() -for batch in ds.to_batches(columns=["id", "text"], batch_size=10000): - process(batch) -``` +Use `select()` to choose the columns your application needs and `limit()` to +set the maximum number of results. The search also returns a ranking value: - -**API disparity between OSS and Enterprise.** +| Search method | Ranking column | Order | +|:--------------|:---------------|:------| +| Vector | `_distance` | Smaller distance first | +| Full-text | `_score` | Higher text relevance first | +| Hybrid | `_relevance_score` | Higher combined relevance first | -LanceDB OSS exposes the `to_pandas()`, `to_arrow()`, and `table.to_lance()` methods for direct Lance dataset access. Enterprise's `RemoteTable` exposes none of these. Only `table.search(...)` and `table.query(...)`. In general, it's always best to go through `search()` and `query()` to keep your code portable across both OSS and Enterprise. - +These values have different meanings. Do not compare a vector distance directly +with a full-text score or treat a reranker score as a probability. -#### Diagnostics {#diagnostics} +## Add filters and reranking {#performance} -To analyze a slow query, inspect what the query engine actually did and the state of the indexes it touched. These two tools surface that information — use them in this order: +[Filters](/search/filtering) restrict which rows qualify, such as documents +from a particular collection. Prefiltering searches within those rows. +Postfiltering removes nonmatching rows from the candidates already retrieved, +so it can leave fewer results than requested. -```python Python icon="python" -print(table.search(emb).where("year > 2000").limit(10).analyze_plan()) -print(table.index_stats("vector_idx")) # num_unindexed_rows should be ~0 -``` +[Reranking](/reranking) reorders the candidates returned by retrieval. Start +with representative queries and expected results, then compare changes to +relevance and latency. A reranker cannot recover a relevant row that retrieval +never returned. -`analyze_plan()` returns the execution plan with per-stage timings, so you can see where the query actually spent its time. `index_stats()` shows how many rows are still unindexed — you want `num_unindexed_rows` to be ~0. In `analyze_plan`, look for: +## Read large result sets in batches {#avoid-materializing-the-whole-table} -| Plan pattern | Fix | -|--------------|-----| -| `LanceScan` with high `bytes_read` / `iops` | Add a missing index, project columns with `select()`, or check that the dataset has been compacted | -| Multiple sequential filters | Reorder filter conditions | +Materializing all rows at once can exhaust application memory. For exports and +training reads, use bounded queries or the batch-reading interfaces described +in [Data loading](/training/data-loading). The SDK buffers Enterprise +query responses before exposing batches. Set a query limit rather than relying +on batch iteration alone to bound memory. -[Explain Analyze](/search/explain) walks through a fully worked before/after example, including how `KNNVectorDistance` and `output_batches` change once indexes are in place. +Direct Lance dataset access through Python's `table.to_lance()` is an OSS +interface. Use the SDK query builders for reads that need to work with both +OSS and Enterprise. + +## Investigate a slow query {#diagnostics} + +Use [query plans](/search/explain) to see the scans, filters and indexes selected +for a query. `explain_plan()` inspects the plan; `analyze_plan()` executes the +query and reports measurements. Compare one change at a time, such as adding +an index, returning fewer columns, or adjusting search parameters. diff --git a/docs/web/search/vector-search.mdx b/docs/web/search/vector-search.mdx index c7324701a..3519df553 100644 --- a/docs/web/search/vector-search.mdx +++ b/docs/web/search/vector-search.mdx @@ -1,8 +1,13 @@ --- title: "Vector Search" sidebarTitle: "Vector Search" -description: "Learn how to run vector search queries in LanceDB. Includes best practices, tips and examples." +description: "Search by vector similarity, choose a distance metric and tune your queries." icon: "arrow-up-right-dots" +availability: + oss: available + enterprise: available + summary: >- + Vector search is available in OSS and Enterprise. --- import { @@ -43,31 +48,68 @@ import { RsBinarySearch, } from '/snippets/search.mdx'; -Vector search is a technique used to search for similar items based on their vector representations, called embeddings. It is also known as similarity search, nearest neighbor search, or approximate nearest neighbor search. +import { PyVectorSearch, TsVectorSearchBody, TsVectorSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsVectorSearchBody, RsExampleTail as RsVectorSearchTail } from '/snippets/search_vector_search.mdx'; -![](/static/assets/images/search/vector-db-basics.png) +Vector search finds similar items by comparing their embeddings: numerical +representations of text, images, audio or other data. An embedding model maps +related items to nearby points in vector space, so you can search by meaning +or similarity instead of matching exact words. -Raw data (e.g. text, images, audio, etc.) is converted into embeddings via an embedding model, which are then stored in a multimodal lakehouse like LanceDB. To perform similarity search at scale, an index is created on the stored embeddings, which can then used to perform fast lookups. +![An embedding model maps data into vectors for similarity search.](/static/assets/images/search/vector-db-basics.png) - -Vector search has **no SQL form**. Run it here and use [SQL](/sql) over the result. - +To run a search, pass a query vector to LanceDB and specify how many matches +to return. Generate the query vector with the same model and settings used for +your stored embeddings. LanceDB ranks the results by distance from the query. -## Supported distance metrics +For small tables, you can search the vectors directly. A [vector index](/indexing/vector-index) +speeds up searches on larger tables by narrowing the candidates to examine. -Distance metrics determine how LanceDB compares vectors to find similar matches. Euclidean or `l2` is the default, and used for general-purpose similarity, `cosine` for unnormalized embeddings, `dot` for normalized embeddings (best performance), or `hamming` for binary vectors. + - -Ensure you always use the same distance metric that your embedding model was trained with. Most modern embedding models use cosine similarity, so `cosine` is often the best choice. However, if your vectors are normalized, you should use `dot` for best performance. - + -The right metric improves both search accuracy and query performance. Currently, LanceDB supports the following metrics: +These examples create a three-row table and find the two closest matches to a +query vector. They use hand-written vectors, so you can try the query without +an embedding model. The temporary table is removed when the program finishes. + +- **Python:** Install with `pip install lancedb`, save as `vector_search.py`, then run `python vector_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `vector_search.mts`, then run `node vector_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin vector_search` from the extracted directory. + + + +{PyVectorSearch} + + +{TsVectorSearchBody + "\n" + TsVectorSearchTail} + + +{RsVectorSearchBody + "\n" + RsVectorSearchTail} + + + +The first result is the puppy record, whose vector exactly matches the query. +The dog record is next. `_distance` is smaller for closer matches. This example +uses an exhaustive scan; the indexing section explains how to search larger tables. + + + +## Supported distance metrics {#supported-distance-metrics} + +A distance metric determines how LanceDB compares vectors. Choose the metric +recommended by your embedding model. Use the same metric when building an index +and querying it. + +LanceDB supports `l2`, `cosine`, `dot` and `hamming`. An unindexed search uses +`l2` by default; Hamming distance is for binary vectors. The table below shows +the value returned in the `_distance` column. Smaller values rank first. | Distance metric | Mathematical form | Notes | |---|---|---| -| `l2` | $$\|x-y\|_2= \sqrt{\sum_i (x_i-y_i)^2}$$ | The $\ell_2$ norm. Measures Euclidean distance between two points in geometric space. | +| `l2` | $\sum_i (x_i-y_i)^2$ | Squared Euclidean distance returned in `_distance`. | | `cosine` | $1-\frac{x\cdot y}{\|x\|_2\|y\|_2}$ | Measures directional difference between vectors. Insensitive to nonzero scaling of vectors. | -| `dot` | $x\cdot y= \underset{i}{\sum} x_i y_i$ | The standard dot product of real vectors. Sensitive to nonzero scaling of vectors; use on normalized vectors. | +| `dot` | $1-\sum_i x_i y_i$ | Dot-product distance. For unit-normalized vectors, this equals cosine distance. | | `hamming` | $\underset{i}{\sum} \,\: \mathbf{1}_{\, [x_i\neq y_i]}$ | Counts the number of bitwise-differing bits between binary vectors. | For indexed search, supported distance metrics vary by index type: @@ -111,12 +153,10 @@ non-`l2` index, explicitly set the matching metric or upgrade the SDK. -Here you can see the same search but using `cosine` similarity instead of `l2` -distance. If the table has a vector index, make sure it was also built with -`cosine`. The result focuses on vector direction rather than absolute distance, -which works better for unnormalized embeddings. +These examples use cosine distance and return up to ten matches. If the table +has a vector index, build it with the same metric. -Set `.limit(...)` on vector searches you run in applications. You can page through results with `.offset(...)` and include LanceDB's internal row id with `.with_row_id()` / `.withRowId()` when you need a stable handle for follow-up operations. +Set `.limit(...)` on vector searches you run in applications. You can page through results with `.offset(...)` and include LanceDB's internal row id with `.with_row_id()` / `.withRowId()` when an operation requires internal row IDs. Use your own ID column for application identifiers. ## Selecting the vector column @@ -153,13 +193,12 @@ The same field-path syntax works when creating an index on a nested vector colum When several columns share a name across structs (for example, `image.embedding` and `text.embedding`), LanceDB still picks the one whose dimension matches your query vector. If two candidates have the same dimension, you must pass the column name explicitly. -## Vector Search With ANN Index +## Search with a vector index {#vector-search-with-ann-index} -Instead of performing an exhaustive search on the entire database for each and every query, approximate nearest neighbour (ANN) algorithms use an index to narrow down the search space, which significantly reduces query latency. - -The trade-off is that the results are not guaranteed to be the true nearest neighbors of the query, but are usually "good enough" for most use cases. - -Use ANN search for large-scale applications where speed matters more than perfect recall. LanceDB uses approximate nearest neighbor algorithms to deliver fast results without examining every vector in your dataset. +An approximate nearest-neighbor (ANN) index narrows the set of vectors examined +for a query. It can reduce query latency, but may miss neighbors that an +exhaustive search would find. Compare recall and latency on queries that +represent your application before choosing an index and its search settings. When a vector index is used, `_distance` is not always the true distance between full vectors. On quantized ANN indexes, LanceDB may compute `_distance` from the compressed representation for speed. Use `refine_factor` when you want reranking on full vectors. @@ -193,8 +232,8 @@ The table below summarizes the behavior of `_distance` in search results based o -For deeper tuning guidance on indexing and performance estimation, see the [vector indexes](/indexing/vector-index) page, -For tuning `nprobes`, see below. +See [vector indexes](/indexing/vector-index) for index configuration. The +following sections cover query-time settings. ### Tuning `approx_mode` @@ -208,13 +247,13 @@ The supported values are: | `normal` | Use the default balance between query latency and recall. This only has an effect for RQ indexes built with `num_bits > 1`. | | `accurate` | Prefer higher recall, which can increase query latency. | -You can change `approx_mode` per query without rebuilding the index. For RQ indexes built with `num_bits=1`, `normal` uses the same one-bit scoring path as `fast`. If you also set `refine_factor`, LanceDB first uses `approx_mode` while finding candidates, then reranks the selected candidates on full vectors. +Only the Rust SDK sets `approx_mode`; the Python and TypeScript SDKs do not expose it. You can change `approx_mode` per query without rebuilding the index. For RQ indexes built with `num_bits=1`, `normal` uses the same one-bit scoring path as `fast`. If you also set `refine_factor`, LanceDB first uses `approx_mode` while finding candidates, then reranks the selected candidates on full vectors. ### Tuning `nprobes` - `nprobes` controls how many partitions are searched at query time. - `nprobes` improves candidate recall, but does not by itself make `_distance` exact. -- By default, LanceDB automatically tunes `nprobes` to achieve the best performance without noticeably sacrificing accuracy. +- The default probe selection is a starting point. Measure recall and latency for your data before overriding it. - In most cases, leave `nprobes` unset and use the auto-tuned value. - Only tune `nprobes` manually when recall is below your target, or when you need even higher performance for your workload. - If recall is too low, increase `nprobes` gradually, but after a certain threshold, increasing `nprobes` yields only marginal accuracy gains. @@ -223,6 +262,8 @@ You can change `approx_mode` per query without rebuilding the index. For RQ inde For filtered ANN searches, you can also set `minimum_nprobes` and `maximum_nprobes`. LanceDB starts with the minimum and can scan more partitions up to the maximum if the filter leaves too few candidates. Calling `nprobes(n)` fixes both values to `n`, which disables that adaptive behavior. +Whether a deployment honors this range depends on its version; one that does not searches exactly +`minimum_nprobes` partitions. ### Vector Search with Prefiltering @@ -242,15 +283,15 @@ This is the default vector search setting. You can use prefiltering to boost que -This filters out rows where label ≤ 2 before doing vector search, then picks specific columns from the top 5 matches. - -The `.where("label > 2")` applies a filter before vector search, `.select(["text", "keywords", "label"])` chooses specific columns to return, and `.limit(5)` restricts results to the top `5` most similar vectors. - -As a result, you'll see a result with just the data you want from the most similar vectors. +The filter limits which rows can match. Use `select()` to choose the returned +columns and `limit()` to set the maximum number of results. ### Vector Search with Postfiltering -Use postfiltering to prioritize vector similarity by searching the full dataset first, then applying metadata filters to the top results. This approach ensures you get the most similar vectors before filtering, which can be crucial when similarity is more important than metadata constraints. +Postfiltering selects vector-search candidates before applying the metadata +predicate. It can return fewer matches than requested, even when other rows in +the table satisfy the predicate. Use prefiltering when you want the nearest +records within the matching subset. @@ -266,11 +307,8 @@ Use postfiltering to prioritize vector similarity by searching the full dataset -Here you can see how to do vector search first to get the most similar vectors, then filter by label > 1 on those results. - -The `prefilter=False` parameter tells LanceDB to apply the filter after vector search instead of before, `.where("label > 1")` filters the top results by metadata, and `.select()` chooses which columns to include. - -In the end, you receive a query result with the best matches that also meet your metadata requirements. +Only retrieved candidates that satisfy the predicate remain in the result. +Increase the candidate limit or use prefiltering if too few records are returned. [Post-filtering](/search/filtering/#post-filtering-with-vector-search) in LanceDB applies @@ -282,7 +320,7 @@ the filter condition after obtaining the nearest neighbors based on vector simil Use multivector search when your documents contain multiple embeddings and you need sophisticated matching between query and document vector pairs. The late interaction approach finds the most relevant combinations across all available embeddings and provides nuanced similarity scoring. Only `cosine` similarity is supported as the distance metric for multivector search operations. -Every query vector must match the inner dimension of the multivector column; LanceDB rejects mismatched query dimensions rather than guessing how to reshape them. +Every query vector must match the inner dimension of the multivector column. @@ -292,7 +330,7 @@ Every query vector must match the inner dimension of the multivector column; Lan Here you can see how to take 2 query vectors and find the best matching pairs between them and document vectors using late interaction. The `np.random.random(size=(2, 256))` creates a 2×256 array with two random query vectors, `.limit(5)` returns the top 5 best document-query combinations, and `.to_pandas()` provides results in a DataFrame format. -**Read more:** [Multivector search](/search/vector-search) +**Read more:** [Multivector search walkthrough](#multivector) ## Advanced Search Scenarios @@ -350,9 +388,11 @@ The search produces an Arrow table with binary vectors ranked by how many bits d ## Scaling Vector Search -### Batch Search +### Query multiple vectors in a batch {#batch-search} -Use batch search to handle multiple query vectors simultaneously. This gives you significant efficiency gains over individual queries. LanceDB processes all vectors in parallel and organizes results with a `query_index` field that maps each result set back to its originating query. +Batch search accepts multiple query vectors in one request. The `query_index` +column identifies which query produced each result. Measure throughput and +latency with your workload when choosing a batch size. @@ -368,26 +408,16 @@ Use batch search to handle multiple query vectors simultaneously. This gives you -This takes 5 query embeddings and finds the top 5 matches for each one in a single batch operation. +Each input vector has its own nearest-neighbor results. Use `query_index` to +associate each row with the corresponding input vector. -The `load_dataset()` loads embeddings from a Hugging Face dataset, `query_embeds` contains `5` query vectors, and `.search(query_embeds)` processes all queries simultaneously. - -The final query result contains all results, including a `query_index` to tell you which query each result came from. - - -When processing batch queries, the results include a `query_index` field -to explicitly associate each result set with its corresponding query in -the input batch. - - -### Search With Asynchronous Indexing +### Include or skip unindexed rows {#search-with-asynchronous-indexing} To optimize for speed over completeness, enable the `fast_search` flag in your query to skip searching unindexed data. -While vector indexing occurs asynchronously, newly added vectors are immediately -searchable through a fallback brute-force search mechanism. This ensures zero -latency between data insertion and searchability, though it may temporarily -increase query response times. +By default, search includes both indexed vectors and unindexed rows visible to +the query. LanceDB scans the unindexed rows directly. `fast_search` skips that +scan, which can reduce query time but omit recent records. @@ -403,13 +433,9 @@ increase query response times. -Here you can see how to turn on fast search mode to skip unindexed vectors and only look through indexed data for speed. +Use this option only when results from indexed data are sufficient. -The `fast_search=True` parameter tells LanceDB to only search indexed vectors, skipping any recently added data that hasn't been indexed yet. - -You'll obtain a query result with the top `5` matches from indexed vectors, but might miss data that was just added. - -## Brute Force Search +## Search without a vector index {#brute-force-search} ### Search With No Index @@ -433,11 +459,13 @@ Choose brute force search when you need guaranteed 100% recall, typically with s -This carries out a brute force search through every vector in the table to find the 3 closest matches to a random 1536-dimensional query. You'll get back a list of the most similar vectors with exact distances. +These queries return the three nearest matches. Without a vector index, LanceDB +calculates distances directly from the stored vectors. ![](/static/assets/images/search/knn_search.png) -As you can imagine, the brute force approach is not scalable for datasets larger than a few hundred thousand vectors, as the latency of the search grows linearly with the size of the dataset. This is where approximate nearest neighbour (ANN) algorithms come in. +Exhaustive search does more work as the table grows. When query latency becomes +too high, add an ANN index and compare its results with the exhaustive baseline. ### Bypass the Vector Index @@ -461,7 +489,8 @@ This skips the approximate index and checks every single vector for exact, groun The `.bypass_vector_index()` method forces LanceDB to perform an exhaustive search through all vectors instead of using the approximate nearest neighbor index, ensuring exact results but at the cost of slower performance. -The output is a query result with the top 5 exact matches, guaranteeing 100% recall but taking longer to run. +The query returns the nearest five rows within its filters and distance bounds. +Use an exhaustive scan as a baseline when measuring ANN recall. This approach is particularly useful when: - Evaluating ANN index quality @@ -596,7 +625,7 @@ tbl.create_index(metric="cosine", vector_column_name="vector") A brute-force scan over a multivector column has to compare every query vector to every document vector in every row, so the cost grows with both the row count and the number of vectors per row. In LanceDB OSS, the query will just run, so a large unindexed multivector table can stall a process for a long time before returning results. -On LanceDB Cloud, the brute-force KNN safety check applies a stricter row threshold to multivector columns — roughly 10× lower than for single-vector columns. So an unindexed multivector table can be rejected with a "vector search would use brute-force KNN" error well before a comparable single-vector table would. LanceDB Enterprise doesn't have the limit. Build the index before you start hitting it from production traffic, even if the dataset is small enough that you'd skip indexing for a single-vector workload. +On LanceDB Cloud, the brute-force KNN safety check applies a stricter row threshold to multivector columns; roughly 10× lower than for single-vector columns. So an unindexed multivector table can be rejected with a "vector search would use brute-force KNN" error well before a comparable single-vector table would. LanceDB Enterprise doesn't have the limit. Build the index before you start hitting it from production traffic, even if the dataset is small enough that you'd skip indexing for a single-vector workload. #### 6. Query a Single Vector @@ -644,7 +673,7 @@ import lancedb from pylate import models ## 1) Load a late-interaction model via PyLate -## PyLate docs show ColBERT() + encode(..., is_query=...) :contentReference[oaicite:2]{index=2} +## PyLate docs show ColBERT() + encode(..., is_query=...) model = models.ColBERT(model_name_or_path="lightonai/GTE-ModernColBERT-v1") ## You can discover dim from one embedding (avoid guessing) @@ -655,7 +684,7 @@ db = lancedb.connect("./pylate_lancedb") schema = pa.schema([ pa.field("doc_id", pa.string()), pa.field("text", pa.string()), - # multivector: list> :contentReference[oaicite:3]{index=3} + # multivector: list> pa.field("mv", pa.list_(pa.list_(pa.float32(), dim))), ]) @@ -667,7 +696,7 @@ docs = [ ## 3) Encode documents with PyLate (token vectors per doc) doc_texts = [d["text"] for d in docs] -doc_embs = model.encode(doc_texts, is_query=False) # list/array of (T, dim) per doc :contentReference[oaicite:4]{index=4} +doc_embs = model.encode(doc_texts, is_query=False) # list/array of (T, dim) per doc rows = [] for d, emb in zip(docs, doc_embs): @@ -678,14 +707,14 @@ tbl = db.create_table("docs", data=rows, schema=schema, mode="overwrite") ## 4) Build an index + query using a query matrix. ## Multivector brute-force scales with rows × vectors-per-row, so build the index -## at much smaller dataset sizes than you would for single-vector search — and +## at much smaller dataset sizes than you would for single-vector search; and ## always before exposing the table to remote traffic. tbl.create_index(vector_column_name="mv", metric="cosine") query = "Tell me about ramen in Japan" -q_emb = np.asarray(model.encode([query], is_query=True)[0], dtype=np.float32) # (Tq, dim) :contentReference[oaicite:5]{index=5} +q_emb = np.asarray(model.encode([query], is_query=True)[0], dtype=np.float32) # (Tq, dim) -out = tbl.search(q_emb).limit(5).to_pandas() # multivector search accepts a matrix :contentReference[oaicite:6]{index=6} +out = tbl.search(q_emb).limit(5).to_pandas() # multivector search accepts a matrix print(out[["doc_id", "text"]]) ``` diff --git a/docs/web/snippets/ingestion_delete.mdx b/docs/web/snippets/ingestion_delete.mdx new file mode 100644 index 000000000..f6f6222ae --- /dev/null +++ b/docs/web/snippets/ingestion_delete.mdx @@ -0,0 +1,4 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExample = "use std::sync::Arc;\n\nuse arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase};\n\nfn make_users_reader(rows: &[(i64, &str)]) -> Box {\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"name\", DataType::Utf8, false),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),\n Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),\n ],\n )\n .unwrap();\n Box::new(RecordBatchIterator::new(\n vec![Ok(batch)].into_iter(),\n schema,\n ))\n}\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let table = db\n .create_table(\n \"users\",\n make_users_reader(&[(1, \"Alice\"), (2, \"Bob\"), (3, \"Charlie\")]),\n )\n .execute()\n .await?;\n let predicate = \"id = 3\";\n let selected: Vec = table\n .query()\n .only_if(predicate)\n .execute()\n .await?\n .try_collect()\n .await?;\n println!(\"Selected: {selected:?}\");\n table.delete(predicate).await?;\n let rows: Vec = table.query().execute().await?.try_collect().await?;\n println!(\"Remaining: {rows:?}\");\n assert_eq!(table.count_rows(None).await?, 2);\n assert_eq!(\n table\n .count_rows(Some(\"id = 1 AND name = 'Alice'\".into()))\n .await?,\n 1\n );\n assert_eq!(\n table\n .count_rows(Some(\"id = 2 AND name = 'Bob'\".into()))\n .await?,\n 1\n );\n assert_eq!(table.count_rows(Some(\"id = 3\".into())).await?, 0);\n Ok(())\n}\n"; + diff --git a/docs/web/snippets/ingestion_examples.mdx b/docs/web/snippets/ingestion_examples.mdx new file mode 100644 index 000000000..0268a906a --- /dev/null +++ b/docs/web/snippets/ingestion_examples.mdx @@ -0,0 +1,66 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PyAddFromDataset = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n from pathlib import Path\n\n import pyarrow as pa\n import pyarrow.parquet as pq\n\n data_path = str(Path(uri) / \"input.parquet\")\n pq.write_table(pa.table({\"id\": [1, 2], \"name\": [\"Alice\", \"Bob\"]}), data_path)\n import pyarrow.dataset as ds\n\n dataset = ds.dataset(data_path, format=\"parquet\")\n table = db.create_table(\"my_table\", schema=dataset.schema, mode=\"overwrite\")\n table.add(dataset)\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyAddProgress = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\ndef report_progress(progress):\n print(f\"Rows written: {progress['output_rows']}\")\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\"users\", [{\"id\": 1, \"name\": \"Alice\"}])\n table.add(\n [{\"id\": 2, \"name\": \"Bob\"}, {\"id\": 3, \"name\": \"Charlie\"}],\n progress=report_progress,\n )\n rows = sorted(table.search().to_list(), key=lambda row: row[\"id\"])\n print(\"After insertion:\", rows)\n"; + +export const PyAppendRows = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\"users\", [{\"id\": 1, \"name\": \"Alice\"}])\n table.add([{\"id\": 2, \"name\": \"Bob\"}, {\"id\": 3, \"name\": \"Charlie\"}])\n rows = sorted(table.search().to_list(), key=lambda row: row[\"id\"])\n print(rows)\n"; + +export const PyCreateEmptyTable = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n schema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n pa.field(\"item\", pa.string()),\n pa.field(\"price\", pa.float32()),\n ]\n )\n tbl = db.create_table(\"test_empty_table\", schema=schema, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateEmptyTablePydantic = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n from lancedb.pydantic import LanceModel, Vector\n\n class Item(LanceModel):\n vector: Vector(2)\n item: str\n price: float\n\n tbl = db.create_table(\n \"test_empty_table_new\", schema=Item.to_arrow_schema(), mode=\"overwrite\"\n )\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableConflictHandling = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n data = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7}]\n db.create_table(\"conflict_table\", data)\n # Idempotent open: reuse the existing table if it exists.\n # The provided data is ignored; the schema is validated against the\n # existing table and a mismatch raises an error.\n tbl = db.create_table(\"conflict_table\", data, exist_ok=True)\n\n # Overwrite: replace the contents with the provided data.\n tbl = db.create_table(\"conflict_table\", data, mode=\"overwrite\")\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableCustomSchema = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n custom_schema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 4)),\n pa.field(\"lat\", pa.float32()),\n pa.field(\"long\", pa.float32()),\n ]\n )\n\n data = [\n {\"vector\": [1.1, 1.2, 1.3, 1.4], \"lat\": 45.5, \"long\": -122.7},\n {\"vector\": [0.2, 1.8, 0.4, 3.6], \"lat\": 40.1, \"long\": -74.1},\n ]\n tbl = db.create_table(\n \"my_table_custom_schema\", data, schema=custom_schema, mode=\"overwrite\"\n )\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableFromArrow = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import numpy as np\n import pyarrow as pa\n\n dim = 16\n total = 2\n schema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float16(), dim)),\n pa.field(\"text\", pa.string()),\n ]\n )\n data = pa.Table.from_arrays(\n [\n pa.array(\n [np.random.randn(dim).astype(np.float16) for _ in range(total)],\n pa.list_(pa.float16(), dim),\n ),\n pa.array([\"foo\", \"bar\"]),\n ],\n [\"vector\", \"text\"],\n )\n tbl = db.create_table(\"f16_tbl\", data, schema=schema, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableFromDicts = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n data = [\n {\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7},\n {\"vector\": [0.2, 1.8], \"lat\": 40.1, \"long\": -74.1},\n ]\n db.create_table(\"test_table\", data, mode=\"overwrite\")\n tbl = db[\"test_table\"]\n tbl.head()\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableFromIterator = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n schema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 4)),\n pa.field(\"item\", pa.utf8()),\n pa.field(\"price\", pa.float32()),\n ]\n )\n\n def make_batches():\n for i in range(5):\n yield pa.RecordBatch.from_arrays(\n [\n pa.array(\n [[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],\n pa.list_(pa.float32(), 4),\n ),\n pa.array([\"foo\", \"bar\"]),\n pa.array([10.0, 20.0]),\n ],\n [\"vector\", \"item\", \"price\"],\n )\n\n table = db.create_table(\"batched_table\", schema=schema)\n table.add(make_batches())\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyCreateTableFromPandas = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pandas as pd\n\n data = pd.DataFrame(\n {\n \"vector\": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],\n \"lat\": [45.5, 40.1],\n \"long\": [-122.7, -74.1],\n }\n )\n db.create_table(\"my_table_pandas\", data, mode=\"overwrite\")\n db[\"my_table_pandas\"].head()\n"; + +export const PyCreateTableFromPolars = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import polars as pl\n\n data = pl.DataFrame(\n {\n \"vector\": [[3.1, 4.1], [5.9, 26.5]],\n \"item\": [\"foo\", \"bar\"],\n \"price\": [10.0, 20.0],\n }\n )\n tbl = db.create_table(\"my_table_pl\", data, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableFromPydantic = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n from lancedb.pydantic import LanceModel, Vector\n\n class Content(LanceModel):\n movie_id: int\n vector: Vector(128)\n genres: str\n title: str\n imdb_id: int\n\n @property\n def imdb_url(self) -> str:\n return f\"https://www.imdb.com/title/tt{self.imdb_id}\"\n\n tbl = db.create_table(\"movielens_small\", schema=Content, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableNestedSchema = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n from lancedb.pydantic import LanceModel, Vector\n from pydantic import BaseModel\n\n class Document(BaseModel):\n content: str\n source: str\n\n class NestedSchema(LanceModel):\n id: str\n vector: Vector(1536)\n document: Document\n\n tbl = db.create_table(\"nested_table\", schema=NestedSchema, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyDeleteRows = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"users\",\n [\n {\"id\": 1, \"name\": \"Alice\"},\n {\"id\": 2, \"name\": \"Bob\"},\n {\"id\": 3, \"name\": \"Charlie\"},\n ],\n )\n predicate = \"id = 3\"\n print(\"Selected:\", table.search().where(predicate).to_list())\n table.delete(predicate)\n rows = sorted(table.search().to_list(), key=lambda row: row[\"id\"])\n print(\"Remaining:\", rows)\n"; + +export const PyDropTable = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n # Create a table first\n data = [{\"vector\": [1.1, 1.2], \"lat\": 45.5}]\n db.create_table(\"my_table\", data, mode=\"overwrite\")\n\n # Drop the table\n db.drop_table(\"my_table\")\n"; + +export const PyInsertIfNotExists = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n )\n\n (table.merge_insert(\"id\").when_not_matched_insert_all().execute(incoming_users))\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyMergeDeleteMissingBySource = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2, 3],\n \"name\": [\"Alice\", \"Bob\", \"Charlie\"],\n \"login_count\": [10, 20, 5],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n )\n\n (\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .when_not_matched_by_source_delete()\n .execute(incoming_users)\n )\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyMergeMatchedUpdateOnly = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n )\n\n (table.merge_insert(\"id\").when_matched_update_all().execute(incoming_users))\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyMergePartialColumns = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n }\n )\n\n (\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .execute(incoming_users)\n )\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyMergeUpdateInsert = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n )\n\n (\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .execute(incoming_users)\n )\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyOpenExistingTable = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n # Create a table first\n data = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7}]\n db.create_table(\"test_table\", data, mode=\"overwrite\")\n\n # List table names\n print(db.list_tables().tables)\n\n # Open existing table\n tbl = db.open_table(\"test_table\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyUpdateOperation = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n table.update(where=\"id = 2\", values={\"name\": \"Bobby\"})\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyUpdateUsingSql = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n table.update(where=\"id = 2\", values_sql={\"login_count\": \"login_count + 1\"})\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyWriteWorkflow = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"users\",\n [\n {\"id\": 1, \"name\": \"Alice\", \"login_count\": 10},\n {\"id\": 2, \"name\": \"Bob\", \"login_count\": 20},\n ],\n )\n table.add([{\"id\": 3, \"name\": \"Charlie\", \"login_count\": 5}])\n table.update(where=\"id = 2\", values={\"name\": \"Bobby\"})\n table.update(where=\"id = 2\", values_sql={\"login_count\": \"login_count + 1\"})\n (\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .execute(\n [\n {\"id\": 3, \"name\": \"Charlie\", \"login_count\": 6},\n {\"id\": 4, \"name\": \"Dana\", \"login_count\": 1},\n ]\n )\n )\n print(\"Rows selected for deletion:\", table.search().where(\"id = 4\").to_list())\n table.delete(\"id = 4\")\n rows = sorted(table.search().to_list(), key=lambda row: row[\"id\"])\n print(rows)\n"; + +export const TsAddProgressBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-ingestion-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"users\", [{ id: 1, name: \"Alice\" }]);\n await table.add(\n [\n { id: 2, name: \"Bob\" },\n { id: 3, name: \"Charlie\" },\n ],\n {\n progress: (progress) =>\n console.log(`Rows written: ${progress.outputRows}`),\n },\n );\n const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);\n console.log(\"After insertion:\", rows);\n"; + +export const TsAddProgressTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsDeleteRowsBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-ingestion-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"users\", [\n { id: 1, name: \"Alice\" },\n { id: 2, name: \"Bob\" },\n { id: 3, name: \"Charlie\" },\n ]);\n const predicate = \"id = 3\";\n console.log(\"Selected:\", await table.query().where(predicate).toArray());\n await table.delete(predicate);\n const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);\n console.log(\"Remaining:\", rows);\n"; + +export const TsDeleteRowsTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsWriteWorkflowBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\n\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-ingestion-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"users\", [\n { id: 1, name: \"Alice\", login_count: 10 },\n { id: 2, name: \"Bob\", login_count: 20 },\n ]);\n await table.add([{ id: 3, name: \"Charlie\", login_count: 5 }]);\n await table.update({ where: \"id = 2\", values: { name: \"Bobby\" } });\n await table.update({\n where: \"id = 2\",\n valuesSql: { login_count: \"login_count + 1\" },\n });\n await table\n .mergeInsert(\"id\")\n .whenMatchedUpdateAll()\n .whenNotMatchedInsertAll()\n .execute([\n { id: 3, name: \"Charlie\", login_count: 6 },\n { id: 4, name: \"Dana\", login_count: 1 },\n ]);\n console.log(\n \"Rows selected for deletion:\",\n await table.query().where(\"id = 4\").toArray(),\n );\n await table.delete(\"id = 4\");\n const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);\n console.log(rows);\n"; + +export const TsWriteWorkflowTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const RsWriteWorkflow = "use std::sync::Arc;\n\nuse arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase};\n\nfn make_users_reader(rows: &[(i64, &str, i64)]) -> Box {\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"name\", DataType::Utf8, false),\n Field::new(\"login_count\", DataType::Int64, false),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),\n Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),\n Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.2))),\n ],\n )\n .unwrap();\n Box::new(RecordBatchIterator::new(\n vec![Ok(batch)].into_iter(),\n schema,\n ))\n}\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let table = db\n .create_table(\n \"users\",\n make_users_reader(&[(1, \"Alice\", 10), (2, \"Bob\", 20)]),\n )\n .execute()\n .await?;\n table\n .add(make_users_reader(&[(3, \"Charlie\", 5)]))\n .execute()\n .await?;\n table\n .update()\n .only_if(\"id = 2\")\n .column(\"name\", \"'Bobby'\")\n .execute()\n .await?;\n table\n .update()\n .only_if(\"id = 2\")\n .column(\"login_count\", \"login_count + 1\")\n .execute()\n .await?;\n let mut merge = table.merge_insert(&[\"id\"]);\n merge\n .when_matched_update_all(None)\n .when_not_matched_insert_all();\n merge\n .execute(make_users_reader(&[(3, \"Charlie\", 6), (4, \"Dana\", 1)]))\n .await?;\n let selected: Vec = table\n .query()\n .only_if(\"id = 4\")\n .execute()\n .await?\n .try_collect()\n .await?;\n println!(\"Rows selected for deletion: {selected:?}\");\n table.delete(\"id = 4\").await?;\n let rows: Vec = table.query().execute().await?.try_collect().await?;\n println!(\"Remaining rows: {rows:?}\");\n assert_eq!(table.count_rows(None).await?, 3);\n assert_eq!(\n table\n .count_rows(Some(\n \"id = 2 AND name = 'Bobby' AND login_count = 21\".into()\n ))\n .await?,\n 1\n );\n assert_eq!(\n table\n .count_rows(Some(\"id = 3 AND login_count = 6\".into()))\n .await?,\n 1\n );\n Ok(())\n}\n"; + diff --git a/docs/web/snippets/ingestion_progress.mdx b/docs/web/snippets/ingestion_progress.mdx new file mode 100644 index 000000000..d910468f6 --- /dev/null +++ b/docs/web/snippets/ingestion_progress.mdx @@ -0,0 +1,4 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExample = "use std::sync::Arc;\n\nuse arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::ExecutableQuery;\n\nfn make_users_reader(rows: &[(i64, &str)]) -> Box {\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"name\", DataType::Utf8, false),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),\n Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),\n ],\n )\n .unwrap();\n Box::new(RecordBatchIterator::new(\n vec![Ok(batch)].into_iter(),\n schema,\n ))\n}\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let table = db\n .create_table(\"users\", make_users_reader(&[(1, \"Alice\")]))\n .execute()\n .await?;\n table\n .add(make_users_reader(&[(2, \"Bob\"), (3, \"Charlie\")]))\n .progress(|progress| println!(\"Rows written: {}\", progress.output_rows()))\n .execute()\n .await?;\n let rows: Vec = table.query().execute().await?.try_collect().await?;\n println!(\"After insertion: {rows:?}\");\n assert_eq!(table.count_rows(None).await?, 3);\n assert_eq!(\n table\n .count_rows(Some(\"id = 1 AND name = 'Alice'\".into()))\n .await?,\n 1\n );\n assert_eq!(\n table\n .count_rows(Some(\"id = 2 AND name = 'Bob'\".into()))\n .await?,\n 1\n );\n assert_eq!(table.count_rows(Some(\"id = 3\".into())).await?, 1);\n Ok(())\n}\n"; + diff --git a/docs/web/snippets/integrations.mdx b/docs/web/snippets/integrations.mdx index 22bbde338..67bb654cf 100644 --- a/docs/web/snippets/integrations.mdx +++ b/docs/web/snippets/integrations.mdx @@ -88,7 +88,7 @@ export const PyFrameworksLlamaindexHybridSearch = "from lancedb.rerankers import export const PyFrameworksLlamaindexQuickStart = "import logging\nimport sys\nimport textwrap\n\nimport openai\n\n# Uncomment to see debug logs\n# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)\n# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))\nfrom llama_index.core import (\n Document,\n SimpleDirectoryReader,\n StorageContext,\n VectorStoreIndex,\n)\nfrom llama_index.vector_stores.lancedb import LanceDBVectorStore\n\nopenai.api_key = \"sk-...\"\n\ndocuments = SimpleDirectoryReader(\"./data/your-data-dir/\").load_data()\nprint(\"Document ID:\", documents[0].doc_id, \"Document Hash:\", documents[0].hash)\n\n## For LanceDB Enterprise :\n# vector_store = LanceDBVectorStore(\n# uri=\"db://db_name\", # your remote DB URI\n# api_key=\"sk_..\", # lancedb enterprise api key\n# region=\"your-region\" # the region you configured\n# host_override=\"https://your-host.com\" # if you have a custom host, otherwise omit this\n# )\n\nvector_store = LanceDBVectorStore(\n uri=\"./lancedb\", mode=\"overwrite\", query_type=\"vector\"\n)\nstorage_context = StorageContext.from_defaults(vector_store=vector_store)\n\nindex = VectorStoreIndex.from_documents(documents, storage_context=storage_context)\nlance_filter = \"metadata.file_name = 'paul_graham_essay.txt' \"\nretriever = index.as_retriever(vector_store_kwargs={\"where\": lance_filter})\nresponse = retriever.retrieve(\"What did the author do growing up?\")\n"; -export const PyFrameworksPydanticBaseExample = "table = db.create_table(\"docs\", schema=LanceDocs, mode=\"overwrite\")\ntable.add(\n [\n {\"text\": \"hello world\", \"vector\": [1.0, 0.0]},\n {\"text\": \"goodbye world\", \"vector\": [0.0, 1.0]},\n ]\n)\nresults = table.search(\"hello world\").limit(1).to_pydantic(LanceDocs)\nprint(results[0].text)\n"; +export const PyFrameworksPydanticBaseExample = "table = db.create_table(\"docs\", schema=LanceDocs, mode=\"overwrite\")\ntable.add(\n [\n {\"text\": \"hello world\", \"vector\": [1.0, 0.0]},\n {\"text\": \"goodbye world\", \"vector\": [0.0, 1.0]},\n ]\n)\nresults = table.search([1.0, 0.0]).limit(1).to_pydantic(LanceDocs)\nprint(results[0].text)\n# hello world\n"; export const PyFrameworksPydanticBaseModel = "class LanceDocs(LanceModel):\n text: str\n vector: Vector(2)\n"; @@ -120,7 +120,7 @@ export const PyPlatformsDuckdbMeanPrice = "duckdb.query(\"SELECT mean(price) FRO export const PyPlatformsDuckdbQueryTable = "import duckdb\n\narrow_table = table.to_lance()\n\nduckdb.query(\"SELECT * FROM arrow_table\")\n"; -export const PyPlatformsPandasAsyncExample = "async def run_pandas_async_example() -> None:\n async_db = await lancedb.connect_async(\n str(Path(tempfile.mkdtemp()) / \"pandas-async\")\n )\n async_df = pd.DataFrame(\n [\n {\"id\": \"10\", \"text\": \"sage\", \"vector\": [0.6, 0.4, 0.8]},\n {\"id\": \"11\", \"text\": \"bard\", \"vector\": [0.2, 0.7, 0.3]},\n ]\n )\n async_table = await async_db.create_table(\n \"creatures_async\", data=async_df, mode=\"overwrite\"\n )\n async_results = await (\n async_table.search([0.6, 0.4, 0.8])\n .select([\"text\", \"_distance\"])\n .limit(1)\n .to_pandas()\n )\n print(async_results)\n\nasyncio.run(run_pandas_async_example())\n"; +export const PyPlatformsPandasAsyncExample = "async def run_pandas_async_example() -> pd.DataFrame:\n async_db = await lancedb.connect_async(\n str(Path(tempfile.mkdtemp()) / \"pandas-async\")\n )\n async_df = pd.DataFrame(\n [\n {\"id\": \"10\", \"text\": \"sage\", \"vector\": [0.6, 0.4, 0.8]},\n {\"id\": \"11\", \"text\": \"bard\", \"vector\": [0.2, 0.7, 0.3]},\n ]\n )\n async_table = await async_db.create_table(\n \"creatures_async\", data=async_df, mode=\"overwrite\"\n )\n # An async search returns its query builder from an await\n query = await async_table.search([0.6, 0.4, 0.8])\n async_results = await query.select([\"text\", \"_distance\"]).limit(1).to_pandas()\n print(async_results)\n return async_results\n\nasync_results = asyncio.run(run_pandas_async_example())\n"; export const PyPlatformsPandasCreateTable = "pandas_df = pd.DataFrame(\n [\n {\"id\": \"1\", \"text\": \"dragon\", \"vector\": [0.9, 0.1, 0.3]},\n {\"id\": \"2\", \"text\": \"griffin\", \"vector\": [0.4, 0.5, 0.2]},\n {\"id\": \"3\", \"text\": \"phoenix\", \"vector\": [0.7, 0.3, 0.6]},\n ]\n)\npandas_db = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"pandas-demo\"))\npandas_table = pandas_db.create_table(\"creatures\", data=pandas_df, mode=\"overwrite\")\n"; @@ -152,23 +152,23 @@ export const PyPlatformsVoxel51LoadDataset = "import fiftyone as fo\nimport fift export const PyPlatformsVoxel51SortBySimilarity = "# Step 4: Query your data\nquery = dataset.first().id # query by sample ID\nview = dataset.sort_by_similarity(\n query,\n brain_key=\"lancedb_index\",\n k=10, # limit to 10 most similar samples\n)\n"; -export const PyRerankingAnswerdotaiUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import AnswerdotaiRerankers\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = AnswerdotaiRerankers()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingAnswerdotaiUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import AnswerdotaiRerankers\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"answerdotai-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = AnswerdotaiRerankers()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; export const PyRerankingCohereUsage = "import os\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CohereReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = CohereReranker(api_key=os.environ[\"COHERE_API_KEY\"])\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingColbertUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import ColbertReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = ColbertReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingColbertUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import ColbertReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"colbert-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = ColbertReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingCrossEncoderUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CrossEncoderReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = CrossEncoderReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingCrossEncoderUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CrossEncoderReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"cross-encoder-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = CrossEncoderReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; export const PyRerankingJinaUsage = "import os\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import JinaReranker\n\nembedder = get_registry().get(\"jina\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = JinaReranker(api_key=os.environ[\"JINA_API_KEY\"])\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingLinearCombinationUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import LinearCombinationReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = LinearCombinationReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingLinearCombinationUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import LinearCombinationReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"linear-combination-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = LinearCombinationReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingMrrUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import MRRReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = MRRReranker(weight_vector=0.7, weight_fts=0.3)\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n\n# Run multivector search across multiple vector columns\nrs1 = tbl.search(\"hello\").limit(10).with_row_id(True).to_arrow()\nrs2 = tbl.search(\"greeting\").limit(10).with_row_id(True).to_arrow()\ncombined = MRRReranker().rerank_multivector([rs1, rs2])\n"; +export const PyRerankingMrrUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import MRRReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"mrr-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = MRRReranker(weight_vector=0.7, weight_fts=0.3)\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n\n# Run multivector search across multiple vector columns\nrs1 = tbl.search(\"hello\").limit(10).with_row_id(True).to_arrow()\nrs2 = tbl.search(\"greeting\").limit(10).with_row_id(True).to_arrow()\ncombined = MRRReranker().rerank_multivector([rs1, rs2])\n"; export const PyRerankingOpenaiUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import OpenaiReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = OpenaiReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingRrfUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import RRFReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = RRFReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingRrfUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import RRFReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"rrf-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = RRFReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; export const PyRerankingVoyageaiUsage = "import os\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import VoyageAIReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = VoyageAIReranker(model_name=\"rerank-2\")\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; diff --git a/docs/web/snippets/provider_examples.mdx b/docs/web/snippets/provider_examples.mdx new file mode 100644 index 000000000..13768e265 --- /dev/null +++ b/docs/web/snippets/provider_examples.mdx @@ -0,0 +1,18 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PyEmbeddingIbmUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import EmbeddingFunctionRegistry\nfrom lancedb.pydantic import LanceModel, Vector\n\nregistry = EmbeddingFunctionRegistry.get_instance()\nregistry.set_var(\"watsonx-api-key\", \"your-api-key\")\nwatsonx_embed = registry.get(\"watsonx\").create(\n name=\"ibm/slate-125m-english-rtrvr\",\n api_key=\"$var:watsonx-api-key\",\n project_id=\"your-project-id\",\n)\n\nclass TextModel(LanceModel):\n text: str = watsonx_embed.SourceField()\n vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"watsonx-demo\"))\ntbl = db.create_table(\"watsonx_test\", schema=TextModel, mode=\"overwrite\")\ntbl.add(data)\n\nrs = tbl.search(\"hello\").limit(1).to_pandas()\nprint(rs.head())\n"; + +export const PyEmbeddingJinaMultimodal = "import os\nimport tempfile\nfrom pathlib import Path\n\nimport lancedb\nimport pandas as pd\nimport requests\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\nos.environ[\"JINA_API_KEY\"] = \"your-api-key\"\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"jina-images\"))\nfunc = get_registry().get(\"jina\").create()\n\nclass Images(LanceModel):\n label: str\n image_uri: str = func.SourceField()\n image_bytes: bytes = func.SourceField()\n vector: Vector(func.ndims()) = func.VectorField()\n vec_from_bytes: Vector(func.ndims()) = func.VectorField()\n\ntable = db.create_table(\"images\", schema=Images)\nlabels = [\"cat\", \"cat\", \"dog\", \"dog\", \"horse\", \"horse\"]\nuris = [\n \"http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg\",\n \"http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg\",\n \"http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg\",\n \"http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg\",\n \"http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg\",\n \"http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg\",\n]\nimage_bytes = [requests.get(uri).content for uri in uris]\ntable.add(\n pd.DataFrame({\"label\": labels, \"image_uri\": uris, \"image_bytes\": image_bytes})\n)\n"; + +export const PyEmbeddingJinaText = "import os\nimport tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import EmbeddingFunctionRegistry\nfrom lancedb.pydantic import LanceModel, Vector\n\nos.environ[\"JINA_API_KEY\"] = \"your-api-key\"\n\njina_embed = (\n EmbeddingFunctionRegistry.get_instance()\n .get(\"jina\")\n .create(name=\"jina-embeddings-v2-base-en\")\n)\n\nclass TextModel(LanceModel):\n text: str = jina_embed.SourceField()\n vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()\n\ndata = [{\"text\": \"hello world\"}, {\"text\": \"goodbye world\"}]\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"jina-text\"))\ntbl = db.create_table(\"test\", schema=TextModel, mode=\"overwrite\")\n\ntbl.add(data)\n"; + +export const PyRerankingCohereUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CohereReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"cohere-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = CohereReranker(api_key=\"your-api-key\")\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + +export const PyRerankingJinaUsage = "import os\nimport tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import JinaReranker\n\nos.environ[\"JINA_API_KEY\"] = \"your-api-key\"\nembedder = get_registry().get(\"jina\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"jina-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = JinaReranker(api_key=\"your-api-key\")\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + +export const PyRerankingOpenaiUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import OpenaiReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"openai-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = OpenaiReranker(api_key=\"your-api-key\")\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + +export const PyRerankingVoyageaiUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import VoyageAIReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"voyageai-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = VoyageAIReranker(api_key=\"your-api-key\", model_name=\"rerank-2\")\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + +export const PyRerankingWatsonxUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import WatsonxReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"watsonx-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\n\n# Supply your API key and project ID (or use space_id instead of project_id).\nreranker = WatsonxReranker(\n api_key=\"your-api-key\",\n project_id=\"your-project-id\",\n)\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + diff --git a/docs/web/snippets/search_examples.mdx b/docs/web/snippets/search_examples.mdx new file mode 100644 index 000000000..a9aeb5e7f --- /dev/null +++ b/docs/web/snippets/search_examples.mdx @@ -0,0 +1,32 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PyFilterSearch = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n \"vector\": [1.0, 0.0],\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n \"vector\": [0.9, 0.1],\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n \"vector\": [0.0, 1.0],\n },\n ],\n )\n rows = (\n table.search([0.0, 1.0])\n .where(\"category = 'animals'\", prefilter=True)\n .select([\"id\", \"text\", \"category\", \"_distance\"])\n .limit(2)\n .to_list()\n )\n print(rows)\n"; + +export const PyHybridSearch = "from tempfile import TemporaryDirectory\n\nimport lancedb\nfrom lancedb.index import FTS\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n \"vector\": [1.0, 0.0],\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n \"vector\": [0.9, 0.1],\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n \"vector\": [0.0, 1.0],\n },\n ],\n )\n from lancedb.rerankers import RRFReranker\n\n table.create_index(\"text\", config=FTS())\n rows = (\n table.search(\n query_type=\"hybrid\", vector_column_name=\"vector\", fts_columns=\"text\"\n )\n .vector([1.0, 0.0])\n .text(\"puppy\")\n .rerank(RRFReranker())\n .limit(2)\n .to_list()\n )\n print(rows)\n"; + +export const PyQueryPlan = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n \"vector\": [1.0, 0.0],\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n \"vector\": [0.9, 0.1],\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n \"vector\": [0.0, 1.0],\n },\n ],\n )\n query = table.search([1.0, 0.0]).select([\"id\", \"text\", \"_distance\"]).limit(2)\n print(\"Plan:\", query.explain_plan())\n print(\"Measured execution:\", query.analyze_plan())\n rows = query.to_list()\n print(rows)\n"; + +export const PyTextSearch = "from tempfile import TemporaryDirectory\n\nimport lancedb\nfrom lancedb.index import FTS\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n },\n ],\n )\n table.create_index(\"text\", config=FTS())\n rows = (\n table.search(\"puppy\", query_type=\"fts\", fts_columns=\"text\")\n .select([\"id\", \"text\", \"_score\"])\n .limit(2)\n .to_list()\n )\n print(rows)\n"; + +export const PyVectorSearch = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n \"vector\": [1.0, 0.0],\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n \"vector\": [0.9, 0.1],\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n \"vector\": [0.0, 1.0],\n },\n ],\n )\n rows = (\n table.search([1.0, 0.0])\n .distance_type(\"l2\")\n .select([\"id\", \"text\", \"_distance\"])\n .limit(2)\n .to_list()\n )\n print(rows)\n"; + +export const TsFilterSearchBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n {\n id: 1,\n text: \"A puppy playing in the park\",\n category: \"animals\",\n vector: [1.0, 0.0],\n },\n {\n id: 2,\n text: \"A dog resting at home\",\n category: \"animals\",\n vector: [0.9, 0.1],\n },\n {\n id: 3,\n text: \"A guide to growing tomatoes\",\n category: \"gardening\",\n vector: [0.0, 1.0],\n },\n ]);\n const rows = await table\n .vectorSearch([0.0, 1.0])\n .where(\"category = 'animals'\")\n .select([\"id\", \"text\", \"category\", \"_distance\"])\n .limit(2)\n .toArray();\n console.log(rows);\n"; + +export const TsFilterSearchTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsHybridSearchBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n {\n id: 1,\n text: \"A puppy playing in the park\",\n category: \"animals\",\n vector: [1.0, 0.0],\n },\n {\n id: 2,\n text: \"A dog resting at home\",\n category: \"animals\",\n vector: [0.9, 0.1],\n },\n {\n id: 3,\n text: \"A guide to growing tomatoes\",\n category: \"gardening\",\n vector: [0.0, 1.0],\n },\n ]);\n await table.createIndex(\"text\", { config: lancedb.Index.fts() });\n const reranker = await lancedb.rerankers.RRFReranker.create();\n const rows = await table\n .vectorSearch([1.0, 0.0])\n .fullTextSearch(\"puppy\", { columns: [\"text\"] })\n .rerank(reranker)\n .limit(2)\n .toArray();\n console.log(rows);\n"; + +export const TsHybridSearchTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsQueryPlanBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n {\n id: 1,\n text: \"A puppy playing in the park\",\n category: \"animals\",\n vector: [1.0, 0.0],\n },\n {\n id: 2,\n text: \"A dog resting at home\",\n category: \"animals\",\n vector: [0.9, 0.1],\n },\n {\n id: 3,\n text: \"A guide to growing tomatoes\",\n category: \"gardening\",\n vector: [0.0, 1.0],\n },\n ]);\n const query = table\n .vectorSearch([1.0, 0.0])\n .select([\"id\", \"text\", \"_distance\"])\n .limit(2);\n console.log(\"Plan:\", await query.explainPlan());\n console.log(\"Measured execution:\", await query.analyzePlan());\n const rows = await query.toArray();\n console.log(rows);\n"; + +export const TsQueryPlanTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsTextSearchBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n { id: 1, text: \"A puppy playing in the park\", category: \"animals\" },\n { id: 2, text: \"A dog resting at home\", category: \"animals\" },\n { id: 3, text: \"A guide to growing tomatoes\", category: \"gardening\" },\n ]);\n await table.createIndex(\"text\", { config: lancedb.Index.fts() });\n const rows = await table\n .query()\n .fullTextSearch(\"puppy\", { columns: [\"text\"] })\n .select([\"id\", \"text\", \"_score\"])\n .limit(2)\n .toArray();\n console.log(rows);\n"; + +export const TsTextSearchTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsVectorSearchBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n {\n id: 1,\n text: \"A puppy playing in the park\",\n category: \"animals\",\n vector: [1.0, 0.0],\n },\n {\n id: 2,\n text: \"A dog resting at home\",\n category: \"animals\",\n vector: [0.9, 0.1],\n },\n {\n id: 3,\n text: \"A guide to growing tomatoes\",\n category: \"gardening\",\n vector: [0.0, 1.0],\n },\n ]);\n const rows = await table\n .vectorSearch([1.0, 0.0])\n .distanceType(\"l2\")\n .select([\"id\", \"text\", \"_distance\"])\n .limit(2)\n .toArray();\n console.log(rows);\n"; + +export const TsVectorSearchTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + diff --git a/docs/web/snippets/search_filter_search.mdx b/docs/web/snippets/search_filter_search.mdx new file mode 100644 index 000000000..efe325aae --- /dev/null +++ b/docs/web/snippets/search_filter_search.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::types::Float32Type;\nuse arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n true,\n ),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]\n .into_iter()\n .map(|v| Some(v.into_iter().map(Some).collect::>())),\n 2,\n ),\n ),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n let query = table\n .vector_search(&[0.0_f32, 1.0])?\n .only_if(\"category = 'animals'\")\n .select(Select::columns(&[\"id\", \"text\", \"category\", \"_distance\"]))\n .limit(2);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/snippets/search_hybrid_search.mdx b/docs/web/snippets/search_hybrid_search.mdx new file mode 100644 index 000000000..492e14f54 --- /dev/null +++ b/docs/web/snippets/search_hybrid_search.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::types::Float32Type;\nuse arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lance_index::scalar::FullTextSearchQuery;\nuse lancedb::index::{Index, scalar::FtsIndexBuilder};\nuse lancedb::query::{ExecutableQuery, QueryBase};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n true,\n ),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]\n .into_iter()\n .map(|v| Some(v.into_iter().map(Some).collect::>())),\n 2,\n ),\n ),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n table\n .create_index(&[\"text\"], Index::FTS(FtsIndexBuilder::default()))\n .execute()\n .await?;\n let query = table\n .vector_search(&[1.0_f32, 0.0])?\n .full_text_search(FullTextSearchQuery::new(\"puppy\".into()).with_column(\"text\".into())?)\n .rerank(Arc::new(lancedb::rerankers::rrf::RRFReranker::default()))\n .limit(2);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/snippets/search_query_plan.mdx b/docs/web/snippets/search_query_plan.mdx new file mode 100644 index 000000000..fbfe156c9 --- /dev/null +++ b/docs/web/snippets/search_query_plan.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::types::Float32Type;\nuse arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n true,\n ),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]\n .into_iter()\n .map(|v| Some(v.into_iter().map(Some).collect::>())),\n 2,\n ),\n ),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n let query = table\n .vector_search(&[1.0_f32, 0.0])?\n .select(Select::columns(&[\"id\", \"text\", \"_distance\"]))\n .limit(2);\n println!(\"Plan: {}\", query.explain_plan(false).await?);\n println!(\"Measured execution: {}\", query.analyze_plan().await?);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/snippets/search_text_search.mdx b/docs/web/snippets/search_text_search.mdx new file mode 100644 index 000000000..86745b652 --- /dev/null +++ b/docs/web/snippets/search_text_search.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lance_index::scalar::FullTextSearchQuery;\nuse lancedb::index::{Index, scalar::FtsIndexBuilder};\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n table\n .create_index(&[\"text\"], Index::FTS(FtsIndexBuilder::default()))\n .execute()\n .await?;\n let query = table\n .query()\n .full_text_search(FullTextSearchQuery::new(\"puppy\".into()).with_column(\"text\".into())?)\n .select(Select::columns(&[\"id\", \"text\", \"_score\"]))\n .limit(2);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/snippets/search_vector_search.mdx b/docs/web/snippets/search_vector_search.mdx new file mode 100644 index 000000000..47966f5d4 --- /dev/null +++ b/docs/web/snippets/search_vector_search.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::types::Float32Type;\nuse arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n true,\n ),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]\n .into_iter()\n .map(|v| Some(v.into_iter().map(Some).collect::>())),\n 2,\n ),\n ),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n let query = table\n .vector_search(&[1.0_f32, 0.0])?\n .distance_type(lancedb::DistanceType::L2)\n .select(Select::columns(&[\"id\", \"text\", \"_distance\"]))\n .limit(2);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/static/code/ingestion-feedback-rust.zip b/docs/web/static/code/ingestion-feedback-rust.zip new file mode 100644 index 0000000000000000000000000000000000000000..a9643851ce999d6fb760335c6d92ee8acc607f43 GIT binary patch literal 38831 zcmV(zK<2+tO9KQH000080000?U8eYY#yyV!0Q_45015yA07GGNXKyZSZ)0ohy;*M~ zNzyI&UcVwy_wly!^L-g$fC1)d-skP&LU4C8&t#V*s!6J3z8BUjG&~fV9kQ* zulq}JNoBaZE$5iq^54GvpWj#8FMajEU+`yn*{vVTZq>@e!;dfj+c$pVP1!MhsXxB_ z?`89C{q>js_T~TB$ycA&_~U8EPlS*Adh_M6JiU~MFHEc5>gn6pUw-G!cC~)`^1ps@ zzx?t){>O7^zm;$NKmX@1Pvw#E?yrStV;)vd{Oiege^p!Ssy8XIZ->vXSfByPQ`_ApR z?aQP53SB)}6|GU#>ZB=9%-(Sbwo+^{x{_?F$r#mH`D|MXlzUaVyPT@4zy3n}EK7ZA zE8gNa{*Pb2{Q7P6_^|r*U%&j%x#d0Z#+PjR~yHAE0dIMITrFJcYJC}Eje~9(Ns&dC?Xwo40&2x`9CjZ zx4yc);`o?IlNK^e{6UBc>lavKl5gXEf}pxfAtG5DCx|2zD$*HAkZ=>U9K7? zRCU80!WwaSF3a&y;F5KSVw59`gVEFa_OPng&g^MoU18yMx#~iz)dpx%iqI;fppXHWS&wn-|qeP?j_oVy}uz zE)-LG@8r4rSaN3^SW|4c8P=|6d@}~6)a>*B_`d$*^6}Nuz7K9q8R{vx!PF(#Fp6A? z_mO){o_q^90S_~PlTxA_SVK|K*b4L4=AM7k?zVrF=g|t{;>aW99(!swt5Acf(Yr?8 zngHESa7G%&TT|;WcsgT&T)e47*=6gBbw18M|EB!da39IP{hF8Fe+UXEZNyAZl?gj{Q@z)<)D@^5^>z8AO{0F-K|rp~Qid;Uu|85h^z*dRXISn3M4rZ?(%aGS9V z9j%96H(!Bkxmf_JmQ=}DS8GuxND_2q{`p7Z?-O=^X+LNo;a0Kt*x;C%if!6)gWntA}6owOYgind83u>?)2sM z_mfvtcID*d>bae~`uBGIbn#Veew?&jwM zd9OwrG7XF(Z~_wZx_Nmkra?Wbf-lc-2EAtjR@Yn;Ih?1e(Wa~C^UisUw)|x0fKk^^ z>*2?zm&ZF``PFXw{*KANck>-Htoe!TWXmcX&Z@f25?KJaUgJdT}bN!@ZbPH)#S$4 zANU8)w@X%DSKqK1pWKMSzTA8pT=Ru{^>AzO3}6;UgbWNCsMV*Md>7i^6+m1!;7bj* zH7gZZH?XVN7u48uJ9yRd?bfT|hg!CvLKlL&e(XQ)cBKMPo}C4xQ=(MDy7}@nu%AC( z-aQqV`I{dKXUD1a$R5-ba1`2}fmO8$;1Qs)%>lof0@Ug~MdLiMw)LPF0OdK-j8T|8 zl#v&ku>=R-hCko-4(lA0{pxlNM|^u&&{8rIuCT>auwk8YC>?CXs$8rxsW+&21w+=p z7n2L1g2LA-&{<-h*5w(YJ=vi%9sfCNC&1w5+hZ)uTIwoU=aP*eJ}F^`s0P8=QQK6F zv5L4hs{y8(vcQ2LYk_SGZb==#rq%tS?Dl_uuZ<6Xx%+-}D1U8t^xkF;xa#)1seCI& zgWm@VF@tbvi*4$_r2(64B5(|WfPu7#4#yC|&q8i7fj-(W9ccPF%=vMPd7{j(Qah~e zu{?hAgdgHQoFSbp_Q= zos5s!q)|wYL3FZ%l~5{6zyPey)Y==nu5;IP&0s925m&eAAE8^dRF4ga zR)b3gWt&9MLpC8w1kS6T6r?&v*T8wf;6v9vS-CXO+5@xRW}6NIe06&cU%&Yd^#Jr< zJ9V33HFKc?(ULH13s8R@xJKP;jRf`YLE8+hDGr;laYZvtqsY|`w(#3l6jL1(^O|y9 z--*3A-Fzn!C|vN(I~`5IY9oM3H$6%CGqNJv2~eA<7KjPpba$<3P;;NMGx*Pkh|J%R znveVB1EapZBgsUu5g0|4tE=vSUf7k^zyc-MDEk9QS8JeTtRdFkI8T6;PqIEA`cL*blSEA|dZR@A@vf1#{?(RP?{IXX@{#{-_;DP1pTjU3i^=-fZ z{bl#E;q99reO^7?J#W_k#`fZ`?D*a70;|R^ke#}?X0l612Xo@)JFs6*z=9?;SpQZ* z3Y~5Z&XP#(&q^=Jd+eD7DXi5KrNF%u8)x?Hbp==raoZkQ+R598-TBCNeff2_{%y5> zZ{r@;qia=*9a}n_4LJg)Q%V}T@4CXGfz|5RGW0AL=qYLMb5Wcv z=xj@kfudcTxdg*f+udVno3$C1|DTujK`gMkFGIG3j_>WkV8t#j%hJ-f1zA~N$dJmR zvZRmr0<*&$SqPVpj=)XHHd|EJDYOcbkBw@PnLO2LuCT-g>@yd}*SdD;;F61_np9vU ztn(e}kl@f1ynbXdRI|mv*_%W?u}O(xg{Ch-*F#pu-0CDWG)|vc9QENT zbLN(6=VEd&LARny*U@z&H$NJvk4p~0!CZp%;b1FHT>%op9!!V1KLk*A>&I2Q`=k6e z3XzKTE`8Ogz48@eXi4jkqTy(wE`c0tAQG^kmV8txhL#A_S7k_XE)q|rR7#t@Dtg(i zTmLqC{N??BQz@t*ca5zAY%(NF1glr2w3BFO3V8a?lG9a6O5P?{z-w@LSSqKP#|KUP z4ad~R3263fT-qDMwOWnmidh$<#WV&}tfJBt5hv?mQZ*i$w1n&sT8IG z;%frk;Dq0@o({&bt2lhy~@r+Z_QbK7Bg`SO36+um9M>^F00b zMU04^7vDp;c4T(U;(jNZYK;e37bYrMdb$)1Y^~TfnB1;(&Jc{?asHJ7a475U0TC4} z72cma`qej}m(^n#TTZw5HM0*2NK{S}U;;oZ)Mz1?R#>Bp+>Gu5&SLQ{c<5qx)m5uQ zjLD>IrxSjTns+$u$GQ87&xSIOe08MKc5#bT84^#<>tF#ZvZoy zlB%z^gF{-Ybv6+k-L&FDkKmV+i|a2>yFF0f6k5IPwQg6Q_tx{rUsqQB06#xYzJpXQ zp3?cmr*oLqu0P!LPl&H_lDBM?lioIDVM%1sC^d}wiHHqyJ# zHu~4`g-GY%Putk74#~vjX?MR{J@P&O=h}{du}VBAit*0dgB~RztQcXEPj3 z$7Z;kLumyDs^P#j%#l$f-ZF6no+qztnMtJ?a{mEPkMXCum2Q0+t#JAD{n{b9MT-&x zL^Yc;AUG^$hl@QU8Z0Q6to+>BgHzCGrL-1TfZ}>ok*#vw zdC;9C_juSCWrOD{GX3~+QDkA3&(qhvQuj$oY}uw7gmfhDjV?p3D4Rf9wU$$I5-=?h zmdGZ{+Jc#Qu{pB~Sa0Zp^Dj@Dbtl!0Pfi2U1j$r)Nx@!%DWxE}Cmu4_d$h`PC0!UM zR4E1y3b6+_m)^S1iy8$imX9}TjxI%OYlP>DIk9%uL5!L=uEWsxZc249EAaB#D-AN5 zW5Z%&ac}{1Ht`}8OWh>375F584S>FDljSN%H?8oqaguMJdhRwBQ*{uz@;OP~26u|p zMy`JD*~h5ae09r5IeoH9Ww0y}b*-Ve-8q*bwzbrTV=zt99SHch8whCCKEq;=7U&D0W#;U2`(E&0qb^bV z()b_%$O86(qU9D@dw5BX%oYFu78yoN6$3J3jIpVhqe(79> zJGi0NQ6DiA4Y>zjhDBY8@E!7u#^Euyv@F8ifWUy6uwwYlBC#MiA-Q~G;Cbi4Wh01` z00bX{lidK|1fY`p^*T-KR^_SO2~E5EYki21{(F13{jr_8{2nqe_8h)!C4}}wU_YM@ ziM}OS-<<=s_TseK`tr^m!?Ms(4=zo&NGo$Kxxxv-fVW;Gc+Wa# zPOyfsaWLKt7v)r+;X>mhyq-+f9!4c`4y9ogAhdnBN2RltU&vegm%Y|XV$ z0GN>5od))v8A2P$Bxe%)=}3$``F(v^JrG|hUq1WRc%4h5qo|$Knk!v<2o4|-TE}1* zApLMuT11fKCcMFBS0g|}s~Aj&z4i*qJROQV6psGuxaR9n@c51K-Ra^;Fp-m@Kwt<4 z@c@5vIm6OukL3cw6Zk7geXa(eD)}yVH{SNzTc!-CKc{{tl?|s$h&Q7ro5SGzZv^2YrToY?N) z-G=rm~?P@EGPi3-3c4u^C6Iunmh}vVwww;MD++p-fTa44`x_ zc&BNsx+|L$Y0ALi{^a5(p3_nicU;hTX061YIji=UDd9nZh?VSxdy&f9JnHE>uhNDja5o8USELnIKZ?u#oc>w&_Kqh+XF=i^oWVj@g zn@+Z~rB>#0BgE$1_4UgWmEC2ks$TxCR-$E%G7PS@9D6XeI7uM1jFXkn3rKz{Qq3U+ z#z0YkhN`8QD`_A43==+I#Vhx_-gsl`Z;dYs;?l<$Ex2vX(y7T)FscXFjTB>u`4T)I zG_+j7G<$Fw$yM`FLgo?nIHkEjZF_#ce@*WhyO5vAX{e9^0*c9Jg|FfLX&roLd7B2uJPyd;;wnVycbaZ68o+F&Wx#V42VCz{ zk!y0_obffv<#N{EVW}l}$fsoJ`I+S9ccpW)+VSSnfG&U2DEBrlz={IfXVWvNS?Iny z{A48=LX!St;2|r{1I;#z|3qDgoG_GW|8QLK-SfkG z|3!IY2>MGn$sP)zM&Y;h`lLVGGdA&lyZY;J-HYF|I~I{{%8x?<;#;-KTj*|7qbMGG z$B)kfyUz*@B*^#h8?CMZ@bh)t`uIpxtQedZOo^A|4R*15ARU}mvt-1V*8<}TS8o~S zMQKjb8U-FRbD5UT<55kV!0fV*KzFLHXUOYVytS68lHZp?sy5^hf!-8wIk<+T+eby5 z8<(DR}k_R&I5?q;!4!tgdpV#gCK@mT_1N*-`PB~nhwZOH&0K0<4 zwUV24L1myFWfW_t3=_dhS-2enDvDy8!NIjFaH8IsxmfP!mwU-a2dBDt=(gkdBonb` z4J59!fFZdTR;oe9qlaD24Nee%)&tK<;JU(5Npnka#7ixA``hYSDvmA~+R|kr)J-OK zPJwte$#Q|)5y%QFW~__egO*9dtzMzpWNMc}$e2o&R8y~~Gl3Qo@%B$Q4WHZ=~vc~!VX^`mLT(QD2v33k}m%dtpM)u0E zyQU4jH?5^;@StL;6ugj?&a28>0_=6JOTbs+3@Lj9ou%32h0TxWois^(e7vNT%Tp?l zV3UpVo&fx)tJYZ3uoTuqE#Z|kzlHUU67QlKTCAF3ZU^V0rd{edBNCY62fp-jrG#nR zVn*Mfp{O_D&J>0wGra|(4&Ei)DoWadEg2tbOA>hpW@wy~iG>w3Q<@>pH0nBQ5A)hd zVzNEN{L-hQml_~s>BCloKM9iSnv{#+EZImYg`+A`>?OQebEv6A@F_OZKDTiIWUe`; zX%O*Vkd{hm-`AV%IHN$vKZpJ+L&acJ zSprtAI-bc%-#`8_m0|zf|2p|#h)AA%GJB4b+i-&x0lqNQu$ZCiCr*@rb~U84*P>MB z7U*g3%A3i*(>aw=fE zWYhvhAAu!Pk*jbOoX*fE8?BLxr3^@sq5@6^e4OEH`wqg>Eon|#HvH?C(=MAQ*=KJX zaM$qT!}{CI+d~(PeaqbQkM}HN^4&Y|D7l~;e=0xjw|4%MySJ51e}3$)IQ{8saPs71 z{*&9waLsn{!SiP7y_v;zutWJZ+u0B9%C~Qjm}5M2@+(ig-7(F7{?>?d^5cEB%yHhf3v-Iw0qZ+EZlaNZ2UugA>XFA{X~;~4jU9`N1m;khNKE#rDdLG72#{&`+r zR#%!UPH5K6O^l6=J8@8QZ3ZULd(aa5=@K8Y)|eqSW-7=4MagyU9N^5@SL(ga+$^1# zn2|*}`kb4ZgimSh$He?S*;`J~-f6jX#mwGZR&)fGHCBNPc?uE)wHp5^u1RuT?_dJ> zDe5Fu3eD=wHY!Q;=-q)9zw3@JWaV8)GbuLzs9-E+|6V2RXGQ4)0&^MOyNR5@P!tG2 zajs;~S|trkLblYJ6u3ZiIRL-J(y9w!)zXyRO7}eg!0y{jH=^{di4wk zhC`LbbU)Uqojw*mUD4j{5ptjVfw%iiu6GCX_0bc0wRkr_sgxx-y)c;^qT=Q{1?6pU zwt%&{+`&eiG$YWBbxqBOn5!`@MAC+M8k(O9@NwflZq-h-_vXvz+`&#&s(Xw+xzO$8H$-Sxk zuF##&L75f7{|BP4CFN;Sj4s6IXUx*&WP)2riC4Cp%*rRzbZX(hpE_ zIV0Fva>+KpDmZW0j23!O@Mq$yeF=_O8a&vU7Q6qyG+aupA0`qJ6JAPZj?`# zzHa{jegktaO;Oh&$mkN63fY#EbaSaz`V&e)K{OqhrfLGhksi6d#NbMbJlDr#@|zy|XjE!0#DrdZDs+mho2I2~rKx5xL5h4R(e+5>fltNER`Wq zb?r^EfwHTG6iwfE%p0oD;Am0EiIaOTV*3`DefOLD0PB19Y5b3Vn2nxr&C$e<6YBXf zLHNf)0RScqT(VoL0%$d7!&Nf9gA6_7Bw4dQ+md?3OD&_8U<8ON2v>!LdPg3n5B<|S z`u?W91@Ldr4@@ese$v`elJ9`-oYXAHum`ESw%lX2x|HCp${va&X0WP)EpA@FPfEvT z@MN^tp%3Egk9MqN=acGwPUV)lQP~%=?qfU${C1T&KGNOZcwU*ZFdO3kUiO=Etd-sJ zw?}^3mDhEAIIcf+-*a<`QVP;ybYve?P8B-FO3M6ol(|%a$F1^-eEeJhU!A;;x=%}NWK)bu~sOc@;$d!b>r;7=d-lkj#1FHPbD_vNV2^% zTkCrFn%{*>OWm-2?5cFkVt!`d4>=3Bzp`?i61Pk~Ht?6^NhT`G*a>M@91KMhwDhed zY9Kbr$UIAcuWUSll)eYLewLj4B-dEd`TZTOIe*C8M?++4bIsQ4Ty@DhYa5L%T3gk6 z!BMj9(HNyj?jv#DS(nh8cTq8cpUx!Py>oOw?XnK1bn|U7Y(wNA)f8Tn;wo;anhqh? zM)_QAs+x3{2J0a!nIC$<=*zj=Gl!hR+o@kuv)slFe-$3E!rXXnVx{_8hC zCQx1usYs=rtJ1PS@?)tR51Jdgl=D-PJbb4ist&phwCarlfpMq#@F;l3)5i^{%Gi;LYfhk*Qrz5or zL3(<3ZlPr6+%wgg2`zt50A?Za&-vJi^ZV>^{9`Oqx#~O@xo9X2xG3RJiYZa@Mpy-( zb<4rz7E4f8nxO`qS8$nIcQ$-Hx978#^nG&hbBgncgZKyCSOhJwGEPI~;tV!D$;}^i zPSV}ZT4}poESP~($`ufMU7b@-C&}TgdAfA@Q%bQ6xBNMM`RKLZ^XBF0n1KHGbZq$T zx4%7MYRUkud6`58>*afOtvOJ=QoHZ5g7J}~*0UtwW+jbBdj}y5)LQ(w*L!cO`4ygc z>l~%eL>m^ok@>LfonGKAm1Z~>RvbC-pP0}(p$s>d;?iW^O$v&w_cG_tdzEyP5$7Q_ zmbR6#NXmjt;EX`xGC#03nE(Z*;@=C5z0akuBpv*dhUm&!zkgcWiAlt_u{ll~N#ZUp zv2jM`Xqup9d|Ea*r%)rq1i|Mk?K~OZ7*-0=>ls4ySp)KO=^!(qrVs4ye;)pU zi@c%;PMWC0<5pYKaD~@0*tL{!CCd@-WpEela%A}g6#3h*L zWSP;F3A-edAGv?#`tofx9t@F|ho{`VHypZ8iJkPX0WhlI^`MKTX(u;Zv4Sxcnd{k< z*UU=EXl+wvh~ea(zjM3C8$1?Y4!u~q9TUTL4j1Q;8@YG@Ht2FooSN~fM1yr#nMz=z z?HaEkTrL&4bbxC6G<|{dy#9kX{c*3RAL$&ipU*Ve%X%owUA#HDiqxyhpd2;!5`c7D zA0`-E>B%K#%Db4ImoEF>N=w7u+2$U4c6sL7>Uh5@^cM0xf+qrrN%N-!g{B_7G|!Bx+NkH|GvF}S zB#3`=*)wkOIRw!wtKc+b5+VUmMV|sukl-8WDi+8x?;bveTTNJ$WoT;5%~umcnCam9 zUW3|`72aQrv01(+n;zVx7!(-MQl;^(s8mie(#iv85GkzDIqC5U*bGpOZzQJxK21r} zYdk%r`Gx-HrR>&YBTG_fjxXj7cbZUaE}Cd+)Pt^tw(mXkEO z(hfMWqxIEG!75cM z0mlrUBU5e!9xx!#bqMeoAJ4n*Pon{1(*Aqt27ylD|FHQUE7jZ_RwdUM(kfAM7Gkps zk}CbRIxs!-6Q-C6CaMeI_p`_KyG%Bb-U!kd@;urDom;x;6+_klzFX_wSgpHNMLM(_ z?V?!Fl9Puefg2iPvP`NbqFAhpJ%G$O?O~c~p7AT^XZyLGke>E; z{b5~p<^sDeE!$njSq9fU@T##AJ%!EbcuSK89g)d-(O4<|g+hbG7HgW|J>5fvS)K!L zvINOiA9c69#jM<{vdmPF6yRujRz5cfIweUBhH)~zt6*B@?348A2SaHvvDSOAt~ys= zHXb=U1~*FsIACEFgVPFII>cXfGdWe|ND{*$pr%s2D90d*El08)s~`rGC{l$nH;q)5 zvHcF)da;7#of@cYW^6vG9$6(7B%!$CryP<6?hMX44$3G&H%503wTA)mbR`3$r|rw3 zyG8ly$Ng5y84j;cyv4p<+MFt_{$#4F-Im9J55GJ>1>knTCvbt#9bIZjx|l>c6IPgI zFgCeTd{qWePVx>aw23qsI2C+r?hGHwZXfqJKSbld=C5XHd!VumXU5}Zj@C&kUdH)> z3c)GBR6wh`&QWv5<0m}0QCUOV0=wv$Y5Lk82Y^+4+e(mL9fI`oJnjClo+}5qpL;eQ z(zo~^w+=|nM2W$Yv`pW2-m#5c$RTz@C94QxY^4haoMGnLVRmA#T&3o7rtkQ1K#45) zarLzN>wf*`>e^gTX+t_jpG(WBl_zho=CC5hsQ^3DYAzjA*EPTwcmO4Xf{iP9mn5mM zC@?DE0XhG<;mv*hK*L4;t!*V`@LbSb@OAB*aa?oG%>*rTykK)c%jL?-1$b?T_ic-i zQWs)v$rc&9_nGH6%V);K#zt}glQ9X15lmJ&Edyk#OovP|eN6*K%MCS`z|yL@M0_Q5 zAAmDwpMUds|FU@)DDCL7N8ey3E3jg##(CItFOSxfH!-H@8U1TMAf@ zBFFT0-ZhU>>G0ha#qwEHoFipw7+2b`VcOZ&o- zfo7#awC@BdZmeSp(hfa%8!WX>CflG~QnkpuoDR%5r*|*Y6SfwgvKf4BDpp}64My5G z>CdjUYA%KrZOYZbg4YsFkq#_LW>1F_zm zy?JOqaeTR5J^gm}#;$D7-;f@-*JR=Rfo{I7hB{nd?J?o>qlW1PPL&%D71^pL;g^ya z)L+COWXKhOSWd;j_iI?}>)Sfs!o&8wxA@Bo+#9T1L4XFnq#7Kg^zY?Fpn=c@7=fh7 zP|HriYev|$BHcAr4n2ug48hKFr?Ir$d?A3LNyf|O*uAubcZF*M%mGgVj5QXzp^Y|j zIR&kkbx61dw3h-Lg0DV0WiGt&voAq#9!c=hh7RMP3GnBBl3eE)Wlcb|2f(k6e3?gMr5WGjR z2E~-R=w+93*9Omgll!zY{k{1P$gI6yY6Oo|@K4fE3Fx(Fm~*UO+oZukJn)o(Q!_&s z%cQg9Zgow-UhkaaAi+0Zg!EF0B$8zeXRnp(t*Y#yq8VOG1N;JTlu^*Z zG@T;%DW^N~ug80wk9gdZP7;%;x$@>)NBZqUsq+djOfQu&cTLaqziEs?_kUA;SHH2Y+affrFT4YL7nHlPL=)n5cl>ef2syyCXJmU2x;+-vd z9q2Toz02eC1cxO>WV*wHnIT)qz$=5%MpsB?!K9(np0*@Yn!#3p0dod=?;&%Z#^-M% zsJ{GqSit@8k6v5$ZgNPuS$Y~hWNbvMc zNzfjK1y*K86y9Xu-`nwAXBqFgT^)aTysLOBob%^*!!CbOkuw2@HXEf2uB1@_y|{#9 zhh>t~OZWwLfIJ>VK#)MOPRjGy5Mh_5Ge@TsVspoR^~HZZPVJB#dA@QDH@5}L-y17p zFJ+g}G0Z@$GQ>0*sgNcaaSDYXZY4^$fuIOR8z9{TC4FAw|O*yoBJ>H2Z7 z$uP|PY>N&*+RsSP>hWPUmY(HrdN`O2cAvpzEYZ*=zZFwS^^`4eBIGAPqbMsdp`0Dq zrl)|t?nZ#s9HtoB;$FDnrO&-df4!)BV4W2Ff-2dhN-a6!qOdF{!&yL#lc0ScCBLAS z?qaPp49}i?_V45SHMjf-&u*KlA-68G$aIA2hJpZ;P{iuLo4Yddf#x*CIn<+VFWIdN2SzX`F|5rEX$3=iQY2AjYDjY~>b376Mc zIHt}(3xEuu4QUo^&#u5NUp(M}W1O+9BWPdl%|LHBGEld-rspcT#dI zP_l#t_ql6{*%d5b{*aAvi_hs2&Y^4`ci=NNppFb6i3 z8Ln`+miV&2O8LC3g~fF)IZZNUu(He`G$26MXzJi|No*6>EnUHi>rU-=mN~Jco>i}kc|Yw6>t$T1j1OF5L_rb7!Sm~@OqU!mvQaegjEPX1mQPfyl+KDIExga@<4 z5rkUf2M?8$i|v9W69ahK{fc}=|Xw&zX$8tGB=N1`q(7~!< zHLw7-jg^lj!&ieRWiH;-mOUJj<}8(o-9U3a+*#D2g8(`+msrXE*^rEIY2?a9&yz3<=Lc?%m zz;Xkl0IR6rg{4bj>_86W;q72orb|BG_HhuY*Pq^~=3#knKi)l+-RgI~YrNT=z5o2% z>P|wM+ez+v@_TktReZPQ&F`yr&Ca|%$i*D`ym)KPn3SiWU~B-!VyH>(?Vyt?vv8dv zHC85O*E95z+#vlD*%D{$r*AP&duM-6%15`G^YZ?BYOCib-kk@}Ch~b^bYZ)0zwypo zoZI;^eI)E4&-1YUhW|c4yo{A%c{mi{I=RCAJ8Gp_%fO%!fQ@1x>j+3$(v5?ZAUU{= zs#3Gc04jwBY{Ap{<3pYM{2Eu~sSGpQ?*1B=_=#wS#Y;Z0s@P>}qN+duQAKYw*V225 zDMRC10X*$F_}Gff0IDjR>XRm?WQ5tY8GfS8dPVo!AKM*2-a{gAaJ$uU02$=)uCHI7 zXlP>o8kap^`rY@k`t9W+c6+$d8O2}Uic79H5SaWY#(TVcoKPndi&2v0SHP$l)DnO{ zkS9Yb4Rg1oO-(jF1B%(MLj{KmE_)MZau|kNuenvWm;1JUc;Jb#0p}FW3;yXE=h`74%Q7xtFZRinBjgr0AN}I zPRo*y6oa;{lXe-$pIxJtTP_CpeR7w^NByr~PS^C-M&RT<>AUj!9#r%96&>qXF7|l0 zdV&gXKD(EfZ7OcDq1hOnYcLgP9nf5|nZ4E-KwY_}RhQJw+9PLQgHvRDl~N2Zmm|$I zGu^RlBzEG&j>CA4miv5L{@CCvufwlDvq{52++ONBNxo-`a;Z{ z?NaX;n>HQhCMwrOc{J%lJwlaQcgbXDIk{qLAv<@KFxs43ucERvW z3aY1B!6k!YJ6KZfY%-;5J>1q5*Mr1DGD#m06DnL+t4;DS^<4PiY$tBL25ob54W_K6 zDs6fRycKTKfm`+5urf&-=*@J+3|UDObq9v!UYwT-*^l^uDKU8S1>0df!#x(u{r;`% z+__trM<<^Xk;Cwxe_&&MQ=p^ZG?JAbbhZUR#HAP#^lq3T>z#9nRf?)midjXHcs(n^ zH)m@8t}EHJ9-S@wP2b+bgP+^Ze@5XLcC>JH{JBjw@7Jo>u87(|7e~TlX8`x%Ry^UF>WB{S7 z3`}M-L(A}1b?|O7BnEOTlUad;_xba6C_tWy(!A`}t?%P0XHA49hvn4CeP;ECM5CoA z!*hO;)ngWvDfXlyFVWfQU7oC zfSoo{`+qGZc_sCQf)?&KTrQh_4rcZPm6#W8#?NNqmM0mkw5Rr`^KIX0a4k7=n{iuf zC8=L_rp&DTj9FCNulO7T#0hMbRi;y}uYN2*GEY;!R>dfSWsoMR^ zq~W=>4>tlM&A-7c7U{gBYHmR6KC+dlg^_*)v6WV307R!&oMRV*cNvaQxT@Bf99M1U zvME-Zm9(LmI=iLsejM0n1QFGgasX|uc&_BdNEa(L-!T?zz+^L+X$s)r*7yqAT>?oK z#&i$alk$)EjP!}xUfh=~-hUuCB@i%Gm}z#_Cr638PL+LcGAI^K+ck!_>RwlGt+wK` zq@f7@#&n4DB<<+z0_rpEq+Z*hkIl#OAlZRAbEMv+b*3&34yZBLNVTP6Z8sE#Haj3! zxo2Z~O#m|R)4D6IXR?A19TPvlzU6%}v@4qylTkMILX@S8jR(%RfXFG8Qirq(Lq+-p zd(Z-a7rY6M1HzN06ZPL;-+BzNCC_u|qbnZjSCdbGQV>*zy-7Vt;7IofJk>eso?{2x z0_c{0g*j{RgD8~8%w?VZd&RX(Oqdo(IrzT2bZE?@rn^4X<~9QI&ZJJAr>^nDBYM8Ny?C`^227_ujG)&LH;Z?EC3s z-F#y+1h1#+tW1~paB$XTz%m^oWKlHC{S%&+Hg z^Bz4xGRWY+do;qeO@rcb5)~$OPb-=(eEY}KLlJY(%YLpeU#sEddk8!OlPpX76pFU z*FwHe7XCfe!Q6gLi?{O(uGpw@i`~POLqs)u8EsO6Box}(La8N$Y`sh;D_vy=tlfJh zf(Q_jEvBJN9Z`V&_Rb0L*#o(BL8iNj&te37OPGYbI{+*?UMivS3Z(GZdv_UG}II~xZWJ2?}t9D17kkvU3nG&7~I(q0XW=ijH;=I-^R56yL%^j+7}5g zf4odaTVxmK_xQK+R5q(}Vec2edsaPD4&HdDUoUyfF}fAcJ*bpO_CNN)dfFT*FV*C#I?B-nBKmqnSj&!wIIh7&2+ zzp;D0tUbJ5|He6#(BNZ=lRJ_T9>k2zDpJ;s>uw;$;`cyc~?u<}qwUCM9L za#341pMMOnIqTandH)HZ6RVv}3^1nMSAoPugalssxv9UE6=+(Zb_{^p~Mt8*q5A9F7i7Kc)Rp7sobv9E^`CEGVDF-@iscFJ07^i$zq++>f@E7s&b2CZ z5OEMx$2xZ9Vrt52HkDe3xB3{6$amHXYl!5=>En#WQ+IqURea{LL9 zek$s3j{d`t`dAh78i)Nb7(J1{_sX%(|G8U>k#w<4+B*)_NZ+&MbI&TJz@1X7rCFC+ zE5_q%HF+-sYE%ZGaGfdY>D-_9#AqZnCvtKk2Ga!TcpoqZ^G|e7t7GxN@bb^~_WSC6 zPjWD0H*buse4t`DI@2ZZMiQx@w<$^|M6#?n=ZZ~@K2%m-aEIjEv%z*V5cMsh+ap?d z>n#8R4ITz46%JaeB5aVpA3c|xvs59}9Bi#NO94|&Iz=0sx3;(JrCse+cl!6@9#_2W z6sLH6X}?|dy+g0{3n4zp?zrCalYVy;Dj6HAgK2vB%T1A(c?&htD4g}nK#;;AYi-Jc zefjmzKgB%`b;85zzqY$~>FeN>Z!K&xeWwa&2O{&VVkK6Jt;t?h?+qX=`v43_#${V< zgsf@fqq8C4)l6%lpA@!FNY_U~XP*On7a3XaIl753(J>Bml9F~Bh`vB`ACAd<+ASVV z`qrTjP-!X04P7w+6~oi1K*|@ueQLgYmlgg`TPf-?(I*eHRjk$0TUCdd>fOH+xcw}! zIxSTHkW7u@udCfLS^2}k*U!SXtC7>U4_BL`Nxc#`oiv}Xi4z||d#B-ss~GSita=6v zKN@Jg8d&{o5Or0zWwkx7^i@mgO^BA$*mr@~x7WhXlL)Kiaoj=VcI5!y#x(BjrxQYV z*4|@=&7Fo&KM*<{^_0Jb@d&`vM8GuUYicPv-JP}+;BHhgsdu9+WhaS>tyYrRh-6E* zTyiyYc??Tm^!!6EX%5c6Hqtz%`hM6ujV>MDzdtCu@Asu1(9cK13@0_oACSBE;{pd2 zzd6Egsw6VS)D}VLf-$*NfF3Z2K$(qlApTOMtda{!GD(zfU0Ksx)G5!jPW_mGoszhA zZk^vb*q@HGzsbU_H&%uSQ1sd4R=7DG0SbI-3=~jxQ9)NxZXH_Hij*|t*GBrMYGb4Z ze7ckLiB3GZ%_Y={4}o~^WZOi6j8h6eww$#h61 zBWaCotc^5#GBN_(7jg-D*ma&(!NVHf!6C!T4L_e;1mFL0^uHgl{s(7zbM@-B7#)mF zzBrqWjRC}^L_5r)EYkhNnp&lfzpDML7-yA(>6c#6c8;}t*80z3fh!@7SB*K9+Yd6r zy>i?6y}t%%$caa!!b@s#hGRMTiF>unSHU7!nuDSTncOFORQdo!b=KmR5dDZ zl_u3fNslX6AU+&kdwE?B_2cnx!0yOVCxU}1C4aNh)D98JW@^-qGfH|~1t z4I~|`kwAMOTczZ`G^dIhTCh3>etKJF2g3y1w}h&-k;li>NxeDB{GO-Y}~TDTrJ8wC`` zp6j)K=|isS99SMbSNfXw{%SfB@ln_Hw$k&0zvcVn%LJJ!!;W*1x=bppLNP3@w$tJ;*dk+73J=TbwIww$FI3R#sFr1hK6 z7D7^L<>IHKYaiaa4{gNp@85bWa*0LR+*7Z)>D+P#f{NAj*4QfBfJgzOWROWK-Z_?5 zOHJy46WCL=*QM)Y3%+PEE$dvm^4_wOV=@Y2U$Cu`+60oii>=!LlTXLBg%Zii@-= z$5(<4(--3JsP~7}`26d)-U_F(jAsryxhTDYeT1iq(umo5o0PH6w_37kDMZS>Hsaj6 z<02_`j;3>QJ}AACDS1&!o0eW-NTp`2bc70q77ScdNKUb9$+j#B4qZC=#ROMvvqUT` z1-axUYm^TRq#=)mE-E!<8XBA(Im29##I{W<1ljNRBTl_ zx+IMQBF*B5RoyVH$D?Xw2FBvODngGOHFmeEpp=rP!$F8AT&;}YEaa3n1qdT&e8B`= zWcICsR{GYagGEzqWv-Wpp4^7d5wbZzzqJ|MbAC1#2m8j@vjB2Xc4py{%%)@)B?Ojl82 zFw#O*!`CTmZLqK*dgfB?>Bsj2wK|X0we9Z*%2wL<^>Hf3u_*H-#BfKefg6_1kMp?O zSzYbqIDu1Ia+QnxD31^K$6(trAuJsS9?Bc{`{oU&?;m3s^2YVRh$20|sO`QIl0P4Fc_hAZa4cuuQ)CXcJMdHplE0*Z@Tvyq!t6{+tx- zQ&sPW!0EA&_;9ZGhX~Wj>-z&i9OjLlv0Hw8EQi^v=V|$8qI1%GRvBxNl zkE?V(m$6n|&GxQ5b~qnji!DA(!$Q7P1ZC^0G9DW492^>wTBQ~bfZY1zJDg1EMn$jT zj~LF9NSI)pMX1o-`|>;dpuXfTjA)lKRU zqMB;$uq*Hs!Dp2$*4iUrS@bCztbG1>E@nr^#RqIS>97Lkrd$BcaS_HSMy}Zx*3p$F z?Hy`_4}?qfy8A$q9+^0UCUp?M_a?;-)8J)*<=$e*knlJ>P=4i(BOC_6c?!LY-!Md9>GJw`=J8FL&P`E+%6Ry{#4osN5AfXkG{skx9t}u1YjFf>O9r zQa2vQ8>GdChu89Ad>Tgs{mnCl+NY&&Gb?;7J3Piez%EF8iDVX&;DTK*s)JU*6X?Rx zdsrQVZzu`Ju=3t!mUdhz!cS9xv6{}P+~=f9v);%3Ez*D%UmkX==Z7ETE4Ssz zV^M+FDjhiY0{k(wvjowtMOzJb-d%=msZP}h16*A+(l^fvdyl%PnI5z!J&CVDzVgTO zdi8p*HpGCDIg>ne50 zr|Ze+25*;>L^gDbGH9%6fb8rN*-kNeucp1k<1v{~czoGjGZU+))nE7PKUXw9t+aS= zVBJd$MZ&i|DhDG8O(HnQ;H7PwB!vLCS?LyBlWkRjae!PTxf!d$&P8nKfj4{7yj{WQ z4X^E$9#fB_S_2tgusDzlSt6yX5d46ce`6hGCx*l7q(YU3?a)UJd)P8$wSfha%v;B5 zW1k)o_tpxyZ1>H9KhraExH}*D1LslwCc0ZQCN* z)D%!aW(1HKL5CeZd>#~vK0VLxaXG%}labH&&&@cOD*_IP zW_dz!4O!VyvUac}55@sfF&a1$mTD`^6QdZ(cv%`}?&p4yT-R72=&DUB4*+cn)dtr5150ODEuK;=VyCa!r>?<-U=6)}#jNkJ!=rzC zN9g4AL)RW0Yj_clTlq^+2YAKeGf0s-xj7$iq;S?$Fe4F|^G3UUaovHkkx;6Ln6 zf9YR4Tc2t7^$EhH17rx81mv&;*Crrk8w#WuNho-40q9L#4SX|`F;YmL_k}=R>w`a| z4KKw+c&z&`&&R6#`t-v8ytwjLM1J2O_pJ%NJtw0s{d@1@ya;BdGAU^|$EX4X)NJym zV~3lb1svRtnMJ0VG4bYgHgOI3P^?V*&fV9Gtm7{@DttRTDZ5Ig>8v@~oMXur>t+^@ ztb=u<^iVW2YYf;wh9rEXQUd7tWTZGt5^rf~|plzmac`w+A1Bb4^sUyQ*)l zF}KPR7chXUUYQ)V&6W0-jF*d2m+ZghQ!zk(~cJKF8%+eNEM z*qv-jR6u3c@Pl}PxD3Gh6jDx};#Nps(sRIPEMlf)nYGgz3F}%Y3f|Vw8!qnl3uAtX z7}p+j_Op4PXhz%LBvH{o%2qna-lkJeFjHuY(yJ}!x!xnh&kr$upsOP4mmOCIe$lc z2Uh{A9W?{!C!dj$cyTqfrEGW_udyQ^2l~dP6h;=`W|XQ3JP9k-^>uoE$g(f+@mYQN z!mC;<(&71f*nL4(Ft&t2V%-&q;A!-SNWG|)0f-eSFy65B*nT-&NxZjJd>w}m}U zPy3z~*|vvzJPSM{s(F7&4U981B*z?u%2CO;OfhX$wOTZ%cyJZ1dD6d~Ki7f;L!0T` z+(vWSx`)liSdYcq=SM|(_e?ow2H~{RAR&~JutkeVp4A|nNSM(}Yf~)mWb0_bM6P`CX zgS62yw{qB!IQU2x#tUUWCFHN)d#TD_@ak8+feZS|FNSOj4!%}Fb6UpmUQgMd^5yq& z`|xt2@+xzGbJ~LOr?#8t=h}M1uP60q*MoNVG*f5pUPIG{^;jY)A+I&X=IU&T*&Kp! zLJ;=Ommcw|z3n{{+xr%`+`;bdj%TJHD|2bc2X@7MKycWC!d8Lo@*xGVsO1GQ0~^Ca78TUR)z`C@VGy?0;(MurxL&bxiIt004}{%P=<8s4d62|s9e00ydi+5x z?k~$T_LpOeFSp-*KWfNdnk8ymU)^Ib=57Yqd1w*HI@E#1jvZ`-(F$}n4Xy2vuEB44 zegh64#^|Ep{ljc8?q~I+bI+Z{%JBbs%;Ru?Mx{*^r z6B-6rmA87{Z!p{Qa@7;X>c^+?>Ha~#e1Gl7p5M!bw|EW$&pqMyUg~&RGr8QGZ7#C= zzk1;n1j6qCD=aLVp9OTu6j1*Wry5}^UyJ}d&TT=kQ#Xx8W#{gPqRzNx7>im_PdjB$ zKRl|-cz66~0?QQ3#!5f!XtifET+ixlL~_|2!qevmx&Li(i@d*&nXxSvMW#b;M=FTT zVtPH`Ny1p1fWHMljixaXjGGDF75dPEzavJkABfG{zjQR@eO41iqQgYBbx*A!fT(*yJI zH$lk#*YiXF${_czrt20cZ}d!!FFzXSY{yQE;!^<8C07W!Ia<*I83N~#wl|$AAe2to zXbyWjNPlL7zCGMmchIX3JJiD=#Q=$Swsmlb)P;8E+7M(5*qyjk0^&^sPV9AJB1R*r zA8~*^Z4UcwmG9a_Tp0e(U%vVTSPz^8lADv&m@ve``#v^J+YE%CS~0Bw3*vJ+LAAo* zDi1FBR%-U^i<{Sn(5wWB3nJ`^iuL%0c7*F&e7PUaT*E)l$SQ#T4KiM3?3iUdz&t6&K&DGlGV6WQb z1wzL*4+X42JJ^Gib#ibrg2U1O10BylGTxt`zW%_q+M;330+ti1(spLr(7c4!OE0Zg zKeV_SY*Jy152nSLEQs7#&sV8lIGqXh0$txf*Ed>FS4Zt4n@w#AvAL=YMs|BRjRthi zHn>q-(H10u)#e0nZR@OU3vXc{$D*cd%Wjlb*hAS_T9PJ$QT18^%bo)^XaQvz#_6yY z5Ha@{4wddAGBeVV-K-P3|NS%X7w7GHyO003-y61P9b+6clibKaJ)q;Pftb=Fz%7tO z=%Z3nM60l@rn)NbV0>v2PQ#Yf`8_!H)3Vau?_cdCim4BVzSW^xtFs|oveMePJXt{M zYKv`xPEMqR*}KC)kr#pLK|a@$;CWrk^I&co#;Gs4@#UM3e|%n?l)s*kUu-W~{Jqzn ztTBT_X6*KGL1`2H&}Z-^wJObGescA`T0jYv06Grn0KyPmLR_-hZF26@skr(Jw+&fN zfY6cu@%++3X2g$_fO-6(6|Yd)!XS{^Swm#Mp$8G?{c_6dZLKSBcxMSArhBi7MA)SVP-;J{T9c}>+iQc4Evn8oU)_^_d%Be` zpT0jWXxZ*#Z5ew9yT{Fl*Ohh5bh#?v4k3n-lP-J}l*^t{|@9tExx#0|vLTiHYExqal0Sgor!|Yhq{F zrq)2st74V0(aDAfLP`}v!Wn^%(xjE;A&dR`ki@kEcmQ3>PuWMu-A-n~8TeKkzD z{cEKs>dndTBhb(@2K?IYzF?;h{fa4gQDu80(e{3;`~CeN2XILJm-FJM-K$)Q^l1_| zC>c8X)a+95cr#6B_hzs#J0OuE!B@l+XZp~CM3~k*Wc|*bo21{}|F|&32Yq__asNfx z9dbTMzk3~XMgS}_kj6&tTBeT=p!}drY7gt|l_v6^%6!|T>fTUNDsV!)o29N_ve~V`Jphpua)5$?p8n1SHe?B}YLWJ^wi|2Pa|0_-*9v9fIu%!Z1aOYh z%P17eix$yz>!AOit>ypye%pV)`RnU~GwomNk^z32?a4O~b!(NEFlC+ z!nW~Amr84}k{xEr=3cS$|5kM-#`kS;hhul|du_D%5v2B-K#i*9#T#N*3D~bOSYzs# z4$L68C}HxP(0R{t|4v;2w@&6y$}s2WPXOKs{0ldw;}Pfa&10zi_S4N ze>y7j%ja&%$GiT!KDiH9SH52`dHN)aso%elTXV$Xkr=0jX~5aU#$g(@6@i{v-kfXY zi6(G&PgW38#itQ~I(4@7s^6b;|5=}o#gqR!$P^uu^^4Tau>tI$JZ;UhUFX4=?Y=(? zzP&2HZO;Gkk2m+f-=1Xf*T4Pjjof~FZ5a4F?}V#iQBbmbuLP2GPXeYRqX;%|5>|m0 z8>dm3cu#d4T3I45cEJ)RKqE4jrwb^8|5mZkn=SClA0&;eptt=_=2E}YF@&B| zhp>0yP_WY_TWd-tn#3Y>H(U#0rWAmLSlmlToqV_fA4wGfV~pvui?vm^E(P>99c&En z;YR0MeX;m&zrO%b?V|y>S=x0`rF%?7eUOLHhOfXZl|zkru);C7F()ParbkMQp_#dH zc(Q!>bPWrTGJ3B)BT(}^65xKY@wR7U)eKW=OtZRX)PV@4K0{}5P0Ye|U)$!x%eei% z)DPReE3$)nQEs4Sswx3SII7Hw0BIW=inoA`7)%^Ozw0qW31)cK+lD&Kma-lqTVECg z>@Tfgk_Y2-Y=#3rZ~_`;5y^Wo_&*SBUbKyj6dORqddI(d&(6^!l!775BEUC_pRcA#5@LzNqkR~)`aF{vi_EbU( zvY{Y!7_Kv$Ir?-G4$=x2XW?b~&fov;JJTDiUaIlVWDx(=BJ2&Sq$Q;>P8^CnAWD=B zYGf%gq+xjV6X|h7p!no4^Os-$dI9m?OQ2TJIPg-WI0Q;;Rz%7oNNw3Q=WK2%VLvyO z-X_L9kR&3q1GakcdHD4sq|>+V{~dmPoL5lXz~Q?h zvUWNIAFBzD@3+@l#P@U|^1hh;Ub`MR3D{X2hl6S#rg%Fuoz+rhyKb)`_b>+CH;nHT zwQs%|ysm8K%ptAU1OE-9(N`F~cxQW;4s;PiqhPS|4sPy{%t0CueggJtX2(@gnza@? zIaF-YG+b}0(^Txa_3E8p3_pL}J^psPkQ><_8O&^u=F{O4ZxD-H>_~8@8N5t@msya4 z*c_Q>i>XnSeir28)P?|0T*^Nm3pk$_+PwA(!$cfultjqhgNh`V4+AwwLI9l}tb=8w zF_;9*sSY#+=nfE-LL>mHd8-=aMC)TwRJ8XtG~y!#!79iBfM-zpY)+mVlo@?Me~IE)MqlI^lFQhQC9 zjJ)Lte0=$3lhwT;AAedmWUH<0Gd~6z?Keezw*#D3Tv2P&C(XVe< z9R*>16tLTxQuk`KiyF2tR|BwB6!G3zMh;uzcH`s2vVb)1zO=jGQFVHoyE~XOM4@ia z96gj-HQQWN>D3MLGrM{Z;9@}9QxO((tS@ce8T98{{>?XZbUk0Rmh2z55^bh?Dq5h;mhGEbi5>i{^`#Lxqlh;$weIhbUOayH(tm8#L~#0f4X^{ z85i1P2a_ht^cdCDc&#&hkF zz0x_Zq{6mV;L$lOHhUF|HLi}L7r-mjKCK{4y{})SaD9LMesM3KcOx6BH|>9$7BCP{ zCg*BEqR|4Dnkd1WQ)$2-JpfL(5?#|Y5MEuVrCA#$J2(7B>*uqN!W&DFbp(3+;P-D& zeB9^t`Svm`x{J`94n~Vyw7-NNIEI>D*_R$>1yai_%RSbSTVL(O0w_b1RXY$E0cB*y zBo$FTZK$80#`DvF_oj0=KX;U_bq*hOa{uq!+lSX{lw)3h?!w?Sop#BeUi4m86F&XT ztIu;BP7l81bRMURB@c!ua?KhJ8(0ACQe=rvwh5au6rMI=Ut(o;>Fas(pQ7PAS=);H zSHfGw=N?l8`BR>Hjuv&s>0OP$eVC9Lc)aat8ev>Fex?TekYeq=hmAVklQ{F%ifn7N zeAZWM`!hkR-On{NByq+kW2wGbN+FbJSwb%%1KTFLu7VEL4E$GvN^w191-eR}{{0?>A)_HYBU9W#RZ4=}))2G|ZireHv3)Ywhk z5>Gm*0C}rnvHuM!PY#VAyqb=UXAwPCJT6`l!d`2(r39j$>6idYQllo*n*t*yhu~-~ zj^I@Q{n{!PePWYQ_y`e13&s%|j-ntqau&pf3v~HrC#= z8~lZ}cDvx#`(HT>1W#}Y(8Oq^gb@MPN4I@|TpOe^ycOxTiZ|J@tr(jP>?E6vjR->d zVV_ccST<4T-)jtotf^O7H2Im`|~$}aq<0Qxcc6nPI{Cm5cbYIy`=}|>3MhQ znpzCYC9y9|H<99XwMjHq>2SwBpf+dsBbgLzh*(f;@@2yE9$iE-HhVs|^dNfKQTNax;CCp2oc z6n^!{Ziw``)`W>%KfO7r!29^>cwSrQH=O#5}R;Dh~P}gWzzz(7c`~pgiaFIvk*)D*~ z00A^MS+f|7*ew7z}y`m1n;`BoC_&e8!+pTfb{_&!`~|FgmA}cc(rw1v_bBE zQ`Bb)9>4>TwM;s+Aj9TC3dkB%?B&cxe>tdHdF<<9r6TGYHWk(_{g4wc;cFA!3P9wi z=jyel&gwcl`G)H=YOJ*E3>ByWaG+a+BLpZUOl7Y@&^p6b-PQ^aIvt6>Lj6|r$mTwH zMjCG>0t>J**q``1#2&gH5;H_I$bxKIH&U=rY3tb#kk|L-8nxj0{>{Am{%!p9jz9a} z&$IX!^0Azd%Lm9CpFbne`}fY{HBWHjNjaxbfD{~Ucv+Y@ka9a6UOi3bYnq^^xeHZ) z2(Vk}R9ap?b-g@0n^SP0tP6~>yQm5tA5luA^-^f9Ov^1Fv6gH;l&X#cSB-UG4wB^| zS07Eg05<_uy3H-{zc*`ET7Gwofz3%-Zfhp(*lV3w_FZiEgs{WRh-D7W2&|+H30b3a zsfba8>HxOARrGa?x=VPvH+(?kR34#agdr|7*Bkx~!e1mvMR-cKm>wIAXM?pGN{ax4 z>MVk&*7Y-iGog)-9k`wR()EW8mhVdh%MYE+2(iwd=?nvK?rXuj1FtXCjusaYvJiFY zhc-0C!k?|zlKjdzSI*9_Gg1~D(sLC+sbrgh2Jpo!B}z*v(YXn*H)bRhjMO4Q`baMT z83V6>Y<0W-v%~WfAhA?R-u*PW03d1zSizpI;2>5qRKd!yJG(jo6~_dPUSlqTU_{?q zw(j*D`rL4LdiKqEQ!a51lwQ~Wf4<)S&wG>cp!drjm)#j1)-WCGi=je)aoJ~rC9fc# z-W!%xv+=p$L%s8oZ)n99@ELQb66^XQ#4d~BIc)^KrQW9#~ zs0E7zV44Rr#v5gMiM0k%P*_zcj>lH(%Aa=+U(avUUl$44`soU}I1_(jKxcT(UEz*J z^Fb{IXp-hwMne`lbu!IHM1+c@j66`WHbO}b+{^eeT&8$F3)bJQ)Od{L2+n?*x%~Dl z5Ak|yX`c(X*Do^v=XTDQQ@#VW`qz+GOQK!ub#LkiBN2=pz4eM8(Lms}|e*65d!;$$e zVid=j`5|Awd|KXt?%x-Q_%m8^7<6gup>`DNrFc=Fgo+uA!?hMGtLc&CMzv{7sXC?2 z5$rmX?5D@Ccgrf#-8t96R*HHXgE|~W%$(EuR5SR$ z(wH*Fm<)1xNWS>M+?Ht{2WX9Dwso|K0h~L>Y>);^IYHTU*tbO zK0Q+N{}Q|V?c8G0UZEG;AZ0qsmqLhJJ%MOgV69Q+$Px55BDwh9$^;Dc;*Ik& zc}M^5tp$4f`I~&X`R(rR_nVieJR8gYj>$Qt!1jSwG+6{@N}|oX~L}zbl9}HU2 zcr>FNAIOyU>2cX+5(RiCoGCo?awvxq99))+jmjJ9A z56%9G4o_3EZ8>eZXZ_UtzIHT^RArbMvOqj8U`reu%?%~DJy5oz;bRXj;{9MJiI!fC z!>cHp2bnKgjAv)+SLi>@4a-1$JIJ}v7?wbSmGg1-86x7YvK2M6<`X~}TR+a0Gm!FZ z!)!IaJ${q<{cQTn?zuFv^#^O@xslm+HIWv^G~EPvbak-OrdRwKAt@N7R;1N|)ikYI z*{E6JqdzOM{prsYM?aL z)2=AT;Wcjbww^fOZ4K^!zue2ie2OL=yFXFLTQLsQjt%oW$gi3 zW_O1}s@N(Le;E--4dHyf#QvrI z&%{yyVUrQ?A?(984OQasAHhcDDND^|^WpFAZ~nY{7z>%!D+I|@Fzv}3?`akf1`V7Y zWAojnR7?Zg#!;u6 zK&PH}DWj^xxMJdio8al?5DIvz0jns5Hrum>%@Vq&hlQYbM&U7a>x;5x7qJZ!0CLz(m~}%u@_eA>QpeiqhnFvQ@1~;A zQ&TNq;^IP$5wtu2CMJYu5uZy3=$gez2Mo&zJFwd5hI82{OAFU;{XYPyPfKrqAvteP z&HhV48m~WDplwubjwR~|fSBx$u%=Z_rQjz#AHZ4(QK|u2AwNT!PHDXZU^ljL>;-sR zh;#+A;XB0+wfC5tF9BQK(ow+2GI`XQ-4p~ec9O6^0$U+daWl7};$7W5416)sz@vOU zzxxL~+27OEdq-KZYODoSR32C+HokOj>^f5%G$4z8TH8XGj(y)fI7Hy>XWzK$9ooKs zz8x>u|6k?)*(YKNBHwSMTpEj9e^#n>#^!9h$K+XZl8R=%Ar^5e$X8386x3hZXDx!K`p1V4ju zK7dfNE-{Be&#PK{WHDY5q(THnfGJ>&R)3X=e9&LxeC+-Th4CdmD^uH8ft3Jmhbg{i z2{A~Cy#c$2=`)>|5+#KSQr;{b00s1POYC%qk^fX6SX4&s?g9WX0}>XMl2JYzM{Bns zVBnY>02mNZ05-h(VTqfCZqH_TfHIfehT`Cv3dQy~nbSHyyYxcc-J{UaC~VT(7CBqP*W zO%xi&j!szcw)NUa@u{WRZMMnzHVlBQcLD5MrVDn*|U>7A7wgVz-U^^Q~ znGpy2&o+HWywLJFK{%W`BPt~(U4%)pA zmDD!2&XW;my2)ZK)~Jv4C#je%5uOd2)+rq!GVsR&rU>c_ez>6`^7{Pmh?<>!?&^b8 zI#$tVBO|MVZKW`Podru885sWEDD8AwqR0=B3Jx~PI^?O9L@ zNU^~x4m!1Gm7|m}?swn*IFiQ~2g;RlM~?jF?N_gLju$3tsZ`2CyuCL*W+mw1#ID*| z*s|wRjZWG$wWrzz%o%Y&El?M4xe6A-G)QJas1&?)h<7^5g`eWyFWo6(R_IZT8YzPP z>SbW*RgjDdf~&0tAR;-w%I3!`1r*1F z+Rx4^I4p6?#u#@n1zR7*K@9M$%5wkHS3tjH*rZQSU`yPK2X$oKBT#{k-TOnl6pcWQ zBL*LVlp@k`PbHXj%Z8sKYOe(kU!Ro?@8#ilsz`phd-k+kQ({~#I6eH})c5HiiFndm z@Jo#2^q0?1cXvPGp1~YUFbqH_Ko>TFM!H^jOLXGY@vH~C9J5VAf(_DovRNx40(R0> zVEC!Ryi~=!XViV4T{~gu1j(z zkNMtUkCK+lc(D5^Cp#QRZcT#ZnDOcXPF)YdW-_l?k@FftA?p~~FW~T=BbaDM9smMc z&*KZZ)zyu2t{{3@Gvnklix#l`(Hx?Es6eF2WC|tFFCgm#h{Js4mhy}TlQe51&%}zy ze$il(0u98vt={>L8tvolx8L8NugwYib5WSIztlW1Km~cRu>ue@fZjgYL)u)-9t3A% zW?c7LKwOJpr--9ecalgX2%Pw_^|FxXU5~t%G-)X<3$*n{dG@E&?4>z83e=vL?Csup z8IDDIw|D%tYoFHG12af{NJzDq*39g~lG0C^ zi56lVwBA@hlwD^ujY^betUcf>vsK*mcBI!H1Tq@Xx&p*H_Vy#!)i$!N-I!r(Bo1sn z5A7j`Q3FVMuB8JIq^hcYtFHaUGImR(JdUvUgs-{SGf*{_^?M_}xp3KutT~n0A)_ zh7?oDfv!EQ{4aK9V|(pU1%bQAq9*3)o0LBBZp(E%BHql_Vg~l^`N@DM>cSK(j#7I9 zqtR9(fLz6FObVyiq0x>hffNF84@d`8-cDh&(+9*F|Bx@=d8_~akXdkJuN0N?5+x;l%K}1+0IhYpY_T=W0A2@pb3Jx3U0h|w=enb}JmG)i z*-PNt_Z#{4czg4t4`0`Am6nDfZ`wu=-+1XZGPY5j(>i3IKsf>LeWZ;a z0^sa1uwiC%2yfI!7M(r&_d|Px-B?-YE3HVy8tRl&HP|^8tEnYm%G{+PP*m!yq=?Kf zv3rN?VmF8JqRxD6vDl)$r6I21;XNoo&Uoa4L7qo8$bdD z#MgUCE)2YS@PW2fKfob;x;^*R5ZX!5PH6^)r~$;=S*c zNn9M$^wN=5Q|D0(I0mniuzL>&zb&(bTJfhiy7Yl{x&@k^pZe3$4C*{y?rwb1s59jK z;2C~rn8bDpdb`pZmV<`=9dd z+-GrrD+%DcoRbk9mOV*nsO!0d5rIcU!!AQCLqCu&CKVnV8@a$gI%{@JGwUZHuO|u@ z#`d^yjlgdDKm7j9BPQqZx4WmC`v?8wR{z}Hm#0sfsvmr7C>U%35Pk2$;{28eFh~J% zj3B^Zk+ZqULdya6LRplsc@Dv6zjurH|2!x{tG!bJIJgTA!2prcXX>#I*9pWyd>yD_ zYm;Xybo2^Nv8bJ$xn$%!p`Gb^H2w4T=_MdCo~BEkhKC{hg(i@B8M6Pkxq0X}&bV}A z?2Xyo%?+cUrOi^<28(@JCg8nJM;*$PDI4BOv4}p|AnByhMrDIMGAN+iM)|?p35fQc z3HX^l7D#&c@f!BVg0j%fHHgC=^uzjK3eZ4EYQevdbczc14P14 zVhdKm84>U`I|W_>GR&YVn%58RPUh_dZM>brr8Vh$hNCN!eEoG5O6Tc!^4v|x1q}?+ z=?EdPC^-ds9N3hQtt*=^WMUTkEL|Sg)>c-@o3=m*-ls58jRa zlil_F;s3&HVY&EXB4}`JopnrH-PXouaCf%?MT<;vXDAekLy^JV-QA%`DaCatRtjZ` zQ?$t7uEmPGTd@w#$9wNh-h1DC=VWD{ot$SU`;T9qWXoFlU#^E$?ah+-n(xgm!jwl- z(F=cO(G6na=U`P+@Xh-JhtY5E@SaN&O~~TwV%!QLbS#KEQA^T=detr^k<%lZYfI>O zAS&2e`+4Hs<63jBR=Z3a{-@rp`CA`q*bj;*@}B$8;QkP|pRbNDGPo0nS_-`=*}~J{ zUDy?5jca;Z@oHB{-T;`BtXlJ>*$6>w{bp$&_S$I=wpdq!@4OK^rx&XW8eqgo_I=TN zzd%7>Ni~ju6g_H{i$a{(M~AOz-BVe^{Fvu5>a-~*el^4|wKuwN)=uAY-dO9NgjD~q z8zHzOkE;MDTSKN{71e>#y`%k(vNgDGYcxpF9n-eDRIt7N#KkA?pyZZ|$HaDu;VR5f zImRVq%eL_d#z#}vUG`k&e%KzmOsXR{T7Z6_TvdD~4WxlzyOV6!!fLh| z5tKOis#LDr6xo-0Gf4w~lAB&G%EBiFaK3*2y=>#Jn1K9E$A(wsm0~ZV8Bun9%&VRWV9M3NbyPD+oZeHgbc~Lr z%A11|2=YxCeLo&M6q;HzD&qs0eXZ$OX|DXoYjAkNcS7!{5LaNr1a=L-nyI`;`|h=G z$Q>5+tjPcsPS=i!XCUa1N|8+Z&M%+djg~&F0Vk*iZESwM!4sAnT}7g{2u4w&HfB?z z70vZz)XNU_f-pv!1)*!=0I!3duJpe?vVE$;N0Uq@D4Zq=D!`)_7^uaXmlGb4O7Q?RL;hv?g4q1dV7D zTbE`c-^5;$T#4jKdy`Rj|DpHyYpU`yhSO1w4fwz%_PSCZ3bue4SNB5;D_}Cc5YrBE zP$#O@;w_|x6%*osiRboR6_@Ll?s>s?yB79%)1_#S%U!?rmq5xeT&HXmIlaD!Y%vFm z-1s!lKI%cA;;kJ65d$Ng#AM)*d+hIsJ2YzrR`xFlOqs1m_W znd})9eYy>-*U@Q>Lsq^sLK+D21?V!1BRM&r+`fiGE!Z`x23kDx)*pyf@s>4Qu_1}b zzL;r8LM>>+pMTDdsxdbmbs)gvjqA}GBkx2EH(TZ(0LnKz;mPo?8a!1s;gni*y#hK? zgRh_(AnQ$i!ISa&KG#SmJSb5P@@~-p@`oa3kxM&8S$*49Esh|R)pyXN)gY>RabWFZJ_ z`X=BgUCL^>)>M#L_if=!ms^qtDL$B}BrZ!lh1jKUb6;*MmgRZHh&m5sK>Xe6!gkte zx7=fYtaku&#+kKzE`2V2R4IATjAVvN0z;DJYDCS`1HM3QCLJ zUpE5xrNQRk&T%YyL=9LCqCsihdT-4>AS4F}@_G}D75NkUIA*ZJ#rU#V| z*OTS&yK7>X#~U~Ta(`SW8&{ zp#>RRo4CxJ_GBChNuKfA?SR2Sa|Ymro9304!}DitL30b>rJJEKbmzxnYf8dp|orI2U*JO_RrDD2=3=!A)B zhf^*S_5HwjBP)_4blfj&ibG9L3Ona#aC#d?omOs@XZKOH_U(Pg>7j`Y%c`S1r>0_I zwR$DP7?JPGO@Rarfq?2c3q*V-nw-)O-s&#B9l$rxQ{1p`Hbf+f57Z<;?RP;*OXrI9 zGq%C=YV@fdfE1txwsa+Gtr~xMq}4-Vrm24ZJfcY&)s(hLm`hhT-ueALhrEzP!S2UY zi#j>#2P^1z!7sMc?`VAaEavds*8mSHp3M&i9nXcx>M=t3jgNpqaaU zN>QC8`J?4Q>@80AVxMRgrS8wSRX?Gy+Mq+uB&(S7EsTz#oSLp}Cw^;&O3aYO3BLcysMUYtqX+te{Fu4$JS9ud(803+uE(6Ko`|a4 z!Vxe1DiI-4V&l(6^tNC2xU?C?C>(IsRbCL@1(G-tL%)T$!}*f)2=;G4k)m^idq!VI z1Xad7sdu^3siJ9ppNSd?U^voS`5KK(bE6AYMx-E$KaStZG)dzev;jVROo||BL8r^b zTvCs<#FW1<&8{tRo1Vn8il34$eX6pWPqZJve#TXG@8Xz2xFBOkTIL)fOcyNVV6iwXZj42OISyIxwVr|!I4SF#49W3zJ025Hb&ErvXlU6UypdG4-v+&%t zkJDoad9QEjWT(_VB$8FB=-pLi=y!T526t(Z#vm=ZZ*?L$k@t|F=Hxyqb?yrXM zYh!Y;Nk;e0i>bY=mZ3MbM>f$hz3>SCCh>Kb}29ywTo@;e;jb15f7;i2tqxx(e7zo^H z=u)qZ%#fpPAG%*o9~#4H_G!psJsK;fsdfP@vGJM_z5 z-NF>FJBPs)TZ^FnT9Pd<%XX>~c7EXFaQTUj;LTCUs93_E1pjjDLklZEHY5|ADM4j@ z{ge7WDzgcvRgyt3|lq*b^dek^4 zw^`^z@>9gEKH4}I?Q}P%Nok-;ni%*Lmy@IAWew91o$7Vogw75Rp_BR31(fQYUNEE9 zHBk)mYuo=Y!xXE3TRl-zwX05xFG$_^!{*Stt!IvTi)W@ZJfw5`L)YW@vE=olIj?sm z0Y<;d_gm3<(-o~sEk5~7+%E{_9A|%m$t@<+$&62djOKWf+B@qjKOH6K`<%zxXBQsz z_7#tVO;n~nFZOSmDspI4c8IlQ6~Z2}BTO=)>_;y0_c&Vx5f8-*+ClP8^} ze)s)*1m~pdS{+meX1{To>ns8J_l~HkIxOfUy(JTL-`ZbgeP3gci}iK#&FgCTp&AyS z5CY4)z9h!p8Mp|!;`=gtb<&yx{>|&8-K|4OJ5De(PHjT!LUcQpg7AXX>z~NIK%(MF z4mI|4gI&?gY`g{cuHUY}Htw93>25V2o~bJ#qmTdq0F0+lHd5kvJy!Pg_4pK6Pa$LG zZsWr3>Eh(5oiKqH{t`!a1wpEmXGNL+9u$blQ0$CK_Din5fTuTvT)nW3zx8YDo|8lY zRaIe>NNqOCnry_2S<2?&r$2T04z* z3-cX?Lsdz50(N8~uUNu>CCmgI5e5E5{J(SRZQQ9PMg{;b(E$L$r#X4JTY${%oI#dW zj#i#l-0mLbT25*UF9{x&P0V;PX}CW45<_t~?1flPBki~r$hBlEJdC)Az6#?pBR$-X zszNhJnTold(Ja`25sOP6-0t*E?#d1oQLk!}1-ZY!$nCx%xO&Zr`x3G}Tw55qQaBX1t zAS_MWe#+(%mj1y4z3RT!tQ4Y>^1y`T(hcx7yNr2w?}h#>Dr9!*{zOsTgd#& zY=dR_GGbHB}gbLQr|&bmla7#ugULKO1%G9PC~@lcIhDcNvPPJ1Bq zOR_Nx&IF|L_Bn7^O}FBM;z)Ax(&@&k_gTW5Y4t^j^a_+M)h0LsJp~hiB8zPkmc=IV zieX%%{7PXER7v{6Qc!nWTH`N|{rPi4P~rZ)r1VkaoOsFn4Y*49uv$v{kuSP@(W`~F zZdHXB;eZi1ZhNVIkM_R=Wv(TR-Xj43=1&9>{}(~7?k+a&Rvu3bJTbKL7em1(|6vFs zWy@Pug2Qs!S(|=@s-2W=2$MA@?<;uPmHmsMXQ}ALT=?%d#6o-!OFn-wq(CvG!)xtW zvN35~*{)Jir!RN4&CzRIQ6wTlH=h?8M-LHgT4l-g39fj8DD~JeK9n`oz@jj+#RnsM z5)vw&E7Qx<$t-l|T^AY}NS?^z+o|_rN6!okSHz;hKT^H@d#Q~kRD;_>zBo>TET3n# z)FuYI>nrfN$vLhVqD`R`9j_hK(pk>Fh`F$Brk!s2_dHS5Cti?w< z@R!9I{Lpgv;KPSXi2iPb+QD$Kpo?;u_7gspj2|Fx*v|1cv=s8o`r>I5;0uyUCD=ua}6fapj=d(X0%c1XI} zvbvq!F63?_Omz89f$huNx0Is3vO1*WCr z*~T%mw`%f3eo6}FW(AI@FD5Er=ta@_C4oF=&Z6PiU{nx!v!jxb!v16GEFmq`H*sdC z3suO&5)b>8*M_-Lya_o;I5pW}(D$AU!%PT?kRkAz{tez$@OJU>t zP~R@C?$xK$XW*I$=d9#+ZI~f3zsc?Q8CoweW>>Zizpi^#MM4n1L7SefM&$!YX*L>7 zZ(0cl<2dVHqA@)OQFB@ykbUdq#3I&XW^@bbGxY1Zi@GW+f`>Vka&dbiNb8H_z5%4g zimMeh7gXR8^PA$I3IyLL%109p79V}iacNP!6>liUphMVFazi4^u$nsDR*=v)Z4s}H zG#hZOPN+Xs54Iay>uTiTxNyI3E&9FFE^ycGp*dAh-lNF}xp&WhkRglC;Z!Go{M3c# zQ*QeP*H^_F9^PL1E?uYl$wPriB*=feXz`VQuS5a>aA$z@FBg46=3h@J@UQ0Ip6cjg z;qXu0Ki&7=DgYo9X!SJO-(LJr-9H`fZ=Fcu|KoT6l>Za&zvTfbf6M<1c6B9Gw7-5r Od3t|8wXpQRdjAJM+uj`j literal 0 HcmV?d00001 diff --git a/docs/web/static/code/ingestion-rust.zip b/docs/web/static/code/ingestion-rust.zip new file mode 100644 index 0000000000000000000000000000000000000000..6e71439d18d16e0c99bf873c4d04d04547349014 GIT binary patch literal 38092 zcmV)2K+L~TO9KQH000080000?T}vxO&#V9d0RI30015yA07GGNXKyZaZ*6QNhgV5CiYlR_4IV64_Ok<;Xqq1dnyog#f+7b+>r&4eNNYqnIp zZTxT)<(Eg*lf}P5JEu8NeBCX9C9+d%#x3P4{b2mk-}z6#9Co}V{1|JnsDh6Rg^)5k zkI7`IUq6F}KiCscO9KQH000080000?U8eYY#yyV!0Q_45015yA07GGNXKyZSZ)0oh zy;*M~NzyI&UcVwy_wly!^L-g$fC1)d-skP&LU4C8&t#V*s!6J3z8BUjG&~f zV9kQ*ulq}JNoBaZE$5iq^54GvpWj#8FMajEU+`yn*{vVTZq>@e!;dfj+c$pVP1!Mh zsXxB_?`89C{q>js_T~TB$ycA&_~U8EPlS*Adh_M6JiU~MFHEc5>gn6pUw-G!cC~)` z^1ps@zx?t){>O7^zm;$NKmX@1Pvw#E?yrStV;)vd{Oiege^p!Ssy8XIZ->vXSfByPQ z`_ApR?aQP53SB)}6|GU#>ZB=9%-(Sbwo+^{x{_?F$r#mH`D|MXlzUaVyPT@4zy3n} zEK7ZAE8gNa{*Pb2{Q7P6_^|r*U%&j%x#d0Z#+PjR~yHAE0dIMITrFJcYJC}Eje~9(Ns&dC?Xwo40&2x z`9CjZx4yc);`o?IlNK^e{6UBc>lavKl5gXEf}pxfAtG5DCx|2zD$*HAkZ=> zU9K7?RCU80!WwaSF3a&y;F5KSVw59`gVEFa_OPng&g^MoU18yMx#~iz)dpx%iqI;f zppXHWS&wn-|qeP?j_o zVy}uzE)-LG@8r4rSaN3^SW|4c8P=|6d@}~6)a>*B_`d$*^6}Nuz7K9q8R{vx!PF(# zFp6A?_mO){o_q^90S_~PlTxA_SVK|K*b4L4=AM7k?zVrF=g|t{;>aW99(!swt5Acf z(Yr?8ngHESa7G%&TT|;WcsgT&T)e47*=6gBbw18M|EB!da39IP{hF8Fe+UXEZNyA< zDw=yL_>Zl?gj{Q@z)<)D@^5^>z8AO{0F-K|rp~Qid;Uu|85h^z*dRXISn3M4rZ?(% zaGS9V9j%96H(!Bkxmf_JmQ=}DS8GuxND_2q{`p7Z?-O=^X+LNo;a0Kt*x;C%if!6) zgWntA}6owOYgind83u> z?)2sM_mfvtcID*d>bae~`uBGIbn#Veew z?&jwMd9OwrG7XF(Z~_wZx_Nmkra?Wbf-lc-2EAtjR@Yn;Ih?1e(Wa~C^UisUw)|x0 zfKk^^>*2?zm&ZF``PFXw{*KANck>-Htoe!TWXmcX&Z@f25?KJaUgJdT}bN!@ZbPH z)#S$4ANU8)w@X%DSKqK1pWKMSzTA8pT=Ru{^>AzO3}6;UgbWNCsMV*Md>7i^6+m1! z;7bj*H7gZZH?XVN7u48uJ9yRd?bfT|hg!CvLKlL&e(XQ)cBKMPo}C4xQ=(MDy7}@n zu%AC(-aQqV`I{dKXUD1a$R5-ba1`2}fmO8$;1Qs)%>lof0@Ug~MdLiMw)LPF0OdK- zj8T|8l#v&ku>=R-hCko-4(lA0{pxlNM|^u&&{8rIuCT>auwk8YC>?CXs$8rxsW+&2 z1w+=p7n2L1g2LA-&{<-h*5w(YJ=vi%9sfCNC&1w5+hZ)uTIwoU=aP*eJ}F^`s0P8= zQQK6Fv5L4hs{y8(vcQ2LYk_SGZb==#rq%tS?Dl_uuZ<6Xx%+-}D1U8t^xkF;xa#)1 zseCI&gWm@VF@tbvi*4$_r2(64B5(|WfPu7#4#yC|&q8i7fj-(W9ccPF%=vMPd7{j( zQah~eu{?hAgdgHQoFS zbp_Q=os5s!q)|wYL3FZ%l~5{6zyPey)Y==nu5;IP&0s925m&eAAE8^d zRF4gaR)b3gWt&9MLpC8w1kS6T6r?&v*T8wf;6v9vS-CXO+5@xRW}6NIe06&cU%&Yd z^#JrFkI8T6;PqIEA`cL*blSEA|dZR@A@vf1#{?(RP?{IXX@{#{-_;DP1pTjU3i z^=-fZ{bl#E;q99reO^7?J#W_k#`fZ`?D*a70;|R^ke#}?X0l612Xo@)JFs6*z=9?; zSpQZ*3Y~5Z&XP#(&q^=Jd+eD7DXi5KrNF%u8)x?Hbp==raoZkQ+R598-TBCNeff2_ z{%y5>Z{r@;qia=*9a}n_4LJg)Q%V}T@4CXGfz|5RGW0AL=qYLM zb5Wcv=xj@kfudcTxdg*f+udVno3$C1|DTujK`gMkFGIG3j_>WkV8t#j%hJ-f1zA~N z$dJmRvZRmr0<*&$SqPVpj=)XHHd|EJDYOcbkBw@PnLO2LuCT-g>@yd}*SdD;;F61_ znp9vUtn(e}kl@f1ynbXdRI|mv*_%W?u}O(xg{Ch-*F#pu-0CDWG)|vc z9QENTbLN(6=VEd&LARny*U@z&H$NJvk4p~0!CZp%;b1FHT>%op9!!V1KLk*A>&I2Q z`=k6e3XzKTE`8Ogz48@eXi4jkqTy(wE`c0tAQG^kmV8txhL#A_S7k_XE)q|rR7#t@ zDtg(iTmLqC{N??BQz@t*ca5zAY%(NF1glr2w3BFO3V8a?lG9a6O5P?{z-w@LSSqKP z#|KUP4ad~R3263fT-qDMwOWnmidh$<#WV&}tfJBt5hv?mQZ*i$w1n& zsT8IG;%frk;Dq0@o({&bt2lhy~@r+Z_QbK7Bg`SO36+um9M> z^F00bMU04^7vDp;c4T(U;(jNZYK;e37bYrMdb$)1Y^~TfnB1;(&Jc{?asHJ7a475U z0TC4}72cma`qej}m(^n#TTZw5HM0*2NK{S}U;;oZ)Mz1?R#>Bp+>Gu5&SLQ{c<5qx z)m5uQjLD>IrxSjTns+$u$GQ87&xSIOe08MKc5#bT84^#<>tF#ZvZoylB%z^gF{-Ybv6+k-L&FDkKmV+i|a2>yFF0f6k5IPwQg6Q_tx{rUsqQB06#xY zzJpXQp3?cmr*oLqu0P!LPl&H_lDBM?lioIDVM%1sC^d}wiH zHqyJ#Hu~4`g-GY%Putk74#~vjX?MR{J@P&O=h}{du}VBAit*0dgB~RztQc zXEPj3$7Z;kLumyDs^P#j%#l$f-ZF6no+qztnMtJ?a{mEPkMXCum2Q0+t#JAD{n{b9 zMT-&xL^Yc;AUG^$hl@QU8Z0Q6to+>BgHzCGrL-1TfZ}>o zk*#vwdC;9C_juSCWrOD{GX3~+QDkA3&(qhvQuj$oY}uw7gmfhDjV?p3D4Rf9wU$$I z5-=?hmdGZ{+Jc#Qu{pB~Sa0Zp^Dj@Dbtl!0Pfi2U1j$r)Nx@!%DWxE}Cmu4_d$h`P zC0!UMR4E1y3b6+_m)^S1iy8$imX9}TjxI%OYlP>DIk9%uL5!L=uEWsxZc249EAaB# zD-AN5W5Z%&ac}{1Ht`}8OWh>375F584S>FDljSN%H?8oqaguMJdhRwBQ*{uz@;OP~ z26u|pMy`JD*~h5ae09r5IeoH9Ww0y}b*-Ve-8q*bwzbrTV=zt99SHch8whCCKEq;=7U&D0W#;U2`(E&0 zqb^bV()b_%$O86(qU9D@dw5BX%oYFu78yoN6$3J3jIpVhq ze(79>JGi0NQ6DiA4Y>zjhDBY8@E!7u#^Euyv@F8ifWUy6uwwYlBC#MiA-Q~G;Cbi4 zWh01`00bX{lidK|1fY`p^*T-KR^_SO2~E5EYki21{(F13{jr_8{2nqe_8h)!C4}}w zU_YM@iM}OS-<<=s_TseK`tr^m!?Ms(4=zo&NGo$Kxxxv-fVW;G zc+Wa#POyfsaWLKt7v)r+;X>mhyq-+f9!4c`4y9ogAhdnBN2RltU&vegm% zY|XV$0GN>5od))v8A2P$Bxe%)=}3$``F(v^JrG|hUq1WRc%4h5qo|$Knk!v<2o4|- zTE}1*ApLMuT11fKCcMFBS0g|}s~Aj&z4i*qJROQV6psGuxaR9n@c51K-Ra^;Fp-m@ zKwt<4@c@5vIm6OukL3cw6Zk7geXa(eD)}yVH{SNzTc!-CKc{{tl?|s$h&Q7ro5SGzZv^2YrT zoY?N)-G=rm~?P@EGPi3-3c4u^C6Iunmh}vVwww;MD++p-fTa z44`x_c&BNsx+|L$Y0ALi{^a5(p3_nicU;hTX061YIji=UDd9nZh?VSxdy&f9JnHE>uhNDja5o8USELnIKZ?u#oc>w&_Kqh+XF=i^o zWVj@gn@+Z~rB>#0BgE$1_4UgWmEC2ks$TxCR-$E%G7PS@9D6XeI7uM1jFXkn3rKz{ zQq3U+#z0YkhN`8QD`_A43==+I#Vhx_-gsl`Z;dYs;?l<$Ex2vX(y7T)FscXFjTB>u z`4T)IG_+j7G<$Fw$yM`FLgo?nIHkEjZF_#ce@*WhyO5vAX{e9^0*c9Jg|FfLX&roLd7B2uJPyd;;wnVycbaZ68o+F&Wx#V4 z2VCz{k!y0_obffv<#N{EVW}l}$fsoJ`I+S9ccpW)+VSSnfG&U2DEBrlz={IfXVWvN zS?Iny{A48=LX!St;2|r{1I;#z|3qDgoG_GW|8QLK z-SfkG|3!IY2>MGn$sP)zM&Y;h`lLVGGdA&lyZY;J-HYF|I~I{{%8x?<;#;-KTj*|7 zqbMGG$B)kfyUz*@B*^#h8?CMZ@bh)t`uIpxtQedZOo^A|4R*15ARU}mvt-1V*8<}T zS8o~SMQKjb8U-FRbD5UT<55kV!0fV*KzFLHXUOYVytS68lHZp?sy5^hf!-8wIk<+T z+eby58<(DR}k_R&I5?q;!4!tgdpV#gCK@mT_1N*-`PB~nhwZOH& z0K0<4wUV24L1myFWfW_t3=_dhS-2enDvDy8!NIjFaH8IsxmfP!mwU-a2dBDt=(gkd zBonb`4J59!fFZdTR;oe9qlaD24Nee%)&tK<;JU(5Npnka#7ixA``hYSDvmA~+R|kr z)J-OKPJwte$#Q|)5y%QFW~__egO*9dtzMzpWNMc}$e2o&R8y~~Gl3Qo@%B$Q4WHZ=~vc~!VX^`mLT(QD2v33k}m%dtp zM)u0EyQU4jH?5^;@StL;6ugj?&a28>0_=6JOTbs+3@Lj9ou%32h0TxWois^(e7vNT z%Tp?lV3UpVo&fx)tJYZ3uoTuqE#Z|kzlHUU67QlKTCAF3ZU^V0rd{edBNCY62fp-j zrG#nRVn*Mfp{O_D&J>0wGra|(4&Ei)DoWadEg2tbOA>hpW@wy~iG>w3Q<@>pH0nBQ z5A)hdVzNEN{L-hQml_~s>BCloKM9iSnv{#+EZImYg`+A`>?OQebEv6A@F_OZKDTiI zWUe`;X%O*Vkd{hm-`AV%IHN$vKZpJ+ zL&acJSprtAI-bc%-#`8_m0|zf|2p|#h)AA%GJB4b+i-&x0lqNQu$ZCiCr*@rb~U84 z*P>MB7U*g3%A3i*(> zaw=fEWYhvhAAu!Pk*jbOoX*fE8?BLxr3^@sq5@6^e4OEH`wqg>Eon|#HvH?C(=MAQ z*=KJXaM$qT!}{CI+d~(PeaqbQkM}HN^4&Y|D7l~;e=0xjw|4%MySJ51e}3$)IQ{8s zaPs71{*&9waLsn{!SiP7y_v;zutWJZ+u0B9%C~Qjm}5M2@+(ig-7(F7{?>?d^5cEB z%yHhf3v-Iw0qZ+EZlaNZ2UugA>XFA{X~;~4jU9`N1m;khNKE#rDdLG72# z{&`+rR#%!UPH5K6O^l6=J8@8QZ3ZULd(aa5=@K8Y)|eqSW-7=4MagyU9N^5@SL(ga z+$^1#n2|*}`kb4ZgimSh$He?S*;`J~-f6jX#mwGZR&)fGHCBNPc?uE)wHp5^u1RuT z?_dJ>De5Fu3eD=wHY!Q;=-q)9zw3@JWaV8)GbuLzs9-E+|6V2RXGQ4)0&^MOyNR5@ zP!tG2ajs;~S|trkLblYJ6u3ZiIRL-J(y9w!)zXyRO7}eg!0y{jH=^{ zdi4wkhC`LbbU)Uqojw*mUD4j{5ptjVfw%iiu6GCX_0bc0wRkr_sgxx-y)c;^qT=Q{ z1?6pUwt%&{+`&eiG$YWBbxqBOn5!`@MAC+M8k(O9@NwflZq-h-_vXvz+`&#&s(Xw+xzO$8H z$-SxkuF##&L75f7{|BP4CFN;Sj4s6IXUx*&WP)2riC4Cp%*rRzbZX z(hpE_IV0Fva>+KpDmZW0j23!O@Mq$yeF=_O8a&vU7Q6qyG+aupA0`qJ6JAP zZj?`#zHa{jegktaO;Oh&$mkN63fY#EbaSaz`V&e)K{OqhrfLGhksi6d#NbMbJlDr# z@|zy|XjE!0#DrdZDs+mho2I2~rKx5xL5h4R(e+5>fltN zER`Wqb?r^EfwHTG6iwfE%p0oD;Am0EiIaOTV*3`DefOLD0PB19Y5b3Vn2nxr&C$e< z6YBXfLHNf)0RScqT(VoL0%$d7!&Nf9gA6_7Bw4dQ+md?3OD&_8U<8ON2v>!LdPg3n z5B<|S`u?W91@Ldr4@@ese$v`elJ9`-oYXAHum`ESw%lX2x|HCp${va&X0WP)EpA@F zPfEvT@MN^tp%3Egk9MqN=acGwPUV)lQP~%=?qfU${C1T&KGNOZcwU*ZFdO3kUiO=E ztd-sJw?}^3mDhEAIIcf+-*a<`QVP;ybYve?P8B-FO3M6ol(|%a$F1^-eEeJhU!A;;x=%}NWK)bu~sOc@;$d!b>r;7=d-lkj#1FHPbD_v zNV2^%TkCrFn%{*>OWm-2?5cFkVt!`d4>=3Bzp`?i61Pk~Ht?6^NhT`G*a>M@91KMh zwDhedY9Kbr$UIAcuWUSll)eYLewLj4B-dEd`TZTOIe*C8M?++4bIsQ4Ty@DhYa5L% zT3gk6!BMj9(HNyj?jv#DS(nh8cTq8cpUx!Py>oOw?XnK1bn|U7Y(wNA)f8Tn;wo;a znhqh?M)_QAs+x3{2J0a!nIC$<=*zj=Gl!hR+o@kuv)slFe-$3E!rXXnVx z{_8hCCQx1usYs=rtJ1PS@?)tR51Jdgl=D-PJbb4ist&phwCarlfpMq#@F;l3)5i^{%Gi;LYfhk*Q zrz5orL3(<3ZlPr6+%wgg2`zt50A?Za&-vJi^ZV>^{9`Oqx#~O@xo9X2xG3RJiYZa@ zMpy-(b<4rz7E4f8nxO`qS8$nIcQ$-Hx978#^nG&hbBgncgZKyCSOhJwGEPI~;tV!D z$;}^iPSV}ZT4}poESP~($`ufMU7b@-C&}TgdAfA@Q%bQ6xBNMM`RKLZ^XBF0n1KHG zbZq$Tx4%7MYRUkud6`58>*afOtvOJ=QoHZ5g7J}~*0UtwW+jbBdj}y5)LQ(w*L!cO z`4ygc>l~%eL>m^ok@>LfonGKAm1Z~>RvbC-pP0}(p$s>d;?iW^O$v&w_cG_tdzEyP z5$7Q_mbR6#NXmjt;EX`xGC#03nE(Z*;@=C5z0akuBpv*dhUm&!zkgcWiAlt_u{ll~ zN#ZUpv2jM`Xqup9d|Ea*r%)rq1i|Mk?K~OZ7*-0=>ls4ySp)KO=^!(qrVs4y ze;)pUi@c%;PMWC0<5pYKaD~@0*tL{!CCd@-WpEela%A}g6 z#3h*LWSP;F3A-edAGv?#`tofx9t@F|ho{`VHypZ8iJkPX0WhlI^`MKTX(u;Zv4Sxc znd{k<*UU=EXl+wvh~ea(zjM3C8$1?Y4!u~q9TUTL4j1Q;8@YG@Ht2FooSN~fM1yr# znMz=z?HaEkTrL&4bbxC6G<|{dy#9kX{c*3RAL$&ipU*Ve%X%owUA#HDiqxyhpd2;! z5`c7DA0`-E>B%K#%Db4ImoEF>N=w7u+2$U4c6sL7>Uh5@^cM0xf+qrrN%N-!g{B_7G|!Bx+NkH| zGvF}SB#3`=*)wkOIRw!wtKc+b5+VUmMV|sukl-8WDi+8x?;bveTTNJ$WoT;5%~umc znCam9UW3|`72aQrv01(+n;zVx7!(-MQl;^(s8mie(#iv85GkzDIqC5U*bGpOZzQJx zK21r}Ydk%r`Gx-HrR>&YBTG_fjxXj7cbZUaE}Cd+)Pt^tw( zmXkEO(hfMWqxIEG z!75cM0mlrUBU5e!9xx!#bqMeoAJ4n*Pon{1(*Aqt27ylD|FHQUE7jZ_RwdUM(kfAM z7Gkpsk}CbRIxs!-6Q-C6CaMeI_p`_KyG%Bb-U!kd@;urDom;x;6+_klzFX_wSgpHN zMLM(_?V?!Fl9Puefg2iPvP`NbqFAhpJ%G$O?O~c~p7AT^XZyLG zke>E;{b5~p<^sDeE!$njSq9fU@T##AJ%!EbcuSK89g)d-(O4<|g+hbG7HgW|J>5fv zS)K!LvINOiA9c69#jM<{vdmPF6yRujRz5cfIweUBhH)~zt6*B@?348A2SaHvvDSOA zt~ys=HXb=U1~*FsIACEFgVPFII>cXfGdWe|ND{*$pr%s2D90d*El08)s~`rGC{l$n zH;q)5vHcF)da;7#of@cYW^6vG9$6(7B%!$CryP<6?hMX44$3G&H%503wTA)mbR`3$ zr|rw3yG8ly$Ng5y84j;cyv4p<+MFt_{$#4F-Im9J55GJ>1>knTCvbt#9bIZjx|l>c z6IPgIFgCeTd{qWePVx>aw23qsI2C+r?hGHwZXfqJKSbld=C5XHd!VumXU5}Zj@C&k zUdH)>3c)GBR6wh`&QWv5<0m}0QCUOV0=wv$Y5Lk82Y^+4+e(mL9fI`oJnjClo+}5q zpL;eQ(zo~^w+=|nM2W$Yv`pW2-m#5c$RTz@C94QxY^4haoMGnLVRmA#T&3o7rtkQ1 zK#45)arLzN>wf*`>e^gTX+t_jpG(WBl_zho=CC5hsQ^3DYAzjA*EPTwcmO4Xf{iP9 zmn5mMC@?DE0XhG<;mv*hK*L4;t!*V`@LbSb@OAB*aa?oG%>*rTykK)c%jL?-1$b?T z_ic-iQWs)v$rc&9_nGH6%V);K#zt}glQ9X15lmJ&Edyk#OovP|eN6*K%MCS`z|yL@ zM0_Q5AAmDwpMUds|FU@)DDCL7N8ey3E3jg##(CItFOSxfH!-H@8U1 zTMAf@BFFT0-ZhU>>G0ha#qwEHoFipw7+2b`Vc zOZ&o-fo7#awC@BdZmeSp(hfa%8!WX>CflG~QnkpuoDR%5r*|*Y6SfwgvKf4BDpp}6 z4My5G>CdjUYA%KrZOYZbg4YsFkq#_LW> z1F_zmy?JOqaeTR5J^gm}#;$D7-;f@-*JR=Rfo{I7hB{nd?J?o>qlW1PPL&%D71^pL z;g^ya)L+COWXKhOSWd;j_iI?}>)Sfs!o&8wxA@Bo+#9T1L4XFnq#7Kg^zY?Fpn=c@ z7=fh7P|HriYev|$BHcAr4n2ug48hKFr?Ir$d?A3LNyf|O*uAubcZF*M%mGgVj5QXz zp^Y|jIR&kkbx61dw3h-Lg0DV0WiGt&voAq#9!c=hh7RMP3GnBBl3eE)Wlcb|2f(k6e3?gMr z5WGjR2E~-R=w+93*9Omgll!zY{k{1P$gI6yY6Oo|@K4fE3Fx(Fm~*UO+oZukJn)o( zQ!_&s%cQg9Zgow-UhkaaAi+0Zg!EF0B$8zeXRnp(t*Y#yq8VOG1N;JT zlu^*ZG@T;%DW^N~ug80wk9gdZP7;%;x$@>)NBZqUsq+djOfQu&cTLaqziEs?_kUA;SHH2Y+affrFT4YL7nHlPL=)n5cl>ef2syyCXJmU2x z;+-vd9q2Toz02eC1cxO>WV*wHnIT)qz$=5%MpsB?!K9(np0*@Yn!#3p0dod=?;&%Z z#^-M%sJ{GqSit@8k6v5$ZgNPuS$Y~hW zNbvMcNzfjK1y*K86y9Xu-`nwAXBqFgT^)aTysLOBob%^*!!CbOkuw2@HXEf2uB1@_ zy|{#9hh>t~OZWwLfIJ>VK#)MOPRjGy5Mh_5Ge@TsVspoR^~HZZPVJB#dA@QDH@5}L z-y17pFJ+g}G0Z@$GQ>0*sgNcaaSDYXZY4^$fuIOR8z9{TC4FAw|O*yoBJ z>H2Z7$uP|PY>N&*+RsSP>hWPUmY(HrdN`O2cAvpzEYZ*=zZFwS^^`4eBIGAPqbMsd zp`0Dqrl)|t?nZ#s9HtoB;$FDnrO&-df4!)BV4W2Ff-2dhN-a6!qOdF{!&yL#lc0Sc zCBLAS?qaPp49}i?_V45SHMjf-&u*KlA-68G$aIA2hJpZ;P{iuLo4Yddf#x*CIn<+VFWIdN2SzX`F|5rEX$3=iQY2AjYDjY~>b z376McIHt}(3xEuu4QUo^&#u5NUp(M}W1O+9BWPdl%|LHBGEld-rspcT#dIP_l#t_ql6{*%d5b{*aAvi_hs2&Y^4`ci=NNpp zFb6i38Ln`+miV&2O8LC3g~fF)IZZNUu(He`G$26MXzJi|No*6>EnUHi>rU-=mN~Jco>i}kc|Yw6>t$T1j1OF5L_rb7!Sm~@OqU!mvQaegjEPX1mQPfyl+KDIEx zga@<45rkUf2M?8$i|v9W69ahK{fc}=|Xw&zX$8tGB=N1`q z(7~!rXE*^rEIY2?a9&yz3<= zLc?%mz;Xkl0IR6rg{4bj>_86W;q72orb|BG_HhuY*Pq^~=3#knKi)l+-RgI~YrNT= zz5o2%>P|wM+ez+v@_TktReZPQ&F`yr&Ca|%$i*D`ym)KPn3SiWU~B-!VyH>(?Vyt? zvv8dvHC85O*E95z+#vlD*%D{$r*AP&duM-6%15`G^YZ?BYOCib-kk@}Ch~b^bYZ)0 zzwypooZI;^eI)E4&-1YUhW|c4yo{A%c{mi{I=RCAJ8Gp_%fO%!fQ@1x>j+3$(v5?Z zAUU{=s#3Gc04jwBY{Ap{<3pYM{2Eu~sSGpQ?*1B=_=#wS#Y;Z0s@P>}qN+duQAKYw z*V225DMRC10X*$F_}Gff0IDjR>XRm?WQ5tY8GfS8dPVo!AKM*2-a{gAaJ$uU02$=) zuCHI7XlP>o8kap^`rY@k`t9W+c6+$d8O2}Uic79H5SaWY#(TVcoKPndi&2v0SHP$l z)DnO{kS9Yb4Rg1oO-(jF1B%(MLj{KmE_)MZau|kNuenvWm;1JUc;Jb#0p}FW3;yXE=h`74%Q7xtFZRinBjgr z0AN}IPRo*y6oa;{lXe-$pIxJtTP_CpeR7w^NByr~PS^C-M&RT<>AUj!9#r%96&>qX zF7|l0dV&gXKD(EfZ7OcDq1hOnYcLgP9nf5|nZ4E-KwY_}RhQJw+9PLQgHvRDl~N2Z zmm|$IGu^RlBzEG&j>CA4miv5L{@CCvufwlDvq{52++ONBNxo-`a;Z{?NaX;n>HQhCMwrOc{J%lJwlaQcgbXDIk{qLAv<@KFxs43ucERvW3aY1B!6k!YJ6KZfY%-;5J>1q5*Mr1DGD#m06DnL+t4;DS^<4PiY$tBL25ob5 z4W_K6Ds6fRycKTKfm`+5urf&-=*@J+3|UDObq9v!UYwT-*^l^uDKU8S1>0df!#x(u z{r;`%+__trM<<^Xk;Cwxe_&&MQ=p^ZG?JAbbhZUR#HAP#^lq3T>z#9nRf?)midjXH zcs(n^H)m@8t}EHJ9-S@wP2b+bgP+^Ze@5XLcC>JH{JBjw@7Jo>u87(|7e~TlX8`x%Ry^UF> zWB{S73`}M-L(A}1b?|O7BnEOTlUad;_xba6C_tWy(!A`}t?%P0XHA49hvn4CeP;EC zM5CoA!*hO;)ngWvDfXlyFVWf zQU7oCfSoo{`+qGZc_sCQf)?&KTrQh_4rcZPm6#W8#?NNqmM0mkw5Rr`^KIX0a4k7= zn{iufC8=L_rp&DTj9FCNulO7T#0hMbRi;y}uYN2*GEY;!R>dfSW zsoMR^q~W=>4>tlM&A-7c7U{gBYHmR6KC+dlg^_*)v6WV307R!&oMRV*cNvaQxT@Bf z99M1UvME-Zm9(LmI=iLsejM0n1QFGgasX|uc&_BdNEa(L-!T?zz+^L+X$s)r*7yqA zT>?oK#&i$alk$)EjP!}xUfh=~-hUuCB@i%Gm}z#_Cr638PL+LcGAI^K+ck!_>RwlG zt+wK`q@f7@#&n4DB<<+z0_rpEq+Z*hkIl#OAlZRAbEMv+b*3&34yZBLNVTP6Z8sE# zHaj3!xo2Z~O#m|R)4D6IXR?A19TPvlzU6%}v@4qylTkMILX@S8jR(%RfXFG8Qirq( zLq+-pd(Z-a7rY6M1HzN06ZPL;-+BzNCC_u|qbnZjSCdbGQV>*zy-7Vt;7IofJk>es zo?{2x0_c{0g*j{RgD8~8%w?VZd&RX(Oqdo(IrzT2bZE?@rn^4X<~9QI&ZJJAr>^nDBYM8Ny?C`^227_ujG)&LH;Z z?EC3s-F#y+1h1#+tW1~paB$XTz%m^oWKlHC{S z%&+Hg^Bz4xGRWY+do;qeO@rcb5)~$OPb-=(eEY}KLlJY(%YLpeU#sEddk8!OlPpX z76pFU*FwHe7XCfe!Q6gLi?{O(uGpw@i`~POLqs)u8EsO6Box}(La8N$Y`sh;D_vy= ztlfJhf(Q_jEvBJN9Z`V&_Rb0L*#o(BL8iNj&te37OPGYbI{+*?UMivS3Z(GZd zv_UG}II~xZWJ2?}t9D17kkvU3nG&7~I(q0XW=ijH;=I-^R56yL%^j z+7}5gf4odaTVxmK_xQK+R5q(}Vec2edsaPD4&HdDUoUyfF}fAcJ*bpO_CNN)dfFT*FV*C#I?B-nBKmqnSj&!wII zh7&2+zp;D0tUbJ5|He6#(BNZ=lRJ_T9>k2zDpJ;s>uw;$;`cyc~?u<}qw zUCM9La#341pMMOnIqTandH)HZ6RVv}3^1nMRa-hWE>V8)LrZ|GYPMm4Xq=Iik1mp(j%lq&wop^MyhmWlJ6W=dUs@`Ow1hH#!o@cSH?fGRpe3xB3{6$amHXYl!5=>En#WQ+IqURe za{LL9ek$s3j{d`t`dAh78i)Nb7(J1{_sX%(|G8U>k#w<4+B*)_NZ+&MbI&TJz@1X7 zrCFC+E5_q%HF+-sYE%ZGaGfdY>D-_9#AqZnCvtKk2Ga!TcpoqZ^G|e7t7GxN@bb^~ z_WSC6PjWD0H*buse4t`DI@2ZZMiQx@w<$^|M6#?n=ZZ~@K2%m-aEIjEv%z*V5cMsh z+ap?d>n#8R4ITz46%JaeB5aVpA3c|xvs59}9Bi#NO94|&Iz=0sx3;(JrCse+cl!6@ z9#_2W6sLH6X}?|dy+g0{3n4zp?zrCalYVy;Dj6HAgK2vB%T1A(c?&htD4g}nK#;;A zYi-JcefjmzKgB%`b;85zzqY$~>FeN>Z!K&xeWwa&2O{&VVkK6Jt;t?h?+qX=`v43_ z#${VBml9F~Bh`vB`ACAd< z+ASVV`qrTjP-!X04P7w+6~oi1K*|@ueQLgYmlgg`TPf-?(I*eHRjk$0TUCdd>fOH+ zxcw}!IxSTHkW7u@udCfLS^2}k*U!SXtC7>U4_BL`Nxc#`oiv}Xi4z||d#B-ss~GSi zta=6vKN@Jg8d&{o5Or0zWwkx7^i@mgO^BA$*mr@~x7WhXlL)Kiaoj=VcI5!y#x(Bj zrxQYV*4|@=&7Fo&KM*<{^_0Jb@d&`vM8GuUYicPv-JP}+;BHhgsdu9+WhaS>tyYrR zh-6E*TyiyYc??Tm^!!6EX%5c6Hqtz%`hM6ujV>MDzdtCu@Asu1(9cK13@0_oACSBE z;{pd2zd6Egsw6VS)D}VLf-$*NfF3Z2K$(qlApTOMtda{!GD(zfU0Ksx)G5!jPW_mG zoszhAZk^vb*q@HGzsbU_H&%uSQ1sd4R=7DG0SbI-3=~jxQ9)NxZXH_Hij*|t*GBrM zYGb4Ze7ckLiB3GZ%_Y={4}o~^WZOi6j8h6eww z$#h61BWaCotc^5#GBN_(7jg-D*ma&(!NVHf!6C!T4L_e;1mFL0^uHgl{s(7zbM@-B z7#)mFzBrqWjRC}^L_5r)EYkhNnp&lfzpDML7-yA(>6c#6c8;}t*80z3fh!@7SB*K9 z+Yd6ry>i?6y}t%%$caa!!b@s#hGRMTiF>unSHU7!nuDSTncOFORQdo!b=Km zR5dDZl_u3fNslX6AU+&kdwE?B_2cnx!0yOVCxU}1C4aNh)D98JW@^-qGf zH|~1t4I~|`kwAMOTczZ`G^dIhTCh3>etKJF2g3y1w}h&-k;li>NxeDB{GO-Y}~TDTrJ z8wC``p6j)K=|isS99SMbSNfXw{%SfB@ln_Hw$k&0zvcVn%LJJ!!;W*1x=bppLNP3@ zw$tJ;*dk+73J=TbwIww$FI3R#sF zr1hK67D7^L<>IHKYaiaa4{gNp@85bWa*0LR+*7Z)>D+P#f{NAj*4QfBfJgzOWROWK z-Z_?5OHJy46WCL=*QM)Y3%+PEE$dvm^4_wOV=@Y2U$Cu`+60oii>=!LlTXLBg%Z zii@-=$5(<4(--3JsP~7}`26d)-U_F(jAsryxhTDYeT1iq(umo5o0PH6w_37kDMZS> zHsaj6<02_`j;3>QJ}AACDS1&!o0eW-NTp`2bc70q77ScdNKUb9$+j#B4qZC=#ROMv zvqUT`1-axUYm^TRq#=)mE-E!<8XBA(Im29##I{W<1ljN zRBTl_x+IMQBF*B5RoyVH$D?Xw2FBvODngGOHFmeEpp=rP!$F8AT&;}YEaa3n1qdT& ze8B`=WcICsR{GYagGEzqWv-Wpp4^7d5wbZzzqJ|MbAC1#2m8j@vjB2Xc4py{%%)@)B? zOjl82Fw#O*!`CTmZLqK*dgfB?>Bsj2wK|X0we9Z*%2wL<^>Hf3u_*H-#BfKefg6_1 zkMp?OSzYbqIDu1Ia+QnxD31^K$6(trAuJsS9?Bc{`{oU&?;m3s^2YVRh$20|sO`QIl0P4Fc_hAZa4cuuQ)CXcJMdHplE0*Z@Tvyq!t6 z{+tx-Q&sPW!0EA&_;9ZGhX~Wj>-z&i9OjLlv0Hw8EQi^v=V|$8qI1%GR zvBxNlkE?V(m$6n|&GxQ5b~qnji!DA(!$Q7P1ZC^0G9DW492^>wTBQ~bfZY1zJDg1E zMn$jTj~LF9NSI)pMX1o-`|>;dpuXfTjA z)lKRUqMB;$uq*Hs!Dp2$*4iUrS@bCztbG1>E@nr^#RqIS>97Lkrd$BcaS_HSMy}Zx z*3p$F?Hy`_4}?qfy8A$q9+^0UCUp?M_a?;-)8J)*<=$e*knlJ>P=4i(BOC_6c?!LY-!Md9>GJw`=J8FL&P`E+%6Ry{#4osN5AfXkG{skx9t}u1YjF zf>O9rQa2vQ8>GdChu89Ad>Tgs{mnCl+NY&&Gb?;7J3Piez%EF8iDVX&;DTK*s)JU* z6X?RxdsrQVZzu`Ju=3t!mUdhz!cS9xv6{}P+~=f9v);%3Ez*D%UmkX==Z7ET zE4SszV^M+FDjhiY0{k(wvjowtMOzJb-d%=msZP}h16*A+(l^fvdyl%PnI5z!J&CVD zzVgTOdi8p*HpGCDIg>ne50r|Ze+25*;>L^gDbGH9%6fb8rN*-kNeucp1k<1v{~czoGjGZU+))nE7PKUXw9 zt+aS=VBJd$MZ&i|DhDG8O(HnQ;H7PwB!vLCS?LyBlWkRjae!PTxf!d$&P8nKfj4{7 zyj{WQ4X^E$9#fB_S_2tgusDzlSt6yX5d46ce`6hGCx*l7q(YU3?a)UJd)P8$wSfha z%v;B5W1k)o_tpxyZ1>H9KhraExH}*D1LslwCc0 zZQCN*)D%!aW(1HKL5CeZd>#~vK0VLxaXG%}labH&&&@cO zD*_IPW_dz!4O!VyvUac}55@sfF&a1$mTD`^6QdZ(cv%`}?&p4yT-R72=&DUB4*+cn)dtr5150ODEuK;=VyCa!r>?<-U=6)}#jNkJ z!=rzCN9g4AL)RW0Yj_clTlq^+2YAKeGf0s-xj7$iq;S?$Fe4F|^G3UUaovHkkx z;6Ln6f9YR4Tc2t7^$EhH17rx81mv&;*Crrk8w#WuNho-40q9L#4SX|`F;YmL_k}=R z>w`a|4KKw+c&z&`&&R6#`t-v8ytwjLM1J2O_pJ%NJtw0s{d@1@ya;BdGAU^|$EX4X z)NJymV~3lb1svRtnMJ0VG4bYgHgOI3P^?V*&fV9Gtm7{@DttRTDZ5Ig>8v@~oMXur z>t+^@tb=u<^iVW2YYf;wh9rEXQUd7tWTZGt5^rf~|plzmac`w+A1Bb4^sU zyQ*)lF}KPR7chXUUYQ)V&6W0-jF*d2m+ZghQ!zk(~cJKF8% z+eNEM*qv-jR6u3c@Pl}PxD3Gh6jDx};#Nps(sRIPEMlf)nYGgz3F}%Y3f|Vw8!qnl z3uAtX7}p+j_Op4PXhz%LBvH{o%2qna-lkJeFjHuY(yJ}!x!xnh&kr$upsOP4mmOC zIe$lc2Uh{A9W?{!C!dj$cyTqfrEGW_udyQ^2l~dP6h;=`W|XQ3JP9k-^>uoE$g(f+ z@mYQN!mC;<(&71f*nL4(Ft&t2V%-&q;A!-SNWG|)0f-eSFy65B*nT-&NxZjJd> zw}m}UPy3z~*|vvzJPSM{s(F7&4U981B*z?u%2CO;OfhX$wOTZ%cyJZ1dD6d~Ki7f; zL!0T`+(vWSx`)liSdYcq=SM|(_e?ow2H~{RAR&~JutkeVp4A|nNSM(}Yf~)mWb0_b zM z6P`CXgS62yw{qB!IQU2x#tUUWCFHN)d#TD_@ak8+feZS|FNSOj4!%}Fb6UpmUQgMd z^5yq&`|xt2@+xzGbJ~LOr?#8t=h}M1uP60q*MoNVG*f5pUPIG{^;jY)A+I&X=IU&T z*&Kp!LJ;=Ommcw|z3n{{+xr%`+`;bdj%TJHD|2bc2X@7MKycWC!d8Lo@*xGVsO1GQ0~^Ca78TUR)z`C@VGy?0;(MurxL&bxiIt004}{%P=<8s4d62|s9e00y zdi+5x?k~$T_LpOeFSp-*KWfNdnk8ymU)^Ib=57Yqd1w*HI@E#1jvZ`-(F$}n4Xy2v zuEB44egh64#^|Ep{ljc8?q~I+bI+Z{%JBbs%;Ru?M zx{*^r6B-6rmA87{Z!p{Qa@7;X>c^+?>Ha~#e1Gl7p5M!bw|EW$&pqMyUg~&RGr8QG zZ7#C=zk1;n1j6qCD=aLVp9OTu6j1*Wry5}^UyJ}d&TT=kQ#Xx8W#{gPqRzNx7>im_ zPdjB$KRl|-cz66~0?QQ3#!5f!XtifET+ixlL~_|2!qevmx&Li(i@d*&nXxSvMW#b; zM=FTTVtPH`Ny1p1fWHMljixaXjGGDF75dPEzavJkABfG{zjQR@eO41iqQgYBbx*A!fT z(*yJIH$lk#*YiXF${_czrt20cZ}d!!FFzXSY{yQE;!^<8C07W!Ia<*I83N~#wl|$A zAe2toXbyWjNPlL7zCGMmchIX3JJiD=#Q=$Swsmlb)P;8E+7M(5*qyjk0^&^sPV9AJ zB1R*rA8~*^Z4UcwmG9a_Tp0e(U%vVTSPz^8lADv&m@ve``#v^J+YE%CS~0Bw3*vJ+ zLAAo*Di1FBR%-U^i<{Sn(5wWB3nJ`^iuL%0c7*F&e7PUaT*E)l$SQ#T4KiM3?3iUdz&t6&K&DGlG zV6WQb1wzL*4+X42JJ^Gib#ibrg2U1O10BylGTxt`zW%_q+M;330+ti1(spLr(7c4! zOE0ZgKeV_SY*Jy152nSLEQs7#&sV8lIGqXh0$txf*Ed>FS4Zt4n@w#AvAL=YMs|BR zjRthiHn>q-(H10u)#e0nZR@OU3vXc{$D*cd%Wjlb*hAS_T9PJ$QT18^%bo)^XaQvz z#_6yY5Ha@{4wddAGBeVV-K-P3|NS%X7w7GHyO003-y61P9b+6clibKaJ)q;Pftb=F zz%7tO=%Z3nM60l@rn)NbV0>v2PQ#Yf`8_!H)3Vau?_cdCim4BVzSW^xtFs|oveMeP zJXt{MYKv`xPEMqR*}KC)kr#pLK|a@$;CWrk^I&co#;Gs4@#UM3e|%n?l)s*kUu-W~ z{JqzntTBT_X6*KGL1`2H&}Z-^wJObGescA`T0jYv06Grn0KyPmLR_-hZF26@skr(J zw+&fNfY6cu@%++3X2g$_fO-6(6|Yd)!XS{^Swm#Mp$8G?{c_6dZLKSBcxMSArhBi7MA)SVP-;J{T9c}>+iQc4Evn8oU)_^_ zd%Be`pT0jWXxZ*#Z5ew9yT{Fl*Ohh5bh#?v4k3n-lP-J}l*^t{|@9tExx#0|vLTiHYExqal0Sgor!| zYhq{Frq)2st74V0(aDAfLP`}v!Wn^%(xjE;A&dR`ki@kEcmQ3>PuWMu-A-n~8T zeKkzD{cEKs>dndTBhb(@2K?IYzF?;h{fa4gQDu80(e{3;`~CeN2XILJm-FJM-K$)Q z^l1_|C>c8X)a+95cr#6B_hzs#J0OuE!B@l+XZp~CM3~k*Wc|*bo21{}|F|&32Yq__ zasNfx9dbTMzk3~XMgS}_kj6&tTBeT=p!}drY7gt|l_v6^%6!|T>fTUNDsV!)o29N_ve~V`Jphpua)5$?p8n1SHe?B}YLWJ^wi|2Pa|0_-*9v9fIu%!Z z1aOYh%P17eix$yz>!AOit>ypye%pV)`RnU~GwomNk^z32?a4O~b!(NEFlC+!nW~Amr84}k{xEr=3cS$|5kM-#`kS;hhul|du_D%5v2B-K#i*9#T#N*3D~bO zSYzs#4$L68C}HxP(0R{t|4v;2w@&6y$}s2WPXOKs{0ldw;}Pfa&10zi_S4Ne>y7j%ja&%$GiT!KDiH9SH52`dHN)aso%elTXV$Xkr=0jX~5aU#$g(@6@i{v z-kfXYi6(G&PgW38#itQ~I(4@7s^6b;|5=}o#gqR!$P^uu^^4Tau>tI$JZ;UhUFX4= z?Y=(?zP&2HZO;Gkk2m+f-=1Xf*T4Pjjof~FZ5a4F?}V#iQBbmbuLP2GPXeYRqX;%| z5>|m08>dm3cu#d4T3I45cEJ)RKqE4jrwb^8|5mZkn=SClA0&;eptt=_=2E}Y zF@&B|hp>0yP_WY_TWd-tn#3Y>H(U#0rWAmLSlmlToqV_fA4wGfV~pvui?vm^E(P>9 z9c&En;YR0MeX;m&zrO%b?V|y>S=x0`rF%?7eUOLHhOfXZl|zkru);C7F()ParbkMQ zp_#dHc(Q!>bPWrTGJ3B)BT(}^65xKY@wR7U)eKW=OtZRX)PV@4K0{}5P0Ye|U)$!x z%eei%)DPReE3$)nQEs4Sswx3SII7Hw0BIW=inoA`7)%^Ozw0qW31)cK+lD&Kma-lq zTVECg>@Tfgk_Y2-Y=#3rZ~_`;5y^Wo_&*SBUbKyj6dORqddI(d&(6^!l!775BEUC_pRcA#5@LzNqkR~)`aF{vi z_EbU(vY{Y!7_Kv$Ir?-G4$=x2XW?b~&fov;JJTDiUaIlVWDx(=BJ2&Sq$Q;>P8^Cn zAWD=BYGf%gq+xjV6X|h7p!no4^Os-$dI9m?OQ2TJIPg-WI0Q;;Rz%7oNNw3Q=WK2% zVLvyO-X_L9kR&3q1GakcdHD4sq|>+V{~dmPoL5lXz~Q?hvUWNIAFBzD@3+@l#P@U|^1hh;Ub`MR3D{X2hl6S#rg%Fuoz+rhyKb)`_b>+C zH;nHTwQs%|ysm8K%ptAU1OE-9(N`F~cxQW;4s;PiqhPS|4sPy{%t0CueggJtX2(@g znza@?IaF-YG+b}0(^Txa_3E8p3_pL}J^psPkQ><_8O&^u=F{O4ZxD-H>_~8@8N5t@ zmsya4*c_Q>i>XnSeir28)P?|0T*^Nm3pk$_+PwA(!$cfultjqhgNh`V4+AwwLI9l} ztb=8wF_;9*sSY#+=nfE-LL>mHd8-=aMC)TwRJ8XtG~y!#!79iBfM-zpY)+mVlo@?< zcr6t<0q9vwrWg*sM2J-^-fng4y8Ni@zPRk){pvB5NxM7~+bk1+*gDL0OLXk;ntA}J z1}U&uiFgSNnH3|+J*J#xtAkwe82aLN{^Q{Db*W3{Me~IE)MqlI^lF zQhQC9jJ)Lte0=$3lhwT;AAedmWUH<0Gd~6z?Keezw*#D3Tv2P&C z(XVe<9R*>16tLTxQuk`KiyF2tR|BwB6!G3zMh;uzcH`s2vVb)1zO=jGQFVHoyE~XO zM4@ia96gj-HQQWN>D3MLGrM{Z;9@}9QxO((tS@ce8T98{{>?XZbUk0Rmh2z55^bh< ze-`LEHiB=XTKw0IT(u<$*#ndrCt--rUMVtfv*sW?Dq5h;mhGEbi5>i{^`#Lxqlh;$weIhbUOayH(tm8#L~#0 zf4X^{85i1P2a_ht^cdCDc& z#&hkFz0x_Zq{6mV;L$lOHhUF|HLi}L7r-mjKCK{4y{})SaD9LMesM3KcOx6BH|>9$ z7BCP{Cg*BEqR|4Dnkd1WQ)$2-JpfL(5?#|Y5MEuVrCA#$J2(7B>*uqN!W&DFbp(3+ z;P-D&eB9^t`Svm`x{J`94n~Vyw7-NNIEI>D*_R$>1yai_%RSbSTVL(O0w_b1RXY$E z0cB*yBo$FTZK$80#`DvF_oj0=KX;U_bq*hOa{uq!+lSX{lw)3h?!w?Sop#BeUi4m8 z6F&XTtIu;BP7l81bRMURB@c!ua?KhJ8(0ACQe=rvwh5au6rMI=Ut(o;>Fas(pQ7PA zS=);HSHfGw=N?l8`BR>Hjuv&s>0OP$eVC9Lc)aat8ev>Fex?TekYeq=hmAVklQ{F% zifn7NeAZWM`!hkR-On{NByq+kW2wGbN+FbJSwb%%1KTFLu7VEL4E$GvN^w191-eR} z{{0?>A)_HYBU9W#RZ4=}))2G|ZireHv3 z)Ywhk5>Gm*0C}rnvHuM!PY#VAyqb=UXAwPCJT6`l!d`2(r39j$>6idYQllo*n*t*y zhu~-~j^I@Q{n{!PePWYQ_y`e13&s%|j-ntqau&pf3v~ zHrC#=8~lZ}cDvx#`(HT>1W#}Y(8Oq^gb@MPN4I@|TpOe^ycOxTiZ|J@tr(jP>?E6v zjR->dVV_ccST<4T-)jtotf^O7H2Im`|~$}aq<0Qxcc6nPI{Cm5cbYIy`=}| z>3MhQnpzCYC9y9|H<99XwMjHq>2SwBpf+dsBbgLzh*(f;@@2yE9$iE-HhVs|^dNfKQTNax;C zCp2oc6n^!{Ziw``)`W>%KfO7r!29^>cwSrQH=O#5}R;Dh~P}gWzzz(7c`~pgiaFIvk z*)D*~00A^MS+f|7*ew7z}y`m1n;`BoC_&e8!+pTfb{_&!`~|FgmA}cc(rw1 zv_bBEQ`Bb)9>4>TwM;s+Aj9TC3dkB%?B&cxe>tdHdF<<9r6TGYHWk(_{g4wc;cFA! z3P9wi=jyel&gwcl`G)H=YOJ*E3>ByWaG+a+BLpZUOl7Y@&^p6b-PQ^aIvt6>Lj6|r z$mTwHMjCG>0t>J**q``1#2&gH5;H_I$bxKIH&U=rY3tb#kk|L-8nxj0{>{Am{%!p9 zjz9a}&$IX!^0Azd%Lm9CpFbne`}fY{HBWHjNjaxbfD{~Ucv+Y@ka9a6UOi3bYnq^^ zxeHZ)2(Vk}R9ap?b-g@0n^SP0tP6~>yQm5tA5luA^-^f9Ov^1Fv6gH;l&X#cSB-UG z4wB^|S07Eg05<_uy3H-{zc*`ET7Gwofz3%-Zfhp(*lV3w_FZiEgs{WRh-D7W2&|+H z30b3asfba8>HxOARrGa?x=VPvH+(?kR34#agdr|7*Bkx~!e1mvMR-cKm>wIAXM?pG zN{ax4>MVk&*7Y-iGog)-9k`wR()EW8mhVdh%MYE+2(iwd=?nvK?rXuj1FtXCjusaY zvJiFYhc-0C!k?|zlKjdzSI*9_Gg1~D(sLC+sbrgh2Jpo!B}z*v(YXn*H)bRhjMO4Q z`baMT83V6>Y<0W-v%~WfAhA?R-u*PW03d1zSizpI;2>5qRKd!yJG(jo6~_dPUSlqT zU_{?qw(j*D`rL4LdiKqEQ!a51lwQ~Wf4<)S&wG>cp!drjm)#j1)-WCGi=je)aoJ~r zC9fc#-W!%xv+=p$L%s8oZ)n99@ELQb66^XQ#4d~BIc)^Kr zQW9#~s0E7zV44Rr#v5gMiM0k%P*_zcj>lH(%Aa=+U(avUUl$44`soU}I1_(jKxcT( zUEz*J^Fb{IXp-hwMne`lbu!IHM1+c@j66`WHbO}b+{^eeT&8$F3)bJQ)Od{L2+n?* zx%~Dl5Ak|yX`c(X*Do^v=XTDQQ@#VW`qz+GOQK!ub#LkiBN2=pz4eM8(Lms}|e*65d z!;$$eVid=j`5|Awd|KXt?%x-Q_%m8^7<6gup>`DNrFc=Fgo+uA!?hMGtLc&CMzv{7 zsXC?25$rmX?5D@Ccgrf#-8t96R*HHXgE|~W%$(Eu zR5SR$(wH*Fm<)1xNWS>M+?Ht{2WX9Dwso|K0h~L>Y>);^IYHT zU*tbOK0Q+N{}Q|V?c8G0UZEG;AZ0qsmqLhJJ%MOgV69Q+$Px55BDwh9$^;Dc z;*Ik&c}M^5tp$4f`I~&X`R(rR_nVieJR8gYj>$Qt!1jSwG+6{@N}|oX~L}zbl z9}HU2cr>FNAIOyU>2cX+5(RiCoGCo?awvxq99))+j zmjJ9A56%9G4o_3EZ8>eZXZ_UtzIHT^RArbMvOqj8U`reu%?%~DJy5oz;bRXj;{9MJ ziI!fC!>cHp2bnKgjAv)+SLi>@4a-1$JIJ}v7?wbSmGg1-86x7YvK2M6<`X~}TR+a0 zGm!FZ!)!IaJ${q<{cQTn?zuFv^#^O@xslm+HIWv^G~EPvbak-OrdRwKAt@N7R;1N| z)ikYI*{E6JqdzOM{prs zYM?aL)2=AT;Wcjbww^fOZ4K^!zue2ie2OL=yFXFLTQLsQjt%o zW$gi3W_O1}s@N(Le;E--4dHyf#QvrI&%{yyVUrQ?A?(984OQasAHhcDDND^|^WpFAZ~nY{7z>%!D+I|@Fzv}3?`akf z1`V7YWAojnR7?Zg z#!;u6K&PH}DWj^xxMJdio8al?5DIvz0jns5Hrum>%@Vq&hlQYbM&U7a>x;5x7qJZ!0CLz(m~}%u@_eA>QpeiqhnFvQ z@1~;AQ&TNq;^IP$5wtu2CMJYu5uZy3=$gez2Mo&zJFwd5hI82{OAFU;{XYPyPfKrq zAvteP&HhV48m~WDplwubjwR~|fSBx$u%=Z_rQjz#AHZ4(QK|u2AwNT!PHDXZU^ljL z>;-sRh;#+A;XB0+wfC5tF9BQK(ow+2GI`XQ-4p~ec9O6^0$U+daWl7};$7W5416)s zz@vOUzxxL~+27OEdq-KZYODoSR32C+HokOj>^f5%G$4z8TH8XGj(y)fI7Hy>XWzK$ z9ooKsz8x>u|6k?)*(YKNBHwSMTpEj9e^#n>#^!9h$K+XZl8R=%Ar^5e$X8386x3hZXDx!K`p z1V4juK7dfNE-{Be&#PK{WHDY5q(THnfGJ>&R)3X=e9&LxeC+-Th4CdmD^uH8ft3Jm zhbg{i2{A~Cy#c$2=`)>|5+#KSQr;{b00s1POYC%qk^fX6SX4&s?g9WX0}>XMl2JYz zM{BnsVBnY>02mNZ05-h(VTqfCZqH_TfHIfehT`Cv3dQy~nbSHyyYxcc-J{UaC~VT( z7C zBqP*WO%xi&j!szcw)NUa@u{WRZMMnzHVlBQcLD5MrVDn*|U>7A7wgVz- zU^^Q~nGpy2&o+HWywLJFK{%W`BPt~(U z4%)pAmDD!2&XW;my2)ZK)~Jv4C#je%5uOd2)+rq!GVsR&rU>c_ez>6`^7{Pmh?<>! z?&^b8I#$tVBO|MVZKW`Podru885sWEDD8AwqR0=B3Jx~PI^ z?O9L@NU^~x4m!1Gm7|m}?swn*IFiQ~2g;RlM~?jF?N_gLju$3tsZ`2CyuCL*W+mw1 z#ID*|*s|wRjZWG$wWrzz%o%Y&El?M4xe6A-G)QJas1&?)h<7^5g`eWyFWo6(R_IZT z8YzPP>SbW*RgjDdf~&0tAR;-w%I3!` z1r*1F+Rx4^I4p6?#u#@n1zR7*K@9M$%5wkHS3tjH*rZQSU`yPK2X$oKBT#{k-TOnl z6pcWQBL*LVlp@k`PbHXj%Z8sKYOe(kU!Ro?@8#ilsz`phd-k+kQ({~#I6eH})c5Hi ziFndm@Jo#2^q0?1cXvPGp1~YUFbqH_Ko>TFM!H^jOLXGY@vH~C9J5VAf(_DovRNx4 z0(R0>VEC!Ryi~=!XViV4T{~g zu1j(zkNMtUkCK+lc(D5^Cp#QRZcT#ZnDOcXPF)YdW-_l?k@FftA?p~~FW~T=BbaDM z9smMc&*KZZ)zyu2t{{3@Gvnklix#l`(Hx?Es6eF2WC|tFFCgm#h{Js4mhy}TlQe51 z&%}zye$il(0u98vt={>L8tvolx8L8NugwYib5WSIztlW1Km~cRu>ue@fZjgYL)u)- z9t3A%W?c7LKwOJpr--9ecalgX2%Pw_^|FxXU5~t%G-)X<3$*n{dG@E&?4>z83e=vL z?Csup8IDDIw|D%tYoFHG12af{NJzDq*39g~ zlG0C^i56lVwBA@hlwD^ujY^betUcf>vsK*mcBI!H1Tq@Xx&p*H_Vy#!)i$!N-I!r( zBo1sn5A7j`Q3FVMuB8JIq^hcYtFHaUGImR(JdUvUgs-{SGf*{_^?M_}xp3KutT~ zn0A)_h7?oDfv!EQ{4aK9V|(pU1%bQAq9*3)o0LBBZp(E%BHql_Vg~l^`N@DM>cSK( zj#7I9qtR9(fLz6FObVyiq0x>hffNF84@d`8-cDh&(+9*F|Bx@=d8_~akXdkJuN0N?5+x;l%K}1+0IhYpY_T=W0A2@pb3Jx3U0h|w=enb} zJmG)i*-PNt_Z#{4czg4t4`0`Am6nDfZ`wu=-+1XZGPY5j(>i3IKsf>L zeWZ;a0^sa1uwiC%2yfI!7M(r&_d|Px-B?-YE3HVy8tRl&HP|^8tEnYm%G{+PP*m!y zq=?Kfv3rN?VmF8JqRxD6vDl)$r6I21;XNoo&Uoa4L7qo z8$bdD#MgUCE)2YS@PW2fKfob;x;^*R5ZX!5PH6^)r~$;=S*cNn9M$^wN=5Q|D0(I0mniuzL>&zb&(bTJfhiy7Yl{x&@k^pZe3$4C*{y?rwb1 zs59jK;2C~rn8bDpdb`pZmV< z`=9dd+-GrrD+%DcoRbk9mOV*nsO!0d5rIcU!!AQCLqCu&CKVnV8@a$gI%{@JGwUZH zuO|u@#`d^yjlgdDKm7j9BPQqZx4WmC`v?8wR{z}Hm#0sfsvmr7C>U%35Pk2$;{28e zFh~J%j3B^Zk+ZqULdya6LRplsc@Dv6zjurH|2!x{tG!bJIJgTA!2prcXX>#I*9pWy zd>yD_Ym;Xybo2^Nv8bJ$xn$%!p`Gb^H2w4T=_MdCo~BEkhKC{hg(i@B8M6Pkxq0X} z&bV}A?2Xyo%?+cUrOi^<28(@JCg8nJM;*$PDI4BOv4}p|AnByhMrDIMGAN+iM)|?p z35fQc3HX^l7D#&c@f!BVg0j%fHHgC=^uzjK3eZ4EYQev&O^-Jp#$UykZ{ zp!+0DpnPReS>d{jSQ^aQGO8k3O1Mp1Jil4J?{u4i=*_d$U3n~GsSz~oXcWlMz++J zf&}%{33PVzWV*+6X5Fl}+1CS(eSYR|e5~f)E2ha~3qTSz%Q(zcB^E<&M57nO&Pq3k zwT0%l#W>@e9)7&qU#G4EEGbrOgfnc#_3Zp-81Hu48TU3gmqTuRP+P}mEA!fNsNw9} z;t&2oVt%r3cmmT5>DAAQ2;=S@zh!h!eI63QJ5|(VOgr+gCV!^8-hI7#ti^k6V{jB& z)oedZdP5ywA(v{SHw~+(3R37D>3349Ceo_bro?f|*yvIx@Vy##X zjihsmPs&zm6BJ8~p|3mdInVvLIe5eORonL{&vl7XA3Rckv!@0xIZ*&IAg(+pHfv$k zo2)2WLJ|!+H-4Jzb3Ft8j=pgcgl3#^M{VHnYxM7$HLv&t z>L>et9VWbk!Y@teA{}t`tn}ASbi3b^($_6ONhZZbDPi9A>;QAV2EK#pN%G8|g2h7| zd<~%-!XN>^w2=>Eaf4y$#UqNodNVKII#pV#U3mW*8uuFq9~2RZj+?=*5SPfxTkIO| zU1R?6V5TMr76R0cM+^~jOs7eueD9yn?9Rv>-ar^!jXgTI*5C!pje%3VS&+k2r8nhL zWt7VGVl~VT^VVaHvIxd`O9;FQekkdGe*m4f2>>SQ>`Gnv`vY$e2dNKQ8-nKtjsqBO zPZF#yLQc#7$%Mb3DVRGLZa@Uze!sXf!q8zmwXCf$-b8H$P$l( z;qx1)vFHWn>ID^(YJ18kV!G|SuSeWC)h{s~7HU5;>A0F361<@)hju#}Cfht-qZJs| zCATU2gmE2rPH`!jr{F_H-+jRx;NJxIH${MGM+bZn&v&}g?+P}6xRoM?7Np8Vw>oG>EOB#=ez1(R?w+^L*RC{TYxpoX9y_9cS3^sXS^+@(3N6kEm4*~D zhZ6R!=Wl}714L%+=^Ar0u?7OIUb!8tvI>pI@^gK<0APN%mza$BrY#7sj-b^6^@{4t z|9Sz{7O?qaBz81b-{%(PObjK%0i_P*`9F2U(+(Rx-*v}SircPG%XeX)7Z%fdO! zIYl6*%r|rGfmygzXWx`kWiocP3N5AgNhL{$TOh&^o)&HE2{?nY_CA_ca=^2%jn$7P z`?+^59ctSu@L_+BQ(6$%q>srWh+8Q;`z4@=y%AH>T`>2y^Q;y71XU$I=jKyfpT!6Y z4@3Y3Mwr*`_U0*PL^P66zcUU%otJUjkJ=Ayo!Ko6Tb#$wH)Ehf5pwj#47#mm2;Bhn znv=hVdCC^7Jb&(bV=I%bzh#fp5UtD+*9u9m-8!TTQR+S)Y%&~H;(6teW9UXNHITU` ze`I7P7i~aWh0vJ_w&+F|A-mjDJSj=!$V%fs%cPOJ_WjufPsMStRSati>J7-eUzy*` zIPM1D_s98U5ekgh*fxDfwPlossRz|Haflksx9=^|y(>sgVZ$)xWu_De=b@J-7M+4J z67<(0p#p1XZ&*og8z^MM%|fqE`eyRV0y*NCF1*3{YuZ!s6fm$|vXd36O^{F(a8F1@s@V6#Cd@na z9UME@)W*yedEAqAATE0%WWOaR7d#7*o4;;ZZrwj+Y73s7ms`9Z95t=KAEOGf*4X0m zLj`_ISFr5)G;#N{&X7?0J+DbIFaqc^Z3L*szUj9-9Kq)MoL(vJl{m*kk|z+p_e$W1 zoqdZJ{0ZyBz*r;aBPZyXfA|#78$)@5oQaUkHrzU$T)0>FL6t80w&Qr;%$8%tNtySp zN>P<&CCez8-?KlWN!p@;RkK#8#822@)h*(cZDxCbUyzrKao?8hAn5$ZZga7OsyPojT;1l2U42gv!?m`@J(=dx5p$l)25PQ8p zyC1Hs{b7evlx3kVe%ZHO3G7qqCa2E%*c!RvWy}n_`VzrHj`+1Mm}M9BLo%(<7_0%STF; zN)UIB?%^Poa(YnPR^kSdPTH8m8L|7tF%{al+^6w9h0xa zV(Oz_^xJ%ybTN#6Oj0JIxK7O0ekQ}y{5ay3k!h%sgfX;YlLFyh8{lI?N+d-q4k#CI zQ8Vrxp7NP_c5SKq^dzx$;*>(!gUV_?*=`{B2_O8{)hX-oj=~(w%i81->>c*&$vS|Z zU@d9XDuQofd?v*Dh$89RxEBJ}O*-1Xe@+il08)l??8O*si{yC7B z?{%XqZe&$1XroROh_}8+MVO4i-*K?ElZFXEvw;ADC@F`kD&|i0ELO_b1|6p1J<&L( z$mn?T22AQVPv=IHG*DTXU&K@k=txD&q6H01G71Il?KFV?CB9EBu3v**wJ(^OHOcYU``ED+Mc&daX{m5<+}P>3z41p> zuH0foL(_j5lO1KR;?po2BOe~+NRF#x3$mqI}> ziRl9c1w@WrhWj+Aj(=@ziEl$DyJC+7Vey&O;1eyB$3ios`yxIHB1kSK;WXf2s~pkr z17r1V*a0?8pPks7hLF)3v75TMZuI|DGr-cS3~}#_PSK(FhzzWg6Lpp5#o!iW@tp6(A)3Q z49jA)IunkUXZ39~grxh#?QVNsN(_I-PHq^Sy5563%VXxGbil^4moE1@E~ zczj&>&Zqd*?-#Q=6af37)9?eC;Wl^P`7fjzhKQE?Dr4%FyqyGw^poys3x5)jRxgd6 z;f{8O;0ro7iICbzoB*taAt)6Q5z>T@ULjT>DfBeg`uYl|Qr0-pQbJLKX$=?z+-m6W zvVx)Y^ zOn}3IypI$)DitM^S>`JIGd21V9ebv^cZx^XXOS}?z9i&2py2g{n&@+DKzG>Jox1rc zArBtNC0DDM(Q1mF5XWY^DnWkG{ZRRlzS#9a=!kUEMN&Yy&AyejKi4BO!YMH|Bcr4G zK02;7fl2M}rja@cZCP3YOj?|b+YO0&mzQry8U~>|i*=yQ4)NLfakUEb#hy33lbamO zq4{Yt)(N&w#arDi8S>g#vStw9l5%S7yw5`nWXFb`*I_e5gE&+HpujT2O4Ih`yrKa$EvPCpXy5C$qi;v04(#>TazqPdLfW^|_3;&&=N&?y4My zn5j<{F7*E~SK-mF?2vj@B(gufn_5X6FF$Kz4td#J=BXi{%OC3tp}#ubY1LmktSqRQ ze)<;HjoG{{w5Y&c8PA-*Q~OD$OyfmM9m8!qWn@RERT#(T`;sf=) zanysTscJ2vB}mdJ$j!V-*r(Q?$Qa$E5i zEJO@p<(yCr5TwNZgw53jCL^oTpFP21`K7n!(~TFHf2a#qdai2p=amPB(U+7yypDqQ zlxUKxjC@YoNqC7kpQ7CTqpKz7jRVTPP)p4Q4`Pf++eokhXh~V z?`J`yTic?mwGSz#y|_U}Kyz!>Se|=HdX9!Q#2EC~`iZ z3VHJLDsOzwcfM3FA}K22q4!RPZ&7c#Rp1h~1!DR;v~mvy1fo zo6XRD0LATtBm92+?3%X$N#L2^($esE7QBxe-6rh7AZfCr+(N%(ef#8a&@#dGFk9Ql z>uC4U65VILZ3IG>^&cu#9XAAn0%BVUwPHpCYpbZV#ZI-=dus#8JuXkcTb1tdbIsdl z3!l~uU!4|}Kk>LbPQ9GyQGzsCI2y=vi6wNN3dW`|9s4zP%sFdh+iu7KUEM$~K=0O$ zGxcYGUwj=_M2Tyv0v}Oe{GV5=^1K}V@!-w(5BA{L|7~Hwf2N8S9=5LhUarng|HhSE z2J}8p0sw9xkN$<5`Y+DO)ynbTxPM2H|Gh8_D4G2KVv43J7WO}f7!MuxFcPKzv-%I6 C&#c@4 literal 0 HcmV?d00001 diff --git a/docs/web/static/code/search-rust.zip b/docs/web/static/code/search-rust.zip new file mode 100644 index 0000000000000000000000000000000000000000..f08531a91d323fad6e28a5412120440a6c2f22a6 GIT binary patch literal 42197 zcmV(+K;6GkO9KQH000080000?UCT=bZdH!}00~?I015yA07GGNXKyZSZ)0ohy;*M~ zNzyI&UcVwy_wly!^L-g$fC1)d-skP&LU4C8&t#V*s!6J3z8BUjG&~fV9kQ* zulq}JNoBaZE$5iq^54GvpWj#8FMajEU+`yn*{vVTZq>@e!;dfj+c$pVP1!MhsXxB_ z?`89C{q>js_T~TB$ycA&_~U8EPlS*Adh_M6JiU~MFHEc5>gn6pUw-G!cC~)`^1ps@ zzx?t){>O7^zm;$NKmX@1Pvw#E?yrStV;)vd{Oiege^p!Ssy8XIZ->vXSfByPQ`_ApR z?aQP53SB)}6|GU#>ZB=9%-(Sbwo+^{x{_?F$r#mH`D|MXlzUaVyPT@4zy3n}EK7ZA zE8gNa{*Pb2{Q7P6_^|r*U%&j%x#d0Z#+PjR~yHAE0dIMITrFJcYJC}Eje~9(Ns&dC?Xwo40&2x`9CjZ zx4yc);`o?IlNK^e{6UBc>lavKl5gXEf}pxfAtG5DCx|2zD$*HAkZ=>U9K7? zRCU80!WwaSF3a&y;F5KSVw59`gVEFa_OPng&g^MoU18yMx#~iz)dpx%iqI;fppXHWS&wn-|qeP?j_oVy}uz zE)-LG@8r4rSaN3^SW|4c8P=|6d@}~6)a>*B_`d$*^6}Nuz7K9q8R{vx!PF(#Fp6A? z_mO){o_q^90S_~PlTxA_SVK|K*b4L4=AM7k?zVrF=g|t{;>aW99(!swt5Acf(Yr?8 zngHESa7G%&TT|;WcsgT&T)e47*=6gBbw18M|EB!da39IP{hF8Fe+UXEZNyAZl?gj{Q@z)<)D@^5^>z8AO{0F-K|rp~Qid;Uu|85h^z*dRXISn3M4rZ?(%aGS9V z9j%96H(!Bkxmf_JmQ=}DS8GuxND_2q{`p7Z?-O=^X+LNo;a0Kt*x;C%if!6)gWntA}6owOYgind83u>?)2sM z_mfvtcID*d>bae~`uBGIbn#Veew?&jwM zd9OwrG7XF(Z~_wZx_Nmkra?Wbf-lc-2EAtjR@Yn;Ih?1e(Wa~C^UisUw)|x0fKk^^ z>*2?zm&ZF``PFXw{*KANck>-Htoe!TWXmcX&Z@f25?KJaUgJdT}bN!@ZbPH)#S$4 zANU8)w@X%DSKqK1pWKMSzTA8pT=Ru{^>AzO3}6;UgbWNCsMV*Md>7i^6+m1!;7bj* zH7gZZH?XVN7u48uJ9yRd?bfT|hg!CvLKlL&e(XQ)cBKMPo}C4xQ=(MDy7}@nu%AC( z-aQqV`I{dKXUD1a$R5-ba1`2}fmO8$;1Qs)%>lof0@Ug~MdLiMw)LPF0OdK-j8T|8 zl#v&ku>=R-hCko-4(lA0{pxlNM|^u&&{8rIuCT>auwk8YC>?CXs$8rxsW+&21w+=p z7n2L1g2LA-&{<-h*5w(YJ=vi%9sfCNC&1w5+hZ)uTIwoU=aP*eJ}F^`s0P8=QQK6F zv5L4hs{y8(vcQ2LYk_SGZb==#rq%tS?Dl_uuZ<6Xx%+-}D1U8t^xkF;xa#)1seCI& zgWm@VF@tbvi*4$_r2(64B5(|WfPu7#4#yC|&q8i7fj-(W9ccPF%=vMPd7{j(Qah~e zu{?hAgdgHQoFSbp_Q= zos5s!q)|wYL3FZ%l~5{6zyPey)Y==nu5;IP&0s925m&eAAE8^dRF4ga zR)b3gWt&9MLpC8w1kS6T6r?&v*T8wf;6v9vS-CXO+5@xRW}6NIe06&cU%&Yd^#Jr< zJ9V33HFKc?(ULH13s8R@xJKP;jRf`YLE8+hDGr;laYZvtqsY|`w(#3l6jL1(^O|y9 z--*3A-Fzn!C|vN(I~`5IY9oM3H$6%CGqNJv2~eA<7KjPpba$<3P;;NMGx*Pkh|J%R znveVB1EapZBgsUu5g0|4tE=vSUf7k^zyc-MDEk9QS8JeTtRdFkI8T6;PqIEA`cL*blSEA|dZR@A@vf1#{?(RP?{IXX@{#{-_;DP1pTjU3i^=-fZ z{bl#E;q99reO^7?J#W_k#`fZ`?D*a70;|R^ke#}?X0l612Xo@)JFs6*z=9?;SpQZ* z3Y~5Z&XP#(&q^=Jd+eD7DXi5KrNF%u8)x?Hbp==raoZkQ+R598-TBCNeff2_{%y5> zZ{r@;qia=*9a}n_4LJg)Q%V}T@4CXGfz|5RGW0AL=qYLMb5Wcv z=xj@kfudcTxdg*f+udVno3$C1|DTujK`gMkFGIG3j_>WkV8t#j%hJ-f1zA~N$dJmR zvZRmr0<*&$SqPVpj=)XHHd|EJDYOcbkBw@PnLO2LuCT-g>@yd}*SdD;;F61_np9vU ztn(e}kl@f1ynbXdRI|mv*_%W?u}O(xg{Ch-*F#pu-0CDWG)|vc9QENT zbLN(6=VEd&LARny*U@z&H$NJvk4p~0!CZp%;b1FHT>%op9!!V1KLk*A>&I2Q`=k6e z3XzKTE`8Ogz48@eXi4jkqTy(wE`c0tAQG^kmV8txhL#A_S7k_XE)q|rR7#t@Dtg(i zTmLqC{N??BQz@t*ca5zAY%(NF1glr2w3BFO3V8a?lG9a6O5P?{z-w@LSSqKP#|KUP z4ad~R3263fT-qDMwOWnmidh$<#WV&}tfJBt5hv?mQZ*i$w1n&sT8IG z;%frk;Dq0@o({&bt2lhy~@r+Z_QbK7Bg`SO36+um9M>^F00b zMU04^7vDp;c4T(U;(jNZYK;e37bYrMdb$)1Y^~TfnB1;(&Jc{?asHJ7a475U0TC4} z72cma`qej}m(^n#TTZw5HM0*2NK{S}U;;oZ)Mz1?R#>Bp+>Gu5&SLQ{c<5qx)m5uQ zjLD>IrxSjTns+$u$GQ87&xSIOe08MKc5#bT84^#<>tF#ZvZoy zlB%z^gF{-Ybv6+k-L&FDkKmV+i|a2>yFF0f6k5IPwQg6Q_tx{rUsqQB06#xYzJpXQ zp3?cmr*oLqu0P!LPl&H_lDBM?lioIDVM%1sC^d}wiHHqyJ# zHu~4`g-GY%Putk74#~vjX?MR{J@P&O=h}{du}VBAit*0dgB~RztQcXEPj3 z$7Z;kLumyDs^P#j%#l$f-ZF6no+qztnMtJ?a{mEPkMXCum2Q0+t#JAD{n{b9MT-&x zL^Yc;AUG^$hl@QU8Z0Q6to+>BgHzCGrL-1TfZ}>ok*#vw zdC;9C_juSCWrOD{GX3~+QDkA3&(qhvQuj$oY}uw7gmfhDjV?p3D4Rf9wU$$I5-=?h zmdGZ{+Jc#Qu{pB~Sa0Zp^Dj@Dbtl!0Pfi2U1j$r)Nx@!%DWxE}Cmu4_d$h`PC0!UM zR4E1y3b6+_m)^S1iy8$imX9}TjxI%OYlP>DIk9%uL5!L=uEWsxZc249EAaB#D-AN5 zW5Z%&ac}{1Ht`}8OWh>375F584S>FDljSN%H?8oqaguMJdhRwBQ*{uz@;OP~26u|p zMy`JD*~h5ae09r5IeoH9Ww0y}b*-Ve-8q*bwzbrTV=zt99SHch8whCCKEq;=7U&D0W#;U2`(E&0qb^bV z()b_%$O86(qU9D@dw5BX%oYFu78yoN6$3J3jIpVhqe(79> zJGi0NQ6DiA4Y>zjhDBY8@E!7u#^Euyv@F8ifWUy6uwwYlBC#MiA-Q~G;Cbi4Wh01` z00bX{lidK|1fY`p^*T-KR^_SO2~E5EYki21{(F13{jr_8{2nqe_8h)!C4}}wU_YM@ ziM}OS-<<=s_TseK`tr^m!?Ms(4=zo&NGo$Kxxxv-fVW;Gc+Wa# zPOyfsaWLKt7v)r+;X>mhyq-+f9!4c`4y9ogAhdnBN2RltU&vegm%Y|XV$ z0GN>5od))v8A2P$Bxe%)=}3$``F(v^JrG|hUq1WRc%4h5qo|$Knk!v<2o4|-TE}1* zApLMuT11fKCcMFBS0g|}s~Aj&z4i*qJROQV6psGuxaR9n@c51K-Ra^;Fp-m@Kwt<4 z@c@5vIm6OukL3cw6Zk7geXa(eD)}yVH{SNzTc!-CKc{{tl?|s$h&Q7ro5SGzZv^2YrToY?N) z-G=rm~?P@EGPi3-3c4u^C6Iunmh}vVwww;MD++p-fTa44`x_ zc&BNsx+|L$Y0ALi{^a5(p3_nicU;hTX061YIji=UDd9nZh?VSxdy&f9JnHE>uhNDja5o8USELnIKZ?u#oc>w&_Kqh+XF=i^oWVj@g zn@+Z~rB>#0BgE$1_4UgWmEC2ks$TxCR-$E%G7PS@9D6XeI7uM1jFXkn3rKz{Qq3U+ z#z0YkhN`8QD`_A43==+I#Vhx_-gsl`Z;dYs;?l<$Ex2vX(y7T)FscXFjTB>u`4T)I zG_+j7G<$Fw$yM`FLgo?nIHkEjZF_#ce@*WhyO5vAX{e9^0*c9Jg|FfLX&roLd7B2uJPyd;;wnVycbaZ68o+F&Wx#V42VCz{ zk!y0_obffv<#N{EVW}l}$fsoJ`I+S9ccpW)+VSSnfG&U2DEBrlz={IfXVWvNS?Iny z{A48=LX!St;2|r{1I;#z|3qDgoG_GW|8QLK-SfkG z|3!IY2>MGn$sP)zM&Y;h`lLVGGdA&lyZY;J-HYF|I~I{{%8x?<;#;-KTj*|7qbMGG z$B)kfyUz*@B*^#h8?CMZ@bh)t`uIpxtQedZOo^A|4R*15ARU}mvt-1V*8<}TS8o~S zMQKjb8U-FRbD5UT<55kV!0fV*KzFLHXUOYVytS68lHZp?sy5^hf!-8wIk<+T+eby5 z8<(DR}k_R&I5?q;!4!tgdpV#gCK@mT_1N*-`PB~nhwZOH&0K0<4 zwUV24L1myFWfW_t3=_dhS-2enDvDy8!NIjFaH8IsxmfP!mwU-a2dBDt=(gkdBonb` z4J59!fFZdTR;oe9qlaD24Nee%)&tK<;JU(5Npnka#7ixA``hYSDvmA~+R|kr)J-OK zPJwte$#Q|)5y%QFW~__egO*9dtzMzpWNMc}$e2o&R8y~~Gl3Qo@%B$Q4WHZ=~vc~!VX^`mLT(QD2v33k}m%dtpM)u0E zyQU4jH?5^;@StL;6ugj?&a28>0_=6JOTbs+3@Lj9ou%32h0TxWois^(e7vNT%Tp?l zV3UpVo&fx)tJYZ3uoTuqE#Z|kzlHUU67QlKTCAF3ZU^V0rd{edBNCY62fp-jrG#nR zVn*Mfp{O_D&J>0wGra|(4&Ei)DoWadEg2tbOA>hpW@wy~iG>w3Q<@>pH0nBQ5A)hd zVzNEN{L-hQml_~s>BCloKM9iSnv{#+EZImYg`+A`>?OQebEv6A@F_OZKDTiIWUe`; zX%O*Vkd{hm-`AV%IHN$vKZpJ+L&acJ zSprtAI-bc%-#`8_m0|zf|2p|#h)AA%GJB4b+i-&x0lqNQu$ZCiCr*@rb~U84*P>MB z7U*g3%A3i*(>aw=fE zWYhvhAAu!Pk*jbOoX*fE8?BLxr3^@sq5@6^e4OEH`wqg>Eon|#HvH?C(=MAQ*=KJX zaM$qT!}{CI+d~(PeaqbQkM}HN^4&Y|D7l~;e=0xjw|4%MySJ51e}3$)IQ{8saPs71 z{*&9waLsn{!SiP7y_v;zutWJZ+u0B9%C~Qjm}5M2@+(ig-7(F7{?>?d^5cEB%yHhf3v-Iw0qZ+EZlaNZ2UugA>XFA{X~;~4jU9`N1m;khNKE#rDdLG72#{&`+r zR#%!UPH5K6O^l6=J8@8QZ3ZULd(aa5=@K8Y)|eqSW-7=4MagyU9N^5@SL(ga+$^1# zn2|*}`kb4ZgimSh$He?S*;`J~-f6jX#mwGZR&)fGHCBNPc?uE)wHp5^u1RuT?_dJ> zDe5Fu3eD=wHY!Q;=-q)9zw3@JWaV8)GbuLzs9-E+|6V2RXGQ4)0&^MOyNR5@P!tG2 zajs;~S|trkLblYJ6u3ZiIRL-J(y9w!)zXyRO7}eg!0y{jH=^{di4wk zhC`LbbU)Uqojw*mUD4j{5ptjVfw%iiu6GCX_0bc0wRkr_sgxx-y)c;^qT=Q{1?6pU zwt%&{+`&eiG$YWBbxqBOn5!`@MAC+M8k(O9@NwflZq-h-_vXvz+`&#&s(Xw+xzO$8H$-Sxk zuF##&L75f7{|BP4CFN;Sj4s6IXUx*&WP)2riC4Cp%*rRzbZX(hpE_ zIV0Fva>+KpDmZW0j23!O@Mq$yeF=_O8a&vU7Q6qyG+aupA0`qJ6JAPZj?`# zzHa{jegktaO;Oh&$mkN63fY#EbaSaz`V&e)K{OqhrfLGhksi6d#NbMbJlDr#@|zy|XjE!0#DrdZDs+mho2I2~rKx5xL5h4R(e+5>fltNER`Wq zb?r^EfwHTG6iwfE%p0oD;Am0EiIaOTV*3`DefOLD0PB19Y5b3Vn2nxr&C$e<6YBXf zLHNf)0RScqT(VoL0%$d7!&Nf9gA6_7Bw4dQ+md?3OD&_8U<8ON2v>!LdPg3n5B<|S z`u?W91@Ldr4@@ese$v`elJ9`-oYXAHum`ESw%lX2x|HCp${va&X0WP)EpA@FPfEvT z@MN^tp%3Egk9MqN=acGwPUV)lQP~%=?qfU${C1T&KGNOZcwU*ZFdO3kUiO=Etd-sJ zw?}^3mDhEAIIcf+-*a<`QVP;ybYve?P8B-FO3M6ol(|%a$F1^-eEeJhU!A;;x=%}NWK)bu~sOc@;$d!b>r;7=d-lkj#1FHPbD_vNV2^% zTkCrFn%{*>OWm-2?5cFkVt!`d4>=3Bzp`?i61Pk~Ht?6^NhT`G*a>M@91KMhwDhed zY9Kbr$UIAcuWUSll)eYLewLj4B-dEd`TZTOIe*C8M?++4bIsQ4Ty@DhYa5L%T3gk6 z!BMj9(HNyj?jv#DS(nh8cTq8cpUx!Py>oOw?XnK1bn|U7Y(wNA)f8Tn;wo;anhqh? zM)_QAs+x3{2J0a!nIC$<=*zj=Gl!hR+o@kuv)slFe-$3E!rXXnVx{_8hC zCQx1usYs=rtJ1PS@?)tR51Jdgl=D-PJbb4ist&phwCarlfpMq#@F;l3)5i^{%Gi;LYfhk*Qrz5or zL3(<3ZlPr6+%wgg2`zt50A?Za&-vJi^ZV>^{9`Oqx#~O@xo9X2xG3RJiYZa@Mpy-( zb<4rz7E4f8nxO`qS8$nIcQ$-Hx978#^nG&hbBgncgZKyCSOhJwGEPI~;tV!D$;}^i zPSV}ZT4}poESP~($`ufMU7b@-C&}TgdAfA@Q%bQ6xBNMM`RKLZ^XBF0n1KHGbZq$T zx4%7MYRUkud6`58>*afOtvOJ=QoHZ5g7J}~*0UtwW+jbBdj}y5)LQ(w*L!cO`4ygc z>l~%eL>m^ok@>LfonGKAm1Z~>RvbC-pP0}(p$s>d;?iW^O$v&w_cG_tdzEyP5$7Q_ zmbR6#NXmjt;EX`xGC#03nE(Z*;@=C5z0akuBpv*dhUm&!zkgcWiAlt_u{ll~N#ZUp zv2jM`Xqup9d|Ea*r%)rq1i|Mk?K~OZ7*-0=>ls4ySp)KO=^!(qrVs4ye;)pU zi@c%;PMWC0<5pYKaD~@0*tL{!CCd@-WpEela%A}g6#3h*L zWSP;F3A-edAGv?#`tofx9t@F|ho{`VHypZ8iJkPX0WhlI^`MKTX(u;Zv4Sxcnd{k< z*UU=EXl+wvh~ea(zjM3C8$1?Y4!u~q9TUTL4j1Q;8@YG@Ht2FooSN~fM1yr#nMz=z z?HaEkTrL&4bbxC6G<|{dy#9kX{c*3RAL$&ipU*Ve%X%owUA#HDiqxyhpd2;!5`c7D zA0`-E>B%K#%Db4ImoEF>N=w7u+2$U4c6sL7>Uh5@^cM0xf+qrrN%N-!g{B_7G|!Bx+NkH|GvF}S zB#3`=*)wkOIRw!wtKc+b5+VUmMV|sukl-8WDi+8x?;bveTTNJ$WoT;5%~umcnCam9 zUW3|`72aQrv01(+n;zVx7!(-MQl;^(s8mie(#iv85GkzDIqC5U*bGpOZzQJxK21r} zYdk%r`Gx-HrR>&YBTG_fjxXj7cbZUaE}Cd+)Pt^tw(mXkEO z(hfMWqxIEG!75cM z0mlrUBU5e!9xx!#bqMeoAJ4n*Pon{1(*Aqt27ylD|FHQUE7jZ_RwdUM(kfAM7Gkps zk}CbRIxs!-6Q-C6CaMeI_p`_KyG%Bb-U!kd@;urDom;x;6+_klzFX_wSgpHNMLM(_ z?V?!Fl9Puefg2iPvP`NbqFAhpJ%G$O?O~c~p7AT^XZyLGke>E; z{b5~p<^sDeE!$njSq9fU@T##AJ%!EbcuSK89g)d-(O4<|g+hbG7HgW|J>5fvS)K!L zvINOiA9c69#jM<{vdmPF6yRujRz5cfIweUBhH)~zt6*B@?348A2SaHvvDSOAt~ys= zHXb=U1~*FsIACEFgVPFII>cXfGdWe|ND{*$pr%s2D90d*El08)s~`rGC{l$nH;q)5 zvHcF)da;7#of@cYW^6vG9$6(7B%!$CryP<6?hMX44$3G&H%503wTA)mbR`3$r|rw3 zyG8ly$Ng5y84j;cyv4p<+MFt_{$#4F-Im9J55GJ>1>knTCvbt#9bIZjx|l>c6IPgI zFgCeTd{qWePVx>aw23qsI2C+r?hGHwZXfqJKSbld=C5XHd!VumXU5}Zj@C&kUdH)> z3c)GBR6wh`&QWv5<0m}0QCUOV0=wv$Y5Lk82Y^+4+e(mL9fI`oJnjClo+}5qpL;eQ z(zo~^w+=|nM2W$Yv`pW2-m#5c$RTz@C94QxY^4haoMGnLVRmA#T&3o7rtkQ1K#45) zarLzN>wf*`>e^gTX+t_jpG(WBl_zho=CC5hsQ^3DYAzjA*EPTwcmO4Xf{iP9mn5mM zC@?DE0XhG<;mv*hK*L4;t!*V`@LbSb@OAB*aa?oG%>*rTykK)c%jL?-1$b?T_ic-i zQWs)v$rc&9_nGH6%V);K#zt}glQ9X15lmJ&Edyk#OovP|eN6*K%MCS`z|yL@M0_Q5 zAAmDwpMUds|FU@)DDCL7N8ey3E3jg##(CItFOSxfH!-H@8U1TMAf@ zBFFT0-ZhU>>G0ha#qwEHoFipw7+2b`VcOZ&o- zfo7#awC@BdZmeSp(hfa%8!WX>CflG~QnkpuoDR%5r*|*Y6SfwgvKf4BDpp}64My5G z>CdjUYA%KrZOYZbg4YsFkq#_LW>1F_zm zy?JOqaeTR5J^gm}#;$D7-;f@-*JR=Rfo{I7hB{nd?J?o>qlW1PPL&%D71^pL;g^ya z)L+COWXKhOSWd;j_iI?}>)Sfs!o&8wxA@Bo+#9T1L4XFnq#7Kg^zY?Fpn=c@7=fh7 zP|HriYev|$BHcAr4n2ug48hKFr?Ir$d?A3LNyf|O*uAubcZF*M%mGgVj5QXzp^Y|j zIR&kkbx61dw3h-Lg0DV0WiGt&voAq#9!c=hh7RMP3GnBBl3eE)Wlcb|2f(k6e3?gMr5WGjR z2E~-R=w+93*9Omgll!zY{k{1P$gI6yY6Oo|@K4fE3Fx(Fm~*UO+oZukJn)o(Q!_&s z%cQg9Zgow-UhkaaAi+0Zg!EF0B$8zeXRnp(t*Y#yq8VOG1N;JTlu^*Z zG@T;%DW^N~ug80wk9gdZP7;%;x$@>)NBZqUsq+djOfQu&cTLaqziEs?_kUA;SHH2Y+affrFT4YL7nHlPL=)n5cl>ef2syyCXJmU2x;+-vd z9q2Toz02eC1cxO>WV*wHnIT)qz$=5%MpsB?!K9(np0*@Yn!#3p0dod=?;&%Z#^-M% zsJ{GqSit@8k6v5$ZgNPuS$Y~hWNbvMc zNzfjK1y*K86y9Xu-`nwAXBqFgT^)aTysLOBob%^*!!CbOkuw2@HXEf2uB1@_y|{#9 zhh>t~OZWwLfIJ>VK#)MOPRjGy5Mh_5Ge@TsVspoR^~HZZPVJB#dA@QDH@5}L-y17p zFJ+g}G0Z@$GQ>0*sgNcaaSDYXZY4^$fuIOR8z9{TC4FAw|O*yoBJ>H2Z7 z$uP|PY>N&*+RsSP>hWPUmY(HrdN`O2cAvpzEYZ*=zZFwS^^`4eBIGAPqbMsdp`0Dq zrl)|t?nZ#s9HtoB;$FDnrO&-df4!)BV4W2Ff-2dhN-a6!qOdF{!&yL#lc0ScCBLAS z?qaPp49}i?_V45SHMjf-&u*KlA-68G$aIA2hJpZ;P{iuLo4Yddf#x*CIn<+VFWIdN2SzX`F|5rEX$3=iQY2AjYDjY~>b376Mc zIHt}(3xEuu4QUo^&#u5NUp(M}W1O+9BWPdl%|LHBGEld-rspcT#dI zP_l#t_ql6{*%d5b{*aAvi_hs2&Y^4`ci=NNppFb6i3 z8Ln`+miV&2O8LC3g~fF)IZZNUu(He`G$26MXzJi|No*6>EnUHi>rU-=mN~Jco>i}kc|Yw6>t$T1j1OF5L_rb7!Sm~@OqU!mvQaegjEPX1mQPfyl+KDIExga@<4 z5rkUf2M?8$i|v9W69ahK{fc}=|Xw&zX$8tGB=N1`q(7~!< zHLw7-jg^lj!&ieRWiH;-mOUJj<}8(o-9U3a+*#D2g8(`+msrXE*^rEIY2?a9&yz3<=Lc?%m zz;Xkl0IR6rg{4bj>_86W;q72orb|BG_HhuY*Pq^~=3#knKi)l+-RgI~YrNT=z5o2% z>P|wM+ez+v@_TktReZPQ&F`yr&Ca|%$i*D`ym)KPn3SiWU~B-!VyH>(?Vyt?vv8dv zHC85O*E95z+#vlD*%D{$r*AP&duM-6%15`G^YZ?BYOCib-kk@}Ch~b^bYZ)0zwypo zoZI;^eI)E4&-1YUhW|c4yo{A%c{mi{I=RCAJ8Gp_%fO%!fQ@1x>j+3$(v5?ZAUU{= zs#3Gc04jwBY{Ap{<3pYM{2Eu~sSGpQ?*1B=_=#wS#Y;Z0s@P>}qN+duQAKYw*V225 zDMRC10X*$F_}Gff0IDjR>XRm?WQ5tY8GfS8dPVo!AKM*2-a{gAaJ$uU02$=)uCHI7 zXlP>o8kap^`rY@k`t9W+c6+$d8O2}Uic79H5SaWY#(TVcoKPndi&2v0SHP$l)DnO{ zkS9Yb4Rg1oO-(jF1B%(MLj{KmE_)MZau|kNuenvWm;1JUc;Jb#0p}FW3;yXE=h`74%Q7xtFZRinBjgr0AN}I zPRo*y6oa;{lXe-$pIxJtTP_CpeR7w^NByr~PS^C-M&RT<>AUj!9#r%96&>qXF7|l0 zdV&gXKD(EfZ7OcDq1hOnYcLgP9nf5|nZ4E-KwY_}RhQJw+9PLQgHvRDl~N2Zmm|$I zGu^RlBzEG&j>CA4miv5L{@CCvufwlDvq{52++ONBNxo-`a;Z{ z?NaX;n>HQhCMwrOc{J%lJwlaQcgbXDIk{qLAv<@KFxs43ucERvW z3aY1B!6k!YJ6KZfY%-;5J>1q5*Mr1DGD#m06DnL+t4;DS^<4PiY$tBL25ob54W_K6 zDs6fRycKTKfm`+5urf&-=*@J+3|UDObq9v!UYwT-*^l^uDKU8S1>0df!#x(u{r;`% z+__trM<<^Xk;Cwxe_&&MQ=p^ZG?JAbbhZUR#HAP#^lq3T>z#9nRf?)midjXHcs(n^ zH)m@8t}EHJ9-S@wP2b+bgP+^Ze@5XLcC>JH{JBjw@7Jo>u87(|7e~TlX8`x%Ry^UF>WB{S7 z3`}M-L(A}1b?|O7BnEOTlUad;_xba6C_tWy(!A`}t?%P0XHA49hvn4CeP;ECM5CoA z!*hO;)ngWvDfXlyFVWfQU7oC zfSoo{`+qGZS@P00{Y0)!{|z%yfaN`uNMl+_|+6QTLH{-C*8{bhlA8H=tr4*-B)t2rBxZS(5V&Y z*u~&ohO-eqsC6dCRol6Yh1F&ytzM?iZt43K2gVmc_B5p&WEMT1D|s=5#Y$aujKvzT z#SHeB0(h=9zJl78K$7V&-KX`W#Ns_ec%m8?cM^-&_y?y1W~B;K$j*}r5R{VlA?ZDrd4kMnV_MBbve5Pg7Ys>Mm{aYT0IDQZ9XT#Y6pS@(FMSg37Qr zsRs$p=pKPtI)kF;*a2w(Y^5h)&Kejz3av16nO*;0aV=9*&D1S2M;x4N0m#ZVSm&)T zN$SH|Pcdq#TFkcf?k)UtZJFStRg=Dk60&5-;wUlXvm^Z1enY+t=3|e=P6micgPFSnTAAIb-p5%J%U3bEp$G>jD>wOo3F17P1?P}q zQJ{^b%3Me5pp-^##)=6Y!eR|P58&CfqROM+owP(6Uhn>*Dl-Rm0;H=AlWjaL2qK+!r+@rU}88X&_0D#y_ z;S1cp>LpA|-`^}0em#eq_rVF0r3L@pqY*x98WgVt1^dC(rsAZ0$2nt~YqeA*8JJy) z08)fT`8L-FXF`x}oqm|2FCOOs%#0`1C1oA8Zr#f`0S1m?3Egu^%KMX{FUJLKO?CTBRCky|c>Jo0hro~%k23KrUxyA0`$|0hfy^IejL6Y`t zZK2c>LbhHefR(PY1J>@n62T!(pXfPR{8(|_zqjkBu_`WK`C)KOK#f5MOV&$g5GQd? z1OLuary`34!M3es$Z<}oSHKnp%;_C=W*TaW8(eRW5$;2;(;+kKq(9^t$sXLgFGu+><*dNusxnPMR7g_LDFW8FWC4?-?}56R8~)nvwtpP*6UHOfaB{ovXRnWa zsrIi=MlS5%XvcbUMCm{|3t)&;E^J9Mgu|%Uk{Zctq3i`c40F zaB@n=q{4G;dCPS?nK~qxaa*}Op7g4HdvE*O>RB4L4sR1+<1wS-Z3NNix`sWpqe&uFeHpgu_;~8QwAAjfbfY8W_ z;^h;zdiceA(lX9dh7o~8p1d(pbo=Gkm(BjY>%*T;5C7KIE5m5{UZwpjV5basOq5&;Wq`$Ntc4|`5uP%(cx80;`)29&&i8$@bj=W5BuNiHp8~Cc$DFC) z9%D@S+mCeiJ2@XbSa~R;PT)6bYbf~*X&96nJE#-&n6%cR2M-JbhCmEBro2>zIB7*} zn-)Gw+A(11%tg^pxsTDY&3cT(W`AT8q59j~?;()`%e1zEi*z8@kfbGz(K<08hm{dO z9v|l>k=9^)%D{w?no1!`L%~n%;pH|=ip@K_bK);=Z3mhU&FErt3aS-R@~BO9Y+RL* z4k1AMn~yRQ&Y4tv%)Q8rSYlrjOvl{blph=3FaK>+_?G|k-fW8#e|PJZyVhE);B~rb zU8zRbv&qmcqn-D*O7SU!1-3M_XCx|GresMUjuJipEzKIK(v@ksb6Dx!v5Yd6a%>wv z0Xg>L%T8(P@WXke>eaEsnBKSNm+g#~pY`fvuYKaH-}A%#`231*e$FZH{#gI{1_t&n z`l?u-;I3N>CrGxH1X!y=2N4HBb*y7oE~cieW>Z;aHx>i7ZnZAnsxBS)!T={g*uT)Y zq+ID{`#1Q*$6fPSiYMoZeoT%(0n$%J{ms#T7*ZcAPhR7&9|of*^7mfJ)A>JlYcY~e zkx6^Up&IEemVEA6r4+bRYPB@$QftL{e61$$WeAMQ02Hn>MLnHg^PU)uq~=6UPQ+lE zARX@m#$f)5?rC*Q&>vp@x!!(Xz3)j5X6)w8r3`b|Wa3vayzK%l|H0Hwk~OXYzL(p#hFl5>_!|C)oX)n+LY zs!6A4WAoPbmc6uuz3NW?Ufkn~x1Hh?k1y@FtG;*WXMQ2X2iYChG=9?WjzT44V|6f1 z4}ZBS5;Je1MjC~)ei;Z-IAn%RS+Fm^{`sf4$DtN?c>ULQ_bz=Mobs)OO(xt_0qsC! zo>i>GO0hNBtLnW0q-7s~;mEjbi;a*qZG3b#1iYGQ-}95g_6h0wNa*ZyfbUcSKji2p z!bHb7&`H|YX(0Ln&3!l~^J%wuIO$u5K0u|V+KW(LO z%0!<$%vP~hOaD?GW-4U=PT=;l!0NOZ{6jJ|iodRQ$AsVy2VXx6+pb1V-#%Pzjwba= z+;q}>z9vq51nr%M8?Iu&i?He$F#Kqs^=e@Cvq98V-Imq%xYAcGr8gm3PGjE%Uf*7e zCr=`*lE-lek=vC6d>hlax1UZ3-B}Bb88&wsLj6GKbkw)~7RDn0PZI&tkgutw=yZ45 zQh>Wr#iZVivXq@9Dz;il<`0rB-Ezs*%;hmGebMs|xuiKb|JtnbnCkms?=-q}c>n&O z?7rWZdO$xP4Ktk7{eD31-j53$RQ%=$yQz}M6jNISp$o?3QUQ9vAOdAJ%7OSxA+Sm= zB*`REx^-nuZ&9Z_)1LHW0(MH`+PQUp=U{(2&i*C~x87K3)KAf8lUw2DbOb2ysWDJM z)kOteMY(loRVz}`@Ln6~QL2rRI_v3f%_llNp-xxa(CfOPmH+vJv4ZxnIuCM%v+hTMuHu>UgHZ}$jmlEwT@3BaS5o>CdTKTH>^H`i!4yIrF zE!#QP@>%OYhXt;LI9@g8RBk`W2=~fy^YEPoT+7y!M>@{LGl_Pe7rRd>iUxI%jG=hN) z()hE2u|bo`&o>QuYsnx>KMONF$a8G=TRA+YbVGaO!>6Y$B-YP&e?Ud|AG|!Rqz>fn zSd@Isce=Kd7fI=M`8q+FC;^eGRW~AOq|_mkGMHwVFVVWRi~!X2woytPY)+{G9A{WV zZ#&Q2{lDI*{@v@wh0=uPk>ST~CGq3Y>U|*UC3E!t;q=3IF8$Am+Q&V@a^bMQ4UxxG zW)eJ~neY8Oqe1O-?e*LkA9_&rPCy;0jXRAIDT< zd8;R5aS0#~mXBQq(U0tUFd6QcOl}gXbL2wC8da&O6}V;@*X2@4n(ACF&IA8BsO3fO zl2lzTO_!umFAQ@rGEz%L_IUURwC_C7sPta0GOe#_*lQ;P0$7}&HmXu^#pmgJ{o@LT zmdGcC2j}mv;2OUjW7CIEUejxiZw$?nCVs{K{kLB+C(jd4a_cSirbw3_)3Hb0dsUm# zHWGHS=Ui%t()zMALm{iug0xrj*+NK4tz7(cbnU}i_o0nA{{35TMJ}-@n|tatH=SF~ zKv1!o-WpqF8xSdAlne`L#XHB+&ZtQpZ~}X(_PTU^Y{3_8oMoL$SKgbT96^r)5Rmi` z=z!xI$6rFwkbnpikh@G{trlt)@kfcyG~jfSX8Uvf;-{40=F%(UgycR#q*hCxAniNY zDpp2~t8>O>DOi>SF-W-8SaFed;P^_gVfsS+9rgaO8lQjt)?49JmJ!H7Cl{q3uaEFl zQJNiFZ<8|C`BqCdErm$A*G8OMcU&aJ&e3!(&IhGeG9@ocY17gx45`$tm5xxs(1L+$ z3dt#UE!mbO!J$jHyqMssZI+0IrJ%WquOm3&Ik}be&WopCy8f`eN%$ZocS4HTNqsH!56_ira z*f$9AgsYV?mxY|trT}5&j4zm=i_Di*&`RIhbg*cut<3fE(35-NTNm>=C4%OtN*;ZY z3FI78(MC5QJC^xLVB%T%VW6=LzF{lF%*^l^q;xtsd$>QIUEZE*pRUc`zz0OvsKiWB zLqoC*Cj=@44m4fX)tc>zjOi*$3`W|NYWO;3tqm47M9*BRJ^lE8pjPLxy0-oOK-o(B zzCKQ#I2L7|gc$B65F}0HWo*5# zKH5aotIaWc1vWs@25)E5tv@FP`&8BYA#i#uBtD$${UO41^7{Tj5QmwbXY7_AAIo9B z>3LfIndqD}-}vpv@9&7qM}=eeeTBceSzp5htEZPg?>6O+iQ@Jfw8A3TB-?EDmP&A~ z`v}aJjET_RB%|9~5UdcHTkJ7PbKxr8#btC3<<_#=k1BoZbVXAvrN_r5%j$G*9*dg)#Yz$P|lj;^LC z{d_&PTx-gSdQZThN(ckVWNbhJ;P4FkEv=v}RD*Mv?x(VC>u3J`hLNqdJH;RE3kz3x7cq(>&sph+FX@4acPZ$s$+bv!b< zdmZU>T86X#`EL7Cw{5ekR~g0Tw^87>x?UT+oPVU%Nq>F>d)RJ=H zB25@n44In+v5(ym`_gCV`(N%bvt{mR4qq5`&BcHJyL_bgxOQRlSSd9{;6TzM7R;f- z8=I71NK%BEX|7gyX;{$`68MI+)YLw~u0yEPOfQf2TI_ZWegEa|`@_Zb$)UH^!T^=K zA_vV2K_W6KnZQ+v=0;EoS4!%}<9LI#*zoXLUW`xUXrRA&rcnE|^lfH^k7b9)_y^bp zX)lq?ViH`i>qT|Y3U~rtIC>AOWAF_n;TTrl+sx9AD@FKe3NTjF8I}8-RB4F&xW7f3 zU*%@?=i|%6ZuR`|V|?YdJb5fC5L=}K=U#w6hIW=9y0vJl;m*6uur1Z88exE|i$?n9 zSz+%{7d6x8^`!6bHON=~cwVnwZ#H+|H(X$nH+-cN<|Ezt%hq&D2Q(-FA}}jB9j4wK z$uJ9LbaJ3rIT$B>K$&|DRH~8MV*oV`r6vO=+GYkSz4;3Z8T7m{>kZJH z_}RivCX0bonkwxCL)F@fy1Gzzz?Fn7)VcPil&Zx!X$m^yRE~)M{Fw6c!W*^3 z)|52MRT>V_ux=K|0|wN9nKU49lJ43S0n}JkX|AbkQ&K=kVt{&r-sZXa%JT5{-(Gk-9-*QZ z?`qX!bFRasfwn3ogUwayl26x@(GA`%Cy8w67G;=M(*W7oC9<7j@?K4QiN|9yq44;! zy=Ep>PpiM~*MF{Pd|GMo-oUz-7>a~%dsGfa5}HJCj=@XYHc1KrZnM%YxF*}G0^WZC!g z6|>Ik+4REw@---#T`Dci z-~zabAdgtVf65k9`tt4TujjEO?0)VS$#sndg09-6@&M4LP;Fq%Kd^Lm)#53|B6j+k zed-!a2-eU)SIqhjJ3RWQcZ5z(KXmO;hL}hadT%2{LE4JR_|bjvED+Ehi9vGomepRo z(Qsh=tssZce%h~J4*tXL^q2m%v-O#FU!NdMIzWbiNk9%uaBTuowxK|pk%WTx7J%N= z)xbA186$<{d0zajk#5pxPWO?hYb<1TcicS_x%ioRiCVLnKFwh9>c_Y$L~0IU0&!1V;y6jC=*{CtQNvw`g3ZeVJ3k8=C2R<0*oB*BE zFyH`_bcYPmCr{;Fa$DHr^tA6;k!^dZ$FsmQqMG-Y)WA4HLvqYfs2r7i%M{aARjWmF ziU;S;nkPNd`ExBuFtl>c&22QNt$WyPjP+Q&eSTDwch8h_W)Mz04H7~r30t&?0 zYo?1Z-Nz2LH=;;HGU0iHGe{dPb1R1piGz=HVZ2c0Q$qgwy_c%|1+RY98@Qma{9<~x z;NWW&G^b?@@AZ`ZDW88Iw+}BTDz7s4H>WKae`>pVey*)I{CZMdeCZLd+S}eUvAu6`%N^|Q?kKzab^!JYW*Q^q3VOlo zqM5J`Y0@);quM<0)qS8IBGP3lzLab{0qb+vX{#PCd z7!L8#S)Y*Wr{tfU!jR4IKbo;oH3eOCoB(*wCA40me;Z;TJ#XairN86;1v z3ZTYnux64A!E~Ps7XLx&8tipQ?zTIVjE;ebXcEPi)kl0W-Fh45M_|Z3zWnrf`=9fb zmiAY-z-z(#peBv=w%S7}gRp5l;HCKr29J#rra0i>+9DRE(4Ya+qQQ*yLXT5JITjLK z80W)6$JkxLNp|`&A|gPyX%5BGYmF7pGejfc<4G_BBiSn1fy)AbI8Yn)IZv{kwsnPb znlBc|-g^f&V1x)H=m7&2Of5H>n@^UXfEuWza&A|i|Z9Dmsq)2^+4Dy zg}x4!mj_9F)^YdOr^oNq;{LomV}Cie_K0V(w;uore~ItV11G z?AXCZ7_C5O)6m)u=^Ff&=QrT+VT>*c-apLt;(k_7I``b!dvhXUPUDcQ1Ky(naF&W3 z5NA!Jv6DFA8;+nEsT(;3G@)T|Re7uD{RXo=FIPQLtbTkNpY9*@^S9T2?D?%+c#G#C z@Z5KO@1>5HHIvJ|+2$g<|Em{XK_L7Nu)@N!`B^}hOab*DajFrv^2G?S^|UMo^~0mOjCaR>Ca_GgY^?Otj#hg%!}YA*MkJTbAw2#1Aoss5 zZjtx*F*CNsqR4c}?MMZ&Sxm17JV_Xf6Y#g-r_nSff^jpUyFwpY@OQ-M^#iea`5${(G zl>v}zd$4_U@0x<^Z+c)p{w4^y|9XDtpBd!-)pXqg<&B=H@#RMYo$c6ZQG5y@y5tHW zH%BX4AVc6h()Ok^1%%Qm8_i*F2kFnO#JEDKVTXD+q!=La&bAH?k-E@QTpNOH z0lO2IN+foIsKsbXDQ>n@>_k=p zLeM@Z;6K2@RJC5G74V-rOR5pjB1@}Bp}Y0edwb8&cIi5DdyM*_$K+mskmN!FWs=-(}p<22O`J0f;8J`<)(>lHyC8RLmQ#G)2x_gZIA6)R&z<0qRV zSfO60_1P;cyt!K29PCw_yg=yK=AnQ!Xa{?cvQ7?8MsPU#f1u;}N5=c()7KxkR$Da8 zS-^5aRoc!>8=9BUdg-P0>W3CrgH0-I@xin>lLe6*>-j3x3#T)|UZCsy=lVtq>guRn zWV5L)AvRZ)!N_h8r_q4U*#5NYjrk+OIBJNmnREIU2U;V(8-CEFnf0xDDomuJ;>*J5}f*kQwnKC14(ZXvHg3wlD~!cGeKtZ|Fh9dB2?U zdRyzt8{T?nXEjdC)}Xx)0ZVWR_?=?CV0KI!o@#m}c;6jz{uGetUY!&R_6Jgz=~`{0 zPb_@#TkmYGF6%tv?|n@(inxh@IZQAkmjLWyYDYLM)w-dLt?AyYA`y1!0hHPgtJb8d z?DpEAV~eV@%~$uNU!QK}^QUi53tG1OSX;*4!R~Q0V!4`{Vx|MLi=Zk|EoH51FGvX{ zmM%UYt{n`yi;0+#^|2mLg4q_^RrW{eZ!3 zY+@of=V-{@HX$NU!kXAwwy8BZGc1vvj@ASS9RX--%^t)X&tBaAYGA&Iu|q;m2l>E5 zzoO9SPpHVU;r-~Rel$IYur1xClaS3R!{oOq(g z->8J~e6liuN$=jC_P!b>-2S!F6ZPig_Yr9583TT4cb~D-hkn5nyr{Chk!X9r)&2hd zj{`WQ{>yps)9zKSMEW#|8rPFkl-ugi8FoZK_X0R9<&2}q~E=cIU@iT8AxLzcP-P-Lz8aRjxauzRzF}w0U>c1 z7Mr+(iWfa>%S>*)%H*60d<_4e{L$#W=#!k|dijWQJC9Z%eV_A&6{LrKwP zWV~iX2l5*b%P>a4ml!D5n+9LBIc~?Xhhcl9kBh6Xu={cBaEkmET|m++$kZ0JP6NmR zrN2cpd(fka6l5y}rLh1NL}Lz{N=0tx?WIdjbv<(O@82$dd6fuxX%0D7z&xM0ALr?0 zKfg}g$rPT*IRB8Dd9!2Q#L0^jz2ePxm6>+AC*#B6zsLVkUN+ zg0ay*c?{L!JoeC#+02Hz_YZNG^+Kb6c))Nn+gG2UQZW&sx6!ccVzYLdW&pCLx{{DQ zG&(=D@~9B8r*IZQcBq!v5oyN;)O=n!TLXN0`1D1e&YQ{hKO8i)L+ul|!y*&NOcbfI z7_78Ii*1cIfyD<2-a5t=Z)+%%2?Nn+!$AAx=|8<2QqDej?a^i*GIi~YWVJ|pSlf-Y z?YV&!sB49?aGi=PJ_0yL>17lOY1&(`w4zuorVZ~pqS;7t41x@3T#W_$7t zMBQ5D<#?ulI&zAaVLH3$ov>|u(xuWGtYn8-vbk66{J&M5iSd11+~L^W`(7I@egvt# zCQzeldGUtWRRZ>F4Az)BrUNs`ElQX?Cv@Jk+`m&-z^#+{lQPWt>n8wj1pbAa((#D% z_~tQGe*5X}xBVJzI>;0qll6<# z&9MROpge8Ovt8%GnC-ql3%r(3>sr$|27< zf z0b`8mvx~J=w=Mup0FW=mNQk*zNa0``|yFv){)IyS?BA2$EvfU z0r)RE4M-E3GC0hfbbBhH1=&y#It4nsYX{l(3(hN^cY69!L@q*#TR<_&ogj5z^^f_rHXnALkVmH}Kwy&B+?i zs}UaaGJ#_U2ohg35jz||w5**D!N+QXNFL5ZoPWvC&SNQcaOi_F62h`M+P$+r1^BX z#2duo7CRE$X$CJ7;AIx1AT~$l*=a-#LIC@R`}8yfMEf?yTo z0KhXSeKsf04a$tZIlPvNoB;GJCQ}RtUn0aR7H_w@bzOc`c3)if?|$`|%A{SMiEWk% zKx`f6x+OYxcuhS3RD%>)tVFy7hRlkQSXAN(TLA@ z$X#uX{!JszB59Pp%RE6{S)0p^#>*nl`=2Ruh@W7#HP;j(QeLrcHg*b`S|(5+(VtHn zy?RUIg^Ppz_P8w6io0hE0X{bcLB$t3@L*_cs6fDEv)4onu|a`b9c`e3L%dbfCR?Gj zTbE^Bmg*GB910Ylk2sFD86UjUe$aWe={rBVaJwrSde9SNgr;FRO-qj~|8sxbKG*8~P)Rwj_uiCB;SaDb0Y86er(CQIfB5y> z4x;=L_ukAQH&3|)+jy=$vR69Cl~mZ)3OqW8#b&Q!vBuR=^a6N=+NTwysrU7Z6t3?t z-!AUu^KN8A^``xA(*gzp%H&)PNHki&QWGV3b1Dt^qX)q0R-$W~2EwZgwKQwPWaoz8 zX#ITlQFvnsvW`HHAN=<9iI4ldKHpxZMRyUJ)4^zwi}sh$1IJL)EBn&JtUzj+Wx2;X za_g&|SO8^cvT6qcBcP1Tn4}`Arw#S<(|CRw@ZNL|=jV>nwa($APVWD_y?uDSMmgs7 z=PnFR(`lFd=|%5lHR03Wy!t%H;q>53PUmsDSn^~8E=88;WSg)lL*Z!? z_9a$km%g4i|0x>2leMk5eEOGO%r;>ni9_ z&A@*(s1(;zR-mio=|B5(;t&^xd{XYcuov39l(|KNp{6?OWTK!PAOLMwY7aLs+c6`k z{{RD=X@KqUV+sajMvdLnE%Bt23Xr!N7W?0z^5oF?!K>-mcoxxP#pB`?A?&qgTS_46 znT`pdBsFR>y(utaatMy*;s{;^(66mx(I++;g^v(Hv|t<&;ae3D6?31wM>=l}n=61< z%nnxN*t$@i0Q#~(Vq@(+yTM;rYqtw-z5kWNK=1^Y08NZmN*EDveRSIg$hAQ#!&{MV zt9X+g+lsN-z)rHs*oYvMANDELhh@{n8<)S8;+TtKz|YY0?D*U7Umjzye1YYFa|V5O z9CAzVK81ta8Zx=>Mk{Lx!Qc>#rRIY+u=OSs%C)Ud(V6MaX>ry9vp;_m7#H6^hO2Mw z>7+-A0%7mW(_4CQo}PD?uBpYaToU`jbQ39FSDQp*l@53818Q@2KaxqohKL2lCSN8j z?*WF_QP5lIy={hbstft3{oBLs)AyVCfa&=2?%@)QKL6VmhxNPH;hNITtRv1Cni!WI zAa=LInaUtfPmeZzNr-(LgM0ypk` z2UaSD9)LV3hpibIT`lmVrLm}wph19JV4WpkXJzU#4Rwun1?(W&67u<(+>D_DBrop3 z_OD_Jz%QWG2p4%op6vp-3=lwLL(VQbBNPLliAG)3niyH=ZzIV;v?;D%?~d-~l`US<9qD3o>jTq=2kJ#a_;A^p}IGmB+pwRw|;d zVN+q<(hoWD623OktpG%Rdahn;>a4D_lW({_qsB_h&QO6G00+87I6{Cz!c_Je1g$e{ z)onehJoMi_(adp0xPQZ%{Nds5&zqwI&;PuBeO%P1=jlf@<1A6^>Jhu+K>4||OT>2iwEhZ(X3Yb3_~025y_txB0QI1$;_DE5XwOJYWz8U6v%B6%!H%N;XhY^$ z4nAjOp*+u{%?KqI?SO?}HS)RuA(X%_9} zV!J2S9%e=?bC6qrI&DA>8l6i;j3TFmDt)Uu?U){xIC^iKgAlJg>&u9jTxPB}{5#uP zBuGUZO}3aG8y#|kwHiu`K&|R5qP^Dj!;3RTkdK`Vp8V4FhaJEWOYC4zom~^L&Yo#u z0~XqRd=GtQMm{_BjCm60@u1)?k2X22GFF-wWk zQcAR#0zQu!nGYki$mBlKv_Qtd>mOTfx&Q2x{{-gza1dU!}E`nf0gI)F*_8j`$u%UW3_<2*;agOL-*Z+UM-2U%-lkuSUOH8pl z^~4%h^_US_2^5!oCaUuamh8P@{Wm+b3qI64ZzG3RYyqD!cPg>2pO);h7*1}@&&#HF zS5)?a;`|}o*o8N-7$>!>rj1&#NPzlzf@Hi=*1=e75Cw(hkK%Z~wk}b6_weQX>i>0- z@L1t=gjKzI7cZkB3oS*NW+Ng(MN&o{s8}1JBnOgc{1`4% z<)4-H@0OH2#&QH_KU`;id&ae|*Q@rq!hHS4^?z^Yd^wIhaKL{Z>F=9w9jV~TrdzGe z;M^X)K%WaPF1Fb&f9LA2rLf2WtTWe6#5^hsiwEFzL&`PjVD@B%XDW5toWMXGk0T;w z{IiDjJJ}%*pKrhZ`me*;{w}H+$C>#qUp{|YUe@m4z=`-XT5=e4X>9g(R1~JlQlEs1 z8H~fV7AwpCk>p0TX-ug)rOgrSI=fg~a+^DrC!mi&0XIO+9QK8&`AzJn$1iuwa@O5B z*TMga!XSf!Ax6xc)B03%XkO`{8DmTaxjZCad|+S>r#X}z$1y>U_k)9=4cl>@gv z7Qq}PbH?qjdPyyXO3Db|M?rw|bi6nBncyb^ZnB!%?U2CyWh^eFzpqZ#tl-Y;}tY7rg?jXxYZMgh6UCd zWsV#{ZzGb6@2yP0P%qv%Ka+Pf8sA!=x4(Xs&o{r_-Ti*^@|0&|+27hZrxe)d(26FD zz)a27!COmY&w%LcEq#tbD;m#`l;Z=L(w<$xdfFfi?@;jDFXKkWXMOPJ8`A%>d(Ve= zMpBPLV_}!AP}PJ5Ffy<*T_)*4>fw7bAU=Z>1irTeR!BU)a6G|4n3&NHt0AsmK)x2h{!{$FJrn!C_c*Qb9Yap3 zl}xSG8eDBfV&NUFP*!9h{Da2!JZ70C0%6!Oa;cPjyzWdqYZRTE3-0c#lC7&s@fz9A$f>#DhZU6$;=Na=eDvaCIY%u6 zRJghk6HEQ8P~CV4RrEWMWv2>JObwK#dipBmIK0M8N~@s|;S)DX_sdjVehWM0Sd{&DpA zC(qlpc5mlTV+6u~L^yVB)HYkowi*qt^Y&pTz%mNRfcFi)f5f}26n~>X`E?GY4`S_{ z0iB04HS@KnxIveM}m3j2P62A9(m0OX^|)Z+qJW z-3J%J^QYgN(PzXF*e_AO2q`xmTL_M#=tAqs8tKr7hvn*ob19a z);r}WPY*m}=)W&`-R{f+=qI^STB?AtX_z@2uf#2t*GsClQcgBH0ARLlxX9z`qQfPn zUiDUEDxM0|bVbl|@5difz=u4*XdHFA33Tdtmoln4j4LKSxCx%76QO{o8nB8|=m$Pq z*sS$?dRQpWxwOAtF;cF@AEfSwxWrWUy3;AkFGx4tNgixJx} z0U(FngjqLKF3$&AF16O3et7v}_wIfQJvG$=CN3`27(vSeU}8de7V)`sfUa4bbilBj zumh`&Za9~XvIcVf*8c;L`m{#)CzA8_)a<_$r1AQbmE=a%=2)_h0Eo#(4r^M~R0@8= z^8u`t5TzQR74kEr>6F%66?S7A$EJn1g-BN*8@^MOQG1WM`4X_zEv*lHER#o_*-b$p zW6KL0Ij|Knr9pEWD&E!2!@w7}4?N13^Sgh*ll{$Sy?4|Dc$(gF^(~e)f&4-l6T=ueamn`u~gEKO0&sLFD@#pGyan>(5HH&e)tS z8{LMp)tiVMi={lPAk@WPN5_=#lvHAO_(KYs53cB-U3o*`=>GoG8yA&Ib@m)+OdJXwFq@k1WP3f>emW2rvb#(dy4K zkq;VyoR8gKp@P4}X9;Z^E3gv4?J&jnEFlIdu{U7%Fny*)RHF7#LCTw@1E7GOZmF{F zF!CQ#35&|8-R%khW) zlgrp1Cv#foXP2g}yPGLGI+{(IFe6V%%abMTmWBjbFnj^c2`$D@@p|L2sD!%gE0vS+1a>H`wk;De!>E8rXLSQf9<~{!>o?+}l*pwiILv+k<$q3Sv?5m78{* zJG?Q1UP{I;dP0v4z(3m-`$L{>k%M-xLnXD1t@C8WnQpRJi#6&ajZ`XTON3{Grgdsu zhz$I(fGL9df*)?koxDE(JL+p^pS${CmDXA`1j)#%U|T5+U}wRSMh3<|#-^TnjdObB z<=S!$c3^?WM7_NcuM&xVV?W2u(}ONOWA?^fbkahi0dfu?T3)b~V;F#;UTr8Y2%CI8 z(rk9BeJ0>To^y@{nDKxW`!QbsF5PruU-b$?!;3kP$&&EjEqp#M;(J&zXx%6E#wQ% zuCM1eulMv7Zoaaw-*0|LT#(-v;d*b}=s*Km033{=>?cB1kC+5M47Cu`$b?QoS#Oxt z&#sL|dcek33rN(4R@K9|ul=_RMwcAnQW<@J4%~}(-kac@?GBX3Mw@tz?o*H1kqauK zRRv%~R*DXJh+--T(g0zjrj3(<+sFIQ@_pR(wJrLt6)wk4fc9AymNom|KQ#@=t#ML= zgvD*F#wm!z3LeDD0&50Ht^qp$H<(31evq*XaHq$A+j zM*}csMU)w>mu}pMy--DC#+bp5n4PTSG6yIPu}s@cLPB1zAb5WIPw$7>H=n!qg^8WM z_a@z2J5`X33WBSx1|T9izRKpuECm$DgWAu|DmW~0%f=XYa4%aQ#i8Z&tjco#(^o*h zWT+-U^I}WfiwAXN-6K#ck1_*8ycCT>~~BE$fmMVZtEU4GoIjv@W1>rsMhEV2|3H%XqN+DJMG|M{Z4m>kgPr6E3;MH^me4z z9t1KP(7FP|I`;M>*VQ(%t=*VmYa|YAJrC_6hfxDad9I}c5TvTAeXFkh#f|ekd4=6A z#mC|LRzO0!tyGYTZiHM1#XJFH5W|_;LPRtlYHV*ZC$06Ow6IOTo z_Of?gzWoj_U;gs>)cD;?i$F~~;Fxxn{)QA&$$|Dlto$!_W@CHpQ3Zj!$D$_Y>6?^3 z@ovj?%q-r_)?x`Xy>#`j!QR76Dr8 zblGBSm;t;F@aB5#V!F7>iqCcHae2c3#|fAc#uEA z9Owy#fMceX)Fk_wtPO1|f&6***SdW2?mDU}-tw9k4<545NYAt^#)?Qse-|0P`{B%# zk(1_0qpwO|V5<6TNM3xz^N`}1U(bpc>6pHIREP;Fu~?H*Lkdh0I|HFGQXneGWT@zr zT6W2s2qYshwvsg2kq-_pTd#;k8}fKwY*zL!k;UVLFsNqWRjXw|P7U6bn!Rb-%Rvmw zAkKxDY&zVk9CYh5mRa;$H{CBaCym==nT~hw!E4qb#SW|-WMglWm)csQG8ZNu|Ja5d zfgwt*fi7BDeZ|^2d`>K>d}kEL%9*Rns#b5^#Il)b&T3KxjEbQa?DDRqfzA{IT9`7+ z5j<^Pqf3fF+#U63XMATAU+*3e7jOS_lx|z7-RvE^;v5(Wz&OGZ4IzP#nlW_<@sBOO zMXHWrpJT+p_k)eXcq59TQIu|-fc-FY*x2_kgMHLrzu=`U$7-)pMb`knqRH4sbx!M$eFEhKy!VlQg9w1L$H0b}%^|!|A6azv?B5UV5q4u`ov*Ya z6>F$d_tjwMSgfX&fGKmAhCorNvyvh*zr^kxvWwju#*2*fwN-74_LhdYf`|8@06F84 z3zido-mrXZ)>KT!LxbD1X_{q^qm1E-04i?|;NiSZChmT2G-slrHr=L?NxIiboi^(P zv+Z3f&SB>sI(jhE#eln0=-QVieo(UNk+=Lx@Wy3Rz+2Zn?uRMQj&IeqBr~O zfHlZT8ols(NU7!OqblHiN~{h77_|W;P(XaWm*m30s|O!wTlE7R!l&DF17Hqd?+b^9 zCj>Wl2KaKe=3I9}6{!@P8t*nptS!#j9bSBiRpK-w)X?IHl!4khOFER6pIsF1pIX4M zu#0?G?am@+WS13R6gJ9%WzilG6P_yKaR%2)A!hdE&la3PyjwqmIW67`zZ?%6PW;}^ z#k^icl@2#+Z%xiN7U(N=zxItQ$&N||0!rfIn5LJGw3<4PV!$zYorK+cIQVUuCDe*P z#nGh?tkW&f^!(JHj;>Yb@p5;Ejz*m!?+4HDI~#8&-7u#s9$86-1GyNnSp%|XV{p{M z6C2e5r&|-Zw%DKY`S)>qUP!rrQ!z~sObtBpU?Xkqae$8k9t53ZsMygBUWiVuA%e>0 zkpN+l&D2X&($uwThF3-X=lk6E1?vBlujhu4`+Hsh-{qW)=&pW13k%0eL-9xG=WIg}nuK)Bof5Zyqr@kH6hL-P}LuAGiAF z?gm1A(*6BlutULM3xMc*7Z&HYJb*z8kYfY^28*1{O%_@Xuoudrgw1mZKKs2}#Q*0( z5nAn?3c$f#a0mv7ls;3Bb+}F-4&v)T6aEe9k?93%2-wAzJ*Q4p5w@)tt zk?}NL>NGqI*)KGK%*&AdFXrZ<-#Fvajj=aociTCPewH>%VH+&=Wto8YIvsT=Q>JWq zE5#!EWP_xWMjMq4^2nfoZX4wXZzmwycP8LR`dA?8-N$R#8w<)p+u0xvd(aQ-gDF4* zA*lub%0ltG2qcXkm59$^o6>+7&~zQnohaQcOWsdAEZ_Kl9u=_{-{$G%C1;ma57G1W z6L;@X@@at8x{(mmRkiYzo9Nxr{R|KZKZz|^1!qLS*X$H{3CJ*ms%Tz6xI3A*6SVPm z3YQ+L?>VEcO!D>DRVbaO-^p_~Ar~|-NT(x&z@p?7XvSbuLbk4KzL1Go?6Y*Wq}FZy zGT_I3&vHyRkKgX^?;b8Ra9#MF>(56=*yn9KdzPykb#w}^hg5tGb#H>dnhll2L z`^D`sbC(8+j)QK)BcYf$l?fxHOx-EJ@HkQT8H}18LopD6uDB!aB}A}z^+-S)tGbbr zuvj^o>)8BrPaoA0Pp9+T`_x%!6-s?Bk%g6wdlPeiM;`aD-1sL=3>TFr8+b+?Xs{n# zmK@@7CpF$KV7ow0=O=MX0Il*@G8p#eMKr;_NJc@1DEg+4eal)?&t=zjy)TAzElk|qu_C?KV~6!3w_rCA*5i?VNX3g- zqLT?^fuh3MIHRC!573&>__SS^l+!>W8sU52Gd}KY^K#Y8J9*YWvi3wsPr6$to+1Xq z!Ko1|M)5zMSM>D!XZubC37b>rh|mL*?Xy-QCXKG*H^=H}J7 zFDsCzqT#o*7gBwjhH6@A^>6!ZZF0T<=s%mr{w7?F+lqVklavCmSn_+`Go_=nRHCAe z930H@m$i9vW>z%~;)x5tf?eF9(WXatx|19rOCP#`#+k6Cu7x7azyrguwo2I~Y9O4= zg2=ig^i^i;t-s0pggvD~dEWsJ^fa_aeSZTcXSZ0Zk>3|nPCN<;6?zN?*JgYW&+0ah z*;y!k!viRE+kPccUp^PoIGVgDKm#s+D6d1M`gqKpVgNgIrd&`!Vzp4tHdAj`ptJy- z7(qC2&E~hUep4>O`EgvF&)dS;6vZK1q&y8h*QPI0ja; zHUzL5R!k^@!!FJ4$F{a9%Ba|N8^G5rU)67a4U9?K!6EfEjK;&-HLbfjwZlsEKZM6z zGl|+7G@B#S8S!TcN1kvquS{dm>c^i{D=+SvctEeMTO@9TnHm`h9J_k_J>(|W@pL0e zRCJ9*izLuxo4xV+S~K(riLmzrg^qG(y(x}YvRYHxE+pOA!ffdd8!l6JN7nwsF<(ZR z_ylAS+z_ml*@_j@Fc;zKss`SbQ+WaRxAZHawpA-=!;|xSdmVbV;fqkA!QD@VG_jA-YYR0!-`o=dJ-h`r4)C!&2M%=?|xU}FzPJvaP2z;&kyj~?n zrK~ttmytgAz22ckQdzVnF>*#VbSK#@!l}l2Y`P4x}{P;!>6|O z3PDVG98RXqeh8%t)aKfn9R=Zvr5fMwmK6`Aa&KhyXL4I;_c!|!KAsOXEy+X=m-phj zPn<|v_fN^*jW@1D-W@ksMDpN`uSZ1uYmXz@{1>-XZyvUDXF6Kwi{`qfAE@a>$0TAc z0MI}-movQH>JR4=df<#sqZZKWUnHR|C-;qSZ%hbuJ(s+9KGVm$yk5;Dx8KxNbUuxZ z@V#xVB)7lJRCGRrjGvIKx77BWqWoPuU8ShVR3oTPHnh_FtA&TARd9*1_0%e$h%LLK zY4%R7CgUU@+=chw_Z&ZeKyCKEzLXPk7&icD#H4(T^dRjjg3H8`XunK=s-L7fF+B`d zkp2xPExt;q)v!T?Kk@2p0x=%c$5hCIi*SQ`N#h42taC-FQiqk!>v9%iu2=7L3KoSJ z2kkp3KG^H*`I2nNaTTXk7b~7UwFO>GT;V1ieD>WT68r;k`f_BI;oh$LhMY!cZkbWil5%!oyqtXo^;K9w%f9DA4&KXt~2DI^6E z>qZJ0&lI)efMMvwO?hP==`TvrXhP6p>?6|2$>mAbxc{)N%GgVP>(Nn!;w0>FQkK2a z{_vgJ|1|*H9qt?Qq(Wt#rqSbZCnnYb%5=-tmdB~OO+`(w@$_xk_|jY% z`#Ohfb>uZoP=2PqQd`?=&BEnIs4freG3WC4VyJXSWyURXO?ZE?v~?x&+JKq?;`IX5 z?g{a3w%k%`*k|s~58%0#Z%QTc1hMO)%T_bjGEIK#2Br3kes{SuDgj;4NvD3RnVg48G(j;qFdqOd1HTOYOw22GvY zkxaS)vGO+Hf)h(ukSV1$PaY#>62qd6pobxhposmtLQ3G}1v{Sra&s_nPN5Z_i~eN# z)?`Ql1z*u(lz`({!sphweYe`mK23GF$ICXKGLQf_l%%;+b2S)rwGnExNPxCg z20*Ob43WL84r%rWq{OL6N(L7a1WT_YelYr2tQhvyAN|OK@u5fk)`xx(>XWnyDa1$c z`@CN`VATpG`IvYJq^nL;O0fV85Qe&y+orMg5Qn|A-9 z7Y|6*Tn29$0|pW1!YzVABZHo3F{rzQyC-lOXt@ql@&!~y%HkQ+dXzI?YSYNFS6Bis|uG@*7(ig>c7G+uYu@_LS! zMCBitn)?eCT1C=oLcJM-SwEmRsoW)q%x#1VnGb5w#@9Z=ocBw@jkVXR^5X z>@GWz3nx8rvx>Fz?aD_Lb7+K}>d@ux#$oOH(V*=SKi$T}+HtpBQb^Hv&&;oME_}}_ zAeJY;JIley#LCtoe+ZxH*UV+h^ehmczUbl$^+{9F)fjafM?OErYo|gOqC)|MiI(B@ z7+yLO>el)^Zo-idAWD4*76oqpm9xKgyIx4!M(#2ch0`Ic$O%*vcvz%K(`iGgyGGT+*% zNZjw>*A;%XegULl%B=xj+w-6CYS8OjV&>{)!7g6-5!z?g>{)FU$3R6Sc5}OelMaIC z)KlQYDpT9g2Zy6Wg~IB$Eb-}>Bjh^|?7Ns|DD@ATuaDuZXO?ia&jzmaW>Erj74GHs zSn$n)H@r#@t4*k2-?a_)jBdVK2MYZJE5>dV-;C1OOM;ocYu%Von|mdFS(+^!4~Juc z@A*8Sm=Rg2G0DmV(Zxw5MQAU@8XTaRjE0vQbLCtq-0~@%5Ak5n=t;4{hi}zu$vd*R z)1D8l)sn#s&W|0YJwYK%b_ky(ICLPrA9)=NID}6v9*y3Bc{>qOu zuO0P9PLhmTZ`U`kUvw_~GkBbozeCGKEgL-#UqWgWK;M2voht(v5-D%Rc z6`{P^qDNQGJ{cQdPQ6w2+>zv{k_v*#BSi@km@=KQarN~f)jtHu=mguv2-Ry}(PYN> z1T3`kyacb`9KuxbY7$lqf(ykLq*}NSgN1;P2I+0(q&F+%SC9# zX6HPyc;XpeCp~Oft_JnyJBAhfmb>P4`K#Xuq(g!nlWub7T;JVMs#9tb!+r%)0Nby2tgA7^No_yFmq#1 z-k4jdz)xai)FMC(B1Ke}`C7uwdr#=j+@g&@wX4V28IrxPeWm`a^SzrmE|x$rfu|*S z$|@N|$_T}L-fw;ZOs2iU5$G4o=X}Ao&Y2JdL~ly`SI1v4$Kwy*E=(n=E|S|b`OlfG zRr=I$2quw7Ch>GJY*C*k;_sYG8#T|ws3C9}<9v0T>`U+Q=XVtO2ezL!N_U=}*9o7L zU-=cJAwQslfq}ukUozU#tk)lZz2BjOgT4Q44+bWz?_g?6=VWVbr4l`5)5GvlbmIym zHrqU*QD!@kzVs)M>=bv3EHNGtd3(bgex?y?gl}QK{nUiT^f^!G!!=nnDk@3(MLZlW zV~Fkct^sA3YQr?~P|Gt#q|Gg{h^cvz9u>BDDHa;E@a;FuD9wfy6sAGx2Gus}=bfBo z-KIS_7P&rT&J_5q@yXMWEmLel=utkfA1(UF9>AvD)t% zPv~~f5VgtS3Sd?Go#3It%ZL9?aG(+m+y@d2to41q*zXf`bTFhhFt?#MF}HFucF=V+ z)^{*8qjPXfRME0sVMF)6tI}iDOJtChm^L^_85Q&=7x0A;J+7n=BAJ}7krg3jJsPS> z+v~iGQKA_hhX%kJ9IaLnrQ@bBi18O(>(*x&Y6*@#Sm#1@a6d`u?#J&b99*cL5@!R? z&K2j~j6)e)mG1|iUSFY@tS>G7mQ1eggY`aGpw;2vmd?vP1Md_}@hG8w-D~AkpBlP+ z%^i~LwRhiA35cxm^Jb+@g-~1vRU2Xm8OP9bzBPdx2DC2;&gZCOo-ZjG?0Hv;$~KxR z2n4lH;W~~Vs~TtMrh+_~9*m$3Ixyt9>ToRv34vHL-qXLOgU z&Z@p~j2Rn?*@p?da@d&=F~6r{gsnAiv4knh5@~#Y%CT<)rhY#V|00X^VoFsFL+@?( zR)SjvDEf0%#InST@nS0mD(=!O#`})o3q_L=0z@XH)od?Ca2zW)NiuE-s_$rxZ1+5j z7eb;?hIm4j9HTj0$D)^{&{;(;!f|1_@eiK^%8=fw;@|+RzaP08c*nBzj)yV&J|F2> zvjDd`VDrC~a_7T7{tm%OgC@0j(O_Y-J%lVYrOPv;e44g_R9Rd3lm;eb+k4z&M zgVny8!zP^fsVkUv*VEW(so^}qAYzFdp2S;YOiNtXmJBrfqyJ~oPfb;fJn~Nz7`>mW zrBa>(aPPK%=`b&UtM7qW5=%a>-E4{FWSF&*<m5QTJdg$ht=OgAeay&P+% z;GuN<|$+$qo|0WAs?n48ww20VU#|!b+;PJdU$5x3|*3Rs0~?qCVQA! zmv7~s3jL+tR$?YkigD6;b8>PYs~}l@Zuk^1R&u*FzhF-z9Tg{og$Yk}X&a+mFhfNm z>9k7eF&+hkiCf>#YkeFNv&#&?k$<}y+1wg0ewwV1tUniyda5{20_t<8jwC~l8}EJ~ zaW01x8FHRa$T_#GhV9}$)AmF`yCIwvpUB3#O{PoSI3Q7cm|b~L^Q-nK1qJBk6ZCN% zNLeCNtN=Qf{Up>dT|o*W$pJa4qUpnPU@1n&*8E4dkQ4SNS*_I`H=ukamcpMvr{Mhw z>=Q`am_3%GTbx=CrW9&WNwBF7UWpK^sf&9Xge(<$^14DW-%DSQHovonfobHegnsTi zpgW)0{Q*6OXfQ9gT7GrL=o0O}MtdS8ST_G|wEb}ZHCnSD1`g&%|1w%tYs+Ic^w%ui z-1P41RMX{kMz#iNa}CJAZuOZHX;`tEXf#Ew+ua>99ox63D5Hg>_|=*)CFy-x@%`VN zrC;@gjtz*We5UN8WZ^LOuga0{AC6Mx7UCb(51i?cVrX{x&(q7#$B`j#OU?(0T3*ol z0_9Gj23DrFPy`WZoVqYe3YXO{*zba<*M;Z~!gQhF9lg5tP-_7!G{tCgmgNoh*Bb0b z@FK5vT}g>>7{X*K5HRCq>PA8{9!E4DL6CF;2X0T|pIYn-;#v1IdyexTpv%?zhLuh& zQ^|8SqUvASk|d6|!0aOpGW{9|7@mG%VL0JpQ0&U)8;qYhdGe-u*7a?RVrJ|M{hSlQ zCX~it7N0QPn4q*#$=r(EhXHkPtdd1qRDeN+@pv#~FGaM=VOM;NAy*QXnX+D_|42fw zoC;TymBn)AWQo4d0U7|>Dv9J8ol}E89BT}X!@gN!$RJ*=Y}T3*AGty=lF4p z2%+P&)^vvy?u`5Jx9fQ$l}l&Dul1lr>o~1}Zxl1W?eELIwA8_+MOFQXGUf!NP9*l_ zWqclO6V-{W4l0P0kIhYFUD>|R=!p=NN4E4G2Qau(8=LM5F1UdUYX``ec3uwdee%k2 z8cjSy($`Iw%P-;f5?9GdH@o&_wxCq;=q+Z!R^4_=wO* z62UmFz6%gmAPDujM^Vl-lKU;h>C}Xmi>6S(@Y2$c#xF0}&`y;{I;B`@Os9BSB4&f{ zgs3j!Q>SccLYthtCE7*T6Ry&ggtawc;8*SktCz9kERB3gNjT8g#zHWZpD2@u? z?(B4HlzmP-c9?VyoRPtt0e-I~w4*DMMkXEM5&{&29OI)CVqt>2d3YRWPl(I~Jet*W z+3+{WKSa?AkTVzWzp=w=A;~{C#*aU_EamMD%h+%~?oA+^nXTGq2~eBCrvkiSpHB2x z9L+#N;VFv7gA5M>3clPSwQap_w++iZDZWsY;7 zf?Ve-cB+cv6zCx-?|Q3^eTz0QHH#BsiXW84AEj8BW>VG@m01!d!kCqh<+x1IKj9^k zcxulIL5^!jU-!-9I=A=8cwH7%%^#esB%IdJQ0p_tY*57efdAJQ9AVIiX1)6Yn0H@* z@h^beI~zOv(6zJDxA~ji*6;L2em&Lg{@F#W|1+jQc#1JK;DfX;1VQ86S_gKJfV63P zh=|23u<@PVXA;y&My8NL9~hn6?vnS@{=)Z}(j&d*cuhsl-gCz!#D=bCVbzGF
  • ; z{awt&F8e)=Wm`>OikH&;R`KcI_}2Oh-xqM*u(K2<%q&V31Wz-QLYdY%=tkRWem!Z? zOYH#CFK~C$j8(W+5zg0T=8_m8mp0L_rO4C)p1=qGPuRzxHNslx{#svtT5^-qebw0l z4`?W(MXCzM{Wcqwfrm&S-Iw?#$5|vLM;-W5Of>zCMJdK795K*ZfefjGjCModYZkSq&O0jH+h`5LI;am%ld0F}2>A*3kMW zQcorFD@n``>9$h9N?{{_+6<33 z3RL~pr?b;cbBm9ERvdGcI8{i$#rc3;ceg9_T)U`;AMEhiBzYW9l~rY~_9rr}Btt2- zjju`%5B~3F!-<=;se(LX%ptZGKr+rk*Np2dRGBy#H zee$xp5h@oQ-XHFNOQtmSthEw=!>>;bhuLAqRZxzvaHJ)5tvIqeoaQ9&&M!WED zerr;C(8&SCGXf?8I}u|eH@RS#Pha=P7`ieyh=kZFrVI`oR%QG3Qt%t&TRfL`zKe+= zE8RH(V1JFLp`sJAV}ds~oxX&plpy8sY5G7mb>1s@7l`Iv9_sDsLiN}vF9qtgpDgh> z6|%y{TlWxInebXR`PmL@Bot&zY_UAad9@|YJ5QECxoo@1 zN|9yFI*2$P$d>ll>FBadL50D<&mwo)@#LjjhMHnGUTJ#$)nYvR?34Gi-HEQ!E!cDm zNG#|3jm{}bhQP5Bn^^HEQ;97-^jx*(D%S0^zfMkqk86xHMJ0UB-rrro3K&A+lU$hsx$rlndLwCrl2@L24}|^2qt` zA|)=!CSIYJ>m*p`R&b5n6ie7fWlTc$yn_3|9Kmfw4FfU7IWY(Clihy3^COg#_}X`X zrydHp;b>v+>Fxm92^3$3rFM)n3P1xkbal%ts=n}IDC>`dW_?*O5mC@asSdl7EoojR zus>=Pa5?VSy{=%Y0^Edp?)i{enzSOw4ZyV^^T*N%6 zZMg+B%y>;BNGA*{StSM)$#3C^DL&DCFvmv9FDO!X$mz@lA^8OY znm#QbP*yC!<1mGtb$)>Bag3z^Uc}y`>y`?O zI?!tDfuHpqk|*HPxcywg^hv0UcaEQ_r`vt;l|TuqDfT+7)W4sX3$WsW^wP6;?~sHo zRs9Q+V(*XyfbKZ^Rp&T$p!NG~uD6&hzm-8zV)s5&yrfPn?Et2L;2IR4aAu42eV71w zt3&vd$4>dl#BD9u91DHZF!2_J5)22wca%2iU=3#_1Icg}j=DpbV1^tc-4UdwMmwPOGk@PJStK4EALTOLfZTp+!Z0Vd_Sk2 zHUD^fBP|tJR4T7RJ2ttQxOFwa;J82hMP}@WFId-yb-X!YuL9QhcGo_ zu+H)LHzX}p=Z%-DnXo?n1<6z2#GX~6zaV*?^uLgFY4J_QX=)ht%{9^b8%G4I4#~2)cS!dA4axP0YzB-E2>iGFi~bHoATOLdBcOi9(W(&1k5~4B zWT2;<1Zo!MaMlvlL&(h2_XahhCD=-4hMiNN^^b(7&(1{6jBsilBZ}-8>kOP$U!0fR zyM84^5R;uA!lE!`&O=f1*w8?n&$}P%4*eYjzx=TAu@-)a#0(wYTOaL&2dov@epi%1-Sr^ zg;b34p>9p9vnVEhp2U3I{&jMtw>RfIt!!_iH8?V!Vc<~nN%FuTns(lqb^%SxnneWO z@rELWkE>8kUimKyeg=7ayi*WTK^hza9rB;j+lVx@jY>2a7~FS=ze{iLDE#~IeJ{oR zze1dqt)b Date: Thu, 15 Oct 2026 14:57:20 -0700 Subject: [PATCH 19/19] docs: correct data workflows, SQL and operations (#4534) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR is part of the documentation revamp. Once the full set of PRs is merged, we’ll publish the combined site to staging for review before updating docs.lancedb.com. The curation, SQL and operations pages contain outdated claims and examples that do not reliably work as written. This corrects those pages against the reviewed SDK and local server evidence, including prepared statements, background jobs, cleanup and monitoring. Integration guides are grouped with their examples in #4533. Function and Secret-object documentation stays unpublished until its release. The two Function pages are listed in `.mintignore` and left out of the navigation, and the sections about them on other pages are MDX comments. Stacked on #4533. Each MDX file has one PR owner. Rebase onto main after #4533 is squash-merged. Local assembly, Mint validation and link checks pass, and the assembled site shows no Function or Secret-object content. A local release rehearsal that uncomments the drafts and restores the navigation also passes. Hold publication until this PR and lancedb/sophon#8688 are on `main`. --- docs/web-tests/py/test_curation_sampling.py | 27 ++ docs/web-tests/py/test_function.py | 127 +++++++-- docs/web-tests/py/test_materialized_view.py | 26 +- docs/web-tests/py/test_multimodal.py | 20 +- docs/web-tests/py/test_tables.py | 60 ++--- docs/web-tests/rs/embedding.rs | 6 +- docs/web/.mintignore | 6 + docs/web/api-reference/rest/index.mdx | 26 +- docs/web/curation/aggregate.mdx | 189 ++++++++------ docs/web/curation/index.mdx | 137 +++++++--- docs/web/curation/join.mdx | 152 +++++++---- docs/web/curation/sampling.mdx | 190 +++++++++----- docs/web/curation/windowing.mdx | 216 +++++++++------- docs/web/deployment/monitoring.mdx | 18 +- docs/web/deployment/storage.mdx | 23 +- docs/web/deployment/support.mdx | 7 +- docs/web/docs.json | 8 +- docs/web/embedding/index.mdx | 15 +- docs/web/enterprise/authentication.mdx | 17 +- docs/web/enterprise/index.mdx | 6 +- .../add-computed-column.mdx | 55 +++- .../feature-engineering/data-evolution.mdx | 39 ++- docs/web/feature-engineering/index.mdx | 128 ++++++--- .../materialized-views.mdx | 113 ++++---- .../feature-engineering/refresh-column.mdx | 35 ++- docs/web/governance/authorization.mdx | 90 +++++-- docs/web/maintenance/background-jobs.mdx | 42 +-- docs/web/maintenance/cleanup.mdx | 55 +++- docs/web/maintenance/compaction.mdx | 23 +- docs/web/management/database.mdx | 242 ++++++++---------- docs/web/management/function.mdx | 28 +- docs/web/management/index.mdx | 44 ++-- docs/web/management/materialized-view.mdx | 49 ++-- docs/web/management/namespace.mdx | 64 +++-- docs/web/management/table.mdx | 118 ++++++--- docs/web/management/view.mdx | 30 ++- docs/web/snippets/curation_sampling.mdx | 4 + docs/web/snippets/embedding.mdx | 6 +- docs/web/snippets/materialized_view.mdx | 4 +- docs/web/snippets/multimodal.mdx | 2 +- docs/web/snippets/tables.mdx | 2 +- docs/web/sql/data-types.mdx | 4 +- docs/web/sql/ddl/function.mdx | 11 +- docs/web/sql/ddl/index.mdx | 32 ++- docs/web/sql/ddl/job.mdx | 4 +- docs/web/sql/ddl/materialized-view.mdx | 23 +- docs/web/sql/ddl/namespace.mdx | 18 +- docs/web/sql/ddl/table.mdx | 34 ++- docs/web/sql/ddl/view.mdx | 2 +- docs/web/sql/dml/copy.mdx | 6 +- docs/web/sql/dml/delete.mdx | 8 +- docs/web/sql/dml/insert.mdx | 2 +- docs/web/sql/dml/refresh.mdx | 26 +- docs/web/sql/dml/update.mdx | 2 +- docs/web/sql/dql/aggregation.mdx | 32 +-- docs/web/sql/dql/explain.mdx | 11 +- docs/web/sql/dql/fts.mdx | 2 +- docs/web/sql/dql/join.mdx | 5 +- docs/web/sql/dql/sampling.mdx | 25 +- docs/web/sql/dql/select.mdx | 13 +- docs/web/sql/dql/subqueries.mdx | 21 +- docs/web/sql/dql/window.mdx | 8 +- docs/web/sql/functions.mdx | 2 +- docs/web/sql/index.mdx | 102 +++----- docs/web/sql/operators.mdx | 18 +- docs/web/sql/prepared-statements.mdx | 62 +++-- docs/web/sql/scalar-functions.mdx | 32 ++- docs/web/tables/branching.mdx | 180 ++++++++----- docs/web/training/data-loading.mdx | 53 ++-- docs/web/training/torch.mdx | 55 +++- docs/web/training/why-lancedb.mdx | 3 +- .../agents/time-travel-rag/index.mdx | 6 +- 72 files changed, 1997 insertions(+), 1254 deletions(-) create mode 100644 docs/web-tests/py/test_curation_sampling.py create mode 100644 docs/web/.mintignore create mode 100644 docs/web/snippets/curation_sampling.mdx diff --git a/docs/web-tests/py/test_curation_sampling.py b/docs/web-tests/py/test_curation_sampling.py new file mode 100644 index 000000000..d50b8a43f --- /dev/null +++ b/docs/web-tests/py/test_curation_sampling.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + + +def test_sample_dataset(): + # --8<-- [start:sample_dataset] + from tempfile import TemporaryDirectory + + import lancedb + from lancedb.permutation import Permutation, permutation_builder + + with TemporaryDirectory(prefix="lancedb-sampling-") as uri: + db = lancedb.connect(uri) + table = db.create_table("records", [{"id": i} for i in range(10)]) + splits = ( + permutation_builder(table).split_random(counts=[3, 7], seed=42).execute() + ) + sample = Permutation.from_tables(table, splits, split=0).with_format("arrow") + selected_ids = [] + for batch in sample.iter(batch_size=3): + selected_ids.extend(batch.column("id").to_pylist()) + print(f"Selected {len(selected_ids)} rows: {selected_ids}") + # --8<-- [end:sample_dataset] + + assert len(selected_ids) == 3 + assert len(set(selected_ids)) == 3 + assert all(0 <= value < 10 for value in selected_ids) diff --git a/docs/web-tests/py/test_function.py b/docs/web-tests/py/test_function.py index 629ab89bc..f66a50aec 100644 --- a/docs/web-tests/py/test_function.py +++ b/docs/web-tests/py/test_function.py @@ -4,16 +4,18 @@ """Examples for docs/web/management/function.mdx. Registering a Function and filling its column need a LanceDB Enterprise -deployment, so `test_function_enterprise` runs only when the connection -variables the Enterprise examples already use are set: LANCEDB_URI, -LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the -deployment needs one. The walkthrough calls its table `function_demo` and its -Function `double`; the test creates both under names unique to its run, never -replaces an existing object, and drops only what it created. +deployment, so `test_function_enterprise` runs only when +LANCEDB_DOCS_TEST_DEPLOYMENT=1 asks for one, with the connection variables the +Enterprise examples already use: LANCEDB_URI, LANCEDB_API_KEY and +LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the deployment needs one. The +walkthrough calls its table `function_demo` and its Function `double`; the test +creates both under names unique to its run, never replaces an existing object, +and drops only what it created. The other tests run locally and check the client side of the same code: what -`@udf` sends for registration, how a registered version binds a column, and -where a local connection stops. They do not run a Function remotely. +`@udf` sends for registration, how a registered version binds a column, where a +local connection stops, and when the Enterprise test connects. They do not run +a Function remotely. """ import json @@ -70,12 +72,17 @@ def registered_version(definition) -> FunctionVersion: return FunctionVersion.from_json(json.dumps(reply)) -def require_env(var_name: str) -> str: - """Skip the test unless the required environment variable is provided.""" - value = os.environ.get(var_name) - if not value: - pytest.skip(f"Set {var_name} to run this Enterprise example") - return value +DEPLOYMENT = ("LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY") +# The connection settings alone never send this test to a deployment. +OPT_IN = "LANCEDB_DOCS_TEST_DEPLOYMENT" + + +def require_deployment(): + if os.environ.get(OPT_IN) != "1": + pytest.skip(f"set {OPT_IN}=1 and {', '.join(DEPLOYMENT)} to use a deployment") + missing = [name for name in DEPLOYMENT if not os.environ.get(name)] + if missing: + pytest.fail(f"{OPT_IN}=1 needs {', '.join(missing)}") def test_function_runs_locally_as_python(): @@ -160,9 +167,7 @@ def test_function_version_binds_a_column(tmp_path): def test_function_enterprise(monkeypatch): - require_env("LANCEDB_URI") - require_env("LANCEDB_API_KEY") - require_env("LANCEDB_HOST_OVERRIDE") + require_deployment() # --8<-- [start:function_connect] import os @@ -240,3 +245,91 @@ def test_function_enterprise(monkeypatch): db.drop_table(table_name) if version is not None: db.drop_function(version.name, version=version.version) + + +class ReachedConnection(Exception): + pass + + +SETTINGS = { + "LANCEDB_URI": "db://docs-test", + "LANCEDB_HOST_OVERRIDE": "https://deployment.invalid", + "LANCEDB_API_KEY": "docs-test-key", +} + + +@pytest.fixture +def connect_attempts(monkeypatch): + """`lancedb.connect`, replaced so that it records each call and stops.""" + attempts = [] + + def connect(**kwargs): + attempts.append(kwargs) + raise ReachedConnection + + monkeypatch.setattr(lancedb, "connect", connect) + for name in (OPT_IN, *DEPLOYMENT, "LANCEDB_REGION"): + monkeypatch.delenv(name, raising=False) + return attempts + + +def enterprise_outcome(monkeypatch): + try: + test_function_enterprise(monkeypatch) + except (pytest.skip.Exception, pytest.fail.Exception, ReachedConnection) as outcome: + return outcome + + +@pytest.mark.parametrize( + "opt_in", + [None, "", "0", "true", "TRUE", "yes", "on", " 1", "1 ", "01", "1.0", "11"], +) +def test_function_enterprise_needs_the_exact_opt_in( + monkeypatch, connect_attempts, opt_in +): + for name, value in SETTINGS.items(): + monkeypatch.setenv(name, value) + if opt_in is not None: + monkeypatch.setenv(OPT_IN, opt_in) + outcome = enterprise_outcome(monkeypatch) + assert isinstance(outcome, pytest.skip.Exception), outcome + assert str(outcome).startswith(f"set {OPT_IN}=1 ") + assert connect_attempts == [] + + +@pytest.mark.parametrize("blank", ["unset", "empty"]) +@pytest.mark.parametrize( + "missing", [*([name] for name in DEPLOYMENT), list(DEPLOYMENT)] +) +def test_function_enterprise_fails_when_asked_without_settings( + monkeypatch, connect_attempts, missing, blank +): + monkeypatch.setenv(OPT_IN, "1") + for name, value in SETTINGS.items(): + if name not in missing: + monkeypatch.setenv(name, value) + elif blank == "empty": + monkeypatch.setenv(name, "") + outcome = enterprise_outcome(monkeypatch) + assert isinstance(outcome, pytest.fail.Exception), outcome + assert str(outcome) == f"{OPT_IN}=1 needs {', '.join(missing)}" + assert connect_attempts == [] + + +@pytest.mark.parametrize("region", [None, "eu-west-1"]) +def test_function_enterprise_connects_when_asked(monkeypatch, connect_attempts, region): + monkeypatch.setenv(OPT_IN, "1") + for name, value in SETTINGS.items(): + monkeypatch.setenv(name, value) + if region: + monkeypatch.setenv("LANCEDB_REGION", region) + outcome = enterprise_outcome(monkeypatch) + assert isinstance(outcome, ReachedConnection), outcome + assert connect_attempts == [ + { + "uri": "db://docs-test", + "api_key": "docs-test-key", + "region": region or "us-east-1", + "host_override": "https://deployment.invalid", + } + ] diff --git a/docs/web-tests/py/test_materialized_view.py b/docs/web-tests/py/test_materialized_view.py index ec48e1033..414d6dc1e 100644 --- a/docs/web-tests/py/test_materialized_view.py +++ b/docs/web-tests/py/test_materialized_view.py @@ -12,7 +12,9 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch): db = lancedb.connect("./.lancedb") table = db.create_table( - "people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}] + "people", + [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}], + storage_options={"new_table_enable_stable_row_ids": "true"}, ) view = db.create_materialized_view( @@ -29,12 +31,11 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch): # --8<-- [start:materialized_view_refresh] table.add([{"name": "bea", "age": 41}]) - result = view.refresh() - print(result.rows_written) + view.refresh() + print(view.table.count_rows()) # 2 # --8<-- [end:materialized_view_refresh] - assert result.mode == "rebuild" - assert result.rows_written == 2 + assert view.table.count_rows() == 2 assert sorted(row["shout"] for row in view.table.to_arrow().to_pylist()) == [ "ADA", "BEA", @@ -45,7 +46,11 @@ def test_materialized_view_with_no_data(tmp_path): import lancedb db = lancedb.connect(tmp_path) - db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]) + db.create_table( + "people", + [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}], + storage_options={"new_table_enable_stable_row_ids": "true"}, + ) view = db.create_materialized_view( "adults", "people", where="age >= 18", with_no_data=True ) @@ -58,11 +63,16 @@ def test_materialized_view_refresh_to_source_version(tmp_path): import lancedb db = lancedb.connect(tmp_path) - table = db.create_table("people", [{"name": "ada", "age": 36}]) + table = db.create_table( + "people", + [{"name": "ada", "age": 36}], + storage_options={"new_table_enable_stable_row_ids": "true"}, + ) view = db.create_materialized_view("adults", "people", where="age >= 18") first = table.version table.add([{"name": "bea", "age": 41}]) - assert view.refresh().rows_written == 2 + view.refresh() + assert view.table.count_rows() == 2 result = view.refresh(source_version=first) assert (result.source_version, result.rows_written) == (first, 1) diff --git a/docs/web-tests/py/test_multimodal.py b/docs/web-tests/py/test_multimodal.py index bf7ec1bc8..cb4a647c3 100644 --- a/docs/web-tests/py/test_multimodal.py +++ b/docs/web-tests/py/test_multimodal.py @@ -215,28 +215,18 @@ async def test_query_to_pandas_kwargs(db_path_factory): .to_pandas(blob_mode="bytes") ) - # Vector / FTS / hybrid queries return "bytes" or "descriptions". - # Lazy BlobFile handles need a plain scan. + # Vector / FTS / hybrid queries can't materialize blob columns, + # so omit them from the projection df_vec = ( tbl.search(query_vector) .limit(10) - .select(["id", "video"]) - .to_pandas(blob_mode="bytes", split_blocks=True, self_destruct=True) + .select(["id", "vector"]) + .to_pandas(split_blocks=True, self_destruct=True) ) - - # to_arrow and to_list take the same blob_mode - hits = tbl.search(query_vector).limit(2).select(["id", "video"]) - arrow_bytes = hits.to_arrow(blob_mode="bytes") - rows = hits.to_list(blob_mode="bytes") # --8<-- [end:query_to_pandas_kwargs] assert len(df_lazy) == 1 assert len(df_bytes) == 1 assert df_bytes["video"].iloc[0] == b"fake_video_bytes_1" assert len(df_vec) == 10 - assert all( - video == f"fake_video_bytes_{i}".encode() - for i, video in zip(df_vec["id"], df_vec["video"]) - ) - assert arrow_bytes["video"].to_pylist() == [row["video"] for row in rows] - assert all(isinstance(row["video"], bytes) for row in rows) + assert "video" not in df_vec.columns diff --git a/docs/web-tests/py/test_tables.py b/docs/web-tests/py/test_tables.py index b3a456adc..d17344aec 100644 --- a/docs/web-tests/py/test_tables.py +++ b/docs/web-tests/py/test_tables.py @@ -835,31 +835,31 @@ def test_update_optimize_cleanup_snippet(tmp_db): # ============================================================================ -def _setup_schema_add_table(tmp_db, data=None): +def _setup_schema_add_table(db): # --8<-- [start:schema_add_setup] - table_name = "schema_evolution_add_example" - if data is None: - data = [ - { - "id": 1, - "name": "Laptop", - "price": 1200.00, - "vector": np.random.random(128).tolist(), - }, - { - "id": 2, - "name": "Smartphone", - "price": 800.00, - "vector": np.random.random(128).tolist(), - }, - { - "id": 3, - "name": "Headphones", - "price": 150.00, - "vector": np.random.random(128).tolist(), - }, - ] - table = tmp_db.create_table(table_name, data, mode="overwrite") + import numpy as np + + data = [ + { + "id": 1, + "name": "Laptop", + "price": 1200.00, + "vector": np.random.random(128).tolist(), + }, + { + "id": 2, + "name": "Smartphone", + "price": 800.00, + "vector": np.random.random(128).tolist(), + }, + { + "id": 3, + "name": "Headphones", + "price": 150.00, + "vector": np.random.random(128).tolist(), + }, + ] + table = db.create_table("schema_evolution_add_example", data, mode="overwrite") # --8<-- [end:schema_add_setup] return table @@ -953,17 +953,7 @@ def test_add_columns_default_values(tmp_db): def test_add_columns_nullable(tmp_db): - table = _setup_schema_add_table( - tmp_db, - data=[ - { - "id": 1, - "name": "Laptop", - "price": 1200.00, - "vector": np.random.random(128).tolist(), - } - ], - ) + table = _setup_schema_add_table(tmp_db) # --8<-- [start:add_columns_nullable] # Add a nullable timestamp column diff --git a/docs/web-tests/rs/embedding.rs b/docs/web-tests/rs/embedding.rs index 33bdec997..67f6bbc80 100644 --- a/docs/web-tests/rs/embedding.rs +++ b/docs/web-tests/rs/embedding.rs @@ -17,7 +17,7 @@ use lancedb::{ #[tokio::main] async fn main() -> Result<()> { let db = connect("./mydb").execute().await?; - let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set"); + let api_key = "your-openai-api-key".to_string(); let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model( api_key, "text-embedding-3-large", @@ -54,7 +54,7 @@ use std::sync::Arc; use lancedb::embeddings::openai::OpenAIEmbeddingFunction; -let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set"); +let api_key = "your-openai-api-key".to_string(); let embedding = Arc::new( OpenAIEmbeddingFunction::new_with_model(api_key, "text-embedding-3-small") .expect("failed to create OpenAI embedding function"), @@ -77,7 +77,7 @@ use lancedb::{ #[tokio::main] async fn main() -> Result<()> { let db = connect("./mydb").execute().await?; - let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set"); + let api_key = "your-openai-api-key".to_string(); let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model( api_key, "text-embedding-3-large", diff --git a/docs/web/.mintignore b/docs/web/.mintignore new file mode 100644 index 000000000..b8da6c17a --- /dev/null +++ b/docs/web/.mintignore @@ -0,0 +1,6 @@ +# Functions are documented before their release. At release, remove these +# entries, add the pages back to the navigation in docs.json, restore the +# /feature-engineering/functions redirect to /management/function, and publish +# the drafts marked "Draft for the Functions release". +management/function.mdx +sql/ddl/function.mdx diff --git a/docs/web/api-reference/rest/index.mdx b/docs/web/api-reference/rest/index.mdx index 805f73cda..b2dd10016 100644 --- a/docs/web/api-reference/rest/index.mdx +++ b/docs/web/api-reference/rest/index.mdx @@ -35,10 +35,13 @@ To authenticate to the Enterprise REST API, you need the endpoint for your deplo ```bash export LANCEDB_API_KEY="" -export LANCEDB_URI="https://your-enterprise-endpoint.com" +export LANCEDB_HOST_OVERRIDE="https://your-enterprise-endpoint.com" export LANCEDB_DATABASE="your-database-name" ``` +`LANCEDB_HOST_OVERRIDE` is the endpoint the SDKs take as `host_override`; the +database name is the part of its `db://` URI after `db://`. + 3. If your Enterprise deployment is private, connect through the private network endpoint provided for your deployment. For example, Azure Private Link deployments commonly use a private IP or an internal DNS name as the endpoint. ### Verify authentication {#verify-authentication} @@ -46,30 +49,37 @@ export LANCEDB_DATABASE="your-database-name" 4. Check that you can reach the deployment and list tables: ```bash -curl -X GET "$LANCEDB_URI/v1/management/table" \ - -H "Content-Type: application/json" \ +curl -X GET "$LANCEDB_HOST_OVERRIDE/v1/table/" \ -H "x-api-key: $LANCEDB_API_KEY" \ -H "x-lancedb-database: $LANCEDB_DATABASE" ``` +The response lists one page of table names, with a token for the next page when +there are more. + If your deployment endpoint already includes the database host name, you can omit the `x-lancedb-database` header. -5. Create a table to confirm write access. Let's call it `words`. +5. Create a table to confirm write access. Let's call it `words`. A table is +created from an Arrow IPC stream, which this writes with PyArrow: ```bash -curl -X POST "$LANCEDB_URI/v1/tables/words" \ +python -c "import pyarrow as pa, sys; t = pa.table({'word': ['hello', 'world']}); w = pa.ipc.new_stream(sys.stdout.buffer, t.schema); w.write_table(t); w.close()" > words.arrow + +curl -X POST "$LANCEDB_HOST_OVERRIDE/v1/table/words/create" \ -H "Content-Type: application/vnd.apache.arrow.stream" \ -H "x-api-key: $LANCEDB_API_KEY" \ - -H "x-lancedb-database: $LANCEDB_DATABASE" + -H "x-lancedb-database: $LANCEDB_DATABASE" \ + --data-binary @words.arrow ``` 6. Check that the table has been created: ```bash -curl -X GET "$LANCEDB_URI/v1/management/table" \ +curl -X POST "$LANCEDB_HOST_OVERRIDE/v1/table/words/describe" \ -H "Content-Type: application/json" \ -H "x-api-key: $LANCEDB_API_KEY" \ - -H "x-lancedb-database: $LANCEDB_DATABASE" + -H "x-lancedb-database: $LANCEDB_DATABASE" \ + -d '{}' ``` That's it -- you're connected! Now, you can start adding data and querying it. diff --git a/docs/web/curation/aggregate.mdx b/docs/web/curation/aggregate.mdx index 91c32fdd4..b6d02d7fb 100644 --- a/docs/web/curation/aggregate.mdx +++ b/docs/web/curation/aggregate.mdx @@ -1,113 +1,144 @@ --- title: "Profiling a dataset" sidebarTitle: "Aggregate" -description: "Counting, grouping and summarizing — finding out what your data actually contains." +description: "Measure coverage, missing values and class balance before selecting a dataset." icon: chart-column keywords: ["aggregate", "group by", "class balance", "count", "profiling", "distinct"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL aggregation. --- -Enterprise Aggregation is SQL, and -[SQL needs a deployment](/basics/sql#sql-is-remote). +Use SQL aggregates to summarize a dataset: count records, find missing values, +and compare groups. These summaries help you understand the data before +selecting records for search, evaluation or training. -Before curating anything, find out what you have. Most of the surprises in a -training set — a class that is 90% of the rows, a column that is half null, a -"unique" id that is not — are one `GROUP BY` away from being visible. +## Example data {#example-data} -## How big, and how complete {#size} +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. + + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. ```sql -SELECT - count(*) AS rows, - count(embedding) AS embedded, - count(caption) AS captioned -FROM images +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); ``` -`count(*)` counts rows; `count(col)` counts **non-null values**. The gap between -them is your coverage, which is the fastest way to check how far a -[column refresh](/feature-engineering/refresh-column) has got without waiting -for the job to report. +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. -## Class balance {#balance} + -The single most useful curation query: +## Count records and missing values {#size} + +Use `count(*)` to count rows and `count(column)` to count non-null values. +Add `DISTINCT` to count unique values: ```sql -SELECT label, count(*) AS n -FROM images +SELECT count(*) AS records, + count(DISTINCT doc_id) AS documents, + count(caption) AS captioned, + count(*) FILTER (WHERE caption IS NULL) AS missing_caption +FROM curation_documents; +``` + +The example returns **6 records, 4 documents, 4 populated captions and 2 missing +captions**. `count(column)` excludes null values; it does not check whether a +string is empty or whether a value is useful for the task. + +## Group records by label {#balance} + +Use `GROUP BY` to calculate a summary for each label: + +```sql +SELECT label, count(*) AS documents +FROM curation_labels GROUP BY label -ORDER BY n DESC +ORDER BY documents DESC, label; ``` -If the top class is an order of magnitude larger than the rest, sampling -uniformly will reproduce that imbalance. Two ways out: draw a fixed number per -class with [windowing](/curation/windowing#per-group), or oversample the rare -ones and accept the duplication. +There are **2 cats and 1 dog** in the label table. Document 40 is absent, so this +query does not measure labeling coverage across all source documents. Use a +[left join](/curation/join#join-type) to find documents without labels. -## Distinct values {#distinct} +Decide whether the distribution matches your intended use. A uniform random +sample tends to preserve the source distribution; it does not balance classes. +To cap the number selected per label, use +[ranking within a group](/curation/windowing#per-group). + +## Count records that match a condition {#conditional} ```sql -SELECT count(DISTINCT source) FROM images +SELECT source, + count(*) AS records, + count(*) FILTER (WHERE score >= 0.9) AS high_score, + count(*) FILTER (WHERE caption IS NULL) AS missing_caption +FROM curation_documents +GROUP BY source +ORDER BY source; ``` -On a high-cardinality column this is expensive, because exact distinctness means -holding every value seen. When an estimate will do: +`FILTER` applies a condition to one aggregate. Here, `books` has 2 records, +1 high-score record and no missing captions. `web` has 4 records, 1 high-score +record and 2 missing captions. Counts are per record, so repeated document IDs +still contribute more than once. + + +## Group numeric values into ranges {#buckets} + +Use `count(DISTINCT column)` for the number of distinct non-null values. +For numeric fields, group by a range or summarize their distribution. For +example, this query counts records in score buckets of width 0.1: ```sql -SELECT approx_distinct(source) FROM images +SELECT floor(score * 10) / 10 AS score_bucket, count(*) AS records +FROM curation_documents +GROUP BY score_bucket +ORDER BY score_bucket; ``` -Cheap enough to run on a table where the exact version is not worth the wait. -The other approximate functions — `approx_median`, `approx_percentile_cont` — -are useful for the same reason when profiling a numeric column. +For time-based data, group by the time interval relevant to the task. The +[aggregation reference](/sql/dql/aggregation) and +[scalar function reference](/sql/scalar-functions) cover additional summaries, +including approximate counts and percentiles. Use an approximate aggregate when an estimate is sufficient; use an exact +count when the result must be precise. -## Conditional counts {#conditional} +## Profile a sample {#sampling} -`FILTER` narrows one aggregate without narrowing the query, which is how you get -several slices in one pass rather than one query each: +A [sample](/curation/sampling) can help you explore a large dataset before a +full scan. Report the sample size and method alongside the result. A sample can miss rare classes, and fragment sampling can reflect the order +in which records were written. -```sql -SELECT - label, - count(*) AS total, - count(*) FILTER (WHERE score > 0.9) AS confident, - count(*) FILTER (WHERE caption IS NULL) AS uncaptioned -FROM images -GROUP BY label -``` +Before accepting a curated dataset, check its required fields and key +constraints directly rather than inferring them from a small sample. -## Grouping by a bucket {#buckets} +## Continue curating {#reference} -Raw values are rarely the grouping you want. `date_bin` buckets timestamps, and -ordinary arithmetic buckets numbers: - -```sql -SELECT date_bin(INTERVAL '1 day', created_at) AS day, count(*) -FROM images GROUP BY day ORDER BY day -``` - -A histogram of a score column is the same trick: - -```sql -SELECT floor(score * 10) / 10 AS bucket, count(*) -FROM images GROUP BY bucket ORDER BY bucket -``` - -## Profiling a sample {#sampling} - -On a very large table, profile a [sample](/curation/sampling) and scale: - -```sql -SELECT label, count(*) * 100 AS estimated -FROM images TABLESAMPLE (1) -GROUP BY label -``` - -Treat the result as an estimate. A 1% sample will miss classes rarer than about -1 in 10,000 entirely, which is exactly the tail you are usually looking for — so -use this to check the shape of the head, not the existence of the tail. - -## Reference {#reference} - -The complete clause and function list is in -[Aggregation](/sql/dql/aggregation). +Use these measurements to choose a [sample](/curation/sampling), +[attach missing metadata](/curation/join), or +[select one record per document](/curation/windowing). +See [Aggregation](/sql/dql/aggregation) for complete SQL syntax. diff --git a/docs/web/curation/index.mdx b/docs/web/curation/index.mdx index 0bfc89b2d..fd8ea73aa 100644 --- a/docs/web/curation/index.mdx +++ b/docs/web/curation/index.mdx @@ -1,54 +1,125 @@ --- title: "Data curation" sidebarTitle: "Overview" -description: "Shaping a dataset before you train on it: profiling, sampling, joining, and ranking." +description: "Prepare datasets for search, evaluation and training by profiling, selecting and combining records." icon: filter keywords: ["curation", "dataset", "training set", "sampling", "deduplication", "class balance"] --- -Curation is the work between having data and having a *dataset*: finding out -what you actually have, drawing a subset you can afford to train on, attaching -labels that live elsewhere, and picking one row per thing when you have several. +Data curation turns a collection of records into a dataset suited to a task. +You might select labeled examples for training, prepare a fixed evaluation +set, or remove duplicate documents from a search corpus. -These pages are task-oriented. Each shows the operation in the context you would -reach for it, and links to the [SQL reference](/sql) for the complete syntax. +Start with the result you need. Then inspect the source data, choose which +records belong, and save the selected data or the rules used to produce it. - -Most curation is analytical -- grouping, joining, ranking -- which the query -builders do not do. It is therefore SQL, and SQL needs a deployment. See -[SQL](/basics/sql#sql-is-remote). - +## Choose a curation task {#questions} -## The four questions {#questions} - -| | | +| Task | What it helps you decide | | --- | --- | -| [What do I have?](/curation/aggregate) | counts, class balance, null coverage, distinct values | -| [What can I afford?](/curation/sampling) | a random subset, reproducible when you need it fixed | -| [What else do I know?](/curation/join) | labels, metadata and scores that live in another table | -| [Which one do I keep?](/curation/windowing) | one row per key, ranking within a group, splits | +| [Profile a dataset](/curation/aggregate) | Whether coverage, missing values and class balance suit the task. | +| [Sample records](/curation/sampling) | Which subset to inspect or use for an experiment. | +| [Join datasets](/curation/join) | How to attach labels, metadata or scores stored elsewhere. | +| [Rank and deduplicate](/curation/windowing) | Which records to keep when several describe the same item. | -## A worked shape {#shape} +The analytical examples in this section use **Enterprise SQL**. Connect with +the [SQL guide](/basics/sql). For sampling through the Python SDK in OSS or +Enterprise, see [Sample without SQL](/curation/sampling#training). -The four compose into the same pipeline most curation ends up being: +## Prepare a labeled document dataset {#shape} + +Suppose you need one labeled record per document for a training experiment. +Inspect the source records, choose the latest version of each document, then +join its label. Finally, check the class balance of the selected records. + +### Example data {#example-data} + +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. + + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. + +```sql +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); +``` + +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. + + + +The following query chooses the latest record for each document, then keeps +only documents with a label: ```sql WITH ranked AS ( - SELECT d.*, c.label, - row_number() OVER (PARTITION BY d.doc_id ORDER BY d.version DESC) AS rn - FROM documents AS d - JOIN labels AS c ON c.doc_id = d.doc_id + SELECT *, row_number() OVER ( + PARTITION BY doc_id ORDER BY version DESC, row_id DESC + ) AS rn + FROM curation_documents ) -SELECT * FROM ranked WHERE rn = 1 +SELECT d.row_id, d.doc_id, d.caption, l.label +FROM ranked AS d +JOIN curation_labels AS l ON l.doc_id = d.doc_id +WHERE d.rn = 1 +ORDER BY d.doc_id; ``` -Join to bring in what you know, window to pick one row per document, filter to -keep it. Add [`TABLESAMPLE`](/curation/sampling) when the result is still larger -than you want, and [aggregate](/curation/aggregate) to check the balance of what -you ended up with. +The result contains documents **10, 20 and 30**, with labels **cat, dog and +cat**. Document 40 is excluded because this particular task requires a label. +The `row_id` tie-breaker makes the choice for document 30 deterministic; +choose a tie-breaker that fits your own data. -## Then what {#next} +The example assumes one label per document. Duplicate keys in the label table +would multiply the result rows. The [joining guide](/curation/join#labels) +shows how to check that assumption. -A curated result you want to keep is a -[materialized view](/management/materialized-view) -- stored, refreshed, and -readable by a [data loader](/training/data-loading) like any other table. +## Check the selected dataset {#check-result} + +Compare the result with the task's requirements: are the expected documents +present, are required fields populated, and is the class distribution suitable? +This example has two cats and one dog. That distribution may be appropriate +for exploration, but it does not establish a representative evaluation set. + +[Profiling](/curation/aggregate) provides the counts to inspect. +[Sampling](/curation/sampling) and [ranking](/curation/windowing) help you +adjust the selection without changing the original records. + +## Save and reuse the result {#next} + +Choose the output according to how you will use it: + +- **A fixed dataset:** save the query result in a new table with + [`CREATE TABLE AS SELECT`](/sql/ddl/table#create). Keep that output unchanged + when comparing experiments. +- **A reusable selection:** save the query as a [view](/management/view). + A view stores the query definition, so it is not a frozen evaluation set. +- **Stored results that you refresh:** use a + [materialized view](/management/materialized-view) when its supported query + and refresh behavior fit the workflow. + +Record the source versions and selection rules alongside the output. Continue +with [data loading](/training/data-loading) to use the curated data in a +training pipeline. diff --git a/docs/web/curation/join.mdx b/docs/web/curation/join.mdx index b4c198ef9..03a45326e 100644 --- a/docs/web/curation/join.mdx +++ b/docs/web/curation/join.mdx @@ -1,91 +1,133 @@ --- title: "Joining datasets" sidebarTitle: "Join" -description: "Bringing labels, metadata and scores together with the rows they describe." +description: "Combine records with labels or metadata, and check for missing or duplicate matches." icon: link keywords: ["join", "labels", "metadata", "enrichment", "namespace"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL joins. --- -Enterprise Joins are SQL, and -[SQL needs a deployment](/basics/sql#sql-is-remote). +A SQL join combines records from two tables using a shared key. Use joins to +attach labels, metadata or model scores to your documents. Choose an inner join +to keep matching records, or a left join to retain records without a match. -Curation usually means combining things that were produced separately — images -in one table, human labels in another, model scores in a third. A join is how -they become one dataset. +## Example data {#example-data} -## Attaching labels {#labels} +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. + + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. ```sql -SELECT i.id, i.embedding, l.label -FROM images AS i -JOIN labels AS l ON l.image_id = i.id +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); ``` -An inner join keeps only rows that matched, which for curation is often exactly -what you want: unlabelled rows are not training data. +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. -When you need to *see* what did not match, use a left join and filter: + + +## Basic join {#join-type} + +An inner join keeps matching records: ```sql -SELECT i.id -FROM images AS i -LEFT JOIN labels AS l ON l.image_id = i.id -WHERE l.image_id IS NULL +SELECT d.row_id, d.doc_id, l.label +FROM curation_documents AS d +JOIN curation_labels AS l ON l.doc_id = d.doc_id +ORDER BY d.row_id; ``` -That is your unlabelled backlog, which is the thing worth counting before you -plan an annotation round. +This returns **5 rows**: both records for document 10, one for document 20, +and both records for document 30. To produce one row per document, +[deduplicate the source records](/curation/windowing#dedup). -## Across namespaces {#namespaces} +### Find records without a match -Tables in different [namespaces](/management/namespace) join by their dotted -paths — the catalog is one catalog, so there is nothing to federate: +A left join keeps every record from the left table and fills missing right-side +values with `NULL`. Add `WHERE l.doc_id IS NULL` to return only unlabeled documents: ```sql -SELECT i.id, u.team -FROM prod.media.images AS i -JOIN prod.reference.uploaders AS u ON u.id = i.uploader_id +SELECT DISTINCT d.doc_id +FROM curation_documents AS d +LEFT JOIN curation_labels AS l ON l.doc_id = d.doc_id +WHERE l.doc_id IS NULL +ORDER BY d.doc_id; ``` -## Joining a search result {#search} +The result is **document 40**. You might send it for annotation, retain it for +an unlabeled-data workflow, or exclude it from a task that requires labels. -Full-text search is a table function, so its result is a relation and joins like -any other. This is how you ask which categories a query matches, rather than -just which rows: +## Handle duplicate keys {#labels} + +This example joins `curation_documents.doc_id` to `curation_labels.doc_id`. +First check whether the label table contains more than one row per document: ```sql -SELECT c.name, count(*) AS hits -FROM fts('documents', 'puppy') AS m -JOIN categories AS c ON c.id = m.category_id -GROUP BY c.name -ORDER BY hits DESC +SELECT doc_id, count(*) AS labels +FROM curation_labels +GROUP BY doc_id +HAVING count(*) > 1; ``` - -Vector search has no SQL form, so a similarity result cannot be joined this way. -Run the search in an SDK, then join over its output — or write the result to a -table first. - +The example returns no rows. If your data returns duplicates, decide whether +they represent valid multiple labels or records that need consolidation. +A join does not enforce one-to-one matching. -## Making it fast {#performance} +## Join tables across namespaces {#namespaces} -Two things dominate join cost on a lakehouse table. +Use a table's qualified path to address it in another +[namespace](/management/namespace). For example, a table named `images` in +`prod.media` is addressed as `prod.media.images`. The same matching-key and +join-type rules apply; your identity needs access to the source objects. -**Index the join key.** A [scalar index](/indexing/scalar-index) on the key lets -the planner look rows up instead of scanning for them. Check with -[`EXPLAIN`](/sql/dql/explain): an indexed key shows a `ScalarIndexQuery`, an -unindexed one shows a `LanceScan` feeding the join. +## Combine search results with metadata {#search} -**Do not carry wide columns through.** Select the columns you need before -joining, not after. Embeddings and blobs are the expensive part of a multimodal -table, and a join can materialize them many more times than the result has rows. +A [SQL full-text search](/sql/dql/fts) returns rows that can be joined to other +tables. Join using an identifier retained in the search result, then check +whether the metadata has one or several matches per result. -```sql -SELECT i.id, l.label -FROM (SELECT id, uploader_id FROM images) AS i -JOIN labels AS l ON l.image_id = i.id -``` +For an SDK search, retain the matching IDs and combine them with metadata in +your application or save the selected records to a table for further SQL work. +See [SQL search interfaces](/basics/sql#no-vector-sql) for supported query forms. -## Reference {#reference} +## Join performance {#performance} -The join types and their semantics are in [Joins](/sql/dql/join). +Check the row count, unmatched keys and duplicate matches before using a +joined dataset. Select only the columns needed for the next step, especially +when source records include large media or embedding columns. + +Use [Explain a query plan](/search/explain) to see how the query reads and +combines data. Check the plan to see whether the query uses indexes and how it performs the +join. An index on a key does not guarantee that the join will use it. + +## Save the result or continue curating {#reference} + +Keep the query for reuse or [save the selected data](/curation#next). +See [Joins](/sql/dql/join) for join types and exact SQL behavior. diff --git a/docs/web/curation/sampling.mdx b/docs/web/curation/sampling.mdx index 4e8b000cd..643b06bdf 100644 --- a/docs/web/curation/sampling.mdx +++ b/docs/web/curation/sampling.mdx @@ -1,98 +1,172 @@ --- -title: "Sampling a dataset" -sidebarTitle: "Sampling" -description: "Drawing a subset you can afford to work with, and fixing it when it has to be reproducible." +title: "SQL sampling" +sidebarTitle: "SQL Sampling" +description: "Choose a sample for exploration or evaluation and preserve the selected records when reproducibility matters." icon: dice keywords: ["sampling", "tablesample", "subset", "eval set", "reproducible", "shuffle"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL `TABLESAMPLE`. For sampling in OSS and Enterprise through the Python SDK, see [Sample without SQL](#training). --- -Most curation starts with a subset. A question you can answer in ten seconds on -1% of the data is worth asking twenty times; the same question on the whole -table is worth asking once. +import { PySampleDataset } from '/snippets/curation_sampling.mdx'; -## A quick look {#quick} +Sampling selects records from a larger table. Use it to explore unfamiliar +data, run a smaller experiment, or prepare an evaluation dataset. Enterprise +SQL provides `TABLESAMPLE` for selecting a percentage or number of rows. -Enterprise +This page covers Enterprise SQL sampling and a complete +[Python SDK example](#training) that runs locally. + +## Example data {#example-data} + +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. + + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. ```sql -SELECT * FROM images TABLESAMPLE (1) -- 1 percent -SELECT * FROM images TABLESAMPLE (500 ROWS) -- 500 rows +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); ``` -A bare number is a percentage; add `ROWS` for a count. Use it while you are -exploring — the cost of the query drops with the sample, and so does the cost -of being wrong about what you wanted to ask. +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. -## Rows or fragments {#methods} + + +## Basic sampling {#quick} + +A percentage selects a fraction of the source; `ROWS` requests a row count: ```sql -SELECT * FROM images TABLESAMPLE BERNOULLI (10) -- rows -SELECT * FROM images TABLESAMPLE SYSTEM (10) -- fragments +SELECT row_id, doc_id FROM curation_documents TABLESAMPLE (50); +SELECT row_id, doc_id FROM curation_documents TABLESAMPLE (3 ROWS); ``` -`BERNOULLI` considers each row independently and reads across the whole table. -`SYSTEM` picks whole Lance fragments and takes the rows in them — far less I/O, -because it skips files rather than rows. +A percentage sample does not guarantee an exact number of rows. Use a count +when the desired sample size is the main constraint, and inspect the result +rather than assuming it contains every category you need. -The trade is uniformity. A fragment sample is only as representative as your -write order. If rows arrived grouped by source, by date, or by label — which is -usual — then a fragment sample inherits that grouping. +`TABLESAMPLE` requires a table that is not WAL-backed. See the +[sampling reference](/sql/dql/sampling#refused) for this restriction; do not +change a production table's write configuration just to try an example. - -Reach for `SYSTEM` when you want a cheap look, and `BERNOULLI` when the number -you compute has to be defensible. Omitting the method gives `BERNOULLI`. - +## Sampling methods {#methods} -## A fixed evaluation set {#fixed} +Choose a method according to how records should be selected: -Without a seed, every execution resamples. That is fine for exploring and wrong -for evaluation — a metric computed on different rows each run is not comparable -with itself. +| Method | Selection | When to use it | +| --- | --- | --- | +| `BERNOULLI` (default) | Select individual rows | When records should be sampled independently of their storage fragments | +| `SYSTEM` | Select whole Lance fragments | When reducing reads matters and grouping by fragment is acceptable | ```sql -SELECT * FROM images TABLESAMPLE (10) REPEATABLE (42) +SELECT row_id, doc_id +FROM curation_documents TABLESAMPLE BERNOULLI (50); + +SELECT row_id, doc_id +FROM curation_documents TABLESAMPLE SYSTEM (50); ``` -The same seed against the same data gives the same rows, which is what makes a -sample usable as a held-out set. If the underlying data changes, so does the -draw — when you need the set frozen against that too, materialize it: +If write order groups records by label, date or source, a fragment sample can +reflect that grouping. On a tiny table stored in one fragment, it may include +all records or none. Row sampling also needs an adequate sample size to +represent rare groups; check that the resulting sample covers the groups you need. + +## Repeatable samples {#fixed} + +A seed requests a repeatable draw: ```sql -CREATE TABLE eval_set AS -SELECT * FROM images TABLESAMPLE (10) REPEATABLE (42) +SELECT row_id, doc_id +FROM curation_documents TABLESAMPLE (50) REPEATABLE (42); ``` -A [materialized view](/management/materialized-view) is the other option when -you want it to track the source on a refresh rather than stay frozen. - -## Sampling both sides of a join {#joins} +Keep the source data and sampling configuration fixed when comparing draws. +For a lasting evaluation set, save the selected records rather than relying +on a seed to preserve membership as the source changes: ```sql -SELECT * FROM a TABLESAMPLE (10) JOIN b TABLESAMPLE (10) ON a.id = b.id +CREATE TABLE curation_eval AS +SELECT * FROM curation_documents TABLESAMPLE (50) REPEATABLE (42); ``` -Each side samples independently, so a row survives only if both of its sides -were drawn. Sampling 10% of each side does **not** give 10% of the join — it -gives roughly 1%. Sample once, after the join, unless you specifically want the -smaller result. +Create this output once with a fresh name and keep it unchanged. Record the +source version and selection policy. Sampling records independently can place +revisions of the same document into different splits; decide the +[grouping policy](/curation/windowing#splits) before using the result for +training or evaluation. -## Shuffling for training {#training} +## Sampling with joins {#joins} -For a training loop rather than a query, the SDKs read a table in a shuffled -order without SQL at all: +Sampling both input tables independently changes which pairs can match. +For a one-to-one match, independent 10% row samples retain a given pair with +about 1% probability, not 10%. -```python -from lancedb.permutation import Permutation +To sample documents while retaining their available labels, sample only the +document table: -permutation = Permutation.identity(table) -dataloader = torch.utils.data.DataLoader(permutation) +```sql +SELECT curation_documents.row_id, curation_documents.doc_id, l.label +FROM curation_documents TABLESAMPLE (50) +JOIN curation_labels AS l ON l.doc_id = curation_documents.doc_id; ``` -This is the right tool for epoch-level shuffling, where you want every row -eventually and only the order varies. `TABLESAMPLE` is for when you want fewer -rows. See [Training](/training/data-loading). +This samples source records, not distinct documents. If you need to sample the +result of a join or deduplication, first save that result as a table suitable +for sampling. `TABLESAMPLE` cannot be applied directly to a subquery result. -## Reference {#reference} +## Sample without SQL {#training} -The full clause, its options and what it refuses are in -[Sampling](/sql/dql/sampling). +The Python SDK provides a permutation builder for selecting and ordering rows +in OSS and Enterprise. A split selects a subset; a shuffle changes the order +in which records are read. `Permutation.identity(table)` retains stored order. + + + +Install the Python SDK with `pip install lancedb`, save the program as +`sample_dataset.py`, and run `python sample_dataset.py`. It creates a temporary +local database with ten rows and reads a random split containing three rows. +The temporary database is removed when the program finishes. + + +{PySampleDataset} + + +The program prints three distinct IDs. It uses a fixed seed, but does not +promise the same IDs across SDK versions or changes to the source dataset. + + + +For shuffled training reads, worker configuration and data loaders, see +[Loading data for model training](/training/data-loading). + +## Related guides {#reference} + +Count the sampled rows and [inspect their distribution](/curation/aggregate) +before using them. Preserve the selected data when the exact membership matters. +See [SQL sampling](/sql/dql/sampling) for syntax and restrictions. diff --git a/docs/web/curation/windowing.mdx b/docs/web/curation/windowing.mdx index fbb3cbbf2..fa84ed5fe 100644 --- a/docs/web/curation/windowing.mdx +++ b/docs/web/curation/windowing.mdx @@ -1,124 +1,162 @@ --- title: "Ranking and deduplication" sidebarTitle: "Windowing" -description: "Picking one row per thing, ranking within a group, and splitting a dataset." +description: "Choose records with explicit ranking rules and deterministic tie handling." icon: chart-simple keywords: ["window", "deduplication", "row_number", "rank", "train test split", "per class"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL window functions. --- -Enterprise Window functions are SQL, and -[SQL needs a deployment](/basics/sql#sql-is-remote). +Window functions calculate values across related rows while keeping each row +in the result. Use them to rank documents, keep one record per item, or compare +a record with the one before it. -A window function computes across a set of related rows and returns a value -**per row** — nothing is collapsed. That is what makes it the tool for "which of -these do I keep", where an aggregate would throw away the rows you are choosing -between. +## Example data {#example-data} -## One row per thing {#dedup} +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. -The most common curation query there is. You have several rows per document — -revisions, re-uploads, repeated annotations — and want one. + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. + +```sql +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); +``` + +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. + + + +## Keep one record per document {#dedup} + +`row_number()` numbers rows within each group. `PARTITION BY` defines the group, +and `ORDER BY` determines which record comes first. This query selects the +latest version of each document, using `row_id` to break ties: ```sql WITH ranked AS ( - SELECT *, - row_number() OVER (PARTITION BY doc_id ORDER BY version DESC) AS rn - FROM revisions + SELECT *, row_number() OVER ( + PARTITION BY doc_id ORDER BY version DESC, row_id DESC + ) AS rn + FROM curation_documents ) -SELECT * FROM ranked WHERE rn = 1 +SELECT row_id, doc_id, version, caption +FROM ranked +WHERE rn = 1 +ORDER BY doc_id; ``` -`PARTITION BY` says what "per thing" means; `ORDER BY` says which one wins. The -outer query is not optional — a window function cannot appear in `WHERE`, -because windows are computed after filtering, so the predicate would have -nothing to read. +This returns row IDs **2, 3, 5 and 6**. For document 30, rows 4 and 5 have the +same version; the example deliberately chooses the larger `row_id`. +Without a complete ordering, the selected row among ties is not determined by +the query. - -Swap `row_number()` for `rank()` when ties should both survive. `row_number()` -always picks exactly one; `rank()` gives tied rows the same rank, so `rank = 1` -can return several. - +Use `rank()` instead of `row_number()` when tied records should all survive, +and order only by the fields that define a tie. Adding a unique tie-breaker +would make every rank distinct. The outer query filters the calculated rank +because window functions are evaluated after the inner query's `WHERE` clause. -## A fixed number per class {#per-group} +## Select the highest-scoring records per class {#per-group} -Class imbalance found by [profiling](/curation/aggregate#balance) is fixed here: -rank within each class, then take the top N. +Deduplicate first, then rank the labeled documents within each class: ```sql -WITH ranked AS ( - SELECT *, row_number() OVER (PARTITION BY label ORDER BY score DESC) AS rn - FROM images +WITH latest AS ( + SELECT *, row_number() OVER ( + PARTITION BY doc_id ORDER BY version DESC, row_id DESC + ) AS version_rank + FROM curation_documents +), ranked AS ( + SELECT d.doc_id, l.label, d.score, + row_number() OVER ( + PARTITION BY l.label ORDER BY d.score DESC, d.doc_id + ) AS class_rank + FROM latest AS d + JOIN curation_labels AS l ON l.doc_id = d.doc_id + WHERE d.version_rank = 1 ) -SELECT * FROM ranked WHERE rn <= 1000 +SELECT doc_id, label, score +FROM ranked +WHERE class_rank <= 1 +ORDER BY label; ``` -That gives at most 1,000 rows per label, taking the highest-scoring ones. Order -by something arbitrary but stable if you want an unbiased draw instead, or -combine with [sampling](/curation/sampling) to randomize within each class. +The result is **document 10 for cat and document 20 for dog**. Change the cap +from `1` to the maximum number you want per class. A class with fewer records +cannot supply the full cap. -## Splitting a dataset {#splits} +This selects by score, not at random. For a random subset, use +[sampling](/curation/sampling) instead of ordering by score or ID. -`ntile` divides each partition into equal buckets, which is a train/test split -that stays balanced per class: +## Training and evaluation splits {#splits} + +Ranking can divide ordered records into groups, but ordering by ID does not +make those groups random or representative. `ntile()` produces approximately +equal-sized buckets within a partition; small classes may not populate every +bucket. + +Choose the split policy for the task. Keep related records, such as revisions +of one document or frames from one video, in the same split to avoid leakage. +A time-based evaluation may instead require a cutoff date. Save the final +assignments or resulting datasets so later updates do not silently change the +evaluation population. + +## Compare neighboring records {#offset} + +`lag()` and `lead()` read neighboring rows in the specified order. For example, +inspect how scores change across a document's records: ```sql -WITH split AS ( - SELECT *, ntile(10) OVER (PARTITION BY label ORDER BY id) AS bucket - FROM images -) -SELECT * FROM split WHERE bucket <= 8 -- train +SELECT doc_id, row_id, score, + lag(score) OVER ( + PARTITION BY doc_id ORDER BY version, row_id + ) AS previous_score +FROM curation_documents +ORDER BY doc_id, version, row_id; ``` -Buckets 9 and 10 are your validation and test sets. Because the partition is by -label, each split keeps the class proportions of the whole. +The first record in each document has no previous score. In this dataset, +row 2 follows a score of 0.80 and row 5 follows a score of 0.70. - -Order by something stable. Ordering by a value that changes — a score you -recompute, a timestamp that gets updated — means the split changes with it, and -a row can move from train to test between runs. - + +## Window query performance {#cost} -## Comparing a row to its neighbours {#offset} +Window operations can require sorting. Filter out records you do not need +before ranking, and inspect the +[query plan](/search/explain) when cost matters. -`lag` and `lead` read the previous and next row in the window, which is how you -find gaps and changes without a self-join: +For cumulative counts, sums and other running summaries, use an aggregate +with an explicit window order and frame. The +[window reference](/sql/dql/window#window) covers those expressions. -```sql -SELECT - id, - created_at, - created_at - lag(created_at) OVER (ORDER BY created_at) AS gap -FROM events -``` +## Continue with the curated data {#reference} -Useful for spotting ingestion gaps, duplicate timestamps, and runs of identical -values that suggest a broken upstream job. - -## Running totals {#running} - -Any [aggregate](/curation/aggregate) becomes a window function with `OVER`, -which is how cumulative coverage is expressed: - -```sql -SELECT - label, - count(*) AS n, - sum(count(*)) OVER (ORDER BY count(*) DESC) AS cumulative -FROM images -GROUP BY label -``` - -Reading down the `cumulative` column tells you how many classes cover most of -your data — usually far fewer than you expect. - -## Cost {#cost} - -Window functions sort within each partition, so on a large table they are among -the more expensive things you can ask for. Filter first, partition on something -selective, and check the plan with -[Explain Analyze](/search/explain). - -## Reference {#reference} - -The full window syntax, framing, and function list are in -[Window functions](/sql/dql/window). +[Profile the selected records](/curation/aggregate) and +[save the result](/curation#next) for downstream use. +See [Window functions](/sql/dql/window) for the full syntax and function list. diff --git a/docs/web/deployment/monitoring.mdx b/docs/web/deployment/monitoring.mdx index 6a66eb0ed..5ad56ce1c 100644 --- a/docs/web/deployment/monitoring.mdx +++ b/docs/web/deployment/monitoring.mdx @@ -11,10 +11,10 @@ LanceDB emits internal metrics (currently object store request counts, bytes tra The bridge is available in the Python and TypeScript SDKs. It is a thin wrapper over LanceDB's `metrics` recorder; your application supplies and configures the OpenTelemetry SDK. -This page covers LanceDB OSS instrumentation. LanceDB Enterprise clusters emit their own metrics from the server side — see [Enterprise monitoring](#enterprise) below. +This page covers LanceDB OSS instrumentation. LanceDB Enterprise clusters emit their own metrics from the server side. See [Enterprise monitoring](#enterprise) below. -## What you get {#what-you-get} +## Available metrics {#what-you-get} Once instrumented, LanceDB registers one observable instrument per metric on your `MeterProvider`. The current catalog covers the object store layer: @@ -22,8 +22,10 @@ Once instrumented, LanceDB registers one observable instrument per metric on you |--------|------|-------------| | `lance_object_store_requests_total` | Counter | Total object store requests, labelled by `operation` and `base` (store scheme). | | `lance_object_store_request_duration_seconds` | Histogram | Request latency in seconds. | -| `lance_object_store_bytes_transferred_total` | Counter | Bytes read from or written to the store. | -| `lance_object_store_retryable_responses_total` | Counter | Requests that returned a retryable error (throttles, transient failures). | +| `lance_object_store_request_bytes_total` | Counter | Bytes read from or written to the store. | +| `lance_object_store_errors_total` | Counter | Object store requests that failed. | +| `lance_object_store_retryable_responses_total` | Counter | Retryable HTTP responses (5xx, 429 and 408) from a store reached over HTTP. | +| `lance_object_store_throttle_total` | Counter | Throttle responses (429 and 503) from a store reached over HTTP. | | `lance_object_store_in_flight_requests` | Gauge | Currently outstanding object store requests. | The recorder is process-global and pull-based: your configured `MetricReader` collects on its own schedule, so there is no hot-path overhead beyond the atomic aggregation that LanceDB does anyway. @@ -37,7 +39,7 @@ The recorder is process-global and pull-based: your configured `MetricReader` co Install LanceDB with the `otel` extra to pull in the OpenTelemetry API, plus an OpenTelemetry SDK of your choice. The SDK is intentionally not bundled, so you configure it and its readers and exporters however your platform expects. ```bash -pip install "lancedb[otel]" opentelemetry-sdk +pip install "lancedb[otel]" opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc ``` Call `instrument_lancedb_metrics()` once at startup, before opening any tables. It returns `True` when the recorder is installed and instruments are registered. @@ -70,7 +72,7 @@ If you omit `meter_provider`, LanceDB uses the global provider returned by `open ## TypeScript {#typescript} -The Node SDK depends on `@opentelemetry/api` directly, so no extra install step is needed to expose the entry point. You still need an OpenTelemetry SDK to actually export. +Install the OpenTelemetry SDK and exporter in your application: ```bash npm install @opentelemetry/sdk-metrics @opentelemetry/exporter-metrics-otlp-grpc @@ -93,7 +95,7 @@ const db = await connect("s3://my-bucket/lancedb"); `instrumentLanceDbMetrics()` also accepts no arguments, in which case it uses the global provider from `@opentelemetry/api`. Calling it more than once is safe: instruments are created only on the first successful call. -## What to watch {#what-to-watch} +## Build dashboards and alerts {#what-to-watch} A few starting points for dashboards and alerts: @@ -116,7 +118,7 @@ export them with data enrichment -- no client-side instrumentation needed. [Contact support](/deployment/support) for the comprehensive metric list and help integrating them with your monitoring dashboard. -## Where to go next {#where-to-go-next} +## Related guides {#where-to-go-next} diff --git a/docs/web/deployment/storage.mdx b/docs/web/deployment/storage.mdx index fd8550ea6..4994e8f0c 100644 --- a/docs/web/deployment/storage.mdx +++ b/docs/web/deployment/storage.mdx @@ -1,11 +1,24 @@ --- -title: "Storage" -sidebarTitle: "Storage" -description: "Choosing and configuring where LanceDB stores its data -- how you deploy an embedded database." +title: "Embedded storage configuration" +sidebarTitle: "Embedded Storage" +description: "Choose and configure local disk or object storage for tables accessed by the embedded LanceDB OSS library." icon: database keywords: ["storage", "s3", "gcs", "azure", "object store", "configuration", "deploy"] +availability: + oss: available + enterprise: unavailable + summary: >- + The embedded library chooses and configures its storage backend from the + connection URI and `storage_options`. In LanceDB Enterprise, storage is part of + the deployment's own configuration instead. --- +This page is about the embedded library, where your connection chooses and +configures the storage backend. A LanceDB Enterprise deployment sets up its own +storage when it is deployed, so a `db://` connection passes none of these +options except an Azure storage account name -- see +[Deployment](/enterprise/deployment) for how a deployment is set up. + LanceDB's storage layer is built on modular, disk-first components. That design makes it flexible enough to run across local NVMe, EBS, EFS, and any object store that exposes an S3-compatible API. It also supports region-specific backends such as [Tencent COS](/storage/configuration#tencent-cos) and cache-acceleration layers such as [GooseFS](/storage/configuration#goosefs). Choosing a backend is a balance between latency, scalability, cost, and operational complexity. Use this guide to pick the right fit for your workload. @@ -176,9 +189,9 @@ Table-level `storage_options` inherit every key from the connection and override -**Inspect the effective options** +**Inspect a table's own options** -On `AsyncTable`, `await table.initial_storage_options()` returns the options the table was opened with, and `await table.latest_storage_options()` returns the current options after any provider-driven refresh. The deprecated `table.storage_options()` method will be removed in a future release. +On `AsyncTable`, `await table.initial_storage_options()` returns the options passed when the table was opened, and `await table.latest_storage_options()` returns them after any provider-driven refresh. Neither includes what the table inherits from its connection's `storage_options`, which still apply. The deprecated `table.storage_options()` method will be removed in a future release. #### General object store options {#general-object-store-options} diff --git a/docs/web/deployment/support.mdx b/docs/web/deployment/support.mdx index f758cab00..5e35d9131 100644 --- a/docs/web/deployment/support.mdx +++ b/docs/web/deployment/support.mdx @@ -9,11 +9,10 @@ keywords: ["support", "help", "discord", "community"] ## LanceDB OSS {#oss} For a quick question, the -[Discord community](https://discord.gg/AUEWnJ7Txb) is fastest -- ask under the -relevant language channel for Python, TypeScript, or Rust, and you reach both -the community and the engineering team. +[Discord community](https://discord.gg/AUEWnJ7Txb) has channels for +Python, TypeScript and Rust where you can ask the community for help. -For anything worth keeping searchable for the next person, use GitHub instead: +Use GitHub for discussions and issues that others can search and follow: | | | | --- | --- | diff --git a/docs/web/docs.json b/docs/web/docs.json index 38c2bc6cd..a8fb44007 100644 --- a/docs/web/docs.json +++ b/docs/web/docs.json @@ -63,10 +63,6 @@ "source": "/feature-engineering/computed-columns", "destination": "/feature-engineering/add-computed-column" }, - { - "source": "/feature-engineering/functions", - "destination": "/management/function" - }, { "source": "/tables/create", "destination": "/ingestion" @@ -332,8 +328,7 @@ "management/table", "tables/branching", "management/materialized-view", - "management/view", - "management/function" + "management/view" ] }, { @@ -383,7 +378,6 @@ "sql/ddl/view", "sql/ddl/materialized-view", "sql/ddl/index", - "sql/ddl/function", "sql/ddl/job" ], "icon": "sitemap" diff --git a/docs/web/embedding/index.mdx b/docs/web/embedding/index.mdx index e3568604b..9f1531c05 100644 --- a/docs/web/embedding/index.mdx +++ b/docs/web/embedding/index.mdx @@ -3,6 +3,13 @@ title: "Managing Embeddings" sidebarTitle: "Managing Embeddings" description: "Use the embedding API in LanceDB -- registry, functions, schemas, and multi-language SDK support." icon: "bars" +availability: + oss: available + enterprise: available + summary: >- + Embedding functions run in your process, whether the table is local or on a + deployment, which stores their configuration but never runs them. Python + and TypeScript also embed query text; in Rust you embed queries yourself. --- import { @@ -30,8 +37,9 @@ the data, so vectors that are close to each other are considered similar. LanceDB provides an embedding function registry in OSS as well as its Enterprise versions ([see below](#embeddings-in-lancedb-enterprise)) -that automatically generates vector embeddings during data ingestion. Automatic query-time embedding -generation is available in LanceDB OSS, with SDK-specific query ergonomics. The API abstracts +that automatically generates vector embeddings during data ingestion. Query text is embedded +automatically too, in the client, for local and Enterprise tables alike, with SDK-specific query +ergonomics. The API abstracts embedding generation, allowing you to focus on your application logic. ## Embedding Registry {#embedding-registry} @@ -41,6 +49,9 @@ Once configured, the embedding function will automatically generate embeddings w into the table. Query-time behavior depends on SDK: Python/TypeScript can query with text directly, while Rust examples typically compute query embeddings explicitly before vector search. +In the Rust examples, replace `your-openai-api-key` with your OpenAI API key. In +application code, load credentials from environment variables or a secrets manager. + {PyOpenaiEmbeddings} diff --git a/docs/web/enterprise/authentication.mdx b/docs/web/enterprise/authentication.mdx index f41603fdf..4d04120fc 100644 --- a/docs/web/enterprise/authentication.mdx +++ b/docs/web/enterprise/authentication.mdx @@ -4,14 +4,15 @@ sidebarTitle: "Authentication" description: "Authentication modes for LanceDB Enterprise: API keys and OAuth credentials." icon: key keywords: ["authentication", "oauth", "api key", "azure managed identity", "client credentials"] +availability: + oss: unavailable + enterprise: available + summary: >- + Requests to a deployment carry an API key or an OAuth token, whichever it is + configured to accept. Embedded LanceDB has no server to sign in to; see + [Architecture](/architecture). --- -Enterprise - -This page describes a LanceDB Enterprise capability. The embedded form -of LanceDB has no server to provide it -- see -[Architecture](/architecture) for what differs between the two. - A deployment authenticates every request before [authorization](/governance/authorization) decides what it may do, against a `db://` remote table, in one of two ways: @@ -50,6 +51,10 @@ from an OIDC issuer and attach them to every request. Token acquisition, caching, and refresh are handled inside the client -- your application code only provides the configuration. +A deployment accepts these tokens only when its administrator has configured +it with your identity provider. Ask them for the issuer URL, client ID and +scopes to use. + In Python, OAuth is supported through `lancedb.connect_async`. The synchronous `connect` entry point continues to use API key authentication for `db://` diff --git a/docs/web/enterprise/index.mdx b/docs/web/enterprise/index.mdx index ed4d232c1..a601e850d 100644 --- a/docs/web/enterprise/index.mdx +++ b/docs/web/enterprise/index.mdx @@ -21,9 +21,13 @@ What it adds over the embedded form: | | | | --- | --- | | [SQL](/sql) | an Arrow Flight SQL endpoint | -| [Functions](/management/function) | Python running on workers | | [Jobs](/basics/jobs) | server-side work with ids, status and events | | [Authorization](/governance/authorization) | per-object grants | | Scale | a cluster rather than your process | +{/* +Draft for the Functions release. Add this row to the table above, after SQL: +| [Functions](/management/function) | Python running on workers | +*/} + See [Architecture](/architecture) for the two shapes side by side. diff --git a/docs/web/feature-engineering/add-computed-column.mdx b/docs/web/feature-engineering/add-computed-column.mdx index 571797d4e..331918715 100644 --- a/docs/web/feature-engineering/add-computed-column.mdx +++ b/docs/web/feature-engineering/add-computed-column.mdx @@ -4,13 +4,18 @@ sidebarTitle: "Computed Columns" description: "Declare a column from a SQL expression. Nothing is evaluated until you refresh it." icon: table-columns keywords: ["computed columns", "add_columns", "transforms", "schema"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support computed columns. Enterprise requires the deployment to enable them; the embedded library evaluates them in your process. --- A computed column is declared with an expression and starts with no values. Declaring it is cheap on any table size, because nothing is evaluated until [a refresh](/feature-engineering/refresh-column) asks for it. -## Declaring one {#declare} +## Add a computed column {#declare} ```python import lancedb @@ -26,14 +31,17 @@ That commits a new table version with the column present and empty. It is an ordinary table version, so the usual [versioning](/management/table#versioning) applies -- including that you can travel back to before the column existed. -## `computed` and `transforms` are different {#computed-vs-transforms} +## Compute now or on refresh {#computed-vs-transforms} `add_columns` takes either, and the difference matters: - **SQL expressions in `transforms`** are evaluated *now*, for every row, as part of the call. Good for a small table, or a column you want populated - immediately. A `FunctionApplication` passed through `transforms` instead - declares empty Function-bound columns; a refresh fills them. + immediately. + {/* Draft for the Functions release: + A `FunctionApplication` passed through `transforms` instead + declares empty Function-bound columns, which a refresh fills; that needs a + deployment with Functions enabled, and the embedded library refuses it. */} - **`computed`** stores the expression and evaluates nothing. The column is committed empty and filled later by [`refresh_column`](/feature-engineering/refresh-column). @@ -48,8 +56,8 @@ the table, including another computed column: ```python table.add_columns(computed={ - "power_score": "(stats.magic + stats.strength) / 2.0", - "is_strong": "power_score > 3.0", + "power_score": "cast(x as double) / 2.0", + "is_strong": "power_score > 0.5", }) ``` @@ -59,12 +67,22 @@ declared earlier in the same mapping, but not one declared after it. The functions available are the ones the [SQL dialect](/sql/functions) documents. -## Its inputs become load-bearing {#input-constraints} +A literal in the expression has to fit the column it meets: `x / 2.0` is +refused when `x` is an integer column, so write `cast(x as double) / 2.0`. + + +In LanceDB Enterprise, keep the expression to top-level columns. A column +refresh can fail on an expression that reads a field inside a struct, such as +`stats.magic + stats.strength`, which the embedded library computes. Try a new +expression on a small table first. + + +## Change columns used by an expression {#input-constraints} While a computed column's declaration references another column, that column -cannot be renamed, retyped, or dropped. The declaration is part of the schema, -and LanceDB will not let you invalidate it out from under itself. +cannot be renamed, retyped, or dropped. Drop the computed column first +if you need to change one of its inputs. This is worth knowing before deriving a column from something you expect to @@ -74,18 +92,33 @@ reshape later. Dropping the computed column releases the constraint. Enterprise +When the value cannot be expressed in SQL -- an embedding, a model call, an +image decode -- [Geneva](/geneva), the feature engineering package for LanceDB +Enterprise, fills the column with a Python function instead. + +{/* +Draft for the Functions release. Replaces the paragraph above: + When the value cannot be expressed in SQL -- an embedding, a model call, an image decode -- the column is filled by a registered [function](/management/function) instead of an expression. Declaring it looks -the same; what differs is what fills it. +the same; what differs is what fills it. It needs a deployment with Functions +enabled. +*/} -## What to read next {#next} +## Related guides {#next} Fill the rows that lack values. + + Fill a column with Python on LanceDB Enterprise. + + {/* + Draft for the Functions release. Replaces the Geneva card above: Fill a column with Python. + */} diff --git a/docs/web/feature-engineering/data-evolution.mdx b/docs/web/feature-engineering/data-evolution.mdx index 5f50bdcbb..f70612cf9 100644 --- a/docs/web/feature-engineering/data-evolution.mdx +++ b/docs/web/feature-engineering/data-evolution.mdx @@ -4,6 +4,12 @@ sidebarTitle: "Data Evolution" description: "Derive new columns from data already in the table, without rewriting what is already there." icon: wand-magic-sparkles keywords: ["data evolution", "derived columns", "add_columns", "zero copy", "feature engineering"] +availability: + oss: available + enterprise: available + summary: >- + Adding columns computed from SQL expressions works in the embedded library + and in LanceDB Enterprise. --- import { @@ -18,20 +24,23 @@ import { RsAddFeatureColumnsSql as RsAddFeatureColumnsSql, } from '/snippets/tables.mdx'; -Data evolution is adding **values** derived from what a table already holds. -[Schema evolution](/management/table#schema) changes the table's shape; this fills the -shape with something new. +Add derived columns to calculate features from data already in a table. +For example, compute a product score from existing fields without reading +and rewriting the table in your application. -The reason it is cheap is the format. Adding a derived column writes only that -column -- the existing data, which is the expensive part on a multimodal table, -is never rewritten. See [why that works](/basics/lance#evolution). +Lance can write the new columns without rewriting the existing columns. +This is useful when the table also contains large images, video or embeddings. +See [Lance format](/basics/lance) for the storage model. -In SQL this is [`ALTER TABLE ... ADD COLUMN ... AS`](/sql/ddl/table#computed-column). +In SQL, on an Enterprise cluster that enables computed columns, this is [`ALTER TABLE ... ADD COLUMN ... AS`](/sql/ddl/table#computed-column). ## Set up the example table {#set-up-the-example-table} +The examples use `db`, a connection opened as in +[Connect to LanceDB](/basics/connect), and create a small product table in it: + {PySchemaAddSetup} @@ -96,16 +105,24 @@ If your transformation cannot be expressed in SQL, compute the values outside ## When SQL is not enough {#beyond-sql} -The expressions above are evaluated once, as part of the call. Two other shapes -exist when that does not fit: +The expressions above are evaluated once, as part of the call. For other workflows: - A [computed column](/feature-engineering/add-computed-column) declares the expression and leaves the column empty, so you choose when the work happens. - That is the right default on a large table. + Use it when you want to schedule the computation separately. +- On LanceDB Enterprise, [Geneva](/geneva) fills a column with Python, for + values SQL cannot express -- an embedding, a model call, an image decode. + +{/* +Draft for the Functions release. Replaces the Geneva item in the list above: - A [function](/management/function) fills a column with Python, for values SQL cannot express -- an embedding, a model call, an image decode. +*/} -## What to read next {#next} +A deployment has to enable computed columns before you use them. +{/* Draft for the Functions release: Functions need enabling there too. */} + +## Related guides {#next} diff --git a/docs/web/feature-engineering/index.mdx b/docs/web/feature-engineering/index.mdx index 4b8200d40..7fe699648 100644 --- a/docs/web/feature-engineering/index.mdx +++ b/docs/web/feature-engineering/index.mdx @@ -3,52 +3,90 @@ title: "Feature engineering" sidebarTitle: "Overview" description: "Derive columns and tables from data already in the lakehouse, without moving it somewhere else to compute." icon: layer-group -keywords: ["feature engineering", "computed columns", "materialized views", "functions", "lakehouse", "jobs"] +keywords: ["feature engineering", "computed columns", "materialized views", "lakehouse", "jobs"] --- -Feature engineering happens where the data already is. Rather than exporting a -table, computing somewhere else and writing the result back, you declare what a -column or a table should contain and let LanceDB fill it. +Use feature engineering to turn raw data into features for search, analysis +and model training. LanceDB stores derived columns and materialized views +alongside the source data, so you can reuse the results across those workflows. + +[Materialized views](/feature-engineering/materialized-views) are ordinary +[catalog objects](/basics/catalog). Create them through the SDK or SQL, +then refresh the derived data when its inputs change. + +{/* +Draft for the Functions release. Replaces the paragraph above: [Functions](/management/function) and [materialized views](/feature-engineering/materialized-views) are ordinary -[catalog objects](/basics/catalog), alongside tables and indexes -- created, -versioned, and refreshed like anything else, not a separate system bolted -alongside. +[catalog objects](/basics/catalog). Create them through the SDK or SQL, +then refresh the derived data when its inputs change. +*/} -Every operation here has a SQL form -- see [DDL](/sql/ddl/table#alter) to declare a column and [REFRESH](/sql/dml/refresh) to fill it. +In LanceDB Enterprise, every operation here also has a SQL form -- see [DDL](/sql/ddl/table#alter) to declare a column and [REFRESH](/sql/dml/refresh) to fill it. -## The shape everything shares {#the-shape} +## Computed columns and materialized views {#the-shape} -Every mechanism here separates **declaring** from **filling**, and that -separation is the point: +Computed columns and materialized views both keep derived data in the +lakehouse, but they fill it differently, and the difference decides what a +refresh costs: -- Declaring is cheap and immediate. It records intent in the schema and commits - a new table version, with no data written. Declaring a column on a - billion-row table costs what declaring one on an empty table costs. -- Filling is a separate call you control, can retry, and -- against a deployment - -- can hand to the server as a [job](/basics/jobs) rather than waiting on. +| | Computed column | Materialized view | +| --- | --- | --- | +| Declaring or creating it | records the expression; no values are written | runs the query and stores its rows, unless created empty | +| A refresh | fills rows with no value and recomputes rows whose inputs changed | applies source changes or rebuilds the view, depending on the query and SDK version | +| A refresh when nothing changed | fills nothing | can be a no-op when incremental refresh is supported | +| In LanceDB Enterprise | runs as a [job](/basics/jobs) | runs as a server-side materialization job | -**Filling does not redo current work.** A refresh gives values to rows that -lack them and recomputes rows whose inputs changed since they were computed. It +### Computed columns fill incrementally {#incremental} + +A [computed column](/feature-engineering/add-computed-column) separates +**declaring** from **filling**: + +- Declaring is cheap and immediate. It records the expression in the schema + and commits a new table version, with no data written. It does not evaluate the expression over existing rows. +- Fill the column with a separate refresh call that you can retry. In LanceDB + Enterprise, submit the refresh as a [job](/basics/jobs) and track its progress. + +**Refresh new and changed rows.** A +[refresh](/feature-engineering/refresh-column) gives values to rows that lack +them and recomputes rows whose inputs changed since they were computed. It leaves the rest alone, so calling it twice is not twice the work, and appending rows means the next refresh handles only the new ones. -The corollary is worth stating plainly: **a filled value catches up with its -inputs only when you refresh.** If a column derives from `price` and a row's +**Computed values update when you refresh.** If a column derives from `price` and a row's `price` is later updated, the derived value keeps its old value until the next -refresh recomputes it. That is a deliberate trade -- it is what makes refreshes -idempotent and cheap -- but it will surprise you if you expect a spreadsheet. +refresh recomputes it. Schedule refreshes after changing the source data when consumers need +up-to-date features. -## Which one to reach for {#which-one} +### Materialized views store query results {#views} + +A [materialized view](/feature-engineering/materialized-views) has no separate +declaration step: creating one runs its query and stores the result, so the view +is ready to read. A refresh updates those stored results. Python SDK 0.40.0 can apply +changes incrementally for supported queries; other queries require a full rebuild. +See [refresh behavior](/feature-engineering/materialized-views#incremental) for +version-specific details. + +Enterprise runs materialization on remote compute. Use the SDK and refresh +options supported by your server version; see +[Enterprise refreshes](/feature-engineering/materialized-views#availability). + +## Choose a feature engineering workflow {#which-one} | | What it produces | Filled by | Available | | --- | --- | --- | --- | -| [Computed column](/feature-engineering/add-computed-column) | a new column on an existing table | a SQL expression | everywhere | -| [Materialized view](/feature-engineering/materialized-views) | a new table derived by a query | a refresh | everywhere | +| [Computed column](/feature-engineering/add-computed-column) | a new column on an existing table | a SQL expression | OSS and Enterprise | +| [Materialized view](/feature-engineering/materialized-views) | a new table derived by a query | a refresh | OSS and Enterprise | +| [Geneva](/geneva) | columns filled by Python | the `geneva` package | Enterprise | + +{/* +Draft for the Functions release. Replace the Geneva row of the table above with: | [Function](/management/function) | a computed column filled by Python | registered code | Enterprise | +| [Geneva](/geneva) | columns filled by Python, for existing pipelines | the `geneva` package | Enterprise | +*/} **Computed columns** are the common case: a column declared with a SQL expression, filled by [a refresh](/feature-engineering/refresh-column). @@ -57,23 +95,34 @@ expression, filled by [a refresh](/feature-engineering/refresh-column). aggregated view of a source -- and bring it up to date on each refresh. The result is an ordinary table: queryable, indexable, searchable. +**Geneva** is LanceDB Enterprise's Python package for work SQL cannot express: +embedding a column, calling a model, decoding an image. It defines features as +Python functions and backfills them as distributed jobs. + +{/* +Draft for the Functions release. Replaces the Geneva paragraph above: + **Functions** run Python where the data is, for work SQL cannot express: embedding a column, calling a model, decoding an image. Registered once, -versioned, then applied as a column. +versioned, then applied as a column. They are the way to do that on a +deployment; [Geneva](/geneva) remains available for existing pipelines. +*/} -They compose. A materialized view can select computed columns, and a computed -column can be filled by a registered function. +A materialized view can select computed columns. +{/* Draft for the Functions release: A computed column can be filled by a registered function. */} ## Where the work runs {#where} Enterprise -Locally, a refresh runs in your process. Against a deployment it runs across the +Locally, a refresh runs in your process. With LanceDB Enterprise it runs across the cluster, and the API reflects that: `refresh_column` raises, and -`refresh_column_async` returns a [job](/basics/jobs) handle instead. +`refresh_column_async` returns a [job](/basics/jobs) handle instead. Computed +columns are off by default in LanceDB Enterprise, and its administrator has to enable +them before a column can be declared there. ```python -job = table.refresh_column_async("embedding") +job = table.refresh_column_async("power_score") result = job.wait() result.rows_assigned ``` @@ -81,24 +130,28 @@ result.rows_assigned A backfill over a large table can therefore be started, left, and checked on from a different process -- the connection does not have to stay open. -That relocation is the main thing to understand about Enterprise feature -engineering. The API is the same; what changes is that the code runs somewhere -else, which is also why a function must declare the environment it needs. +Enterprise manages the compute for the refresh. You still control when to +submit it; declaring a computed column does not schedule recurring refreshes. +{/* Draft for the Functions release: That is also why a function must declare the environment it needs. */} ## Recovering from a failure {#failures} +{/* +Draft for the Functions release. + A function that raises fails the job. There is no per-row retry or skip policy in the function definition, so a function calling a flaky external service should handle that itself, inside the function body. +*/} -The alternative is to let the job fail, fix the cause, and run the refresh +When a refresh fails, fix the cause and run the refresh again. The new refresh fills what is still empty or stale. How much of the failed run was kept depends on where it ran (see [Refresh a column](/feature-engineering/refresh-column#failures)), and a retry can recompute values that were already written, so do not count on it costing only the rows that did not complete. -## Where to go next {#next} +## Related guides {#next} @@ -110,7 +163,10 @@ only the rows that did not complete. Bring a derived table up to date with its source. + {/* + Draft for the Functions release: Register Python to fill a column with. + */} diff --git a/docs/web/feature-engineering/materialized-views.mdx b/docs/web/feature-engineering/materialized-views.mdx index 7b00a6ce4..ab949adb1 100644 --- a/docs/web/feature-engineering/materialized-views.mdx +++ b/docs/web/feature-engineering/materialized-views.mdx @@ -1,9 +1,14 @@ --- title: "Refresh a materialized view" sidebarTitle: "Materialized View Refresh" -description: "A lakehouse object: a table defined by a query over another table, brought up to date by each refresh." +description: "Create a materialized view and refresh it when the source data changes." icon: table keywords: ["materialized view", "create_materialized_view", "refresh", "full refresh"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support materialized views. OSS refreshes run in your process; Enterprise runs the work on remote compute. See [Enterprise refreshes](#availability) for server compatibility requirements. --- import { @@ -11,12 +16,9 @@ import { PyMaterializedViewRefresh as MaterializedViewRefresh, } from '/snippets/materialized_view.mdx'; -A materialized view is a lakehouse object: a table whose contents are defined by -a query over another table. The query is recorded in the view's schema; -creating the view computes its rows, and a refresh brings them up to date. - -The result is an ordinary table. It can be queried, indexed, and searched, and -it appears in `table_names()` — nothing about reading it is special. +A materialized view stores the result of a query as a table. Use it to keep a +filtered or transformed dataset that you can query, index and search repeatedly. +Refresh the view after its source data changes. In SQL this is [`REFRESH MATERIALIZED VIEW`](/sql/refresh#refresh-materialized-view). @@ -36,89 +38,80 @@ reach the view through a refresh: {MaterializedViewRefresh} - -These results were verified with the development SDK 0.40.0b13, built from -lancedb main. The published 0.39.0 differs: the source must be created with -stable row ids (the `new_table_enable_stable_row_ids` storage option), -`create_materialized_view` creates the view empty and has no `with_no_data`, -and `refresh()` still accepts `full`. Its first `refresh()` writes every -matching row, two in this example. - +The example works with Python SDK 0.40.0. It creates the source with stable +row IDs, which that release requires for ordinary materialized-view sources. `select` takes column names, `(alias, expression)` pairs, or a dict of the same. Omitting it selects every source column, expanded against the schema as it was at creation time. `where` filters source rows; `limit` caps the view. -## Local sources no longer need stable row ids {#stable-row-ids} +## Prepare the source table {#stable-row-ids} -A local refresh does not track which source rows the view came from, so a local -view's source does not need stable row ids (`new_table_enable_stable_row_ids`): -the example above creates its source with the defaults. Earlier development -builds and the published 0.39.0 required them, and a deployment may still -require them; see [Against a deployment](#availability). +In Python SDK 0.40.0, create the source with +`storage_options={"new_table_enable_stable_row_ids": "true"}`, as the example +above does. Stable row IDs let the view track source rows across updates and +compaction. Set this option when creating the table; it cannot be added later. -The source is still checked, when the view is created and on every refresh: one -with an LSM write spec, or with retained write-ahead-log rows not yet compacted, -is refused. +A source with an LSM write specification or retained, uncompacted write-ahead-log +rows is not accepted by the local materialized-view implementation. -## Every local refresh rebuilds the view {#incremental} +## Refresh behavior {#incremental} -Locally, `refresh()` runs the view's query against the source and replaces -everything the view holds with the result, in one commit. It does not reconcile -changes: `rows_written` is the size of the whole result — two after adding Bea, -not one — and a refresh with nothing new to read still rewrites the view and -commits a new table version. +With Python SDK 0.40.0, a refresh applies source changes incrementally when the +query supports it, and rebuilds the view otherwise. In the example, adding Bea +writes one new view row. Refreshing again without source changes writes none. +Use `refresh(full=True)` to force a rebuild. -`source_version=N` refreshes against that source version instead of the latest, -which is how a view reproduces a known point in time. There is no `full` -argument; every local refresh is a full one. +`rows_written` counts rows written by that refresh, not all rows in the view. +Use `view.table.count_rows()` to get the total. Pass `source_version=N` to +refresh against a particular source version instead of the latest. -## Against a deployment {#availability} + +Refresh behavior has changed between SDK versions. The earlier 0.40.0b13 +development build rebuilt every local view on refresh and did not accept +`full`; 0.39.0 created views empty. Use the reference for your installed SDK +when maintaining an existing pipeline. + + +## Enterprise refreshes {#availability} Enterprise -`create_materialized_view` and `refresh` also work on a `db://` connection. -There the SDK sends them to the deployment as SQL -- `CREATE MATERIALIZED VIEW` -and `REFRESH MATERIALIZED VIEW` -- and the work runs on the cluster rather than -in your process. `source_version` is for local views only; the SDK refuses it -on a remote refresh. +Enterprise runs materialized-view creation and refresh on remote compute. +For SQL, see [`CREATE MATERIALIZED VIEW`](/sql/ddl/materialized-view) and +[`REFRESH MATERIALIZED VIEW`](/sql/dml/refresh#refresh-materialized-view). -Materialized views are off by default on a deployment and have to be enabled -there. How a deployment refreshes a view -- rebuilding the whole view, as above, -or reconciling what changed in the source -- depends on its LanceDB Enterprise -version and configuration, and so do the options -[`REFRESH MATERIALIZED VIEW`](/sql/dml/refresh#refresh-materialized-view) -accepts and whether the source table needs stable row ids. +Use an SDK version supported by your Enterprise deployment. Materialized-view +APIs and stored definitions have changed across releases; a local SDK example +alone does not establish compatibility with an older server. Before upgrading +an existing pipeline, confirm the supported refresh options and whether its +views need to be recreated with your administrator. -Stable row ids are set up on the deployment, not by the SDK: a `db://` -connection does not send `new_table_enable_stable_row_ids`, and no SDK call adds -them to an existing table. Depending on its version and configuration, a -deployment may give new tables stable row ids, may refuse a view created through -SQL -- as the SDK creates one -- over a source without them, or may convert an -existing source when a view is created through its REST API. Check with the -deployment's administrator what it requires before relying on a source. +Stable row IDs and feature configuration for Enterprise sources are managed on +the server. The source-table storage option in the local example does not +configure the Enterprise deployment, and Python SDK 0.40.0's `create_table` on a +`db://` connection does not accept `storage_options`: leave it out, and confirm +with your administrator that the deployment gives new tables stable row IDs. -`SHOW MATERIALIZED VIEWS` lists them, which is cheaper than -`list_materialized_views` -- that method finds views by reading every table's -schema, so it costs one open per table. +`list_materialized_views` requests the server's view listing. On a local +database it reads table schemas to find views. ## Working with the view {#working-with-the-view} `view.table` is the underlying table, for querying, indexing, or search. -`view.definition` is the recorded query — the select list, predicate, and -limit — which is how you check what a view actually contains without inferring -it from the rows. Both are properties; on an async view, the definition is read +`view.definition` is the recorded query: the select list, predicate, and +limit. Use it to inspect the query without inferring it from the rows. Both are properties; on an async view, the definition is read with `await view.definition()`. An existing view is reopened with `db.open_materialized_view(name)`. -## What to read next {#next} +## Related guides {#next} Derive a column instead of a whole table. - Every refresh commits a table version. + Inspect the table history and read a particular version. diff --git a/docs/web/feature-engineering/refresh-column.mdx b/docs/web/feature-engineering/refresh-column.mdx index d0f9f06a3..ae430e5f8 100644 --- a/docs/web/feature-engineering/refresh-column.mdx +++ b/docs/web/feature-engineering/refresh-column.mdx @@ -1,13 +1,19 @@ --- title: "Refresh a column" sidebarTitle: "Column Refresh" -description: "Fill the rows of a computed column that lack values — in process, or as a server-side job." +description: "Populate a computed column and update it when its inputs change." icon: arrows-rotate keywords: ["refresh_column", "backfill", "job", "idempotent", "REFRESH COLUMN"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support column refresh. The embedded library runs it in your process; Enterprise runs it as a server job that must be enabled for your Enterprise cluster. --- -A [declared column](/feature-engineering/add-computed-column) starts empty. A -refresh gives values to the rows that lack them. +Refresh a [computed column](/feature-engineering/add-computed-column) to fill +new rows and recompute values whose inputs changed. The example below continues +with the `doubled` column from the computed-column guide. ```python Python icon="python" @@ -37,8 +43,7 @@ The SQL form is [`REFRESH COLUMN`](/sql/refresh#refresh-column), with the same r A refresh fills rows that have no value and recomputes rows whose inputs changed since they were computed. Every other row is left alone, so a second -refresh with nothing changed does nothing. Two consequences follow, and both -are load-bearing. +refresh with nothing changed does nothing. This supports two common workflows. **Appending is cheap.** Rows added since the last refresh are filled by the next one; filled rows whose inputs are unchanged are skipped. Backfilling a @@ -63,16 +68,16 @@ REFRESH COLUMN embedding ON documents WHERE created_at > '2026-01-01' ``` -**Enterprise job type:** `computed_refresh` for a SQL-expression column, -`function_refresh` for one filled by a registered -[function](/management/function). +**Enterprise job type:** `computed_refresh` for a SQL-expression column. +{/* Draft for the Functions release: `function_refresh` for one filled by a registered [function](/management/function). */} -## As a job {#async} +## Run a refresh on Enterprise {#async} Enterprise -Against a deployment the refresh runs on the cluster. `refresh_column` raises +With an Enterprise cluster that enables computed columns, the refresh runs on the +cluster. `refresh_column` raises there; use `refresh_column_async`, which returns a [job](/basics/jobs) you can wait on or poll instead of holding a connection open: @@ -92,7 +97,7 @@ does not raise, which is what makes it safe in a loop; `wait()` raises A long backfill can therefore be started, left, and checked on from a different process. -### Resourcing the job {#resources} +### Configure workers and resources {#resources} SQL exposes the worker budget directly, which is the lever when a backfill is too slow or too expensive: @@ -111,7 +116,7 @@ rather than failing the job. What a failure keeps depends on where the refresh ran. `refresh_column` in your process writes the column in one commit, so a failure leaves the column as it -was. A refresh job on a deployment commits finished batches as it goes, so some +was. A refresh job in LanceDB Enterprise commits finished batches as it goes, so some values can remain after a failure. Either way, recover by fixing the cause and refreshing again. The new refresh @@ -121,12 +126,16 @@ that were already written -- for example when the record of the inputs they were computed from was not saved -- so do not count on a retry costing only the rows that did not complete. +{/* +Draft for the Functions release. + For a column filled by a [function](/management/function), the usual cause is the worker environment rather than the data. See [environment mismatches](/management/function#mismatch-symptoms) for the symptoms, which rarely name their cause. +*/} -## What to read next {#next} +## Related guides {#next} diff --git a/docs/web/governance/authorization.mdx b/docs/web/governance/authorization.mdx index 044db158e..9ef978c38 100644 --- a/docs/web/governance/authorization.mdx +++ b/docs/web/governance/authorization.mdx @@ -4,15 +4,13 @@ sidebarTitle: "Authorization" description: "Who may do what: privileges granted on catalog objects to principals, groups, roles, and API keys." icon: shield-halved keywords: ["authorization", "authz", "privileges", "rbac", "access control"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports catalog privileges when its administrator configures enforcement. Embedded LanceDB has no catalog authorization layer: access is whatever the object store allows. --- -Enterprise - -This page describes a LanceDB Enterprise capability. Embedded LanceDB has no -such layer -- access is whatever the object store enforces on the bucket or -directory, which is coarser and has no concept of a table. See -[Architecture](/architecture) for what differs between the two. - [Authentication](/enterprise/authentication) establishes who is calling. Authorization decides what they may do. @@ -28,15 +26,16 @@ A grant is held by one of four kinds of subject, distinguished by a prefix: | `p:` | a principal, by ID -- `P:` by name | | `g:` | a group, by ID -- `G:` by name | | `r:` | a role | -| `a:` | an API key | +| `a:` | the principal holding an API key, named by the key | Granting to a group or role rather than a principal is what keeps the entry count manageable: a principal's effective privileges are the union of what it holds directly and what it inherits. -Granting to an API key is narrower than granting to the principal that owns -it -- useful when a key is issued for one job and should not carry that -principal's full authority. +`P:`, `G:` and `a:` are ways of naming a subject when you make a request. The +entry is stored against the ID they resolve to, so a grant to `a:` belongs to +the principal holding that key and carries that principal's full authority, +not a narrower one. ## Objects {#objects} @@ -45,10 +44,28 @@ Objects are addressed by kind and path: ``` system database:db -namespace:db/ns -table:db/ns/table +namespace:db:ns +table:db:ns$table +view:db:ns$view ``` +{/* +Draft for the Functions release. Replaces the object kinds above: + +``` +system +database:db +namespace:db:ns +table:db:ns$table +view:db:ns$view +function:db:ns$function +secret:db:ns$secret +``` +*/} + +A nested namespace joins its path with `$`, as in `namespace:db:parent$child` +or `table:db:parent$child$table`. + The hierarchy is not decorative. Reaching a table requires `USAGE` on the namespace containing it and on the database containing that -- a grant on the table alone is not enough to traverse to it. @@ -57,10 +74,11 @@ table alone is not enough to traverse to it. | Privilege | Granted on | Allows | | --- | --- | --- | -| `OWNERSHIP` | any object | ownership, conveying full authority | +| `OWNERSHIP` | any object but `system` | ownership, conveying full authority | +| `USAGE` | system | listing jobs and their events, and looking up other identities | | `USAGE` | database | accessing the namespaces inside | | `USAGE` | namespace | accessing the tables inside | -| `SELECT` | table | reading rows | +| `SELECT` | table, view | reading rows | | `INSERT` | table | appending rows | | `UPDATE` | table | modifying existing rows | | `DELETE` | table | deleting rows | @@ -69,7 +87,19 @@ table alone is not enough to traverse to it. | `CREATE_TABLE` | namespace | creating a table in it | | `CREATE_MATERIALIZED_VIEW` | namespace | creating a materialized view in it | | `CREATE_VIEW` | namespace | creating a non-materialized view in it | -| `OPERATE` | system | reading server configuration | +| `OPERATE` | system | describing and controlling jobs, managing caches, and reading server configuration | + +{/* +Draft for the Functions release. Add these rows to the table above. +After the `USAGE` row for namespace: +| `USAGE` | function | binding it to a column, and refreshing a column it computes | +| `USAGE` | secret | binding and resolving it | +After the `UPDATE` row for table: +| `UPDATE` | secret | rotating it, without the right to bind or read it | +After the `CREATE_VIEW` row: +| `CREATE_FUNCTION` | namespace | registering a function in it | +| `CREATE_SECRET` | namespace | creating a secret in it | +*/} The four data privileges map onto the four [ingestion operations](/ingestion), so a principal that may append but not @@ -87,28 +117,32 @@ grant the other. A subtlety worth knowing, because it explains grants that appear to work on objects that do not exist yet. -When `USAGE` is checked on a database and **no entries exist for it at all**, -the database is assumed not to exist yet, and `CREATE_DATABASE` on `system` is -checked instead. The same applies to the default namespace: absent any -entries, the check falls back to `CREATE_NAMESPACE` on the containing -database. +When you create an object, the deployment checks the database and each +namespace that will contain it. One that **has no owner** -- because it does +not exist yet, or predates per-object authorization -- is treated as one you are +creating: the check falls back to `CREATE_DATABASE` on `system` for the +database, or to `CREATE_NAMESPACE` on its parent for a namespace, and you become +its owner. One that has an owner needs `USAGE` as usual. -So a principal holding `CREATE_DATABASE` can act on a database that has not -been created -- which is what makes creation possible at all, and why an empty -ACL set is not the same as a deny. +So a principal holding `CREATE_DATABASE` can create a table in a database that +has not been created -- which is what makes creation possible at all, and why +an empty ACL set is not the same as a deny. The fallback applies only when +creating; other operations on a database or namespace need `USAGE` on it. ## Listing grants {#listing} Entries can be listed filtered by object, by subject, or both: ``` -object = table:db/ns/documents +object = table:db:ns$documents subject = G:data-science ``` -Results paginate through a continuation token. One stored `(object, subject)` -tuple carrying several privileges expands into several entries in the -response, so the count reflects privileges rather than tuples. +Results paginate through a continuation token. A page holds at most 100 stored +`(object, subject)` tuples, and a tuple carrying several privileges expands +into several entries in the response, so a page can return more than 100 +entries: the count reflects privileges rather than tuples. Keep requesting with +the returned token until a page comes back without one. Listing by subject answers "what can this team reach"; listing by object answers "who can reach this table". Both are worth running before changing a diff --git a/docs/web/maintenance/background-jobs.mdx b/docs/web/maintenance/background-jobs.mdx index ba88b7137..151828e5d 100644 --- a/docs/web/maintenance/background-jobs.mdx +++ b/docs/web/maintenance/background-jobs.mdx @@ -1,22 +1,29 @@ --- title: "Background jobs" sidebarTitle: "Background Jobs" -description: "Jobs a deployment runs on its own — visible in SHOW JOBS, not started by a call you make." +description: "Understand the Enterprise background jobs that maintain tables, indexes and caches." icon: gears keywords: ["background jobs", "job type", "wal", "spfresh", "prewarm", "archive"] +availability: + oss: unavailable + enterprise: available + summary: >- + LanceDB Enterprise schedules background maintenance jobs according to your + cluster configuration. --- -Enterprise - -Every job documented elsewhere in these docs -- [Build & Repair](/indexing/reindexing), +The jobs documented elsewhere in these docs -- [Build & Repair](/indexing/reindexing), [Compaction](/maintenance/compaction), [Cleanup](/maintenance/cleanup), -[Column Refresh](/feature-engineering/refresh-column) -- is one you cause, -directly or by a schedule you set. This page is the rest: jobs the deployment -runs to keep itself correct and fast, which you may see named in -[`SHOW JOBS`](/sql/ddl/job#show) without having asked for any of them. +[Column Refresh](/feature-engineering/refresh-column) -- start from a call you +make or, for compaction, cleanup and index updates, from the deployment's own +maintenance. This page is the rest: jobs the deployment runs to keep itself +correct and fast, which you may see named in [`SHOW JOBS`](/sql/ddl/job#show) +without having asked for any of them. -There is no action to take on these beyond recognizing the name. None of them -has a parameter you set or a call you make. +These jobs are configured by the deployment's operators rather than through +the table SDK. If a maintenance job repeatedly fails or stops making progress, +share its job ID and errors with your Enterprise support team. Operators can +also start cache-warming jobs through the admin API. ## Write-ahead log maintenance {#wal} @@ -41,7 +48,7 @@ log from growing without bound: | --- | --- | | `prewarm_page_cache` | Loads table data into a query node's page cache ahead of a request needing it. | | `prewarm_index_cache` | The same, for index structures. | -| `startup_prewarm_idx` | Runs cache prewarming as part of a query node's own startup, rather than on a schedule. | +| `startup_prewarm_idx` | Runs cache prewarming as part of a plan executor's own startup, rather than on a schedule, when the deployment enables it. | ## Registry housekeeping {#registry} @@ -51,15 +58,16 @@ log from growing without bound: | `job_events_export` | Copies terminal jobs' event histories into a curated system table for metering. Registry-scoped, like the archive sweep. | These two are why an old job you completed weeks ago no longer shows in -`SHOW JOBS` -- it was archived, not deleted; its events survive in the exported -table. +`SHOW JOBS` -- it was archived, not deleted. Where the deployment enables the +export, its events also survive in the exported table. ## Object teardown {#teardown} | Job type | What it does | | --- | --- | -| `drop_object` | Deletes a dropped catalog object's content, then clears it from the database's cleanup backlog. Covers namespaces, views, and functions -- everything [`drop_table`](/management/table#drop) does not, since that one fences a directory with its own marker instead. | +| `drop_object` | Deletes a dropped catalog object's content, then clears it from the database's cleanup backlog. Whether a dropped table runs as this job or as [`drop_table`](/management/table#drop), which fences the table's directory with its own marker instead, depends on how the deployment stores its catalog. | -You trigger this indirectly, by dropping a [namespace](/management/namespace), -[view](/sql/ddl/view#drop), or [function](/management/function#drop) -- the teardown -itself runs as this job rather than inline with the call. +You trigger it indirectly, by dropping a table, a [view](/sql/ddl/view#drop) or +a [namespace](/management/namespace); the teardown runs as this job rather than +inline with the call. +{/* Draft for the Functions release: Dropping a [function](/management/function#drop) starts it the same way. */} diff --git a/docs/web/maintenance/cleanup.mdx b/docs/web/maintenance/cleanup.mdx index 71447a7f7..60a2f6bb0 100644 --- a/docs/web/maintenance/cleanup.mdx +++ b/docs/web/maintenance/cleanup.mdx @@ -4,6 +4,11 @@ sidebarTitle: "Cleanup" description: "Pruning old versions and the files only they referenced." icon: broom keywords: ["cleanup", "retention", "prune", "delete_unverified", "vacuum"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support cleanup. Embedded `optimize()` uses a retention window of 7 days by default; Enterprise uses configured background cleanup and retention, and does not support `optimize()`. --- Every write keeps the version before it, which is what makes @@ -14,7 +19,8 @@ those versions referenced. **Enterprise job type:** `cleanup`. Distinct from `compact` -- compaction merges files, cleanup removes them. `optimize()` in OSS runs both in one call; -Enterprise schedules them as separate jobs. +Enterprise schedules them as separate jobs, and runs cleanup only when the +deployment's configuration enables it. ## Retention {#retention} @@ -26,17 +32,43 @@ table.optimize(cleanup_older_than=timedelta(days=30)) ``` `cleanup_older_than` defaults to **7 days**. Versions older than that are -removed, along with the files only they referenced. +removed, along with the files only they referenced. In LanceDB Enterprise, the +retention period comes from its configuration instead. -The latest version is never removed, whatever you pass -- `timedelta(0)` prunes -everything except it. +The latest version is never removed, whatever you pass. `timedelta(0)` makes +every older version eligible for removal, but cleanup still keeps tagged +versions and the files branches still read, as the warning below describes. -Pruning is what makes [time travel](/management/table#versioning) finite. A -version outside the retention window is gone, and a tag or branch pointing at -it no longer resolves. +Pruning is what makes [time travel](/management/table#versioning) finite: an +eligible version outside the retention window is gone. Cleanup keeps a version, +whatever its age, when it is: -If you rely on reading old versions, set the window to cover the range you +- the **latest** version; +- a **tagged** version. + +Cleanup also keeps the files a **branch** still reads, and the version the +branch was forked from while the branch's remaining versions read the parent's +files. That does not keep the fork for good: once the branch has replaced the +data it inherited and its own older versions are pruned, cleanup prunes the +fork version like any other. To keep a version recoverable, tag it, subject to +the policy below, or keep the window wide enough to cover it. + +Under the default policy, a tag can also stop cleanup. When a tagged version is +older than the window and is not the latest, cleanup refuses to run: it raises +an error naming the tag and prunes nothing. To let it proceed, move the tag to a +version inside the window, delete the tag, or widen the window. + +A tag on the latest version does not stop cleanup. But `optimize()` compacts +before it prunes, and compacting commits a new version whenever it merges +files, so a tag on what was the latest version can become an older one by the +time cleanup runs. `optimize()` then raises after compacting, and its index +update does not run. + +A deployment's background cleanup follows the same rules. Its own background +jobs, such as compaction and index updates, also commit new versions, so a tag +on what is now the latest version can become an older one without a call from +you. If you rely on reading old versions, set the window to cover the range you need before running this, not after. @@ -46,8 +78,8 @@ Compaction writes new, merged files before the old ones are removed, so disk usage can rise before it falls. Space is reclaimed only when cleanup prunes the versions that referenced the superseded files. -If you need to reclaim space aggressively, shorten the retention window rather -than waiting for the default: +If you need to reclaim space aggressively in the embedded library, shorten the +retention window rather than waiting for the default: ```python table.optimize(cleanup_older_than=timedelta(days=1)) @@ -63,6 +95,9 @@ old, because it cannot tell them apart from a live write. table.optimize(delete_unverified=True) ``` +This option belongs to `optimize()` in the embedded library. A deployment's +cleanup jobs never set it. + `delete_unverified=True` removes them regardless of age. Only set it when you can guarantee **no other process is writing to the table** -- otherwise it can diff --git a/docs/web/maintenance/compaction.mdx b/docs/web/maintenance/compaction.mdx index b6697921c..757a344fc 100644 --- a/docs/web/maintenance/compaction.mdx +++ b/docs/web/maintenance/compaction.mdx @@ -3,18 +3,27 @@ title: "Compaction and cleanup" sidebarTitle: "Compaction" description: "Merge small files, prune old versions, and fold new rows into existing indexes." icon: broom +availability: + oss: available + enterprise: available + summary: >- + Tables are compacted in the embedded library and in LanceDB Enterprise. In the + embedded library you call `optimize()`; a deployment compacts tables in the + background, and `optimize()` is not available there. --- import { PyStorageRebuildVectorIndex } from '/snippets/doc_storage_examples.mdx'; Every write commits a new table version, and versions accumulate: small files from frequent appends, superseded files from updates and deletes, and rows added -since an index was built. `optimize()` addresses all three. +since an index was built. In the embedded library, `optimize()` addresses all +three; a deployment handles them in the background and does not support +`optimize()`. -It is modelled after `VACUUM` in PostgreSQL — routine housekeeping that a table -needs periodically rather than a repair for something broken. +Run optimization as regular table maintenance, particularly after many small +writes or a large ingest. -## What it does {#what-it-does} +## Optimize a table {#what-it-does} ```python table.optimize() @@ -60,7 +69,9 @@ and can help after a large ingest that changed the data distribution. ## When to run it {#when} -There is no background compactor: `optimize()` runs when you call it. +In the embedded library there is no background compactor: `optimize()` runs +when you call it. A deployment compacts a table automatically once enough small +fragments accumulate, unless its operator turns this off. Frequent small appends are what make it necessary, because each one commits files. A table written in a few large batches needs it rarely; one taking @@ -92,4 +103,4 @@ table.optimize() # 7-day default retention table.optimize(cleanup_older_than=timedelta(days=1)) # reclaim space sooner ``` -The best practice is to run `optimize()` after large writes or on a schedule. It also bundles incremental index updates — see [Build & Repair](/indexing/reindexing) for the breakdown. Updates also move rows out of the vector index, so they remain searchable but unindexed — rebuild the index after large update batches. +The best practice is to run `optimize()` after large writes or on a schedule. It also bundles incremental index updates. See [Build & Repair](/indexing/reindexing) for the breakdown. Updates also move rows out of the vector index, so they remain searchable but unindexed. Update or rebuild the index after large update batches. diff --git a/docs/web/management/database.mdx b/docs/web/management/database.mdx index 1f77e5b10..a4b76f195 100644 --- a/docs/web/management/database.mdx +++ b/docs/web/management/database.mdx @@ -1,97 +1,73 @@ --- title: "Database" sidebarTitle: "Database" -description: "Connecting to LanceDB: the URI forms, their options, and what each one changes." +description: "Connect to LanceDB and configure storage, credentials and read consistency." icon: database keywords: ["connect", "database", "uri", "api key", "storage options", "consistency"] +availability: + oss: available + enterprise: available + summary: >- + OSS opens tables in a local directory or object storage. Enterprise connects + to remote compute with a database URI and credentials. --- -A database is the root of a [catalog](/basics/catalog). A connection opens -exactly one, and every other object is reached through it. +import { PyConnectEnterprisePlaceholders, TsConnectEnterprisePlaceholders } from '/snippets/connection.mdx'; -Connecting is the one decision that reaches furthest: the URI you pass decides -where queries execute, which features exist, and what a table handle actually -is. +A database connection provides access to tables and namespaces in a +[catalog](/basics/catalog). Use it to create and open tables, list catalog +objects, and configure how your application reaches the data. -## The three URI forms {#uris} +## Connection URIs {#uris} + +| URI | Offering | Where queries run | +| --- | --- | --- | +| `./.lancedb` | OSS, with local storage | In your application | +| `s3://bucket/path`, `gs://bucket/path`, `az://container/path` | OSS, with object storage | In your application | +| `db://my-database` | Enterprise | On remote compute | + +[Connect to LanceDB](/basics/connect) has examples for each connection type. +A connection to object storage still uses the embedded library: your process +reads and writes the bucket directly. Enterprise runs operations remotely and +manages the supporting compute and maintenance. + +## Connect to Enterprise {#deployment} + +Replace the placeholders with the database URI, key, region and endpoint +provided by your administrator. + + + + {PyConnectEnterprisePlaceholders} + + + { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } + {TsConnectEnterprisePlaceholders} + + + +| Python option | TypeScript option | Purpose | +| --- | --- | --- | +| `api_key` | `apiKey` | Authenticate the connection | +| `region` | `region` | Select the database's region | +| `host_override` | `hostOverride` | Set the Enterprise endpoint | +| `client_config` | `clientConfig` | Configure retries, timeouts and connection pooling. A retry can [repeat an append](/ingestion/insert#performance). | + +Use your deployment's region rather than relying on the `us-east-1` default. +For application code, load credentials from environment variables or a secrets +manager. See [Authentication](/enterprise/authentication) for OAuth and +[SQL](/basics/sql) for the separate SQL connection settings. + +## Storage options {#storage-options} + +For OSS tables in object storage, `storage_options` configures credentials, +endpoints and options for new tables. This example enables stable row IDs for +tables created through the connection: ```python Python icon="python" import lancedb -db = lancedb.connect("./.lancedb") # local directory -db = lancedb.connect("s3://bucket/path") # object storage -db = lancedb.connect("db://my-database", api_key="...") # a deployment -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; - -const db = await lancedb.connect("./.lancedb"); -const db = await lancedb.connect("s3://bucket/path"); -const db = await lancedb.connect("db://my-database", { apiKey: "..." }); -``` - - -The first two are **embedded**: your process opens Lance tables directly from -storage and does the work itself. The third talks to a **deployment**. - -| | Embedded | Deployment | -| --- | --- | --- | -| Where queries run | in your process | on the server | -| Table handle | `LanceTable` | `RemoteTable` | -| [SQL](/basics/sql) | -- | yes | -| Server-side [jobs](/basics/jobs) | -- | yes | -| [Functions](/management/function) | -- | yes | -| [Authorization](/governance/authorization) | storage permissions | granted per object | - -![](/static/assets/images/overview/understanding-tables.png) - -Both handles expose the same table API -- create, open, mutate, query -- so -application code moves between them by changing the URI. What differs is where -resolution and execution happen, and therefore which operations exist at all. - -## Connecting to a deployment {#deployment} - -Enterprise - - -```python Python icon="python" -db = lancedb.connect( - "db://my-database", - api_key="...", - region="us-east-1", -) -``` - -```typescript TypeScript icon="square-js" -const db = await lancedb.connect("db://my-database", { - apiKey: "...", - region: "us-east-1", -}); -``` - - -| Option | Meaning | -| --- | --- | -| `api_key` / `apiKey` | authenticates the connection -- see [Authentication](/enterprise/authentication) | -| `region` | which region hosts the database. Defaults to `us-east-1` | -| `host_override` / `hostOverride` | point at a specific endpoint instead of the regional one | -| `sql_host_override` | the same, for the [SQL endpoint](/sql#connecting). Python only | -| `client_config` / `clientConfig` | retries, timeouts, and connection pooling | - -An API key belongs in the environment, not in source. `region` is easy to get -wrong silently -- the default is `us-east-1` whether or not that is where your -database lives, and a wrong region fails as a connection error rather than a -helpful one. - -## Storage options {#storage-options} - -`storage_options` configures the underlying object store -- credentials, -endpoints, and defaults for tables created through this connection: - - -```python Python icon="python" db = lancedb.connect( "s3://bucket/path", storage_options={"new_table_enable_stable_row_ids": "true"}, @@ -99,95 +75,89 @@ db = lancedb.connect( ``` ```typescript TypeScript icon="square-js" +import * as lancedb from "@lancedb/lancedb"; + const db = await lancedb.connect("s3://bucket/path", { storageOptions: { new_table_enable_stable_row_ids: "true" }, }); ``` - -`new_table_enable_stable_row_ids` is worth setting deliberately: it gives stable -row ids to the tables this connection creates, and no SDK call adds them to an -existing table. A `db://` connection does not send it; a deployment decides -stable row ids itself (see -[Against a deployment](/feature-engineering/materialized-views#availability)). - +Set stable row IDs before creating a table that needs them, such as an ordinary +materialized-view source in Python SDK 0.40.0. The option does not change tables +that already exist. Enterprise storage settings are managed on the server. -See [Storage](/deployment/storage) for the full set, including per-cloud credentials. +See [Storage](/deployment/storage) for provider credentials and other options. ## Read consistency {#consistency} -`read_consistency_interval` controls how often a connection checks for versions -committed by other writers. +For OSS connections, `read_consistency_interval` controls how often reads check +for table versions committed by other writers. For example, check at most once +every five seconds: ```python Python icon="python" from datetime import timedelta +import lancedb -db = lancedb.connect(uri, read_consistency_interval=timedelta(seconds=5)) +db = lancedb.connect( + "./.lancedb", read_consistency_interval=timedelta(seconds=5) +) ``` ```typescript TypeScript icon="square-js" -const db = await lancedb.connect(uri, { readConsistencyInterval: 5 }); +import * as lancedb from "@lancedb/lancedb"; + +const db = await lancedb.connect("./.lancedb", { readConsistencyInterval: 5 }); ``` -The default reads a snapshot taken when the table was opened, so a long-lived -connection **will not see other processes' writes** until it reopens the table. -That is the right default for a batch job and the wrong one for a service. +By default, an OSS table handle does not automatically pick up another +process's writes. Reopen the table, check out its latest version, or configure +a consistency interval when you need those updates. A zero interval checks on +every read, with the cost of a metadata check. -Set it when several processes write the same table and readers must observe -those writes. The cost is a metadata check on that interval. See -[Consistency](/tables/consistency). +Enterprise manages read freshness on the server. See +[Consistency](/tables/consistency) for its behavior and the distinction between +seeing your own writes and seeing another client's writes. -## Database-level operations {#operations} +## Database operations {#operations} -| | Python | TypeScript | SQL | -| --- | --- | --- | --- | -| list tables | `table_names`, `list_tables` | `tableNames`, `listTables` | -- | -| the URI it opened | `uri` | `display()` | -- | -| drop every table | `drop_all_tables` | `dropAllTables` | -- | -| drop the database | `drop_database` | -- | -- | -| close | -- | `close`, `isOpen` | -- | +The connection provides methods to create, open, list and drop tables. - -```python Python icon="python" -db.table_names() -tbl = db.open_table("documents") -db.drop_table("documents") -``` +| Operation | Python | TypeScript | +| --- | --- | --- | +| Create a table | `create_table` | `createTable` | +| Open a table | `open_table` | `openTable` | +| List tables | `table_names`, `list_tables` | `tableNames`, `listTables` | +| Drop a table | `drop_table` | `dropTable` | -```typescript TypeScript icon="square-js" -await db.tableNames(); -const tbl = await db.openTable("documents"); -await db.dropTable("documents"); -``` - -```sql SQL -DROP TABLE documents; -``` - - -Each takes a `namespace_path` to act inside a [namespace](/management/namespace) rather -than at the root. +Use the namespace and pagination options documented in +[Namespaces](/management/namespace) when working outside the root namespace +or listing a large catalog. [Tables](/management/table) covers table schemas, +versions and lifecycle operations. -`drop_database()` removes every table in the database. There is no confirmation -and no undo -- table [versioning](/management/table#versioning) does not survive the table -being dropped. +Dropping a table removes its history as well as its current data. You cannot +recover a dropped table through table versioning. The OSS `drop_database()` +and `drop_all_tables()` operations remove all tables without a confirmation prompt. ## Async connections {#async} -`connect_async` returns an `AsyncConnection` with the same surface as -coroutines. The TypeScript SDK is async throughout, so its `connect` already is -one. +Python's `connect_async` returns an `AsyncConnection`. Await its operations +when working in an asynchronous application: ```python -db = await lancedb.connect_async("db://my-database", api_key="...") -tbl = await db.open_table("documents") +import lancedb + +db = await lancedb.connect_async("./.lancedb") +table = await db.open_table("documents") ``` -This is a different axis from the `_async` operations that return a -[job](/basics/jobs): `connect_async` is about not blocking your event loop, -while `create_index_async` is about work that outlives the call. +TypeScript's connection and table operations already return promises. + +An async connection and a background job serve different purposes. Async APIs +let other tasks run while your application waits for an operation. A +[job](/basics/jobs) represents work that can continue on the server after the +submission call returns. diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx index 40e5bcd00..5734d2545 100644 --- a/docs/web/management/function.mdx +++ b/docs/web/management/function.mdx @@ -4,27 +4,31 @@ sidebarTitle: "Function" description: "Register a Python function once and apply it as a column, for work SQL cannot express." icon: code keywords: ["udf", "functions", "create_function", "embedding", "feature engineering"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports Functions when enabled for your Enterprise cluster; the embedded library cannot register or run them. --- import { PyFunctionDefine as FunctionDefine, PyFunctionLocalCall as FunctionLocalCall, - PyFunctionConnect as FunctionConnect, PyFunctionCreateTable as FunctionCreateTable, PyFunctionRegister as FunctionRegister, PyFunctionApply as FunctionApply, PyFunctionRefresh as FunctionRefresh, PyFunctionInspect as FunctionInspect, } from '/snippets/function.mdx'; - -Enterprise +import { PyConnectEnterprisePlaceholders as FunctionConnect } from '/snippets/connection.mdx'; Some columns cannot be written as a SQL expression: an embedding, a model call, an image decode. A function is a Python callable registered with the database, versioned, and then applied as a column — the code runs where the data is rather than pulling rows to your process. -Registering and running a function needs a LanceDB Enterprise connection. On a +Registering and running a function needs a LanceDB Enterprise deployment with +Functions enabled, which they are not by default. On a local connection `create_function` raises `NotImplementedError`; to derive a column locally, declare a [SQL computed column](/feature-engineering/add-computed-column) instead. @@ -33,6 +37,10 @@ column locally, declare a In SQL, a function is registered with [`CREATE FUNCTION`](/sql/ddl/function#create) and listed with [`SHOW FUNCTIONS`](/sql/ddl/function#show). +Functions are the way to fill columns with Python in LanceDB Enterprise. If you +already use [Geneva](/geneva), it remains available; see +[Geneva and Functions](/geneva#functions) for how its concepts map onto these. + The steps below take a column of numbers to a filled column of their doubles. ## Define a function {#define} @@ -62,9 +70,9 @@ sibling columns, so a null field is a value rather than a missing row. ## Connect and create a table {#table} -The remaining steps run against your Enterprise deployment. This connection -reads its settings from environment variables, so no credential appears in the -code: +The remaining steps run against your Enterprise deployment. Replace the database +URI, API key, region and endpoint below with your deployment details. In application +code, load credentials from environment variables or a secrets manager: {FunctionConnect} @@ -75,7 +83,7 @@ A nullable column requires a nullable parameter, such as `Optional[float]`; a non-nullable column can bind to either. This example uses `value: float`, so the table declares `value` non-nullable. -Computed-column inputs depend on the deployment version. Where supported, +Computed-column inputs depend on the Enterprise server version. Where supported, they bind to non-nullable parameters even though the physical column is nullable: refresh skips rows whose computed input is still null. Check your deployment's support before using another computed column as an input. @@ -212,6 +220,10 @@ the name was already gone. Deleting the object's content can finish after the call returns, as a [background job](/maintenance/background-jobs). `drop_function_async` returns that job with the result, so you can wait for it. +A deployment refuses the drop unless it also allows changes to a function's +lifecycle, which is a separate setting, off by default even where Functions are +enabled. + ## What to read next {#next} diff --git a/docs/web/management/index.mdx b/docs/web/management/index.mdx index 1d4d4eeba..b6bc07e88 100644 --- a/docs/web/management/index.mdx +++ b/docs/web/management/index.mdx @@ -6,17 +6,17 @@ icon: folder-tree keywords: ["ddl", "create", "drop", "list", "catalog", "management"] --- -Every [catalog object](/basics/catalog) is created, listed, and dropped the same -few ways. This section is the reference for those operations; the pages linked -from it cover what each object is for. +Use the SDKs or SQL to manage [catalog objects](/basics/catalog), including +tables, namespaces and views. The tables below provide a quick reference to +the available methods; each object's guide covers its setup and behavior. -Operations that can take a while have an `_async` form returning a -[job](/basics/jobs). +Some long-running operations also provide a submission method that returns a +[job](/basics/jobs). See the operation's guide for how to track completion. -## The surface at a glance {#surface} +## Management APIs {#surface} -SQL requires a deployment -- a local database has no engine to plan a statement. +SQL is available through the Enterprise Flight SQL endpoint. See [SQL](/basics/sql#sql-is-remote). @@ -59,8 +59,8 @@ See [Tables](/management/table) for creating one and shaping its schema. | drop | `drop_table` | `dropTable` | `DROP TABLE` | -`list_materialized_views` reads every table's schema to find them, so it costs -one open per table. `SHOW MATERIALIZED VIEWS` asks the server instead. +Local `list_materialized_views` reads table schemas to find views. On +Enterprise, it requests the server's view listing. See [Materialized views](/management/materialized-view) for creating one, and @@ -71,13 +71,17 @@ it current. | | Python | TypeScript | SQL | | --- | --- | --- | --- | -| create | -- | -- | `CREATE VIEW` | -| drop | -- | -- | `DROP VIEW` | +| create | `create_view` | `createView` | `CREATE VIEW` | +| list | `list_views` | `listViews` | -- | +| drop | `drop_view`, `drop_view_async` | `dropView`, `dropViewAsync` | `DROP VIEW` | -Enterprise Views exist only in SQL -- there is no SDK -method for them, because the thing a view stores *is* a SQL statement. See +Enterprise Views exist only on an Enterprise cluster that +enables them; the SDK methods send the view's SQL query to the deployment. See [Views](/management/view). +{/* +Draft for the Functions release. + ### Functions {#functions} | | Python | TypeScript | SQL | @@ -87,9 +91,11 @@ method for them, because the thing a view stores *is* a SQL statement. See | list | `list_functions` | -- | `SHOW FUNCTIONS` | | drop | `drop_function` | -- | -- | -Enterprise A function runs on the server, so there is -no local form. The TypeScript SDK has no function methods; use Python or SQL. +Enterprise A function runs on the server, and only on +a deployment that enables functions, so there is no local form. The TypeScript +SDK has no function methods; use Python or SQL. See [Functions](/management/function). +*/} ### Jobs {#jobs} @@ -119,7 +125,7 @@ names it. For a multi-column or named index, use the SDK. Indexes are a performance decision rather than a structural one, so they are covered under [Maintenance](/indexing). -## Namespaces qualify everything {#namespace-paths} +## Work within a namespace {#namespace-paths} Most connection-level operations take a `namespace_path` (`namespacePath`) to act inside a namespace rather than at the database root: @@ -131,8 +137,8 @@ tbl = db.open_table("clicks", namespace_path=["prod", "events"]) ``` ```typescript TypeScript icon="square-js" -await db.tableNames({ namespacePath: ["prod", "events"] }); -const tbl = await db.openTable("clicks", { namespacePath: ["prod", "events"] }); +await db.tableNames(["prod", "events"]); +const tbl = await db.openTable("clicks", ["prod", "events"]); ``` ```sql SQL @@ -142,7 +148,7 @@ SELECT * FROM prod.events.clicks In SQL the path is written as dotted identifiers instead. -## Dropping is not undoable {#dropping} +## Delete catalog objects {#dropping} `drop_table` removes the table and every version of it. Table diff --git a/docs/web/management/materialized-view.mdx b/docs/web/management/materialized-view.mdx index cbdb703b1..1475bd93a 100644 --- a/docs/web/management/materialized-view.mdx +++ b/docs/web/management/materialized-view.mdx @@ -4,6 +4,11 @@ sidebarTitle: "Materialized View" description: "Create, open, list and drop a table whose contents are defined by a query." icon: table keywords: ["materialized view", "create_materialized_view", "catalog", "ddl"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support materialized views. See [Enterprise compatibility](#deployment) for connection and server requirements. --- A materialized view is a table whose contents are defined by a query over @@ -20,7 +25,7 @@ This page covers creating and managing one. Keeping it current is In SQL, see [`REFRESH MATERIALIZED VIEW`](/sql/dml/refresh#refresh-materialized-view) and [`SHOW MATERIALIZED VIEWS`](/sql/ddl/materialized-view#show). -## Creating one {#create} +## Create a materialized view {#create} ```python Python icon="python" @@ -28,24 +33,24 @@ view = db.create_materialized_view( "recent_docs", source="documents", select=["id", "title", "embedding"], - where="created_at > '2026-01-01'", + where="created_at > TIMESTAMP '2026-01-01 00:00:00'", ) ``` ```typescript TypeScript icon="square-js" const view = await db.createMaterializedView("recent_docs", "documents", { select: ["id", "title", "embedding"], - where: "created_at > '2026-01-01'", + where: "created_at > TIMESTAMP '2026-01-01 00:00:00'", }); ``` -`source` names the table it derives from. A local source does not need stable -row ids or any other storage option; a deployment may require stable row ids -(see [Against a deployment](#deployment)). `select`, `where` and `limit` record -the query -- there is no separate definition step. +`source` names the table the view reads. With Python SDK 0.40.0, create that +table with stable row IDs before creating the view. The +[complete example](/feature-engineering/materialized-views#create-and-refresh) +shows the setup. `select`, `where` and `limit` define the query. -## Opening, listing and dropping {#manage} +## Open, list and drop views {#manage} | | Python | TypeScript | SQL | | --- | --- | --- | --- | @@ -56,34 +61,30 @@ the query -- there is no separate definition step. | drop | `drop_table` | `dropTable` | `DROP TABLE` | -`list_materialized_views` finds views by reading every table's schema, so it -costs one open per table. `SHOW MATERIALIZED VIEWS` asks the server instead and -is cheaper on a catalog with many tables. +On a local database, `list_materialized_views` reads table schemas to find +views. On Enterprise, it asks the server for the view listing, as +`SHOW MATERIALIZED VIEWS` does. A view is dropped with `drop_table`, because a materialized view **is** a table. That also means dropping one removes stored rows -- unlike dropping a [view](/management/view), which removes only a saved query. -## Working with the view {#working} +## Read the view and its definition {#working} `view.table` is the underlying table, for querying, indexing, or search. -`view.definition` is the recorded query -- select list, predicate, limit --- which is how you check what a view contains without inferring it from the -rows. Both are properties; on an async view, the definition is read with +`view.definition` contains the query's select list, predicate and limit. Both are properties; on an async view, the definition is read with `await view.definition()`. -## Against a deployment {#deployment} +## Enterprise compatibility {#deployment} -Enterprise `create_materialized_view` works on a -`db://` connection as well as a local one. There the SDK creates and refreshes -the view through SQL, and the work runs on the cluster rather than in your -process. Materialized views are off by default on a deployment and have to be -enabled there, and a deployment may require a view's source to have stable row -ids; see -[Against a deployment](/feature-engineering/materialized-views#availability). +Enterprise supports creating and refreshing materialized views on remote +compute. Use the SDK version supported by your deployment, or the +[SQL interface](/sql/ddl/materialized-view). See +[Enterprise refreshes](/feature-engineering/materialized-views#availability) +for compatibility considerations when upgrading an existing pipeline. -## What to read next {#next} +## Related guides {#next} diff --git a/docs/web/management/namespace.mdx b/docs/web/management/namespace.mdx index f1310b6da..a3485f46d 100644 --- a/docs/web/management/namespace.mdx +++ b/docs/web/management/namespace.mdx @@ -4,6 +4,12 @@ sidebarTitle: "Namespace" description: "Grouping tables into a hierarchy, and addressing them through it." icon: folder-tree keywords: ["namespace", "catalog", "hierarchy", "directory namespace", "rest namespace"] +availability: + oss: available + enterprise: available + summary: >- + Namespaces work in the embedded library, through directory or REST + catalogs, and in LanceDB Enterprise. --- import { @@ -15,8 +21,10 @@ import { RsNamespaceTableOps, } from '/snippets/connection.mdx'; -Despite its name, LanceDB is not a "database" in the traditional sense -- it is a **Multimodal Lakehouse** that builds on the table abstraction, -similar to many other lakehouse projects. LanceDB exposes a catalog-level abstraction over the Lance table format, via a *namespace spec*. +LanceDB organizes Lance tables through a catalog-level abstraction defined by +the *namespace spec*. LanceDB OSS is an embedded library for accessing those +tables, and LanceDB Enterprise is a multimodal lakehouse that provides shared +compute and catalog services on top of the Lance format. If you're coming from traditional databases or lakehouses, you can think of a namespace as the catalog path that says where a table name lives. Lance provides the **file** and **table** formats to store and manage your data and indexes. @@ -29,7 +37,7 @@ language: Python uses `namespace_path`, Rust uses builder methods like `.namespa TypeScript uses `namespacePath` arguments. -In SQL a namespaced table is addressed as a [dotted path](/sql/dql/select#namespaces). +In LanceDB Enterprise SQL, a namespaced table is addressed as a [dotted path](/sql/dql/select#namespaces). ## Namespace hierarchy {#namespace-hierarchy} @@ -47,14 +55,15 @@ If you want to go deeper, see the Lance format [namespace documentation](https:/ A namespace path is a list of components. For example, `["prod", "search"]` means the `search` namespace inside the `prod` namespace. The empty path, `[]`, means the root namespace. -Each component is a name, not a filesystem path segment. Namespace names can't be empty, and each -component can contain only letters, numbers, underscores, hyphens, and periods. That keeps the same -identifier usable across local directory namespaces and REST namespace identifiers. +Each component is a name, not a filesystem path segment, and can't be empty. Keep each one to +letters, numbers, underscores, hyphens, and periods, so the same identifier is usable everywhere: a +local directory namespace accepts other characters too, but other namespace implementations and +deployments can refuse them. [Naming rules](/basics/naming) has the lengths and the other limits. ## Directory namespaces {#directory-namespaces} The simplest namespace model in LanceDB is a single root namespace, often represented by one -directory: +directory. Namespaces can nest inside it, as in this hierarchy: ```bash ./local_lancedb (root) @@ -64,7 +73,12 @@ directory: └── data (table) ``` -As a user of LanceDB OSS, you might never notice namespaces at first, because LanceDB exposes the single-level hierarchy shown above, with the data stored in the `data/` directory, where the root namespace is implicit. Connecting to this namespace is as simple as connecting to the catalog root: +The hierarchy is logical, not a tree of folders: a directory namespace records it in a +`__manifest` table in the root directory, and stores each table in a directory of its own there. + +As a user of LanceDB OSS, you might never notice namespaces at first, because a plain connection +uses only the root namespace, which is implicit. Connecting to this namespace is as simple as +connecting to the catalog root: ```python Python icon="python" @@ -89,7 +103,7 @@ let db = connect("./local_lancedb").execute().await?; ``` -This creates the default namespace directory (`data/`) under the specified root path. +A table created through this connection is stored directly under that path, as `.lance`. You can also explicitly connect to a namespace using `lancedb.connect_namespace(...)` with the directory namespace implementation: @@ -152,7 +166,7 @@ println!("Created table: {}", table.name()); - For simple use cases in LanceDB OSS, you don't need to go too deep into namespaces. -- To integrate LanceDB with external catalogs and to use it as a true **multimodal lakehouse**, it's useful to understand the different namespace implementations and how to use them in your organization's setup. +- To integrate LanceDB with external catalogs, understand how each namespace implementation resolves table names and locations in your organization's setup. ## Remote or external catalog namespaces {#remote-or-external-catalog-namespaces} @@ -169,9 +183,11 @@ LanceDB Enterprise REST requests use the `x-api-key` header for API-key authenti that route multiple databases through the same endpoint can also use headers such as `x-lancedb-database` or `x-lancedb-database-prefix` for database context. +Replace the catalog URL and credential placeholders below with your values. In +application code, load credentials from environment variables or a secrets manager. + ```python Python icon="python" -import os import lancedb # Remote namespace-backed catalog root (RestNamespace) @@ -180,9 +196,9 @@ db = lancedb.connect_namespace( "rest", { "uri": "https://.internal..com", - "headers.x-api-key": os.environ["API_KEY"], + "headers.x-api-key": "your-api-key", # or: - # "headers.Authorization": f"Bearer {os.environ['REST_AUTH_TOKEN']}", + # "headers.Authorization": "Bearer your-access-token", }, ) ``` @@ -195,9 +211,9 @@ import * as lancedb from "@lancedb/lancedb"; const db = await lancedb.connectNamespace("rest", { uri: "https://.internal..com", headers: { - "x-api-key": process.env.API_KEY ?? "", + "x-api-key": "your-api-key", // or: - // Authorization: `Bearer ${process.env.REST_AUTH_TOKEN}`, + // Authorization: "Bearer your-access-token", }, }); ``` @@ -214,12 +230,12 @@ properties.insert( ); properties.insert( "headers.x-api-key".to_string(), - std::env::var("API_KEY")?, + "your-api-key".to_string(), ); // or: // properties.insert( // "headers.Authorization".to_string(), -// format!("Bearer {}", std::env::var("REST_AUTH_TOKEN")?), +// "Bearer your-access-token".to_string(), // ); let db = lancedb::connect_namespace("rest", properties) .execute() @@ -227,7 +243,8 @@ let db = lancedb::connect_namespace("rest", properties) ``` [LanceDB Enterprise](/enterprise) operates a REST namespace server on top of the Lance format, so any REST client that can speak the REST namespace API -contract can be used to interact with it. For authentication examples in LanceDB Enterprise, visit +contract can be used to interact with it. It does not serve the contract's `exists` routes, so checks such as `namespace_exists` +fail against it. For authentication examples in LanceDB Enterprise, visit the [Namespaces in SDKs](#namespaces-in-lancedb-enterprise) page. ## Best practices {#best-practices} @@ -279,8 +296,8 @@ The SDK methods expose the namespace path in the idiom of each language: Using namespaces is **optional** in LanceDB, and most basic use cases do not require to work with them. -An empty namespace (`[]`), which is the default, means "root namespace", and the data will be stored in -the `data/` directory under the specified root path. +An empty namespace (`[]`), which is the default, means the root namespace, whose tables are stored +directly under the specified root path. ## Namespace management APIs {#namespace-management-apis} @@ -315,12 +332,13 @@ doesn't exist, or contains data: - Drop mode: `fail` reports an error when the namespace doesn't exist, and `skip` treats a missing namespace as a successful no-op. - Drop behavior: `restrict` keeps non-empty namespaces from being dropped, and `cascade` drops child namespaces and tables first. -Namespace path components can't be empty. Each component can contain only letters, numbers, -underscores, hyphens, and periods. +Namespace path components can't be empty, and are best kept to letters, numbers, underscores, +hyphens, and periods, as [above](#namespace-paths-and-names). Listing APIs return the immediate children of the requested namespace path. Use `limit` with the returned `page_token` to page through large catalogs; pass an empty namespace path (`[]`) when you -want to list from the root namespace. +want to list from the root namespace. In Python, a listing returns a response: the names are in its +`namespaces` or `tables`, and `page_token` is set when there is another page. ## Namespaces in LanceDB Enterprise {#namespaces-in-lancedb-enterprise} diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index 447b59029..1b8ede070 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -4,6 +4,13 @@ sidebarTitle: "Table" description: "The object most operations act on: creating, opening, renaming, cloning and dropping one." icon: table keywords: ["table", "create_table", "open_table", "clone", "rename", "drop"] +availability: + oss: available + enterprise: available + summary: >- + Tables are created, evolved, versioned and dropped in the embedded library + and in LanceDB Enterprise. Renaming, cloning, dropping every table and pruning old + versions differ between the two. --- import { @@ -113,18 +120,20 @@ import { PyQueryToPandasKwargs as QueryToPandasKwargs, } from '/snippets/multimodal.mdx'; -A table is rows and columns, versioned. It is what searches read, what writes -commit to, and what indexes and branches belong to. +A table stores data in rows and columns and keeps a history of committed +versions. Queries read table data, writes update it, and indexes accelerate +searches over its columns. Branches let you make changes separately from the +main table history. -This page covers the table as a catalog object. Getting data into one is -[Ingest](/ingestion); changing its shape is -[Schema evolution](/management/table#schema). +This page covers table management, data types, schema changes and version +history. For loading records, see [Ingestion](/ingestion). For changes to +columns, see [Schema evolution](/management/table#schema). -Tables are created and dropped in SQL with [`CREATE TABLE` and `DROP TABLE`](/sql/ddl/table). +In LanceDB Enterprise, tables are also created and dropped in SQL with [`CREATE TABLE` and `DROP TABLE`](/sql/ddl/table). -## What a table carries {#carries} +## Table structure and metadata {#carries} | | | | --- | --- | @@ -133,7 +142,7 @@ Tables are created and dropped in SQL with [`CREATE TABLE` and `DROP TABLE`](/sq | **Indexes** | vector, full-text and scalar, built over its columns | | **Branches** | isolated lines of change you can [merge or discard](/tables/branching) | -## Creating {#create} +## Create a table {#create} ```python Python icon="python" @@ -156,19 +165,18 @@ frame, or Pydantic models. [Ingesting data](/ingestion) covers each, plus `exist_ok` and `mode="overwrite"` for re-runnable scripts. -An explicit schema is where nullability is decided. Creation is also the only -point at which the SDK can give a table **stable row ids**, which a deployment -may require on a [materialized view](/management/materialized-view) source. See -[Limits](/basics/limits#schema). +An explicit schema is where nullability is decided. In the embedded library, +creation is also the only point at which a table can be given **stable row +ids**; in LanceDB Enterprise, whether a new table gets them is the deployment's +setting. See [Limits](/basics/limits#schema). -## Opening and listing {#open} +## Open and list tables {#open} ```python Python icon="python" tbl = db.open_table("documents") db.table_names() -db.table_exists("documents") ``` ```typescript TypeScript icon="square-js" @@ -179,9 +187,18 @@ await db.tableNames(); Each accepts `namespace_path` to act inside a [namespace](/management/namespace) -rather than at the database root. TypeScript has no `tableExists`. +rather than at the database root. -## Renaming and cloning {#rename-clone} +To find out whether a table exists, open it and handle the error, or look for +its name in a listing. A missing table raises `ValueError` on a local directory +or a deployment, and `lance_namespace.TableNotFoundError` on a connection from +`connect_namespace`. A listing returns one page of names at a time, so check +every page. Python's `table_exists` is no substitute: a local directory or a +deployment raises `NotImplementedError` for it, and on a connection from +`connect_namespace("dir", ...)` or `connect_namespace("rest", ...)` it fails with +an `AttributeError`. TypeScript has no `tableExists`. + +## Rename and clone tables {#rename-clone} ```python db.rename_table("documents", "docs") @@ -194,6 +211,10 @@ db.clone_table( ) ``` +Renaming is not supported on a local directory connection. In LanceDB Enterprise, +whether a table can be renamed or cloned at all depends on how the deployment +stores its catalog. + A **shallow** clone -- the default -- references the source's data files rather than copying them, so it is fast and cheap regardless of table size. It is the practical way to branch an experiment off a known version, or to pin a training @@ -204,18 +225,20 @@ A shallow clone depends on the source's files continuing to exist. If the source is [compacted](/maintenance/compaction) past the retention window, the version the clone points at can be pruned. Local OSS connections support shallow clones only; `is_shallow=False` is not implemented. If the copy must outlive the -source's history, copy its data into an independent table instead. For remote -connections, confirm which clone modes your deployment supports. +source's history, copy its data into an independent table instead. A deployment +that supports cloning also makes shallow clones only, and one whose catalog +does not support it refuses every clone. -**Enterprise job type:** `drop_table`. Fences the table's directory with a -marker and deletes its content in the background -- see -[Background jobs](/maintenance/background-jobs#teardown) for the namespace, -view, and function equivalent. +**Enterprise job type:** on an Enterprise cluster that drops tables in the background, +the teardown runs as a job: `drop_table`, which fences the table's directory +with a marker and deletes its content, or `drop_object`, depending on how the +deployment stores its catalog. See +[Background jobs](/maintenance/background-jobs#teardown). -## Dropping {#drop} +## Drop a table {#drop} ```python Python icon="python" @@ -238,13 +261,16 @@ DROP TABLE documents Dropping removes the table and **every version of it**. [Versioning](/management/table#versioning) does not survive a drop, and neither -`drop_all_tables` nor `drop_database` asks for confirmation. +`drop_all_tables` nor `drop_database` asks for confirmation. Both work only on +a local database; on an Enterprise connection the SDK raises instead. On a large table the drop can take a while, so `drop_table_async` returns a -[job](/basics/jobs) rather than blocking. +[job](/basics/jobs) rather than blocking when the deployment drops tables in the +background. In the embedded library, and on an Enterprise cluster that drops inline, the +returned job has already finished. -## The full surface {#surface} +## Table management APIs {#surface} | | Python | TypeScript | SQL | | --- | --- | --- | --- | @@ -456,6 +482,8 @@ When you call `to_pandas()` on a local LanceDB table that contains Blob API colu - `"bytes"`: eagerly materializes each blob as `bytes`. Use this when you need the raw payload in the DataFrame, for example to decode an image or audio clip in-process. - `"descriptions"`: returns blob descriptors (offsets, sizes, and positions) instead of the data itself. Use this when you want to plan I/O without paying the cost of loading every blob. +`"bytes"` is not supported on in-memory tables. + Extra keyword arguments are forwarded to the underlying PyArrow / Lance pandas conversion, so you can also pass options like `split_blocks` or `self_destruct`: @@ -464,13 +492,10 @@ Extra keyword arguments are forwarded to the underlying PyArrow / Lance pandas c -Query builders also accept `blob_mode` on `to_pandas()`, `to_arrow()`, and `to_list()`: +Query builders also accept `blob_mode` on their `to_pandas()` method: - Plain scans support `"lazy"`, `"bytes"`, and `"descriptions"` with filters, projections, aliases, `limit`, and `offset`. -- Vector, FTS, hybrid, and take queries support `"bytes"` and `"descriptions"`. Lazy `BlobFile` handles need a plain scan. -- In `to_arrow()` and `to_list()`, `"lazy"` returns blob descriptors that carry the row id `fetch_blobs` reads, and `"descriptions"` returns them without it. -- A plain scan's `to_pandas()` without `timeout` reads the local dataset through Lance's scanner, so it also returns `"bytes"` for legacy blob columns and under namespace query pushdown. In-memory tables have no local dataset, so they take the `to_arrow()` route. -- Every other `"bytes"` route raises `ValueError` for legacy blob columns and blobs inside lists, and `NotImplementedError` on remote tables and namespace query pushdown, where the blob fetch could read a different table version than the query. +- Vector, FTS, hybrid, and ordered queries can't materialize blob columns through `to_pandas()`; omit blob columns from the projection for those query shapes. - This works on both sync and async query builders. Extra PyArrow kwargs like `split_blocks` and `self_destruct` are still forwarded. @@ -880,8 +905,13 @@ Remove several columns at once for efficiency: - -Dropping columns cannot be undone. Make sure you have backups or are certain before removing columns. + +Dropping a column changes the latest version only. Until +[cleanup](/maintenance/cleanup) prunes the versions from before the drop, +[restoring](#versioning) one of them brings the column and its values back -- +along with the rest of the table as it was then. Once those versions are +pruned, the drop cannot be undone, so check before dropping a column you may +need. ## Versioning {#versioning} @@ -1041,8 +1071,9 @@ Numeric table versions like `v3` or `v17` are precise but hard to remember. Tags let you attach human-readable labels (e.g., `"prod"`, `"baseline"`, `"q3-evaluation"`) to specific versions and check those out by name. They are conceptually similar to git tags, and unlike numeric versions, **tagged versions -are preserved when old versions are pruned** (see the cleanup note at the bottom -of this page). +are never pruned** -- a tagged version that is older than the retention window and +is not the latest stops cleanup instead (see the cleanup note at the bottom of this +page). The tags API supports the standard CRUD operations — create, list, update, delete — plus checking out by tag name. @@ -1138,10 +1169,17 @@ mixing in lower-level Lance file operations. **System Operations** System operations like `optimize()`, index updates, and table compaction also increment table version numbers. -In LanceDB OSS and Enterprise, `optimize()` can prune older versions based on its retention setting (`cleanup_older_than`, 7 days by default), -which is when old-version files are removed and disk space is reclaimed. +In LanceDB OSS, `optimize()` can prune older versions based on its retention setting (`cleanup_older_than`, 7 days by default), +which is when old-version files are removed and disk space is reclaimed. A deployment prunes them in the background +when its configuration enables [cleanup](/maintenance/cleanup); `optimize()` is not available there. -**Tagged versions are exempt from cleanup.** A version with a tag pointing at it is -retained regardless of age, and its files are not removed by `optimize()`. To make -a tagged version eligible for pruning, [delete the tag](#versioning-tag-based-versioning) first. +**Tagged versions are never pruned.** A version with a tag pointing at it is +retained regardless of age. When a tagged version is older than the retention window and +is not the latest, cleanup refuses to run instead of pruning around it: `optimize()` raises +an error naming the tag and prunes nothing. A tag on the latest version does not cause +this, but `optimize()` compacts before it prunes, and compacting commits a new version +whenever it merges files, so a tag on what was the latest version can become an older +one by the time cleanup runs; the error then comes after compacting, and the index +update does not run. To let cleanup proceed, [move the tag](#versioning-tag-based-versioning) +to a version inside the retention window, delete it, or widen the window. diff --git a/docs/web/management/view.mdx b/docs/web/management/view.mdx index bfb3a0543..de6cd38a5 100644 --- a/docs/web/management/view.mdx +++ b/docs/web/management/view.mdx @@ -4,29 +4,31 @@ sidebarTitle: "View" description: "A saved query, evaluated when read. The non-materialized counterpart to a materialized view." icon: eye keywords: ["view", "create view", "sql", "catalog"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports views when enabled for your Enterprise cluster; the embedded library cannot create them. --- -Enterprise - -This page describes a LanceDB Enterprise capability. The embedded form -of LanceDB has no server to provide it -- see -[Architecture](/architecture) for what differs between the two. - -A view stores a **query**, not rows. Reading it runs the query against current -data, so a view is never stale -- and never faster than the query it wraps. +A view saves a query that you can refer to by name. Reading the view executes +that query against the source tables, using their read-consistency behavior. +Views do not store a separate copy of the result. It is the counterpart to a [materialized view](/management/materialized-view), which stores rows and is brought up to date by a refresh. Both are [catalog objects](/basics/catalog) -defined by a query; the difference is entirely when that query runs. +defined by a query; choose a materialized view when you need to store and reuse the result. | | View | Materialized view | | --- | --- | --- | | Stores | the query | the rows | -| Freshness | always current | as of the last refresh | -| Read cost | the full query, every read | a table scan | +| Freshness | follows source-table reads | as of the last refresh | +| Read cost | executes the saved query | queries the stored result | | Write cost | none | a refresh | -Views are created in SQL, over the [Flight SQL endpoint](/sql#connecting) -- -there is no SDK method, because the thing a view stores *is* a SQL statement. -See [View](/sql/ddl/view) for `CREATE VIEW`, `DROP VIEW`, and listing. +Views are created in LanceDB Enterprise, either in SQL over the +[Flight SQL endpoint](/sql#connecting) or with the SDKs' `create_view` / +`createView`, which send the query to the deployment. Either way, the deployment +must have views enabled. See [View](/sql/ddl/view) for `CREATE VIEW`, +`DROP VIEW`, and listing. diff --git a/docs/web/snippets/curation_sampling.mdx b/docs/web/snippets/curation_sampling.mdx new file mode 100644 index 000000000..9c3d482b8 --- /dev/null +++ b/docs/web/snippets/curation_sampling.mdx @@ -0,0 +1,4 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PySampleDataset = "from tempfile import TemporaryDirectory\n\nimport lancedb\nfrom lancedb.permutation import Permutation, permutation_builder\n\nwith TemporaryDirectory(prefix=\"lancedb-sampling-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\"records\", [{\"id\": i} for i in range(10)])\n splits = (\n permutation_builder(table).split_random(counts=[3, 7], seed=42).execute()\n )\n sample = Permutation.from_tables(table, splits, split=0).with_format(\"arrow\")\n selected_ids = []\n for batch in sample.iter(batch_size=3):\n selected_ids.extend(batch.column(\"id\").to_pylist())\n print(f\"Selected {len(selected_ids)} rows: {selected_ids}\")\n"; + diff --git a/docs/web/snippets/embedding.mdx b/docs/web/snippets/embedding.mdx index 8db4472dc..da3388ea7 100644 --- a/docs/web/snippets/embedding.mdx +++ b/docs/web/snippets/embedding.mdx @@ -34,13 +34,13 @@ export const TsRegisterModelFallback = "const registry = getRegistry();\nregistr export const TsRegisterSecret = "const registry = getRegistry();\nregistry.setVar(\"api_key\", \"sk-...\");\n\nconst func = registry.get(\"openai\")!.create({\n apiKey: \"$var:api_key\",\n});\n"; -export const RsCreateEmbeddingFunction = "use std::sync::Arc;\n\nuse lancedb::embeddings::openai::OpenAIEmbeddingFunction;\n\nlet api_key = std::env::var(\"OPENAI_API_KEY\").expect(\"OPENAI_API_KEY is not set\");\nlet embedding = Arc::new(\n OpenAIEmbeddingFunction::new_with_model(api_key, \"text-embedding-3-small\")\n .expect(\"failed to create OpenAI embedding function\"),\n);\n"; +export const RsCreateEmbeddingFunction = "use std::sync::Arc;\n\nuse lancedb::embeddings::openai::OpenAIEmbeddingFunction;\n\nlet api_key = \"your-openai-api-key\".to_string();\nlet embedding = Arc::new(\n OpenAIEmbeddingFunction::new_with_model(api_key, \"text-embedding-3-small\")\n .expect(\"failed to create OpenAI embedding function\"),\n);\n"; export const RsEmbeddingFunction = "use std::{borrow::Cow, sync::Arc};\n\nuse arrow_array::{Array, FixedSizeListArray, Float32Array};\nuse arrow_schema::{DataType, Field, Schema};\nuse lancedb::{\n connect,\n embeddings::{EmbeddingDefinition, EmbeddingFunction},\n Result,\n};\n\n#[derive(Debug, Clone)]\nstruct MyTextEmbedder {\n dim: usize,\n}\n\nimpl EmbeddingFunction for MyTextEmbedder {\n fn name(&self) -> &str {\n \"my-embedder\"\n }\n\n fn source_type(&self) -> Result> {\n Ok(Cow::Owned(DataType::Utf8))\n }\n\n fn dest_type(&self) -> Result> {\n Ok(Cow::Owned(DataType::new_fixed_size_list(\n DataType::Float32,\n self.dim as i32,\n true,\n )))\n }\n\n fn compute_source_embeddings(&self, source: Arc) -> Result> {\n let values = Arc::new(Float32Array::from(vec![1.0f32; source.len() * self.dim]));\n let field = Arc::new(Field::new(\"item\", DataType::Float32, true));\n Ok(Arc::new(FixedSizeListArray::new(\n field,\n self.dim as i32,\n values,\n None,\n )))\n }\n\n fn compute_query_embeddings(&self, _input: Arc) -> Result> {\n unimplemented!()\n }\n}\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n db.embedding_registry()\n .register(\"my-embedder\", Arc::new(MyTextEmbedder { dim: 3 }))?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n db.create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\n \"text\",\n \"my-embedder\",\n Some(\"vector\"),\n ))?\n .execute()\n .await?;\n\n Ok(())\n}\n"; -export const RsManualQueryEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = std::env::var(\"OPENAI_API_KEY\").expect(\"OPENAI_API_KEY is not set\");\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n // Manually generate embeddings for the query (Enterprise path)\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n // --8<-- [start:manual_query_search]\n // query_vector is assumed to already be generated by your embedding function\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n // --8<-- [end:manual_query_search]\n\n Ok(())\n}\n"; +export const RsManualQueryEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = \"your-openai-api-key\".to_string();\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n // Manually generate embeddings for the query (Enterprise path)\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n // --8<-- [start:manual_query_search]\n // query_vector is assumed to already be generated by your embedding function\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n // --8<-- [end:manual_query_search]\n\n Ok(())\n}\n"; export const RsManualQuerySearch = "// query_vector is assumed to already be generated by your embedding function\nlet mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\nwhile let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n}\n"; -export const RsOpenaiEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = std::env::var(\"OPENAI_API_KEY\").expect(\"OPENAI_API_KEY is not set\");\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n\n Ok(())\n}\n"; +export const RsOpenaiEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = \"your-openai-api-key\".to_string();\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n\n Ok(())\n}\n"; diff --git a/docs/web/snippets/materialized_view.mdx b/docs/web/snippets/materialized_view.mdx index 41f9def31..975f7635b 100644 --- a/docs/web/snippets/materialized_view.mdx +++ b/docs/web/snippets/materialized_view.mdx @@ -1,6 +1,6 @@ {/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} -export const PyMaterializedViewCreate = "import lancedb\n\ndb = lancedb.connect(\"./.lancedb\")\ntable = db.create_table(\n \"people\", [{\"name\": \"ada\", \"age\": 36}, {\"name\": \"kid\", \"age\": 7}]\n)\n\nview = db.create_materialized_view(\n \"adults\",\n \"people\",\n select=[\"name\", (\"shout\", \"upper(name)\")],\n where=\"age >= 18\",\n)\nprint(view.table.count_rows())\n# 1\n"; +export const PyMaterializedViewCreate = "import lancedb\n\ndb = lancedb.connect(\"./.lancedb\")\ntable = db.create_table(\n \"people\",\n [{\"name\": \"ada\", \"age\": 36}, {\"name\": \"kid\", \"age\": 7}],\n storage_options={\"new_table_enable_stable_row_ids\": \"true\"},\n)\n\nview = db.create_materialized_view(\n \"adults\",\n \"people\",\n select=[\"name\", (\"shout\", \"upper(name)\")],\n where=\"age >= 18\",\n)\nprint(view.table.count_rows())\n# 1\n"; -export const PyMaterializedViewRefresh = "table.add([{\"name\": \"bea\", \"age\": 41}])\n\nresult = view.refresh()\nprint(result.rows_written)\n# 2\n"; +export const PyMaterializedViewRefresh = "table.add([{\"name\": \"bea\", \"age\": 41}])\n\nview.refresh()\nprint(view.table.count_rows())\n# 2\n"; diff --git a/docs/web/snippets/multimodal.mdx b/docs/web/snippets/multimodal.mdx index 90e275db7..f41af6629 100644 --- a/docs/web/snippets/multimodal.mdx +++ b/docs/web/snippets/multimodal.mdx @@ -16,7 +16,7 @@ export const PyMultimodalImports = "import lancedb\nimport pyarrow as pa\nimport export const PyProcessResults = "# Convert back to PIL Image\nfor _, row in results.iterrows():\n image_bytes = row['image_blob']\n image = Image.open(io.BytesIO(image_bytes))\n print(f\"Retrieved image: {row['filename']}, Size: {image.size}\")\n # You can now use 'image' with other libraries or display it\n"; -export const PyQueryToPandasKwargs = "# Plain scan query: blob_mode is supported end to end\ndf_lazy = (\n tbl.search()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"lazy\")\n)\n\n# Same call shape works on async query builders\ndf_bytes = await (\n tbl_async.query()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\")\n)\n\n# Vector / FTS / hybrid queries return \"bytes\" or \"descriptions\".\n# Lazy BlobFile handles need a plain scan.\ndf_vec = (\n tbl.search(query_vector)\n .limit(10)\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\", split_blocks=True, self_destruct=True)\n)\n\n# to_arrow and to_list take the same blob_mode\nhits = tbl.search(query_vector).limit(2).select([\"id\", \"video\"])\narrow_bytes = hits.to_arrow(blob_mode=\"bytes\")\nrows = hits.to_list(blob_mode=\"bytes\")\n"; +export const PyQueryToPandasKwargs = "# Plain scan query: blob_mode is supported end to end\ndf_lazy = (\n tbl.search()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"lazy\")\n)\n\n# Same call shape works on async query builders\ndf_bytes = await (\n tbl_async.query()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\")\n)\n\n# Vector / FTS / hybrid queries can't materialize blob columns,\n# so omit them from the projection\ndf_vec = (\n tbl.search(query_vector)\n .limit(10)\n .select([\"id\", \"vector\"])\n .to_pandas(split_blocks=True, self_destruct=True)\n)\n"; export const PySearchData = "# Search for similar images\nquery_vector = np.random.rand(128).astype(np.float32)\nresults = tbl.search(query_vector).limit(1).to_pandas()\n"; diff --git a/docs/web/snippets/tables.mdx b/docs/web/snippets/tables.mdx index 6b9b98280..28b9ce172 100644 --- a/docs/web/snippets/tables.mdx +++ b/docs/web/snippets/tables.mdx @@ -90,7 +90,7 @@ export const PyMergeUpdateInsert = "import pyarrow as pa\n\ntable = db.create_ta export const PyOpenExistingTable = "db = tmp_db\n# Create a table first\ndata = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7}]\ndb.create_table(\"test_table\", data, mode=\"overwrite\")\n\n# List table names\nprint(db.list_tables().tables)\n\n# Open existing table\ntbl = db.open_table(\"test_table\")\n"; -export const PySchemaAddSetup = "table_name = \"schema_evolution_add_example\"\nif data is None:\n data = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 3,\n \"name\": \"Headphones\",\n \"price\": 150.00,\n \"vector\": np.random.random(128).tolist(),\n },\n ]\ntable = tmp_db.create_table(table_name, data, mode=\"overwrite\")\n"; +export const PySchemaAddSetup = "import numpy as np\n\ndata = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 3,\n \"name\": \"Headphones\",\n \"price\": 150.00,\n \"vector\": np.random.random(128).tolist(),\n },\n]\ntable = db.create_table(\"schema_evolution_add_example\", data, mode=\"overwrite\")\n"; export const PySchemaAlterSetup = "table_name = \"schema_evolution_alter_example\"\nif data is None:\n data = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200,\n \"discount_price\": 1080.0,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800,\n \"discount_price\": 720.0,\n \"vector\": np.random.random(128).tolist(),\n },\n ]\nschema = pa.schema(\n {\n \"id\": pa.int64(),\n \"name\": pa.string(),\n \"price\": pa.int32(),\n \"discount_price\": pa.float64(),\n \"vector\": pa.list_(pa.float32(), 128),\n }\n)\ntable = tmp_db.create_table(table_name, data, schema=schema, mode=\"overwrite\")\n"; diff --git a/docs/web/sql/data-types.mdx b/docs/web/sql/data-types.mdx index 70bafdb96..8c7fa1781 100644 --- a/docs/web/sql/data-types.mdx +++ b/docs/web/sql/data-types.mdx @@ -6,7 +6,7 @@ icon: shapes keywords: ["sql types", "arrow", "cast", "pseudo-columns", "vector", "blob", "json"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) There is no SQL type layer over Lance storage. The table provider hands DataFusion the dataset's Arrow schema directly, so a column is the same object @@ -77,7 +77,7 @@ SELECT id FROM documents WHERE vector IS NOT NULL A list representation does not imply that ordinary arithmetic operators perform vector arithmetic. Use supported distance functions for similarity -calculations. SQL vector search depends on the deployment version and query +calculations. SQL vector search depends on the Enterprise server version and query plan; see [SQL](/basics/sql#no-vector-sql). diff --git a/docs/web/sql/ddl/function.mdx b/docs/web/sql/ddl/function.mdx index 39a661fc3..f0782008d 100644 --- a/docs/web/sql/ddl/function.mdx +++ b/docs/web/sql/ddl/function.mdx @@ -6,17 +6,22 @@ icon: function keywords: ["create function", "drop function", "show functions", "sql", "udf"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ## CREATE FUNCTION {#create} ```sql -CREATE FUNCTION f(x BIGINT) RETURNS BIGINT +CREATE FUNCTION f(x int64) RETURNS int64 LANGUAGE python AS 'def f(x): return x * 2' WITH (pip = 'numpy==1.26.4', python_version = '3.11') ``` +Parameter and result types are Arrow type names, not SQL ones: `int64`, +`float64`, `utf8`, `bool`, `date32`, and lists such as `list` or +`fixed_size_list`. `BIGINT` is refused. A parameter accepts nulls +unless it is declared `NOT NULL`. + The body is Python source. Its dependencies are declared with it and are part of the **immutable function version** -- changing one produces a new version rather than altering the registered one, so a column filled by an earlier @@ -27,7 +32,7 @@ version is unaffected. | `pip` | pip requirements for the worker environment | | `conda` | conda packages instead of pip | | `conda_channels` | channels in priority order; requires `conda` | -| `python_version` | worker Python version, defaulting to yours | +| `python_version` | worker Python version; `3.12` when omitted, whatever Python you send the statement from | | `gpu` | `'true'` when every execution needs a GPU | **Errors and limits** diff --git a/docs/web/sql/ddl/index.mdx b/docs/web/sql/ddl/index.mdx index 2ac18dd4c..3181210c0 100644 --- a/docs/web/sql/ddl/index.mdx +++ b/docs/web/sql/ddl/index.mdx @@ -1,30 +1,38 @@ --- title: "Index" sidebarTitle: "Index" -description: "CREATE INDEX and DROP INDEX — one column, server-chosen name." +description: "CREATE INDEX and DROP INDEX — one column, with the algorithm chosen by USING." icon: bolt keywords: ["create index", "drop index", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ## CREATE INDEX {#create} ```sql -CREATE INDEX ON documents (title) +CREATE INDEX ON documents USING btree (title) +CREATE INDEX text_search ON documents USING fts (text) ``` +`USING` is required: it names the algorithm. The scalar algorithms are `btree`, +`bitmap` and `label_list`; `fts` builds a full-text index; and the vector +algorithms are `ivf_pq`, `ivf_rq`, `ivf_flat`, `ivf_hnsw_sq` and `ivf_hnsw_pq`, +where the deployment supports them. Index options go in a `WITH (...)` clause. + -SQL `CREATE INDEX` covers **one column** and takes **no user-chosen name** -- -the server names it. For a multi-column or explicitly-named index, or to -choose the algorithm, use the SDK's -[`create_index`](/indexing). +In LanceDB Enterprise, SQL `CREATE INDEX` covers **one column**. A name between +`INDEX` and `ON` names the index, where the deployment's configuration allows +it; without one, the server names it after the column, as in `title_idx`. For a +multi-column index, use the SDK's [`create_index`](/indexing). -**Enterprise job type:** `create_ivf_pq`, `create_btree`, `create_fts`, and one -more per algorithm -- the type names which algorithm is building, not a -different feature. See [Build & Repair](/indexing/reindexing). +**Enterprise job type:** `reindex_` followed by the algorithm, such as +`reindex_btree` or `reindex_ivf_pq`; the name says which algorithm is building, +not a different feature. An SDK build that must not replace an existing index +runs as `create_` followed by the algorithm instead. See +[Build & Repair](/indexing/reindexing). ## DROP INDEX {#drop} @@ -35,7 +43,9 @@ DROP INDEX documents.title_idx Takes the qualified `table.index` name. Find it with the SDK's [`list_indices`](/indexing) if you created the index without -naming it yourself. +naming it yourself. A deployment builds the index in the background after +`CREATE INDEX` returns; until that build finishes, the index is not listed and +`DROP INDEX` reports it as not found. ## What to read next {#next} diff --git a/docs/web/sql/ddl/job.mdx b/docs/web/sql/ddl/job.mdx index ae99b014e..bc985d34d 100644 --- a/docs/web/sql/ddl/job.mdx +++ b/docs/web/sql/ddl/job.mdx @@ -6,7 +6,7 @@ icon: hourglass-half keywords: ["show jobs", "show errors", "pause job", "resume job", "cancel job", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) A [job](/basics/jobs) is not a catalog object -- it has no schema and is not namespaced -- but it has its own SQL statements, grouped here the same way the @@ -73,7 +73,7 @@ being ignored. Cancellation is a request, matching the SDK's [`cancel()`](/basics/jobs#managing): it does not guarantee the job stops before doing more work, and asking twice or cancelling a finished job does nothing. -`CANCEL JOB` itself can be unavailable on a deployment with no job registry +`CANCEL JOB` itself can be unavailable on an Enterprise cluster with no job registry configured, which is a deployment condition rather than something you can work around from SQL. diff --git a/docs/web/sql/ddl/materialized-view.mdx b/docs/web/sql/ddl/materialized-view.mdx index 51b5ef55b..19dba5aa4 100644 --- a/docs/web/sql/ddl/materialized-view.mdx +++ b/docs/web/sql/ddl/materialized-view.mdx @@ -6,13 +6,13 @@ icon: table keywords: ["materialized view", "create materialized view", "show materialized views", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) A materialized view stores a query's result as rows, kept current by a refresh -- the counterpart to a [view](/sql/ddl/view), which stores only the query. -Materialized views are off by default on a deployment and have to be enabled +Materialized views are off by default in LanceDB Enterprise and have to be enabled there. Which statements and query shapes a deployment accepts, whether it needs the source table to have stable row ids, and how it refreshes a view depend on its LanceDB Enterprise version and configuration. @@ -24,10 +24,9 @@ its LanceDB Enterprise version and configuration. CREATE MATERIALIZED VIEW AS SELECT ... ``` -On a `db://` connection, the SDK's -[`create_materialized_view`](/management/materialized-view#create) sends this -statement. Once created, a view is queried like a table, refreshed with SQL and -listed with SQL. +Once created, a materialized view can be queried like a table. Use the +statements below to list, refresh and drop it. For SDK methods, see +[Materialized views](/management/materialized-view#create). ## SHOW MATERIALIZED VIEWS {#show} @@ -36,8 +35,8 @@ SHOW MATERIALIZED VIEWS ``` -Cheaper than the SDK equivalent. `list_materialized_views` finds views by -reading every table's schema, costing one open per table; this asks the server. +This asks the server for its views, as the SDK's `list_materialized_views` does +on a `db://` connection. ## Refreshing one {#refresh} @@ -47,7 +46,7 @@ REFRESH MATERIALIZED VIEW chunks ``` See [REFRESH](/sql/dml/refresh#refresh-materialized-view) for the full syntax, -including `CASCADE` and `full = true`. +and compatibility notes for older server versions. ## Dropping one {#drop} @@ -57,8 +56,8 @@ stored rows, unlike dropping a [view](/sql/ddl/view#drop). **Enterprise job type:** `mat_view_refresh` for a refresh, or `sql` on a -deployment that rebuilds views through SQL; `drop_table` for dropping one, same -as any table. +deployment that rebuilds views through SQL. Dropping one runs the same job as +dropping any table, `drop_table` or `drop_object`. ## What to read next {#next} @@ -68,6 +67,6 @@ as any table. Creating one from the SDK. - How a refresh works, locally and on a deployment. + How a refresh works, locally and in LanceDB Enterprise. diff --git a/docs/web/sql/ddl/namespace.mdx b/docs/web/sql/ddl/namespace.mdx index daedf9720..efd209b02 100644 --- a/docs/web/sql/ddl/namespace.mdx +++ b/docs/web/sql/ddl/namespace.mdx @@ -6,7 +6,7 @@ icon: folder-tree keywords: ["namespace", "schema", "create schema", "drop schema", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) A [namespace](/management/namespace) is addressed in SQL as a `SCHEMA` -- the standard's name for the same concept. @@ -14,14 +14,16 @@ standard's name for the same concept. ## CREATE SCHEMA {#create} ```sql -CREATE SCHEMA public -CREATE SCHEMA db1.ns1.ns2 +CREATE SCHEMA analytics +CREATE SCHEMA db1.`ns1$ns2` CREATE SCHEMA IF NOT EXISTS `ns1$ns2` ``` -A dotted path creates the whole chain: `db1.ns1.ns2` creates `ns2` inside -`ns1` inside `db1`, provided the parents already exist. A name containing the -namespace path delimiter itself is backtick-quoted, as in `` `ns1$ns2` `` above. +A nested namespace is one backtick-quoted name with `$` between its +components: `` db1.`ns1$ns2` `` creates `ns2` inside `ns1` in database `db1`, +provided `ns1` already exists. The part before the dot is the database, so a +three-part path such as `db1.ns1.ns2` is rejected. `public` is the database's +root namespace, which is never created or dropped. `IF NOT EXISTS` makes the statement a no-op rather than an error when the namespace is already there. @@ -29,14 +31,14 @@ namespace is already there. ## DROP SCHEMA {#drop} ```sql -DROP SCHEMA db1.public +DROP SCHEMA db1.analytics DROP SCHEMA IF EXISTS `a$b` DROP SCHEMA `a$b` RESTRICT ``` `CASCADE` is rejected: **"DROP SCHEMA CASCADE is not supported; drop the -objects inside first."** A namespace's tables, views and functions must be +objects inside first."** A namespace's tables, views and other objects must be dropped explicitly before the namespace itself can go. `RESTRICT` -- the default -- states that requirement rather than changing it. diff --git a/docs/web/sql/ddl/table.mdx b/docs/web/sql/ddl/table.mdx index bd64458be..390c24b65 100644 --- a/docs/web/sql/ddl/table.mdx +++ b/docs/web/sql/ddl/table.mdx @@ -6,7 +6,7 @@ icon: table keywords: ["create table", "alter table", "drop table", "describe", "clone", "ddl"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ## CREATE TABLE {#create} @@ -30,16 +30,15 @@ The table's columns and types, without a plan -- the SQL counterpart of ## ALTER TABLE {#alter} -`ALTER TABLE ... ADD COLUMN` covers three different things, distinguished by -what follows the column name. +What `ALTER TABLE ... ADD COLUMN` does depends on what follows the column +name. ### A plain column {#plain-column} -```sql -ALTER TABLE t ADD COLUMN c INT -``` - -Added empty and nullable. Nothing is computed. +SQL `ADD COLUMN` requires an `AS` binding, so a plain column such as +`ALTER TABLE t ADD COLUMN c INT` is rejected. Add an empty, nullable column +with the SDK instead -- see +[Add nullable columns](/management/table#schema-add-nullable-columns). ### A computed column {#computed-column} @@ -52,6 +51,9 @@ The column is committed empty and the expression is stored, not evaluated -- declaring one on a billion-row table costs what declaring one on an empty table costs. [`REFRESH COLUMN`](/sql/dml/refresh) fills it. +{/* +Draft for the Functions release. + ### A function column {#function-column} ```sql @@ -86,6 +88,7 @@ ALTER TABLE t ADD COLUMN c AS f(x) WITH (num_workers = '2') `WITH` on a plain computed column accepts only `version`. Resource options such as `num_workers` apply to function columns, where there are workers to size. +*/} ### Table properties {#properties} @@ -95,19 +98,23 @@ ALTER TABLE t SET UNENFORCED PRIMARY KEY (id) Records which column identifies a row. **Unenforced** is not a hedge in the naming: uniqueness is not checked on write, and declaring it does not make -duplicates impossible. It tells the planner what to assume. +duplicates impossible. It tells the planner what to assume. The column must be +declared `NOT NULL`. ```sql ALTER TABLE t SET WAL -ALTER TABLE t SET WAL WITH (sharding = 'bucket', buckets = 4) +ALTER TABLE t SET WAL WITH (sharding = 'bucket', column = 'id', buckets = 4) ALTER TABLE t SET WAL WITH (maintained_indexes = '') ALTER TABLE t UNSET WAL ``` A write-ahead log makes small frequent writes cheap by batching them out of the commit path. `sharding` and `buckets` spread the log so concurrent writers do -not contend on one place; `maintained_indexes` chooses which indexes are kept -current against WAL rows. See +not contend on one place; `bucket` sharding names its `column`, which must be +the table's unenforced primary key, and inserts into the table then need a +scalar index on that key. `maintained_indexes` chooses which indexes are kept +current against WAL rows. A table with a computed column cannot take a +write-ahead log. See [Background jobs](/maintenance/background-jobs#wal) for how the log gets folded back into the table. @@ -162,7 +169,8 @@ Removes the table and every version of it. There is no undrop. -**Enterprise job type:** `drop_table`. See +**Enterprise job type:** `drop_table` or `drop_object`, depending on how the +deployment stores its catalog. See [Background jobs](/maintenance/background-jobs#teardown). diff --git a/docs/web/sql/ddl/view.mdx b/docs/web/sql/ddl/view.mdx index fc0807058..8b4337fac 100644 --- a/docs/web/sql/ddl/view.mdx +++ b/docs/web/sql/ddl/view.mdx @@ -6,7 +6,7 @@ icon: eye keywords: ["create view", "drop view", "sql", "catalog"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) A view stores a **query**, not rows. Reading it runs the query against current data, so a view is never stale -- and never faster than the query it wraps. It diff --git a/docs/web/sql/dml/copy.mdx b/docs/web/sql/dml/copy.mdx index 837b92285..658b20b59 100644 --- a/docs/web/sql/dml/copy.mdx +++ b/docs/web/sql/dml/copy.mdx @@ -6,7 +6,11 @@ icon: file-export keywords: ["copy", "export", "parquet", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) + +Recent deployment versions refuse `COPY` over the SQL endpoint with "COPY is not +supported". On those, export a result by reading it through a Flight SQL client +or an SDK and writing the files from your application. + ```sql COPY (SELECT id, title FROM documents) TO 's3://bucket/export/' STORED AS PARQUET diff --git a/docs/web/sql/dml/delete.mdx b/docs/web/sql/dml/delete.mdx index da71dd9eb..90965062b 100644 --- a/docs/web/sql/dml/delete.mdx +++ b/docs/web/sql/dml/delete.mdx @@ -6,12 +6,16 @@ icon: trash keywords: ["delete", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ```sql -DELETE FROM documents WHERE created_at < '2025-01-01' +DELETE FROM documents WHERE created_at < TIMESTAMP '2025-01-01 00:00:00' ``` +Compare a timestamp column with a typed literal, as above, or with +`CAST('2025-01-01' AS TIMESTAMP)`. A bare string such as `'2025-01-01'` is +rejected in a `DELETE` predicate, even though a `SELECT` accepts it. + Deletion marks rows rather than erasing them, so the data remains recoverable for as long as the version holding it survives [retention](/maintenance/cleanup). diff --git a/docs/web/sql/dml/insert.mdx b/docs/web/sql/dml/insert.mdx index 2be017f06..61fcb9994 100644 --- a/docs/web/sql/dml/insert.mdx +++ b/docs/web/sql/dml/insert.mdx @@ -6,7 +6,7 @@ icon: plus keywords: ["insert", "insert into", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ```sql INSERT INTO documents (id, title) VALUES (1, 'Merlin'), (2, 'Arthur') diff --git a/docs/web/sql/dml/refresh.mdx b/docs/web/sql/dml/refresh.mdx index 2d00c1104..1a14adef8 100644 --- a/docs/web/sql/dml/refresh.mdx +++ b/docs/web/sql/dml/refresh.mdx @@ -6,7 +6,7 @@ icon: arrows-rotate keywords: ["refresh column", "refresh materialized view", "backfill", "cascade"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) LanceDB extensions with no DataFusion equivalent. They are what makes [feature engineering](/feature-engineering) reachable from SQL: a column @@ -52,33 +52,35 @@ REFRESH COLUMN embedding ON documents `num_workers` sets a count; the budget options size each worker. They are alternatives rather than a pair. +{/* +Draft for the Functions release. + `error_policy` and `num_workers` are refused for a column filled by a registered [function](/management/function) with its own resource budget -- use the CPU, memory and GPU options there. +*/} ## REFRESH MATERIALIZED VIEW {#refresh-materialized-view} ```sql -REFRESH MATERIALIZED VIEW [CASCADE] [WITH (