rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-14 07:42:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into gatekeeper/fix-4299-1

Browse Source
This commit is contained in:
Gatefixer
2026-10-13 22:20:42 +00:00
parent d81eed4586 0212cfd3d6
commit bedbb35787
501 changed files with 71456 additions and 15406 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/Cargo.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.40.0-beta.11"
version = "0.41.0-beta.6"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
@@ -8,7 +8,7 @@ license.workspace = true
repository.workspace = true
keywords.workspace = true
categories.workspace = true
rust-version = "1.91.0"
rust-version = "1.94.0"
[lib]
name = "_lancedb"
python/pyproject.toml
+3 -2
View File
@@ -10,7 +10,7 @@ dependencies = [
"pyarrow>=16",
"pydantic>=2.7.4,<3",
"tqdm>=4.27.0",
"lance-namespace>=0.3.2"
"lance-namespace>=0.13.0"
]
description = "lancedb"
authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
@@ -63,7 +63,7 @@ tests = [
"polars>=0.19, <=1.32.3",
"pyarrow<25",
"pyarrow-stubs>=16.0",
"pylance==9.0.0",
"pylance==14.0.0b12",
"requests>=2.31.0",
"datafusion>=54,<55",
"opentelemetry-sdk>=1.30.0",
@@ -73,6 +73,7 @@ dev = [
"pre-commit>=3.5.0",
"pyright>=1.1.350",
'typing-extensions>=4.0.0; python_version < "3.11"',
"maturin>=1.10",
]
docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings-python"]
clip = ["torch", "pillow>=12.1.1", "open-clip-torch"]
python/python/lancedb/__init__.py
+4 -5
View File
@@ -46,7 +46,6 @@ from .materialized_view import (
AsyncMaterializedView,
MaterializedView,
MaterializedViewDefinition,
vector_dedup as vector_dedup,
)
from .view import ViewDescription as ViewDescription
from .table import AsyncTable, Table
@@ -62,7 +61,6 @@ from .namespace import (
from .catalog import (
AsyncCatalog,
Catalog,
ListDatabasesResponse,
connect_catalog,
connect_catalog_async,
)
@@ -323,8 +321,8 @@ def tokenize(
language: str = "English",
max_token_length: Optional[int] = 40,
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
stem: Optional[bool] = None,
remove_stop_words: Optional[bool] = None,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
@@ -335,6 +333,8 @@ def tokenize(
This does not require an FTS index. The tokenizer options match
:class:`lancedb.index.FTS`. ``custom_stop_words`` accepts a list of strings.
When ``stem`` or ``remove_stop_words`` is None, the base tokenizer's default
applies: False for ``code`` and ``ngram``, True otherwise.
"""
return _tokenize(
@@ -578,7 +578,6 @@ async def connect_async(
__all__ = [
"Catalog",
"AsyncCatalog",
"ListDatabasesResponse",
"connect_catalog",
"connect_catalog_async",
"AsyncMaterializedView",
python/python/lancedb/_blob.py
+75 -19
View File
@@ -6,6 +6,7 @@
from __future__ import annotations
import io
import operator
from collections.abc import Awaitable, Callable, Iterable
from typing import TYPE_CHECKING, Optional, Union
@@ -110,7 +111,9 @@ def projection_includes_blob_column(
if projection is None:
return True
for output, source in _iter_projection_pairs(projection):
if output in columns or source in columns:
if output in columns or any(
column == source or column.startswith(f"{source}.") for column in columns
):
return True
return False
@@ -122,14 +125,16 @@ def blob_v2_projection_sources(
blob_columns = row_addressable_blob_v2_paths(schema)
if not blob_columns:
return {}
columns = set(blob_columns)
if projection is None:
return {column: column for column in blob_columns}
return {
output: source
for output, source in _iter_projection_pairs(projection)
if source in columns
}
sources = {}
for output, source in _iter_projection_pairs(projection):
for column in blob_columns:
if column == source:
sources[output] = column
elif column.startswith(f"{source}."):
sources[f"{output}{column[len(source) :]}"] = column
return sources
def v2_projection_needs_row_id(
@@ -180,7 +185,7 @@ async def replace_v2_blob_columns_with_bytes(
fetch_blobs: FetchBlobsAsync,
) -> pa.Table:
for output_name, source_name in blob_sources.items():
if output_name not in tbl.column_names:
if not _has_column_path(tbl.schema, output_name):
continue
blobs = await fetch_blobs(source_name, tbl)
tbl = _set_blob_column(tbl, output_name, blobs)
@@ -193,7 +198,7 @@ def replace_v2_blob_columns_with_bytes_sync(
fetch_blobs: FetchBlobsSync,
) -> pa.Table:
for output_name, source_name in blob_sources.items():
if output_name not in tbl.column_names:
if not _has_column_path(tbl.schema, output_name):
continue
blobs = fetch_blobs(source_name, tbl)
tbl = _set_blob_column(tbl, output_name, blobs)
@@ -287,9 +292,46 @@ def _iter_projection_pairs(
yield name, source if source is not None else expr.to_sql()
def _set_blob_column(tbl: pa.Table, output_name: str, blobs: pa.Array) -> pa.Table:
def _has_column_path(schema: pa.Schema, path: str) -> bool:
if schema.get_field_index(path) >= 0:
return True
top_name, *rest = path.split(".")
index = schema.get_field_index(top_name)
if index < 0:
return False
field = schema.field(index)
for name in rest:
if not pa.types.is_struct(field.type):
return False
index = field.type.get_field_index(name)
if index < 0:
return False
field = field.type.field(index)
return True
def _set_blob_column(
tbl: pa.Table, output_name: str, blobs: pa.Array | pa.ChunkedArray
) -> pa.Table:
index = tbl.schema.get_field_index(output_name)
return tbl.set_column(index, pa.field(output_name, blobs.type), [blobs])
if index >= 0:
return tbl.set_column(index, pa.field(output_name, blobs.type), [blobs])
parent_path, leaf_name = output_name.rsplit(".", 1)
if isinstance(blobs, pa.ChunkedArray):
blobs = blobs.combine_chunks()
def replace_blob(children: list, child_fields: list) -> None:
for i, field in enumerate(child_fields):
if field.name == leaf_name:
children[i] = blobs
child_fields[i] = pa.field(
leaf_name, blobs.type, nullable=field.nullable
)
return
raise KeyError(f"Blob field '{leaf_name}' not found in struct '{parent_path}'")
return _transform_struct_column(tbl, parent_path, replace_blob)
def _embed_row_id_in_column(tbl: pa.Table, path: str, row_ids: pa.Array) -> pa.Table:
@@ -391,16 +433,30 @@ def _leaf_struct_column(tbl: pa.Table, path: str) -> pa.StructArray:
def _normalize_blob_row_ids(
row_ids: Union[list[int], pa.Table], blob_column: str
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table], blob_column: str
) -> list[int]:
if isinstance(row_ids, pa.Table):
return read_row_ids_from_hits(row_ids, blob_column)
if isinstance(row_ids, (pa.Array, pa.ChunkedArray)):
raise ValueError(
"pass a query table with _rowid, not a column array "
"(use fetch_blobs('image', hits), not fetch_blobs('image', hits['image']))"
)
return list(row_ids)
values = read_row_ids_from_hits(row_ids, blob_column)
elif isinstance(row_ids, (pa.Array, pa.ChunkedArray)):
if not pa.types.is_integer(row_ids.type):
raise ValueError(
"row id arrays must have an integer type; pass a query table with "
"_rowid when using a blob descriptor column"
)
values = row_ids.to_pylist()
else:
values = list(row_ids)
normalized = []
for value in values:
try:
row_id = operator.index(value)
except TypeError as exc:
raise ValueError("row ids must be integers without nulls") from exc
if row_id < 0:
raise ValueError("row ids must be non-negative")
normalized.append(row_id)
return normalized
def _wrap_blob_files(handles: Iterable[object]) -> list[Optional[BlobFile]]:
python/python/lancedb/_lancedb.pyi
+76 -26
View File
@@ -1,6 +1,6 @@
from datetime import date, datetime, timedelta
from decimal import Decimal
from typing import Dict, List, Optional, Tuple, Any, TypedDict, Union, Literal
from typing import Dict, List, Optional, Sequence, Tuple, Any, TypedDict, Union, Literal
from uuid import UUID
import pyarrow as pa
@@ -58,8 +58,8 @@ def tokenize(
language: str = "English",
max_token_length: Optional[int] = 40,
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
stem: Optional[bool] = None,
remove_stop_words: Optional[bool] = None,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
@@ -149,12 +149,24 @@ class Connection(object):
limit: Optional[int],
) -> list[str]: ... # Deprecated: Use list_tables instead
async def open_job(self, job_id: str) -> Job: ...
async def create_function_async(self, request_json: str) -> Job: ...
async def get_function(self, name: str, version: str) -> str: ...
async def list_functions(self) -> List[str]: ...
async def drop_function(self, name: str, version: str) -> bool: ...
async def create_function_async(
self, request_json: str, namespace_path: Optional[List[str]] = None
) -> Job: ...
async def get_function(
self, name: str, version: str, namespace_path: Optional[List[str]] = None
) -> str: ...
def list_functions(
self,
namespace_path: Optional[List[str]] = None,
*,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> FunctionListing: ...
async def drop_function(
self, name: str, version: str, namespace_path: Optional[List[str]] = None
) -> bool: ...
async def drop_function_async(
self, name: str, version: str
self, name: str, version: str, namespace_path: Optional[List[str]] = None
) -> Tuple[bool, Job]: ...
async def create_secret(
self, name: str, value: str, namespace_path: Optional[List[str]] = None
@@ -162,9 +174,13 @@ class Connection(object):
async def alter_secret(
self, name: str, value: str, namespace_path: Optional[List[str]] = None
) -> None: ...
async def list_secrets(
self, namespace_path: Optional[List[str]] = None
) -> List[str]: ...
def list_secrets(
self,
namespace_path: Optional[List[str]] = None,
*,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> NameListing: ...
async def drop_secret(
self, name: str, namespace_path: Optional[List[str]] = None
) -> None: ...
@@ -183,10 +199,16 @@ class Connection(object):
async def drop_view_async(
self, name: str, namespace_path: Optional[List[str]] = None
) -> Job: ...
async def list_views(
self, namespace_path: Optional[List[str]] = None
) -> List[str]: ...
async def list_jobs(self) -> List[JobInfo]: ...
def list_views(
self,
namespace_path: Optional[List[str]] = None,
*,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> NameListing: ...
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> JobListing: ...
async def cancel_job(self, job_id: str) -> bool: ...
async def pause_job(self, job_id: str) -> str: ...
async def resume_job(self, job_id: str) -> str: ...
@@ -195,6 +217,7 @@ class Connection(object):
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[pa.RecordBatch] = None,
) -> SqlQuery: ...
async def describe_query(self, query_id: UUID) -> QueryDescription: ...
async def create_table(
@@ -247,8 +270,6 @@ class Connection(object):
filter: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
*,
vector_source_json: Optional[str] = None,
) -> Table: ...
async def create_materialized_view_async(
self,
@@ -258,10 +279,10 @@ class Connection(object):
filter: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
*,
vector_source_json: Optional[str] = None,
) -> Job: ...
async def list_materialized_views(self) -> List[str]: ...
def list_materialized_views(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> NameListing: ...
async def drop_materialized_view(
self, name: str, namespace_path: Optional[List[str]] = None
) -> None: ...
@@ -402,6 +423,7 @@ class Table:
def name(self) -> str: ...
def __repr__(self) -> str: ...
def is_open(self) -> bool: ...
def _is_native(self) -> bool: ...
def close(self) -> None: ...
async def schema(self) -> pa.Schema: ...
async def add(
@@ -492,10 +514,10 @@ class Table:
limit: Optional[int] = None,
) -> FunctionErrors: ...
async def refresh_materialized_view(
self, full: bool = False, source_version: Optional[int] = None
self, source_version: Optional[int] = None
) -> RefreshMaterializedViewResult: ...
async def refresh_materialized_view_async(
self, full: bool = False, source_version: Optional[int] = None
self, source_version: Optional[int] = None
) -> Job: ...
async def materialized_view_definition(self) -> str: ...
async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...
@@ -541,7 +563,7 @@ class Table:
async def fetch_blob_ranges(
self,
column: str,
requests: List[Tuple[int, int, int]],
requests: Sequence[Sequence[int]],
) -> pa.LargeBinaryArray: ...
async def fetch_blob_files(
self, column: str, row_ids: list[int]
@@ -705,6 +727,7 @@ class VectorQuery:
def use_lsm(self, enable: bool): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
def search_effort(self, search_effort: Optional[float]): ...
def minimum_nprobes(self, minimum_nprobes: int): ...
def maximum_nprobes(self, maximum_nprobes: int): ...
def bypass_vector_index(self): ...
@@ -730,6 +753,7 @@ class HybridQuery:
def distance_type(self, distance_type: str): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
def search_effort(self, search_effort: Optional[float]): ...
def minimum_nprobes(self, minimum_nprobes: int): ...
def maximum_nprobes(self, maximum_nprobes: int): ...
def bypass_vector_index(self): ...
@@ -756,8 +780,10 @@ class PyQueryRequest:
use_lsm: Optional[bool]
column: Optional[str]
query_vector: Optional[List[pa.Array]]
nprobes: Optional[int]
minimum_nprobes: Optional[int]
maximum_nprobes: Optional[int]
search_effort: Optional[float]
lower_bound: Optional[float]
upper_bound: Optional[float]
ef: Optional[int]
@@ -938,9 +964,15 @@ class Catalog:
async def drop_database(
self, name: str, *, ignore_missing: bool = False
) -> None: ...
async def list_databases(
self, *, limit: Optional[int] = None, page_token: Optional[str] = None
) -> tuple[list[str], Optional[str]]: ...
def list_databases(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> DatabaseNames: ...
class DatabaseNames:
def num_page_results(self) -> int: ...
def page_token(self) -> Optional[str]: ...
def __aiter__(self) -> DatabaseNames: ...
async def __anext__(self) -> str: ...
async def connect_catalog(
endpoint: str,
@@ -951,3 +983,21 @@ async def connect_catalog(
read_consistency_interval: Optional[float] = None,
oauth_config: Optional[Any] = None,
) -> Catalog: ...
class NameListing:
def num_page_results(self) -> int: ...
def page_token(self) -> Optional[str]: ...
def __aiter__(self) -> NameListing: ...
async def __anext__(self) -> str: ...
class FunctionListing:
def num_page_results(self) -> int: ...
def page_token(self) -> Optional[str]: ...
def __aiter__(self) -> FunctionListing: ...
async def __anext__(self) -> str: ...
class JobListing:
def num_page_results(self) -> int: ...
def page_token(self) -> Optional[str]: ...
def __aiter__(self) -> JobListing: ...
async def __anext__(self) -> JobInfo: ...
python/python/lancedb/catalog.py
+91 -19
View File
@@ -3,9 +3,8 @@
"""Remote catalogs manage databases through a server's root namespace."""
from dataclasses import dataclass
from datetime import timedelta
from typing import Any, Optional, Union
from typing import Any, AsyncIterator, Iterator, Optional, Union
from . import _lancedb
from .background_loop import LOOP
@@ -14,12 +13,66 @@ from .remote import ClientConfig, OAuthConfig
from .remote.db import RemoteDBConnection
@dataclass
class ListDatabasesResponse:
"""A page of database names and an optional continuation token."""
class AsyncDatabaseNames(AsyncIterator[str]):
"""An async iterator of database names with pagination state.
databases: list[str]
page_token: Optional[str] = None
Returned by [list_databases][lancedb.catalog.AsyncCatalog.list_databases].
"""
def __init__(self, inner: _lancedb.DatabaseNames):
self._inner = inner
def __aiter__(self) -> "AsyncDatabaseNames":
return self
async def __anext__(self) -> str:
return await self._inner.__anext__()
def num_page_results(self) -> int:
"""Return the number of names available without another REST request."""
return self._inner.num_page_results()
def page_token(self) -> Optional[str]:
"""Return the token for the next REST request.
Before the first request, this is the supplied starting token (``None``
starts at the beginning). After the final page is fetched, it is ``None``,
even if names remain cached. Drain the cache before saving a token to
avoid skipping those names when resuming. A failed request terminates
iteration and leaves its token available for resuming a new iterator.
"""
return self._inner.page_token()
class DatabaseNames(Iterator[str]):
"""A synchronous iterator of database names with pagination state.
Returned by [Catalog.list_databases][lancedb.catalog.Catalog.list_databases].
"""
def __init__(self, inner: AsyncDatabaseNames):
self._inner = inner
def __iter__(self) -> "DatabaseNames":
return self
def __next__(self) -> str:
try:
return LOOP.run(self._inner.__anext__())
except StopAsyncIteration:
raise StopIteration from None
def num_page_results(self) -> int:
"""Return the number of names available without another REST request."""
return self._inner.num_page_results()
def page_token(self) -> Optional[str]:
"""Return the token for the next REST request.
See [page_token][lancedb.catalog.AsyncDatabaseNames.page_token]
for initial, cached, final-page, and error behavior.
"""
return self._inner.page_token()
class AsyncCatalog:
@@ -50,14 +103,25 @@ class AsyncCatalog:
"""Connect to an existing database by its logical name."""
return AsyncConnection(await self._inner.connect_database(name))
async def list_databases(
self, *, limit: Optional[int] = None, page_token: Optional[str] = None
) -> ListDatabasesResponse:
"""List a page of databases. Pass the returned token for the next page."""
names, token = await self._inner.list_databases(
limit=limit, page_token=page_token
def list_databases(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> AsyncDatabaseNames:
"""Iterate lazily over all database names using ``async for``.
``page_token`` resumes from a saved token; ``None`` starts at the beginning.
``page_limit`` sets the maximum names per REST response (not the total);
``None`` uses the server default. Request errors are raised during iteration.
Examples
--------
```python
async for name in catalog.list_databases():
print(name)
```
"""
return AsyncDatabaseNames(
self._inner.list_databases(page_token=page_token, page_limit=page_limit)
)
return ListDatabasesResponse(names, token)
async def drop_database(self, name: str, *, ignore_missing: bool = False) -> None:
"""Drop an empty database. A nonempty database is an error."""
@@ -73,7 +137,8 @@ class Catalog:
```python
catalog = lancedb.connect_catalog("https://my-server.example", api_key="secret")
db = catalog.create_database("analytics", exist_ok=True)
page = catalog.list_databases(limit=20)
for name in catalog.list_databases():
print(name)
```
"""
@@ -118,10 +183,17 @@ class Catalog:
)
def list_databases(
self, *, limit: Optional[int] = None, page_token: Optional[str] = None
) -> ListDatabasesResponse:
"""List a page of databases. Pass the returned token for the next page."""
return LOOP.run(self._inner.list_databases(limit=limit, page_token=page_token))
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> DatabaseNames:
"""Iterate lazily over database names, fetching more results as needed.
``page_token`` resumes from a saved token; ``None`` starts at the beginning.
``page_limit`` sets the maximum names per REST response (not the total);
``None`` uses the server default. Request errors are raised during iteration.
"""
return DatabaseNames(
self._inner.list_databases(page_token=page_token, page_limit=page_limit)
)
def drop_database(self, name: str, *, ignore_missing: bool = False) -> None:
"""Drop an empty database. A nonempty database is an error."""
python/python/lancedb/db.py
+445 -121
View File
@@ -31,6 +31,7 @@ if sys.version_info >= (3, 12):
else:
from overrides import EnforceOverrides, override # type: ignore
from lancedb.listing import AsyncListing, Listing
from lancedb.embeddings.registry import EmbeddingFunctionRegistry
from lancedb.common import data_to_reader, sanitize_uri, validate_schema
@@ -52,12 +53,11 @@ from .functions import FunctionVersion, UdfDefinition
from .job import AsyncJob, Job, _typed_job
from .sql import AsyncQuery as AsyncSqlQuery
from .sql import Query as SqlQuery
from .sql import QueryDescription
from .sql import QueryDescription, QueryParameters, to_parameter_batch
from .materialized_view import (
AsyncMaterializedView,
MaterializedView,
MaterializedViewSource,
VectorDedupSource,
SelectArg,
normalize_select,
)
@@ -122,7 +122,48 @@ def _view_description(
class DBConnection(EnforceOverrides):
"""An active LanceDB connection interface."""
"""An active LanceDB connection interface.
Use [close][lancedb.db.DBConnection.close] to release the connection's
underlying resources, or use the connection as a context manager to close it
automatically when leaving the block, including when an exception is raised.
Examples
--------
>>> import lancedb
>>> with lancedb.connect("memory://") as db:
... assert db.is_open()
>>> db.is_open()
False
"""
def __enter__(self) -> DBConnection:
return self
def __exit__(self, *_) -> None:
self.close()
@abstractmethod
def is_open(self) -> bool:
"""Return True if the connection is open."""
pass
@abstractmethod
def close(self) -> None:
"""Close the connection, releasing any underlying resources.
It is safe to call this method multiple times. Database operations on a
closed connection raise ``RuntimeError: Connection is closed``.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("memory://")
>>> db.close()
>>> db.is_open()
False
"""
pass
def list_namespaces(
self,
@@ -276,7 +317,7 @@ class DBConnection(EnforceOverrides):
page_token: Optional[str] = None,
limit: Optional[int] = None,
) -> ListTablesResponse:
"""List all tables in this database with pagination support.
"""List a page of tables in this database.
Parameters
----------
@@ -287,7 +328,8 @@ class DBConnection(EnforceOverrides):
Token for pagination. Use the token from a previous response
to get the next page of results.
limit: int, optional
The maximum number of results to return.
The maximum number of results to return, default 100.
Zero returns an empty page without a continuation token.
Returns
-------
@@ -302,7 +344,7 @@ class DBConnection(EnforceOverrides):
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
@@ -316,8 +358,9 @@ class DBConnection(EnforceOverrides):
page_token: str, optional
The token to use for pagination. If not present, start from the beginning.
Typically, this token is last table name from the previous page.
limit: int, default 10
The size of the page to return.
limit: int, optional
The maximum number of tables to return. None returns all tables;
zero returns an empty list.
Returns
-------
@@ -368,9 +411,10 @@ class DBConnection(EnforceOverrides):
- [LanceModel][lancedb.pydantic.LanceModel]
mode: str; default "create"
The mode to use when creating the table.
Can be either "create" or "overwrite".
Can be "create", "overwrite", or "exist_ok".
By default, if the table already exists, an exception is raised.
If you want to overwrite the table, use mode="overwrite".
To open an existing table without adding data, use mode="exist_ok".
exist_ok: bool, default False
If a table by the same name already exists, then raise an exception
if exist_ok=False. If exist_ok=True, then open the existing table;
@@ -571,21 +615,12 @@ class DBConnection(EnforceOverrides):
table. The view is a normal table: it can be queried, indexed and
searched, and it appears in ``table_names``.
Ordinary source tables must have stable row ids (create them with the
``new_table_enable_stable_row_ids`` storage option): they keep the
view's provenance valid across source compactions, and cannot be
enabled after a table exists. Native dedup sources also support physical
row IDs interpreted against their pinned snapshot.
Parameters
----------
name: str
The name of the view.
source: str or VectorDedupSource
The name of the source table, or an indexed dedup source declared
with [vector_dedup][lancedb.vector_dedup]. Dedup sources capture a
fixed snapshot and preserve the source table. They accept no
additional select, where or limit options.
source: str
The name of the source table.
select: list or dict, optional
The view's columns: column names, ``(alias, SQL expression)``
pairs, or a dict of the same. Omitting it selects every source
@@ -636,11 +671,16 @@ class DBConnection(EnforceOverrides):
"materialized views are not supported on this connection type"
)
def list_materialized_views(self) -> List[str]:
def list_materialized_views(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> Listing[str]:
"""The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per
table.
Local listings inspect table schemas one page at a time.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
@@ -784,6 +824,7 @@ class DBConnection(EnforceOverrides):
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
"""Build and register a scalar Python UDF, then return its version.
@@ -800,6 +841,9 @@ class DBConnection(EnforceOverrides):
Function needs, each naming a Secret and the environment variable
its value arrives in. The Function's source is unchanged by this;
it reads the variable the way it already did.
namespace_path : list of str, optional
The namespace holding the Function. None or an empty list is the
root namespace.
Examples
--------
@@ -815,13 +859,16 @@ class DBConnection(EnforceOverrides):
)
```
"""
return self.create_function_async(definition, secrets=secrets).wait()
return self.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
).wait()
def create_function_async(
self,
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
"""Submit a scalar Python UDF for building and registration.
@@ -833,17 +880,30 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def get_function(self, name: str, *, version: str) -> FunctionVersion:
def get_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
"""Open one exact immutable Function version from the remote catalog."""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
)
def list_functions(self) -> List[FunctionVersion]:
"""List every published immutable Function version.
def list_functions(
self,
*,
namespace_path: Optional[List[str]] = None,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> Listing[FunctionVersion]:
"""List every published immutable Function version in a namespace.
Results are ordered by Function name then version. Local connections
raise ``NotImplementedError``.
Functions in child namespaces are not included. Results are ordered by
Function name then version. Local connections raise
``NotImplementedError``.
Examples
--------
@@ -852,12 +912,22 @@ class DBConnection(EnforceOverrides):
```python
[(function.name, function.version) for function in db.list_functions()]
```
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
)
def drop_function(self, name: str, *, version: str) -> bool:
def drop_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> bool:
"""Drop a Function name and the object it was bound to.
The requested version must exist in the currently named object. Returns
@@ -870,7 +940,13 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]:
def drop_function_async(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> Tuple[bool, Job]:
"""Drop a Function name and return its cleanup job.
The name is unbound before this returns; the object's content may still
@@ -911,11 +987,21 @@ class DBConnection(EnforceOverrides):
"Secret operations are not supported for this connection type"
)
def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
def list_secrets(
self,
*,
namespace_path: Optional[List[str]] = None,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> Listing[str]:
"""The names of every Secret in this database.
Names only. No method returns a stored credential, by construction
rather than by policy. Local connections raise ``NotImplementedError``.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
raise NotImplementedError(
"Secret operations are not supported for this connection type"
@@ -1013,12 +1099,22 @@ class DBConnection(EnforceOverrides):
"View operations are not supported for this connection type"
)
def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
def list_views(
self,
*,
namespace_path: Optional[List[str]] = None,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> Listing[str]:
"""The names of the views in one namespace.
Names only; a definition comes from
[describe_view][lancedb.db.DBConnection.describe_view]. Local
connections raise ``NotImplementedError``.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
raise NotImplementedError(
"View operations are not supported for this connection type"
@@ -1037,8 +1133,15 @@ class DBConnection(EnforceOverrides):
"""
raise NotImplementedError("open_job is not supported for this connection type")
def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> Listing[JobInfo]:
"""List server-side jobs across the database's tables.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
raise NotImplementedError("list_jobs is not supported for this connection type")
def cancel_job(self, job_id: str) -> bool:
@@ -1077,6 +1180,7 @@ class DBConnection(EnforceOverrides):
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> pa.RecordBatchReader:
"""Execute SQL and return a blocking Arrow reader.
@@ -1087,6 +1191,7 @@ class DBConnection(EnforceOverrides):
return self.execute_query_async(
query,
default_namespace_path=default_namespace_path,
parameters=parameters,
).reader()
def execute_query_async(
@@ -1094,9 +1199,32 @@ class DBConnection(EnforceOverrides):
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> SqlQuery:
"""Start executing SQL and return its query handle.
``parameters`` binds values to the statement's ``$1`` / ``$name``
placeholders. They travel as Arrow rather than as SQL text, so a
float keeps its exact value and type and a vector stays a compact
fixed-size list. A list binds by position, a dict binds by name, and
a one-row ``pyarrow.RecordBatch`` or ``pyarrow.Table`` binds both
ways. A 1-D numpy or pyarrow array becomes a fixed-size list of its
element type, the shape of a vector column:
.. code-block:: python
db.execute_query(
"SELECT id FROM docs ORDER BY distance(vector, $1) LIMIT $2",
parameters=[np.array([0.1, 0.2, 0.3], dtype=np.float32), 10],
)
Parameters are supported in queries, not in DDL or DML. A
parameterized query runs on the call that returns its rows:
cancelling it, or dropping its handle before reading, stops it on the
server. Its status follows that stream as well: ``describe()``
reports ``running`` until the reader has received every row, so open
``reader()`` directly rather than waiting for ``finished``.
Local connections do not support SQL.
"""
raise NotImplementedError("SQL is not supported for this connection type")
@@ -1247,6 +1375,14 @@ class LanceDBConnection(DBConnection):
def __repr__(self) -> str:
return f"{self.__class__.__name__}(uri={self._conn.uri!r})"
@override
def is_open(self) -> bool:
return self._conn.is_open()
@override
def close(self) -> None:
self._conn.close()
@override
def serialize(self) -> str:
import json
@@ -1356,7 +1492,7 @@ class LanceDBConnection(DBConnection):
page_token: Optional[str] = None,
limit: Optional[int] = None,
) -> ListTablesResponse:
"""List all tables in this database with pagination support.
"""List a page of tables in this database.
Parameters
----------
@@ -1367,7 +1503,8 @@ class LanceDBConnection(DBConnection):
Token for pagination. Use the token from a previous response
to get the next page of results.
limit: int, optional
The maximum number of results to return.
The maximum number of results to return, default 100.
Zero returns an empty page without a continuation token.
Returns
-------
@@ -1386,7 +1523,7 @@ class LanceDBConnection(DBConnection):
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
@@ -1401,8 +1538,9 @@ class LanceDBConnection(DBConnection):
The namespace to list tables in.
page_token: str, optional
The token to use for pagination.
limit: int, default 10
The maximum number of tables to return.
limit: int, optional
The maximum number of tables to return. None returns all tables;
zero returns an empty list.
Returns
-------
@@ -1469,8 +1607,6 @@ class LanceDBConnection(DBConnection):
"""
if namespace_path is None:
namespace_path = []
if mode.lower() not in ["create", "overwrite"]:
raise ValueError("mode must be either 'create' or 'overwrite'")
validate_table_name(name)
tbl = LanceTable.create(
@@ -1578,10 +1714,7 @@ class LanceDBConnection(DBConnection):
Examples
--------
>>> import lancedb
>>> db = lancedb.connect(
... "./.lancedb",
... storage_options={"new_table_enable_stable_row_ids": "true"},
... )
>>> db = lancedb.connect("./.lancedb")
>>> data = [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]
>>> table = db.create_table("people", data)
>>> view = db.create_materialized_view(
@@ -1636,9 +1769,20 @@ class LanceDBConnection(DBConnection):
return view
@override
def list_materialized_views(self) -> List[str]:
"""The names of the materialized views in this database."""
return LOOP.run(self._conn.list_materialized_views())
def list_materialized_views(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> Listing[str]:
"""The names of the materialized views in this database.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return Listing(
self._conn.list_materialized_views(
page_token=page_token, page_limit=page_limit
)
)
@override
def drop_materialized_view(
@@ -1808,25 +1952,78 @@ class LanceDBConnection(DBConnection):
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
job = LOOP.run(
self._conn.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
)
)
return Job(job)
@override
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
def get_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
return LOOP.run(
self._conn.get_function(
name, version=version, namespace_path=namespace_path
)
)
@override
def list_functions(self) -> List[FunctionVersion]:
return LOOP.run(self._conn.list_functions())
def list_functions(
self,
*,
namespace_path: Optional[List[str]] = None,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> Listing[FunctionVersion]:
"""List functions.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return Listing(
self._conn.list_functions(
namespace_path=namespace_path,
page_token=page_token,
page_limit=page_limit,
)
)
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
def drop_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> bool:
return LOOP.run(
self._conn.drop_function(
name, version=version, namespace_path=namespace_path
)
)
@override
def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]:
dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version))
def drop_function_async(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> Tuple[bool, Job]:
dropped, job = LOOP.run(
self._conn.drop_function_async(
name, version=version, namespace_path=namespace_path
)
)
return dropped, Job(job)
@override
@@ -1842,8 +2039,26 @@ class LanceDBConnection(DBConnection):
LOOP.run(self._conn.alter_secret(name, value, namespace_path=namespace_path))
@override
def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path))
def list_secrets(
self,
*,
namespace_path: Optional[List[str]] = None,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> Listing[str]:
"""List secrets.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return Listing(
self._conn.list_secrets(
namespace_path=namespace_path,
page_token=page_token,
page_limit=page_limit,
)
)
@override
def drop_secret(
@@ -1886,13 +2101,40 @@ class LanceDBConnection(DBConnection):
)
@override
def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
return LOOP.run(self._conn.list_views(namespace_path=namespace_path))
def list_views(
self,
*,
namespace_path: Optional[List[str]] = None,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> Listing[str]:
"""List views.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return Listing(
self._conn.list_views(
namespace_path=namespace_path,
page_token=page_token,
page_limit=page_limit,
)
)
@override
def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
return LOOP.run(self._conn.list_jobs())
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> Listing[JobInfo]:
"""List server-side jobs across the database's tables.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return Listing(
self._conn.list_jobs(page_token=page_token, page_limit=page_limit)
)
@override
def cancel_job(self, job_id: str) -> bool:
@@ -2128,7 +2370,7 @@ class AsyncConnection(object):
page_token: Optional[str] = None,
limit: Optional[int] = None,
) -> ListTablesResponse:
"""List all tables in this database with pagination support.
"""List a page of tables in this database.
Parameters
----------
@@ -2139,7 +2381,8 @@ class AsyncConnection(object):
Token for pagination. Use the token from a previous response
to get the next page of results.
limit: int, optional
The maximum number of results to return.
The maximum number of results to return, default 100.
Zero returns an empty page without a continuation token.
Returns
-------
@@ -2176,8 +2419,9 @@ class AsyncConnection(object):
This can be combined with limit to implement pagination by setting this to
the last table name from the previous page.
limit: int, default 10
The number of results to return.
limit: int, optional
The maximum number of tables to return. None returns all tables;
zero returns an empty list.
Returns
-------
@@ -2236,11 +2480,12 @@ class AsyncConnection(object):
- pyarrow.Schema
- [LanceModel][lancedb.pydantic.LanceModel]
mode: Literal["create", "overwrite"]; default "create"
mode: Literal["create", "overwrite", "exist_ok"]; default "create"
The mode to use when creating the table.
Can be either "create" or "overwrite".
Can be "create", "overwrite", or "exist_ok".
By default, if the table already exists, an exception is raised.
If you want to overwrite the table, use mode="overwrite".
To open an existing table without adding data, use mode="exist_ok".
exist_ok: bool, default False
If a table by the same name already exists, then raise an exception
if exist_ok=False. If exist_ok=True, then open the existing table;
@@ -2511,19 +2756,7 @@ class AsyncConnection(object):
where: Optional[str],
limit: Optional[int],
) -> Tuple[str, Dict[str, str]]:
if not isinstance(source, VectorDedupSource):
return source, {}
if select is not None or where is not None or limit is not None:
raise ValueError(
"vector_dedup cannot be combined with select, where or limit"
)
version = source.dataset_version
if version is None:
table = await self.open_table(source.source)
version = await table.version()
return source.source, {
"vector_source_json": source._native_source_json(version)
}
return source, {}
async def create_materialized_view(
self,
@@ -2591,13 +2824,22 @@ class AsyncConnection(object):
await view.definition()
return view
async def list_materialized_views(self) -> List[str]:
def list_materialized_views(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> AsyncListing[str]:
"""The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per
table.
Local listings inspect table schemas one page at a time.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return await self._inner.list_materialized_views()
return AsyncListing(
self._inner.list_materialized_views(
page_token=page_token, page_limit=page_limit
)
)
async def drop_materialized_view(
self,
@@ -2787,6 +3029,7 @@ class AsyncConnection(object):
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> AsyncJob[FunctionVersion]:
"""Submit a scalar Python UDF for building and registration.
@@ -2794,36 +3037,72 @@ class AsyncConnection(object):
artifact. Waiting on the job returns the immutable Function version.
``secrets`` is a sequence of
[EnvVarSecret][lancedb.secrets.EnvVarSecret], each naming a Secret and
the environment variable its value arrives in. Local connections raise
``NotImplementedError``.
the environment variable its value arrives in. ``namespace_path`` is
the namespace the Function is registered in; None or an empty list is
the root. Local connections raise ``NotImplementedError``.
"""
if not isinstance(definition, UdfDefinition):
raise TypeError("create_function_async requires a @udf definition")
request = definition.bind_secrets(secrets)
inner = await self._inner.create_function_async(request.to_canonical_json())
inner = await self._inner.create_function_async(
request.to_canonical_json(), namespace_path
)
return _typed_job(inner, FunctionVersion.from_json)
async def get_function(self, name: str, *, version: str) -> FunctionVersion:
async def get_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
"""Open one exact immutable Function version from the remote catalog."""
return FunctionVersion.from_json(await self._inner.get_function(name, version))
return FunctionVersion.from_json(
await self._inner.get_function(name, version, namespace_path)
)
async def list_functions(self) -> List[FunctionVersion]:
"""List every published immutable Function version.
def list_functions(
self,
*,
namespace_path: Optional[List[str]] = None,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> AsyncListing[FunctionVersion]:
"""List every published immutable Function version in a namespace.
Results are ordered by Function name then version. Local connections
raise ``NotImplementedError``.
Functions in child namespaces are not included. Results are ordered by
Function name then version. Local connections raise
``NotImplementedError``.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return [
FunctionVersion.from_json(value)
for value in await self._inner.list_functions()
]
return AsyncListing(
self._inner.list_functions(
namespace_path=namespace_path,
page_token=page_token,
page_limit=page_limit,
),
FunctionVersion.from_json,
)
async def drop_function(self, name: str, *, version: str) -> bool:
async def drop_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> bool:
"""Drop a Function name and the object it was bound to."""
return await self._inner.drop_function(name, version)
return await self._inner.drop_function(name, version, namespace_path)
async def drop_function_async(
self, name: str, *, version: str
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> Tuple[bool, AsyncJob]:
"""Drop a Function name and return its cleanup job.
@@ -2831,7 +3110,9 @@ class AsyncConnection(object):
be being deleted. Await :meth:`AsyncJob.wait` to wait for that to
finish.
"""
dropped, job = await self._inner.drop_function_async(name, version)
dropped, job = await self._inner.drop_function_async(
name, version, namespace_path
)
return dropped, AsyncJob(job)
async def create_secret(
@@ -2862,12 +3143,25 @@ class AsyncConnection(object):
list(validate_namespace_path(namespace_path)),
)
async def list_secrets(
self, *, namespace_path: Optional[List[str]] = None
) -> List[str]:
"""The names of every Secret in this database. Names only."""
return await self._inner.list_secrets(
list(validate_namespace_path(namespace_path))
def list_secrets(
self,
*,
namespace_path: Optional[List[str]] = None,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> AsyncListing[str]:
"""The names of every Secret in this database. Names only.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return AsyncListing(
self._inner.list_secrets(
namespace_path=namespace_path,
page_token=page_token,
page_limit=page_limit,
)
)
async def drop_secret(
@@ -2939,15 +3233,39 @@ class AsyncConnection(object):
await self._inner.drop_view_async(name, namespace_path=namespace_path)
)
async def list_views(
self, *, namespace_path: Optional[List[str]] = None
) -> List[str]:
"""The names of the views in one namespace."""
return await self._inner.list_views(list(namespace_path or []))
def list_views(
self,
*,
namespace_path: Optional[List[str]] = None,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> AsyncListing[str]:
"""The names of the views in one namespace.
async def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
return await self._inner.list_jobs()
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return AsyncListing(
self._inner.list_views(
namespace_path=namespace_path,
page_token=page_token,
page_limit=page_limit,
)
)
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> AsyncListing[JobInfo]:
"""List server-side jobs across the database's tables.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return AsyncListing(
self._inner.list_jobs(page_token=page_token, page_limit=page_limit)
)
async def cancel_job(self, job_id: str) -> bool:
"""Request cancellation of a server-side job by id.
@@ -2980,6 +3298,7 @@ class AsyncConnection(object):
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> AsyncRecordBatchReader:
"""Execute SQL and return an asynchronous Arrow reader.
@@ -2990,6 +3309,7 @@ class AsyncConnection(object):
submitted = await self.execute_query_async(
query,
default_namespace_path=default_namespace_path,
parameters=parameters,
)
return await submitted.reader()
@@ -2998,17 +3318,21 @@ class AsyncConnection(object):
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> AsyncSqlQuery:
"""Start executing SQL and return its query handle.
The database from ``connect_async`` is used for unqualified database
references. The namespace defaults to ``["public"]``. Local
connections raise ``NotImplementedError``.
references. The namespace defaults to ``["public"]``. ``parameters``
binds values to ``$1`` / ``$name`` placeholders; see
[DBConnection.execute_query_async][lancedb.db.DBConnection.execute_query_async].
Local connections raise ``NotImplementedError``.
"""
return AsyncSqlQuery(
await self._inner.execute_query_async(
query,
default_namespace_path=default_namespace_path,
parameters=to_parameter_batch(parameters),
)
)
python/python/lancedb/embeddings/base.py
+5 -1
View File
@@ -182,7 +182,11 @@ class EmbeddingFunction(BaseModel, ABC):
def VectorField(self, **kwargs):
"""
Creates a pydantic Field that can automatically annotate
the target vector column for this embedding function
the target vector column for this embedding function.
The field can be annotated as ``list[float]`` for compatibility with
static type checkers. LanceDB will infer the fixed vector dimension from
this embedding function.
"""
return Field(json_schema_extra={"vector_column_for": self}, **kwargs)
python/python/lancedb/functions.py
+11 -2
View File
@@ -356,16 +356,23 @@ _ObjectVersion = Annotated[
class FunctionVersion(_RemoteValue):
"""A pinned object revision, independent of its executable image digest."""
"""A pinned object revision, independent of its executable image digest.
``name`` and ``namespace_path`` are the Function's parts, as a
[SecretReference][lancedb.functions.SecretReference] carries a Secret's;
they are what [get_function][lancedb.db.DBConnection.get_function] takes.
An empty ``namespace_path`` is the root namespace.
"""
name: str
namespace_path: tuple[str, ...] = ()
object_id: str
location: str
version: _ObjectVersion
image: FunctionImage
signature: FunctionSignature
secret_bindings: tuple[SecretBinding, ...] = ()
created_at: str
created_at_millis: int
metadata: Mapping[str, str]
disabled: bool
@@ -444,6 +451,7 @@ class FunctionVersion(_RemoteValue):
return FunctionApplication(
function=FunctionVersionRef(
name=self.name,
namespace_path=self.namespace_path,
object_id=self.object_id,
location=self.location,
version=self.version,
@@ -502,6 +510,7 @@ class FunctionRegistrationRequest(_RemoteValue):
class FunctionVersionRef(_OpenRemoteValue):
name: str
namespace_path: tuple[str, ...] = ()
object_id: str
location: str
version: _ObjectVersion
python/python/lancedb/index.py
+44 -6
View File
@@ -112,9 +112,42 @@ class Fm:
class FTS:
"""Describe a FTS index configuration.
`FTS` is a full-text search index that can be used on `String` columns
`FTS` supports string columns, lists of strings, and JSON columns created
from PyArrow ``pa.json_()``. JSON columns use Lance's JSON-aware tokenizer
and are stored as ``LargeBinary`` with the ``lance.json`` logical type.
Arbitrary binary columns are not supported.
For example, it works with `title`, `description`, `content`, etc.
JSON queries use ``path,str,value`` triples. Paths are relative to the JSON
column and use dots for nested object keys. For example,
``meta.tag,str,nature`` matches the string value at ``$.meta.tag``;
it does not search other paths or JSON key names. The value is processed
with the configured text tokenizer. Use ``str``, not ``string``.
Create and query a JSON index through the modern API::
import pyarrow as pa
from lancedb.index import FTS
table = db.create_table(
"documents",
pa.table({"doc": pa.array(
['{"meta":{"tag":"nature"}}'], type=pa.json_()
)}),
)
table.create_index("doc", config=FTS())
table.search(
"meta.tag,str,nature", query_type="fts", fts_columns="doc"
).to_list()
Create the index on the JSON column itself, not a JSON subpath such as
``doc.meta.tag``. JSON-path scalar indexes do not provide FTS path scoping.
Plain text such as ``nature`` is not a valid JSON FTS query. The current
Lance tokenizer can panic on malformed triples; see
https://github.com/lance-format/lance/issues/8812.
Remote tables use the same ``FTS()`` configuration and require a server
that supports JSON FTS. Wait for ``table.wait_for_index(["doc_idx"])``
before searching a newly created remote index with its default name.
Examples
--------
@@ -138,6 +171,7 @@ class FTS:
- "simple": Splits text by whitespace and punctuation.
- "whitespace": Split text by whitespace, but not punctuation.
- "raw": No tokenization. The entire text is treated as a single token.
- "code": Tokenizes source code and identifiers.
- "ngram": N-gram tokenizer for substring-style matching.
- "icu": ICU dictionary-based word segmentation.
- "icu/split": ICU segmentation with simple-style delimiter splitting.
@@ -151,12 +185,16 @@ class FTS:
ignored.
lower_case : bool, default True
Whether to convert the token to lower case. This makes queries case-insensitive.
stem : bool, default True
stem : bool, optional
Whether to stem the token. Stemming reduces words to their root form.
For example, in English "running" and "runs" would both be reduced to "run".
remove_stop_words : bool, default True
``None`` uses the base tokenizer's default: False for ``code`` and
``ngram``, True otherwise.
remove_stop_words : bool, optional
Whether to remove stop words. Stop words are common words that are often
removed from text before indexing. For example, in English "the" and "and".
``None`` uses the base tokenizer's default: False for ``code`` and
``ngram``, True otherwise.
custom_stop_words : list of str, optional
Custom words replace the built-in language stop words
and only take effect when ``remove_stop_words`` is True. ``None`` uses
@@ -197,8 +235,8 @@ class FTS:
language: str = "English"
max_token_length: Optional[int] = 40
lower_case: bool = True
stem: bool = True
remove_stop_words: bool = True
stem: Optional[bool] = None
remove_stop_words: Optional[bool] = None
ascii_folding: bool = True
ngram_min_length: int = 3
ngram_max_length: int = 3
python/python/lancedb/listing.py
+85
View File
@@ -0,0 +1,85 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Lazy, resumable resource listings."""
from __future__ import annotations
from typing import Any, AsyncIterator, Callable, Generic, Iterator, Optional, TypeVar
from .background_loop import LOOP
T = TypeVar("T")
class AsyncListing(AsyncIterator[T], Generic[T]):
"""A lazy async iterator with inspectable pagination state.
Returned by connection resource-listing methods. For example::
async for name in connection.list_views(page_limit=20):
print(name)
Requests happen during iteration; request errors terminate the iterator.
"""
def __init__(self, inner: Any, convert: Callable[[Any], T] = lambda value: value):
self._inner = inner
self._convert = convert
def __aiter__(self) -> AsyncListing[T]:
return self
async def __anext__(self) -> T:
return self._convert(await self._inner.__anext__())
def num_page_results(self) -> int:
"""Return the number of cached items available without another request."""
return self._inner.num_page_results()
def page_token(self) -> Optional[str]:
"""Return the token for the next request.
Initially this is the supplied starting token (``None`` starts at the
beginning). After fetching the final page it is ``None``, even if items
remain cached. Drain the cache before saving a token to avoid skipping
items when resuming. A failed request terminates iteration and retains
its token for retrying with a new iterator.
"""
return self._inner.page_token()
class Listing(Iterator[T], Generic[T]):
"""A lazy synchronous iterator with inspectable pagination state.
For example::
for name in connection.list_views(page_limit=20):
print(name)
Use ``list(connection.list_views())`` to collect all results.
"""
def __init__(self, inner: AsyncListing[T]):
self._inner = inner
def __iter__(self) -> Listing[T]:
return self
def __next__(self) -> T:
try:
return LOOP.run(self._inner.__anext__())
except StopAsyncIteration:
raise StopIteration from None
def num_page_results(self) -> int:
"""Return the number of cached items available without another request."""
return self._inner.num_page_results()
def page_token(self) -> Optional[str]:
"""Return the next request's token.
See [AsyncListing.page_token][lancedb.listing.AsyncListing.page_token]
for initial, cached, final-page, and error behavior.
"""
return self._inner.page_token()
python/python/lancedb/materialized_view.py
+18 -122
View File
@@ -6,11 +6,11 @@ maintained by refresh. See ``DBConnection.create_materialized_view``."""
from __future__ import annotations
import json
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union
import json
from dataclasses import dataclass
from .background_loop import LOOP
from .job import AsyncJob, Job, _typed_job
@@ -21,7 +21,6 @@ if TYPE_CHECKING:
from .table import AsyncTable, LanceTable
DEFINITION_META_KEY = b"mv.definition"
_MAX_FLOAT32 = float.fromhex("0x1.fffffep+127")
SelectArg = Union[
str,
@@ -31,104 +30,12 @@ SelectArg = Union[
]
@dataclass(frozen=True)
class VectorDedupSource:
"""A declarative source of retained original rows from an indexed snapshot.
Construct with [vector_dedup][lancedb.vector_dedup] and pass it as the
``source`` of ``create_materialized_view`` or
``create_materialized_view_async``. An omitted ``dataset_version`` is
captured once when creation is submitted, not when this object is made.
Refreshes keep that captured version. The source table is never modified.
"""
source: str
column: str
distance_threshold: float
dataset_version: Optional[int] = None
def __post_init__(self):
for name in ("source", "column"):
value = getattr(self, name)
if not isinstance(value, str) or not value:
raise ValueError(f"{name} must be a non-empty string")
if self.dataset_version is not None and (
type(self.dataset_version) is not int
or not 0 < self.dataset_version < 2**64
):
raise ValueError("dataset_version must be a positive uint64 integer")
value = self.distance_threshold
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise TypeError("distance_threshold must be a number")
try:
value = float(value)
except OverflowError as exc:
raise ValueError("distance_threshold must fit a finite float32") from exc
if not math.isfinite(value) or abs(value) > _MAX_FLOAT32:
raise ValueError("distance_threshold must fit a finite float32")
object.__setattr__(self, "distance_threshold", value)
def _native_source_json(self, version: int) -> str:
return json.dumps(
{
"kind": "Dedup",
"dataset_version": version,
"column": self.column,
"distance_threshold": repr(self.distance_threshold),
},
allow_nan=False,
)
MaterializedViewSource = str
def vector_dedup(
source: str,
*,
column: str,
distance_threshold: float,
dataset_version: Optional[int] = None,
) -> VectorDedupSource:
"""Declare an indexed dedup source for a materialized view.
The default policy keeps the smallest unassigned snapshot row ID and removes
only its direct qualifying neighbors: an A-B-C chain without an A-C edge
retains A and C. Rows with no pair, including null vectors, survive.
Only same-segment/same-IVF-partition pairs are considered; quantized distances
use reconstructed index vectors. The index must cover the selected snapshot.
``source`` is an exact table name in the connection's root namespace, just
as for ``create_materialized_view``. ``dataset_version=None`` captures the
source's version when creation is submitted. Pass an explicit positive
version to reproduce a previous snapshot. The result contains original
source columns and rows. Do not combine this source with ``select``,
``where`` or ``limit``; query the resulting table for further transforms.
Creation follows the usual MV contract: by default it waits for the result.
Use ``with_no_data=True`` to declare only, or
``create_materialized_view_async`` to get a job handle. On Cloud/Enterprise,
execution uses server-side MV registry jobs; local connections run in process.
Examples
--------
Given a connected ``db`` and an indexed ``images.phash`` column::
from lancedb import vector_dedup
view = db.create_materialized_view(
"images_clean",
vector_dedup("images", column="phash", distance_threshold=4),
)
cleaned = view.table
"""
return VectorDedupSource(source, column, distance_threshold, dataset_version)
MaterializedViewSource = Union[str, VectorDedupSource]
DEFINITION_FORMAT = 4
DEFINITION_FORMAT = 2
"""The newest stored layout this version reads: ``{"format": N, "query": "<SQL>"}``,
format 2 being a query with ``GROUP BY``, format 3 native pairs and format 4
native dedup results. A ``kind`` key beside it is for
format 2 being a query with ``GROUP BY``. A ``kind`` key beside it is for
readers older than the format number."""
@@ -272,38 +179,31 @@ class AsyncMaterializedView:
return _definition_from_json(raw)
async def refresh(
self, *, full: bool = False, source_version: Optional[int] = None
self, *, source_version: Optional[int] = None
) -> "RefreshMaterializedViewResult":
"""Recompute the view from its source.
The refresh is incremental when the source's changes can be
reconciled into the view -- rows added, changed or removed since the
last one -- and otherwise rebuilds. ``full=True`` forces a rebuild;
Every refresh rebuilds the complete view. For local views,
``source_version`` refreshes to that source version instead of the
latest.
Concurrent refreshes of one view do not duplicate its rows. Two that
plan the same source rows conflict on commit, and the loser raises
rather than writing them a second time.
latest; remote SQL refreshes do not support source-version pinning.
"""
return await self._table._inner.refresh_materialized_view(
full=full, source_version=source_version
source_version=source_version
)
async def refresh_async(
self, *, full: bool = False, source_version: Optional[int] = None
self, *, source_version: Optional[int] = None
) -> "AsyncJob[RefreshMaterializedViewResult]":
"""Submit a refresh and return its job without waiting.
The job may already be complete for a local view. On LanceDB Cloud
and Enterprise, its ``id`` is the server job identifier returned by
the refresh endpoint.
The job may already be complete for a local view. Remote refreshes are
submitted through the SQL service.
"""
from ._lancedb import RefreshMaterializedViewResult
return _typed_job(
await self._table._inner.refresh_materialized_view_async(
full=full, source_version=source_version
source_version=source_version
),
RefreshMaterializedViewResult.from_json,
)
@@ -335,22 +235,18 @@ class MaterializedView:
return LOOP.run(self._async.definition())
def refresh(
self, *, full: bool = False, source_version: Optional[int] = None
self, *, source_version: Optional[int] = None
) -> "RefreshMaterializedViewResult":
"""Recompute the view from its source. See
[AsyncMaterializedView.refresh][lancedb.materialized_view.AsyncMaterializedView.refresh]."""
return LOOP.run(self._async.refresh(full=full, source_version=source_version))
return LOOP.run(self._async.refresh(source_version=source_version))
def refresh_async(
self, *, full: bool = False, source_version: Optional[int] = None
self, *, source_version: Optional[int] = None
) -> "Job[RefreshMaterializedViewResult]":
"""Submit a refresh and return its job without waiting.
See
[AsyncMaterializedView.refresh_async][lancedb.materialized_view.AsyncMaterializedView.refresh_async].
"""
return Job(
LOOP.run(
self._async.refresh_async(full=full, source_version=source_version)
)
)
return Job(LOOP.run(self._async.refresh_async(source_version=source_version)))
python/python/lancedb/namespace.py
+67 -13
View File
@@ -11,7 +11,7 @@ through a namespace abstraction.
from __future__ import annotations
import sys
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
from typing import Any, Dict, Iterable, List, Optional, Union
from uuid import UUID
if sys.version_info >= (3, 12):
@@ -19,8 +19,7 @@ if sys.version_info >= (3, 12):
else:
from overrides import override
if TYPE_CHECKING:
from lancedb.query import Query
from lancedb.query import Query
from datetime import timedelta
import pyarrow as pa
@@ -52,8 +51,9 @@ from lancedb.background_loop import LOOP
from lancedb.arrow import AsyncRecordBatchReader
from lancedb.db import AsyncConnection, DBConnection
from lancedb.job import AsyncJob, Job
from lancedb.listing import AsyncListing, Listing
from lancedb.sql import AsyncQuery as AsyncSqlQuery
from lancedb.sql import QueryDescription
from lancedb.sql import QueryDescription, QueryParameters
from lance_namespace import (
LanceNamespace,
connect as namespace_connect,
@@ -105,6 +105,11 @@ def _query_to_namespace_request(
if not isinstance(query, Query):
raise TypeError(f"Expected Query, got {type(query)}")
if query.search_effort is not None:
raise ValueError(
"namespace QueryTable does not support search_effort; "
"execute the query locally"
)
vector_request = None
if query.vector is not None:
@@ -190,8 +195,12 @@ def _query_to_namespace_request(
kwargs["with_row_id"] = query.with_row_id
if query.ef is not None:
kwargs["ef"] = query.ef
if query.nprobes is not None:
kwargs["nprobes"] = query.nprobes
if query.minimum_nprobes is not None:
kwargs["nprobes"] = query.minimum_nprobes
kwargs["minimum_nprobes"] = query.minimum_nprobes
if query.maximum_nprobes is not None:
kwargs["maximum_nprobes"] = query.maximum_nprobes
if query.refine_factor is not None:
kwargs["refine_factor"] = query.refine_factor
if query.lower_bound is not None:
@@ -205,6 +214,13 @@ def _query_to_namespace_request(
if query.distance_type is not None:
kwargs["distance_type"] = query.distance_type
if kwargs.get("maximum_nprobes") == 0:
# The generated 0.13 model rejects LanceDB's unbounded sentinel;
# model_copy preserves direct wire pass-through to the Lance receiver.
maximum_nprobes = kwargs.pop("maximum_nprobes")
return QueryTableRequest(**kwargs).model_copy(
update={"maximum_nprobes": maximum_nprobes}
)
return QueryTableRequest(**kwargs)
@@ -489,6 +505,14 @@ class LanceNamespaceDBConnection(DBConnection):
)
self._uri = self._inner.uri
@override
def is_open(self) -> bool:
return self._inner.is_open()
@override
def close(self) -> None:
self._inner.close()
@override
def serialize(self) -> str:
import json
@@ -514,13 +538,15 @@ class LanceNamespaceDBConnection(DBConnection):
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
"""
List table names in the database.
If limit is None, return all table names; zero returns an empty list.
.. deprecated::
Use :meth:`list_tables` instead, which provides proper pagination support.
"""
@@ -690,9 +716,20 @@ class LanceNamespaceDBConnection(DBConnection):
return view
@override
def list_materialized_views(self) -> List[str]:
"""The names of the materialized views in the root namespace."""
return LOOP.run(self._inner.list_materialized_views())
def list_materialized_views(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> Listing[str]:
"""The names of the materialized views in the root namespace.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return Listing(
self._inner.list_materialized_views(
page_token=page_token, page_limit=page_limit
)
)
@override
def drop_materialized_view(
@@ -1138,13 +1175,15 @@ class AsyncLanceNamespaceDBConnection:
async def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
"""
List table names in the namespace.
If limit is None, return all table names; zero returns an empty list.
.. deprecated::
Use :meth:`list_tables` instead, which provides proper pagination support.
"""
@@ -1289,9 +1328,20 @@ class AsyncLanceNamespaceDBConnection:
await view.definition()
return view
async def list_materialized_views(self) -> List[str]:
"""The names of the materialized views in the root namespace."""
return await self._inner.list_materialized_views()
def list_materialized_views(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> AsyncListing[str]:
"""The names of the materialized views in the root namespace.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return AsyncListing(
self._inner.list_materialized_views(
page_token=page_token, page_limit=page_limit
)
)
async def drop_materialized_view(
self, name: str, namespace_path: Optional[List[str]] = None
@@ -1554,11 +1604,13 @@ class AsyncLanceNamespaceDBConnection:
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> AsyncRecordBatchReader:
"""Execute SQL when supported by the underlying connection."""
return await self._inner.execute_query(
query,
default_namespace_path=default_namespace_path,
parameters=parameters,
)
async def execute_query_async(
@@ -1566,6 +1618,7 @@ class AsyncLanceNamespaceDBConnection:
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> AsyncSqlQuery:
"""Start executing SQL when supported by the underlying connection.
@@ -1574,6 +1627,7 @@ class AsyncLanceNamespaceDBConnection:
return await self._inner.execute_query_async(
query,
default_namespace_path=default_namespace_path,
parameters=parameters,
)
async def describe_query(self, query_id: UUID) -> QueryDescription:
python/python/lancedb/pydantic.py
+39
View File
@@ -88,6 +88,17 @@ def Vector(
... pa.field("url", pa.utf8(), False),
... pa.field("embeddings", pa.list_(pa.float32(), 768))
... ])
Notes
-----
``Vector`` creates a type dynamically, so calls such as ``Vector(768)`` are
not valid static type annotations. For an embedding field, use the standard
``list[float]`` annotation when running mypy; ``VectorField`` supplies the
fixed dimension to LanceDB::
class MyModel(LanceModel):
text: str = embeddings.SourceField()
vector: list[float] = embeddings.VectorField()
"""
# TODO: make a public parameterized type.
@@ -313,6 +324,10 @@ def _unwrap_optional_annotation(annotation: Any) -> Any | None:
def _pydantic_to_arrow_type(field: FieldInfo) -> pa.DataType:
"""Convert a Pydantic FieldInfo to Arrow DataType"""
embedding_vector_type = _embedding_vector_to_arrow_type(field)
if embedding_vector_type is not None:
return embedding_vector_type
unwrapped = _unwrap_optional_annotation(field.annotation)
if unwrapped is not None:
return _pydantic_type_to_arrow_type(unwrapped, field)
@@ -326,8 +341,32 @@ def _pydantic_to_arrow_type(field: FieldInfo) -> pa.DataType:
return _pydantic_type_to_arrow_type(field.annotation, field)
def _embedding_vector_to_arrow_type(field: FieldInfo) -> pa.DataType | None:
"""Infer a fixed-size vector type from ``VectorField`` metadata."""
if not _is_embedding_vector_annotation(field):
return None
function = get_extras(field, "vector_column_for")
return pa.list_(pa.float32(), function.ndims())
def _is_embedding_vector_annotation(field: FieldInfo) -> bool:
if get_extras(field, "vector_column_for") is None:
return False
annotation = _unwrap_optional_annotation(field.annotation)
if annotation is None:
annotation = field.annotation
origin = getattr(annotation, "__origin__", None)
args = getattr(annotation, "__args__", ())
return origin is list and args == (float,)
def is_nullable(field: FieldInfo) -> bool:
"""Check if a Pydantic FieldInfo is nullable."""
if _is_embedding_vector_annotation(field):
return True
if _unwrap_optional_annotation(field.annotation) is not None:
return True
if isinstance(field.annotation, (_GenericAlias, GenericAlias)):
python/python/lancedb/query.py
+167 -31
View File
@@ -39,8 +39,8 @@ from .expr import Expr
from .rerankers.base import Reranker
from .rerankers.rrf import RRFReranker
from .rerankers.util import check_reranker_result
from .schema import is_blob_like_field, schema_has_blob_field
from .util import flatten_columns
from .schema import blob_column_paths, is_blob_like_field, schema_has_blob_field
from .util import _validate_query_vector, flatten_columns
from . import _wal_hybrid # WAL-PK-FUSION: delete.
from ._blob import (
BLOB_MODE_TO_HANDLING,
@@ -51,6 +51,7 @@ from ._blob import (
finalize_blob_query_table,
replace_v2_blob_columns_with_bytes,
replace_v2_blob_columns_with_bytes_sync,
strip_auto_row_ids,
validate_blob_mode,
)
from .types import BlobMode, QueryProjection
@@ -474,12 +475,12 @@ class MatchQuery(FullTextQuery):
----------
query : str
The query string to match against.
column : str
The name of the column to match against.
column : str, optional
The column to match against. None leaves column selection to query planning.
boost : float, default 1.0
The boost factor for the query.
The score of each matching document is multiplied by this value.
fuzziness : int, optional
fuzziness : int or None, default 0
The maximum edit distance for each term in the match query.
Defaults to 0 (exact match).
If None, fuzziness is applied automatically by the rules:
@@ -504,9 +505,9 @@ class MatchQuery(FullTextQuery):
"""
query: str
column: str
column: Optional[str] = None
boost: float = pydantic.Field(1.0, kw_only=True)
fuzziness: int = pydantic.Field(0, kw_only=True)
fuzziness: Optional[int] = pydantic.Field(0, kw_only=True)
max_expansions: int = pydantic.Field(50, kw_only=True)
operator: FullTextOperator = pydantic.Field(FullTextOperator.OR, kw_only=True)
prefix_length: int = pydantic.Field(0, kw_only=True)
@@ -692,8 +693,9 @@ class Query(pydantic.BaseModel):
if True then apply the filter after vector / FTS search. This is ignored for
plain SQL filtering.
nprobes : Optional[int]
The number of IVF partitions to search. If this is None then a default
number of partitions will be used.
The legacy number of IVF partitions to search. Lance sets both probe
bounds to this value. If this is None then Lance's default probe settings
will be used.
- A higher number makes search more accurate but also slower.
@@ -772,22 +774,28 @@ class Query(pydantic.BaseModel):
# distance type to use for vector search
distance_type: Optional[str] = None
# legacy number of IVF partitions to search
#
# Lance sets both probe bounds to this value. Explicit bounds can override it.
nprobes: Optional[int] = None
# which columns to return in the results (dict values may be str or Expr)
columns: QueryProjection = None
# minimum number of IVF partitions to search
#
# If None then a default value (20) will be used.
# If None then Lance's default will be used.
minimum_nprobes: Optional[int] = None
# maximum number of IVF partitions to search
#
# If None then a default value (20) will be used.
#
# If 0 then no limit will be applied and all partitions could be searched
# if needed to satisfy the limit.
# If None then Lance's default will be used. If 0 then no limit will be applied
# and all partitions could be searched if needed to satisfy the limit.
maximum_nprobes: Optional[int] = None
# Optional initial IVF effort; None preserves Auto and explicit nprobes.
search_effort: Optional[float] = None
# lower bound for distance search
lower_bound: Optional[float] = None
@@ -846,8 +854,10 @@ class Query(pydantic.BaseModel):
query.vector_column = req.column
query.vector = req.query_vector
query.distance_type = req.distance_type
query.nprobes = req.nprobes
query.minimum_nprobes = req.minimum_nprobes
query.maximum_nprobes = req.maximum_nprobes
query.search_effort = req.search_effort
query.lower_bound = req.lower_bound
query.upper_bound = req.upper_bound
query.ef = req.ef
@@ -907,6 +917,9 @@ class LanceQueryBuilder(ABC):
fast_search: bool
Skip flat search of unindexed data.
"""
if query_type != "fts":
_validate_query_vector(query)
if ordering_field_name is not None:
import warnings
@@ -1619,11 +1632,14 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
str_query: Optional[str] = None,
fast_search: bool = None,
):
_validate_query_vector(query)
super().__init__(table)
self._query = query
self._distance_type = None
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
self._search_effort = None
self._lower_bound = None
self._upper_bound = None
self._refine_factor = None
@@ -1686,22 +1702,44 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
See discussion in [Querying an ANN Index](https://lancedb.com/docs/indexing/)
for tuning advice.
This method sets both the minimum and maximum number of probes to the same
value. See `minimum_nprobes` and `maximum_nprobes` for more fine-grained
control.
The value is retained as `nprobes` through client and server request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
Parameters
----------
nprobes: int
The number of probes to use.
The number of probes to use. Must be greater than 0.
Returns
-------
LanceVectorQueryBuilder
The LanceQueryBuilder object.
"""
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._nprobes = nprobes
return self
def search_effort(self, search_effort: Optional[float]) -> LanceVectorQueryBuilder:
"""Set the initial IVF search effort to a finite value in [0, 1].
None (the default) uses Auto, internally 0.5. Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found. Any explicit
effort, including 0.5, overrides nprobes in either call order. Explicit
minimum and maximum bounds still apply. This is not a recall guarantee
and does not change refinement or HNSW ef. MemWAL LSM queries do not
yet support this option.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("memory://")
>>> table = db.create_table("effort_vectors", [{"vector": [0.1, 0.2]}])
>>> query = table.search([0.1, 0.2]).search_effort(0.75)
"""
self._search_effort = search_effort
return self
def minimum_nprobes(self, minimum_nprobes: int) -> LanceVectorQueryBuilder:
@@ -1843,8 +1881,10 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
limit=self._limit,
distance_type=self._distance_type,
columns=self._columns,
nprobes=self._nprobes,
minimum_nprobes=self._minimum_nprobes,
maximum_nprobes=self._maximum_nprobes,
search_effort=self._search_effort,
lower_bound=self._lower_bound,
upper_bound=self._upper_bound,
refine_factor=self._refine_factor,
@@ -2201,8 +2241,10 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._fts_columns = fts_columns
self._norm = None
self._reranker = None
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
self._search_effort = None
self._refine_factor = None
self._distance_type = None
self._phrase_query = None
@@ -2492,18 +2534,48 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
Higher values will yield better recall (more likely to find vectors if
they exist) at the expense of latency.
The value is retained as `nprobes` until Lance sets both probe bounds to it.
Explicit minimum or maximum settings can override their respective bounds.
Parameters
----------
nprobes: int
The number of probes to use.
The number of probes to use. Must be greater than 0.
Returns
-------
LanceHybridQueryBuilder
The LanceHybridQueryBuilder object.
"""
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._nprobes = nprobes
return self
def search_effort(self, search_effort: Optional[float]) -> LanceHybridQueryBuilder:
"""Set the initial IVF search effort to a finite value in [0, 1].
None (the default) uses Auto, internally 0.5. Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found. Any explicit
effort, including 0.5, overrides nprobes in either call order. Explicit
minimum and maximum bounds still apply. This is not a recall guarantee
and does not change refinement or HNSW ef. MemWAL LSM queries do not
yet support this option.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("memory://")
>>> table = db.create_table(
... "effort_hybrid", [{"vector": [0.1, 0.2], "text": "dog"}]
... )
>>> query = (
... table.search(query_type="hybrid").vector([0.1, 0.2])
... .text("dog").search_effort(0.75)
... )
"""
self._search_effort = search_effort
return self
def minimum_nprobes(self, minimum_nprobes: int) -> LanceHybridQueryBuilder:
@@ -2782,6 +2854,10 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._fts_query.phrase_query(True)
if self._distance_type:
self._vector_query.metric(self._distance_type)
if self._nprobes is not None:
self._vector_query.nprobes(self._nprobes)
if self._search_effort is not None:
self._vector_query.search_effort(self._search_effort)
if self._minimum_nprobes is not None:
self._vector_query.minimum_nprobes(self._minimum_nprobes)
if self._maximum_nprobes is not None:
@@ -3065,14 +3141,18 @@ class AsyncQueryBase(object):
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
blob_mode: str, default "lazy"
Controls how blob columns are returned for plain scan queries.
Vector, FTS, hybrid, and other non-native query shapes keep the
existing Arrow conversion path and only support blob descriptions.
Controls how blob columns are returned. Remote queries support
"descriptions"; "bytes" and "lazy" are not yet supported.
**kwargs
Forwarded to pyarrow.Table.to_pandas after query execution and
optional flattening.
"""
validate_blob_mode(blob_mode)
if self._table is not None and not self._table._inner._is_native():
return await self._remote_to_pandas(
flatten=flatten, timeout=timeout, blob_mode=blob_mode, **kwargs
)
if hasattr(self._inner, "output_schema"):
schema = await self.output_schema()
if _blob_mode_requires_native_pandas(blob_mode, schema):
@@ -3100,6 +3180,31 @@ class AsyncQueryBase(object):
)
return tbl.to_pandas(**kwargs)
async def _remote_to_pandas(
self,
*,
flatten: Optional[Union[int, bool]],
timeout: Optional[timedelta],
blob_mode: BlobMode,
**kwargs,
) -> "pd.DataFrame":
# A live remote table can be replaced between query and blob fetch.
# Row ids and version numbers do not identify the producing table.
tbl = await self.to_arrow(timeout=timeout)
projected_blob_paths = set(blob_column_paths(tbl.schema))
if not projected_blob_paths:
return flatten_columns(tbl, flatten).to_pandas(**kwargs)
if blob_mode == "descriptions":
tbl = strip_auto_row_ids(tbl, self._blob_paths)
return flatten_columns(tbl, flatten).to_pandas(**kwargs)
raise NotImplementedError(
f"remote to_pandas(blob_mode={blob_mode!r}) cannot safely materialize "
"blob columns without a stable table snapshot; "
"use blob_mode='descriptions'"
)
async def _plain_scan_to_pandas(
self,
blob_mode: BlobMode,
@@ -3652,7 +3757,9 @@ class AsyncVectorQueryBase:
def nprobes(self, nprobes: int) -> Self:
"""
Set the number of partitions to search (probe)
Set the legacy IVF probe parameter
The number of probes must be greater than 0.
This argument is only used when the vector column has an IVF-based index.
If there is no index then this value is ignored.
@@ -3662,20 +3769,49 @@ class AsyncVectorQueryBase:
The partition whose centroids are closest to the query vector will be
exhaustiely searched to find matches. This parameter controls how many
partitions should be searched.
partitions will be searched.
Increasing this value will increase the recall of your query but will
also increase the latency of your query. The default value is 20. This
default is good for many cases but the best value to use will depend on
your data and the recall that you need to achieve.
also increase the latency of your query. If this method is not called,
Lance's adaptive probe defaults are used.
For best results we recommend tuning this parameter with a benchmark against
your actual data to find the smallest possible value that will still give
you the desired recall.
LanceDB retains this as `nprobes` through local and remote request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
"""
self._inner.nprobes(nprobes)
return self
def search_effort(self, search_effort: Optional[float]) -> Self:
"""Set the initial IVF search effort to a finite value in [0, 1].
None preserves Auto (internally 0.5) and fixed nprobes. Any explicit
effort, including 0.5, overrides nprobes in either call order; explicit
minimum and maximum bounds remain effective. Zero starts at the caller
minimum and one searches all partitions within the maximum. Later
probing may continue to obtain enough candidates. This is not a recall
guarantee and does not change refinement or HNSW ef. MemWAL LSM queries
do not yet support this option.
Examples
--------
>>> import asyncio
>>> import lancedb
>>> async def build_query():
... db = await lancedb.connect_async("memory://")
... table = await db.create_table(
... "effort_async", [{"vector": [0.1, 0.2]}]
... )
... return table.vector_search([0.1, 0.2]).search_effort(0.75)
>>> query = asyncio.run(build_query())
"""
self._inner.search_effort(search_effort)
return self
def minimum_nprobes(self, minimum_nprobes: int) -> Self:
"""Set the minimum number of probes to use.
python/python/lancedb/remote/db.py
+155 -34
View File
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from __future__ import annotations
from dataclasses import replace
from datetime import timedelta
import json
@@ -34,11 +36,12 @@ from lancedb.remote import ClientConfig, RetryConfig, TimeoutConfig, TlsConfig
import pyarrow as pa
from ..common import DATA
from lancedb.listing import Listing
from ..db import DBConnection, LOOP
from ..functions import FunctionVersion, UdfDefinition
from ..job import AsyncJob, Job
from ..sql import Query as SqlQuery
from ..sql import QueryDescription
from ..sql import QueryDescription, QueryParameters
from ..materialized_view import MaterializedView, MaterializedViewSource, SelectArg
from ..secrets import EnvVarSecret, SecretInfo
from ..view import ViewDescription
@@ -376,7 +379,7 @@ class RemoteDBConnection(DBConnection):
page_token: Optional[str] = None,
limit: Optional[int] = None,
) -> ListTablesResponse:
"""List all tables in this database with pagination support.
"""List a page of tables in this database.
Parameters
----------
@@ -387,7 +390,8 @@ class RemoteDBConnection(DBConnection):
Token for pagination. Use the token from a previous response
to get the next page of results.
limit: int, optional
The maximum number of results to return.
The maximum number of results to return, default 100.
Zero returns an empty page without a continuation token.
Returns
-------
@@ -406,7 +410,7 @@ class RemoteDBConnection(DBConnection):
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
@@ -422,8 +426,9 @@ class RemoteDBConnection(DBConnection):
Empty list represents root namespace.
page_token: str
The last token to start the new page.
limit: int, default 10
The maximum number of tables to return for each page.
limit: int, optional
The maximum number of tables to return. None returns all tables;
zero returns an empty list.
Returns
-------
@@ -483,12 +488,12 @@ class RemoteDBConnection(DBConnection):
namespace_path = []
if storage_options is not None:
logging.info(
"storage_options is ignored in LanceDb Cloud"
"storage_options is ignored for remote tables"
" (storage is managed; set storage_options on connect() instead)"
)
if index_cache_size is not None:
logging.info(
"index_cache_size is ignored in LanceDb Cloud"
"index_cache_size is ignored for remote tables"
" (there is no local cache to configure)"
)
@@ -608,6 +613,9 @@ class RemoteDBConnection(DBConnection):
One of "error", "drop", "fill".
fill_value: float
The value to use when filling vectors. Only used if on_bad_vectors="fill".
embedding_functions: list of EmbeddingFunctionConfig, optional
The embedding functions to store in the table schema and use to
generate vectors when creating the table or adding data.
Returns
-------
@@ -682,12 +690,6 @@ class RemoteDBConnection(DBConnection):
if namespace_path is None:
namespace_path = []
validate_table_name(name)
if embedding_functions is not None:
logging.warning(
"embedding_functions is not yet supported on LanceDB Cloud."
"Please vote https://github.com/lancedb/lancedb/issues/626 "
"for this feature."
)
from .table import RemoteTable
@@ -700,6 +702,7 @@ class RemoteDBConnection(DBConnection):
schema=schema,
on_bad_vectors=on_bad_vectors,
fill_value=fill_value,
embedding_functions=embedding_functions,
)
)
return RemoteTable(
@@ -771,8 +774,20 @@ class RemoteDBConnection(DBConnection):
return view
@override
def list_materialized_views(self) -> List[str]:
return LOOP.run(self._conn.list_materialized_views())
def list_materialized_views(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> Listing[str]:
"""List materialized views.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return Listing(
self._conn.list_materialized_views(
page_token=page_token, page_limit=page_limit
)
)
@override
def drop_materialized_view(
@@ -862,25 +877,78 @@ class RemoteDBConnection(DBConnection):
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
job = LOOP.run(
self._conn.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
)
)
return Job(job)
@override
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
def get_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
return LOOP.run(
self._conn.get_function(
name, version=version, namespace_path=namespace_path
)
)
@override
def list_functions(self) -> List[FunctionVersion]:
return LOOP.run(self._conn.list_functions())
def list_functions(
self,
*,
namespace_path: Optional[List[str]] = None,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> Listing[FunctionVersion]:
"""List functions.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return Listing(
self._conn.list_functions(
namespace_path=namespace_path,
page_token=page_token,
page_limit=page_limit,
)
)
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
def drop_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> bool:
return LOOP.run(
self._conn.drop_function(
name, version=version, namespace_path=namespace_path
)
)
@override
def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]:
dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version))
def drop_function_async(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> Tuple[bool, Job]:
dropped, job = LOOP.run(
self._conn.drop_function_async(
name, version=version, namespace_path=namespace_path
)
)
return dropped, Job(job)
@override
@@ -902,8 +970,26 @@ class RemoteDBConnection(DBConnection):
return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path))
@override
def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path))
def list_secrets(
self,
*,
namespace_path: Optional[List[str]] = None,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> Listing[str]:
"""List secrets.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return Listing(
self._conn.list_secrets(
namespace_path=namespace_path,
page_token=page_token,
page_limit=page_limit,
)
)
@override
def drop_secret(
@@ -939,13 +1025,40 @@ class RemoteDBConnection(DBConnection):
return Job(job)
@override
def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
return LOOP.run(self._conn.list_views(namespace_path=namespace_path))
def list_views(
self,
*,
namespace_path: Optional[List[str]] = None,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
) -> Listing[str]:
"""List views.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return Listing(
self._conn.list_views(
namespace_path=namespace_path,
page_token=page_token,
page_limit=page_limit,
)
)
@override
def list_jobs(self) -> List["JobInfo"]:
"""List server-side jobs across the database's tables."""
return LOOP.run(self._conn.list_jobs())
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> Listing[JobInfo]:
"""List server-side jobs across the database's tables.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
return Listing(
self._conn.list_jobs(page_token=page_token, page_limit=page_limit)
)
@override
def cancel_job(self, job_id: str) -> bool:
@@ -979,18 +1092,22 @@ class RemoteDBConnection(DBConnection):
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> SqlQuery:
"""Start executing SQL through this remote connection.
Unqualified tables use this connection's database and the
``["public"]`` namespace by default. Fully qualified table names may
reference other databases available to the same deployment.
``parameters`` binds values to ``$1`` / ``$name`` placeholders; see
[DBConnection.execute_query_async][lancedb.db.DBConnection.execute_query_async].
"""
return SqlQuery(
LOOP.run(
self._conn.execute_query_async(
query,
default_namespace_path=default_namespace_path,
parameters=parameters,
)
)
)
@@ -1017,6 +1134,10 @@ class RemoteDBConnection(DBConnection):
"""
return LOOP.run(self._conn.namespace_client())
async def close(self):
"""Close the connection to the database."""
@override
def is_open(self) -> bool:
return self._conn.is_open()
@override
def close(self) -> None:
self._conn.close()
python/python/lancedb/remote/table.py
+62 -54
View File
@@ -42,6 +42,7 @@ from lancedb.index import (
BTree,
Bitmap,
HnswFlat,
HnswPq,
HnswSq,
IvfFlat,
IvfPq,
@@ -63,6 +64,7 @@ from lancedb.table import _normalize_progress
from ..query import (
AnalyzePlanDistributedMetrics,
DocumentGranularity,
FullTextQuery,
LanceQueryBuilder,
LanceTakeQueryBuilder,
LanceVectorQueryBuilder,
@@ -77,6 +79,7 @@ from ..table import (
Tags,
)
from ..types import BaseTokenizerType
from ..util import infer_vector_column_name
class RemoteTable(Table):
@@ -237,12 +240,12 @@ class RemoteTable(Table):
return LOOP.run(self._table.list_versions())
def to_arrow(self) -> pa.Table:
"""to_arrow() is not yet supported on LanceDB cloud."""
raise NotImplementedError("to_arrow() is not yet supported on LanceDB cloud.")
"""to_arrow() is not supported for remote tables."""
raise NotImplementedError("to_arrow() is not supported for remote tables.")
def to_pandas(self, blob_mode: BlobMode = "lazy", **kwargs):
"""to_pandas() is not yet supported on LanceDB cloud."""
raise NotImplementedError("to_pandas() is not yet supported on LanceDB cloud.")
"""to_pandas() is not supported for remote tables."""
raise NotImplementedError("to_pandas() is not supported for remote tables.")
def checkout(self, version: Union[int, str]):
result = LOOP.run(self._table.checkout(version))
@@ -351,8 +354,8 @@ class RemoteTable(Table):
language: str = "English",
max_token_length: Optional[int] = 40,
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
stem: Optional[bool] = None,
remove_stop_words: Optional[bool] = None,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
@@ -499,14 +502,8 @@ class RemoteTable(Table):
if accelerator is not None:
logging.warning(
"GPU accelerator is not yet supported on LanceDB cloud."
"If you have 100M+ vectors to index,"
"please contact us at contact@lancedb.com"
)
if replace is not None:
logging.warning(
"replace is not supported on LanceDB cloud."
"Existing indexes will always be replaced."
"GPU accelerator is not supported for remote tables "
"and will be ignored."
)
idx_type = index_type.upper()
@@ -526,9 +523,11 @@ class RemoteTable(Table):
elif idx_type == "IVF_SQ":
config = IvfSq(distance_type=metric, num_partitions=num_partitions)
elif idx_type == "IVF_HNSW_PQ":
raise ValueError(
"IVF_HNSW_PQ is not supported on LanceDB cloud."
"Please use IVF_HNSW_SQ instead."
config = HnswPq(
distance_type=metric,
num_partitions=num_partitions,
num_sub_vectors=num_sub_vectors,
num_bits=num_bits,
)
elif idx_type == "IVF_HNSW_SQ":
config = HnswSq(distance_type=metric, num_partitions=num_partitions)
@@ -657,7 +656,7 @@ class RemoteTable(Table):
capped at the number of CPU cores. Lower this if bulk ingestion is
using too much memory.
allow_external_blob_outside_bases: bool, default False
Not supported on LanceDB Cloud. Setting this raises.
Not supported for remote tables. Setting this raises.
Returns
-------
@@ -687,7 +686,7 @@ class RemoteTable(Table):
vector_column_name: Optional[str] = None,
query_type="auto",
fts_columns: Optional[Union[str, List[str]]] = None,
fast_search: bool = False,
fast_search: Optional[bool] = None,
) -> LanceVectorQueryBuilder:
"""Create a search query to find the nearest neighbors
of the given query vector. We currently support
@@ -734,11 +733,17 @@ class RemoteTable(Table):
- If the table has multiple vector columns then the *vector_column_name*
needs to be specified. Otherwise, an error is raised.
query_type: str, default "auto"
The type of search: "auto", "vector", "fts", or "hybrid".
With "auto", a string query uses the embedding function for the
inferred or specified vector column if available, otherwise it uses
full-text search.
fast_search: bool, optional
Skip a flat search of unindexed data. This may improve
search performance but search results will not include unindexed data.
- *default False*.
- *default None*, which leaves fast search disabled.
Returns
-------
@@ -753,12 +758,27 @@ class RemoteTable(Table):
- and also the "_distance" column which is the distance between the query
vector and the returned vector.
"""
if isinstance(query, FullTextQuery):
query_type = "fts"
elif (
query_type == "auto"
and isinstance(query, str)
and not self.embedding_functions
):
query_type = "fts"
vector_column_name = infer_vector_column_name(
schema=self.schema,
query_type=query_type,
query=query,
vector_column_name=vector_column_name,
)
return LanceQueryBuilder.create(
self,
query,
query_type,
vector_column_name=vector_column_name,
fts_columns=fts_columns,
fts_columns=fts_columns or [],
fast_search=fast_search,
)
@@ -933,26 +953,20 @@ class RemoteTable(Table):
def cleanup_old_versions(self, *_):
"""
cleanup_old_versions() is a no-op on LanceDB Cloud.
Tables are automatically cleaned up and optimized.
cleanup_old_versions() is a no-op for remote tables.
"""
warnings.warn(
"cleanup_old_versions() is a no-op on LanceDB Cloud. "
"Tables are automatically cleaned up and optimized.",
"cleanup_old_versions() is a no-op for remote tables.",
stacklevel=2,
)
pass
def compact_files(self, *_):
"""
compact_files() is a no-op on LanceDB Cloud.
Tables are automatically compacted and optimized.
compact_files() is a no-op for remote tables.
"""
warnings.warn(
"compact_files() is a no-op on LanceDB Cloud. "
"Tables are automatically compacted and optimized.",
"compact_files() is a no-op for remote tables.",
stacklevel=2,
)
pass
@@ -963,17 +977,13 @@ class RemoteTable(Table):
cleanup_older_than: Optional[timedelta] = None,
delete_unverified: bool = False,
):
"""
optimize() is a no-op on LanceDB Cloud.
Indices are optimized automatically.
"""
warnings.warn(
"optimize() is a no-op on LanceDB Cloud. "
"Indices are optimized automatically.",
stacklevel=2,
"""optimize() is not supported for remote tables."""
return LOOP.run(
self._table.optimize(
cleanup_older_than=cleanup_older_than,
delete_unverified=delete_unverified,
)
)
pass
def count_rows(self, filter: Optional[str] = None) -> int:
return LOOP.run(self._table.count_rows(filter))
@@ -1021,7 +1031,7 @@ class RemoteTable(Table):
return LOOP.run(self._table.drop_columns(columns))
def set_unenforced_primary_key(self, columns: Union[str, Iterable[str]]) -> None:
"""Not supported on LanceDB Cloud."""
"""Not supported for remote tables."""
return LOOP.run(self._table.set_unenforced_primary_key(columns))
def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None:
@@ -1067,7 +1077,7 @@ class RemoteTable(Table):
)
def close_lsm_writers(self) -> None:
"""No-op on LanceDB Cloud (no local shard writers)."""
"""No-op for remote tables (no local shard writers)."""
return LOOP.run(self._table.close_lsm_writers())
def drop_index(self, index_name: str):
@@ -1135,30 +1145,28 @@ class RemoteTable(Table):
return LanceTakeQueryBuilder(self._table.take_row_ids(row_ids))
def uses_v2_manifest_paths(self) -> bool:
raise NotImplementedError(
"uses_v2_manifest_paths() is not supported on the LanceDB Cloud"
)
return LOOP.run(self._table.uses_v2_manifest_paths())
def migrate_v2_manifest_paths(self):
raise NotImplementedError(
"migrate_v2_manifest_paths() is not supported on the LanceDB Cloud"
)
return LOOP.run(self._table.migrate_manifest_paths_v2())
def blob_columns(self) -> list[str]:
return LOOP.run(self._table.blob_columns())
def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> pa.LargeBinaryArray:
return LOOP.run(self._table.fetch_blobs(column, row_ids))
def fetch_blob_ranges(self, column: str, requests) -> pa.LargeBinaryArray:
raise NotImplementedError(
"fetch_blob_ranges() is not supported on LanceDB Cloud"
)
return LOOP.run(self._table.fetch_blob_ranges(column, list(requests)))
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> "list[Optional[BlobFile]]":
return LOOP.run(self._table.fetch_blob_files(column, row_ids))
python/python/lancedb/schema.py
+51 -4
View File
@@ -5,7 +5,7 @@
"""Schema helpers for Lance blob columns."""
import importlib
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Optional
import pyarrow as pa
import pyarrow.ipc
@@ -181,13 +181,60 @@ def _resolve_blob_type():
return _resolved_blob_type
def blob(name: str, nullable: bool = True) -> pa.Field:
def blob(
name: str,
nullable: bool = True,
*,
inline_size_threshold: Optional[int] = None,
dedicated_size_threshold: Optional[int] = None,
pack_file_size_threshold: Optional[int] = None,
) -> pa.Field:
"""Create a Lance blob v2 column field.
When pylance is installed this is ``lance.blob.BlobType``.
Parameters
----------
name:
Field name.
nullable:
Whether the column accepts null values.
inline_size_threshold:
Maximum payload size in bytes to keep inline in the data file
before using packed blob storage. Zero is allowed.
dedicated_size_threshold:
Maximum payload size in bytes to store in packed blob storage
before using dedicated blob storage. Must be positive.
pack_file_size_threshold:
Maximum size in bytes of a single packed blob sidecar file before
a new one is started. Must be positive.
The thresholds are forwarded to ``lance.blob_field``, which validates
them and stores them as field metadata, so pylance is required whenever
any threshold is set. When all thresholds are omitted the behavior is
unchanged.
"""
blob_type = _resolve_blob_type()
return pa.field(name, blob_type(), nullable=nullable)
if (
inline_size_threshold is None
and dedicated_size_threshold is None
and pack_file_size_threshold is None
):
blob_type = _resolve_blob_type()
return pa.field(name, blob_type(), nullable=nullable)
try:
lance = importlib.import_module("lance")
except ModuleNotFoundError as err:
raise ValueError(
"Setting blob tier thresholds requires pylance to be installed"
) from err
_resolve_blob_type()
return lance.blob_field(
name,
nullable=nullable,
inline_size_threshold=inline_size_threshold,
dedicated_size_threshold=dedicated_size_threshold,
pack_file_size_threshold=pack_file_size_threshold,
)
def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType:
python/python/lancedb/sql.py
+79 -1
View File
@@ -3,8 +3,10 @@
"""Handles to SQL queries running on a remote database."""
from typing import Any, Mapping, Optional, Sequence, Union
from uuid import UUID
import numpy as np
import pyarrow as pa
from lancedb.background_loop import LOOP
@@ -14,6 +16,82 @@ from .arrow import AsyncRecordBatchReader
QueryDescription = _lancedb.QueryDescription
QueryParameters = Union[pa.RecordBatch, pa.Table, Sequence[Any], Mapping[str, Any]]
"""Values for a statement's ``$1`` / ``$name`` placeholders.
A sequence binds by position (``$1``, ``$2``, ...), a mapping binds by name
(``$name``), and a one-row ``pyarrow.RecordBatch`` or ``pyarrow.Table`` binds
both ways: column ``i`` is ``$<i + 1>`` and, when its name is not a number,
also ``$<name>``.
"""
def _parameter_array(value: Any) -> pa.Array:
"""One parameter as a one-element array of its own type."""
if isinstance(value, pa.Scalar):
return pa.array([value.as_py()], type=value.type)
# A 1-D array is a vector: keep its element type and make it the
# fixed-size list a vector column stores, rather than a list of Python
# floats that would come back as float64.
if isinstance(value, pa.ChunkedArray):
value = value.combine_chunks()
if isinstance(value, np.ndarray):
if value.ndim == 0:
return pa.array(value.reshape(1))
if value.ndim != 1:
raise ValueError(
"a numpy query parameter must be a scalar or a 1-D vector, "
f"got {value.ndim} dimensions"
)
value = pa.array(value)
if isinstance(value, pa.Array):
return pa.FixedSizeListArray.from_arrays(value, len(value))
return pa.array([value])
def to_parameter_batch(
parameters: Optional[QueryParameters],
) -> Optional[pa.RecordBatch]:
"""Turn ``parameters`` into the one-row batch a parameterized query sends.
Scalars keep the type pyarrow infers for them (numpy scalars keep their
dtype), a ``pyarrow.Scalar`` keeps its own type, and a 1-D numpy or
pyarrow array becomes a ``FixedSizeList`` of its element type, which is
what a vector column stores.
"""
if parameters is None:
return None
batch: pa.RecordBatch
if isinstance(parameters, pa.RecordBatch):
batch = parameters
elif isinstance(parameters, pa.Table):
batches = parameters.combine_chunks().to_batches()
batch = (
batches[0]
if len(batches) == 1
else pa.RecordBatch.from_pylist([], schema=parameters.schema)
)
elif isinstance(parameters, Mapping):
batch = pa.RecordBatch.from_arrays(
[_parameter_array(value) for value in parameters.values()],
names=[str(name) for name in parameters.keys()],
)
elif isinstance(parameters, Sequence) and not isinstance(parameters, (str, bytes)):
batch = pa.RecordBatch.from_arrays(
[_parameter_array(value) for value in parameters],
names=[str(position + 1) for position in range(len(parameters))],
)
else:
raise TypeError(
"query parameters must be a sequence, a mapping, or a one-row "
f"pyarrow RecordBatch or Table, not {type(parameters).__name__}"
)
if batch.num_rows != 1:
raise ValueError(
f"query parameters must be exactly one row, got {batch.num_rows}"
)
return batch
class AsyncQuery:
"""A handle to a submitted SQL query on an asynchronous connection."""
@@ -85,4 +163,4 @@ class Query:
LOOP.run(self._inner.cancel())
__all__ = ["AsyncQuery", "Query", "QueryDescription"]
__all__ = ["AsyncQuery", "Query", "QueryDescription", "QueryParameters"]
python/python/lancedb/table.py
+233 -51
View File
@@ -261,6 +261,101 @@ IndexConfigType = Union[
KNOWN_METRICS = {"l2", "cosine", "dot", "hamming"}
def _blob_value_to_storage(value: Any) -> Optional[dict]:
"""Keep a Python blob's inline data or external URI before Arrow infers it."""
if value is None:
return None
if isinstance(value, (bytes, bytearray, memoryview)):
return {"data": bytes(value)}
if isinstance(value, str):
if not value:
raise ValueError("Blob uri cannot be empty")
return {"uri": value}
if isinstance(value, dict):
unknown = value.keys() - {"data", "uri", "position", "size"}
if unknown:
raise ValueError(f"Unknown blob fields: {sorted(unknown)}")
return value
try:
from lance.blob import Blob
except ModuleNotFoundError as err:
if err.name not in ("lance", "lance.blob"):
raise
else:
if isinstance(value, Blob):
return {
"data": value.data,
"uri": value.uri,
"position": value.position,
"size": value.size,
}
raise TypeError(f"Unsupported blob value: {type(value).__name__}")
def _blob_input_to_arrow(data: Any, schema: Optional[pa.Schema]) -> Optional[pa.Table]:
"""Convert Python rows with blob fields before Arrow loses their value types."""
if schema is None:
return None
if isinstance(data, list) and data and isinstance(data[0], dict):
names = {
field.name
for field in schema
if is_blob_v2_field(field)
and any(isinstance(row, dict) and field.name in row for row in data)
}
if not names:
return None
values = {name: [row.get(name) for row in data] for name in names}
rows = [{**row, **{name: None for name in names}} for row in data]
table = pa.Table.from_pylist(rows)
elif _check_for_pandas(data) and isinstance(data, pd.DataFrame):
names = {
field.name
for field in schema
if is_blob_v2_field(field) and field.name in data.columns
}
if not names:
return None
values = {
name: [
None
if value is pd.NA or isinstance(value, float) and np.isnan(value)
else value
for value in data[name]
]
for name in names
}
table = pa.Table.from_pandas(
data.assign(**{name: None for name in names}), preserve_index=False
).replace_schema_metadata(None)
else:
return None
for field in schema:
if field.name not in names:
continue
storage_type = (
field.type.storage_type
if isinstance(field.type, pa.ExtensionType)
else field.type
)
storage = pa.array(
[_blob_value_to_storage(value) for value in values[field.name]],
type=storage_type,
)
column = (
pa.ExtensionArray.from_storage(field.type, storage)
if isinstance(field.type, pa.ExtensionType)
else storage
)
table = table.set_column(
table.schema.get_field_index(field.name), field, column
)
return table
def _into_pyarrow_reader(
data, schema: Optional[pa.Schema] = None
) -> pa.RecordBatchReader:
@@ -303,9 +398,14 @@ def _into_pyarrow_reader(
return pa.Table.from_batches(data).to_reader()
else:
data = _serialize_json_values(data, schema)
return pa.Table.from_pylist(data).to_reader()
table = _blob_input_to_arrow(data, schema)
return (
table if table is not None else pa.Table.from_pylist(data)
).to_reader()
elif _check_for_pandas(data) and isinstance(data, pd.DataFrame):
table = pa.Table.from_pandas(data, preserve_index=False)
table = _blob_input_to_arrow(data, schema)
if table is None:
table = pa.Table.from_pandas(data, preserve_index=False)
# Do not serialize Pandas metadata
meta = table.schema.metadata if table.schema.metadata is not None else {}
meta = {k: v for k, v in meta.items() if k != b"pandas"}
@@ -1502,8 +1602,8 @@ class Table(ABC):
language: str = "English",
max_token_length: Optional[int] = 40,
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
stem: Optional[bool] = None,
remove_stop_words: Optional[bool] = None,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
@@ -1553,6 +1653,7 @@ class Table(ABC):
- "simple": Splits text by whitespace and punctuation.
- "whitespace": Split text by whitespace, but not punctuation.
- "raw": No tokenization. The entire text is treated as a single token.
- "code": Tokenizes source code and identifiers.
- "ngram": N-Gram tokenizer.
- "icu": ICU dictionary-based word segmentation.
- "icu/split": ICU segmentation with simple-style delimiter splitting.
@@ -1567,12 +1668,16 @@ class Table(ABC):
lower_case : bool, default True
Whether to convert the token to lower case. This makes queries
case-insensitive.
stem : bool, default True
stem : bool, optional
Whether to stem the token. Stemming reduces words to their root form.
For example, in English "running" and "runs" would both be reduced to "run".
remove_stop_words : bool, default True
``None`` uses the base tokenizer's default: False for ``code`` and
``ngram``, True otherwise.
remove_stop_words : bool, optional
Whether to remove stop words. Stop words are common words that are often
removed from text before indexing. For example, in English "the" and "and".
``None`` uses the base tokenizer's default: False for ``code`` and
``ngram``, True otherwise.
custom_stop_words : list of str, optional
Custom words that replace the built-in language stop words. ``None``
uses the built-in list; an empty list explicitly uses no stop words.
@@ -1957,15 +2062,20 @@ class Table(ABC):
@abstractmethod
def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> pa.LargeBinaryArray:
"""Materialize full blob bytes for ``column`` at the given rows.
The result has the same length and order as ``row_ids``. Null blobs
produce null slots; valid empty blobs produce ``b""``.
``_rowid`` values stay valid after compaction when the table has stable
row ids.
``row_ids`` may be a list, an integer Arrow array or chunked array,
or a query table with row ids. ``_rowid`` values stay valid after
compaction when the table has stable row ids.
One missing or deleted row ID rejects the entire batch; no partial
results are returned.
Blobs inside lists cannot be fetched by row id yet.
@@ -1979,17 +2089,20 @@ class Table(ABC):
def fetch_blob_ranges(
self,
column: str,
requests: Sequence[Tuple[int, int, int]],
requests: Sequence[Sequence[int]],
) -> pa.LargeBinaryArray:
"""Materialize row-specific byte ranges from a blob v2 column.
Each request is a ``(row_id, offset, length)`` tuple. Requests may be
repeated or reordered, including multiple ranges for the same blob.
Each request is a three-item ``(row_id, offset, length)`` sequence.
Requests may be repeated or reordered, including multiple ranges for
the same blob.
The result has the same length and order as ``requests``; null blobs
produce null slots and empty ranges on non-null blobs produce ``b""``.
``_rowid`` values stay valid after compaction when the table has stable
row ids.
One missing or deleted row ID rejects the entire batch; no partial
results are returned.
Row IDs can be obtained from a query with ``with_row_id(True)``. This
API is currently supported only by local tables.
@@ -1997,18 +2110,22 @@ class Table(ABC):
@abstractmethod
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> "list[Optional[BlobFile]]":
"""Open lazy, seekable :class:`~lancedb._blob.BlobFile` handles.
Prefer this over :meth:`fetch_blobs` for large payloads. ``row_ids`` is
a ``list[int]`` or a query ``pyarrow.Table`` carrying row identity via
``_rowid`` or a ``_lance_row_id`` field on the blob descriptor. Null
rows are ``None``. Remote tables require LanceDB Cloud server 0.5.0 or
newer.
a ``list[int]``, an integer Arrow array or chunked array, or a query
``pyarrow.Table`` carrying row identity via ``_rowid`` or a
``_lance_row_id`` field on the blob descriptor. Null blob values return
``None``. Remote tables require LanceDB Cloud server 0.5.0 or newer.
``_rowid`` values stay valid after compaction when the table has stable
row ids.
One missing or deleted row ID rejects the entire batch; no partial
handles are returned.
"""
@abstractmethod
@@ -2523,6 +2640,10 @@ class Table(ABC):
to nullable. Currently, you cannot change a nullable column to
non-nullable.
Each alteration must specify at least one of "rename", "data_type",
or "nullable". The legacy key "name" is also accepted for renaming.
Unknown keys raise ValueError before any alterations are applied.
Returns
-------
AlterColumnsResult
@@ -2910,19 +3031,23 @@ class LanceTable(Table):
return LOOP.run(self._table.blob_columns())
def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> pa.LargeBinaryArray:
return LOOP.run(self._table.fetch_blobs(column, row_ids))
def fetch_blob_ranges(
self,
column: str,
requests: Sequence[Tuple[int, int, int]],
requests: Sequence[Sequence[int]],
) -> pa.LargeBinaryArray:
return LOOP.run(self._table.fetch_blob_ranges(column, list(requests)))
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> "list[Optional[BlobFile]]":
return LOOP.run(self._table.fetch_blob_files(column, row_ids))
@@ -3710,8 +3835,8 @@ class LanceTable(Table):
language: str = "English",
max_token_length: Optional[int] = 40,
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
stem: Optional[bool] = None,
remove_stop_words: Optional[bool] = None,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
@@ -4238,9 +4363,9 @@ class LanceTable(Table):
)
if storage_options is None:
storage_options = {}
storage_options["new_table_enable_v2_manifest_paths"] = (
storage_options["new_table_enable_v2_manifest_paths"] = str(
enable_v2_manifest_paths
)
).lower()
self._table = LOOP.run(
self._conn._conn.create_table(
@@ -4337,6 +4462,7 @@ class LanceTable(Table):
and not self._route_pushdown_to_rust
and self.current_branch() is None
and query.take_offsets is None
and query.search_effort is None
):
from lancedb.namespace import _execute_server_side_query
@@ -4672,7 +4798,7 @@ class LanceTable(Table):
[LanceTable.uses_v2_manifest_paths][lancedb.table.LanceTable.uses_v2_manifest_paths]
to check if the table is already using the new path style.
"""
LOOP.run(self._table.migrate_v2_manifest_paths())
LOOP.run(self._table.migrate_manifest_paths_v2())
@deprecation.deprecated(
deprecated_in="0.33.1",
@@ -5347,11 +5473,16 @@ class AsyncTable:
async def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
"""Read the LsmWriteSpec currently installed on this table.
Returns ``None`` when the MemWAL LSM write path is not enabled (no
spec has been set, or it was removed with `unset_lsm_write_spec`).
The returned spec mirrors what was passed to `set_lsm_write_spec`,
except that ``maintained_indexes`` always reports the concrete list
resolved when the spec was set — ``None`` never round-trips.
Returns ``None`` when the LSM write path is not enabled at all — no
spec has been set, or one was removed with `unset_lsm_write_spec`.
That is a different answer from a spec whose ``maintained_indexes``
is ``None``, which is an installed spec selecting indexes
automatically.
The spec read back is the one that was installed, selection
included: ``None`` maintains every supported index the table has now
or gains later, ``[]`` maintains none, and a non-empty list maintains
exactly those. All three round-trip.
"""
return await self._inner.get_lsm_write_spec()
@@ -5494,7 +5625,11 @@ class AsyncTable:
return AsyncQuery(self._inner.query(), self)
async def to_lance(self, **kwargs) -> lance.LanceDataset:
"""Return the Lance dataset backing this table.
"""Return the Lance dataset backing a local table.
Remote tables cannot be opened as Lance datasets by the client. Use
[to_arrow][lancedb.table.AsyncTable.to_arrow] or
[query][lancedb.table.AsyncTable.query] to read them through the server.
Parameters
----------
@@ -5511,6 +5646,12 @@ class AsyncTable:
>>> async def get_lance_dataset(table):
... return await table.to_lance()
"""
if not self._inner._is_native():
raise NotImplementedError(
"to_lance() is not supported for remote tables; "
"query the server instead"
)
try:
import lance
except ImportError:
@@ -5540,7 +5681,8 @@ class AsyncTable:
Parameters
----------
blob_mode: str, default "lazy"
Controls how Lance blob columns are returned.
Controls how Lance blob columns are returned. Remote tables support
"descriptions"; "bytes" and "lazy" are not yet supported.
**kwargs
Forwarded to PyArrow / Lance pandas conversion.
@@ -5558,6 +5700,9 @@ class AsyncTable:
)
return arrow_tbl.to_pandas(**kwargs)
if not self._inner._is_native():
return await self.query().to_pandas(blob_mode=blob_mode, **kwargs)
if blob_mode == "lazy" and get_uri_scheme(await self.uri()) == "memory":
return (await self.to_arrow()).to_pandas(**kwargs)
if blob_mode == "bytes" and blob_v2_column_paths(schema):
@@ -5897,6 +6042,9 @@ class AsyncTable:
"""
schema = await self.schema()
data = _serialize_json_values(data, schema)
blob_table = _blob_input_to_arrow(data, schema)
if blob_table is not None:
data = blob_table
if on_bad_vectors is None:
on_bad_vectors = "error"
if fill_value is None:
@@ -6010,7 +6158,17 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[str] = None,
query: None = None,
vector_column_name: Optional[str] = None,
query_type: QueryType = "auto",
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
) -> AsyncQuery: ...
@overload
async def search(
self,
query: str,
vector_column_name: Optional[str] = None,
query_type: Literal["auto"] = ...,
ordering_field_name: Optional[str] = None,
@@ -6020,7 +6178,7 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[str] = None,
query: str,
vector_column_name: Optional[str] = None,
query_type: Literal["hybrid"] = ...,
ordering_field_name: Optional[str] = None,
@@ -6030,7 +6188,7 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[Union[VEC, "PIL.Image.Image", Tuple]] = None,
query: Union[VEC, "PIL.Image.Image", Tuple],
vector_column_name: Optional[str] = None,
query_type: Literal["auto"] = ...,
ordering_field_name: Optional[str] = None,
@@ -6040,7 +6198,7 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[str] = None,
query: str,
vector_column_name: Optional[str] = None,
query_type: Literal["fts"] = ...,
ordering_field_name: Optional[str] = None,
@@ -6050,9 +6208,7 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[
Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery]
] = None,
query: Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery],
vector_column_name: Optional[str] = None,
query_type: Literal["vector"] = ...,
ordering_field_name: Optional[str] = None,
@@ -6068,7 +6224,7 @@ class AsyncTable:
query_type: QueryType = "auto",
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
) -> Union[AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]:
) -> Union[AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]:
"""Create a search query to find the nearest neighbors
of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/)
and [full-text search](https://lancedb.com/docs/search/full-text-search/).
@@ -6083,8 +6239,9 @@ class AsyncTable:
- *default None*.
Acceptable types are: list, np.ndarray, PIL.Image.Image
- If None then the select/where/limit clauses are applied to filter
the table
- If None then a plain [AsyncQuery][lancedb.query.AsyncQuery] is
returned, equivalent to calling [query][lancedb.table.AsyncTable.query].
The select/where/limit clauses are applied to filter the table.
vector_column_name: str, optional
The name of the vector column to search.
@@ -6112,10 +6269,13 @@ class AsyncTable:
Returns
-------
LanceQueryBuilder
AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, or AsyncVectorQuery
A query builder object representing the query.
"""
if query is None:
return self.query()
def is_embedding(query):
return isinstance(query, (list, np.ndarray, pa.Array, pa.ChunkedArray))
@@ -6302,9 +6462,12 @@ class AsyncTable:
)
if query.distance_type is not None:
async_query = async_query.distance_type(query.distance_type)
if query.nprobes is not None:
async_query = async_query.nprobes(query.nprobes)
if query.search_effort is not None:
async_query = async_query.search_effort(query.search_effort)
if query.minimum_nprobes is not None and query.maximum_nprobes is not None:
# Set both to the minimum first to avoid min > max error.
async_query = async_query.nprobes(
async_query = async_query.minimum_nprobes(
query.minimum_nprobes
).maximum_nprobes(query.maximum_nprobes)
elif query.minimum_nprobes is not None:
@@ -6343,6 +6506,7 @@ class AsyncTable:
)
and not self._route_pushdown_to_rust
and query.take_offsets is None
and query.search_effort is None
):
from lancedb.namespace import _execute_server_side_query
@@ -6765,6 +6929,10 @@ class AsyncTable:
to nullable. Currently, you cannot change a nullable column to
non-nullable.
Each alteration must specify at least one of "rename", "data_type",
or "nullable". The legacy key "name" is also accepted for renaming.
Unknown keys raise ValueError before any alterations are applied.
Returns
-------
AlterColumnsResult
@@ -6808,9 +6976,11 @@ class AsyncTable:
"""
versions = await self._inner.list_versions()
for v in versions:
ts_nanos = v["timestamp"]
v["timestamp"] = datetime.fromtimestamp(ts_nanos // 1e9) + timedelta(
microseconds=(ts_nanos % 1e9) // 1e3
# Use integer math: float division on ~1e18 nanosecond
# values loses sub-millisecond precision.
seconds, nanos = divmod(v["timestamp"], 1_000_000_000)
v["timestamp"] = datetime.fromtimestamp(seconds) + timedelta(
microseconds=nanos // 1000
)
return versions
@@ -6931,7 +7101,9 @@ class AsyncTable:
return await self._inner.blob_columns()
async def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> pa.LargeBinaryArray:
return await self._inner.fetch_blobs(
column, _normalize_blob_row_ids(row_ids, column)
@@ -6940,12 +7112,14 @@ class AsyncTable:
async def fetch_blob_ranges(
self,
column: str,
requests: Sequence[Tuple[int, int, int]],
requests: Sequence[Sequence[int]],
) -> pa.LargeBinaryArray:
return await self._inner.fetch_blob_ranges(column, list(requests))
async def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> "list[Optional[BlobFile]]":
handles = await self._inner.fetch_blob_files(
column, _normalize_blob_row_ids(row_ids, column)
@@ -7205,6 +7379,13 @@ class TableStatistics:
and manifests.
num_rows: int
The total number of rows in the table.
num_deleted_rows: Optional[int]
The total number of rows marked as deleted across all fragments of the
table. These rows are not counted in ``num_rows``, but still occupy space
on disk until the table is compacted, so a large value here indicates
that the table should be optimized. Fragments in which every row was
deleted are dropped outright, so their rows are not counted here.
``None`` when the backend does not report deletion counts.
num_indices: int
The total number of indices in the table.
fragment_stats: FragmentStatistics
@@ -7213,6 +7394,7 @@ class TableStatistics:
total_bytes: int
num_rows: int
num_deleted_rows: Optional[int]
num_indices: int
fragment_stats: FragmentStatistics
python/python/lancedb/util.py
+20
View File
@@ -10,6 +10,7 @@ import os
import pathlib
import warnings
from datetime import date, datetime
from decimal import Decimal
from functools import singledispatch
from typing import Tuple, Union, Optional, Any, List
from urllib.parse import urlparse
@@ -280,6 +281,14 @@ def infer_vector_column_dim(data_type: pa.DataType) -> Optional[int]:
return None
def _validate_query_vector(query: Any) -> None:
"""Reject empty vector inputs before vector-column inference or execution."""
if (isinstance(query, list) and not query) or (
isinstance(query, np.ndarray) and query.size == 0
):
raise ValueError("Query vector must not be empty")
def _query_vector_dim(query: Optional[Any]) -> Optional[int]:
if query is None:
return None
@@ -301,6 +310,9 @@ def infer_vector_column_name(
query: Optional[Any], # inferred later in query builder
vector_column_name: Optional[str],
):
if query_type != "fts":
_validate_query_vector(query)
if vector_column_name is not None:
return vector_column_name
@@ -362,6 +374,14 @@ def _(value: float):
return str(value)
@value_to_sql.register(Decimal)
def _(value: Decimal):
if not value.is_finite():
raise ValueError("Non-finite Decimal values cannot be converted to SQL")
# Quote exact decimal text so the target-column cast avoids float rounding.
return value_to_sql(format(value, "f"))
@value_to_sql.register(bool)
def _(value: bool):
return str(value).upper()
python/python/tests/test_blob.py
+358 -2
View File
@@ -7,13 +7,15 @@ import sys
import textwrap
import lance
import pandas as pd
import pyarrow as pa
import pyarrow.compute as pc
import pytest
from lance.blob import BlobType as LanceBlobType
from lance.blob import Blob, BlobType as LanceBlobType
import lancedb
from lancedb._blob import (
_set_blob_column,
blob_v2_projection_sources,
read_row_ids_from_hits,
stash_auto_row_ids,
@@ -68,21 +70,25 @@ def _row_ids_by_id(table):
def _assert_missing_blob_row_ids(exc_info):
message = str(exc_info.value)
assert "row ids" in message
assert "row ids but some do not exist in the table" in message
assert "rowaddr" not in message
assert "fragment" not in message
def _assert_fetch_apis_reject_missing_row_ids(table, row_ids):
expected = f"first missing row ids: [{row_ids[0]}]"
with pytest.raises(ValueError) as exc_info:
table.fetch_blobs("image", row_ids)
_assert_missing_blob_row_ids(exc_info)
assert expected in str(exc_info.value)
with pytest.raises(ValueError) as exc_info:
table.fetch_blob_files("image", row_ids)
_assert_missing_blob_row_ids(exc_info)
assert expected in str(exc_info.value)
with pytest.raises(ValueError) as exc_info:
table.fetch_blob_ranges("image", [(row_id, 0, 1) for row_id in row_ids])
_assert_missing_blob_row_ids(exc_info)
assert expected in str(exc_info.value)
def test_blob_factory_declares_v2_field():
@@ -93,6 +99,37 @@ def test_blob_factory_declares_v2_field():
assert type(field.type) is LanceBlobType
def test_blob_thresholds_land_in_field_metadata():
field = lancedb.blob(
"payload",
inline_size_threshold=8 * 1024,
dedicated_size_threshold=1 << 20,
pack_file_size_threshold=1 << 26,
)
assert field.type.extension_name == "lance.blob.v2"
metadata = field.metadata
assert metadata[b"lance-encoding:blob-inline-size-threshold"] == b"8192"
assert metadata[b"lance-encoding:blob-dedicated-size-threshold"] == b"1048576"
assert metadata[b"lance-encoding:blob-pack-file-size-threshold"] == b"67108864"
def test_blob_without_thresholds_has_no_threshold_metadata():
field = lancedb.blob("image")
metadata = field.metadata or {}
assert not any(b"threshold" in key for key in metadata)
def test_blob_thresholds_are_validated():
with pytest.raises(ValueError, match="inline_size_threshold must be non-negative"):
lancedb.blob("payload", inline_size_threshold=-1)
with pytest.raises(ValueError, match="dedicated_size_threshold must be positive"):
lancedb.blob("payload", dedicated_size_threshold=0)
with pytest.raises(ValueError, match="pack_file_size_threshold must be positive"):
lancedb.blob("payload", pack_file_size_threshold=0)
with pytest.raises(TypeError, match="inline_size_threshold must be an int"):
lancedb.blob("payload", inline_size_threshold="8192")
def test_blob_type_works_without_pylance():
script = _HIDE_LANCE_BLOB + textwrap.dedent(
"""\
@@ -296,6 +333,121 @@ def test_blob_v2_projection_sources_use_typed_column_name():
}
def test_blob_v2_projection_sources_include_selected_struct():
schema = pa.schema(
[
pa.field(
"info", pa.struct([pa.field("name", pa.string()), lancedb.blob("blob")])
)
]
)
assert blob_v2_projection_sources(schema, ["info"]) == {"info.blob": "info.blob"}
assert blob_v2_projection_sources(schema, {"renamed": "info"}) == {
"renamed.blob": "info.blob"
}
@pytest.mark.parametrize("parent_path", ["info", "outer.info"])
def test_set_blob_column_missing_nested_field_raises(parent_path):
row = {"name": "first"}
for name in reversed(parent_path.split(".")):
row = {name: row}
table = pa.Table.from_pylist([row])
blobs = pa.array([b"hello"], type=pa.large_binary())
with pytest.raises(KeyError) as exc_info:
_set_blob_column(table, f"{parent_path}.blob", blobs)
assert parent_path in str(exc_info.value)
assert "'blob'" in str(exc_info.value)
def _nested_blob_table(db, name):
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field(
"info", pa.struct([pa.field("name", pa.string()), lancedb.blob("blob")])
),
]
)
table = db.create_table(name, schema=schema)
info = pa.StructArray.from_arrays(
[
pa.array(["first", "second", "third"]),
pa.array([b"hello", None, b"hidden"], type=pa.large_binary()),
],
names=["name", "blob"],
mask=pa.array([False, False, True]),
)
table.add(pa.table({"id": pa.array([0, 1, 2]), "info": info}))
return table
def test_nested_blob_to_pandas_bytes(tmp_path):
table = _nested_blob_table(lancedb.connect(tmp_path), "nested_pandas_bytes")
expected = [
{"name": "first", "blob": b"hello"},
{"name": "second", "blob": None},
None,
]
assert table.to_pandas(blob_mode="bytes")["info"].tolist() == expected
assert table.search().to_pandas(blob_mode="bytes")["info"].tolist() == expected
assert (
table.search().select(["info"]).to_pandas(blob_mode="bytes")["info"].tolist()
== expected
)
assert (
table.search()
.select({"renamed": "info"})
.to_pandas(blob_mode="bytes")["renamed"]
.tolist()
== expected
)
assert table.search().select({"b": "info.blob"}).to_pandas(blob_mode="bytes")[
"b"
].tolist() == [b"hello", None, None]
@pytest.mark.asyncio
async def test_nested_blob_to_pandas_bytes_async(tmp_path):
db = await lancedb.connect_async(tmp_path)
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field(
"info", pa.struct([pa.field("name", pa.string()), lancedb.blob("blob")])
),
]
)
table = await db.create_table("nested_pandas_bytes_async", schema=schema)
await table.add(
pa.table(
{
"id": pa.array([0, 1]),
"info": pa.StructArray.from_arrays(
[
pa.array(["first", "second"]),
pa.array([b"hello", None], type=pa.large_binary()),
],
names=["name", "blob"],
),
}
)
)
expected = [
{"name": "first", "blob": b"hello"},
{"name": "second", "blob": None},
]
assert (await table.to_pandas(blob_mode="bytes"))["info"].tolist() == expected
assert (await table.query().to_pandas(blob_mode="bytes"))[
"info"
].tolist() == expected
def _legacy_v1_table(name):
# Legacy v1 blob columns are only writable at file version <= 2.1.
db = lancedb.connect(
@@ -719,6 +871,38 @@ def test_fetch_blobs_accepts_query_result():
assert {blobs[i].as_py() for i in range(len(blobs))} == {b"gamma"}
def test_fetch_blob_apis_accept_integer_arrow_row_ids():
table = _blob_table(
"arrow_row_ids",
[{"id": 1, "image": b"alpha"}, {"id": 2, "image": b"beta"}],
)
by_id = _row_ids_by_id(table)
ids = [by_id[2], by_id[1]]
for row_ids in (
pa.array(ids, type=pa.uint64()),
pa.chunked_array([ids[:1], ids[1:]], type=pa.int64()),
):
assert table.fetch_blobs("image", row_ids).to_pylist() == [b"beta", b"alpha"]
files = table.fetch_blob_files("image", row_ids)
assert [file.read() for file in files] == [b"beta", b"alpha"]
def test_fetch_blob_apis_validate_arrow_and_negative_row_ids():
table = _blob_table("invalid_arrow_row_ids", [{"id": 1, "image": b"alpha"}])
hits = table.search().with_row_id(True).to_arrow()
for fetch in (table.fetch_blobs, table.fetch_blob_files):
with pytest.raises(ValueError, match="non-negative"):
fetch("image", [-1])
with pytest.raises(ValueError, match="non-negative"):
fetch("image", pa.array([-1], type=pa.int64()))
with pytest.raises(ValueError, match="without nulls"):
fetch("image", pa.array([None], type=pa.int64()))
with pytest.raises(ValueError, match="integer type"):
fetch("image", hits["image"])
def test_fetch_blobs_after_compact_with_stable_row_ids(tmp_path):
db = lancedb.connect(
tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"}
@@ -850,6 +1034,15 @@ def test_fetch_blob_ranges_aligns_repeated_ranges_and_nulls():
assert ranges.to_pylist() == [b"cde", None, b"ab", b"cde", b""]
def test_fetch_blob_ranges_accepts_three_item_sequences():
table = _blob_table("range_sequences", [{"id": 1, "image": b"abcdef"}])
row_id = _row_ids_by_id(table)[1]
ranges = table.fetch_blob_ranges("image", [[row_id, 1, 2], (row_id, 4, 2)])
assert ranges.to_pylist() == [b"bc", b"ef"]
def test_fetch_blob_ranges_validates_requests():
table = _blob_table("range_validation", [{"id": 1, "image": b"abc"}])
row_id = _row_ids_by_id(table)[1]
@@ -864,6 +1057,30 @@ def test_fetch_blob_ranges_validates_requests():
table.fetch_blob_ranges("image", [(2**64 - 1, 0, 1)])
_assert_missing_blob_row_ids(exc_info)
for index, name in enumerate(("row id", "offset", "length")):
request = [row_id, 0, 1]
request[index] = -1
with pytest.raises(
ValueError, match=f"blob range {name} must be a non-negative"
):
table.fetch_blob_ranges("image", [request])
with pytest.raises(ValueError, match="three values"):
table.fetch_blob_ranges("image", [[row_id, 0]])
def test_fetch_blob_apis_reject_row_past_fragment_end():
table = _blob_table(
"past_fragment_end",
[
{"id": 1, "image": b"a"},
{"id": 2, "image": b"b"},
{"id": 3, "image": b"c"},
],
)
_assert_fetch_apis_reject_missing_row_ids(table, [3])
_assert_fetch_apis_reject_missing_row_ids(table, [3, 0])
def test_fetch_blob_apis_reject_missing_fragment_row_addr():
table = _blob_table("missing_frag", [{"id": 1, "image": b"x"}])
@@ -881,6 +1098,19 @@ def test_fetch_blob_apis_reject_deleted_row_ids():
_assert_fetch_apis_reject_missing_row_ids(table, [by_id[2], by_id[1]])
def test_fetch_blob_apis_reject_deleted_stable_row_ids(tmp_path):
db = lancedb.connect(
tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"}
)
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table("stable_deleted", schema=schema)
table.add([{"id": 1, "image": b"one"}, {"id": 2, "image": b"two"}])
by_id = _row_ids_by_id(table)
table.delete("id = 2")
_assert_fetch_apis_reject_missing_row_ids(table, [by_id[2], by_id[1]])
def test_fetch_blob_ranges_empty_requests_returns_empty_array():
table = _blob_table("range_empty", [{"id": 1, "image": b"x"}])
assert table.fetch_blob_ranges("image", []).to_pylist() == []
@@ -1438,3 +1668,129 @@ def test_add_external_uri_string_round_trips_with_flag(tmp_path):
hits = table.search().to_arrow()
blobs = table.fetch_blobs("image", hits)
assert blobs[0].as_py() == payload
@pytest.mark.parametrize("as_pandas", [False, True])
def test_add_bytes_and_uri_in_one_batch_preserves_external_uri(tmp_path, as_pandas):
payload = b"external-payload"
blob_path = tmp_path / "payload.bin"
blob_path.write_bytes(payload)
rows = [
{"id": 1, "image": b"inline"},
{"id": 2, "image": blob_path.as_uri()},
]
db = lancedb.connect(tmp_path / "db")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table("mixed", schema=schema)
table.add(
pd.DataFrame(rows) if as_pandas else rows,
allow_external_blob_outside_bases=True,
)
by_id = _row_ids_by_id(table)
assert table.fetch_blobs("image", [by_id[1], by_id[2]]).to_pylist() == [
b"inline",
payload,
]
@pytest.mark.parametrize("as_pandas", [False, True])
def test_add_mixed_python_blob_values_preserves_external_uris(tmp_path, as_pandas):
payload = b"external-payload"
blob_path = tmp_path / "payload.bin"
blob_path.write_bytes(payload)
uri = blob_path.as_uri()
rows = [
{"id": 1, "image": b"inline"},
{"id": 2, "image": uri},
{"id": 3, "image": {"data": b"from-dict"}},
{"id": 4, "image": Blob.from_bytes(b"from-blob")},
{"id": 5, "image": Blob.from_uri(uri)},
{"id": 6, "image": {"uri": uri}},
{"id": 7, "image": None},
]
db = lancedb.connect(tmp_path / "db")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table("mixed", schema=schema)
table.add(
pd.DataFrame(rows) if as_pandas else rows,
allow_external_blob_outside_bases=True,
)
by_id = _row_ids_by_id(table)
assert table.fetch_blobs("image", [by_id[i] for i in range(1, 8)]).to_pylist() == [
b"inline",
payload,
b"from-dict",
b"from-blob",
payload,
payload,
None,
]
@pytest.mark.parametrize("as_pandas", [False, True])
def test_merge_insert_mixed_python_blobs_validates_external_uri(tmp_path, as_pandas):
blob_path = tmp_path / "payload.bin"
blob_path.write_bytes(b"external-payload")
table = _blob_table("merge_mixed_uri", [{"id": 1, "image": b"before"}])
rows = [
{"id": 1, "image": b"updated"},
{"id": 2, "image": blob_path.as_uri()},
]
with pytest.raises(ValueError, match="outside registered external bases"):
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(pd.DataFrame(rows) if as_pandas else rows)
)
assert table.count_rows() == 1
@pytest.mark.parametrize("as_pandas", [False, True])
def test_merge_insert_python_blob_dicts_and_objects(as_pandas):
table = _blob_table("merge_python_blobs", [{"id": 1, "image": b"before"}])
rows = [
{"id": 1, "image": b"updated"},
{"id": 2, "image": {"data": b"from-dict"}},
{"id": 3, "image": Blob.from_bytes(b"from-blob")},
]
result = (
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(pd.DataFrame(rows) if as_pandas else rows)
)
assert result.num_updated_rows == 1
assert result.num_inserted_rows == 2
by_id = _row_ids_by_id(table)
assert table.fetch_blobs("image", [by_id[i] for i in range(1, 4)]).to_pylist() == [
b"updated",
b"from-dict",
b"from-blob",
]
def test_create_table_with_python_blob_dict_and_object():
db = lancedb.connect("memory:///")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table(
"create_python_blobs",
data=[
{"id": 1, "image": {"data": b"from-dict"}},
{"id": 2, "image": Blob.from_bytes(b"from-blob")},
],
schema=schema,
)
by_id = _row_ids_by_id(table)
assert table.fetch_blobs("image", [by_id[1], by_id[2]]).to_pylist() == [
b"from-dict",
b"from-blob",
]
python/python/tests/test_catalog.py
+129 -4
View File
@@ -55,7 +55,7 @@ def test_catalog_sync_scope_and_serialization(catalog_server):
(200, {}),
(200, {"tables": []}),
(200, {"tables": []}),
(200, {"namespaces": ["team/search"], "page_token": "next"}),
(200, {"namespaces": ["team/search"]}),
(204, None),
]
)
@@ -85,13 +85,12 @@ def test_catalog_sync_scope_and_serialization(catalog_server):
with pytest.raises(ValueError, match="sql_host_override must use"):
connection.execute_query_async("SELECT 1")
assert restored.table_names() == []
page = catalog.list_databases(limit=1, page_token="a/b")
assert page == lancedb.ListDatabasesResponse(["team/search"], "next")
assert list(catalog.list_databases()) == ["team/search"]
catalog.drop_database("team/search", ignore_missing=True)
assert requests[0][0] == "/v1/namespace/team%2Fsearch/create"
assert requests[0][2] == {"mode": "ExistOk"}
assert requests[1][0] == "/v1/namespace/team%2Fsearch/describe"
assert requests[4][0] == "/v1/namespace/$/list?limit=1&page_token=a%2Fb"
assert requests[4][0] == "/v1/namespace/$/list"
assert requests[5][2] == {"mode": "Skip", "behavior": "Restrict"}
for i, (_, headers, _) in enumerate(requests):
headers = {key.lower(): value for key, value in headers.items()}
@@ -138,3 +137,129 @@ async def test_catalog_async_and_errors(catalog_server):
def test_catalog_requires_remote_endpoint(endpoint):
with pytest.raises(ValueError, match="endpoint"):
lancedb.connect_catalog(endpoint)
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.asyncio
@pytest.mark.parametrize("page_token,page_limit", [(None, None), ("start/token", 2)])
async def test_database_names_iteration(
catalog_server, asynchronous, page_token, page_limit
):
endpoint, requests, responses = catalog_server
responses.extend(
[
(200, {"namespaces": ["first", "second"], "page_token": "a/b"}),
(200, {"namespaces": [], "page_token": "next"}),
(200, {"namespaces": ["last", "final"], "page_token": ""}),
]
)
catalog = (
await lancedb.connect_catalog_async(endpoint)
if asynchronous
else lancedb.connect_catalog(endpoint)
)
names = catalog.list_databases(page_token=page_token, page_limit=page_limit)
assert requests == []
assert names.num_page_results() == 0
assert names.page_token() == page_token
for cached, expected in [(1, "first"), (0, "second")]:
assert (await names.__anext__() if asynchronous else next(names)) == expected
assert names.num_page_results() == cached
assert names.page_token() == "a/b"
assert len(requests) == 1
assert (await names.__anext__() if asynchronous else next(names)) == "last"
assert names.num_page_results() == 1
assert names.page_token() is None
assert (await names.__anext__() if asynchronous else next(names)) == "final"
assert names.num_page_results() == 0
assert names.page_token() is None
for _ in range(2):
with pytest.raises(StopAsyncIteration if asynchronous else StopIteration):
if asynchronous:
await names.__anext__()
else:
next(names)
from urllib.parse import parse_qs, urlsplit
for request, token in zip(requests, [page_token, "a/b", "next"]):
url = urlsplit(request[0])
assert url.path == "/v1/namespace/$/list"
expected = {}
if token is not None:
expected["page_token"] = [token]
if page_limit is not None:
expected["limit"] = [str(page_limit)]
assert parse_qs(url.query) == expected
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("fail", [False, True])
@pytest.mark.asyncio
async def test_database_names_empty_or_error(catalog_server, asynchronous, fail):
endpoint, requests, responses = catalog_server
responses.append(
(401, {"error": "unauthorized"}) if fail else (200, {"namespaces": []})
)
catalog = (
await lancedb.connect_catalog_async(endpoint)
if asynchronous
else lancedb.connect_catalog(endpoint)
)
names = catalog.list_databases()
if fail:
with pytest.raises(HttpError):
if asynchronous:
await names.__anext__()
else:
next(names)
assert ([name async for name in names] if asynchronous else list(names)) == []
assert len(requests) == 1
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.asyncio
async def test_database_names_resume(catalog_server, asynchronous):
endpoint, requests, responses = catalog_server
responses.extend(
[
(200, {"namespaces": ["first"], "page_token": "resume/token"}),
(200, {"namespaces": ["second"]}),
]
)
catalog = (
await lancedb.connect_catalog_async(endpoint)
if asynchronous
else lancedb.connect_catalog(endpoint)
)
names = catalog.list_databases(page_limit=1)
assert (await names.__anext__() if asynchronous else next(names)) == "first"
assert names.num_page_results() == 0
resumed = catalog.list_databases(page_token=names.page_token(), page_limit=1)
assert resumed.page_token() == "resume/token"
assert len(requests) == 1
assert ([name async for name in resumed] if asynchronous else list(resumed)) == [
"second"
]
assert resumed.page_token() is None
assert resumed.num_page_results() == 0
assert requests[1][0] == "/v1/namespace/$/list?limit=1&page_token=resume%2Ftoken"
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("page_limit", [0, 2**32 - 1])
@pytest.mark.asyncio
async def test_database_names_invalid_limit(catalog_server, asynchronous, page_limit):
endpoint, requests, _ = catalog_server
catalog = (
await lancedb.connect_catalog_async(endpoint)
if asynchronous
else lancedb.connect_catalog(endpoint)
)
names = catalog.list_databases(page_limit=page_limit)
with pytest.raises(ValueError, match="limit"):
if asynchronous:
await names.__anext__()
else:
next(names)
assert names.num_page_results() == 0
assert requests == []
python/python/tests/test_db.py
+133
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import contextlib
import inspect
import re
import sys
@@ -244,6 +245,40 @@ def test_db_contains_and_len_include_all_table_name_pages(tmp_db: lancedb.DBConn
assert "does_not_exist" not in tmp_db
@pytest.mark.asyncio
async def test_table_listing_defaults_and_zero_limit(tmp_path):
names = [f"t{i:03}" for i in range(130)]
for name in names:
(tmp_path / f"{name}.lance").mkdir()
db = lancedb.connect(tmp_path)
assert db.table_names() == names
assert db.table_names(limit=None) == names
assert db.table_names(limit=0) == []
first = db.list_tables()
assert first.tables == names[:100]
assert first.page_token
second = db.list_tables(page_token=first.page_token)
assert second.tables == names[100:]
assert second.page_token is None
assert db.list_tables(limit=200).tables == names
empty = db.list_tables(limit=0)
assert empty.tables == []
assert empty.page_token is None
adb = await lancedb.connect_async(tmp_path)
assert await adb.table_names() == names
assert await adb.table_names(limit=0) == []
first = await adb.list_tables()
assert first.tables == names[:100]
second = await adb.list_tables(page_token=first.page_token)
assert second.tables == names[100:]
assert second.page_token is None
empty = await adb.list_tables(limit=0)
assert empty.tables == []
assert empty.page_token is None
def test_db_contains_stops_after_matching_table_page(
tmp_db: lancedb.DBConnection, monkeypatch
):
@@ -350,6 +385,40 @@ async def test_create_table_from_iterator_async(mem_db_async: lancedb.AsyncConne
assert await table.count_rows() == 10
def test_create_exist_ok_mode(tmp_db: lancedb.DBConnection):
db = tmp_db
data = [{"vector": [1.1, 1.2]}, {"vector": [0.2, 1.8]}]
table = db.create_table("t1", data, mode="exist_ok")
assert table.count_rows() == 2
original = table.to_arrow()
reopened = db.create_table("t1", [{"vector": [9.0, 8.0]}], mode="exist_ok")
assert reopened.to_arrow().equals(original)
def test_create_empty_exist_ok_mode(tmp_db: lancedb.DBConnection):
schema = pa.schema([pa.field("id", pa.int64())])
table = tmp_db.create_table("empty", schema=schema, mode="exist_ok")
assert table.schema == schema
assert table.count_rows() == 0
table.add([{"id": 7}])
reopened = tmp_db.create_table("empty", schema=schema, mode="exist_ok")
assert reopened.to_arrow().to_pylist() == [{"id": 7}]
bad_schema = pa.schema([pa.field("other", pa.int64())])
with pytest.raises(ValueError):
tmp_db.create_table("empty", schema=bad_schema, mode="exist_ok")
def test_create_table_invalid_mode(tmp_db: lancedb.DBConnection):
db = tmp_db
data = [{"vector": [1.1, 1.2]}, {"vector": [0.2, 1.8]}]
with pytest.raises(ValueError, match="Invalid mode bogus"):
db.create_table("tb", data, mode="bogus")
assert "tb" not in db
def test_create_exist_ok(tmp_db: lancedb.DBConnection):
from conftest import pandas_string_type
@@ -405,6 +474,49 @@ async def test_connect(tmp_path):
assert str(db) == f"ListingDatabase(uri={tmp_path}, read_consistency_interval=5s)"
@pytest.fixture(params=["memory", "local", "namespace", "from_inner"])
def sync_db(request, tmp_path):
if request.param == "memory":
return lancedb.connect("memory://")
if request.param == "namespace":
return lancedb.connect_namespace("dir", {"root": str(tmp_path)})
db = lancedb.connect(tmp_path)
if request.param == "from_inner":
return lancedb.db.LanceDBConnection.from_inner(db._inner, None)
return db
def test_sync_close(sync_db):
assert sync_db.is_open()
assert sync_db.close() is None
assert not sync_db.is_open()
assert sync_db.close() is None
with pytest.warns(DeprecationWarning, match="table_names"):
with pytest.raises(RuntimeError, match="Connection is closed"):
sync_db.table_names()
with pytest.raises(RuntimeError, match="Connection is closed"):
sync_db.list_tables()
with pytest.raises(RuntimeError, match="Connection is closed"):
sync_db.open_table("test")
@pytest.mark.parametrize("raise_error", [False, True])
def test_sync_context_manager(sync_db, raise_error):
with contextlib.ExitStack() as stack:
if raise_error:
stack.enter_context(pytest.raises(ValueError, match="test error"))
with sync_db as db:
assert db is sync_db
assert db.is_open()
if raise_error:
raise ValueError("test error")
assert not sync_db.is_open()
with pytest.raises(RuntimeError, match="Connection is closed"):
sync_db.list_tables()
@pytest.mark.asyncio
async def test_close(mem_db_async: lancedb.AsyncConnection):
assert mem_db_async.is_open()
@@ -495,6 +607,27 @@ async def test_create_exist_ok_async(tmp_db_async: lancedb.AsyncConnection):
# await db.create_table("test", schema=bad_schema, exist_ok=True)
@pytest.mark.parametrize("enable_v2_manifest_paths", [False, True])
@pytest.mark.parametrize("empty", [False, True])
def test_create_table_deprecated_v2_manifest_paths(
tmp_path, enable_v2_manifest_paths, empty
):
db = lancedb.connect(tmp_path)
data = None if empty else [{"id": 1}]
with pytest.warns(DeprecationWarning, match="enable_v2_manifest_paths"):
table = db.create_table(
"test",
data=data,
schema=pa.schema([("id", pa.int64())]),
enable_v2_manifest_paths=enable_v2_manifest_paths,
)
assert table.uses_v2_manifest_paths() == enable_v2_manifest_paths
assert table.to_arrow().to_pylist() == ([] if empty else data)
assert db.open_table("test").uses_v2_manifest_paths() == enable_v2_manifest_paths
@pytest.mark.asyncio
async def test_create_table_v2_manifest_paths_async(tmp_path):
db_with_v2_paths = await lancedb.connect_async(
python/python/tests/test_e2e_remote_db.py
+65
View File
@@ -2,13 +2,78 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import os
from datetime import timedelta
from uuid import uuid4
import lancedb
import numpy as np
import pyarrow as pa
import pytest
from lancedb import LanceDBConnection
from lancedb.index import FTS
# TODO: setup integ test mark and script
@pytest.mark.skipif(
not os.environ.get("LANCEDB_URI"), reason="Set LANCEDB_URI to a JSON FTS server"
)
def test_json_fts_against_server():
"""Requires a writable database and capacity to run the distributed indexer.
Configure LANCEDB_URI, LANCEDB_API_KEY, and optionally LANCEDB_REGION and
LANCEDB_HOST_OVERRIDE before running this test.
"""
uri = os.environ["LANCEDB_URI"]
assert uri.startswith("db://"), "LANCEDB_URI must identify a remote database"
conn = lancedb.connect(
uri,
api_key=os.environ.get("LANCEDB_API_KEY"),
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ.get("LANCEDB_HOST_OVERRIDE"),
)
name = f"json_fts_{uuid4().hex}"
table = conn.create_table(
name,
pa.table(
{
"id": [0, 1, 2],
"doc": pa.array(
[
'{"meta":{"tag":"nature"},"author":"alice"}',
'{"meta":{"tag":"city"},"author":"nature"}',
'{"meta":{"tag":"nature"},"author":"bob"}',
],
type=pa.json_(),
),
"raw": pa.array([b"nature"] * 3, type=pa.large_binary()),
}
),
)
try:
table.create_index("doc", config=FTS(), name="doc_fts")
table.wait_for_index(["doc_fts"], timeout=timedelta(minutes=5))
stats = table.index_stats("doc_fts")
assert stats.num_indexed_rows == 3
assert stats.num_unindexed_rows == 0
assert stats.index_type == "FTS"
for query, expected in [
("meta.tag,str,nature", [0, 2]),
("meta.tag,str,alice", []),
("meta.tag,str,tag", []),
]:
search = table.search(query, query_type="fts", fts_columns="doc")
assert sorted(row["id"] for row in search.to_list()) == expected
plan = search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("raw", config=FTS())
finally:
conn.drop_table(name)
@pytest.mark.skip(reason="Need to set up a local server")
def test_against_local_server():
conn = LanceDBConnection("lancedb+http://localhost:10024")
python/python/tests/test_first_class_function_slice1.py
+23
View File
@@ -162,6 +162,29 @@ def test_function_version_binds_named_columns_as_one_immutable_application():
] == [("text", "column", "documents.body")]
def test_function_version_names_its_namespace_as_a_separate_field():
"""A Function is named by its parts, as a Secret is; a root one states no
namespace."""
value = job_result("remote_function_job.json")
root = FunctionVersion.from_json(json.dumps(value))
assert root.namespace_path == ()
assert "namespace_path" not in json.loads(root.to_canonical_json())
nested = FunctionVersion(**{**value, "namespace_path": ["analytics", "features"]})
assert nested.name == "embed"
assert nested.namespace_path == ("analytics", "features")
assert json.loads(nested.to_canonical_json())["namespace_path"] == [
"analytics",
"features",
]
application = nested(text=col("documents.body"))
assert application.function.name == "embed"
assert application.function.namespace_path == ("analytics", "features")
assert json.loads(application.to_canonical_json())["function"][
"namespace_path"
] == ["analytics", "features"]
def test_function_version_binding_validates_names_and_direct_columns():
version = FunctionVersion.from_json(
json.dumps(job_result("remote_function_job.json"))
python/python/tests/test_first_class_function_slice2.py
+119 -9
View File
@@ -1499,7 +1499,7 @@ def test_local_function_catalog_operations_are_not_supported(tmp_path):
with pytest.raises(NotImplementedError, match=message):
db.get_function("normalize_score", version=FUNCTION_VERSION)
with pytest.raises(NotImplementedError, match=message):
db.list_functions()
list(db.list_functions())
with pytest.raises(NotImplementedError, match=message):
db.drop_function("normalize_score", version=FUNCTION_VERSION)
@@ -1539,8 +1539,10 @@ def _mock_remote_function_catalog():
else (None, None)
)
if function_action[1] == "create":
*namespace_path, name = function_action[0].split("$")
state["version"] = {
"name": "normalize_score",
"name": name,
**({"namespace_path": namespace_path} if namespace_path else {}),
"version": FUNCTION_VERSION,
"object_id": "fixture",
"location": "memory:///fixture",
@@ -1553,7 +1555,7 @@ def _mock_remote_function_catalog():
)["image"],
"signature": body["signature"],
"secret_bindings": body.get("secret_bindings", []),
"created_at": "2026-08-21T00:00:00Z",
"created_at_millis": 1787270400000, # 2026-08-21T00:00:00Z
}
response = {"job_id": "job-register"}
status = 202
@@ -1565,10 +1567,18 @@ def _mock_remote_function_catalog():
"job_state": "DONE",
"result": state["version"],
}
elif self.path == "/v1/function/normalize_score/describe":
# A namespaced Function's id is its namespace path and name joined
# by `$`, so the name is the last segment of it.
elif (
function_action[1] == "describe"
and function_action[0].split("$")[-1] == "normalize_score"
):
assert body == {"version": FUNCTION_VERSION}
response = state["version"]
elif self.path == "/v1/function/normalize_score/drop":
elif (
function_action[1] == "drop"
and function_action[0].split("$")[-1] == "normalize_score"
):
assert body == {"version": FUNCTION_VERSION}
response = {"dropped": True}
elif secret_action[1] in ("create", "alter"):
@@ -1602,7 +1612,8 @@ def _mock_remote_function_catalog():
assert query["page_token"] == "next"
self._write_response(200, {"secrets": [{"name": "hf-prod"}]})
return
if url.path != "/v1/namespace/$/function/list":
parts = url.path.strip("/").split("/")
if parts[:2] != ["v1", "namespace"] or parts[3:] != ["function", "list"]:
self._write_response(404, {"error": "not found"})
return
assert query["include_definition"] == "true"
@@ -1709,7 +1720,7 @@ def test_remote_secret_verbs_round_trip():
)
assert db.create_secret("openai-prod", "sk-live-0001") is None
assert db.alter_secret("openai-prod", "sk-live-0002") is None
assert db.list_secrets() == ["openai-prod", "hf-prod"]
assert list(db.list_secrets()) == ["openai-prod", "hf-prod"]
assert db.drop_secret("openai-prod") is None
routes = [path for path, _ in state["requests"]]
@@ -1770,7 +1781,7 @@ def test_remote_list_functions_paginates_and_returns_typed_versions():
)
created = db.create_function(normalize_score)
state["requests"].clear()
functions = db.list_functions()
functions = list(db.list_functions())
assert functions == [created]
assert state["requests"] == [
@@ -1794,7 +1805,7 @@ async def test_async_remote_list_functions_returns_typed_versions():
registration = await db.create_function_async(normalize_score)
created = await registration.wait()
state["requests"].clear()
functions = await db.list_functions()
functions = [item async for item in db.list_functions()]
assert functions == [created]
assert [path for path, _ in state["requests"]] == [
@@ -1840,3 +1851,102 @@ async def test_async_remote_drop_function_sends_exact_version():
{"version": FUNCTION_VERSION},
)
]
def test_remote_function_verbs_address_a_namespace_in_the_path():
"""A Function's namespace is part of the identifier each route addresses;
no body carries it, so a namespaced request differs only in its path."""
namespace = ["analytics", "features"]
with _mock_remote_function_catalog() as (host, state):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
created = db.create_function(normalize_score, namespace_path=namespace)
# The returned parts are what the other verbs take, so a created
# Function reopens from its own fields.
assert created.name == "normalize_score"
assert created.namespace_path == tuple(namespace)
reopened = db.get_function(
created.name,
version=created.version,
namespace_path=list(created.namespace_path),
)
assert reopened == created
assert reopened(value=lancedb.col("score")).function.namespace_path == tuple(
namespace
)
list(db.list_functions(namespace_path=namespace))
assert (
db.drop_function(
"normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
)
is True
)
dropped, _ = db.drop_function_async(
"normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
)
assert dropped is True
assert [path for path, _ in state["requests"]] == [
"/v1/function/analytics$features$normalize_score/create",
"/v1/jobs/describe",
"/v1/function/analytics$features$normalize_score/describe",
"/v1/namespace/analytics$features/function/list",
"/v1/namespace/analytics$features/function/list",
"/v1/function/analytics$features$normalize_score/drop",
"/v1/function/analytics$features$normalize_score/drop",
]
@pytest.mark.asyncio
async def test_async_remote_function_verbs_address_a_namespace_in_the_path():
namespace = ["analytics", "features"]
with _mock_remote_function_catalog() as (host, state):
db = await lancedb.connect_async(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
registration = await db.create_function_async(
normalize_score, namespace_path=namespace
)
await registration.wait()
await db.get_function(
"normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
)
[item async for item in db.list_functions(namespace_path=namespace)]
assert (
await db.drop_function(
"normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
)
is True
)
assert [path for path, _ in state["requests"]] == [
"/v1/function/analytics$features$normalize_score/create",
"/v1/jobs/describe",
"/v1/function/analytics$features$normalize_score/describe",
"/v1/namespace/analytics$features/function/list",
"/v1/namespace/analytics$features/function/list",
"/v1/function/analytics$features$normalize_score/drop",
]
def test_a_function_namespace_path_must_be_a_list_of_segments():
"""A bare string is refused rather than read as a sequence of one-character
segments, and nothing reaches the service."""
with _mock_remote_function_catalog() as (host, state):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
with pytest.raises(TypeError):
list(db.list_functions(namespace_path="analytics"))
assert state["requests"] == []
python/python/tests/test_fts.py
+174 -1
View File
@@ -13,6 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import random
import shutil
@@ -44,6 +45,75 @@ from utils import exception_output
TEST_LANGUAGE_MODEL_HOME = Path(__file__).parent / "models"
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
@pytest.mark.parametrize("block_size", [128, 256])
def test_json_fts(tmp_path, block_size):
db = ldb.connect(tmp_path)
data = pa.table(
{
"id": [0, 1, 2, 3],
"doc": pa.array(
[
'{"title":"brown fox","meta":{"tag":"nature"},"author":"alice"}',
'{"title":"blue car","meta":{"tag":"city"}}',
'{"title":"brown bear","meta":{"tag":"nature"}}',
'{"title":"red fox","author":"alice"}',
],
type=pa.json_(),
),
}
)
table = db.create_table("json_fts", data)
table.create_index("doc", config=FTS(block_size=block_size), name="doc_fts")
table = db.open_table("json_fts")
index = table.list_indices()[0]
assert index.name == "doc_fts"
assert index.index_type == "FTS"
assert index.columns == ["doc"]
stats = table.index_stats("doc_fts")
assert stats.num_indexed_rows == 4
assert stats.num_unindexed_rows == 0
for query, expected in [
("title,str,brown", [0, 2]),
("meta.tag,str,nature", [0, 2]),
("author,str,alice", [0, 3]),
("title,str,alice", []),
("title,str,title", []),
]:
search = table.search(query, query_type="fts", fts_columns="doc")
assert sorted(row["id"] for row in search.to_list()) == expected
plan = search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
@pytest.mark.asyncio
async def test_json_fts_async(tmp_path):
db = await ldb.connect_async(tmp_path)
table = await db.create_table(
"json_fts",
pa.table({"doc": pa.array(['{"meta":{"tag":"nature"}}'], type=pa.json_())}),
)
await table.create_index("doc", config=FTS())
search = table.query().nearest_to_text(MatchQuery("meta.tag,str,nature", "doc"))
assert len(await search.to_list()) == 1
plan = await search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
@pytest.mark.parametrize("dtype", [pa.binary(), pa.large_binary()])
def test_fts_rejects_raw_binary(tmp_path, dtype):
table = ldb.connect(tmp_path).create_table(
"binary_fts", pa.table({"doc": pa.array([b'{"title":"brown"}'], type=dtype)})
)
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("doc", config=FTS())
assert table.list_indices() == []
@pytest.fixture
def table(tmp_path) -> ldb.table.LanceTable:
# Use local random state to avoid affecting other tests
@@ -894,6 +964,78 @@ def test_tokenize_uses_simple_index_tokenizer(mem_db: DBConnection):
]
@pytest.mark.parametrize("legacy", [False, True])
@pytest.mark.parametrize(
("base_tokenizer", "options", "expected_filters", "expected_tokens"),
[
(
"code",
{},
(False, False),
[("running", 0), ("in", 1), ("cafe", 2)],
),
(
"code",
{"stem": None, "remove_stop_words": None},
(False, False),
[("running", 0), ("in", 1), ("cafe", 2)],
),
(
"code",
{"stem": True},
(True, False),
[("run", 0), ("in", 1), ("cafe", 2)],
),
(
"code",
{"remove_stop_words": True},
(False, True),
[("running", 0), ("cafe", 2)],
),
(
"code",
{"stem": True, "remove_stop_words": True},
(True, True),
[("run", 0), ("cafe", 2)],
),
(
"simple",
{},
(True, True),
[("run", 0), ("cafe", 2)],
),
(
"simple",
{"stem": False, "remove_stop_words": False},
(False, False),
[("running", 0), ("in", 1), ("cafe", 2)],
),
],
)
def test_fts_tokenizer_filter_defaults(
mem_db, legacy, base_tokenizer, options, expected_filters, expected_tokens
):
text = "Running in café"
table = mem_db.create_table("test_filter_defaults", data=[{"text": text}])
if legacy:
with pytest.warns(DeprecationWarning, match="create_fts_index"):
table.create_fts_index("text", base_tokenizer=base_tokenizer, **options)
else:
table.create_index("text", config=FTS(base_tokenizer=base_tokenizer, **options))
details = table.list_indices()[0].index_details
assert (details["stem"], details["remove_stop_words"]) == expected_filters
assert [
(token.text, token.position) for token in table.tokenize(text, column="text")
] == expected_tokens
assert [
(token.text, token.position)
for token in ldb.tokenize(text, base_tokenizer=base_tokenizer, **options)
] == expected_tokens
if base_tokenizer == "code" and not expected_filters[1]:
assert table.search("in", query_type="fts").to_list()[0]["text"] == text
def test_tokenize_uses_icu_index_tokenizer_by_name(mem_db: DBConnection):
data = pa.table({"text": ["Hello, こんにちは世界!"]})
table = mem_db.create_table("test_tokenize_icu", data=data)
@@ -1137,6 +1279,34 @@ def test_fts_lindera_tokenizer(
assert [row["text"] for row in results] == ["成田国際空港"]
@pytest.mark.parametrize("fuzziness", [None, 0, 1, 2])
def test_match_query_fuzziness_to_json(fuzziness):
query = MatchQuery("puppy", "text", fuzziness=fuzziness)
assert query.fuzziness == fuzziness
assert json.loads(query.to_json())["match"]["fuzziness"] == fuzziness
@pytest.mark.parametrize(
"options, expected",
[
pytest.param({}, ["ab"], id="default-exact"),
pytest.param({"fuzziness": 0}, ["ab"], id="explicit-exact"),
pytest.param({"fuzziness": None}, ["ab"], id="automatic"),
pytest.param({"fuzziness": 1}, ["ab", "ac"], id="one-edit"),
pytest.param({"fuzziness": 2}, ["ab", "ac", "zz"], id="two-edits"),
],
)
def test_match_query_fuzziness_search(tmp_db, options, expected):
# Automatic fuzziness is zero for two-character terms.
table = tmp_db.create_table("fuzzy", pa.table({"text": ["ab", "ac", "zz"]}))
table.create_index("text", config=FTS(stem=False, remove_stop_words=False))
results = table.search(MatchQuery("ab", "text", **options)).to_list()
assert sorted(row["text"] for row in results) == sorted(expected)
def test_fts_query_to_json():
"""Test that FTS query to_json() produces valid JSON strings with exact format."""
@@ -1267,7 +1437,10 @@ def test_fts_query_to_json():
json_str = multi_match.to_json()
expected = (
'{"multi_match":{"query":"python","columns":["tags","title"],'
'"boost":[1.0,1.0]}}'
'"boost":[1.0,1.0],"match_queries":[{"column":"tags","terms":"python",'
'"boost":1.0,"fuzziness":0,"max_expansions":50,"operator":"Or",'
'"prefix_length":0},{"column":"title","terms":"python","boost":1.0,'
'"fuzziness":0,"max_expansions":50,"operator":"Or","prefix_length":0}]}}'
)
assert json_str == expected
python/python/tests/test_hybrid_query.py
+14
View File
@@ -355,6 +355,20 @@ def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table):
)
def test_hybrid_query_nprobes_remains_independent(sync_table: Table):
query = sync_table.search(query_type="hybrid").nprobes(20)
assert query._nprobes == 20
assert query._minimum_nprobes is None
assert query._maximum_nprobes is None
query = query.minimum_nprobes(5).nprobes(20)
assert query._nprobes == 20
assert query._minimum_nprobes == 5
assert query._maximum_nprobes is None
def test_hybrid_query_distance_range(sync_table: Table):
reranker = RRFReranker(return_score="all")
result = (
python/python/tests/test_listing.py
+194
View File
@@ -0,0 +1,194 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import asyncio
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
from pathlib import Path
from threading import Thread
from urllib.parse import parse_qs, urlsplit
import lancedb
from lancedb.remote.errors import HttpError
import pytest
@pytest.fixture
def listing_server():
requests, responses = [], []
class Handler(BaseHTTPRequestHandler):
def handle_request(self):
body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
requests.append((self.command, self.path, json.loads(body) if body else {}))
status, response = responses.pop(0)
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps(response).encode())
do_GET = handle_request
do_POST = handle_request
with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server:
thread = Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
yield f"http://127.0.0.1:{server.server_port}", requests, responses
finally:
server.shutdown()
thread.join()
def response_page(kind, names, token):
if kind == "secrets":
items = [{"name": name} for name in names]
elif kind == "jobs":
items = [
{
"job_id": name,
"table": "t",
"job_type": "create_index",
"state": "in_progress",
"created_at_millis": 1,
}
for name in names
]
elif kind == "functions":
fixture = (
Path(__file__).resolve().parents[3]
/ "rust/lancedb/tests/fixtures/first_class_functions/v1"
/ "remote_function_version.canonical.json"
)
definition = json.loads(fixture.read_text())
items = [
{"name": "embed", "version": "1", "definition": definition} for _ in names
]
else:
items = names
key = "views" if kind == "materialized_views" else kind
return {key: items, "page_token": token}
async def connection(endpoint, asynchronous):
kwargs = dict(
api_key="fake",
host_override=endpoint,
client_config={"retry_config": {"retries": 0}},
)
if asynchronous:
return await lancedb.connect_async("db://dev", **kwargs)
return lancedb.connect("db://dev", **kwargs)
async def advance(items, asynchronous):
return await items.__anext__() if asynchronous else next(items)
KINDS = ["secrets", "views", "jobs", "functions", "materialized_views"]
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", KINDS)
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_listing_lazy_pages_and_resume(listing_server, kind, asynchronous):
endpoint, requests, responses = listing_server
responses.extend(
(200, response_page(kind, names, token))
for names, token in [(["a", "b"], "empty"), ([], "last"), (["c", "d"], "")]
)
db = await connection(endpoint, asynchronous)
method = getattr(db, "list_" + kind)
kwargs = (
{} if kind in ("jobs", "materialized_views") else {"namespace_path": ["team"]}
)
items = method(page_token="start/token", page_limit=2, **kwargs)
assert requests == []
assert items.page_token() == "start/token"
assert items.num_page_results() == 0
first = await advance(items, asynchronous)
if kind == "functions":
assert first.name == "embed"
assert first.version == "1"
elif kind == "jobs":
assert first.job_id == "a"
assert first.state == "running"
else:
assert first == "a"
assert items.num_page_results() == 1
assert items.page_token() == "empty"
await advance(items, asynchronous)
assert items.num_page_results() == 0
assert len(requests) == 1
resumed = method(page_token=items.page_token(), page_limit=2, **kwargs)
await advance(resumed, asynchronous)
assert len(requests) == 3
assert resumed.page_token() is None
assert resumed.num_page_results() == 1
remaining = [item async for item in resumed] if asynchronous else list(resumed)
assert len(remaining) == 1
assert resumed.num_page_results() == 0
assert ([item async for item in resumed] if asynchronous else list(resumed)) == []
for (verb, url, body), token in zip(requests, ["start/token", "empty", "last"]):
path = urlsplit(url)
if kind == "jobs":
assert verb == "POST"
assert path.path == "/v1/jobs/list"
assert body == {"page_token": token, "limit": 2}
else:
namespace = "$" if kind == "materialized_views" else "team"
resource = kind.removesuffix("s")
assert path.path == f"/v1/namespace/{namespace}/{resource}/list"
expected = {"page_token": [token], "limit": ["2"]}
if kind == "functions":
expected["include_definition"] = ["true"]
assert parse_qs(path.query) == expected
assert responses == []
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", KINDS)
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_listing_error_retains_token(listing_server, kind, asynchronous):
endpoint, requests, responses = listing_server
responses.extend([(401, {}), (200, response_page(kind, ["a"], None))])
db = await connection(endpoint, asynchronous)
method = getattr(db, "list_" + kind)
items = method(page_token="retry")
with pytest.raises(HttpError):
await advance(items, asynchronous)
assert items.page_token() == "retry"
assert items.num_page_results() == 0
assert ([item async for item in items] if asynchronous else list(items)) == []
assert len(requests) == 1
resumed = method(page_token=items.page_token())
assert len([item async for item in resumed] if asynchronous else list(resumed)) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", KINDS)
async def test_listing_serializes_concurrent_advances(listing_server, kind):
endpoint, requests, responses = listing_server
responses.append((200, response_page(kind, ["a", "b"], None)))
db = await connection(endpoint, True)
items = getattr(db, "list_" + kind)()
results = await asyncio.gather(
items.__anext__(), items.__anext__(), items.__anext__(), return_exceptions=True
)
assert (
len([result for result in results if isinstance(result, StopAsyncIteration)])
== 1
)
assert len(requests) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", KINDS)
@pytest.mark.parametrize("page_limit", [0, 2**32 - 1])
async def test_listing_invalid_limit_does_not_fetch(listing_server, kind, page_limit):
endpoint, requests, _ = listing_server
db = await connection(endpoint, True)
items = getattr(db, "list_" + kind)(page_limit=page_limit)
with pytest.raises(ValueError, match="limit"):
await items.__anext__()
assert requests == []
python/python/tests/test_lsm_write_spec.py
+7 -7
View File
@@ -52,9 +52,9 @@ def test_set_lsm_write_spec_validates(tmp_path):
with pytest.raises(Exception, match="num_buckets"):
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1025))
# Happy path then mutation rejected.
# Happy path, then a set over the installed spec is refused.
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4))
with pytest.raises(Exception, match="mutation"):
with pytest.raises(Exception, match="already set"):
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8))
@@ -176,18 +176,18 @@ def test_get_lsm_write_spec(tmp_path):
table.unset_lsm_write_spec()
assert table.get_lsm_write_spec() is None
# Identity round-trips (column recovered from the schema). Leaving the
# maintained set to be inferred picks up the index on the table, so the
# spec reads back naming it rather than as "infer".
# Identity round-trips (column recovered from the schema). An unnamed
# maintained set reads back as None: what is stored is the intent to
# maintain every index, not the set it resolves to now.
table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
spec = table.get_lsm_write_spec()
assert spec.spec_type == "identity"
assert spec.column == "id"
assert spec.maintained_indexes == [idx_name]
assert spec.maintained_indexes is None
table.unset_lsm_write_spec()
# Unsharded round-trips (no routing column). Opting out is distinct from
# the inferred default.
# the unnamed default.
table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
spec = table.get_lsm_write_spec()
assert spec.spec_type == "unsharded"
python/python/tests/test_materialized_views.py
+66 -199
View File
@@ -93,28 +93,10 @@ def test_remote_list_uses_namespace_route():
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert db.list_materialized_views() == ["daily_sales"]
assert list(db.list_materialized_views()) == ["daily_sales"]
assert requests == ["/v1/namespace/$/materialized_view/list"]
def test_remote_create_async_returns_server_job():
with mock_remote_materialized_view_create() as (host, requests):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
job = db.create_materialized_view_async("adults", "people", where="age >= 18")
assert job.id == "mv-create-123"
assert requests == [
(
"/v1/materialized_view/adults/create",
{"query": 'SELECT * FROM "people" WHERE age >= 18', "with_no_data": False},
)
]
def test_remote_drop_async_returns_server_job():
with mock_remote_materialized_view_create() as (host, requests):
db = lancedb.connect(
@@ -208,7 +190,7 @@ def test_create_and_refresh_jobs(tmp_path):
drop_job = db.drop_materialized_view_async("adults")
assert drop_job.id is None
assert drop_job.wait() is None
assert "adults" not in db.list_materialized_views()
assert "adults" not in list(db.list_materialized_views())
def test_definition_round_trips(tmp_path):
@@ -221,29 +203,29 @@ def test_definition_round_trips(tmp_path):
)
def test_incremental_refresh_after_append(tmp_path):
def test_full_refresh_after_append(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view("copy", "people", with_no_data=True)
view.refresh()
db.open_table("people").add([{"name": "alan", "age": 41}])
result = view.refresh()
assert result.mode == "incremental"
assert result.rows_written == 1
assert result.mode == "rebuild"
assert result.rows_written == 4
assert view.table.count_rows() == 4
assert view.refresh().mode == "no_op"
assert view.refresh().mode == "rebuild"
def test_incremental_refresh_after_update(tmp_path):
def test_full_refresh_after_update(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view("copy", "people", with_no_data=True)
view.refresh()
db.open_table("people").update(where="name = 'kid'", values={"age": 8})
result = view.refresh()
assert result.mode == "incremental"
assert result.rows_written == 1
assert result.mode == "rebuild"
assert result.rows_written == 3
rows = view.table.search().to_list()
assert sorted(row["age"] for row in rows) == [8, 36, 85]
@@ -264,18 +246,43 @@ def test_legacy_storage_source_update_rebuilds(tmp_path):
assert sorted(row["age"] for row in rows) == [8, 36]
def test_direct_writes_to_a_view_are_refused(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view("copy", "people")
table = view.table
row = [{"name": "eve", "age": 1}]
refused = pytest.raises(NotImplementedError, match="is a materialized view")
with refused:
table.add(row)
with refused:
table.update(where="name = 'kid'", values={"age": 8})
with refused:
table.delete("age < 18")
with refused:
table.merge_insert("name").when_not_matched_insert_all().execute(row)
with refused:
table.add_columns({"older": "age + 1"})
with refused:
table.drop_columns(["age"])
assert table.count_rows() == 3
db.open_table("people").add(row)
assert view.refresh().rows_written == 4
def test_list_and_not_a_view(tmp_path):
db = make_db(tmp_path)
db.create_materialized_view("adults", "people", where="age >= 18")
assert db.list_materialized_views() == ["adults"]
assert list(db.list_materialized_views()) == ["adults"]
with pytest.raises(ValueError, match="not a materialized view"):
db.open_materialized_view("people")
with pytest.raises(ValueError, match="not a materialized view"):
db.drop_materialized_view("people")
db.drop_materialized_view("adults")
assert db.list_materialized_views() == []
assert list(db.list_materialized_views()) == []
def test_invalid_expression_fails_at_create(tmp_path):
@@ -303,7 +310,7 @@ async def test_async_create_refresh_and_open(tmp_path):
reopened = await db.open_materialized_view("shouts")
definition = await reopened.definition()
assert definition.query == "SELECT upper(name) AS shout FROM people"
assert await db.list_materialized_views() == ["shouts"]
assert [item async for item in db.list_materialized_views()] == ["shouts"]
@pytest.mark.asyncio
@@ -327,12 +334,12 @@ async def test_async_create_and_refresh_jobs(tmp_path):
drop_job = await db.drop_materialized_view_async("adults")
assert drop_job.id is None
assert await drop_job.wait() is None
assert "adults" not in await db.list_materialized_views()
assert "adults" not in [item async for item in db.list_materialized_views()]
@pytest.mark.asyncio
async def test_async_incremental(tmp_path):
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
async def test_async_refresh_rebuilds(tmp_path):
db = await lancedb.connect_async(tmp_path)
await db.create_table("people", [{"name": "ada", "age": 36}])
view = await db.create_materialized_view("copy", "people", with_no_data=True)
await view.refresh()
@@ -340,15 +347,15 @@ async def test_async_incremental(tmp_path):
table = await db.open_table("people")
await table.add([{"name": "alan", "age": 41}])
result = await view.refresh()
assert result.mode == "incremental"
assert result.rows_written == 1
assert result.mode == "rebuild"
assert result.rows_written == 2
def test_source_requires_stable_row_ids(tmp_path):
def test_source_does_not_require_stable_row_ids(tmp_path):
db = lancedb.connect(tmp_path)
db.create_table("plain", [{"x": 1}])
with pytest.raises(Exception, match="stable row ids"):
db.create_materialized_view("v", "plain")
view = db.create_materialized_view("v", "plain")
assert view.table.count_rows() == 1
def test_bare_select_names_are_quoted(tmp_path):
@@ -373,7 +380,7 @@ def test_scalar_select_is_one_column(tmp_path):
view = db.create_materialized_view("just_name", "people", select="name")
view.refresh()
rows = view.table.search().to_list()
assert set(rows[0]) - {"__source_row_id"} == {"name"}
assert set(rows[0]) == {"name"}
assert sorted(row["name"] for row in rows) == ["ada", "grace", "kid"]
@@ -384,7 +391,7 @@ async def test_async_scalar_select_is_one_column(tmp_path):
view = await db.create_materialized_view("just_name", "people", select="name")
await view.refresh()
rows = await view.table.query().to_list()
assert set(rows[0]) - {"__source_row_id"} == {"name"}
assert set(rows[0]) == {"name"}
def test_limit_above_i64_max_is_refused(tmp_path):
@@ -417,7 +424,7 @@ def test_namespace_connection_materialized_views(tmp_path):
view = db.create_materialized_view("adults", "people", where="age >= 18")
view.refresh()
assert view.table.count_rows() == 1
assert db.list_materialized_views() == ["adults"]
assert list(db.list_materialized_views()) == ["adults"]
reopened = db.open_materialized_view("adults")
assert reopened.definition.query.startswith("SELECT name, age FROM ")
@@ -433,7 +440,7 @@ def test_namespace_connection_materialized_views(tmp_path):
assert db.drop_materialized_view_async("job_view").wait() is None
db.drop_materialized_view("adults")
assert db.list_materialized_views() == []
assert list(db.list_materialized_views()) == []
@pytest.mark.asyncio
@@ -452,7 +459,7 @@ async def test_async_namespace_connection_materialized_views(tmp_path):
view = await db.create_materialized_view("adults", "people", where="age >= 18")
await view.refresh()
assert await view.table.count_rows() == 1
assert await db.list_materialized_views() == ["adults"]
assert [item async for item in db.list_materialized_views()] == ["adults"]
reopened = await db.open_materialized_view("adults")
assert (await reopened.definition()).query.startswith("SELECT name, age FROM ")
@@ -478,7 +485,7 @@ async def test_async_namespace_connection_materialized_views(tmp_path):
drop_job = await db.drop_materialized_view_async("job_view")
assert await drop_job.wait() is None
await db.drop_materialized_view("adults")
assert await db.list_materialized_views() == []
assert [item async for item in db.list_materialized_views()] == []
def test_stored_queries_and_legacy_layouts_are_read():
@@ -520,166 +527,26 @@ def test_stored_queries_and_legacy_layouts_are_read():
"SELECT * FROM people"
)
for fmt, function in [(3, "vector_duplicate_pairs"), (4, "vector_dedup")]:
native = f"SELECT * FROM {function}('images', 3, 'phash', 4)"
assert read({"format": fmt, "query": native}).query == native
# A newer writer's layout is reported, never guessed at.
for newer in (
{"format": 5, "query": query},
{"format": 3, "query": query},
{"kind": "select_v3", "source_table": "people"},
):
with pytest.raises(NotImplementedError, match="cannot refresh"):
read(newer)
def indexed_dedup_source(db, name="images"):
import pyarrow as pa
from lancedb.index import IvfFlat
vectors = pa.FixedSizeListArray.from_arrays(
pa.array(
[v for i in range(256) for v in (float(i if i < 3 else i * 10), 0.0)],
type=pa.float32(),
),
2,
)
table = db.create_table(name, pa.table({"id": range(256), "vector": vectors}))
table.create_index("vector", config=IvfFlat(num_partitions=1))
return table
@pytest.mark.parametrize("stable", [False, True])
def test_vector_dedup_source_captures_snapshot_and_preserves_source(tmp_path, stable):
db = lancedb.connect(
tmp_path,
storage_options={"new_table_enable_stable_row_ids": str(stable).lower()},
)
source = indexed_dedup_source(db, "images.with.dot")
version = source.version
expression = lancedb.vector_dedup(
source.name, column="vector", distance_threshold=1
)
view = db.create_materialized_view("clean", expression)
assert "vector_dedup(" in view.definition.query
assert f", {version}, 'vector', 1.0)" in view.definition.query
assert view.table.count_rows() == 255
assert [r["id"] for r in view.table.search().where("id < 3").to_list()] == [0, 2]
assert source.version == version
assert source.count_rows() == 256
source.delete("id = 0")
view.refresh_async().wait()
assert view.table.search().where("id = 0").to_list() == [
{"id": 0, "vector": [0.0, 0.0]}
]
assert expression.dataset_version is None # reusable immutable declaration
def test_vector_dedup_create_job_and_explicit_snapshot(tmp_path):
db = lancedb.connect(tmp_path)
source = indexed_dedup_source(db)
version = source.version
source.delete("id = 0")
job = db.create_materialized_view_async(
"clean",
lancedb.vector_dedup(
"images", column="vector", distance_threshold=1, dataset_version=version
),
)
assert job.wait() is None
view = db.open_materialized_view("clean")
assert view.table.count_rows() == 255
assert len(view.table.search().where("id = 0").to_list()) == 1
@pytest.mark.asyncio
async def test_async_vector_dedup_declaration_and_creation(tmp_path):
sync_db = lancedb.connect(tmp_path)
source = indexed_dedup_source(sync_db)
db = await lancedb.connect_async(tmp_path)
expression = lancedb.vector_dedup("images", column="vector", distance_threshold=1)
view = await db.create_materialized_view("clean", expression, with_no_data=True)
assert await view.table.count_rows() == 0
# The snapshot is captured at declaration even if refresh comes later.
source.delete("id = 0")
job = await view.refresh_async()
await job.wait()
assert await view.table.count_rows() == 255
another = await db.create_materialized_view_async(
"later", expression, with_no_data=True
)
await another.wait()
later = await db.open_materialized_view("later")
assert (await later.definition()).query != (await view.definition()).query
assert await later.table.count_rows() == 0
@pytest.mark.parametrize(
"options", [{"select": ["id"]}, {"select": []}, {"where": "id > 0"}, {"limit": 1}]
)
def test_vector_dedup_rejects_extra_clauses_before_opening_source(tmp_path, options):
db = lancedb.connect(tmp_path)
expression = lancedb.vector_dedup("missing", column="vector", distance_threshold=1)
with pytest.raises(ValueError, match="cannot be combined"):
db.create_materialized_view("bad", expression, **options)
assert db.list_tables().tables == []
@pytest.mark.parametrize("version", [0, -1, True, 1.5, "2", 2**64])
def test_vector_dedup_rejects_invalid_snapshot(version):
with pytest.raises(ValueError, match="dataset_version"):
lancedb.vector_dedup(
"images", column="vector", distance_threshold=1, dataset_version=version
)
@pytest.mark.parametrize(
"threshold", [float("nan"), float("inf"), -float("inf"), 1e100, 10**1000, True, "4"]
)
def test_vector_dedup_rejects_invalid_threshold(threshold):
with pytest.raises((ValueError, TypeError), match="distance_threshold"):
lancedb.vector_dedup("images", column="vector", distance_threshold=threshold)
def test_vector_dedup_descriptor_is_frozen():
from dataclasses import FrozenInstanceError
expression = lancedb.vector_dedup("images", column="vector", distance_threshold=1)
with pytest.raises(FrozenInstanceError):
expression.column = "other"
@pytest.mark.parametrize("with_no_data", [False, True])
def test_remote_vector_dedup_uses_mv_endpoint_without_sql_client(with_no_data):
with mock_remote_materialized_view_create() as (host, requests):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
sql_host_override="grpc://127.0.0.1:1",
client_config={"retry_config": {"retries": 0}},
)
job = db.create_materialized_view_async(
"clean",
lancedb.vector_dedup(
"images.with'dot",
column="vector'field",
distance_threshold=4,
dataset_version=7,
),
with_no_data=with_no_data,
)
assert job.id == "mv-create-123"
assert requests == [
(
"/v1/materialized_view/clean/create",
{
"query": (
"SELECT * FROM vector_dedup('`images.with''dot`', "
"7, 'vector''field', 4.0)"
),
"with_no_data": with_no_data,
},
)
]
def test_local_materialized_view_listing_pages_and_resume(tmp_path):
db = make_db(tmp_path)
db.create_materialized_view("a_view", "people", with_no_data=True)
db.create_materialized_view("z_view", "people", with_no_data=True)
db.create_table("ordinary", [{"id": 1}])
names = db.list_materialized_views(page_limit=1)
assert names.num_page_results() == 0
first = next(names)
assert names.num_page_results() == 0
assert names.page_token() is not None
resumed = db.list_materialized_views(page_token=names.page_token(), page_limit=1)
assert sorted([first, *resumed]) == ["a_view", "z_view"]
assert resumed.page_token() is None
assert list(resumed) == []
python/python/tests/test_namespace.py
+192 -1
View File
@@ -6,11 +6,17 @@
import tempfile
import shutil
import importlib
import http.server
import json
import threading
from urllib.parse import parse_qs, unquote, urlparse
import pytest
import pyarrow as pa
import lancedb
from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
from lancedb.namespace import _MAX_QUERY_K
from lancedb.listing import AsyncListing, Listing
from lancedb.namespace import _MAX_QUERY_K, _query_to_namespace_request
from lancedb.query import Query
from lancedb.table import AsyncTable, LanceTable
@@ -72,6 +78,132 @@ def _namespace_lance_table(namespace_client: _NamespaceClient) -> LanceTable:
return table
@pytest.mark.asyncio
@pytest.mark.parametrize("namespace_path", [[], ["test_ns"]])
async def test_namespace_table_names_defaults_and_limits(tmp_path, namespace_path):
db = lancedb.connect_namespace("dir", {"root": str(tmp_path)})
if namespace_path:
db.create_namespace(namespace_path)
names = [f"t{idx:03}" for idx in range(12)]
schema = pa.schema([pa.field("id", pa.int64())])
for name in names:
db.create_table(name, schema=schema, namespace_path=namespace_path)
assert list(db.table_names(namespace_path=namespace_path)) == names
assert list(db.table_names(limit=None, namespace_path=namespace_path)) == names
assert list(db.table_names(limit=0, namespace_path=namespace_path)) == []
assert list(db.table_names(limit=2, namespace_path=namespace_path)) == names[:2]
assert (
list(
db.table_names(page_token=names[1], limit=2, namespace_path=namespace_path)
)
== names[2:4]
)
adb = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
assert list(await adb.table_names(namespace_path=namespace_path)) == names
assert (
list(await adb.table_names(limit=None, namespace_path=namespace_path)) == names
)
assert list(await adb.table_names(limit=0, namespace_path=namespace_path)) == []
assert (
list(await adb.table_names(limit=2, namespace_path=namespace_path)) == names[:2]
)
assert (
list(
await adb.table_names(
page_token=names[1], limit=2, namespace_path=namespace_path
)
)
== names[2:4]
)
@pytest.fixture
def paginated_rest_namespace():
names = [f"t{idx:03}" for idx in reversed(range(12))]
requests = []
class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
url = urlparse(self.path)
query = parse_qs(url.query)
requests.append((unquote(url.path), query))
token = query.get("page_token", [None])[0]
if token not in {None, "opaque-next"}:
self.send_error(400, "Unknown page token")
return
start = 0 if token is None else 10
body = {"tables": names[start : start + 10]}
if start == 0:
body["page_token"] = "opaque-next"
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps(body).encode())
def log_message(self, *args):
pass
with http.server.HTTPServer(("127.0.0.1", 0), Handler) as server:
thread = threading.Thread(target=server.serve_forever)
thread.start()
try:
yield {"uri": f"http://127.0.0.1:{server.server_port}"}, requests
finally:
server.shutdown()
thread.join()
@pytest.mark.asyncio
@pytest.mark.parametrize("namespace_path", [[], ["n"]])
async def test_rest_namespace_table_names_collects_pages(
paginated_rest_namespace, namespace_path
):
properties, requests = paginated_rest_namespace
names = [f"t{idx:03}" for idx in range(12)]
db = lancedb.connect_namespace("rest", properties)
first = db.list_tables(namespace_path=namespace_path)
second = db.list_tables(namespace_path=namespace_path, page_token=first.page_token)
assert len(first.tables) == 10
assert first.page_token == "opaque-next"
assert sorted(first.tables + second.tables) == names
assert second.page_token is None
assert list(db.table_names(namespace_path=namespace_path)) == names
assert list(db.table_names(limit=None, namespace_path=namespace_path)) == names
assert list(db.table_names(limit=11, namespace_path=namespace_path)) == names[:11]
assert (
list(db.table_names(page_token="t005", limit=2, namespace_path=namespace_path))
== names[6:8]
)
adb = lancedb.connect_namespace_async("rest", properties)
assert list(await adb.table_names(namespace_path=namespace_path)) == names
assert (
list(await adb.table_names(limit=None, namespace_path=namespace_path)) == names
)
assert (
list(await adb.table_names(limit=11, namespace_path=namespace_path))
== names[:11]
)
assert (
list(
await adb.table_names(
page_token="t005", limit=2, namespace_path=namespace_path
)
)
== names[6:8]
)
request_count = len(requests)
assert list(db.table_names(limit=0, namespace_path=namespace_path)) == []
assert list(await adb.table_names(limit=0, namespace_path=namespace_path)) == []
assert len(requests) == request_count
identifier = "$" if not namespace_path else "n"
assert all(path == f"/v1/namespace/{identifier}/table/list" for path, _ in requests)
class TestNamespaceConnection:
"""Test namespace-based LanceDB connection using DirectoryNamespace."""
@@ -104,6 +236,18 @@ class TestNamespaceConnection:
assert isinstance(db, lancedb.LanceNamespaceDBConnection)
assert len(list(db.table_names())) == 0
def test_list_materialized_views(self):
db = lancedb.connect_namespace("dir", {"root": self.temp_dir})
db.create_table("source", [{"id": 1}])
db.create_materialized_view("view", "source", with_no_data=True)
names = db.list_materialized_views(page_limit=1)
assert isinstance(names, Listing)
assert names.num_page_results() == 0
assert list(names) == ["view"]
assert names.num_page_results() == 0
assert names.page_token() is None
def test_sync_builtin_namespace_uses_rust_without_python_client(self, monkeypatch):
"""Built-in sync namespace connections should not construct or call the
Python namespace client for normal namespace/table management."""
@@ -605,6 +749,18 @@ class TestAsyncNamespaceConnection:
table_names = await db.table_names()
assert len(list(table_names)) == 0
async def test_list_materialized_views(self):
db = lancedb.connect_namespace_async("dir", {"root": self.temp_dir})
await db.create_table("source", [{"id": 1}])
await db.create_materialized_view("view", "source", with_no_data=True)
names = db.list_materialized_views(page_limit=1)
assert isinstance(names, AsyncListing)
assert names.num_page_results() == 0
assert [name async for name in names] == ["view"]
assert names.num_page_results() == 0
assert names.page_token() is None
async def test_async_builtin_namespace_uses_rust_without_python_client(
self, monkeypatch
):
@@ -956,6 +1112,29 @@ class TestPushdownOperations:
]
assert all(r.k <= 2**31 - 1 for r in namespace_client.requests)
def test_probe_fields_pass_through_independently(self):
request = _query_to_namespace_request(
["geneva", "hist"],
Query(
vector=[1.0, 2.0],
nprobes=20,
minimum_nprobes=3,
maximum_nprobes=10,
),
)
assert request.nprobes == 20
assert request.minimum_nprobes == 3
assert request.maximum_nprobes == 10
request = _query_to_namespace_request(
["geneva", "hist"],
Query(vector=[1.0, 2.0], nprobes=5, maximum_nprobes=0),
)
assert request.nprobes == 5
assert request.maximum_nprobes == 0
assert request.to_dict()["maximum_nprobes"] == 0
@pytest.mark.asyncio
class TestAsyncPushdownOperations:
@@ -1030,3 +1209,15 @@ def test_local_table_to_arrow_and_to_pandas_are_unchanged(tmp_path):
assert table.to_arrow().column("id").to_pylist() == [1, 2]
assert table.to_pandas()["id"].tolist() == [1, 2]
@pytest.mark.parametrize("effort", [0.0, 0.5, 1.0])
def test_namespace_request_rejects_search_effort(effort):
from lancedb.query import Query
with pytest.raises(
ValueError, match="namespace QueryTable does not support search_effort"
):
_query_to_namespace_request(
["test"], Query(vector=[1.0, 0.0], search_effort=effort)
)
python/python/tests/test_pydantic.py
+20
View File
@@ -9,6 +9,7 @@ from typing import List, Optional, Tuple
import pyarrow as pa
import pydantic
import pytest
from lancedb.conftest import MockTextEmbeddingFunction
from lancedb.pydantic import (
LanceModel,
MultiVector,
@@ -425,6 +426,25 @@ def test_bare_vector_raises_clear_error():
exec("class TestModel(LanceModel):\n vector: Vector", namespace)
def test_embedding_vector_list_annotation():
embedding = MockTextEmbeddingFunction.create()
class StaticTypingModel(LanceModel):
text: str = embedding.SourceField()
vector: list[float] = embedding.VectorField()
schema = pydantic_to_schema(StaticTypingModel)
assert schema == pa.schema(
[
pa.field("text", pa.utf8(), False),
pa.field("vector", pa.list_(pa.float32(), embedding.ndims()), True),
]
)
model = StaticTypingModel(text="hello", vector=[0.0] * embedding.ndims())
assert model.vector == [0.0] * embedding.ndims()
def test_fixed_size_list_field():
class TestModel(pydantic.BaseModel):
vec: Vector(16)
python/python/tests/test_query.py
+265 -28
View File
@@ -24,6 +24,7 @@ from lancedb.pydantic import LanceModel, Vector
from lancedb.query import (
AsyncFTSQuery,
AsyncHybridQuery,
AsyncQuery,
AsyncQueryBase,
AsyncVectorQuery,
ColumnOrdering,
@@ -974,6 +975,19 @@ def test_query_builder_with_filter(table):
assert all(np.array(rs[0]["vector"]) == [3, 4])
@pytest.mark.parametrize("query_type", ["vector", "hybrid"])
@pytest.mark.parametrize("nprobes", [0, -1])
def test_nprobes_nonpositive_sync(table, query_type, nprobes):
if query_type == "hybrid":
query = table.search(query_type="hybrid").vector([0, 0]).text("a")
else:
query = table.search([0, 0])
with pytest.raises(
ValueError, match="^Invalid input, nprobes must be greater than 0$"
):
query.nprobes(nprobes)
def test_invalid_nprobes_sync(table):
with pytest.raises(ValueError, match="minimum_nprobes must be greater than 0"):
LanceVectorQueryBuilder(table, [0, 0], "vector").minimum_nprobes(0).to_list()
@@ -981,12 +995,22 @@ def test_invalid_nprobes_sync(table):
ValueError,
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
LanceVectorQueryBuilder(table, [0, 0], "vector").maximum_nprobes(5).to_list()
(
LanceVectorQueryBuilder(table, [0, 0], "vector")
.minimum_nprobes(10)
.maximum_nprobes(5)
.to_list()
)
with pytest.raises(
ValueError,
match="minimum_nprobes must be less than or equal to maximum_nprobes",
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
LanceVectorQueryBuilder(table, [0, 0], "vector").minimum_nprobes(100).to_list()
(
LanceVectorQueryBuilder(table, [0, 0], "vector")
.maximum_nprobes(5)
.minimum_nprobes(100)
.to_list()
)
def test_nprobes_works_sync(table):
@@ -1005,6 +1029,18 @@ def test_multiple_nprobes_calls_works_sync(table):
).minimum_nprobes(20).to_list()
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
async def test_nprobes_zero_async(table_async: AsyncTable, hybrid):
query = table_async.query().nearest_to([0, 0])
if hybrid:
query = query.nearest_to_text("dog")
with pytest.raises(
ValueError, match="^Invalid input, nprobes must be greater than 0$"
):
query.nprobes(0)
@pytest.mark.asyncio
async def test_invalid_nprobes_async(table_async: AsyncTable):
with pytest.raises(ValueError, match="minimum_nprobes must be greater than 0"):
@@ -1013,12 +1049,22 @@ async def test_invalid_nprobes_async(table_async: AsyncTable):
ValueError,
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
await table_async.vector_search([0, 0]).maximum_nprobes(5).to_list()
await (
table_async.vector_search([0, 0])
.minimum_nprobes(10)
.maximum_nprobes(5)
.to_list()
)
with pytest.raises(
ValueError,
match="minimum_nprobes must be less than or equal to maximum_nprobes",
):
await table_async.vector_search([0, 0]).minimum_nprobes(100).to_list()
await (
table_async.vector_search([0, 0])
.maximum_nprobes(5)
.minimum_nprobes(100)
.to_list()
)
def test_query_builder_with_prefilter(table):
@@ -1279,6 +1325,27 @@ async def test_query_to_polars_async(table_async: AsyncTable):
assert df.shape == (0, num_columns)
@pytest.mark.asyncio
@pytest.mark.parametrize("search_kwargs", [{}, {"query": None}])
@pytest.mark.parametrize("with_vector", [False, True])
async def test_async_search_without_query(mem_db_async, search_kwargs, with_vector):
data = [{"id": i} for i in range(10)]
if with_vector:
for row in data:
row["vector"] = [float(row["id"]), 1.0]
table = await mem_db_async.create_table("test", data)
query = await table.search(**search_kwargs)
assert isinstance(query, AsyncQuery)
assert await query.limit(3).to_arrow() == await table.query().limit(3).to_arrow()
query = await table.search(**search_kwargs)
assert await query.where("id >= 8").select(["id"]).limit(3).to_list() == [
{"id": 8},
{"id": 9},
]
@pytest.mark.asyncio
async def test_none_query(table_async: AsyncTable):
with pytest.raises(ValueError):
@@ -1588,8 +1655,9 @@ def test_query_serialization_sync(table: lancedb.table.Table):
q,
vector_column="vector",
vector=[5.0, 6.0],
minimum_nprobes=10,
maximum_nprobes=10,
nprobes=10,
minimum_nprobes=None,
maximum_nprobes=None,
refine_factor=5,
)
@@ -1607,8 +1675,19 @@ def test_query_serialization_sync(table: lancedb.table.Table):
q,
vector_column="vector",
vector=[5.0, 6.0],
minimum_nprobes=50,
maximum_nprobes=50,
nprobes=50,
minimum_nprobes=None,
maximum_nprobes=None,
)
q = table.search([5.0, 6.0]).minimum_nprobes(5).nprobes(50).to_query_object()
check_set_props(
q,
vector_column="vector",
vector=[5.0, 6.0],
nprobes=50,
minimum_nprobes=5,
maximum_nprobes=None,
)
q = table.search([5.0, 6.0]).maximum_nprobes(10).to_query_object()
@@ -1620,6 +1699,15 @@ def test_query_serialization_sync(table: lancedb.table.Table):
minimum_nprobes=None,
)
q = table.search([5.0, 6.0]).nprobes(5).maximum_nprobes(0).to_query_object()
check_set_props(
q,
vector_column="vector",
vector=[5.0, 6.0],
nprobes=5,
maximum_nprobes=0,
)
q = table.search([5.0, 6.0]).distance_range(0.0, 1.0).to_query_object()
check_set_props(
q, vector_column="vector", vector=[5.0, 6.0], lower_bound=0.0, upper_bound=1.0
@@ -1669,8 +1757,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
limit=10,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
)
@@ -1680,8 +1768,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1692,8 +1780,9 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
postfilter=False,
minimum_nprobes=50,
maximum_nprobes=50,
nprobes=50,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1712,8 +1801,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
filter="id = 1",
postfilter=True,
vector=sample_vector,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
)
@@ -1727,8 +1816,9 @@ async def test_query_serialization_async(table_async: AsyncTable):
check_set_props(
q,
vector=sample_vector,
minimum_nprobes=10,
maximum_nprobes=10,
nprobes=10,
minimum_nprobes=None,
maximum_nprobes=None,
refine_factor=5,
postfilter=False,
with_row_id=False,
@@ -1741,7 +1831,24 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
minimum_nprobes=5,
maximum_nprobes=20,
maximum_nprobes=None,
postfilter=False,
with_row_id=False,
bypass_vector_index=False,
limit=10,
)
q = (
(await table_async.search([5.0, 6.0]))
.nprobes(5)
.maximum_nprobes(0)
.to_query_object()
)
check_set_props(
q,
vector=sample_vector,
nprobes=5,
maximum_nprobes=0,
postfilter=False,
with_row_id=False,
bypass_vector_index=False,
@@ -1759,8 +1866,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
lower_bound=0.0,
upper_bound=1.0,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1772,8 +1879,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
distance_type="cosine",
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1785,8 +1892,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
ef=7,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1798,8 +1905,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
bypass_vector_index=True,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
limit=10,
)
@@ -2091,6 +2198,12 @@ def test_ensure_vector_query_nested_empty_list():
ensure_vector_query([[]])
@pytest.mark.parametrize("query", [[], np.array([], dtype=np.float32)])
def test_vector_query_builder_empty_vector(table, query):
with pytest.raises(ValueError, match="^Query vector must not be empty$"):
LanceVectorQueryBuilder(table, query, "vector")
def test_fast_search(tmp_path):
db = lancedb.connect(tmp_path)
@@ -2152,3 +2265,127 @@ def test_blob_v2_to_batches_row_id(tmp_db):
assert "_rowid" in hits.column_names
blobs = table.fetch_blobs("blob", hits)
assert [blobs[i].as_py() for i in range(len(blobs))] == [b"one", b"two"]
@pytest.mark.parametrize("metric", ["l2", "cosine", "dot"])
@pytest.mark.parametrize(
"effort,probes", [(None, 4), (0, 1), (0.25, 2), (0.5, 4), (0.75, 8), (1, 16)]
)
def test_search_effort_probe_budget(tmp_path, monkeypatch, metric, effort, probes):
import re
from lancedb.index import IvfFlat
monkeypatch.setenv("LANCE_AUTO_PROBE_MARGIN", "0")
monkeypatch.setenv("LANCE_AUTO_MIN_INITIAL_NPROBES", "4")
monkeypatch.setenv("LANCE_AUTO_MAX_INITIAL_NPROBES", "4")
rng = np.random.default_rng(42)
vectors = np.repeat(np.eye(16, dtype=np.float32), 32, axis=0)
vectors += rng.normal(0, 0.001, vectors.shape).astype(np.float32)
data = pa.table(
{
"id": np.arange(512),
"text": ["dog"] * 512,
"vector": pa.FixedSizeListArray.from_arrays(pa.array(vectors.ravel()), 16),
}
)
table = lancedb.connect(tmp_path).create_table("effort", data.slice(0, 256))
table.add(data.slice(256))
table.create_index(
"vector", config=IvfFlat(distance_type=metric, num_partitions=16)
)
vector = np.eye(16, dtype=np.float32)[0]
def search(value=effort):
return table.search(vector).distance_type(metric).search_effort(value).limit(8)
def partitions(query):
vector_plan = query.analyze_plan().split("FTS Search Plan:", 1)[0]
values = re.findall(r"partitions_searched=(\d+)", vector_plan)
assert values
return sum(map(int, values))
assert partitions(search()) == probes
exact = set(
table.search(vector)
.distance_type(metric)
.bypass_vector_index()
.limit(8)
.to_arrow()["id"]
.to_pylist()
)
actual = set(search().to_arrow()["id"].to_pylist())
assert len(actual & exact) / 8 >= 0.5
expected = 1 if effort is None else probes
assert partitions(search().nprobes(1)) == expected
assert (
partitions(
table.search(vector)
.distance_type(metric)
.nprobes(1)
.search_effort(effort)
.limit(8)
)
== expected
)
assert partitions(search().nprobes(1).search_effort(None)) == 1
assert partitions(search().maximum_nprobes(2)) <= 2
assert partitions(search().minimum_nprobes(3).maximum_nprobes(3)) == 3
assert partitions(search(0).nprobes(16).maximum_nprobes(0)) == 1
if effort is None:
assert search().to_arrow() == search(0.5).to_arrow()
filtered = search(0).maximum_nprobes(2).where("id % 8 = 0", prefilter=True)
assert partitions(filtered) <= 2
assert all(row["id"] % 8 == 0 for row in filtered.to_list())
if metric == "l2" and effort == 0.5:
table.create_index("text", config=FTS())
hybrid = (
table.search(query_type="hybrid")
.vector(vector)
.text("dog")
.nprobes(1)
.search_effort(0.5)
.limit(8)
)
assert partitions(hybrid) == 4
assert partitions(hybrid.search_effort(None)) == 1
@pytest.mark.parametrize(
"effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")]
)
def test_search_effort_invalid_sync(table, effort):
with pytest.raises(ValueError, match="search_effort must be finite and in"):
table.search([0, 0]).search_effort(effort).to_list()
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
@pytest.mark.parametrize("effort", [None, 0, 0.25, 0.5, 0.75, 1])
async def test_search_effort_async_roundtrip(table_async, hybrid, effort):
query = table_async.vector_search([0, 0])
if hybrid:
query = query.nearest_to_text("dog")
query = query.nprobes(5).search_effort(effort).minimum_nprobes(2).maximum_nprobes(3)
request = query.to_query_object()
assert request.search_effort == effort
assert request.nprobes == 5
assert request.minimum_nprobes == 2
assert request.maximum_nprobes == 3
assert query.search_effort(None).to_query_object().search_effort is None
query.search_effort(effort)
if not hybrid:
assert len(await query.limit(1).to_list()) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
@pytest.mark.parametrize(
"effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")]
)
async def test_search_effort_invalid_async(table_async, hybrid, effort):
query = table_async.vector_search([0, 0])
if hybrid:
query = query.nearest_to_text("dog")
with pytest.raises(ValueError, match="search_effort must be finite and in"):
query.search_effort(effort)
python/python/tests/test_remote_db.py
+1081 -53
View File
File diff suppressed because it is too large Load Diff
python/python/tests/test_sql.py
+110 -3
View File
@@ -1,8 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import threading
from uuid import UUID
import numpy as np
import pytest
import pyarrow as pa
@@ -11,7 +13,7 @@ from lancedb import _lancedb
from lancedb.arrow import AsyncRecordBatchReader
from lancedb.db import AsyncConnection
from lancedb.remote.db import RemoteDBConnection
from lancedb.sql import AsyncQuery, Query
from lancedb.sql import AsyncQuery, Query, to_parameter_batch
NIL_QUERY_ID = UUID(int=0)
@@ -24,12 +26,20 @@ class FakeNativeQuery:
class FakeNativeConnection:
async def execute_query_async(self, query, *, default_namespace_path=None):
def __init__(self):
self.parameters = []
async def execute_query_async(
self, query, *, default_namespace_path=None, parameters=None
):
self.parameters.append(parameters)
return FakeNativeQuery()
class FakeAsyncConnection:
async def execute_query_async(self, query, *, default_namespace_path=None):
async def execute_query_async(
self, query, *, default_namespace_path=None, parameters=None
):
return AsyncQuery(FakeNativeQuery())
@@ -160,3 +170,100 @@ def test_execute_query_async_rejects_invalid_namespace_components(
remote_connection().execute_query_async(
"SELECT 1", default_namespace_path=default_namespace_path
)
def test_positional_parameters_keep_their_types():
vector = np.array([0.1, 0.2, 0.3], dtype=np.float32)
batch = to_parameter_batch(
[42, 1.5, "a", np.float32(0.25), vector, pa.scalar(7, pa.int16())]
)
assert batch.num_rows == 1
assert batch.schema.names == ["1", "2", "3", "4", "5", "6"]
assert batch.schema.types == [
pa.int64(),
pa.float64(),
pa.string(),
pa.float32(),
pa.list_(pa.float32(), 3),
pa.int16(),
]
sent = batch.column(4).flatten().to_numpy()
assert sent.dtype == np.float32
assert sent.tobytes() == vector.tobytes()
def test_named_parameters_and_batches():
half = pa.array(np.array([1.0, 2.0], dtype=np.float16))
batch = to_parameter_batch({"vector": half, "k": 3})
assert batch.schema.names == ["vector", "k"]
assert batch.schema.field("vector").type == pa.list_(pa.float16(), 2)
row = pa.record_batch({"id": [1]})
assert to_parameter_batch(row) is row
assert to_parameter_batch(pa.table({"id": [1]})).equals(row)
assert to_parameter_batch(None) is None
@pytest.mark.parametrize(
"parameters",
[pa.record_batch({"id": [1, 2]}), pa.table({"id": pa.array([], pa.int64())})],
)
def test_parameters_are_exactly_one_row(parameters):
with pytest.raises(ValueError, match="exactly one row"):
to_parameter_batch(parameters)
def test_parameters_must_be_a_collection():
with pytest.raises(TypeError, match="query parameters"):
to_parameter_batch("SELECT 1")
@pytest.mark.asyncio
async def test_async_execute_query_sends_parameters_as_a_batch():
native = FakeNativeConnection()
connection = AsyncConnection(native)
await connection.execute_query("SELECT $1", parameters=[1])
await connection.execute_query_async("SELECT 1")
sent, unparameterized = native.parameters
assert sent.equals(pa.record_batch({"1": [1]}))
assert unparameterized is None
def test_parameters_reach_the_server_as_the_values_that_were_sent():
"""A round trip through the bindings against a Flight server that answers
each exchange with the parameter row it received."""
flight = pytest.importorskip("pyarrow.flight")
class EchoServer(flight.FlightServerBase):
def __init__(self):
super().__init__("grpc://127.0.0.1:0")
self.commands = []
self.lock = threading.Lock()
def do_exchange(self, context, descriptor, reader, writer):
parameters = reader.read_all()
with self.lock:
self.commands.append(descriptor.command)
writer.begin(parameters.schema)
writer.write_table(parameters)
writer.close()
vector = np.array([0.1, 1e-45, -0.0], dtype=np.float32)
with EchoServer() as server:
db = lancedb.connect(
"db://analytics",
api_key="test-key",
host_override="http://localhost:10024",
sql_host_override=f"grpc://127.0.0.1:{server.port}",
)
echoed = db.execute_query(
"SELECT echo", parameters={"vector": vector, "k": 10}
).read_all()
(command,) = server.commands
assert b"CommandStatementQuery" in command
assert b"SELECT echo" in command
assert echoed.schema.field("vector").type == pa.list_(pa.float32(), 3)
received = echoed.column("vector").combine_chunks().flatten().to_numpy()
assert received.view(np.uint32).tolist() == vector.view(np.uint32).tolist()
assert echoed.column("k").to_pylist() == [10]
python/python/tests/test_table.py
+231 -14
View File
@@ -12,7 +12,7 @@ import warnings
import weakref
from concurrent.futures import ThreadPoolExecutor
from datetime import date, datetime, timedelta
from decimal import Decimal
from decimal import Decimal, localcontext
from time import sleep
from typing import List
from unittest.mock import patch
@@ -113,6 +113,32 @@ def test_basic(mem_db: DBConnection):
assert table.to_arrow() == expected_data
@pytest.mark.parametrize("enable_v2", [False, True])
def test_migrate_v2_manifest_paths(tmp_path, enable_v2):
db = lancedb.connect(
tmp_path,
storage_options={"new_table_enable_v2_manifest_paths": str(enable_v2).lower()},
)
table = db.create_table("calls", [{"id": 1, "vector": [1.0, 1.0]}])
table.add([{"id": 2, "vector": [2.0, 2.0]}])
expected_data = table.to_arrow()
expected_versions = table.list_versions()
assert table.uses_v2_manifest_paths() == enable_v2
# Migration is also safe to repeat on a table already using v2 paths.
for _ in range(2):
table.migrate_v2_manifest_paths()
assert table.uses_v2_manifest_paths()
reopened = db.open_table("calls")
assert reopened.uses_v2_manifest_paths()
assert reopened.to_arrow() == expected_data
assert reopened.list_versions() == expected_versions
manifests = list((tmp_path / "calls.lance" / "_versions").glob("*.manifest"))
assert len(manifests) == len(expected_versions)
assert all(len(path.stem) == 20 and path.stem.isdigit() for path in manifests)
def test_search_preserves_nulls_from_sliced_arrow_table(mem_db: DBConnection):
data = pa.table(
{
@@ -967,6 +993,63 @@ def test_polars(mem_db: DBConnection):
assert len(filtered_result) == 2
@pytest.mark.asyncio
async def test_list_versions_timestamp_precision():
# 2026-10-03T01:03:49.274Z in nanoseconds. Float math turns .274000 into
# .273999, so make sure the conversion is exact.
ts_nanos = 1790989429274000000
class FakeInner:
async def list_versions(self):
return [{"version": 1, "timestamp": ts_nanos, "metadata": {}}]
table = table_module.AsyncTable(FakeInner())
versions = await table.list_versions()
expected = datetime.fromtimestamp(ts_nanos // 1_000_000_000)
assert versions[0]["timestamp"] == expected + timedelta(microseconds=274000)
assert versions[0]["timestamp"].microsecond == 274000
def test_list_versions_timestamp_precision_sync(mem_db: DBConnection):
# The sync LanceTable.list_versions delegates to AsyncTable.list_versions,
# which is the path the parity report hit.
ts_nanos = 1790989429274000000
class FakeInner:
async def list_versions(self):
return [{"version": 1, "timestamp": ts_nanos, "metadata": {}}]
table = mem_db.create_table("ts_precision", data=[{"id": 1}])
table._table = table_module.AsyncTable(FakeInner())
versions = table.list_versions()
expected = datetime.fromtimestamp(ts_nanos // 1_000_000_000)
assert versions[0]["timestamp"] == expected + timedelta(microseconds=274000)
assert versions[0]["timestamp"].microsecond == 274000
def test_list_versions_timestamp_precision_remote():
# RemoteTable.list_versions also delegates to AsyncTable.list_versions.
from lancedb.remote.table import RemoteTable
ts_nanos = 1790989429274000000
class FakeInner:
def name(self):
return "ts_precision"
async def list_versions(self):
return [{"version": 1, "timestamp": ts_nanos, "metadata": {}}]
table = RemoteTable(table_module.AsyncTable(FakeInner()), "dev")
versions = table.list_versions()
expected = datetime.fromtimestamp(ts_nanos // 1_000_000_000)
assert versions[0]["timestamp"] == expected + timedelta(microseconds=274000)
assert versions[0]["timestamp"].microsecond == 274000
def test_versioning(mem_db: DBConnection):
table = mem_db.create_table(
"test",
@@ -2529,6 +2612,51 @@ def test_update_with_arrow_scalar(mem_db: DBConnection):
assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0, 3.0, 4.0]]
@pytest.mark.parametrize("use_arrow_scalar", [False, True])
@pytest.mark.parametrize(
"decimal_type, value",
[
(pa.decimal128(10, 2), "9.99"),
(pa.decimal128(10, 2), "-9.99"),
(pa.decimal128(10, 2), "0.00"),
(pa.decimal128(10, 2), "1.23E+5"),
(pa.decimal128(38, 18), "1E-18"),
(pa.decimal128(38, 18), "12345678901234567.123456789012345678"),
(pa.decimal128(38, 18), "12345678901234567890.123456789012345678"),
(pa.decimal128(38, 18), "-12345678901234567890.123456789012345678"),
(
pa.decimal256(76, 38),
"12345678901234567890123456789012345678."
"12345678901234567890123456789012345678",
),
(
pa.decimal256(76, 38),
"-12345678901234567890123456789012345678."
"12345678901234567890123456789012345678",
),
],
)
def test_update_decimal(mem_db: DBConnection, decimal_type, value, use_arrow_scalar):
expected = Decimal(value)
table = mem_db.create_table(
"my_table",
pa.table(
{
"id": [1, 2],
"price": pa.array([Decimal("1"), expected], type=decimal_type),
}
),
)
update = table.to_arrow()["price"][1] if use_arrow_scalar else expected
with localcontext() as context:
context.prec = 6
result = table.update(where="id = 1", values={"price": update})
assert result.rows_updated == 1
actual = table.to_arrow()
assert actual.schema.field("price").type == decimal_type
assert actual["price"].to_pylist() == [expected, expected]
def test_update_types(mem_db: DBConnection):
table = mem_db.create_table(
"my_table",
@@ -3751,6 +3879,24 @@ def test_empty_query(mem_db: DBConnection):
assert df.num_rows == 42
@pytest.mark.parametrize("query", [[], np.array([], dtype=np.float32)])
@pytest.mark.parametrize("vector_column_name", [None, "vector"])
@pytest.mark.parametrize("query_type", ["auto", "vector"])
@pytest.mark.parametrize("multiple_vector_columns", [False, True])
def test_search_empty_vector(
mem_db, query, vector_column_name, query_type, multiple_vector_columns
):
fields = [pa.field("vector", pa.list_(pa.float32(), 8))]
if multiple_vector_columns:
fields.append(pa.field("vec2", pa.list_(pa.float32(), 4)))
table = mem_db.create_table("empty_vector_query", schema=pa.schema(fields))
with pytest.raises(ValueError, match="^Query vector must not be empty$"):
table.search(
query, vector_column_name=vector_column_name, query_type=query_type
).limit(3).to_arrow()
def test_search_with_schema_inf_single_vector(mem_db: DBConnection):
class MyTable(LanceModel):
text: str
@@ -4144,14 +4290,74 @@ async def test_add_columns_with_schema(mem_db_async: AsyncConnection):
)
def test_alter_columns(mem_db: DBConnection):
@pytest.mark.parametrize("rename_key", ["rename", "name"])
def test_alter_columns(mem_db: DBConnection, rename_key):
data = pa.table({"id": [0, 1]})
table = mem_db.create_table("my_table", data=data)
alter_columns_res = table.alter_columns({"path": "id", "rename": "new_id"})
alter_columns_res = table.alter_columns({"path": "id", rename_key: "new_id"})
assert alter_columns_res.version == 2
assert table.to_arrow().column_names == ["new_id"]
INVALID_COLUMN_ALTERATIONS = [
(({"path": "id"},), "One of rename, nullable or data_type"),
(
({"path": "id", "nulable": False},), # spellchecker:disable-line
"Unknown column alteration key 'nulable'", # spellchecker:disable-line
),
(
(
{
"path": "id",
"rename": "new_id",
"nulable": False, # spellchecker:disable-line
},
),
"Unknown column alteration key 'nulable'", # spellchecker:disable-line
),
(
({"path": "id", "rename": "new_id"}, {"path": "id"}),
"One of rename, nullable or data_type",
),
]
def test_alter_columns_nullable_false(mem_db: DBConnection):
table = mem_db.create_table("my_table", data=pa.table({"id": [0, 1]}))
result = table.alter_columns({"path": "id", "nullable": False})
assert result.version == 2
assert not table.schema.field("id").nullable
@pytest.mark.parametrize("alterations, match", INVALID_COLUMN_ALTERATIONS)
def test_alter_columns_rejects_invalid(mem_db: DBConnection, alterations, match):
table = mem_db.create_table("my_table", data=pa.table({"id": [0, 1]}))
initial_version = table.version
initial_schema = table.schema
with pytest.raises(ValueError, match=match):
table.alter_columns(*alterations)
assert table.version == initial_version
assert table.schema == initial_schema
@pytest.mark.asyncio
@pytest.mark.parametrize("alterations, match", INVALID_COLUMN_ALTERATIONS)
async def test_alter_columns_rejects_invalid_async(
mem_db_async: AsyncConnection, alterations, match
):
table = await mem_db_async.create_table("my_table", data=pa.table({"id": [0, 1]}))
initial_version = await table.version()
initial_schema = await table.schema()
with pytest.raises(ValueError, match=match):
await table.alter_columns(*alterations)
assert await table.version() == initial_version
assert await table.schema() == initial_schema
def test_update_field_metadata(mem_db: DBConnection):
data = pa.table({"id": [0, 1], "category": ["a", "b"]})
table = mem_db.create_table("my_table", data=data)
@@ -4177,10 +4383,11 @@ def test_update_field_metadata(mem_db: DBConnection):
@pytest.mark.asyncio
async def test_alter_columns_async(mem_db_async: AsyncConnection):
@pytest.mark.parametrize("rename_key", ["rename", "name"])
async def test_alter_columns_async(mem_db_async: AsyncConnection, rename_key):
data = pa.table({"id": [0, 1]})
table = await mem_db_async.create_table("my_table", data=data)
alter_columns_res = await table.alter_columns({"path": "id", "rename": "new_id"})
alter_columns_res = await table.alter_columns({"path": "id", rename_key: "new_id"})
assert alter_columns_res.version == 2
assert (await table.to_arrow()).column_names == ["new_id"]
alter_columns_res = await table.alter_columns(
@@ -4379,22 +4586,25 @@ def test_stats(mem_db: DBConnection):
"my_table",
data=[{"text": "foo", "id": 0}, {"text": "bar", "id": 1}],
)
assert len(table) == 2
table.add([{"text": "baz", "id": 1}])
assert len(table) == 3
stats = table.stats()
print(f"{stats=}")
assert stats == {
# Full on-disk size of the data file, footer and metadata included.
"total_bytes": 637,
"num_rows": 2,
# Full on-disk size of the data files, footer and metadata included:
# 637 bytes for the two-row fragment plus 415 for the one-row fragment.
"total_bytes": 1052,
"num_rows": 3,
"num_deleted_rows": 0,
"num_indices": 0,
"fragment_stats": {
"num_fragments": 1,
"num_small_fragments": 1,
"num_fragments": 2,
"num_small_fragments": 2,
"lengths": {
"min": 2,
"min": 1,
"max": 2,
"mean": 2,
"p25": 2,
"mean": 1,
"p25": 1,
"p50": 2,
"p75": 2,
"p99": 2,
@@ -4402,12 +4612,19 @@ def test_stats(mem_db: DBConnection):
},
}
# Both rows with id = 1 are deleted, but that empties the second fragment,
# which is dropped outright rather than kept with a deletion file, so only
# the row marked in the surviving fragment is counted.
table.delete("id = 1")
# Index files count toward total_bytes too (only deletion files and
# manifests are excluded).
table.create_index("id", config=BTree())
stats_with_index = table.stats()
assert stats_with_index["num_indices"] == 1
assert stats_with_index["total_bytes"] > stats["total_bytes"]
assert stats_with_index["num_rows"] == 1
assert stats_with_index["num_deleted_rows"] == 1
assert stats_with_index["fragment_stats"]["num_fragments"] == 1
def test_create_table_empty_list_with_schema(mem_db: DBConnection):
python/python/tests/test_util.py
+36
View File
@@ -4,6 +4,7 @@
import os
import pathlib
from decimal import Decimal, localcontext
from typing import Optional
import lance
@@ -157,6 +158,41 @@ def test_value_to_sql_string(tmp_path):
assert table.to_pandas().query("search == @value")["replace"].item() == value
@pytest.mark.parametrize(
"value, expected",
[
("9.99", "'9.99'"),
("-9.99", "'-9.99'"),
("0", "'0'"),
("-0.00", "'-0.00'"),
("1.23E+5", "'123000'"),
("1E-18", "'0.000000000000000001'"),
(
"12345678901234567890.123456789012345678",
"'12345678901234567890.123456789012345678'",
),
(
"12345678901234567890123456789012345678."
"12345678901234567890123456789012345678",
"'12345678901234567890123456789012345678."
"12345678901234567890123456789012345678'",
),
],
)
def test_value_to_sql_decimal(value, expected):
with localcontext() as context:
context.prec = 6
assert value_to_sql(Decimal(value)) == expected
@pytest.mark.parametrize("value", ["NaN", "sNaN", "NaN123456", "Infinity", "-Infinity"])
def test_value_to_sql_decimal_non_finite(value):
with pytest.raises(
ValueError, match="^Non-finite Decimal values cannot be converted to SQL$"
):
value_to_sql(Decimal(value))
def test_value_to_sql_dict():
# Simple flat struct
assert value_to_sql({"a": 1, "b": "hello"}) == "named_struct('a', 1, 'b', 'hello')"
python/src/catalog.rs
+45 -15
View File
@@ -1,12 +1,13 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::{Arc, Mutex};
use std::time::Duration;
use lancedb::catalog::{
CatalogConnection, CreateDatabaseRequest, DropDatabaseRequest, ListDatabasesRequest,
};
use pyo3::exceptions::PyValueError;
use futures::{StreamExt, future::poll_fn};
use lancedb::catalog::{CatalogConnection, CreateDatabaseRequest, DropDatabaseRequest};
use pyo3::exceptions::{PyStopAsyncIteration, PyValueError};
use pyo3::{Bound, PyAny, PyRef, PyResult, Python, pyclass, pyfunction, pymethods};
use crate::connection::{Connection, PyClientConfig};
@@ -67,19 +68,48 @@ impl Catalog {
})
}
#[pyo3(signature = (*, limit=None, page_token=None))]
fn list_databases<'py>(
self_: PyRef<'py, Self>,
limit: Option<u32>,
page_token: Option<String>,
) -> PyResult<Bound<'py, PyAny>> {
#[pyo3(signature = (*, page_token=None, page_limit=None))]
fn list_databases(&self, page_token: Option<String>, page_limit: Option<u32>) -> DatabaseNames {
DatabaseNames {
inner: Arc::new(Mutex::new(
self.inner.list_databases(page_token, page_limit),
)),
next_lock: Arc::new(tokio::sync::Mutex::new(())),
}
}
}
#[pyclass]
pub struct DatabaseNames {
inner: Arc<Mutex<lancedb::catalog::DatabaseNames>>,
next_lock: Arc<tokio::sync::Mutex<()>>,
}
#[pymethods]
impl DatabaseNames {
fn num_page_results(&self, py: Python<'_>) -> usize {
py.detach(|| self.inner.lock().unwrap().num_page_results())
}
fn page_token(&self, py: Python<'_>) -> Option<String> {
py.detach(|| self.inner.lock().unwrap().page_token().map(str::to_owned))
}
fn __aiter__(self_: PyRef<'_, Self>) -> PyRef<'_, Self> {
self_
}
fn __anext__(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
let next_lock = self_.next_lock.clone();
future_into_py(self_.py(), async move {
let mut request = ListDatabasesRequest::default();
request.limit = limit;
request.page_token = page_token;
let response = inner.list_databases(request).await.infer_error()?;
Ok((response.databases, response.page_token))
// Serialize advances while allowing synchronous state inspection
// between polls, including while a REST request is pending.
let _guard = next_lock.lock().await;
poll_fn(|cx| inner.lock().unwrap().poll_next_unpin(cx))
.await
.ok_or_else(|| PyStopAsyncIteration::new_err(""))?
.infer_error()
})
}
}
python/src/connection.rs
+84 -67
View File
@@ -14,9 +14,10 @@ use crate::{
table::Table,
};
use arrow::{
array::RecordBatch,
datatypes::Schema,
ffi_stream::ArrowArrayStreamReader,
pyarrow::{FromPyArrow, ToPyArrow},
pyarrow::{FromPyArrow, PyArrowType, ToPyArrow},
};
use lancedb::{
connection::Connection as LanceConnection,
@@ -148,18 +149,22 @@ impl Connection {
self.get_inner().map(|inner| inner.uri().to_string())
}
#[pyo3(signature = (query, *, default_namespace_path=None))]
#[pyo3(signature = (query, *, default_namespace_path=None, parameters=None))]
pub fn execute_query_async<'a>(
self_: PyRef<'a, Self>,
query: String,
default_namespace_path: Option<Bound<'_, PyAny>>,
parameters: Option<PyArrowType<RecordBatch>>,
) -> PyResult<Bound<'a, PyAny>> {
let inner = self_.get_inner()?.clone();
let default_namespace_path = parse_default_namespace_path(default_namespace_path)?;
future_into_py(self_.py(), async move {
let operation = inner
let mut operation = inner
.execute_query_async(query)
.default_namespace_path(default_namespace_path);
if let Some(PyArrowType(parameters)) = parameters {
operation = operation.parameters(parameters);
}
operation
.execute()
.await
@@ -407,8 +412,7 @@ impl Connection {
})
}
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, *, vector_source_json=None))]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
pub fn create_materialized_view(
self_: PyRef<'_, Self>,
name: String,
@@ -417,17 +421,10 @@ impl Connection {
filter: Option<String>,
limit: Option<u64>,
with_no_data: bool,
vector_source_json: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let mut builder = inner.create_materialized_view(name, source);
if let Some(source) = vector_source_json {
builder = builder.vector_source(
serde_json::from_str(&source)
.map_err(|e| PyValueError::new_err(e.to_string()))?,
);
}
if let Some(projections) = projections {
builder = builder.select(projections);
}
@@ -443,8 +440,7 @@ impl Connection {
})
}
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, *, vector_source_json=None))]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
pub fn create_materialized_view_async(
self_: PyRef<'_, Self>,
name: String,
@@ -453,17 +449,10 @@ impl Connection {
filter: Option<String>,
limit: Option<u64>,
with_no_data: bool,
vector_source_json: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let mut builder = inner.create_materialized_view(name, source);
if let Some(source) = vector_source_json {
builder = builder.vector_source(
serde_json::from_str(&source)
.map_err(|e| PyValueError::new_err(e.to_string()))?,
);
}
if let Some(projections) = projections {
builder = builder.select(projections);
}
@@ -480,12 +469,16 @@ impl Connection {
})
}
pub fn list_materialized_views(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let views = inner.list_materialized_views().await.infer_error()?;
Ok(views)
})
#[pyo3(signature = (*, page_token=None, page_limit=None))]
pub fn list_materialized_views(
&self,
page_token: Option<String>,
page_limit: Option<u32>,
) -> PyResult<crate::listing::NameListing> {
Ok(crate::listing::NameListing::new(
self.get_inner()?
.list_materialized_views(crate::listing::options(page_token, page_limit)),
))
}
#[pyo3(signature = (name, namespace_path=None))]
@@ -754,31 +747,37 @@ impl Connection {
})
}
#[pyo3(signature = (request_json, namespace_path=None))]
pub fn create_function_async(
self_: PyRef<'_, Self>,
request_json: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json)
.infer_error()?;
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner
.create_function_async(request)
.create_function_async(request, &namespace_path)
.await
.infer_error()
.map(crate::job::Job::new_typed)
})
}
#[pyo3(signature = (name, version, namespace_path=None))]
pub fn get_function(
self_: PyRef<'_, Self>,
name: String,
version: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner
.get_function(name, version)
.get_function(name, version, &namespace_path)
.await
.infer_error()?
.to_canonical_json()
@@ -786,39 +785,50 @@ impl Connection {
})
}
pub fn list_functions(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner
.list_functions()
.await
.infer_error()?
.into_iter()
.map(|function| function.to_canonical_json().infer_error())
.collect::<PyResult<Vec<_>>>()
})
#[pyo3(signature = (namespace_path=None, *, page_token=None, page_limit=None))]
pub fn list_functions(
&self,
namespace_path: Option<Vec<String>>,
page_token: Option<String>,
page_limit: Option<u32>,
) -> PyResult<crate::listing::FunctionListing> {
Ok(crate::listing::FunctionListing::new(
self.get_inner()?.list_functions(
&namespace_path.unwrap_or_default(),
crate::listing::options(page_token, page_limit),
),
))
}
#[pyo3(signature = (name, version, namespace_path=None))]
pub fn drop_function(
self_: PyRef<'_, Self>,
name: String,
version: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner.drop_function(name, version).await.infer_error()
inner
.drop_function(name, version, &namespace_path)
.await
.infer_error()
})
}
#[pyo3(signature = (name, version, namespace_path=None))]
pub fn drop_function_async(
self_: PyRef<'_, Self>,
name: String,
version: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner
.drop_function_async(name, version)
.drop_function_async(name, version, &namespace_path)
.await
.infer_error()
.map(|(dropped, job)| (dropped, crate::job::Job::new(job)))
@@ -859,16 +869,19 @@ impl Connection {
})
}
#[pyo3(signature = (namespace_path=None))]
#[pyo3(signature = (namespace_path=None, *, page_token=None, page_limit=None))]
pub fn list_secrets(
self_: PyRef<'_, Self>,
&self,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner.list_secrets(&namespace_path).await.infer_error()
})
page_token: Option<String>,
page_limit: Option<u32>,
) -> PyResult<crate::listing::NameListing> {
Ok(crate::listing::NameListing::new(
self.get_inner()?.list_secrets(
&namespace_path.unwrap_or_default(),
crate::listing::options(page_token, page_limit),
),
))
}
#[pyo3(signature = (name, namespace_path=None))]
@@ -969,27 +982,31 @@ impl Connection {
})
}
#[pyo3(signature = (namespace_path=None))]
#[pyo3(signature = (namespace_path=None, *, page_token=None, page_limit=None))]
pub fn list_views(
self_: PyRef<'_, Self>,
&self,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner.list_views(&namespace_path).await.infer_error()
})
page_token: Option<String>,
page_limit: Option<u32>,
) -> PyResult<crate::listing::NameListing> {
Ok(crate::listing::NameListing::new(
self.get_inner()?.list_views(
&namespace_path.unwrap_or_default(),
crate::listing::options(page_token, page_limit),
),
))
}
pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let jobs = inner.list_jobs().await.infer_error()?;
Ok(jobs
.into_iter()
.map(crate::job::JobInfo::from)
.collect::<Vec<_>>())
})
#[pyo3(signature = (*, page_token=None, page_limit=None))]
pub fn list_jobs(
&self,
page_token: Option<String>,
page_limit: Option<u32>,
) -> PyResult<crate::listing::JobListing> {
Ok(crate::listing::JobListing::new(
self.get_inner()?
.list_jobs(crate::listing::options(page_token, page_limit)),
))
}
pub fn cancel_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
python/src/index.rs
+8 -4
View File
@@ -54,8 +54,6 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
.with_position(params.with_position)
.lower_case(params.lower_case)
.max_token_length(params.max_token_length)
.remove_stop_words(params.remove_stop_words)
.stem(params.stem)
.ascii_folding(params.ascii_folding)
.ngram_min_length(params.ngram_min_length)
.ngram_max_length(params.ngram_max_length)
@@ -65,6 +63,12 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
DocumentGranularity::try_from(params.document_granularity.as_str())
.map_err(|err| PyValueError::new_err(err.to_string()))?,
);
if let Some(stem) = params.stem {
inner_opts = inner_opts.stem(stem);
}
if let Some(remove_stop_words) = params.remove_stop_words {
inner_opts = inner_opts.remove_stop_words(remove_stop_words);
}
if let Some(memory_limit) = params.memory_limit {
inner_opts = inner_opts.memory_limit_mb(memory_limit);
}
@@ -215,8 +219,8 @@ struct FtsParams {
language: String,
max_token_length: Option<usize>,
lower_case: bool,
stem: bool,
remove_stop_words: bool,
stem: Option<bool>,
remove_stop_words: Option<bool>,
custom_stop_words: Option<Vec<String>>,
ascii_folding: bool,
ngram_min_length: u32,
python/src/lib.rs
+6
View File
@@ -29,6 +29,7 @@ pub mod expr;
pub mod header;
pub mod index;
pub mod job;
mod listing;
pub mod namespace;
pub mod oauth;
pub mod otel;
@@ -64,6 +65,11 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
env_logger::init_from_env(env);
m.add_class::<Connection>()?;
m.add_class::<catalog::Catalog>()?;
m.add_class::<catalog::DatabaseNames>()?;
m.add_class::<listing::NameListing>()?;
m.add_class::<listing::FunctionListing>()?;
m.add_class::<listing::JobListing>()?;
m.add_function(wrap_pyfunction!(catalog::connect_catalog, m)?)?;
m.add_class::<Session>()?;
m.add_class::<Table>()?;
python/src/listing.rs
+131
View File
@@ -0,0 +1,131 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use crate::error::PythonErrorExt;
use crate::runtime::future_into_py;
use futures::{StreamExt, future::poll_fn};
use lancedb::listing::{Listing, ListingOptions};
use pyo3::exceptions::PyStopAsyncIteration;
use pyo3::{Bound, PyAny, PyRef, PyResult, Python, pyclass, pymethods};
use std::sync::{Arc, Mutex};
pub fn options(page_token: Option<String>, page_limit: Option<u32>) -> ListingOptions {
let mut options = ListingOptions::default();
options.page_token = page_token;
options.page_limit = page_limit;
options
}
#[pyclass]
pub struct NameListing {
inner: Arc<Mutex<Listing<String>>>,
next_lock: Arc<tokio::sync::Mutex<()>>,
}
impl NameListing {
pub fn new(inner: Listing<String>) -> Self {
Self {
inner: Arc::new(Mutex::new(inner)),
next_lock: Arc::new(tokio::sync::Mutex::new(())),
}
}
}
#[pymethods]
impl NameListing {
fn num_page_results(&self, py: Python<'_>) -> usize {
py.detach(|| self.inner.lock().unwrap().num_page_results())
}
fn page_token(&self, py: Python<'_>) -> Option<String> {
py.detach(|| self.inner.lock().unwrap().page_token().map(str::to_owned))
}
fn __aiter__(self_: PyRef<'_, Self>) -> PyRef<'_, Self> {
self_
}
fn __anext__(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
let next_lock = self_.next_lock.clone();
future_into_py(self_.py(), async move {
let _guard = next_lock.lock().await;
let value = poll_fn(|cx| inner.lock().unwrap().poll_next_unpin(cx))
.await
.ok_or_else(|| PyStopAsyncIteration::new_err(""))?
.infer_error()?;
Ok(value)
})
}
}
#[pyclass]
pub struct FunctionListing {
inner: Arc<Mutex<Listing<lancedb::function::FunctionVersion>>>,
next_lock: Arc<tokio::sync::Mutex<()>>,
}
impl FunctionListing {
pub fn new(inner: Listing<lancedb::function::FunctionVersion>) -> Self {
Self {
inner: Arc::new(Mutex::new(inner)),
next_lock: Arc::new(tokio::sync::Mutex::new(())),
}
}
}
#[pymethods]
impl FunctionListing {
fn num_page_results(&self, py: Python<'_>) -> usize {
py.detach(|| self.inner.lock().unwrap().num_page_results())
}
fn page_token(&self, py: Python<'_>) -> Option<String> {
py.detach(|| self.inner.lock().unwrap().page_token().map(str::to_owned))
}
fn __aiter__(self_: PyRef<'_, Self>) -> PyRef<'_, Self> {
self_
}
fn __anext__(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
let next_lock = self_.next_lock.clone();
future_into_py(self_.py(), async move {
let _guard = next_lock.lock().await;
let value = poll_fn(|cx| inner.lock().unwrap().poll_next_unpin(cx))
.await
.ok_or_else(|| PyStopAsyncIteration::new_err(""))?
.infer_error()?;
value.to_canonical_json().infer_error()
})
}
}
#[pyclass]
pub struct JobListing {
inner: Arc<Mutex<Listing<lancedb::database::JobInfo>>>,
next_lock: Arc<tokio::sync::Mutex<()>>,
}
impl JobListing {
pub fn new(inner: Listing<lancedb::database::JobInfo>) -> Self {
Self {
inner: Arc::new(Mutex::new(inner)),
next_lock: Arc::new(tokio::sync::Mutex::new(())),
}
}
}
#[pymethods]
impl JobListing {
fn num_page_results(&self, py: Python<'_>) -> usize {
py.detach(|| self.inner.lock().unwrap().num_page_results())
}
fn page_token(&self, py: Python<'_>) -> Option<String> {
py.detach(|| self.inner.lock().unwrap().page_token().map(str::to_owned))
}
fn __aiter__(self_: PyRef<'_, Self>) -> PyRef<'_, Self> {
self_
}
fn __anext__(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
let next_lock = self_.next_lock.clone();
future_into_py(self_.py(), async move {
let _guard = next_lock.lock().await;
let value = poll_fn(|cx| inner.lock().unwrap().poll_next_unpin(cx))
.await
.ok_or_else(|| PyStopAsyncIteration::new_err(""))?
.infer_error()?;
Ok(crate::job::JobInfo::from(value))
})
}
}
python/src/query.rs
+44 -14
View File
@@ -71,7 +71,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
match class_name(&ob)?.as_str() {
"MatchQuery" => {
let query = ob.getattr("query")?.extract()?;
let column = ob.getattr("column")?.extract()?;
let column = ob.getattr("column")?.extract::<Option<String>>()?;
let boost = ob.getattr("boost")?.extract()?;
let fuzziness = ob.getattr("fuzziness")?.extract()?;
let max_expansions = ob.getattr("max_expansions")?.extract()?;
@@ -88,7 +88,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
let mut query =
MatchQuery::new(query)
.with_column(Some(column))
.with_column(column)
.with_boost(boost)
.with_fuzziness(fuzziness)
.with_max_expansions(max_expansions)
@@ -180,6 +180,9 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
.expect("Failed to import namespace");
match self.0 {
FtsQuery::CombinedFields(_) => Err(PyValueError::new_err(
"Combined-fields FTS queries are not yet supported in Python",
)),
FtsQuery::Match(query) => {
let kwargs = PyDict::new(py);
kwargs.set_item("boost", query.boost)?;
@@ -196,7 +199,7 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
}
namespace
.getattr(intern!(py, "MatchQuery"))?
.call((query.terms, query.column.unwrap()), Some(&kwargs))
.call((query.terms, query.column), Some(&kwargs))
}
FtsQuery::Phrase(query) => {
let kwargs = PyDict::new(py);
@@ -333,10 +336,10 @@ pub struct PyQueryRequest {
pub use_lsm: Option<bool>,
pub column: Option<String>,
pub query_vector: Option<PyQueryVectors>,
pub nprobes: Option<usize>,
pub minimum_nprobes: Option<usize>,
// None means user did not set it and default should be used (currently 20)
// Some(0) means user set it to None and there is no limit
pub maximum_nprobes: Option<usize>,
pub search_effort: Option<f64>,
pub lower_bound: Option<f32>,
pub upper_bound: Option<f32>,
pub ef: Option<usize>,
@@ -366,8 +369,10 @@ impl From<AnyQuery> for PyQueryRequest {
use_lsm: query_request.use_lsm,
column: None,
query_vector: None,
nprobes: None,
minimum_nprobes: None,
maximum_nprobes: None,
search_effort: None,
lower_bound: None,
upper_bound: None,
ef: None,
@@ -393,11 +398,10 @@ impl From<AnyQuery> for PyQueryRequest {
use_lsm: vector_query.base.use_lsm,
column: vector_query.column,
query_vector: Some(PyQueryVectors(vector_query.query_vector)),
minimum_nprobes: Some(vector_query.minimum_nprobes),
maximum_nprobes: match vector_query.maximum_nprobes {
None => Some(0),
Some(value) => Some(value),
},
nprobes: vector_query.nprobes,
minimum_nprobes: vector_query.minimum_nprobes,
maximum_nprobes: vector_query.maximum_nprobes,
search_effort: vector_query.search_effort,
lower_bound: vector_query.lower_bound,
upper_bound: vector_query.upper_bound,
ef: vector_query.ef,
@@ -992,8 +996,18 @@ impl VectorQuery {
self.inner = self.inner.clone().refine_factor(refine_factor);
}
pub fn nprobes(&mut self, nprobe: u32) {
self.inner = self.inner.clone().nprobes(nprobe as usize);
pub fn nprobes(&mut self, nprobe: u32) -> PyResult<()> {
self.inner = self.inner.clone().nprobes(nprobe as usize).infer_error()?;
Ok(())
}
pub fn search_effort(&mut self, search_effort: Option<f64>) -> PyResult<()> {
self.inner = self
.inner
.clone()
.search_effort(search_effort)
.infer_error()?;
Ok(())
}
pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> {
@@ -1185,8 +1199,20 @@ impl HybridQuery {
self.inner_vec.refine_factor(refine_factor);
}
pub fn nprobes(&mut self, nprobe: u32) {
self.inner_vec.nprobes(nprobe);
pub fn nprobes(&mut self, nprobe: u32) -> PyResult<()> {
self.inner_vec.nprobes(nprobe)
}
pub fn search_effort(&mut self, search_effort: Option<f64>) -> PyResult<()> {
self.inner_vec.search_effort(search_effort)
}
pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> {
self.inner_vec.minimum_nprobes(minimum_nprobes)
}
pub fn maximum_nprobes(&mut self, maximum_nprobes: u32) -> PyResult<()> {
self.inner_vec.maximum_nprobes(maximum_nprobes)
}
pub fn ef(&mut self, ef: u32) {
@@ -1233,6 +1259,10 @@ impl HybridQuery {
req.query_vector = vec_req.query_vector;
req.column = vec_req.column;
req.distance_type = vec_req.distance_type;
req.nprobes = vec_req.nprobes;
req.minimum_nprobes = vec_req.minimum_nprobes;
req.maximum_nprobes = vec_req.maximum_nprobes;
req.search_effort = vec_req.search_effort;
req.ef = vec_req.ef;
req.refine_factor = vec_req.refine_factor;
req.lower_bound = vec_req.lower_bound;
python/src/table.rs
+68 -31
View File
@@ -267,7 +267,7 @@ fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
/// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()`
/// classmethods, then optionally chain `with_maintained_indexes(...)` and
/// `with_writer_config_defaults(...)`. A fresh spec maintains every index the
/// MemWAL supports, resolved on install.
/// table has, including ones created later.
#[pyclass(module = "lancedb._lancedb", from_py_object)]
#[derive(Clone, Debug)]
pub struct LsmWriteSpec {
@@ -307,9 +307,9 @@ impl LsmWriteSpec {
}
}
/// Set which indexes the MemWAL maintains. `None` (the default)
/// resolves every supported index on install; a list is verbatim,
/// and an empty list maintains nothing.
/// Set which indexes the MemWAL maintains. `None` (the default) is
/// every index the table has, including ones created later; a list is
/// verbatim, and an empty list maintains nothing.
#[pyo3(signature = (indexes))]
pub fn with_maintained_indexes(&self, indexes: Option<Vec<String>>) -> Self {
Self {
@@ -592,8 +592,6 @@ impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewRes
fn from(result: lancedb::RefreshMaterializedViewResult) -> Self {
let mode = match result.mode {
lancedb::RefreshMode::Rebuild => "rebuild",
lancedb::RefreshMode::Incremental => "incremental",
lancedb::RefreshMode::NoOp => "no_op",
};
Self {
mode: mode.to_string(),
@@ -825,8 +823,8 @@ impl From<LanceDbFtsToken> for FtsToken {
language = "English".to_string(),
max_token_length = Some(40),
lower_case = true,
stem = true,
remove_stop_words = true,
stem = None,
remove_stop_words = None,
custom_stop_words = None,
ascii_folding = true,
ngram_min_length = 3,
@@ -840,15 +838,15 @@ pub fn tokenize(
language: String,
max_token_length: Option<u32>,
lower_case: bool,
stem: bool,
remove_stop_words: bool,
stem: Option<bool>,
remove_stop_words: Option<bool>,
custom_stop_words: Option<Vec<String>>,
ascii_folding: bool,
ngram_min_length: u32,
ngram_max_length: u32,
prefix_only: bool,
) -> PyResult<Vec<FtsToken>> {
let params = FtsIndexBuilder::default()
let mut params = FtsIndexBuilder::default()
.base_tokenizer(base_tokenizer)
.language(&language)
.map_err(|_| {
@@ -859,13 +857,17 @@ pub fn tokenize(
})?
.max_token_length(max_token_length.map(|value| value as usize))
.lower_case(lower_case)
.stem(stem)
.remove_stop_words(remove_stop_words)
.ascii_folding(ascii_folding)
.ngram_min_length(ngram_min_length)
.ngram_max_length(ngram_max_length)
.ngram_prefix_only(prefix_only)
.custom_stop_words(custom_stop_words);
if let Some(stem) = stem {
params = params.stem(stem);
}
if let Some(remove_stop_words) = remove_stop_words {
params = params.remove_stop_words(remove_stop_words);
}
let tokens = lancedb_tokenize(&query, &params).infer_error()?;
Ok(tokens.into_iter().map(FtsToken::from).collect())
}
@@ -915,6 +917,11 @@ impl Table {
self.inner.is_some()
}
/// Whether this table has a local Lance dataset that the client may open.
pub fn _is_native(&self) -> PyResult<bool> {
Ok(self.inner_ref()?.as_native().is_some())
}
/// Closes the table, releasing any resources associated with it.
pub fn close(&mut self) {
self.inner.take();
@@ -1268,6 +1275,7 @@ impl Table {
let dict = PyDict::new(py);
dict.set_item("total_bytes", stats.total_bytes)?;
dict.set_item("num_rows", stats.num_rows)?;
dict.set_item("num_deleted_rows", stats.num_deleted_rows)?;
dict.set_item("num_indices", stats.num_indices)?;
let fragment_stats = PyDict::new(py);
@@ -1454,14 +1462,30 @@ impl Table {
pub fn fetch_blob_ranges(
self_: PyRef<'_, Self>,
column: String,
requests: Vec<(u64, u64, u64)>,
requests: Vec<Vec<i128>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
let requests = requests
.into_iter()
.map(|values| {
let [row_id, offset, length] = <[i128; 3]>::try_from(values).map_err(|_| {
PyValueError::new_err("each blob range request must have three values")
})?;
let to_u64 = |value, name| {
u64::try_from(value).map_err(|_| {
PyValueError::new_err(format!(
"blob range {name} must be a non-negative 64-bit integer"
))
})
};
Ok(BlobRangeRequest::new(
to_u64(row_id, "row id")?,
to_u64(offset, "offset")?,
to_u64(length, "length")?,
))
})
.collect::<PyResult<Vec<_>>>()?;
future_into_py(self_.py(), async move {
let requests = requests
.into_iter()
.map(|(row_id, offset, length)| BlobRangeRequest::new(row_id, offset, length))
.collect::<Vec<_>>();
let blobs: LargeBinaryArray = inner
.fetch_blob_ranges(column, requests)
.await
@@ -1709,7 +1733,10 @@ impl Table {
future_into_py(self_.py(), async move {
inner
.as_native()
.ok_or_else(|| PyValueError::new_err("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "uses_v2_manifest_paths is not supported for remote tables.".into(),
})
.infer_error()?
.uses_v2_manifest_paths()
.await
.infer_error()
@@ -1721,7 +1748,10 @@ impl Table {
future_into_py(self_.py(), async move {
inner
.as_native()
.ok_or_else(|| PyValueError::new_err("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "migrate_manifest_paths_v2 is not supported for remote tables.".into(),
})
.infer_error()?
.migrate_manifest_paths_v2()
.await
.infer_error()
@@ -1817,10 +1847,9 @@ impl Table {
})
}
#[pyo3(signature = (full=false, source_version=None))]
#[pyo3(signature = (source_version=None))]
pub fn refresh_materialized_view(
self_: PyRef<'_, Self>,
full: bool,
source_version: Option<u64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
@@ -1828,7 +1857,7 @@ impl Table {
let view = lancedb::MaterializedView::from_table(inner)
.await
.infer_error()?;
let mut builder = view.refresh().full(full);
let mut builder = view.refresh();
if let Some(version) = source_version {
builder = builder.source_version(version);
}
@@ -1837,10 +1866,9 @@ impl Table {
})
}
#[pyo3(signature = (full=false, source_version=None))]
#[pyo3(signature = (source_version=None))]
pub fn refresh_materialized_view_async(
self_: PyRef<'_, Self>,
full: bool,
source_version: Option<u64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
@@ -1848,7 +1876,7 @@ impl Table {
let view = lancedb::MaterializedView::from_table(inner)
.await
.infer_error()?;
let mut builder = view.refresh().full(full);
let mut builder = view.refresh();
if let Some(version) = source_version {
builder = builder.source_version(version);
}
@@ -1863,11 +1891,9 @@ impl Table {
let view = lancedb::MaterializedView::from_table(inner)
.await
.infer_error()?;
view.definition().to_json().map_err(|err| {
PyRuntimeError::new_err(format!(
"failed to serialize materialized-view definition: {err}"
))
})
Ok(lancedb::materialized_view::definition_metadata_from_sql(
view.definition_sql(),
))
})
}
@@ -1897,6 +1923,17 @@ impl Table {
let alterations = alterations
.iter()
.map(|alteration| {
for key in alteration.keys().iter() {
let key = key.extract::<String>()?;
if !matches!(
key.as_str(),
"path" | "rename" | "name" | "nullable" | "data_type"
) {
return Err(PyValueError::new_err(format!(
"Unknown column alteration key '{key}'"
)));
}
}
let path = alteration
.get_item("path")?
.ok_or_else(|| PyValueError::new_err("Missing path"))?
python/uv.lock
Generated
+42 -16
View File
@@ -1849,19 +1849,19 @@ wheels = [
[[package]]
name = "lance-namespace"
version = "0.8.6"
version = "0.13.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "lance-namespace-urllib3-client" },
]
sdist = { url = "https://files.pythonhosted.org/packages/af/12/f7ab93b29be3edbf5fc3610714bf2d06088e7f4524bfb38dfd6852458b08/lance_namespace-0.8.6.tar.gz", hash = "sha256:18232e721c8188145f4ec9389cc2dfbeeabf54a619d94885ea1b3375bee9f4af", size = 11529, upload-time = "2026-06-12T17:36:41.651Z" }
sdist = { url = "https://files.pythonhosted.org/packages/c0/e7/d5d46594678ee479c0eda830c47b2f5c46133bd100e84e9aa6e01306eca7/lance_namespace-0.13.0.tar.gz", hash = "sha256:24554a0997bdb39595c6e4cb3ac6722069f6cd3bd1a74e7acbba7bfaf774a40d", size = 11636, upload-time = "2026-09-14T02:47:03.871523Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/a0/1b/5b1668ee2dc8910965f390640359112a31157092fcf8e000b89c79b58708/lance_namespace-0.8.6-py3-none-any.whl", hash = "sha256:571eae34f9aad70e5b05020416c2860889b9ec82993ccd0eb015e7b39c3ea309", size = 13383, upload-time = "2026-06-12T17:36:43.456Z" },
{ url = "https://files.pythonhosted.org/packages/e7/47/cfa33cca1ba7c749fd2918cfc5c8ded788f378cc6424d23e4fead5a14125/lance_namespace-0.13.0-py3-none-any.whl", hash = "sha256:438c7b17aef421c21c138196e715f2510d62f07e865372047f87c1e75e618c7a", size = 13507, upload-time = "2026-09-14T02:47:01.765655Z" },
]
[[package]]
name = "lance-namespace-urllib3-client"
version = "0.8.6"
version = "0.13.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pydantic" },
@@ -1869,9 +1869,9 @@ dependencies = [
{ name = "typing-extensions" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c7/80/fb224b4a89c1c1638cde949cb6cce6c3aca7759effbfea46a3d9c3960b21/lance_namespace_urllib3_client-0.8.6.tar.gz", hash = "sha256:b6fb1d306e74a7576e5309919020be744527de484a63dbf5eed10f8b368548df", size = 228772, upload-time = "2026-06-12T17:36:42.609Z" }
sdist = { url = "https://files.pythonhosted.org/packages/69/25/9aaa4a5e8999693fb0f227c2c0c4b97bd8f0539066408cdff0b89d41b2d5/lance_namespace_urllib3_client-0.13.0.tar.gz", hash = "sha256:1e8a79c6e4e6277033597fd76aa0e2f33d909ca1436c935936e9e569221f43ef", size = 238348, upload-time = "2026-09-14T02:47:04.521878Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/c5/90/1e27de15cd1b16785a1c7312beb0a59e75c8344a815f600f58173a565bd1/lance_namespace_urllib3_client-0.8.6-py3-none-any.whl", hash = "sha256:9d78249c3fb15aa3d15d668f78f04a275af3d08d800a7027492f37996ac4968b", size = 369950, upload-time = "2026-06-12T17:36:40.438Z" },
{ url = "https://files.pythonhosted.org/packages/60/4c/7b8f0712a7fe1b8655b711342bc4989696635a79e7027ee56ba3cae23e99/lance_namespace_urllib3_client-0.13.0-py3-none-any.whl", hash = "sha256:fb361eb4f6c7f2d1f9e92809609657b73e38241393e9c4516e12d6b6ee643a0a", size = 406848, upload-time = "2026-09-14T02:47:02.715954Z" },
]
[[package]]
@@ -1900,6 +1900,7 @@ clip = [
{ name = "torch" },
]
dev = [
{ name = "maturin" },
{ name = "pre-commit" },
{ name = "pyright" },
{ name = "ruff" },
@@ -1980,7 +1981,8 @@ requires-dist = [
{ name = "huggingface-hub", marker = "extra == 'embeddings'", specifier = ">=0.19.0" },
{ name = "ibm-watsonx-ai", marker = "python_full_version >= '3.10' and extra == 'embeddings'", specifier = ">=1.1.2" },
{ name = "instructorembedding", marker = "extra == 'embeddings'", specifier = ">=1.0.1" },
{ name = "lance-namespace", specifier = ">=0.3.2" },
{ name = "lance-namespace", specifier = ">=0.13.0" },
{ name = "maturin", marker = "extra == 'dev'", specifier = ">=1.10" },
{ name = "mkdocs", marker = "extra == 'docs'" },
{ name = "mkdocs-jupyter", marker = "extra == 'docs'" },
{ name = "mkdocs-material", marker = "extra == 'docs'" },
@@ -2005,7 +2007,7 @@ requires-dist = [
{ name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" },
{ name = "pydantic", specifier = ">=2.7.4,<3" },
{ name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==9.0.0" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b12" },
{ name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },
{ name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" },
{ name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" },
@@ -2156,6 +2158,30 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" },
]
[[package]]
name = "maturin"
version = "1.15.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "tomli", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b9/c8/22e5e21b2679c9bce6415ca578034ca2cc9316be0642ae21e051a2d5198c/maturin-1.15.0.tar.gz", hash = "sha256:94b26cc8e8aba61a5f2099715fe640e18c5f678e9a500408b38761263954228a", size = 385504, upload-time = "2026-08-24T12:11:22.665Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/14/69/5c01b461044eb1f45ddcce006706eb88110c793cdb11c7ae0b5e08492e94/maturin-1.15.0-py3-none-linux_armv6l.whl", hash = "sha256:6bf6dc62e22d4dcfd5a51244ff0d58975fa4979c48209fe84159617648956d82", size = 10206220, upload-time = "2026-08-24T12:10:53.327Z" },
{ url = "https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:cd35772633f489841132bc8e71d6fc7f842df30b9c05cd5cdf1ee1ddcb744cc7", size = 19416513, upload-time = "2026-08-24T12:10:56.126Z" },
{ url = "https://files.pythonhosted.org/packages/51/36/e23a21cb34a648b711036b9b2fe1d4f3f4ee24f8db54215d73f1a9a3a3ec/maturin-1.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c40b4eae7bf5ef1f4b1af8d623fe4105016f93578fb15b764e741d08ec3b92dd", size = 10014962, upload-time = "2026-08-24T12:10:58.486Z" },
{ url = "https://files.pythonhosted.org/packages/8c/80/33b15cb2d8f30f12c807955e8f2fd775027692904e30ec0784744ce8cd83/maturin-1.15.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:7eb066372f541f8eb4909c79c5d9bd0b9e8125980bdf1ec9e8aba23c6c8d6c55", size = 10196223, upload-time = "2026-08-24T12:11:00.696Z" },
{ url = "https://files.pythonhosted.org/packages/fe/91/b495e19e2f5c503b540452b2039115e7b2363867e8c5ad4179eb752fa92c/maturin-1.15.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:653020a63525bb224e5ab0adf02e17a2e08bc86dbea7fc1399c9a56d7529b99e", size = 10541186, upload-time = "2026-08-24T12:11:02.857Z" },
{ url = "https://files.pythonhosted.org/packages/5d/2b/2abff58037188d852b124871b1f0d720e1c2bfb3d4f1b03d87c52cd66488/maturin-1.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:0ebf9767892725083138e671c34482c660317a2f3d6a29fc0e0f34e9d8c99136", size = 10083468, upload-time = "2026-08-24T12:11:05.012Z" },
{ url = "https://files.pythonhosted.org/packages/3f/07/b7e9f8be99a6627849e81ac7b6694876bce8f50a92995fe17e3cf2610f0a/maturin-1.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7ab7eebffd7b8debca2265985de4eaeb332141276d24b9560b5ad484d4b3add1", size = 10047786, upload-time = "2026-08-24T12:11:07.042Z" },
{ url = "https://files.pythonhosted.org/packages/ca/ab/167e3cb7accee11b507dbe53e0e87aeccb376d44ae66284c96ee4df3a9fd/maturin-1.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:126e12e618b4db42f68c779a56d41f82a390145ba36ac3f621d057eb34f5ad9d", size = 13315332, upload-time = "2026-08-24T12:11:09.433Z" },
{ url = "https://files.pythonhosted.org/packages/14/4d/801379f646cbc6b00998e5289b0630a886be3a4ee4c75b6bc9b87478a7f1/maturin-1.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f9d33e6c3f9615c8caceecbbbd440f8eb25a3ddeb687077682cd5eca2e9ae15", size = 10807183, upload-time = "2026-08-24T12:11:11.73Z" },
{ url = "https://files.pythonhosted.org/packages/89/27/2e612e1cbd1580e9e94d4722c227b5180dca27b32b955a34b79918aa1292/maturin-1.15.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:bf29beddd0c6708f112db51d5275fc28b28b9e9c9c5faae387eaef662918b176", size = 10413274, upload-time = "2026-08-24T12:11:14.04Z" },
{ url = "https://files.pythonhosted.org/packages/70/d8/202a7b4d75a51f20f84ec9ce3b7345b12b822207072164e2b1c6ef665125/maturin-1.15.0-py3-none-win32.whl", hash = "sha256:da649988be98e87e009e51b1bf0d301b6a301bc0cecbdd60d40d8ba60748d1ca", size = 8928744, upload-time = "2026-08-24T12:11:16.269Z" },
{ url = "https://files.pythonhosted.org/packages/40/dc/4e90da594986ba78dd3bc8a5921ecdcdb11085b22b02a412caab3b225601/maturin-1.15.0-py3-none-win_amd64.whl", hash = "sha256:552c2be4afd43fe8d5c9f3ec8d4c4756d973b8dcbe94c14084390301f50243e1", size = 10335085, upload-time = "2026-08-24T12:11:18.326Z" },
{ url = "https://files.pythonhosted.org/packages/8b/10/15d4314edf130955edf2dc237aa393a8a7c10f2b9b57b89fa2f61f915659/maturin-1.15.0-py3-none-win_arm64.whl", hash = "sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48", size = 9713795, upload-time = "2026-08-24T12:11:20.83Z" },
]
[[package]]
name = "mdit-py-plugins"
version = "0.6.1"
@@ -3912,8 +3938,8 @@ crypto = [
[[package]]
name = "pylance"
version = "9.0.0"
source = { registry = "https://pypi.org/simple" }
version = "14.0.0b12"
source = { registry = "https://pypi.fury.io/lance-format/" }
dependencies = [
{ name = "lance-namespace" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
@@ -3922,12 +3948,12 @@ dependencies = [
{ name = "pyarrow" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/e9/be/45733acd64801991852aac8e658601fd8fc12f76ceb81e57fca690896b90/pylance-9.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:8257213501d3298c5b6a344d60938e4bbe4de9f00cd3265371a56d1dc3dd15ca", size = 68377982, upload-time = "2026-07-24T16:53:45.247Z" },
{ url = "https://files.pythonhosted.org/packages/d0/3e/1ef707cb215cc7268c63ad84a91344ad6313d3984b343eeb19a9b708698e/pylance-9.0.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:804eedfa1fda2e703cca8580c76f0b44a1b849b725a8908c06f8acfca811732f", size = 71844362, upload-time = "2026-07-24T16:56:07.6Z" },
{ url = "https://files.pythonhosted.org/packages/c8/fb/a499e5c53ddb75c7de44100fd2bb1f7cc735966200d20292eed7af9ef552/pylance-9.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a0b75595e3766c1d5f4c90abdc52337b4de60f1a52d123a4f8c4e5bcbbbfa8f", size = 75663088, upload-time = "2026-07-24T17:10:31.283Z" },
{ url = "https://files.pythonhosted.org/packages/e9/80/0714e09f64a68dbdf62558955e737a7436df763b9834a6aba506861b5352/pylance-9.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d2f69c5c390ae3710c35a429905fe15f769951777fafb1b72a359e035ec121f7", size = 71866858, upload-time = "2026-07-24T16:56:35.937Z" },
{ url = "https://files.pythonhosted.org/packages/4b/3c/78d3a6d6ca0d843b7c3ac0c30d9cd2cf4635b0c2cabe6ec66583d5bfc1a1/pylance-9.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:836268a7832d62f3d5ccbe1c3fca239621971d90297bcfff14a70b3cb6842aa8", size = 75642656, upload-time = "2026-07-24T17:13:11.879Z" },
{ url = "https://files.pythonhosted.org/packages/cf/c1/dc9c9a31e171530ec0add024d922488c046437d32a7087c29e254a7eacc7/pylance-9.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:96441d27a5ed3805300388ccf8f31835cd280c98751f80b6a1a11dcd6808fc43", size = 81668288, upload-time = "2026-07-24T17:05:38.707Z" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1Ixasc/pylance-14.0.0b12-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:84350a3c5c1eea3398f4c4fad7cfe85b831d831a5e43b5ca2a783dbdb0e11969" },
{ url = "https://pypi.fury.io/lance-format/-/ver_sCEkG/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76b386c182a6a8e7b86fbef28f84d1dd39e4626ac8681a0cf36f40a45dd7cb87" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1vdT1C/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6aa548c7937fcdfa8f1943a2d59041a42f8b500199880dbb509c7195fa253381" },
{ url = "https://pypi.fury.io/lance-format/-/ver_2fXu1V/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:79f82d6a6c50f37aab338470e3733436e6a27fbb5972d29fcf453ece56d3b3c6" },
{ url = "https://pypi.fury.io/lance-format/-/ver_24UavN/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:de0305999257002785f60169e6926896584d66f1da050bb5afd1d48c6e3d9bb9" },
{ url = "https://pypi.fury.io/lance-format/-/ver_XNynX/pylance-14.0.0b12-cp310-abi3-win_amd64.whl", hash = "sha256:4d0cdb689e898ce71e8e434d00a3b97639cea8fb97da21f1c921543be363268f" },
]
[[package]]