rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-13 07:12:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into gatekeeper/fix-4260-1

Browse Source
This commit is contained in:
Gatefixer
2026-10-06 16:28:54 +00:00
parent 3869ba89b7 17e9a657e6
commit aa912492db
127 changed files with 7396 additions and 13625 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/Cargo.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.40.0-beta.12"
version = "0.40.0-beta.13"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
@@ -8,7 +8,7 @@ license.workspace = true
repository.workspace = true
keywords.workspace = true
categories.workspace = true
rust-version = "1.91.0"
rust-version = "1.94.0"
[lib]
name = "_lancedb"
python/python/lancedb/__init__.py
-3
View File
@@ -46,7 +46,6 @@ from .materialized_view import (
AsyncMaterializedView,
MaterializedView,
MaterializedViewDefinition,
vector_dedup as vector_dedup,
)
from .view import ViewDescription as ViewDescription
from .table import AsyncTable, Table
@@ -62,7 +61,6 @@ from .namespace import (
from .catalog import (
AsyncCatalog,
Catalog,
ListDatabasesResponse,
connect_catalog,
connect_catalog_async,
)
@@ -578,7 +576,6 @@ async def connect_async(
__all__ = [
"Catalog",
"AsyncCatalog",
"ListDatabasesResponse",
"connect_catalog",
"connect_catalog_async",
"AsyncMaterializedView",
python/python/lancedb/_lancedb.pyi
+11 -9
View File
@@ -255,8 +255,6 @@ class Connection(object):
filter: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
*,
vector_source_json: Optional[str] = None,
) -> Table: ...
async def create_materialized_view_async(
self,
@@ -266,8 +264,6 @@ class Connection(object):
filter: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
*,
vector_source_json: Optional[str] = None,
) -> Job: ...
async def list_materialized_views(self) -> List[str]: ...
async def drop_materialized_view(
@@ -500,10 +496,10 @@ class Table:
limit: Optional[int] = None,
) -> FunctionErrors: ...
async def refresh_materialized_view(
self, full: bool = False, source_version: Optional[int] = None
self, source_version: Optional[int] = None
) -> RefreshMaterializedViewResult: ...
async def refresh_materialized_view_async(
self, full: bool = False, source_version: Optional[int] = None
self, source_version: Optional[int] = None
) -> Job: ...
async def materialized_view_definition(self) -> str: ...
async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...
@@ -946,9 +942,15 @@ class Catalog:
async def drop_database(
self, name: str, *, ignore_missing: bool = False
) -> None: ...
async def list_databases(
self, *, limit: Optional[int] = None, page_token: Optional[str] = None
) -> tuple[list[str], Optional[str]]: ...
def list_databases(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> DatabaseNames: ...
class DatabaseNames:
def num_page_results(self) -> int: ...
def page_token(self) -> Optional[str]: ...
def __aiter__(self) -> DatabaseNames: ...
async def __anext__(self) -> str: ...
async def connect_catalog(
endpoint: str,
python/python/lancedb/catalog.py
+91 -19
View File
@@ -3,9 +3,8 @@
"""Remote catalogs manage databases through a server's root namespace."""
from dataclasses import dataclass
from datetime import timedelta
from typing import Any, Optional, Union
from typing import Any, AsyncIterator, Iterator, Optional, Union
from . import _lancedb
from .background_loop import LOOP
@@ -14,12 +13,66 @@ from .remote import ClientConfig, OAuthConfig
from .remote.db import RemoteDBConnection
@dataclass
class ListDatabasesResponse:
"""A page of database names and an optional continuation token."""
class AsyncDatabaseNames(AsyncIterator[str]):
"""An async iterator of database names with pagination state.
databases: list[str]
page_token: Optional[str] = None
Returned by [list_databases][lancedb.catalog.AsyncCatalog.list_databases].
"""
def __init__(self, inner: _lancedb.DatabaseNames):
self._inner = inner
def __aiter__(self) -> "AsyncDatabaseNames":
return self
async def __anext__(self) -> str:
return await self._inner.__anext__()
def num_page_results(self) -> int:
"""Return the number of names available without another REST request."""
return self._inner.num_page_results()
def page_token(self) -> Optional[str]:
"""Return the token for the next REST request.
Before the first request, this is the supplied starting token (``None``
starts at the beginning). After the final page is fetched, it is ``None``,
even if names remain cached. Drain the cache before saving a token to
avoid skipping those names when resuming. A failed request terminates
iteration and leaves its token available for resuming a new iterator.
"""
return self._inner.page_token()
class DatabaseNames(Iterator[str]):
"""A synchronous iterator of database names with pagination state.
Returned by [Catalog.list_databases][lancedb.catalog.Catalog.list_databases].
"""
def __init__(self, inner: AsyncDatabaseNames):
self._inner = inner
def __iter__(self) -> "DatabaseNames":
return self
def __next__(self) -> str:
try:
return LOOP.run(self._inner.__anext__())
except StopAsyncIteration:
raise StopIteration from None
def num_page_results(self) -> int:
"""Return the number of names available without another REST request."""
return self._inner.num_page_results()
def page_token(self) -> Optional[str]:
"""Return the token for the next REST request.
See [page_token][lancedb.catalog.AsyncDatabaseNames.page_token]
for initial, cached, final-page, and error behavior.
"""
return self._inner.page_token()
class AsyncCatalog:
@@ -50,14 +103,25 @@ class AsyncCatalog:
"""Connect to an existing database by its logical name."""
return AsyncConnection(await self._inner.connect_database(name))
async def list_databases(
self, *, limit: Optional[int] = None, page_token: Optional[str] = None
) -> ListDatabasesResponse:
"""List a page of databases. Pass the returned token for the next page."""
names, token = await self._inner.list_databases(
limit=limit, page_token=page_token
def list_databases(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> AsyncDatabaseNames:
"""Iterate lazily over all database names using ``async for``.
``page_token`` resumes from a saved token; ``None`` starts at the beginning.
``page_limit`` sets the maximum names per REST response (not the total);
``None`` uses the server default. Request errors are raised during iteration.
Examples
--------
```python
async for name in catalog.list_databases():
print(name)
```
"""
return AsyncDatabaseNames(
self._inner.list_databases(page_token=page_token, page_limit=page_limit)
)
return ListDatabasesResponse(names, token)
async def drop_database(self, name: str, *, ignore_missing: bool = False) -> None:
"""Drop an empty database. A nonempty database is an error."""
@@ -73,7 +137,8 @@ class Catalog:
```python
catalog = lancedb.connect_catalog("https://my-server.example", api_key="secret")
db = catalog.create_database("analytics", exist_ok=True)
page = catalog.list_databases(limit=20)
for name in catalog.list_databases():
print(name)
```
"""
@@ -118,10 +183,17 @@ class Catalog:
)
def list_databases(
self, *, limit: Optional[int] = None, page_token: Optional[str] = None
) -> ListDatabasesResponse:
"""List a page of databases. Pass the returned token for the next page."""
return LOOP.run(self._inner.list_databases(limit=limit, page_token=page_token))
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
) -> DatabaseNames:
"""Iterate lazily over database names, fetching more results as needed.
``page_token`` resumes from a saved token; ``None`` starts at the beginning.
``page_limit`` sets the maximum names per REST response (not the total);
``None`` uses the server default. Request errors are raised during iteration.
"""
return DatabaseNames(
self._inner.list_databases(page_token=page_token, page_limit=page_limit)
)
def drop_database(self, name: str, *, ignore_missing: bool = False) -> None:
"""Drop an empty database. A nonempty database is an error."""
python/python/lancedb/db.py
+59 -35
View File
@@ -57,7 +57,6 @@ from .materialized_view import (
AsyncMaterializedView,
MaterializedView,
MaterializedViewSource,
VectorDedupSource,
SelectArg,
normalize_select,
)
@@ -122,7 +121,48 @@ def _view_description(
class DBConnection(EnforceOverrides):
"""An active LanceDB connection interface."""
"""An active LanceDB connection interface.
Use [close][lancedb.db.DBConnection.close] to release the connection's
underlying resources, or use the connection as a context manager to close it
automatically when leaving the block, including when an exception is raised.
Examples
--------
>>> import lancedb
>>> with lancedb.connect("memory://") as db:
... assert db.is_open()
>>> db.is_open()
False
"""
def __enter__(self) -> DBConnection:
return self
def __exit__(self, *_) -> None:
self.close()
@abstractmethod
def is_open(self) -> bool:
"""Return True if the connection is open."""
pass
@abstractmethod
def close(self) -> None:
"""Close the connection, releasing any underlying resources.
It is safe to call this method multiple times. Database operations on a
closed connection raise ``RuntimeError: Connection is closed``.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("memory://")
>>> db.close()
>>> db.is_open()
False
"""
pass
def list_namespaces(
self,
@@ -368,9 +408,10 @@ class DBConnection(EnforceOverrides):
- [LanceModel][lancedb.pydantic.LanceModel]
mode: str; default "create"
The mode to use when creating the table.
Can be either "create" or "overwrite".
Can be "create", "overwrite", or "exist_ok".
By default, if the table already exists, an exception is raised.
If you want to overwrite the table, use mode="overwrite".
To open an existing table without adding data, use mode="exist_ok".
exist_ok: bool, default False
If a table by the same name already exists, then raise an exception
if exist_ok=False. If exist_ok=True, then open the existing table;
@@ -571,21 +612,12 @@ class DBConnection(EnforceOverrides):
table. The view is a normal table: it can be queried, indexed and
searched, and it appears in ``table_names``.
Ordinary source tables must have stable row ids (create them with the
``new_table_enable_stable_row_ids`` storage option): they keep the
view's provenance valid across source compactions, and cannot be
enabled after a table exists. Native dedup sources also support physical
row IDs interpreted against their pinned snapshot.
Parameters
----------
name: str
The name of the view.
source: str or VectorDedupSource
The name of the source table, or an indexed dedup source declared
with [vector_dedup][lancedb.vector_dedup]. Dedup sources capture a
fixed snapshot and preserve the source table. They accept no
additional select, where or limit options.
source: str
The name of the source table.
select: list or dict, optional
The view's columns: column names, ``(alias, SQL expression)``
pairs, or a dict of the same. Omitting it selects every source
@@ -1275,6 +1307,14 @@ class LanceDBConnection(DBConnection):
def __repr__(self) -> str:
return f"{self.__class__.__name__}(uri={self._conn.uri!r})"
@override
def is_open(self) -> bool:
return self._conn.is_open()
@override
def close(self) -> None:
self._conn.close()
@override
def serialize(self) -> str:
import json
@@ -1497,8 +1537,6 @@ class LanceDBConnection(DBConnection):
"""
if namespace_path is None:
namespace_path = []
if mode.lower() not in ["create", "overwrite"]:
raise ValueError("mode must be either 'create' or 'overwrite'")
validate_table_name(name)
tbl = LanceTable.create(
@@ -1606,10 +1644,7 @@ class LanceDBConnection(DBConnection):
Examples
--------
>>> import lancedb
>>> db = lancedb.connect(
... "./.lancedb",
... storage_options={"new_table_enable_stable_row_ids": "true"},
... )
>>> db = lancedb.connect("./.lancedb")
>>> data = [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]
>>> table = db.create_table("people", data)
>>> view = db.create_materialized_view(
@@ -2301,11 +2336,12 @@ class AsyncConnection(object):
- pyarrow.Schema
- [LanceModel][lancedb.pydantic.LanceModel]
mode: Literal["create", "overwrite"]; default "create"
mode: Literal["create", "overwrite", "exist_ok"]; default "create"
The mode to use when creating the table.
Can be either "create" or "overwrite".
Can be "create", "overwrite", or "exist_ok".
By default, if the table already exists, an exception is raised.
If you want to overwrite the table, use mode="overwrite".
To open an existing table without adding data, use mode="exist_ok".
exist_ok: bool, default False
If a table by the same name already exists, then raise an exception
if exist_ok=False. If exist_ok=True, then open the existing table;
@@ -2576,19 +2612,7 @@ class AsyncConnection(object):
where: Optional[str],
limit: Optional[int],
) -> Tuple[str, Dict[str, str]]:
if not isinstance(source, VectorDedupSource):
return source, {}
if select is not None or where is not None or limit is not None:
raise ValueError(
"vector_dedup cannot be combined with select, where or limit"
)
version = source.dataset_version
if version is None:
table = await self.open_table(source.source)
version = await table.version()
return source.source, {
"vector_source_json": source._native_source_json(version)
}
return source, {}
async def create_materialized_view(
self,
python/python/lancedb/materialized_view.py
+18 -122
View File
@@ -6,11 +6,11 @@ maintained by refresh. See ``DBConnection.create_materialized_view``."""
from __future__ import annotations
import json
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union
import json
from dataclasses import dataclass
from .background_loop import LOOP
from .job import AsyncJob, Job, _typed_job
@@ -21,7 +21,6 @@ if TYPE_CHECKING:
from .table import AsyncTable, LanceTable
DEFINITION_META_KEY = b"mv.definition"
_MAX_FLOAT32 = float.fromhex("0x1.fffffep+127")
SelectArg = Union[
str,
@@ -31,104 +30,12 @@ SelectArg = Union[
]
@dataclass(frozen=True)
class VectorDedupSource:
"""A declarative source of retained original rows from an indexed snapshot.
Construct with [vector_dedup][lancedb.vector_dedup] and pass it as the
``source`` of ``create_materialized_view`` or
``create_materialized_view_async``. An omitted ``dataset_version`` is
captured once when creation is submitted, not when this object is made.
Refreshes keep that captured version. The source table is never modified.
"""
source: str
column: str
distance_threshold: float
dataset_version: Optional[int] = None
def __post_init__(self):
for name in ("source", "column"):
value = getattr(self, name)
if not isinstance(value, str) or not value:
raise ValueError(f"{name} must be a non-empty string")
if self.dataset_version is not None and (
type(self.dataset_version) is not int
or not 0 < self.dataset_version < 2**64
):
raise ValueError("dataset_version must be a positive uint64 integer")
value = self.distance_threshold
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise TypeError("distance_threshold must be a number")
try:
value = float(value)
except OverflowError as exc:
raise ValueError("distance_threshold must fit a finite float32") from exc
if not math.isfinite(value) or abs(value) > _MAX_FLOAT32:
raise ValueError("distance_threshold must fit a finite float32")
object.__setattr__(self, "distance_threshold", value)
def _native_source_json(self, version: int) -> str:
return json.dumps(
{
"kind": "Dedup",
"dataset_version": version,
"column": self.column,
"distance_threshold": repr(self.distance_threshold),
},
allow_nan=False,
)
MaterializedViewSource = str
def vector_dedup(
source: str,
*,
column: str,
distance_threshold: float,
dataset_version: Optional[int] = None,
) -> VectorDedupSource:
"""Declare an indexed dedup source for a materialized view.
The default policy keeps the smallest unassigned snapshot row ID and removes
only its direct qualifying neighbors: an A-B-C chain without an A-C edge
retains A and C. Rows with no pair, including null vectors, survive.
Only same-segment/same-IVF-partition pairs are considered; quantized distances
use reconstructed index vectors. The index must cover the selected snapshot.
``source`` is an exact table name in the connection's root namespace, just
as for ``create_materialized_view``. ``dataset_version=None`` captures the
source's version when creation is submitted. Pass an explicit positive
version to reproduce a previous snapshot. The result contains original
source columns and rows. Do not combine this source with ``select``,
``where`` or ``limit``; query the resulting table for further transforms.
Creation follows the usual MV contract: by default it waits for the result.
Use ``with_no_data=True`` to declare only, or
``create_materialized_view_async`` to get a job handle. On Cloud/Enterprise,
execution uses server-side MV registry jobs; local connections run in process.
Examples
--------
Given a connected ``db`` and an indexed ``images.phash`` column::
from lancedb import vector_dedup
view = db.create_materialized_view(
"images_clean",
vector_dedup("images", column="phash", distance_threshold=4),
)
cleaned = view.table
"""
return VectorDedupSource(source, column, distance_threshold, dataset_version)
MaterializedViewSource = Union[str, VectorDedupSource]
DEFINITION_FORMAT = 4
DEFINITION_FORMAT = 2
"""The newest stored layout this version reads: ``{"format": N, "query": "<SQL>"}``,
format 2 being a query with ``GROUP BY``, format 3 native pairs and format 4
native dedup results. A ``kind`` key beside it is for
format 2 being a query with ``GROUP BY``. A ``kind`` key beside it is for
readers older than the format number."""
@@ -272,38 +179,31 @@ class AsyncMaterializedView:
return _definition_from_json(raw)
async def refresh(
self, *, full: bool = False, source_version: Optional[int] = None
self, *, source_version: Optional[int] = None
) -> "RefreshMaterializedViewResult":
"""Recompute the view from its source.
The refresh is incremental when the source's changes can be
reconciled into the view -- rows added, changed or removed since the
last one -- and otherwise rebuilds. ``full=True`` forces a rebuild;
Every refresh rebuilds the complete view. For local views,
``source_version`` refreshes to that source version instead of the
latest.
Concurrent refreshes of one view do not duplicate its rows. Two that
plan the same source rows conflict on commit, and the loser raises
rather than writing them a second time.
latest; remote SQL refreshes do not support source-version pinning.
"""
return await self._table._inner.refresh_materialized_view(
full=full, source_version=source_version
source_version=source_version
)
async def refresh_async(
self, *, full: bool = False, source_version: Optional[int] = None
self, *, source_version: Optional[int] = None
) -> "AsyncJob[RefreshMaterializedViewResult]":
"""Submit a refresh and return its job without waiting.
The job may already be complete for a local view. On LanceDB Cloud
and Enterprise, its ``id`` is the server job identifier returned by
the refresh endpoint.
The job may already be complete for a local view. Remote refreshes are
submitted through the SQL service.
"""
from ._lancedb import RefreshMaterializedViewResult
return _typed_job(
await self._table._inner.refresh_materialized_view_async(
full=full, source_version=source_version
source_version=source_version
),
RefreshMaterializedViewResult.from_json,
)
@@ -335,22 +235,18 @@ class MaterializedView:
return LOOP.run(self._async.definition())
def refresh(
self, *, full: bool = False, source_version: Optional[int] = None
self, *, source_version: Optional[int] = None
) -> "RefreshMaterializedViewResult":
"""Recompute the view from its source. See
[AsyncMaterializedView.refresh][lancedb.materialized_view.AsyncMaterializedView.refresh]."""
return LOOP.run(self._async.refresh(full=full, source_version=source_version))
return LOOP.run(self._async.refresh(source_version=source_version))
def refresh_async(
self, *, full: bool = False, source_version: Optional[int] = None
self, *, source_version: Optional[int] = None
) -> "Job[RefreshMaterializedViewResult]":
"""Submit a refresh and return its job without waiting.
See
[AsyncMaterializedView.refresh_async][lancedb.materialized_view.AsyncMaterializedView.refresh_async].
"""
return Job(
LOOP.run(
self._async.refresh_async(full=full, source_version=source_version)
)
)
return Job(LOOP.run(self._async.refresh_async(source_version=source_version)))
python/python/lancedb/namespace.py
+8
View File
@@ -489,6 +489,14 @@ class LanceNamespaceDBConnection(DBConnection):
)
self._uri = self._inner.uri
@override
def is_open(self) -> bool:
return self._inner.is_open()
@override
def close(self) -> None:
self._inner.close()
@override
def serialize(self) -> str:
import json
python/python/lancedb/query.py
+13 -3
View File
@@ -40,7 +40,7 @@ from .rerankers.base import Reranker
from .rerankers.rrf import RRFReranker
from .rerankers.util import check_reranker_result
from .schema import is_blob_like_field, schema_has_blob_field
from .util import flatten_columns
from .util import _validate_query_vector, flatten_columns
from . import _wal_hybrid # WAL-PK-FUSION: delete.
from ._blob import (
BLOB_MODE_TO_HANDLING,
@@ -907,6 +907,9 @@ class LanceQueryBuilder(ABC):
fast_search: bool
Skip flat search of unindexed data.
"""
if query_type != "fts":
_validate_query_vector(query)
if ordering_field_name is not None:
import warnings
@@ -1619,6 +1622,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
str_query: Optional[str] = None,
fast_search: bool = None,
):
_validate_query_vector(query)
super().__init__(table)
self._query = query
self._distance_type = None
@@ -1693,13 +1697,15 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
Parameters
----------
nprobes: int
The number of probes to use.
The number of probes to use. Must be greater than 0.
Returns
-------
LanceVectorQueryBuilder
The LanceQueryBuilder object.
"""
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
return self
@@ -2495,13 +2501,15 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
Parameters
----------
nprobes: int
The number of probes to use.
The number of probes to use. Must be greater than 0.
Returns
-------
LanceHybridQueryBuilder
The LanceHybridQueryBuilder object.
"""
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
return self
@@ -3654,6 +3662,8 @@ class AsyncVectorQueryBase:
"""
Set the number of partitions to search (probe)
The number of probes must be greater than 0.
This argument is only used when the vector column has an IVF-based index.
If there is no index then this value is ignored.
python/python/lancedb/remote/db.py
+6 -2
View File
@@ -1054,6 +1054,10 @@ class RemoteDBConnection(DBConnection):
"""
return LOOP.run(self._conn.namespace_client())
async def close(self):
"""Close the connection to the database."""
@override
def is_open(self) -> bool:
return self._conn.is_open()
@override
def close(self) -> None:
self._conn.close()
python/python/lancedb/table.py
+39 -17
View File
@@ -2617,6 +2617,10 @@ class Table(ABC):
to nullable. Currently, you cannot change a nullable column to
non-nullable.
Each alteration must specify at least one of "rename", "data_type",
or "nullable". The legacy key "name" is also accepted for renaming.
Unknown keys raise ValueError before any alterations are applied.
Returns
-------
AlterColumnsResult
@@ -4332,9 +4336,9 @@ class LanceTable(Table):
)
if storage_options is None:
storage_options = {}
storage_options["new_table_enable_v2_manifest_paths"] = (
storage_options["new_table_enable_v2_manifest_paths"] = str(
enable_v2_manifest_paths
)
).lower()
self._table = LOOP.run(
self._conn._conn.create_table(
@@ -4766,7 +4770,7 @@ class LanceTable(Table):
[LanceTable.uses_v2_manifest_paths][lancedb.table.LanceTable.uses_v2_manifest_paths]
to check if the table is already using the new path style.
"""
LOOP.run(self._table.migrate_v2_manifest_paths())
LOOP.run(self._table.migrate_manifest_paths_v2())
@deprecation.deprecated(
deprecated_in="0.33.1",
@@ -6107,7 +6111,17 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[str] = None,
query: None = None,
vector_column_name: Optional[str] = None,
query_type: QueryType = "auto",
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
) -> AsyncQuery: ...
@overload
async def search(
self,
query: str,
vector_column_name: Optional[str] = None,
query_type: Literal["auto"] = ...,
ordering_field_name: Optional[str] = None,
@@ -6117,7 +6131,7 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[str] = None,
query: str,
vector_column_name: Optional[str] = None,
query_type: Literal["hybrid"] = ...,
ordering_field_name: Optional[str] = None,
@@ -6127,7 +6141,7 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[Union[VEC, "PIL.Image.Image", Tuple]] = None,
query: Union[VEC, "PIL.Image.Image", Tuple],
vector_column_name: Optional[str] = None,
query_type: Literal["auto"] = ...,
ordering_field_name: Optional[str] = None,
@@ -6137,7 +6151,7 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[str] = None,
query: str,
vector_column_name: Optional[str] = None,
query_type: Literal["fts"] = ...,
ordering_field_name: Optional[str] = None,
@@ -6147,9 +6161,7 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[
Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery]
] = None,
query: Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery],
vector_column_name: Optional[str] = None,
query_type: Literal["vector"] = ...,
ordering_field_name: Optional[str] = None,
@@ -6165,7 +6177,7 @@ class AsyncTable:
query_type: QueryType = "auto",
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
) -> Union[AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]:
) -> Union[AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]:
"""Create a search query to find the nearest neighbors
of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/)
and [full-text search](https://lancedb.com/docs/search/full-text-search/).
@@ -6180,8 +6192,9 @@ class AsyncTable:
- *default None*.
Acceptable types are: list, np.ndarray, PIL.Image.Image
- If None then the select/where/limit clauses are applied to filter
the table
- If None then a plain [AsyncQuery][lancedb.query.AsyncQuery] is
returned, equivalent to calling [query][lancedb.table.AsyncTable.query].
The select/where/limit clauses are applied to filter the table.
vector_column_name: str, optional
The name of the vector column to search.
@@ -6209,10 +6222,13 @@ class AsyncTable:
Returns
-------
LanceQueryBuilder
AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, or AsyncVectorQuery
A query builder object representing the query.
"""
if query is None:
return self.query()
def is_embedding(query):
return isinstance(query, (list, np.ndarray, pa.Array, pa.ChunkedArray))
@@ -6862,6 +6878,10 @@ class AsyncTable:
to nullable. Currently, you cannot change a nullable column to
non-nullable.
Each alteration must specify at least one of "rename", "data_type",
or "nullable". The legacy key "name" is also accepted for renaming.
Unknown keys raise ValueError before any alterations are applied.
Returns
-------
AlterColumnsResult
@@ -6905,9 +6925,11 @@ class AsyncTable:
"""
versions = await self._inner.list_versions()
for v in versions:
ts_nanos = v["timestamp"]
v["timestamp"] = datetime.fromtimestamp(ts_nanos // 1e9) + timedelta(
microseconds=(ts_nanos % 1e9) // 1e3
# Use integer math: float division on ~1e18 nanosecond
# values loses sub-millisecond precision.
seconds, nanos = divmod(v["timestamp"], 1_000_000_000)
v["timestamp"] = datetime.fromtimestamp(seconds) + timedelta(
microseconds=nanos // 1000
)
return versions
python/python/lancedb/util.py
+11
View File
@@ -280,6 +280,14 @@ def infer_vector_column_dim(data_type: pa.DataType) -> Optional[int]:
return None
def _validate_query_vector(query: Any) -> None:
"""Reject empty vector inputs before vector-column inference or execution."""
if (isinstance(query, list) and not query) or (
isinstance(query, np.ndarray) and query.size == 0
):
raise ValueError("Query vector must not be empty")
def _query_vector_dim(query: Optional[Any]) -> Optional[int]:
if query is None:
return None
@@ -301,6 +309,9 @@ def infer_vector_column_name(
query: Optional[Any], # inferred later in query builder
vector_column_name: Optional[str],
):
if query_type != "fts":
_validate_query_vector(query)
if vector_column_name is not None:
return vector_column_name
python/python/tests/test_catalog.py
+129 -4
View File
@@ -55,7 +55,7 @@ def test_catalog_sync_scope_and_serialization(catalog_server):
(200, {}),
(200, {"tables": []}),
(200, {"tables": []}),
(200, {"namespaces": ["team/search"], "page_token": "next"}),
(200, {"namespaces": ["team/search"]}),
(204, None),
]
)
@@ -85,13 +85,12 @@ def test_catalog_sync_scope_and_serialization(catalog_server):
with pytest.raises(ValueError, match="sql_host_override must use"):
connection.execute_query_async("SELECT 1")
assert restored.table_names() == []
page = catalog.list_databases(limit=1, page_token="a/b")
assert page == lancedb.ListDatabasesResponse(["team/search"], "next")
assert list(catalog.list_databases()) == ["team/search"]
catalog.drop_database("team/search", ignore_missing=True)
assert requests[0][0] == "/v1/namespace/team%2Fsearch/create"
assert requests[0][2] == {"mode": "ExistOk"}
assert requests[1][0] == "/v1/namespace/team%2Fsearch/describe"
assert requests[4][0] == "/v1/namespace/$/list?limit=1&page_token=a%2Fb"
assert requests[4][0] == "/v1/namespace/$/list"
assert requests[5][2] == {"mode": "Skip", "behavior": "Restrict"}
for i, (_, headers, _) in enumerate(requests):
headers = {key.lower(): value for key, value in headers.items()}
@@ -138,3 +137,129 @@ async def test_catalog_async_and_errors(catalog_server):
def test_catalog_requires_remote_endpoint(endpoint):
with pytest.raises(ValueError, match="endpoint"):
lancedb.connect_catalog(endpoint)
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.asyncio
@pytest.mark.parametrize("page_token,page_limit", [(None, None), ("start/token", 2)])
async def test_database_names_iteration(
catalog_server, asynchronous, page_token, page_limit
):
endpoint, requests, responses = catalog_server
responses.extend(
[
(200, {"namespaces": ["first", "second"], "page_token": "a/b"}),
(200, {"namespaces": [], "page_token": "next"}),
(200, {"namespaces": ["last", "final"], "page_token": ""}),
]
)
catalog = (
await lancedb.connect_catalog_async(endpoint)
if asynchronous
else lancedb.connect_catalog(endpoint)
)
names = catalog.list_databases(page_token=page_token, page_limit=page_limit)
assert requests == []
assert names.num_page_results() == 0
assert names.page_token() == page_token
for cached, expected in [(1, "first"), (0, "second")]:
assert (await names.__anext__() if asynchronous else next(names)) == expected
assert names.num_page_results() == cached
assert names.page_token() == "a/b"
assert len(requests) == 1
assert (await names.__anext__() if asynchronous else next(names)) == "last"
assert names.num_page_results() == 1
assert names.page_token() is None
assert (await names.__anext__() if asynchronous else next(names)) == "final"
assert names.num_page_results() == 0
assert names.page_token() is None
for _ in range(2):
with pytest.raises(StopAsyncIteration if asynchronous else StopIteration):
if asynchronous:
await names.__anext__()
else:
next(names)
from urllib.parse import parse_qs, urlsplit
for request, token in zip(requests, [page_token, "a/b", "next"]):
url = urlsplit(request[0])
assert url.path == "/v1/namespace/$/list"
expected = {}
if token is not None:
expected["page_token"] = [token]
if page_limit is not None:
expected["limit"] = [str(page_limit)]
assert parse_qs(url.query) == expected
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("fail", [False, True])
@pytest.mark.asyncio
async def test_database_names_empty_or_error(catalog_server, asynchronous, fail):
endpoint, requests, responses = catalog_server
responses.append(
(401, {"error": "unauthorized"}) if fail else (200, {"namespaces": []})
)
catalog = (
await lancedb.connect_catalog_async(endpoint)
if asynchronous
else lancedb.connect_catalog(endpoint)
)
names = catalog.list_databases()
if fail:
with pytest.raises(HttpError):
if asynchronous:
await names.__anext__()
else:
next(names)
assert ([name async for name in names] if asynchronous else list(names)) == []
assert len(requests) == 1
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.asyncio
async def test_database_names_resume(catalog_server, asynchronous):
endpoint, requests, responses = catalog_server
responses.extend(
[
(200, {"namespaces": ["first"], "page_token": "resume/token"}),
(200, {"namespaces": ["second"]}),
]
)
catalog = (
await lancedb.connect_catalog_async(endpoint)
if asynchronous
else lancedb.connect_catalog(endpoint)
)
names = catalog.list_databases(page_limit=1)
assert (await names.__anext__() if asynchronous else next(names)) == "first"
assert names.num_page_results() == 0
resumed = catalog.list_databases(page_token=names.page_token(), page_limit=1)
assert resumed.page_token() == "resume/token"
assert len(requests) == 1
assert ([name async for name in resumed] if asynchronous else list(resumed)) == [
"second"
]
assert resumed.page_token() is None
assert resumed.num_page_results() == 0
assert requests[1][0] == "/v1/namespace/$/list?limit=1&page_token=resume%2Ftoken"
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("page_limit", [0, 2**32 - 1])
@pytest.mark.asyncio
async def test_database_names_invalid_limit(catalog_server, asynchronous, page_limit):
endpoint, requests, _ = catalog_server
catalog = (
await lancedb.connect_catalog_async(endpoint)
if asynchronous
else lancedb.connect_catalog(endpoint)
)
names = catalog.list_databases(page_limit=page_limit)
with pytest.raises(ValueError, match="limit"):
if asynchronous:
await names.__anext__()
else:
next(names)
assert names.num_page_results() == 0
assert requests == []
python/python/tests/test_db.py
+99
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import contextlib
import inspect
import re
import sys
@@ -350,6 +351,40 @@ async def test_create_table_from_iterator_async(mem_db_async: lancedb.AsyncConne
assert await table.count_rows() == 10
def test_create_exist_ok_mode(tmp_db: lancedb.DBConnection):
db = tmp_db
data = [{"vector": [1.1, 1.2]}, {"vector": [0.2, 1.8]}]
table = db.create_table("t1", data, mode="exist_ok")
assert table.count_rows() == 2
original = table.to_arrow()
reopened = db.create_table("t1", [{"vector": [9.0, 8.0]}], mode="exist_ok")
assert reopened.to_arrow().equals(original)
def test_create_empty_exist_ok_mode(tmp_db: lancedb.DBConnection):
schema = pa.schema([pa.field("id", pa.int64())])
table = tmp_db.create_table("empty", schema=schema, mode="exist_ok")
assert table.schema == schema
assert table.count_rows() == 0
table.add([{"id": 7}])
reopened = tmp_db.create_table("empty", schema=schema, mode="exist_ok")
assert reopened.to_arrow().to_pylist() == [{"id": 7}]
bad_schema = pa.schema([pa.field("other", pa.int64())])
with pytest.raises(ValueError):
tmp_db.create_table("empty", schema=bad_schema, mode="exist_ok")
def test_create_table_invalid_mode(tmp_db: lancedb.DBConnection):
db = tmp_db
data = [{"vector": [1.1, 1.2]}, {"vector": [0.2, 1.8]}]
with pytest.raises(ValueError, match="Invalid mode bogus"):
db.create_table("tb", data, mode="bogus")
assert "tb" not in db
def test_create_exist_ok(tmp_db: lancedb.DBConnection):
from conftest import pandas_string_type
@@ -405,6 +440,49 @@ async def test_connect(tmp_path):
assert str(db) == f"ListingDatabase(uri={tmp_path}, read_consistency_interval=5s)"
@pytest.fixture(params=["memory", "local", "namespace", "from_inner"])
def sync_db(request, tmp_path):
if request.param == "memory":
return lancedb.connect("memory://")
if request.param == "namespace":
return lancedb.connect_namespace("dir", {"root": str(tmp_path)})
db = lancedb.connect(tmp_path)
if request.param == "from_inner":
return lancedb.db.LanceDBConnection.from_inner(db._inner, None)
return db
def test_sync_close(sync_db):
assert sync_db.is_open()
assert sync_db.close() is None
assert not sync_db.is_open()
assert sync_db.close() is None
with pytest.warns(DeprecationWarning, match="table_names"):
with pytest.raises(RuntimeError, match="Connection is closed"):
sync_db.table_names()
with pytest.raises(RuntimeError, match="Connection is closed"):
sync_db.list_tables()
with pytest.raises(RuntimeError, match="Connection is closed"):
sync_db.open_table("test")
@pytest.mark.parametrize("raise_error", [False, True])
def test_sync_context_manager(sync_db, raise_error):
with contextlib.ExitStack() as stack:
if raise_error:
stack.enter_context(pytest.raises(ValueError, match="test error"))
with sync_db as db:
assert db is sync_db
assert db.is_open()
if raise_error:
raise ValueError("test error")
assert not sync_db.is_open()
with pytest.raises(RuntimeError, match="Connection is closed"):
sync_db.list_tables()
@pytest.mark.asyncio
async def test_close(mem_db_async: lancedb.AsyncConnection):
assert mem_db_async.is_open()
@@ -495,6 +573,27 @@ async def test_create_exist_ok_async(tmp_db_async: lancedb.AsyncConnection):
# await db.create_table("test", schema=bad_schema, exist_ok=True)
@pytest.mark.parametrize("enable_v2_manifest_paths", [False, True])
@pytest.mark.parametrize("empty", [False, True])
def test_create_table_deprecated_v2_manifest_paths(
tmp_path, enable_v2_manifest_paths, empty
):
db = lancedb.connect(tmp_path)
data = None if empty else [{"id": 1}]
with pytest.warns(DeprecationWarning, match="enable_v2_manifest_paths"):
table = db.create_table(
"test",
data=data,
schema=pa.schema([("id", pa.int64())]),
enable_v2_manifest_paths=enable_v2_manifest_paths,
)
assert table.uses_v2_manifest_paths() == enable_v2_manifest_paths
assert table.to_arrow().to_pylist() == ([] if empty else data)
assert db.open_table("test").uses_v2_manifest_paths() == enable_v2_manifest_paths
@pytest.mark.asyncio
async def test_create_table_v2_manifest_paths_async(tmp_path):
db_with_v2_paths = await lancedb.connect_async(
python/python/tests/test_lsm_write_spec.py
+7 -7
View File
@@ -52,9 +52,9 @@ def test_set_lsm_write_spec_validates(tmp_path):
with pytest.raises(Exception, match="num_buckets"):
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1025))
# Happy path then mutation rejected.
# Happy path, then a set over the installed spec is refused.
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4))
with pytest.raises(Exception, match="mutation"):
with pytest.raises(Exception, match="already set"):
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8))
@@ -176,18 +176,18 @@ def test_get_lsm_write_spec(tmp_path):
table.unset_lsm_write_spec()
assert table.get_lsm_write_spec() is None
# Identity round-trips (column recovered from the schema). Leaving the
# maintained set to be inferred picks up the index on the table, so the
# spec reads back naming it rather than as "infer".
# Identity round-trips (column recovered from the schema). An unnamed
# maintained set reads back as None: what is stored is the intent to
# maintain every index, not the set it resolves to now.
table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
spec = table.get_lsm_write_spec()
assert spec.spec_type == "identity"
assert spec.column == "id"
assert spec.maintained_indexes == [idx_name]
assert spec.maintained_indexes is None
table.unset_lsm_write_spec()
# Unsharded round-trips (no routing column). Opting out is distinct from
# the inferred default.
# the unnamed default.
table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
spec = table.get_lsm_write_spec()
assert spec.spec_type == "unsharded"
python/python/tests/test_materialized_views.py
+17 -191
View File
@@ -97,24 +97,6 @@ def test_remote_list_uses_namespace_route():
assert requests == ["/v1/namespace/$/materialized_view/list"]
def test_remote_create_async_returns_server_job():
with mock_remote_materialized_view_create() as (host, requests):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
job = db.create_materialized_view_async("adults", "people", where="age >= 18")
assert job.id == "mv-create-123"
assert requests == [
(
"/v1/materialized_view/adults/create",
{"query": 'SELECT * FROM "people" WHERE age >= 18', "with_no_data": False},
)
]
def test_remote_drop_async_returns_server_job():
with mock_remote_materialized_view_create() as (host, requests):
db = lancedb.connect(
@@ -221,29 +203,29 @@ def test_definition_round_trips(tmp_path):
)
def test_incremental_refresh_after_append(tmp_path):
def test_full_refresh_after_append(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view("copy", "people", with_no_data=True)
view.refresh()
db.open_table("people").add([{"name": "alan", "age": 41}])
result = view.refresh()
assert result.mode == "incremental"
assert result.rows_written == 1
assert result.mode == "rebuild"
assert result.rows_written == 4
assert view.table.count_rows() == 4
assert view.refresh().mode == "no_op"
assert view.refresh().mode == "rebuild"
def test_incremental_refresh_after_update(tmp_path):
def test_full_refresh_after_update(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view("copy", "people", with_no_data=True)
view.refresh()
db.open_table("people").update(where="name = 'kid'", values={"age": 8})
result = view.refresh()
assert result.mode == "incremental"
assert result.rows_written == 1
assert result.mode == "rebuild"
assert result.rows_written == 3
rows = view.table.search().to_list()
assert sorted(row["age"] for row in rows) == [8, 36, 85]
@@ -331,8 +313,8 @@ async def test_async_create_and_refresh_jobs(tmp_path):
@pytest.mark.asyncio
async def test_async_incremental(tmp_path):
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
async def test_async_refresh_rebuilds(tmp_path):
db = await lancedb.connect_async(tmp_path)
await db.create_table("people", [{"name": "ada", "age": 36}])
view = await db.create_materialized_view("copy", "people", with_no_data=True)
await view.refresh()
@@ -340,15 +322,15 @@ async def test_async_incremental(tmp_path):
table = await db.open_table("people")
await table.add([{"name": "alan", "age": 41}])
result = await view.refresh()
assert result.mode == "incremental"
assert result.rows_written == 1
assert result.mode == "rebuild"
assert result.rows_written == 2
def test_source_requires_stable_row_ids(tmp_path):
def test_source_does_not_require_stable_row_ids(tmp_path):
db = lancedb.connect(tmp_path)
db.create_table("plain", [{"x": 1}])
with pytest.raises(Exception, match="stable row ids"):
db.create_materialized_view("v", "plain")
view = db.create_materialized_view("v", "plain")
assert view.table.count_rows() == 1
def test_bare_select_names_are_quoted(tmp_path):
@@ -373,7 +355,7 @@ def test_scalar_select_is_one_column(tmp_path):
view = db.create_materialized_view("just_name", "people", select="name")
view.refresh()
rows = view.table.search().to_list()
assert set(rows[0]) - {"__source_row_id"} == {"name"}
assert set(rows[0]) == {"name"}
assert sorted(row["name"] for row in rows) == ["ada", "grace", "kid"]
@@ -384,7 +366,7 @@ async def test_async_scalar_select_is_one_column(tmp_path):
view = await db.create_materialized_view("just_name", "people", select="name")
await view.refresh()
rows = await view.table.query().to_list()
assert set(rows[0]) - {"__source_row_id"} == {"name"}
assert set(rows[0]) == {"name"}
def test_limit_above_i64_max_is_refused(tmp_path):
@@ -520,166 +502,10 @@ def test_stored_queries_and_legacy_layouts_are_read():
"SELECT * FROM people"
)
for fmt, function in [(3, "vector_duplicate_pairs"), (4, "vector_dedup")]:
native = f"SELECT * FROM {function}('images', 3, 'phash', 4)"
assert read({"format": fmt, "query": native}).query == native
# A newer writer's layout is reported, never guessed at.
for newer in (
{"format": 5, "query": query},
{"format": 3, "query": query},
{"kind": "select_v3", "source_table": "people"},
):
with pytest.raises(NotImplementedError, match="cannot refresh"):
read(newer)
def indexed_dedup_source(db, name="images"):
import pyarrow as pa
from lancedb.index import IvfFlat
vectors = pa.FixedSizeListArray.from_arrays(
pa.array(
[v for i in range(256) for v in (float(i if i < 3 else i * 10), 0.0)],
type=pa.float32(),
),
2,
)
table = db.create_table(name, pa.table({"id": range(256), "vector": vectors}))
table.create_index("vector", config=IvfFlat(num_partitions=1))
return table
@pytest.mark.parametrize("stable", [False, True])
def test_vector_dedup_source_captures_snapshot_and_preserves_source(tmp_path, stable):
db = lancedb.connect(
tmp_path,
storage_options={"new_table_enable_stable_row_ids": str(stable).lower()},
)
source = indexed_dedup_source(db, "images.with.dot")
version = source.version
expression = lancedb.vector_dedup(
source.name, column="vector", distance_threshold=1
)
view = db.create_materialized_view("clean", expression)
assert "vector_dedup(" in view.definition.query
assert f", {version}, 'vector', 1.0)" in view.definition.query
assert view.table.count_rows() == 255
assert [r["id"] for r in view.table.search().where("id < 3").to_list()] == [0, 2]
assert source.version == version
assert source.count_rows() == 256
source.delete("id = 0")
view.refresh_async().wait()
assert view.table.search().where("id = 0").to_list() == [
{"id": 0, "vector": [0.0, 0.0]}
]
assert expression.dataset_version is None # reusable immutable declaration
def test_vector_dedup_create_job_and_explicit_snapshot(tmp_path):
db = lancedb.connect(tmp_path)
source = indexed_dedup_source(db)
version = source.version
source.delete("id = 0")
job = db.create_materialized_view_async(
"clean",
lancedb.vector_dedup(
"images", column="vector", distance_threshold=1, dataset_version=version
),
)
assert job.wait() is None
view = db.open_materialized_view("clean")
assert view.table.count_rows() == 255
assert len(view.table.search().where("id = 0").to_list()) == 1
@pytest.mark.asyncio
async def test_async_vector_dedup_declaration_and_creation(tmp_path):
sync_db = lancedb.connect(tmp_path)
source = indexed_dedup_source(sync_db)
db = await lancedb.connect_async(tmp_path)
expression = lancedb.vector_dedup("images", column="vector", distance_threshold=1)
view = await db.create_materialized_view("clean", expression, with_no_data=True)
assert await view.table.count_rows() == 0
# The snapshot is captured at declaration even if refresh comes later.
source.delete("id = 0")
job = await view.refresh_async()
await job.wait()
assert await view.table.count_rows() == 255
another = await db.create_materialized_view_async(
"later", expression, with_no_data=True
)
await another.wait()
later = await db.open_materialized_view("later")
assert (await later.definition()).query != (await view.definition()).query
assert await later.table.count_rows() == 0
@pytest.mark.parametrize(
"options", [{"select": ["id"]}, {"select": []}, {"where": "id > 0"}, {"limit": 1}]
)
def test_vector_dedup_rejects_extra_clauses_before_opening_source(tmp_path, options):
db = lancedb.connect(tmp_path)
expression = lancedb.vector_dedup("missing", column="vector", distance_threshold=1)
with pytest.raises(ValueError, match="cannot be combined"):
db.create_materialized_view("bad", expression, **options)
assert db.list_tables().tables == []
@pytest.mark.parametrize("version", [0, -1, True, 1.5, "2", 2**64])
def test_vector_dedup_rejects_invalid_snapshot(version):
with pytest.raises(ValueError, match="dataset_version"):
lancedb.vector_dedup(
"images", column="vector", distance_threshold=1, dataset_version=version
)
@pytest.mark.parametrize(
"threshold", [float("nan"), float("inf"), -float("inf"), 1e100, 10**1000, True, "4"]
)
def test_vector_dedup_rejects_invalid_threshold(threshold):
with pytest.raises((ValueError, TypeError), match="distance_threshold"):
lancedb.vector_dedup("images", column="vector", distance_threshold=threshold)
def test_vector_dedup_descriptor_is_frozen():
from dataclasses import FrozenInstanceError
expression = lancedb.vector_dedup("images", column="vector", distance_threshold=1)
with pytest.raises(FrozenInstanceError):
expression.column = "other"
@pytest.mark.parametrize("with_no_data", [False, True])
def test_remote_vector_dedup_uses_mv_endpoint_without_sql_client(with_no_data):
with mock_remote_materialized_view_create() as (host, requests):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
sql_host_override="grpc://127.0.0.1:1",
client_config={"retry_config": {"retries": 0}},
)
job = db.create_materialized_view_async(
"clean",
lancedb.vector_dedup(
"images.with'dot",
column="vector'field",
distance_threshold=4,
dataset_version=7,
),
with_no_data=with_no_data,
)
assert job.id == "mv-create-123"
assert requests == [
(
"/v1/materialized_view/clean/create",
{
"query": (
"SELECT * FROM vector_dedup('`images.with''dot`', "
"7, 'vector''field', 4.0)"
),
"with_no_data": with_no_data,
},
)
]
python/python/tests/test_query.py
+53
View File
@@ -24,6 +24,7 @@ from lancedb.pydantic import LanceModel, Vector
from lancedb.query import (
AsyncFTSQuery,
AsyncHybridQuery,
AsyncQuery,
AsyncQueryBase,
AsyncVectorQuery,
ColumnOrdering,
@@ -974,6 +975,19 @@ def test_query_builder_with_filter(table):
assert all(np.array(rs[0]["vector"]) == [3, 4])
@pytest.mark.parametrize("query_type", ["vector", "hybrid"])
@pytest.mark.parametrize("nprobes", [0, -1])
def test_nprobes_nonpositive_sync(table, query_type, nprobes):
if query_type == "hybrid":
query = table.search(query_type="hybrid").vector([0, 0]).text("a")
else:
query = table.search([0, 0])
with pytest.raises(
ValueError, match="^Invalid input, nprobes must be greater than 0$"
):
query.nprobes(nprobes)
def test_invalid_nprobes_sync(table):
with pytest.raises(ValueError, match="minimum_nprobes must be greater than 0"):
LanceVectorQueryBuilder(table, [0, 0], "vector").minimum_nprobes(0).to_list()
@@ -1005,6 +1019,18 @@ def test_multiple_nprobes_calls_works_sync(table):
).minimum_nprobes(20).to_list()
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
async def test_nprobes_zero_async(table_async: AsyncTable, hybrid):
query = table_async.query().nearest_to([0, 0])
if hybrid:
query = query.nearest_to_text("dog")
with pytest.raises(
ValueError, match="^Invalid input, nprobes must be greater than 0$"
):
query.nprobes(0)
@pytest.mark.asyncio
async def test_invalid_nprobes_async(table_async: AsyncTable):
with pytest.raises(ValueError, match="minimum_nprobes must be greater than 0"):
@@ -1279,6 +1305,27 @@ async def test_query_to_polars_async(table_async: AsyncTable):
assert df.shape == (0, num_columns)
@pytest.mark.asyncio
@pytest.mark.parametrize("search_kwargs", [{}, {"query": None}])
@pytest.mark.parametrize("with_vector", [False, True])
async def test_async_search_without_query(mem_db_async, search_kwargs, with_vector):
data = [{"id": i} for i in range(10)]
if with_vector:
for row in data:
row["vector"] = [float(row["id"]), 1.0]
table = await mem_db_async.create_table("test", data)
query = await table.search(**search_kwargs)
assert isinstance(query, AsyncQuery)
assert await query.limit(3).to_arrow() == await table.query().limit(3).to_arrow()
query = await table.search(**search_kwargs)
assert await query.where("id >= 8").select(["id"]).limit(3).to_list() == [
{"id": 8},
{"id": 9},
]
@pytest.mark.asyncio
async def test_none_query(table_async: AsyncTable):
with pytest.raises(ValueError):
@@ -2091,6 +2138,12 @@ def test_ensure_vector_query_nested_empty_list():
ensure_vector_query([[]])
@pytest.mark.parametrize("query", [[], np.array([], dtype=np.float32)])
def test_vector_query_builder_empty_vector(table, query):
with pytest.raises(ValueError, match="^Query vector must not be empty$"):
LanceVectorQueryBuilder(table, query, "vector")
def test_fast_search(tmp_path):
db = lancedb.connect(tmp_path)
python/python/tests/test_remote_db.py
+237 -7
View File
@@ -16,8 +16,9 @@ import uuid
from packaging.version import Version
import lancedb
import numpy as np
from lancedb.conftest import MockTextEmbeddingFunction
from lancedb.query import ColumnOrdering
from lancedb.query import AsyncQuery, ColumnOrdering
from lancedb.remote import ClientConfig
from lancedb.remote.errors import HttpError, RetryError
import pytest
@@ -119,6 +120,77 @@ async def test_async_remote_db():
assert table_names == []
@pytest.mark.parametrize(
"alteration, match",
[
({"path": "id"}, "One of rename, nullable or data_type"),
(
{"path": "id", "nulable": False}, # spellchecker:disable-line
"Unknown column alteration key 'nulable'", # spellchecker:disable-line
),
(
{
"path": "id",
"rename": "new_id",
"nulable": False, # spellchecker:disable-line
},
"Unknown column alteration key 'nulable'", # spellchecker:disable-line
),
],
)
def test_remote_alter_columns_rejects_invalid_before_request(alteration, match):
requests = []
def handler(request):
requests.append(request.path)
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"version": 1, "schema": {"fields": []}}')
with mock_lancedb_connection(handler) as db:
table = db.open_table("test")
requests.clear()
with pytest.raises(ValueError, match=match):
table.alter_columns(alteration)
assert requests == []
@pytest.mark.asyncio
@pytest.mark.parametrize(
"alteration, match",
[
({"path": "id"}, "One of rename, nullable or data_type"),
(
{"path": "id", "nulable": False}, # spellchecker:disable-line
"Unknown column alteration key 'nulable'", # spellchecker:disable-line
),
],
)
async def test_async_remote_alter_columns_rejects_invalid_before_request(
alteration, match
):
requests = []
def handler(request):
requests.append(request.path)
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"version": 1, "schema": {"fields": []}}')
async with mock_lancedb_connection_async(handler) as db:
table = await db.open_table("test")
requests.clear()
with pytest.raises(ValueError, match=match):
await table.alter_columns(alteration)
assert requests == []
@pytest.mark.asyncio
async def test_async_checkout():
def handler(request):
@@ -1521,6 +1593,60 @@ def test_query_sync_empty_query():
assert data == expected
@pytest.mark.parametrize("query", [[], np.array([], dtype=np.float32)])
@pytest.mark.parametrize("vector_column_name", [None, "vector"])
@pytest.mark.parametrize("query_type", ["auto", "vector"])
def test_query_sync_empty_vector(query, vector_column_name, query_type):
def handler(body):
pytest.fail("An empty query vector must be rejected before sending a query")
with query_test_table(handler) as table:
with pytest.raises(ValueError, match="^Query vector must not be empty$"):
table.search(
query, vector_column_name=vector_column_name, query_type=query_type
).limit(3).to_arrow()
@pytest.mark.asyncio
@pytest.mark.parametrize("search_kwargs", [{}, {"query": None}])
async def test_async_search_without_query(search_kwargs):
expected = pa.table({"id": [7, 8, 9]})
def handler(request):
if request.path == "/v1/table/test/describe/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"version": 1, "schema": {"fields": []}}')
elif request.path == "/v1/table/test/query/":
body = json.loads(
request.rfile.read(int(request.headers["Content-Length"]))
)
assert body == {
"k": 3,
"filter": "id >= 7",
"vector": [],
"columns": ["id"],
"prefilter": True,
"version": None,
}
request.send_response(200)
request.send_header("Content-Type", "application/vnd.apache.arrow.file")
request.end_headers()
with pa.ipc.new_file(request.wfile, schema=expected.schema) as writer:
writer.write_table(expected)
else:
request.send_response(404)
request.end_headers()
async with mock_lancedb_connection_async(handler) as db:
table = await db.open_table("test")
query = await table.search(**search_kwargs)
assert isinstance(query, AsyncQuery)
result = await query.where("id >= 7").select(["id"]).limit(3).to_arrow()
assert result == expected
def test_query_sync_maximal():
def handler(body):
assert body == {
@@ -1581,6 +1707,53 @@ def test_query_sync_maximal():
)
@pytest.mark.parametrize("hybrid", [False, True])
def test_query_sync_nprobes_zero(hybrid):
query_requests = []
def handler(body):
query_requests.append(body)
return pa.table({"id": []})
with query_test_table(handler) as table:
if hybrid:
query = table.search(query_type="hybrid").vector([1, 2, 3]).text("dog")
else:
query = table.search([1, 2, 3])
with pytest.raises(
ValueError, match="^Invalid input, nprobes must be greater than 0$"
):
query.nprobes(0)
assert query_requests == []
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
async def test_query_async_nprobes_zero(hybrid):
requests = []
def handler(request):
requests.append(request.path)
if request.path == "/v1/table/test/describe/":
send_json(request, {"version": 1, "schema": {"fields": []}})
else:
request.send_response(404)
request.end_headers()
async with mock_lancedb_connection_async(handler) as db:
table = await db.open_table("test")
query = table.query().nearest_to([1, 2, 3])
if hybrid:
query = query.nearest_to_text("dog")
with pytest.raises(
ValueError, match="^Invalid input, nprobes must be greater than 0$"
):
query.nprobes(0)
assert requests == ["/v1/table/test/describe/"]
def test_query_sync_nprobes():
def handler(body):
assert body == {
@@ -2145,15 +2318,46 @@ async def test_header_provider_overrides_static_headers():
def test_close():
"""Test that close() works without AttributeError."""
import asyncio
def handler(req):
req.send_response(200)
req.end_headers()
with mock_lancedb_connection(handler) as db:
asyncio.run(db.close())
assert db.close() is None
assert not db.is_open()
assert db.close() is None
with pytest.warns(DeprecationWarning, match="table_names"):
with pytest.raises(RuntimeError, match="Connection is closed"):
db.table_names()
with pytest.raises(RuntimeError, match="Connection is closed"):
db.list_tables()
with pytest.raises(RuntimeError, match="Connection is closed"):
db.open_table("test")
@pytest.mark.parametrize("raise_error", [False, True])
def test_sync_context_manager(raise_error):
def handler(req):
req.send_response(200)
req.send_header("Content-Type", "application/json")
req.end_headers()
req.wfile.write(b'{"tables": []}')
with mock_lancedb_connection(handler) as db:
with contextlib.ExitStack() as stack:
if raise_error:
stack.enter_context(pytest.raises(ValueError, match="test error"))
with db as conn:
assert conn is db
assert conn.is_open()
assert conn.list_tables().tables == []
if raise_error:
raise ValueError("test error")
assert not db.is_open()
with pytest.raises(RuntimeError, match="Connection is closed"):
db.list_tables()
@pytest.mark.parametrize("exception", [KeyboardInterrupt, SystemExit, GeneratorExit])
@@ -2391,6 +2595,27 @@ def blob_query_response_table():
)
def blob_descriptor_take_table():
"""What `fetch_blob_files` reads for rows 10, 20, 30, and 40."""
queried = blob_query_response_table()
image_field = queried.schema.field("image")
images = pa.StructArray.from_arrays(
[
pa.array([0, 0, 0, 0], type=pa.uint8()),
pa.array([0, 0, 0, 0], type=pa.uint64()),
pa.array([5, 0, 5, 0], type=pa.uint64()),
pa.array([0, 0, 0, 0], type=pa.uint32()),
pa.array(["", "", "", ""], type=pa.string()),
],
fields=image_field.type,
mask=pa.array([False, True, False, False]),
)
return pa.Table.from_arrays(
[images, pa.array([10, 20, 30, 40], type=pa.uint64())],
schema=pa.schema([image_field, pa.field("_rowid", pa.uint64())]),
)
@contextlib.contextmanager
def blob_remote_table(*, server_version=Version("0.5.0")):
def handler(request):
@@ -2429,9 +2654,14 @@ def blob_remote_table(*, server_version=Version("0.5.0")):
elif request.path == "/v1/table/test/query/":
content_len = int(request.headers.get("Content-Length", 0))
body = json.loads(request.rfile.read(content_len))
assert body["columns"] == ["id", "image"]
assert body["with_row_id"] is True
response_table = blob_query_response_table()
if body["columns"] == ["image"]:
# fetch_blob_files sizes its handles from a descriptor take.
assert body["filter"].startswith("_rowid IN")
response_table = blob_descriptor_take_table()
else:
assert body["columns"] == ["id", "image"]
response_table = blob_query_response_table()
request.send_response(200)
request.send_header("Content-Type", "application/vnd.apache.arrow.file")
request.end_headers()
python/python/tests/test_table.py
+166 -4
View File
@@ -113,6 +113,32 @@ def test_basic(mem_db: DBConnection):
assert table.to_arrow() == expected_data
@pytest.mark.parametrize("enable_v2", [False, True])
def test_migrate_v2_manifest_paths(tmp_path, enable_v2):
db = lancedb.connect(
tmp_path,
storage_options={"new_table_enable_v2_manifest_paths": str(enable_v2).lower()},
)
table = db.create_table("calls", [{"id": 1, "vector": [1.0, 1.0]}])
table.add([{"id": 2, "vector": [2.0, 2.0]}])
expected_data = table.to_arrow()
expected_versions = table.list_versions()
assert table.uses_v2_manifest_paths() == enable_v2
# Migration is also safe to repeat on a table already using v2 paths.
for _ in range(2):
table.migrate_v2_manifest_paths()
assert table.uses_v2_manifest_paths()
reopened = db.open_table("calls")
assert reopened.uses_v2_manifest_paths()
assert reopened.to_arrow() == expected_data
assert reopened.list_versions() == expected_versions
manifests = list((tmp_path / "calls.lance" / "_versions").glob("*.manifest"))
assert len(manifests) == len(expected_versions)
assert all(len(path.stem) == 20 and path.stem.isdigit() for path in manifests)
def test_search_preserves_nulls_from_sliced_arrow_table(mem_db: DBConnection):
data = pa.table(
{
@@ -967,6 +993,63 @@ def test_polars(mem_db: DBConnection):
assert len(filtered_result) == 2
@pytest.mark.asyncio
async def test_list_versions_timestamp_precision():
# 2026-10-03T01:03:49.274Z in nanoseconds. Float math turns .274000 into
# .273999, so make sure the conversion is exact.
ts_nanos = 1790989429274000000
class FakeInner:
async def list_versions(self):
return [{"version": 1, "timestamp": ts_nanos, "metadata": {}}]
table = table_module.AsyncTable(FakeInner())
versions = await table.list_versions()
expected = datetime.fromtimestamp(ts_nanos // 1_000_000_000)
assert versions[0]["timestamp"] == expected + timedelta(microseconds=274000)
assert versions[0]["timestamp"].microsecond == 274000
def test_list_versions_timestamp_precision_sync(mem_db: DBConnection):
# The sync LanceTable.list_versions delegates to AsyncTable.list_versions,
# which is the path the parity report hit.
ts_nanos = 1790989429274000000
class FakeInner:
async def list_versions(self):
return [{"version": 1, "timestamp": ts_nanos, "metadata": {}}]
table = mem_db.create_table("ts_precision", data=[{"id": 1}])
table._table = table_module.AsyncTable(FakeInner())
versions = table.list_versions()
expected = datetime.fromtimestamp(ts_nanos // 1_000_000_000)
assert versions[0]["timestamp"] == expected + timedelta(microseconds=274000)
assert versions[0]["timestamp"].microsecond == 274000
def test_list_versions_timestamp_precision_remote():
# RemoteTable.list_versions also delegates to AsyncTable.list_versions.
from lancedb.remote.table import RemoteTable
ts_nanos = 1790989429274000000
class FakeInner:
def name(self):
return "ts_precision"
async def list_versions(self):
return [{"version": 1, "timestamp": ts_nanos, "metadata": {}}]
table = RemoteTable(table_module.AsyncTable(FakeInner()), "dev")
versions = table.list_versions()
expected = datetime.fromtimestamp(ts_nanos // 1_000_000_000)
assert versions[0]["timestamp"] == expected + timedelta(microseconds=274000)
assert versions[0]["timestamp"].microsecond == 274000
def test_versioning(mem_db: DBConnection):
table = mem_db.create_table(
"test",
@@ -3751,6 +3834,24 @@ def test_empty_query(mem_db: DBConnection):
assert df.num_rows == 42
@pytest.mark.parametrize("query", [[], np.array([], dtype=np.float32)])
@pytest.mark.parametrize("vector_column_name", [None, "vector"])
@pytest.mark.parametrize("query_type", ["auto", "vector"])
@pytest.mark.parametrize("multiple_vector_columns", [False, True])
def test_search_empty_vector(
mem_db, query, vector_column_name, query_type, multiple_vector_columns
):
fields = [pa.field("vector", pa.list_(pa.float32(), 8))]
if multiple_vector_columns:
fields.append(pa.field("vec2", pa.list_(pa.float32(), 4)))
table = mem_db.create_table("empty_vector_query", schema=pa.schema(fields))
with pytest.raises(ValueError, match="^Query vector must not be empty$"):
table.search(
query, vector_column_name=vector_column_name, query_type=query_type
).limit(3).to_arrow()
def test_search_with_schema_inf_single_vector(mem_db: DBConnection):
class MyTable(LanceModel):
text: str
@@ -4144,14 +4245,74 @@ async def test_add_columns_with_schema(mem_db_async: AsyncConnection):
)
def test_alter_columns(mem_db: DBConnection):
@pytest.mark.parametrize("rename_key", ["rename", "name"])
def test_alter_columns(mem_db: DBConnection, rename_key):
data = pa.table({"id": [0, 1]})
table = mem_db.create_table("my_table", data=data)
alter_columns_res = table.alter_columns({"path": "id", "rename": "new_id"})
alter_columns_res = table.alter_columns({"path": "id", rename_key: "new_id"})
assert alter_columns_res.version == 2
assert table.to_arrow().column_names == ["new_id"]
INVALID_COLUMN_ALTERATIONS = [
(({"path": "id"},), "One of rename, nullable or data_type"),
(
({"path": "id", "nulable": False},), # spellchecker:disable-line
"Unknown column alteration key 'nulable'", # spellchecker:disable-line
),
(
(
{
"path": "id",
"rename": "new_id",
"nulable": False, # spellchecker:disable-line
},
),
"Unknown column alteration key 'nulable'", # spellchecker:disable-line
),
(
({"path": "id", "rename": "new_id"}, {"path": "id"}),
"One of rename, nullable or data_type",
),
]
def test_alter_columns_nullable_false(mem_db: DBConnection):
table = mem_db.create_table("my_table", data=pa.table({"id": [0, 1]}))
result = table.alter_columns({"path": "id", "nullable": False})
assert result.version == 2
assert not table.schema.field("id").nullable
@pytest.mark.parametrize("alterations, match", INVALID_COLUMN_ALTERATIONS)
def test_alter_columns_rejects_invalid(mem_db: DBConnection, alterations, match):
table = mem_db.create_table("my_table", data=pa.table({"id": [0, 1]}))
initial_version = table.version
initial_schema = table.schema
with pytest.raises(ValueError, match=match):
table.alter_columns(*alterations)
assert table.version == initial_version
assert table.schema == initial_schema
@pytest.mark.asyncio
@pytest.mark.parametrize("alterations, match", INVALID_COLUMN_ALTERATIONS)
async def test_alter_columns_rejects_invalid_async(
mem_db_async: AsyncConnection, alterations, match
):
table = await mem_db_async.create_table("my_table", data=pa.table({"id": [0, 1]}))
initial_version = await table.version()
initial_schema = await table.schema()
with pytest.raises(ValueError, match=match):
await table.alter_columns(*alterations)
assert await table.version() == initial_version
assert await table.schema() == initial_schema
def test_update_field_metadata(mem_db: DBConnection):
data = pa.table({"id": [0, 1], "category": ["a", "b"]})
table = mem_db.create_table("my_table", data=data)
@@ -4177,10 +4338,11 @@ def test_update_field_metadata(mem_db: DBConnection):
@pytest.mark.asyncio
async def test_alter_columns_async(mem_db_async: AsyncConnection):
@pytest.mark.parametrize("rename_key", ["rename", "name"])
async def test_alter_columns_async(mem_db_async: AsyncConnection, rename_key):
data = pa.table({"id": [0, 1]})
table = await mem_db_async.create_table("my_table", data=data)
alter_columns_res = await table.alter_columns({"path": "id", "rename": "new_id"})
alter_columns_res = await table.alter_columns({"path": "id", rename_key: "new_id"})
assert alter_columns_res.version == 2
assert (await table.to_arrow()).column_names == ["new_id"]
alter_columns_res = await table.alter_columns(
python/src/catalog.rs
+45 -15
View File
@@ -1,12 +1,13 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::{Arc, Mutex};
use std::time::Duration;
use lancedb::catalog::{
CatalogConnection, CreateDatabaseRequest, DropDatabaseRequest, ListDatabasesRequest,
};
use pyo3::exceptions::PyValueError;
use futures::{StreamExt, future::poll_fn};
use lancedb::catalog::{CatalogConnection, CreateDatabaseRequest, DropDatabaseRequest};
use pyo3::exceptions::{PyStopAsyncIteration, PyValueError};
use pyo3::{Bound, PyAny, PyRef, PyResult, Python, pyclass, pyfunction, pymethods};
use crate::connection::{Connection, PyClientConfig};
@@ -67,19 +68,48 @@ impl Catalog {
})
}
#[pyo3(signature = (*, limit=None, page_token=None))]
fn list_databases<'py>(
self_: PyRef<'py, Self>,
limit: Option<u32>,
page_token: Option<String>,
) -> PyResult<Bound<'py, PyAny>> {
#[pyo3(signature = (*, page_token=None, page_limit=None))]
fn list_databases(&self, page_token: Option<String>, page_limit: Option<u32>) -> DatabaseNames {
DatabaseNames {
inner: Arc::new(Mutex::new(
self.inner.list_databases(page_token, page_limit),
)),
next_lock: Arc::new(tokio::sync::Mutex::new(())),
}
}
}
#[pyclass]
pub struct DatabaseNames {
inner: Arc<Mutex<lancedb::catalog::DatabaseNames>>,
next_lock: Arc<tokio::sync::Mutex<()>>,
}
#[pymethods]
impl DatabaseNames {
fn num_page_results(&self, py: Python<'_>) -> usize {
py.detach(|| self.inner.lock().unwrap().num_page_results())
}
fn page_token(&self, py: Python<'_>) -> Option<String> {
py.detach(|| self.inner.lock().unwrap().page_token().map(str::to_owned))
}
fn __aiter__(self_: PyRef<'_, Self>) -> PyRef<'_, Self> {
self_
}
fn __anext__(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
let next_lock = self_.next_lock.clone();
future_into_py(self_.py(), async move {
let mut request = ListDatabasesRequest::default();
request.limit = limit;
request.page_token = page_token;
let response = inner.list_databases(request).await.infer_error()?;
Ok((response.databases, response.page_token))
// Serialize advances while allowing synchronous state inspection
// between polls, including while a REST request is pending.
let _guard = next_lock.lock().await;
poll_fn(|cx| inner.lock().unwrap().poll_next_unpin(cx))
.await
.ok_or_else(|| PyStopAsyncIteration::new_err(""))?
.infer_error()
})
}
}
python/src/connection.rs
+2 -18
View File
@@ -407,8 +407,7 @@ impl Connection {
})
}
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, *, vector_source_json=None))]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
pub fn create_materialized_view(
self_: PyRef<'_, Self>,
name: String,
@@ -417,17 +416,10 @@ impl Connection {
filter: Option<String>,
limit: Option<u64>,
with_no_data: bool,
vector_source_json: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let mut builder = inner.create_materialized_view(name, source);
if let Some(source) = vector_source_json {
builder = builder.vector_source(
serde_json::from_str(&source)
.map_err(|e| PyValueError::new_err(e.to_string()))?,
);
}
if let Some(projections) = projections {
builder = builder.select(projections);
}
@@ -443,8 +435,7 @@ impl Connection {
})
}
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, *, vector_source_json=None))]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
pub fn create_materialized_view_async(
self_: PyRef<'_, Self>,
name: String,
@@ -453,17 +444,10 @@ impl Connection {
filter: Option<String>,
limit: Option<u64>,
with_no_data: bool,
vector_source_json: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let mut builder = inner.create_materialized_view(name, source);
if let Some(source) = vector_source_json {
builder = builder.vector_source(
serde_json::from_str(&source)
.map_err(|e| PyValueError::new_err(e.to_string()))?,
);
}
if let Some(projections) = projections {
builder = builder.select(projections);
}
python/src/lib.rs
+1
View File
@@ -64,6 +64,7 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
env_logger::init_from_env(env);
m.add_class::<Connection>()?;
m.add_class::<catalog::Catalog>()?;
m.add_class::<catalog::DatabaseNames>()?;
m.add_function(wrap_pyfunction!(catalog::connect_catalog, m)?)?;
m.add_class::<Session>()?;
m.add_class::<Table>()?;
python/src/query.rs
+5 -4
View File
@@ -995,8 +995,9 @@ impl VectorQuery {
self.inner = self.inner.clone().refine_factor(refine_factor);
}
pub fn nprobes(&mut self, nprobe: u32) {
self.inner = self.inner.clone().nprobes(nprobe as usize);
pub fn nprobes(&mut self, nprobe: u32) -> PyResult<()> {
self.inner = self.inner.clone().nprobes(nprobe as usize).infer_error()?;
Ok(())
}
pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> {
@@ -1188,8 +1189,8 @@ impl HybridQuery {
self.inner_vec.refine_factor(refine_factor);
}
pub fn nprobes(&mut self, nprobe: u32) {
self.inner_vec.nprobes(nprobe);
pub fn nprobes(&mut self, nprobe: u32) -> PyResult<()> {
self.inner_vec.nprobes(nprobe)
}
pub fn ef(&mut self, ef: u32) {
python/src/table.rs
+18 -13
View File
@@ -592,8 +592,6 @@ impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewRes
fn from(result: lancedb::RefreshMaterializedViewResult) -> Self {
let mode = match result.mode {
lancedb::RefreshMode::Rebuild => "rebuild",
lancedb::RefreshMode::Incremental => "incremental",
lancedb::RefreshMode::NoOp => "no_op",
};
Self {
mode: mode.to_string(),
@@ -1817,10 +1815,9 @@ impl Table {
})
}
#[pyo3(signature = (full=false, source_version=None))]
#[pyo3(signature = (source_version=None))]
pub fn refresh_materialized_view(
self_: PyRef<'_, Self>,
full: bool,
source_version: Option<u64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
@@ -1828,7 +1825,7 @@ impl Table {
let view = lancedb::MaterializedView::from_table(inner)
.await
.infer_error()?;
let mut builder = view.refresh().full(full);
let mut builder = view.refresh();
if let Some(version) = source_version {
builder = builder.source_version(version);
}
@@ -1837,10 +1834,9 @@ impl Table {
})
}
#[pyo3(signature = (full=false, source_version=None))]
#[pyo3(signature = (source_version=None))]
pub fn refresh_materialized_view_async(
self_: PyRef<'_, Self>,
full: bool,
source_version: Option<u64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
@@ -1848,7 +1844,7 @@ impl Table {
let view = lancedb::MaterializedView::from_table(inner)
.await
.infer_error()?;
let mut builder = view.refresh().full(full);
let mut builder = view.refresh();
if let Some(version) = source_version {
builder = builder.source_version(version);
}
@@ -1863,11 +1859,9 @@ impl Table {
let view = lancedb::MaterializedView::from_table(inner)
.await
.infer_error()?;
view.definition().to_json().map_err(|err| {
PyRuntimeError::new_err(format!(
"failed to serialize materialized-view definition: {err}"
))
})
Ok(lancedb::materialized_view::definition_metadata_from_sql(
view.definition_sql(),
))
})
}
@@ -1897,6 +1891,17 @@ impl Table {
let alterations = alterations
.iter()
.map(|alteration| {
for key in alteration.keys().iter() {
let key = key.extract::<String>()?;
if !matches!(
key.as_str(),
"path" | "rename" | "name" | "nullable" | "data_type"
) {
return Err(PyValueError::new_err(format!(
"Unknown column alteration key '{key}'"
)));
}
}
let path = alteration
.get_item("path")?
.ok_or_else(|| PyValueError::new_err("Missing path"))?
python/vendor/lance-arrow/Cargo.toml
Vendored
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lance-arrow"
version = "13.1.0-beta.1"
version = "14.0.0-beta.1"
edition = "2024"
authors = ["Lance Devs <dev@lance.org>"]
license = "Apache-2.0"
python/vendor/lance-arrow/README.lancedb.md
Vendored
+2 -2
View File
@@ -1,5 +1,5 @@
This directory contains `rust/lance-arrow` from Lance tag `v13.1.0-beta.1`
(commit `e5a3553b1699a7062ff7afe4ee268d596bbf2e0d`), licensed under Apache-2.0.
This directory contains `rust/lance-arrow` from Lance tag `v14.0.0-beta.1`
(commit `ae2903729f02ec8e519a4e53797a052393f25ee3`), licensed under Apache-2.0.
LanceDB carries a local fix in `src/lib.rs` for sliced child validity bitmaps
until a Lance release includes the fix. The patch is selected in the root
python/vendor/lance-arrow/src/lib.rs
Vendored
+5 -8
View File
@@ -442,13 +442,9 @@ pub trait FixedSizeBinaryArrayExt {
/// let int_values = UInt8Array::from_iter(0..10);
/// let fixed_size_list_arr = FixedSizeBinaryArray::try_new_from_values(&int_values, 2).unwrap();
/// assert_eq!(fixed_size_list_arr,
/// FixedSizeBinaryArray::from(vec![
/// Some(vec![0, 1].as_slice()),
/// Some(vec![2, 3].as_slice()),
/// Some(vec![4, 5].as_slice()),
/// Some(vec![6, 7].as_slice()),
/// Some(vec![8, 9].as_slice())
/// ]))
/// FixedSizeBinaryArray::try_from_iter(
/// [[0u8, 1], [2, 3], [4, 5], [6, 7], [8, 9]].into_iter()
/// ).unwrap())
/// ```
fn try_new_from_values(values: &UInt8Array, stride: i32) -> Result<FixedSizeBinaryArray>;
}
@@ -3035,7 +3031,8 @@ mod tests {
vec![Some(b"1".as_slice()), Some(b"23".as_slice())]
);
let fixed_size = FixedSizeBinaryArray::from(vec![b"abcd", b"efgh"]);
let fixed_size =
FixedSizeBinaryArray::try_from_iter([b"abcd", b"efgh"].into_iter()).unwrap();
assert_eq!(
iter_binary_array(&fixed_size).unwrap().collect::<Vec<_>>(),
vec![Some(b"abcd".as_slice()), Some(b"efgh".as_slice())]