rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-13 07:12:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into gatekeeper/fix-4260-1

Browse Source 
Preserve sliced child validity with Lance v14.0.0-beta.10 and its JSONB dependency.
This commit is contained in:
Gatefixer
2026-10-09 20:43:58 +00:00
parent d4f4cc5443 f305a05fd4
commit f75f7f2112
85 changed files with 4319 additions and 835 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.41.0-beta.0"
version = "0.41.0-beta.2"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
python/pyproject.toml
+2 -2
View File
@@ -10,7 +10,7 @@ dependencies = [
"pyarrow>=16",
"pydantic>=2.7.4,<3",
"tqdm>=4.27.0",
"lance-namespace>=0.3.2"
"lance-namespace>=0.13.0"
]
description = "lancedb"
authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
@@ -63,7 +63,7 @@ tests = [
"polars>=0.19, <=1.32.3",
"pyarrow<25",
"pyarrow-stubs>=16.0",
"pylance==14.0.0b6",
"pylance==14.0.0b10",
"requests>=2.31.0",
"datafusion>=54,<55",
"opentelemetry-sdk>=1.30.0",
python/python/lancedb/__init__.py
+4 -2
View File
@@ -321,8 +321,8 @@ def tokenize(
language: str = "English",
max_token_length: Optional[int] = 40,
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
stem: Optional[bool] = None,
remove_stop_words: Optional[bool] = None,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
@@ -333,6 +333,8 @@ def tokenize(
This does not require an FTS index. The tokenizer options match
:class:`lancedb.index.FTS`. ``custom_stop_words`` accepts a list of strings.
When ``stem`` or ``remove_stop_words`` is None, the base tokenizer's default
applies: False for ``code`` and ``ngram``, True otherwise.
"""
return _tokenize(
python/python/lancedb/_lancedb.pyi
+5 -2
View File
@@ -58,8 +58,8 @@ def tokenize(
language: str = "English",
max_token_length: Optional[int] = 40,
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
stem: Optional[bool] = None,
remove_stop_words: Optional[bool] = None,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
@@ -217,6 +217,7 @@ class Connection(object):
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[pa.RecordBatch] = None,
) -> SqlQuery: ...
async def describe_query(self, query_id: UUID) -> QueryDescription: ...
async def create_table(
@@ -422,6 +423,7 @@ class Table:
def name(self) -> str: ...
def __repr__(self) -> str: ...
def is_open(self) -> bool: ...
def _is_native(self) -> bool: ...
def close(self) -> None: ...
async def schema(self) -> pa.Schema: ...
async def add(
@@ -776,6 +778,7 @@ class PyQueryRequest:
use_lsm: Optional[bool]
column: Optional[str]
query_vector: Optional[List[pa.Array]]
nprobes: Optional[int]
minimum_nprobes: Optional[int]
maximum_nprobes: Optional[int]
lower_bound: Optional[float]
python/python/lancedb/db.py
+54 -17
View File
@@ -53,7 +53,7 @@ from .functions import FunctionVersion, UdfDefinition
from .job import AsyncJob, Job, _typed_job
from .sql import AsyncQuery as AsyncSqlQuery
from .sql import Query as SqlQuery
from .sql import QueryDescription
from .sql import QueryDescription, QueryParameters, to_parameter_batch
from .materialized_view import (
AsyncMaterializedView,
MaterializedView,
@@ -317,7 +317,7 @@ class DBConnection(EnforceOverrides):
page_token: Optional[str] = None,
limit: Optional[int] = None,
) -> ListTablesResponse:
"""List all tables in this database with pagination support.
"""List a page of tables in this database.
Parameters
----------
@@ -328,7 +328,8 @@ class DBConnection(EnforceOverrides):
Token for pagination. Use the token from a previous response
to get the next page of results.
limit: int, optional
The maximum number of results to return.
The maximum number of results to return, default 100.
Zero returns an empty page without a continuation token.
Returns
-------
@@ -343,7 +344,7 @@ class DBConnection(EnforceOverrides):
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
@@ -357,8 +358,9 @@ class DBConnection(EnforceOverrides):
page_token: str, optional
The token to use for pagination. If not present, start from the beginning.
Typically, this token is last table name from the previous page.
limit: int, default 10
The size of the page to return.
limit: int, optional
The maximum number of tables to return. None returns all tables;
zero returns an empty list.
Returns
-------
@@ -1178,6 +1180,7 @@ class DBConnection(EnforceOverrides):
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> pa.RecordBatchReader:
"""Execute SQL and return a blocking Arrow reader.
@@ -1188,6 +1191,7 @@ class DBConnection(EnforceOverrides):
return self.execute_query_async(
query,
default_namespace_path=default_namespace_path,
parameters=parameters,
).reader()
def execute_query_async(
@@ -1195,9 +1199,32 @@ class DBConnection(EnforceOverrides):
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> SqlQuery:
"""Start executing SQL and return its query handle.
``parameters`` binds values to the statement's ``$1`` / ``$name``
placeholders. They travel as Arrow rather than as SQL text, so a
float keeps its exact value and type and a vector stays a compact
fixed-size list. A list binds by position, a dict binds by name, and
a one-row ``pyarrow.RecordBatch`` or ``pyarrow.Table`` binds both
ways. A 1-D numpy or pyarrow array becomes a fixed-size list of its
element type, the shape of a vector column:
.. code-block:: python
db.execute_query(
"SELECT id FROM docs ORDER BY distance(vector, $1) LIMIT $2",
parameters=[np.array([0.1, 0.2, 0.3], dtype=np.float32), 10],
)
Parameters are supported in queries, not in DDL or DML. A
parameterized query runs on the call that returns its rows:
cancelling it, or dropping its handle before reading, stops it on the
server. Its status follows that stream as well: ``describe()``
reports ``running`` until the reader has received every row, so open
``reader()`` directly rather than waiting for ``finished``.
Local connections do not support SQL.
"""
raise NotImplementedError("SQL is not supported for this connection type")
@@ -1465,7 +1492,7 @@ class LanceDBConnection(DBConnection):
page_token: Optional[str] = None,
limit: Optional[int] = None,
) -> ListTablesResponse:
"""List all tables in this database with pagination support.
"""List a page of tables in this database.
Parameters
----------
@@ -1476,7 +1503,8 @@ class LanceDBConnection(DBConnection):
Token for pagination. Use the token from a previous response
to get the next page of results.
limit: int, optional
The maximum number of results to return.
The maximum number of results to return, default 100.
Zero returns an empty page without a continuation token.
Returns
-------
@@ -1495,7 +1523,7 @@ class LanceDBConnection(DBConnection):
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
@@ -1510,8 +1538,9 @@ class LanceDBConnection(DBConnection):
The namespace to list tables in.
page_token: str, optional
The token to use for pagination.
limit: int, default 10
The maximum number of tables to return.
limit: int, optional
The maximum number of tables to return. None returns all tables;
zero returns an empty list.
Returns
-------
@@ -2341,7 +2370,7 @@ class AsyncConnection(object):
page_token: Optional[str] = None,
limit: Optional[int] = None,
) -> ListTablesResponse:
"""List all tables in this database with pagination support.
"""List a page of tables in this database.
Parameters
----------
@@ -2352,7 +2381,8 @@ class AsyncConnection(object):
Token for pagination. Use the token from a previous response
to get the next page of results.
limit: int, optional
The maximum number of results to return.
The maximum number of results to return, default 100.
Zero returns an empty page without a continuation token.
Returns
-------
@@ -2389,8 +2419,9 @@ class AsyncConnection(object):
This can be combined with limit to implement pagination by setting this to
the last table name from the previous page.
limit: int, default 10
The number of results to return.
limit: int, optional
The maximum number of tables to return. None returns all tables;
zero returns an empty list.
Returns
-------
@@ -3267,6 +3298,7 @@ class AsyncConnection(object):
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> AsyncRecordBatchReader:
"""Execute SQL and return an asynchronous Arrow reader.
@@ -3277,6 +3309,7 @@ class AsyncConnection(object):
submitted = await self.execute_query_async(
query,
default_namespace_path=default_namespace_path,
parameters=parameters,
)
return await submitted.reader()
@@ -3285,17 +3318,21 @@ class AsyncConnection(object):
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> AsyncSqlQuery:
"""Start executing SQL and return its query handle.
The database from ``connect_async`` is used for unqualified database
references. The namespace defaults to ``["public"]``. Local
connections raise ``NotImplementedError``.
references. The namespace defaults to ``["public"]``. ``parameters``
binds values to ``$1`` / ``$name`` placeholders; see
[DBConnection.execute_query_async][lancedb.db.DBConnection.execute_query_async].
Local connections raise ``NotImplementedError``.
"""
return AsyncSqlQuery(
await self._inner.execute_query_async(
query,
default_namespace_path=default_namespace_path,
parameters=to_parameter_batch(parameters),
)
)
python/python/lancedb/functions.py
+1 -1
View File
@@ -372,7 +372,7 @@ class FunctionVersion(_RemoteValue):
image: FunctionImage
signature: FunctionSignature
secret_bindings: tuple[SecretBinding, ...] = ()
created_at: str
created_at_millis: int
metadata: Mapping[str, str]
disabled: bool
python/python/lancedb/index.py
+44 -6
View File
@@ -112,9 +112,42 @@ class Fm:
class FTS:
"""Describe a FTS index configuration.
`FTS` is a full-text search index that can be used on `String` columns
`FTS` supports string columns, lists of strings, and JSON columns created
from PyArrow ``pa.json_()``. JSON columns use Lance's JSON-aware tokenizer
and are stored as ``LargeBinary`` with the ``lance.json`` logical type.
Arbitrary binary columns are not supported.
For example, it works with `title`, `description`, `content`, etc.
JSON queries use ``path,str,value`` triples. Paths are relative to the JSON
column and use dots for nested object keys. For example,
``meta.tag,str,nature`` matches the string value at ``$.meta.tag``;
it does not search other paths or JSON key names. The value is processed
with the configured text tokenizer. Use ``str``, not ``string``.
Create and query a JSON index through the modern API::
import pyarrow as pa
from lancedb.index import FTS
table = db.create_table(
"documents",
pa.table({"doc": pa.array(
['{"meta":{"tag":"nature"}}'], type=pa.json_()
)}),
)
table.create_index("doc", config=FTS())
table.search(
"meta.tag,str,nature", query_type="fts", fts_columns="doc"
).to_list()
Create the index on the JSON column itself, not a JSON subpath such as
``doc.meta.tag``. JSON-path scalar indexes do not provide FTS path scoping.
Plain text such as ``nature`` is not a valid JSON FTS query. The current
Lance tokenizer can panic on malformed triples; see
https://github.com/lance-format/lance/issues/8812.
Remote tables use the same ``FTS()`` configuration and require a server
that supports JSON FTS. Wait for ``table.wait_for_index(["doc_idx"])``
before searching a newly created remote index with its default name.
Examples
--------
@@ -138,6 +171,7 @@ class FTS:
- "simple": Splits text by whitespace and punctuation.
- "whitespace": Split text by whitespace, but not punctuation.
- "raw": No tokenization. The entire text is treated as a single token.
- "code": Tokenizes source code and identifiers.
- "ngram": N-gram tokenizer for substring-style matching.
- "icu": ICU dictionary-based word segmentation.
- "icu/split": ICU segmentation with simple-style delimiter splitting.
@@ -151,12 +185,16 @@ class FTS:
ignored.
lower_case : bool, default True
Whether to convert the token to lower case. This makes queries case-insensitive.
stem : bool, default True
stem : bool, optional
Whether to stem the token. Stemming reduces words to their root form.
For example, in English "running" and "runs" would both be reduced to "run".
remove_stop_words : bool, default True
``None`` uses the base tokenizer's default: False for ``code`` and
``ngram``, True otherwise.
remove_stop_words : bool, optional
Whether to remove stop words. Stop words are common words that are often
removed from text before indexing. For example, in English "the" and "and".
``None`` uses the base tokenizer's default: False for ``code`` and
``ngram``, True otherwise.
custom_stop_words : list of str, optional
Custom words replace the built-in language stop words
and only take effect when ``remove_stop_words`` is True. ``None`` uses
@@ -197,8 +235,8 @@ class FTS:
language: str = "English"
max_token_length: Optional[int] = 40
lower_case: bool = True
stem: bool = True
remove_stop_words: bool = True
stem: Optional[bool] = None
remove_stop_words: Optional[bool] = None
ascii_folding: bool = True
ngram_min_length: int = 3
ngram_max_length: int = 3
python/python/lancedb/namespace.py
+23 -4
View File
@@ -53,7 +53,7 @@ from lancedb.db import AsyncConnection, DBConnection
from lancedb.job import AsyncJob, Job
from lancedb.listing import AsyncListing, Listing
from lancedb.sql import AsyncQuery as AsyncSqlQuery
from lancedb.sql import QueryDescription
from lancedb.sql import QueryDescription, QueryParameters
from lance_namespace import (
LanceNamespace,
connect as namespace_connect,
@@ -190,8 +190,12 @@ def _query_to_namespace_request(
kwargs["with_row_id"] = query.with_row_id
if query.ef is not None:
kwargs["ef"] = query.ef
if query.nprobes is not None:
kwargs["nprobes"] = query.nprobes
if query.minimum_nprobes is not None:
kwargs["nprobes"] = query.minimum_nprobes
kwargs["minimum_nprobes"] = query.minimum_nprobes
if query.maximum_nprobes is not None:
kwargs["maximum_nprobes"] = query.maximum_nprobes
if query.refine_factor is not None:
kwargs["refine_factor"] = query.refine_factor
if query.lower_bound is not None:
@@ -205,6 +209,13 @@ def _query_to_namespace_request(
if query.distance_type is not None:
kwargs["distance_type"] = query.distance_type
if kwargs.get("maximum_nprobes") == 0:
# The generated 0.13 model rejects LanceDB's unbounded sentinel;
# model_copy preserves direct wire pass-through to the Lance receiver.
maximum_nprobes = kwargs.pop("maximum_nprobes")
return QueryTableRequest(**kwargs).model_copy(
update={"maximum_nprobes": maximum_nprobes}
)
return QueryTableRequest(**kwargs)
@@ -522,13 +533,15 @@ class LanceNamespaceDBConnection(DBConnection):
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
"""
List table names in the database.
If limit is None, return all table names; zero returns an empty list.
.. deprecated::
Use :meth:`list_tables` instead, which provides proper pagination support.
"""
@@ -1157,13 +1170,15 @@ class AsyncLanceNamespaceDBConnection:
async def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
"""
List table names in the namespace.
If limit is None, return all table names; zero returns an empty list.
.. deprecated::
Use :meth:`list_tables` instead, which provides proper pagination support.
"""
@@ -1584,11 +1599,13 @@ class AsyncLanceNamespaceDBConnection:
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> AsyncRecordBatchReader:
"""Execute SQL when supported by the underlying connection."""
return await self._inner.execute_query(
query,
default_namespace_path=default_namespace_path,
parameters=parameters,
)
async def execute_query_async(
@@ -1596,6 +1613,7 @@ class AsyncLanceNamespaceDBConnection:
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> AsyncSqlQuery:
"""Start executing SQL when supported by the underlying connection.
@@ -1604,6 +1622,7 @@ class AsyncLanceNamespaceDBConnection:
return await self._inner.execute_query_async(
query,
default_namespace_path=default_namespace_path,
parameters=parameters,
)
async def describe_query(self, query_id: UUID) -> QueryDescription:
python/python/lancedb/query.py
+69 -25
View File
@@ -39,7 +39,7 @@ from .expr import Expr
from .rerankers.base import Reranker
from .rerankers.rrf import RRFReranker
from .rerankers.util import check_reranker_result
from .schema import is_blob_like_field, schema_has_blob_field
from .schema import blob_column_paths, is_blob_like_field, schema_has_blob_field
from .util import _validate_query_vector, flatten_columns
from . import _wal_hybrid # WAL-PK-FUSION: delete.
from ._blob import (
@@ -51,6 +51,7 @@ from ._blob import (
finalize_blob_query_table,
replace_v2_blob_columns_with_bytes,
replace_v2_blob_columns_with_bytes_sync,
strip_auto_row_ids,
validate_blob_mode,
)
from .types import BlobMode, QueryProjection
@@ -479,7 +480,7 @@ class MatchQuery(FullTextQuery):
boost : float, default 1.0
The boost factor for the query.
The score of each matching document is multiplied by this value.
fuzziness : int, optional
fuzziness : int or None, default 0
The maximum edit distance for each term in the match query.
Defaults to 0 (exact match).
If None, fuzziness is applied automatically by the rules:
@@ -506,7 +507,7 @@ class MatchQuery(FullTextQuery):
query: str
column: str
boost: float = pydantic.Field(1.0, kw_only=True)
fuzziness: int = pydantic.Field(0, kw_only=True)
fuzziness: Optional[int] = pydantic.Field(0, kw_only=True)
max_expansions: int = pydantic.Field(50, kw_only=True)
operator: FullTextOperator = pydantic.Field(FullTextOperator.OR, kw_only=True)
prefix_length: int = pydantic.Field(0, kw_only=True)
@@ -692,8 +693,9 @@ class Query(pydantic.BaseModel):
if True then apply the filter after vector / FTS search. This is ignored for
plain SQL filtering.
nprobes : Optional[int]
The number of IVF partitions to search. If this is None then a default
number of partitions will be used.
The legacy number of IVF partitions to search. Lance sets both probe
bounds to this value. If this is None then Lance's default probe settings
will be used.
- A higher number makes search more accurate but also slower.
@@ -772,20 +774,23 @@ class Query(pydantic.BaseModel):
# distance type to use for vector search
distance_type: Optional[str] = None
# legacy number of IVF partitions to search
#
# Lance sets both probe bounds to this value. Explicit bounds can override it.
nprobes: Optional[int] = None
# which columns to return in the results (dict values may be str or Expr)
columns: QueryProjection = None
# minimum number of IVF partitions to search
#
# If None then a default value (20) will be used.
# If None then Lance's default will be used.
minimum_nprobes: Optional[int] = None
# maximum number of IVF partitions to search
#
# If None then a default value (20) will be used.
#
# If 0 then no limit will be applied and all partitions could be searched
# if needed to satisfy the limit.
# If None then Lance's default will be used. If 0 then no limit will be applied
# and all partitions could be searched if needed to satisfy the limit.
maximum_nprobes: Optional[int] = None
# lower bound for distance search
@@ -846,6 +851,7 @@ class Query(pydantic.BaseModel):
query.vector_column = req.column
query.vector = req.query_vector
query.distance_type = req.distance_type
query.nprobes = req.nprobes
query.minimum_nprobes = req.minimum_nprobes
query.maximum_nprobes = req.maximum_nprobes
query.lower_bound = req.lower_bound
@@ -1626,6 +1632,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
super().__init__(table)
self._query = query
self._distance_type = None
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
self._lower_bound = None
@@ -1690,9 +1697,9 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
See discussion in [Querying an ANN Index](https://lancedb.com/docs/indexing/)
for tuning advice.
This method sets both the minimum and maximum number of probes to the same
value. See `minimum_nprobes` and `maximum_nprobes` for more fine-grained
control.
The value is retained as `nprobes` through client and server request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
Parameters
----------
@@ -1706,8 +1713,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
"""
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
self._nprobes = nprobes
return self
def minimum_nprobes(self, minimum_nprobes: int) -> LanceVectorQueryBuilder:
@@ -1849,6 +1855,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
limit=self._limit,
distance_type=self._distance_type,
columns=self._columns,
nprobes=self._nprobes,
minimum_nprobes=self._minimum_nprobes,
maximum_nprobes=self._maximum_nprobes,
lower_bound=self._lower_bound,
@@ -2207,6 +2214,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._fts_columns = fts_columns
self._norm = None
self._reranker = None
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
self._refine_factor = None
@@ -2498,6 +2506,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
Higher values will yield better recall (more likely to find vectors if
they exist) at the expense of latency.
The value is retained as `nprobes` until Lance sets both probe bounds to it.
Explicit minimum or maximum settings can override their respective bounds.
Parameters
----------
nprobes: int
@@ -2510,8 +2521,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
"""
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
self._nprobes = nprobes
return self
def minimum_nprobes(self, minimum_nprobes: int) -> LanceHybridQueryBuilder:
@@ -2790,6 +2800,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._fts_query.phrase_query(True)
if self._distance_type:
self._vector_query.metric(self._distance_type)
if self._nprobes is not None:
self._vector_query.nprobes(self._nprobes)
if self._minimum_nprobes is not None:
self._vector_query.minimum_nprobes(self._minimum_nprobes)
if self._maximum_nprobes is not None:
@@ -3073,14 +3085,18 @@ class AsyncQueryBase(object):
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
blob_mode: str, default "lazy"
Controls how blob columns are returned for plain scan queries.
Vector, FTS, hybrid, and other non-native query shapes keep the
existing Arrow conversion path and only support blob descriptions.
Controls how blob columns are returned. Remote queries support
"descriptions"; "bytes" and "lazy" are not yet supported.
**kwargs
Forwarded to pyarrow.Table.to_pandas after query execution and
optional flattening.
"""
validate_blob_mode(blob_mode)
if self._table is not None and not self._table._inner._is_native():
return await self._remote_to_pandas(
flatten=flatten, timeout=timeout, blob_mode=blob_mode, **kwargs
)
if hasattr(self._inner, "output_schema"):
schema = await self.output_schema()
if _blob_mode_requires_native_pandas(blob_mode, schema):
@@ -3108,6 +3124,31 @@ class AsyncQueryBase(object):
)
return tbl.to_pandas(**kwargs)
async def _remote_to_pandas(
self,
*,
flatten: Optional[Union[int, bool]],
timeout: Optional[timedelta],
blob_mode: BlobMode,
**kwargs,
) -> "pd.DataFrame":
# A live remote table can be replaced between query and blob fetch.
# Row ids and version numbers do not identify the producing table.
tbl = await self.to_arrow(timeout=timeout)
projected_blob_paths = set(blob_column_paths(tbl.schema))
if not projected_blob_paths:
return flatten_columns(tbl, flatten).to_pandas(**kwargs)
if blob_mode == "descriptions":
tbl = strip_auto_row_ids(tbl, self._blob_paths)
return flatten_columns(tbl, flatten).to_pandas(**kwargs)
raise NotImplementedError(
f"remote to_pandas(blob_mode={blob_mode!r}) cannot safely materialize "
"blob columns without a stable table snapshot; "
"use blob_mode='descriptions'"
)
async def _plain_scan_to_pandas(
self,
blob_mode: BlobMode,
@@ -3660,7 +3701,7 @@ class AsyncVectorQueryBase:
def nprobes(self, nprobes: int) -> Self:
"""
Set the number of partitions to search (probe)
Set the legacy IVF probe parameter
The number of probes must be greater than 0.
@@ -3672,16 +3713,19 @@ class AsyncVectorQueryBase:
The partition whose centroids are closest to the query vector will be
exhaustiely searched to find matches. This parameter controls how many
partitions should be searched.
partitions will be searched.
Increasing this value will increase the recall of your query but will
also increase the latency of your query. The default value is 20. This
default is good for many cases but the best value to use will depend on
your data and the recall that you need to achieve.
also increase the latency of your query. If this method is not called,
Lance's adaptive probe defaults are used.
For best results we recommend tuning this parameter with a benchmark against
your actual data to find the smallest possible value that will still give
you the desired recall.
LanceDB retains this as `nprobes` through local and remote request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
"""
self._inner.nprobes(nprobes)
return self
python/python/lancedb/remote/db.py
+14 -8
View File
@@ -41,7 +41,7 @@ from ..db import DBConnection, LOOP
from ..functions import FunctionVersion, UdfDefinition
from ..job import AsyncJob, Job
from ..sql import Query as SqlQuery
from ..sql import QueryDescription
from ..sql import QueryDescription, QueryParameters
from ..materialized_view import MaterializedView, MaterializedViewSource, SelectArg
from ..secrets import EnvVarSecret, SecretInfo
from ..view import ViewDescription
@@ -379,7 +379,7 @@ class RemoteDBConnection(DBConnection):
page_token: Optional[str] = None,
limit: Optional[int] = None,
) -> ListTablesResponse:
"""List all tables in this database with pagination support.
"""List a page of tables in this database.
Parameters
----------
@@ -390,7 +390,8 @@ class RemoteDBConnection(DBConnection):
Token for pagination. Use the token from a previous response
to get the next page of results.
limit: int, optional
The maximum number of results to return.
The maximum number of results to return, default 100.
Zero returns an empty page without a continuation token.
Returns
-------
@@ -409,7 +410,7 @@ class RemoteDBConnection(DBConnection):
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
@@ -425,8 +426,9 @@ class RemoteDBConnection(DBConnection):
Empty list represents root namespace.
page_token: str
The last token to start the new page.
limit: int, default 10
The maximum number of tables to return for each page.
limit: int, optional
The maximum number of tables to return. None returns all tables;
zero returns an empty list.
Returns
-------
@@ -486,12 +488,12 @@ class RemoteDBConnection(DBConnection):
namespace_path = []
if storage_options is not None:
logging.info(
"storage_options is ignored in LanceDb Cloud"
"storage_options is ignored for remote tables"
" (storage is managed; set storage_options on connect() instead)"
)
if index_cache_size is not None:
logging.info(
"index_cache_size is ignored in LanceDb Cloud"
"index_cache_size is ignored for remote tables"
" (there is no local cache to configure)"
)
@@ -1090,18 +1092,22 @@ class RemoteDBConnection(DBConnection):
query: str,
*,
default_namespace_path: Optional[List[str]] = None,
parameters: Optional[QueryParameters] = None,
) -> SqlQuery:
"""Start executing SQL through this remote connection.
Unqualified tables use this connection's database and the
``["public"]`` namespace by default. Fully qualified table names may
reference other databases available to the same deployment.
``parameters`` binds values to ``$1`` / ``$name`` placeholders; see
[DBConnection.execute_query_async][lancedb.db.DBConnection.execute_query_async].
"""
return SqlQuery(
LOOP.run(
self._conn.execute_query_async(
query,
default_namespace_path=default_namespace_path,
parameters=parameters,
)
)
)
python/python/lancedb/remote/table.py
+33 -52
View File
@@ -42,6 +42,7 @@ from lancedb.index import (
BTree,
Bitmap,
HnswFlat,
HnswPq,
HnswSq,
IvfFlat,
IvfPq,
@@ -239,12 +240,12 @@ class RemoteTable(Table):
return LOOP.run(self._table.list_versions())
def to_arrow(self) -> pa.Table:
"""to_arrow() is not yet supported on LanceDB cloud."""
raise NotImplementedError("to_arrow() is not yet supported on LanceDB cloud.")
"""to_arrow() is not supported for remote tables."""
raise NotImplementedError("to_arrow() is not supported for remote tables.")
def to_pandas(self, blob_mode: BlobMode = "lazy", **kwargs):
"""to_pandas() is not yet supported on LanceDB cloud."""
raise NotImplementedError("to_pandas() is not yet supported on LanceDB cloud.")
"""to_pandas() is not supported for remote tables."""
raise NotImplementedError("to_pandas() is not supported for remote tables.")
def checkout(self, version: Union[int, str]):
result = LOOP.run(self._table.checkout(version))
@@ -353,8 +354,8 @@ class RemoteTable(Table):
language: str = "English",
max_token_length: Optional[int] = 40,
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
stem: Optional[bool] = None,
remove_stop_words: Optional[bool] = None,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
@@ -501,14 +502,8 @@ class RemoteTable(Table):
if accelerator is not None:
logging.warning(
"GPU accelerator is not yet supported on LanceDB cloud."
"If you have 100M+ vectors to index,"
"please contact us at contact@lancedb.com"
)
if replace is not None:
logging.warning(
"replace is not supported on LanceDB cloud."
"Existing indexes will always be replaced."
"GPU accelerator is not supported for remote tables "
"and will be ignored."
)
idx_type = index_type.upper()
@@ -528,9 +523,11 @@ class RemoteTable(Table):
elif idx_type == "IVF_SQ":
config = IvfSq(distance_type=metric, num_partitions=num_partitions)
elif idx_type == "IVF_HNSW_PQ":
raise ValueError(
"IVF_HNSW_PQ is not supported on LanceDB cloud."
"Please use IVF_HNSW_SQ instead."
config = HnswPq(
distance_type=metric,
num_partitions=num_partitions,
num_sub_vectors=num_sub_vectors,
num_bits=num_bits,
)
elif idx_type == "IVF_HNSW_SQ":
config = HnswSq(distance_type=metric, num_partitions=num_partitions)
@@ -659,7 +656,7 @@ class RemoteTable(Table):
capped at the number of CPU cores. Lower this if bulk ingestion is
using too much memory.
allow_external_blob_outside_bases: bool, default False
Not supported on LanceDB Cloud. Setting this raises.
Not supported for remote tables. Setting this raises.
Returns
-------
@@ -689,7 +686,7 @@ class RemoteTable(Table):
vector_column_name: Optional[str] = None,
query_type="auto",
fts_columns: Optional[Union[str, List[str]]] = None,
fast_search: bool = False,
fast_search: Optional[bool] = None,
) -> LanceVectorQueryBuilder:
"""Create a search query to find the nearest neighbors
of the given query vector. We currently support
@@ -746,7 +743,7 @@ class RemoteTable(Table):
Skip a flat search of unindexed data. This may improve
search performance but search results will not include unindexed data.
- *default False*.
- *default None*, which leaves fast search disabled.
Returns
-------
@@ -781,7 +778,7 @@ class RemoteTable(Table):
query,
query_type,
vector_column_name=vector_column_name,
fts_columns=fts_columns,
fts_columns=fts_columns or [],
fast_search=fast_search,
)
@@ -956,26 +953,20 @@ class RemoteTable(Table):
def cleanup_old_versions(self, *_):
"""
cleanup_old_versions() is a no-op on LanceDB Cloud.
Tables are automatically cleaned up and optimized.
cleanup_old_versions() is a no-op for remote tables.
"""
warnings.warn(
"cleanup_old_versions() is a no-op on LanceDB Cloud. "
"Tables are automatically cleaned up and optimized.",
"cleanup_old_versions() is a no-op for remote tables.",
stacklevel=2,
)
pass
def compact_files(self, *_):
"""
compact_files() is a no-op on LanceDB Cloud.
Tables are automatically compacted and optimized.
compact_files() is a no-op for remote tables.
"""
warnings.warn(
"compact_files() is a no-op on LanceDB Cloud. "
"Tables are automatically compacted and optimized.",
"compact_files() is a no-op for remote tables.",
stacklevel=2,
)
pass
@@ -986,17 +977,13 @@ class RemoteTable(Table):
cleanup_older_than: Optional[timedelta] = None,
delete_unverified: bool = False,
):
"""
optimize() is a no-op on LanceDB Cloud.
Indices are optimized automatically.
"""
warnings.warn(
"optimize() is a no-op on LanceDB Cloud. "
"Indices are optimized automatically.",
stacklevel=2,
"""optimize() is not supported for remote tables."""
return LOOP.run(
self._table.optimize(
cleanup_older_than=cleanup_older_than,
delete_unverified=delete_unverified,
)
)
pass
def count_rows(self, filter: Optional[str] = None) -> int:
return LOOP.run(self._table.count_rows(filter))
@@ -1044,7 +1031,7 @@ class RemoteTable(Table):
return LOOP.run(self._table.drop_columns(columns))
def set_unenforced_primary_key(self, columns: Union[str, Iterable[str]]) -> None:
"""Not supported on LanceDB Cloud."""
"""Not supported for remote tables."""
return LOOP.run(self._table.set_unenforced_primary_key(columns))
def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None:
@@ -1090,7 +1077,7 @@ class RemoteTable(Table):
)
def close_lsm_writers(self) -> None:
"""No-op on LanceDB Cloud (no local shard writers)."""
"""No-op for remote tables (no local shard writers)."""
return LOOP.run(self._table.close_lsm_writers())
def drop_index(self, index_name: str):
@@ -1158,14 +1145,10 @@ class RemoteTable(Table):
return LanceTakeQueryBuilder(self._table.take_row_ids(row_ids))
def uses_v2_manifest_paths(self) -> bool:
raise NotImplementedError(
"uses_v2_manifest_paths() is not supported on the LanceDB Cloud"
)
return LOOP.run(self._table.uses_v2_manifest_paths())
def migrate_v2_manifest_paths(self):
raise NotImplementedError(
"migrate_v2_manifest_paths() is not supported on the LanceDB Cloud"
)
return LOOP.run(self._table.migrate_manifest_paths_v2())
def blob_columns(self) -> list[str]:
return LOOP.run(self._table.blob_columns())
@@ -1176,9 +1159,7 @@ class RemoteTable(Table):
return LOOP.run(self._table.fetch_blobs(column, row_ids))
def fetch_blob_ranges(self, column: str, requests) -> pa.LargeBinaryArray:
raise NotImplementedError(
"fetch_blob_ranges() is not supported on LanceDB Cloud"
)
return LOOP.run(self._table.fetch_blob_ranges(column, list(requests)))
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
python/python/lancedb/sql.py
+79 -1
View File
@@ -3,8 +3,10 @@
"""Handles to SQL queries running on a remote database."""
from typing import Any, Mapping, Optional, Sequence, Union
from uuid import UUID
import numpy as np
import pyarrow as pa
from lancedb.background_loop import LOOP
@@ -14,6 +16,82 @@ from .arrow import AsyncRecordBatchReader
QueryDescription = _lancedb.QueryDescription
QueryParameters = Union[pa.RecordBatch, pa.Table, Sequence[Any], Mapping[str, Any]]
"""Values for a statement's ``$1`` / ``$name`` placeholders.
A sequence binds by position (``$1``, ``$2``, ...), a mapping binds by name
(``$name``), and a one-row ``pyarrow.RecordBatch`` or ``pyarrow.Table`` binds
both ways: column ``i`` is ``$<i + 1>`` and, when its name is not a number,
also ``$<name>``.
"""
def _parameter_array(value: Any) -> pa.Array:
"""One parameter as a one-element array of its own type."""
if isinstance(value, pa.Scalar):
return pa.array([value.as_py()], type=value.type)
# A 1-D array is a vector: keep its element type and make it the
# fixed-size list a vector column stores, rather than a list of Python
# floats that would come back as float64.
if isinstance(value, pa.ChunkedArray):
value = value.combine_chunks()
if isinstance(value, np.ndarray):
if value.ndim == 0:
return pa.array(value.reshape(1))
if value.ndim != 1:
raise ValueError(
"a numpy query parameter must be a scalar or a 1-D vector, "
f"got {value.ndim} dimensions"
)
value = pa.array(value)
if isinstance(value, pa.Array):
return pa.FixedSizeListArray.from_arrays(value, len(value))
return pa.array([value])
def to_parameter_batch(
parameters: Optional[QueryParameters],
) -> Optional[pa.RecordBatch]:
"""Turn ``parameters`` into the one-row batch a parameterized query sends.
Scalars keep the type pyarrow infers for them (numpy scalars keep their
dtype), a ``pyarrow.Scalar`` keeps its own type, and a 1-D numpy or
pyarrow array becomes a ``FixedSizeList`` of its element type, which is
what a vector column stores.
"""
if parameters is None:
return None
batch: pa.RecordBatch
if isinstance(parameters, pa.RecordBatch):
batch = parameters
elif isinstance(parameters, pa.Table):
batches = parameters.combine_chunks().to_batches()
batch = (
batches[0]
if len(batches) == 1
else pa.RecordBatch.from_pylist([], schema=parameters.schema)
)
elif isinstance(parameters, Mapping):
batch = pa.RecordBatch.from_arrays(
[_parameter_array(value) for value in parameters.values()],
names=[str(name) for name in parameters.keys()],
)
elif isinstance(parameters, Sequence) and not isinstance(parameters, (str, bytes)):
batch = pa.RecordBatch.from_arrays(
[_parameter_array(value) for value in parameters],
names=[str(position + 1) for position in range(len(parameters))],
)
else:
raise TypeError(
"query parameters must be a sequence, a mapping, or a one-row "
f"pyarrow RecordBatch or Table, not {type(parameters).__name__}"
)
if batch.num_rows != 1:
raise ValueError(
f"query parameters must be exactly one row, got {batch.num_rows}"
)
return batch
class AsyncQuery:
"""A handle to a submitted SQL query on an asynchronous connection."""
@@ -85,4 +163,4 @@ class Query:
LOOP.run(self._inner.cancel())
__all__ = ["AsyncQuery", "Query", "QueryDescription"]
__all__ = ["AsyncQuery", "Query", "QueryDescription", "QueryParameters"]
python/python/lancedb/table.py
+30 -10
View File
@@ -1602,8 +1602,8 @@ class Table(ABC):
language: str = "English",
max_token_length: Optional[int] = 40,
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
stem: Optional[bool] = None,
remove_stop_words: Optional[bool] = None,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
@@ -1653,6 +1653,7 @@ class Table(ABC):
- "simple": Splits text by whitespace and punctuation.
- "whitespace": Split text by whitespace, but not punctuation.
- "raw": No tokenization. The entire text is treated as a single token.
- "code": Tokenizes source code and identifiers.
- "ngram": N-Gram tokenizer.
- "icu": ICU dictionary-based word segmentation.
- "icu/split": ICU segmentation with simple-style delimiter splitting.
@@ -1667,12 +1668,16 @@ class Table(ABC):
lower_case : bool, default True
Whether to convert the token to lower case. This makes queries
case-insensitive.
stem : bool, default True
stem : bool, optional
Whether to stem the token. Stemming reduces words to their root form.
For example, in English "running" and "runs" would both be reduced to "run".
remove_stop_words : bool, default True
``None`` uses the base tokenizer's default: False for ``code`` and
``ngram``, True otherwise.
remove_stop_words : bool, optional
Whether to remove stop words. Stop words are common words that are often
removed from text before indexing. For example, in English "the" and "and".
``None`` uses the base tokenizer's default: False for ``code`` and
``ngram``, True otherwise.
custom_stop_words : list of str, optional
Custom words that replace the built-in language stop words. ``None``
uses the built-in list; an empty list explicitly uses no stop words.
@@ -3808,8 +3813,8 @@ class LanceTable(Table):
language: str = "English",
max_token_length: Optional[int] = 40,
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
stem: Optional[bool] = None,
remove_stop_words: Optional[bool] = None,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
@@ -5597,7 +5602,11 @@ class AsyncTable:
return AsyncQuery(self._inner.query(), self)
async def to_lance(self, **kwargs) -> lance.LanceDataset:
"""Return the Lance dataset backing this table.
"""Return the Lance dataset backing a local table.
Remote tables cannot be opened as Lance datasets by the client. Use
[to_arrow][lancedb.table.AsyncTable.to_arrow] or
[query][lancedb.table.AsyncTable.query] to read them through the server.
Parameters
----------
@@ -5614,6 +5623,12 @@ class AsyncTable:
>>> async def get_lance_dataset(table):
... return await table.to_lance()
"""
if not self._inner._is_native():
raise NotImplementedError(
"to_lance() is not supported for remote tables; "
"query the server instead"
)
try:
import lance
except ImportError:
@@ -5643,7 +5658,8 @@ class AsyncTable:
Parameters
----------
blob_mode: str, default "lazy"
Controls how Lance blob columns are returned.
Controls how Lance blob columns are returned. Remote tables support
"descriptions"; "bytes" and "lazy" are not yet supported.
**kwargs
Forwarded to PyArrow / Lance pandas conversion.
@@ -5661,6 +5677,9 @@ class AsyncTable:
)
return arrow_tbl.to_pandas(**kwargs)
if not self._inner._is_native():
return await self.query().to_pandas(blob_mode=blob_mode, **kwargs)
if blob_mode == "lazy" and get_uri_scheme(await self.uri()) == "memory":
return (await self.to_arrow()).to_pandas(**kwargs)
if blob_mode == "bytes" and blob_v2_column_paths(schema):
@@ -6420,9 +6439,10 @@ class AsyncTable:
)
if query.distance_type is not None:
async_query = async_query.distance_type(query.distance_type)
if query.nprobes is not None:
async_query = async_query.nprobes(query.nprobes)
if query.minimum_nprobes is not None and query.maximum_nprobes is not None:
# Set both to the minimum first to avoid min > max error.
async_query = async_query.nprobes(
async_query = async_query.minimum_nprobes(
query.minimum_nprobes
).maximum_nprobes(query.maximum_nprobes)
elif query.minimum_nprobes is not None:
python/python/tests/test_db.py
+34
View File
@@ -245,6 +245,40 @@ def test_db_contains_and_len_include_all_table_name_pages(tmp_db: lancedb.DBConn
assert "does_not_exist" not in tmp_db
@pytest.mark.asyncio
async def test_table_listing_defaults_and_zero_limit(tmp_path):
names = [f"t{i:03}" for i in range(130)]
for name in names:
(tmp_path / f"{name}.lance").mkdir()
db = lancedb.connect(tmp_path)
assert db.table_names() == names
assert db.table_names(limit=None) == names
assert db.table_names(limit=0) == []
first = db.list_tables()
assert first.tables == names[:100]
assert first.page_token
second = db.list_tables(page_token=first.page_token)
assert second.tables == names[100:]
assert second.page_token is None
assert db.list_tables(limit=200).tables == names
empty = db.list_tables(limit=0)
assert empty.tables == []
assert empty.page_token is None
adb = await lancedb.connect_async(tmp_path)
assert await adb.table_names() == names
assert await adb.table_names(limit=0) == []
first = await adb.list_tables()
assert first.tables == names[:100]
second = await adb.list_tables(page_token=first.page_token)
assert second.tables == names[100:]
assert second.page_token is None
empty = await adb.list_tables(limit=0)
assert empty.tables == []
assert empty.page_token is None
def test_db_contains_stops_after_matching_table_page(
tmp_db: lancedb.DBConnection, monkeypatch
):
python/python/tests/test_e2e_remote_db.py
+65
View File
@@ -2,13 +2,78 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import os
from datetime import timedelta
from uuid import uuid4
import lancedb
import numpy as np
import pyarrow as pa
import pytest
from lancedb import LanceDBConnection
from lancedb.index import FTS
# TODO: setup integ test mark and script
@pytest.mark.skipif(
not os.environ.get("LANCEDB_URI"), reason="Set LANCEDB_URI to a JSON FTS server"
)
def test_json_fts_against_server():
"""Requires a writable database and capacity to run the distributed indexer.
Configure LANCEDB_URI, LANCEDB_API_KEY, and optionally LANCEDB_REGION and
LANCEDB_HOST_OVERRIDE before running this test.
"""
uri = os.environ["LANCEDB_URI"]
assert uri.startswith("db://"), "LANCEDB_URI must identify a remote database"
conn = lancedb.connect(
uri,
api_key=os.environ.get("LANCEDB_API_KEY"),
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ.get("LANCEDB_HOST_OVERRIDE"),
)
name = f"json_fts_{uuid4().hex}"
table = conn.create_table(
name,
pa.table(
{
"id": [0, 1, 2],
"doc": pa.array(
[
'{"meta":{"tag":"nature"},"author":"alice"}',
'{"meta":{"tag":"city"},"author":"nature"}',
'{"meta":{"tag":"nature"},"author":"bob"}',
],
type=pa.json_(),
),
"raw": pa.array([b"nature"] * 3, type=pa.large_binary()),
}
),
)
try:
table.create_index("doc", config=FTS(), name="doc_fts")
table.wait_for_index(["doc_fts"], timeout=timedelta(minutes=5))
stats = table.index_stats("doc_fts")
assert stats.num_indexed_rows == 3
assert stats.num_unindexed_rows == 0
assert stats.index_type == "FTS"
for query, expected in [
("meta.tag,str,nature", [0, 2]),
("meta.tag,str,alice", []),
("meta.tag,str,tag", []),
]:
search = table.search(query, query_type="fts", fts_columns="doc")
assert sorted(row["id"] for row in search.to_list()) == expected
plan = search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("raw", config=FTS())
finally:
conn.drop_table(name)
@pytest.mark.skip(reason="Need to set up a local server")
def test_against_local_server():
conn = LanceDBConnection("lancedb+http://localhost:10024")
python/python/tests/test_first_class_function_slice2.py
+1 -1
View File
@@ -1555,7 +1555,7 @@ def _mock_remote_function_catalog():
)["image"],
"signature": body["signature"],
"secret_bindings": body.get("secret_bindings", []),
"created_at": "2026-08-21T00:00:00Z",
"created_at_millis": 1787270400000, # 2026-08-21T00:00:00Z
}
response = {"job_id": "job-register"}
status = 202
python/python/tests/test_fts.py
+170
View File
@@ -13,6 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import random
import shutil
@@ -44,6 +45,75 @@ from utils import exception_output
TEST_LANGUAGE_MODEL_HOME = Path(__file__).parent / "models"
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
@pytest.mark.parametrize("block_size", [128, 256])
def test_json_fts(tmp_path, block_size):
db = ldb.connect(tmp_path)
data = pa.table(
{
"id": [0, 1, 2, 3],
"doc": pa.array(
[
'{"title":"brown fox","meta":{"tag":"nature"},"author":"alice"}',
'{"title":"blue car","meta":{"tag":"city"}}',
'{"title":"brown bear","meta":{"tag":"nature"}}',
'{"title":"red fox","author":"alice"}',
],
type=pa.json_(),
),
}
)
table = db.create_table("json_fts", data)
table.create_index("doc", config=FTS(block_size=block_size), name="doc_fts")
table = db.open_table("json_fts")
index = table.list_indices()[0]
assert index.name == "doc_fts"
assert index.index_type == "FTS"
assert index.columns == ["doc"]
stats = table.index_stats("doc_fts")
assert stats.num_indexed_rows == 4
assert stats.num_unindexed_rows == 0
for query, expected in [
("title,str,brown", [0, 2]),
("meta.tag,str,nature", [0, 2]),
("author,str,alice", [0, 3]),
("title,str,alice", []),
("title,str,title", []),
]:
search = table.search(query, query_type="fts", fts_columns="doc")
assert sorted(row["id"] for row in search.to_list()) == expected
plan = search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
@pytest.mark.asyncio
async def test_json_fts_async(tmp_path):
db = await ldb.connect_async(tmp_path)
table = await db.create_table(
"json_fts",
pa.table({"doc": pa.array(['{"meta":{"tag":"nature"}}'], type=pa.json_())}),
)
await table.create_index("doc", config=FTS())
search = table.query().nearest_to_text(MatchQuery("meta.tag,str,nature", "doc"))
assert len(await search.to_list()) == 1
plan = await search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
@pytest.mark.parametrize("dtype", [pa.binary(), pa.large_binary()])
def test_fts_rejects_raw_binary(tmp_path, dtype):
table = ldb.connect(tmp_path).create_table(
"binary_fts", pa.table({"doc": pa.array([b'{"title":"brown"}'], type=dtype)})
)
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("doc", config=FTS())
assert table.list_indices() == []
@pytest.fixture
def table(tmp_path) -> ldb.table.LanceTable:
# Use local random state to avoid affecting other tests
@@ -894,6 +964,78 @@ def test_tokenize_uses_simple_index_tokenizer(mem_db: DBConnection):
]
@pytest.mark.parametrize("legacy", [False, True])
@pytest.mark.parametrize(
("base_tokenizer", "options", "expected_filters", "expected_tokens"),
[
(
"code",
{},
(False, False),
[("running", 0), ("in", 1), ("cafe", 2)],
),
(
"code",
{"stem": None, "remove_stop_words": None},
(False, False),
[("running", 0), ("in", 1), ("cafe", 2)],
),
(
"code",
{"stem": True},
(True, False),
[("run", 0), ("in", 1), ("cafe", 2)],
),
(
"code",
{"remove_stop_words": True},
(False, True),
[("running", 0), ("cafe", 2)],
),
(
"code",
{"stem": True, "remove_stop_words": True},
(True, True),
[("run", 0), ("cafe", 2)],
),
(
"simple",
{},
(True, True),
[("run", 0), ("cafe", 2)],
),
(
"simple",
{"stem": False, "remove_stop_words": False},
(False, False),
[("running", 0), ("in", 1), ("cafe", 2)],
),
],
)
def test_fts_tokenizer_filter_defaults(
mem_db, legacy, base_tokenizer, options, expected_filters, expected_tokens
):
text = "Running in café"
table = mem_db.create_table("test_filter_defaults", data=[{"text": text}])
if legacy:
with pytest.warns(DeprecationWarning, match="create_fts_index"):
table.create_fts_index("text", base_tokenizer=base_tokenizer, **options)
else:
table.create_index("text", config=FTS(base_tokenizer=base_tokenizer, **options))
details = table.list_indices()[0].index_details
assert (details["stem"], details["remove_stop_words"]) == expected_filters
assert [
(token.text, token.position) for token in table.tokenize(text, column="text")
] == expected_tokens
assert [
(token.text, token.position)
for token in ldb.tokenize(text, base_tokenizer=base_tokenizer, **options)
] == expected_tokens
if base_tokenizer == "code" and not expected_filters[1]:
assert table.search("in", query_type="fts").to_list()[0]["text"] == text
def test_tokenize_uses_icu_index_tokenizer_by_name(mem_db: DBConnection):
data = pa.table({"text": ["Hello, こんにちは世界!"]})
table = mem_db.create_table("test_tokenize_icu", data=data)
@@ -1137,6 +1279,34 @@ def test_fts_lindera_tokenizer(
assert [row["text"] for row in results] == ["成田国際空港"]
@pytest.mark.parametrize("fuzziness", [None, 0, 1, 2])
def test_match_query_fuzziness_to_json(fuzziness):
query = MatchQuery("puppy", "text", fuzziness=fuzziness)
assert query.fuzziness == fuzziness
assert json.loads(query.to_json())["match"]["fuzziness"] == fuzziness
@pytest.mark.parametrize(
"options, expected",
[
pytest.param({}, ["ab"], id="default-exact"),
pytest.param({"fuzziness": 0}, ["ab"], id="explicit-exact"),
pytest.param({"fuzziness": None}, ["ab"], id="automatic"),
pytest.param({"fuzziness": 1}, ["ab", "ac"], id="one-edit"),
pytest.param({"fuzziness": 2}, ["ab", "ac", "zz"], id="two-edits"),
],
)
def test_match_query_fuzziness_search(tmp_db, options, expected):
# Automatic fuzziness is zero for two-character terms.
table = tmp_db.create_table("fuzzy", pa.table({"text": ["ab", "ac", "zz"]}))
table.create_index("text", config=FTS(stem=False, remove_stop_words=False))
results = table.search(MatchQuery("ab", "text", **options)).to_list()
assert sorted(row["text"] for row in results) == sorted(expected)
def test_fts_query_to_json():
"""Test that FTS query to_json() produces valid JSON strings with exact format."""
python/python/tests/test_hybrid_query.py
+14
View File
@@ -355,6 +355,20 @@ def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table):
)
def test_hybrid_query_nprobes_remains_independent(sync_table: Table):
query = sync_table.search(query_type="hybrid").nprobes(20)
assert query._nprobes == 20
assert query._minimum_nprobes is None
assert query._maximum_nprobes is None
query = query.minimum_nprobes(5).nprobes(20)
assert query._nprobes == 20
assert query._minimum_nprobes == 5
assert query._maximum_nprobes is None
def test_hybrid_query_distance_range(sync_table: Table):
reranker = RRFReranker(return_score="all")
result = (
python/python/tests/test_namespace.py
+155 -1
View File
@@ -6,12 +6,17 @@
import tempfile
import shutil
import importlib
import http.server
import json
import threading
from urllib.parse import parse_qs, unquote, urlparse
import pytest
import pyarrow as pa
import lancedb
from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
from lancedb.listing import AsyncListing, Listing
from lancedb.namespace import _MAX_QUERY_K
from lancedb.namespace import _MAX_QUERY_K, _query_to_namespace_request
from lancedb.query import Query
from lancedb.table import AsyncTable, LanceTable
@@ -73,6 +78,132 @@ def _namespace_lance_table(namespace_client: _NamespaceClient) -> LanceTable:
return table
@pytest.mark.asyncio
@pytest.mark.parametrize("namespace_path", [[], ["test_ns"]])
async def test_namespace_table_names_defaults_and_limits(tmp_path, namespace_path):
db = lancedb.connect_namespace("dir", {"root": str(tmp_path)})
if namespace_path:
db.create_namespace(namespace_path)
names = [f"t{idx:03}" for idx in range(12)]
schema = pa.schema([pa.field("id", pa.int64())])
for name in names:
db.create_table(name, schema=schema, namespace_path=namespace_path)
assert list(db.table_names(namespace_path=namespace_path)) == names
assert list(db.table_names(limit=None, namespace_path=namespace_path)) == names
assert list(db.table_names(limit=0, namespace_path=namespace_path)) == []
assert list(db.table_names(limit=2, namespace_path=namespace_path)) == names[:2]
assert (
list(
db.table_names(page_token=names[1], limit=2, namespace_path=namespace_path)
)
== names[2:4]
)
adb = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
assert list(await adb.table_names(namespace_path=namespace_path)) == names
assert (
list(await adb.table_names(limit=None, namespace_path=namespace_path)) == names
)
assert list(await adb.table_names(limit=0, namespace_path=namespace_path)) == []
assert (
list(await adb.table_names(limit=2, namespace_path=namespace_path)) == names[:2]
)
assert (
list(
await adb.table_names(
page_token=names[1], limit=2, namespace_path=namespace_path
)
)
== names[2:4]
)
@pytest.fixture
def paginated_rest_namespace():
names = [f"t{idx:03}" for idx in reversed(range(12))]
requests = []
class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
url = urlparse(self.path)
query = parse_qs(url.query)
requests.append((unquote(url.path), query))
token = query.get("page_token", [None])[0]
if token not in {None, "opaque-next"}:
self.send_error(400, "Unknown page token")
return
start = 0 if token is None else 10
body = {"tables": names[start : start + 10]}
if start == 0:
body["page_token"] = "opaque-next"
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps(body).encode())
def log_message(self, *args):
pass
with http.server.HTTPServer(("127.0.0.1", 0), Handler) as server:
thread = threading.Thread(target=server.serve_forever)
thread.start()
try:
yield {"uri": f"http://127.0.0.1:{server.server_port}"}, requests
finally:
server.shutdown()
thread.join()
@pytest.mark.asyncio
@pytest.mark.parametrize("namespace_path", [[], ["n"]])
async def test_rest_namespace_table_names_collects_pages(
paginated_rest_namespace, namespace_path
):
properties, requests = paginated_rest_namespace
names = [f"t{idx:03}" for idx in range(12)]
db = lancedb.connect_namespace("rest", properties)
first = db.list_tables(namespace_path=namespace_path)
second = db.list_tables(namespace_path=namespace_path, page_token=first.page_token)
assert len(first.tables) == 10
assert first.page_token == "opaque-next"
assert sorted(first.tables + second.tables) == names
assert second.page_token is None
assert list(db.table_names(namespace_path=namespace_path)) == names
assert list(db.table_names(limit=None, namespace_path=namespace_path)) == names
assert list(db.table_names(limit=11, namespace_path=namespace_path)) == names[:11]
assert (
list(db.table_names(page_token="t005", limit=2, namespace_path=namespace_path))
== names[6:8]
)
adb = lancedb.connect_namespace_async("rest", properties)
assert list(await adb.table_names(namespace_path=namespace_path)) == names
assert (
list(await adb.table_names(limit=None, namespace_path=namespace_path)) == names
)
assert (
list(await adb.table_names(limit=11, namespace_path=namespace_path))
== names[:11]
)
assert (
list(
await adb.table_names(
page_token="t005", limit=2, namespace_path=namespace_path
)
)
== names[6:8]
)
request_count = len(requests)
assert list(db.table_names(limit=0, namespace_path=namespace_path)) == []
assert list(await adb.table_names(limit=0, namespace_path=namespace_path)) == []
assert len(requests) == request_count
identifier = "$" if not namespace_path else "n"
assert all(path == f"/v1/namespace/{identifier}/table/list" for path, _ in requests)
class TestNamespaceConnection:
"""Test namespace-based LanceDB connection using DirectoryNamespace."""
@@ -981,6 +1112,29 @@ class TestPushdownOperations:
]
assert all(r.k <= 2**31 - 1 for r in namespace_client.requests)
def test_probe_fields_pass_through_independently(self):
request = _query_to_namespace_request(
["geneva", "hist"],
Query(
vector=[1.0, 2.0],
nprobes=20,
minimum_nprobes=3,
maximum_nprobes=10,
),
)
assert request.nprobes == 20
assert request.minimum_nprobes == 3
assert request.maximum_nprobes == 10
request = _query_to_namespace_request(
["geneva", "hist"],
Query(vector=[1.0, 2.0], nprobes=5, maximum_nprobes=0),
)
assert request.nprobes == 5
assert request.maximum_nprobes == 0
assert request.to_dict()["maximum_nprobes"] == 0
@pytest.mark.asyncio
class TestAsyncPushdownOperations:
python/python/tests/test_query.py
+88 -28
View File
@@ -995,12 +995,22 @@ def test_invalid_nprobes_sync(table):
ValueError,
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
LanceVectorQueryBuilder(table, [0, 0], "vector").maximum_nprobes(5).to_list()
(
LanceVectorQueryBuilder(table, [0, 0], "vector")
.minimum_nprobes(10)
.maximum_nprobes(5)
.to_list()
)
with pytest.raises(
ValueError,
match="minimum_nprobes must be less than or equal to maximum_nprobes",
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
LanceVectorQueryBuilder(table, [0, 0], "vector").minimum_nprobes(100).to_list()
(
LanceVectorQueryBuilder(table, [0, 0], "vector")
.maximum_nprobes(5)
.minimum_nprobes(100)
.to_list()
)
def test_nprobes_works_sync(table):
@@ -1039,12 +1049,22 @@ async def test_invalid_nprobes_async(table_async: AsyncTable):
ValueError,
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
await table_async.vector_search([0, 0]).maximum_nprobes(5).to_list()
await (
table_async.vector_search([0, 0])
.minimum_nprobes(10)
.maximum_nprobes(5)
.to_list()
)
with pytest.raises(
ValueError,
match="minimum_nprobes must be less than or equal to maximum_nprobes",
):
await table_async.vector_search([0, 0]).minimum_nprobes(100).to_list()
await (
table_async.vector_search([0, 0])
.maximum_nprobes(5)
.minimum_nprobes(100)
.to_list()
)
def test_query_builder_with_prefilter(table):
@@ -1635,8 +1655,9 @@ def test_query_serialization_sync(table: lancedb.table.Table):
q,
vector_column="vector",
vector=[5.0, 6.0],
minimum_nprobes=10,
maximum_nprobes=10,
nprobes=10,
minimum_nprobes=None,
maximum_nprobes=None,
refine_factor=5,
)
@@ -1654,8 +1675,19 @@ def test_query_serialization_sync(table: lancedb.table.Table):
q,
vector_column="vector",
vector=[5.0, 6.0],
minimum_nprobes=50,
maximum_nprobes=50,
nprobes=50,
minimum_nprobes=None,
maximum_nprobes=None,
)
q = table.search([5.0, 6.0]).minimum_nprobes(5).nprobes(50).to_query_object()
check_set_props(
q,
vector_column="vector",
vector=[5.0, 6.0],
nprobes=50,
minimum_nprobes=5,
maximum_nprobes=None,
)
q = table.search([5.0, 6.0]).maximum_nprobes(10).to_query_object()
@@ -1667,6 +1699,15 @@ def test_query_serialization_sync(table: lancedb.table.Table):
minimum_nprobes=None,
)
q = table.search([5.0, 6.0]).nprobes(5).maximum_nprobes(0).to_query_object()
check_set_props(
q,
vector_column="vector",
vector=[5.0, 6.0],
nprobes=5,
maximum_nprobes=0,
)
q = table.search([5.0, 6.0]).distance_range(0.0, 1.0).to_query_object()
check_set_props(
q, vector_column="vector", vector=[5.0, 6.0], lower_bound=0.0, upper_bound=1.0
@@ -1716,8 +1757,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
limit=10,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
)
@@ -1727,8 +1768,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1739,8 +1780,9 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
postfilter=False,
minimum_nprobes=50,
maximum_nprobes=50,
nprobes=50,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1759,8 +1801,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
filter="id = 1",
postfilter=True,
vector=sample_vector,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
)
@@ -1774,8 +1816,9 @@ async def test_query_serialization_async(table_async: AsyncTable):
check_set_props(
q,
vector=sample_vector,
minimum_nprobes=10,
maximum_nprobes=10,
nprobes=10,
minimum_nprobes=None,
maximum_nprobes=None,
refine_factor=5,
postfilter=False,
with_row_id=False,
@@ -1788,7 +1831,24 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
minimum_nprobes=5,
maximum_nprobes=20,
maximum_nprobes=None,
postfilter=False,
with_row_id=False,
bypass_vector_index=False,
limit=10,
)
q = (
(await table_async.search([5.0, 6.0]))
.nprobes(5)
.maximum_nprobes(0)
.to_query_object()
)
check_set_props(
q,
vector=sample_vector,
nprobes=5,
maximum_nprobes=0,
postfilter=False,
with_row_id=False,
bypass_vector_index=False,
@@ -1806,8 +1866,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
lower_bound=0.0,
upper_bound=1.0,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1819,8 +1879,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
distance_type="cosine",
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1832,8 +1892,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
ef=7,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1845,8 +1905,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
bypass_vector_index=True,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
limit=10,
)
python/python/tests/test_remote_db.py
+588 -26
View File
@@ -13,13 +13,19 @@ import threading
import time
from unittest.mock import MagicMock, patch
import uuid
from urllib.parse import parse_qs, urlparse
from packaging.version import Version
import lancedb
import numpy as np
from lancedb.conftest import MockNonNormTextEmbeddingFunction, MockTextEmbeddingFunction
from lancedb.embeddings import EmbeddingFunctionConfig, EmbeddingFunctionRegistry
from lancedb.query import AsyncQuery, ColumnOrdering, LanceVectorQueryBuilder
from lancedb.query import (
AsyncQuery,
ColumnOrdering,
LanceVectorQueryBuilder,
MatchQuery,
)
from lancedb.remote import ClientConfig
from lancedb.remote.errors import HttpError, RetryError
import pytest
@@ -100,6 +106,67 @@ async def mock_lancedb_connection_async(handler, **client_config):
handle.join()
def paginated_table_listing_handler(request):
url = urlparse(request.path)
assert url.path in {
"/v1/table/",
"/v1/namespace/$/table/list",
"/v1/namespace/n/table/list",
}
query = parse_qs(url.query)
limit = int(query.get("limit", [10])[0])
assert limit > 0
token = query.get("page_token", ["page-0"])[0]
start = int(token.removeprefix("page-"))
end = min(start + limit, 130)
body = {"tables": [f"t{i:03}" for i in range(start, end)]}
if end < 130:
body["page_token"] = f"page-{end}"
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(json.dumps(body).encode())
@pytest.mark.parametrize("namespace", [[], ["n"]])
def test_remote_table_listing_defaults_and_zero_limit(namespace):
names = [f"t{i:03}" for i in range(130)]
with mock_lancedb_connection(paginated_table_listing_handler) as db:
assert db.table_names(namespace_path=namespace) == names
assert db.table_names(limit=None, namespace_path=namespace) == names
assert db.table_names(limit=0, namespace_path=namespace) == []
first = db.list_tables(namespace_path=namespace)
assert first.tables == names[:100]
assert first.page_token == "page-100"
second = db.list_tables(namespace_path=namespace, page_token=first.page_token)
assert second.tables == names[100:]
assert second.page_token is None
assert db.list_tables(namespace_path=namespace, limit=200).tables == names
empty = db.list_tables(namespace_path=namespace, limit=0)
assert empty.tables == []
assert empty.page_token is None
@pytest.mark.asyncio
@pytest.mark.parametrize("namespace", [[], ["n"]])
async def test_async_remote_table_listing_defaults_and_zero_limit(namespace):
names = [f"t{i:03}" for i in range(130)]
async with mock_lancedb_connection_async(paginated_table_listing_handler) as db:
assert await db.table_names(namespace_path=namespace) == names
assert await db.table_names(limit=0, namespace_path=namespace) == []
first = await db.list_tables(namespace_path=namespace)
assert first.tables == names[:100]
assert first.page_token == "page-100"
second = await db.list_tables(
namespace_path=namespace, page_token=first.page_token
)
assert second.tables == names[100:]
assert second.page_token is None
empty = await db.list_tables(namespace_path=namespace, limit=0)
assert empty.tables == []
assert empty.page_token is None
@pytest.mark.asyncio
async def test_async_remote_db():
def handler(request):
@@ -1018,6 +1085,77 @@ def test_table_create_indices():
table.drop_index("custom_fts_idx")
@pytest.mark.parametrize("legacy", [False, True])
@pytest.mark.parametrize(
("base_tokenizer", "options", "expected_filters"),
[
("code", {}, (False, False)),
("code", {"stem": None, "remove_stop_words": None}, (False, False)),
("code", {"stem": True}, (True, False)),
("code", {"remove_stop_words": True}, (False, True)),
("code", {"stem": True, "remove_stop_words": True}, (True, True)),
("simple", {}, (True, True)),
("simple", {"stem": False, "remove_stop_words": False}, (False, False)),
],
)
def test_remote_fts_tokenizer_filter_defaults(
legacy, base_tokenizer, options, expected_filters
):
from lancedb.index import FTS
received_requests = []
def handler(request):
if request.path == "/v1/table/test/create_index/":
content_len = int(request.headers["Content-Length"])
received_requests.append(json.loads(request.rfile.read(content_len)))
request.send_response(200)
request.end_headers()
elif request.path == "/v1/table/test/create/?mode=create":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b"{}")
elif request.path == "/v1/table/test/describe/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
json.dumps(
{
"version": 1,
"schema": {
"fields": [
{
"name": "text",
"type": {"type": "string"},
"nullable": False,
}
]
},
}
).encode()
)
else:
request.send_response(404)
request.end_headers()
with mock_lancedb_connection(handler) as db:
table = db.create_table("test", [{"text": "Running in cafes"}])
if legacy:
with pytest.warns(DeprecationWarning, match="create_fts_index"):
table.create_fts_index("text", base_tokenizer=base_tokenizer, **options)
else:
table.create_index(
"text", config=FTS(base_tokenizer=base_tokenizer, **options)
)
assert len(received_requests) == 1
params = received_requests[0]
assert params["base_tokenizer"] == base_tokenizer
assert (params["stem"], params["remove_stop_words"]) == expected_filters
def test_remote_create_index_async_returns_job():
from lancedb.index import BTree
@@ -1272,7 +1410,73 @@ def test_remote_job_wait_raises_on_failure():
job.wait()
def test_remote_create_index_new_api():
@pytest.mark.parametrize("server_version", ["0.2.0", "0.6.0"])
def test_remote_json_fts_config(server_version):
from lancedb.index import FTS
received_requests = []
def handler(request):
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.send_header("phalanx-version", server_version)
request.end_headers()
if request.path == "/v1/table/test/describe/":
fields = [
{
"name": "doc",
"type": {"type": "large_binary"},
"nullable": True,
"metadata": {"ARROW:extension:name": "lance.json"},
},
{
"name": "raw",
"type": {"type": "large_binary"},
"nullable": True,
},
]
request.wfile.write(
json.dumps({"version": 1, "schema": {"fields": fields}}).encode()
)
elif request.path == "/v1/table/test/create_index/":
received_requests.append(read_json_body(request))
request.wfile.write(b"{}")
else:
raise AssertionError(f"Unexpected request: {request.path}")
with mock_lancedb_connection(handler) as db:
table = db.open_table("test")
table.create_index(
"doc",
config=FTS(
with_position=True,
base_tokenizer="whitespace",
stem=False,
remove_stop_words=False,
lower_case=False,
block_size=256,
),
name="doc_fts",
replace=False,
)
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("raw", config=FTS())
assert len(received_requests) == 1
request = received_requests[0]
assert request["column"] == "doc"
assert request["index_type"] == "FTS"
assert request["name"] == "doc_fts"
assert request["replace"] is False
assert request["with_position"] is True
assert request["base_tokenizer"] == "whitespace"
assert request["stem"] is False
assert request["remove_stop_words"] is False
assert request["lower_case"] is False
assert request["block_size"] == 256
def test_remote_create_index_new_api(caplog):
received_requests = []
def handler(request):
@@ -1333,7 +1537,8 @@ def test_remote_create_index_new_api():
from lancedb.index import BTree, FTS, IvfPq, IvfRq
with mock_lancedb_connection(handler) as db:
table = db.create_table("test", [{"id": 1}])
table = db.create_table("test", [{"id": 1}], embedding_functions=[])
assert "embedding_functions is not yet supported" not in caplog.text
# New API: column-first, config= kwarg. Should NOT emit DeprecationWarning.
import warnings as _warnings
@@ -1355,6 +1560,8 @@ def test_remote_create_index_new_api():
vector_column_name="vector",
index_type="IVF_RQ",
num_partitions=8,
replace=False,
accelerator="cuda",
)
assert len(received_requests) == 6
@@ -1368,6 +1575,160 @@ def test_remote_create_index_new_api():
]
assert received_requests[2]["block_size"] == 256
assert received_requests[4]["replace"] is False
assert received_requests[5]["replace"] is False
assert "replace is not supported" not in caplog.text
assert caplog.messages == [
"GPU accelerator is not supported for remote tables and will be ignored."
]
def remote_unsupported_handler(request):
assert request.path == "/v1/table/test/describe/"
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
json.dumps(
dict(
version=1,
schema=dict(
fields=[
dict(name="id", type={"type": "int64"}, nullable=False),
dict(
name="vector",
type={
"type": "fixed_size_list",
"fields": [
dict(
name="item",
type={"type": "float"},
nullable=True,
)
],
"length": 2,
},
nullable=False,
),
]
),
)
).encode()
)
def remote_unsupported_operations():
from lancedb.index import HnswPq
return [
("uses_v2_manifest_paths", (), {}, "uses_v2_manifest_paths"),
("migrate_v2_manifest_paths", (), {}, "migrate_manifest_paths_v2"),
("set_unenforced_primary_key", (["id"],), {}, "set_unenforced_primary_key"),
("fetch_blob_ranges", ("image", [(1, 0, 1)]), {}, "fetch_blob_ranges"),
("optimize", (), {}, "optimize"),
("create_index", ("vector",), {"config": HnswPq()}, "IVF_HNSW_PQ"),
]
@pytest.mark.parametrize(
"method,args,kwargs,operation", remote_unsupported_operations()
)
def test_remote_unsupported_operations_sync(method, args, kwargs, operation):
with mock_lancedb_connection(remote_unsupported_handler) as db:
table = db.open_table("test")
with pytest.raises(NotImplementedError) as error:
getattr(table, method)(*args, **kwargs)
expected = f"LanceDBError: not supported: {operation} is not supported "
expected += "for remote tables."
if operation == "IVF_HNSW_PQ":
expected += " Please use IVF_HNSW_SQ instead."
assert str(error.value) == expected
@pytest.mark.asyncio
@pytest.mark.parametrize(
"method,args,kwargs,operation", remote_unsupported_operations()
)
async def test_remote_unsupported_operations_async(method, args, kwargs, operation):
async with mock_lancedb_connection_async(remote_unsupported_handler) as db:
table = await db.open_table("test")
if method == "migrate_v2_manifest_paths":
method = "migrate_manifest_paths_v2"
with pytest.raises(NotImplementedError) as error:
await getattr(table, method)(*args, **kwargs)
expected = f"LanceDBError: not supported: {operation} is not supported "
expected += "for remote tables."
if operation == "IVF_HNSW_PQ":
expected += " Please use IVF_HNSW_SQ instead."
assert str(error.value) == expected
def test_remote_unsupported_legacy_hnsw_pq():
with mock_lancedb_connection(remote_unsupported_handler) as db:
table = db.open_table("test")
with pytest.warns(DeprecationWarning, match="create_index"):
with pytest.raises(
NotImplementedError,
match="IVF_HNSW_PQ is not supported for remote tables. "
"Please use IVF_HNSW_SQ instead.",
):
table.create_index(
vector_column_name="vector", index_type="IVF_HNSW_PQ"
)
def test_remote_python_only_messages(caplog):
with mock_lancedb_connection(remote_unsupported_handler) as db:
with caplog.at_level("INFO"):
table = db.open_table("test", storage_options={}, index_cache_size=1)
assert "storage_options is ignored for remote tables" in caplog.text
assert "index_cache_size is ignored for remote tables" in caplog.text
for method in ["to_arrow", "to_pandas"]:
with pytest.raises(NotImplementedError) as error:
getattr(table, method)()
assert str(error.value) == f"{method}() is not supported for remote tables."
for method in ["compact_files", "cleanup_old_versions"]:
with pytest.warns(UserWarning) as warning:
getattr(table, method)()
assert (
str(warning[0].message) == f"{method}() is a no-op for remote tables."
)
def test_remote_embedding_config_does_not_warn_or_drop_metadata(caplog):
received_tables = []
def handler(request):
assert request.path == "/v1/table/test/create/?mode=create"
if request.headers.get("Transfer-Encoding") == "chunked":
body = bytearray()
while size := int(request.rfile.readline().split(b";")[0], 16):
body.extend(request.rfile.read(size))
assert request.rfile.read(2) == b"\r\n"
assert request.rfile.read(2) == b"\r\n"
else:
body = request.rfile.read(int(request.headers["Content-Length"]))
received_tables.append(pa.ipc.open_stream(body).read_all())
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b"{}")
config = EmbeddingFunctionConfig(
source_column="text",
vector_column="vector",
function=MockNonNormTextEmbeddingFunction.create(),
)
schema = pa.schema(
[pa.field("text", pa.string()), pa.field("vector", pa.list_(pa.float32(), 10))]
)
with mock_lancedb_connection(handler) as db:
db.create_table("test", schema=schema, embedding_functions=[config])
assert caplog.messages == []
assert len(received_tables) == 1
metadata = EmbeddingFunctionRegistry.get_instance().get_table_metadata([config])
assert (
received_tables[0].schema.metadata == pa.schema([], metadata=metadata).metadata
)
def test_table_wait_for_index_timeout():
@@ -1674,6 +2035,108 @@ def query_test_table(
yield table
@pytest.mark.parametrize(
"query,search_kwargs,expected_column",
[
pytest.param([0.5] * 8, {}, "vector", id="vector-8"),
pytest.param([0.5] * 4, {}, "vec2", id="vector-4"),
pytest.param(np.array([0.5] * 4), {}, "vec2", id="numpy-vector"),
pytest.param([[0.5] * 4, [0.2] * 4], {}, "vec2", id="batch-vectors"),
pytest.param(
[0.5] * 4, {"vector_column_name": "vec2"}, "vec2", id="explicit-column"
),
pytest.param(None, {}, None, id="scan"),
pytest.param("hello", {"query_type": "fts"}, None, id="fts"),
pytest.param(
"hello", {"query_type": "fts", "fts_columns": "text"}, None, id="fts-column"
),
pytest.param(MatchQuery("hello", "text"), {}, None, id="structured-fts"),
],
)
def test_query_object_sync_matches_local(mem_db, query, search_kwargs, expected_column):
vector_columns = {"vector": 8, "vec2": 4}
schema = pa.schema(
[
pa.field(name, pa.list_(pa.float32(), dim))
for name, dim in vector_columns.items()
]
)
local = mem_db.create_table("test", schema=schema)
local_query = local.search(query, **search_kwargs).limit(2).to_query_object()
assert local_query.vector_column == expected_column
assert local_query.fast_search is None
with query_test_table(None, vector_columns=vector_columns) as remote:
remote_query = remote.search(query, **search_kwargs).limit(2).to_query_object()
assert remote_query == local_query
@pytest.mark.parametrize("fast_search", [None, False, True])
@pytest.mark.parametrize("query,query_type", [([1, 2, 3], "vector"), ("hello", "fts")])
def test_query_object_sync_preserves_fast_search(fast_search, query, query_type):
vector_columns = {"vector": 3} if query_type == "vector" else {}
with query_test_table(None, vector_columns=vector_columns) as table:
query_obj = table.search(
query, query_type=query_type, fast_search=fast_search
).to_query_object()
assert query_obj.fast_search is fast_search
def test_query_sync_rejects_ambiguous_vector_column(mem_db):
vector_columns = {"vector": 4, "vec2": 4}
schema = pa.schema(
[
pa.field(name, pa.list_(pa.float32(), dim))
for name, dim in vector_columns.items()
]
)
local = mem_db.create_table("test", schema=schema)
with pytest.raises(ValueError, match="Candidates:.*vector.*vec2") as local_error:
local.search([0.5] * 4)
with query_test_table(None, vector_columns=vector_columns) as remote:
with pytest.raises(ValueError) as remote_error:
remote.search([0.5] * 4)
assert str(remote_error.value) == str(local_error.value)
@pytest.mark.parametrize("vector_columns", [{}, {"v1": 4, "v2": 8}])
@pytest.mark.parametrize("query_type", ["auto", "fts"])
def test_query_sync_fts_without_embeddings(vector_columns, query_type):
seen = []
def handler(body):
seen.append(body)
assert body["full_text_query"] == {"query": "hello", "columns": []}
assert body["vector"] == []
assert "vector_column" not in body
return pa.table({"id": [1]})
with query_test_table(handler, vector_columns=vector_columns) as table:
assert table.search("hello", query_type=query_type).to_list() == [{"id": 1}]
assert len(seen) == 1
def test_query_sync_auto_with_embedding_function():
def handler(body):
assert body["vector_column"] == "vector"
assert body["vector"] == [0.0] * 10
assert "full_text_query" not in body
return pa.table({"id": [1]})
config = EmbeddingFunctionConfig(
source_column="text",
vector_column="vector",
function=MockTextEmbeddingFunction.create(),
)
with query_test_table(
handler, vector_columns={"vector": 10}, embedding_functions=[config]
) as table:
query = table.search("hello")
assert query.to_query_object().vector_column == "vector"
assert query.to_list() == [{"id": 1}]
def test_head():
def handler(body):
assert body == {
@@ -1701,9 +2164,6 @@ def test_query_sync_minimal():
"ef": None,
"vector": [1.0, 2.0, 3.0],
"vector_column": "vector",
"nprobes": 20,
"minimum_nprobes": 20,
"maximum_nprobes": 20,
"version": None,
}
@@ -1875,8 +2335,6 @@ def test_query_sync_maximal():
"refine_factor": 10,
"vector": [1.0, 2.0, 3.0],
"nprobes": 5,
"minimum_nprobes": 5,
"maximum_nprobes": 5,
"lower_bound": None,
"upper_bound": None,
"ef": None,
@@ -1972,7 +2430,7 @@ async def test_query_async_nprobes_zero(hybrid):
assert requests == ["/v1/table/test/describe/"]
def test_query_sync_nprobes():
def test_query_sync_probe_bounds():
def handler(body):
assert body == {
"k": 10,
@@ -1984,7 +2442,6 @@ def test_query_sync_nprobes():
"upper_bound": None,
"ef": None,
"vector": [1.0, 2.0, 3.0],
"nprobes": 5,
"minimum_nprobes": 5,
"maximum_nprobes": 15,
"version": None,
@@ -2001,7 +2458,28 @@ def test_query_sync_nprobes():
)
def test_query_sync_no_max_nprobes():
def test_query_sync_nprobes_passthrough():
def handler(body):
assert body["nprobes"] == 20
assert "minimum_nprobes" not in body
assert "maximum_nprobes" not in body
return pa.table({"id": [1]})
with query_test_table(handler) as table:
table.search([1, 2, 3]).nprobes(20).to_list()
def test_query_sync_nprobes_with_unbounded_maximum():
def handler(body):
assert body["nprobes"] == 20
assert body["maximum_nprobes"] == 0
return pa.table({"id": [1]})
with query_test_table(handler) as table:
table.search([1, 2, 3]).nprobes(20).maximum_nprobes(0).to_list()
def test_query_sync_minimum_nprobes_only():
def handler(body):
assert body == {
"k": 10,
@@ -2013,9 +2491,7 @@ def test_query_sync_no_max_nprobes():
"upper_bound": None,
"ef": None,
"vector": [1.0, 2.0, 3.0],
"nprobes": 5,
"minimum_nprobes": 5,
"maximum_nprobes": 0,
"version": None,
}
@@ -2025,7 +2501,6 @@ def test_query_sync_no_max_nprobes():
(
table.search([1, 2, 3], vector_column_name="vector2", fast_search=True)
.minimum_nprobes(5)
.maximum_nprobes(0)
.to_list()
)
@@ -2180,9 +2655,6 @@ def test_query_sync_hybrid():
"refine_factor": None,
"vector": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
"vector_column": "vector",
"nprobes": 20,
"minimum_nprobes": 20,
"maximum_nprobes": 20,
"lower_bound": None,
"upper_bound": None,
"ef": None,
@@ -2878,14 +3350,18 @@ def blob_remote_table(*, server_version=Version("0.5.0")):
elif request.path == "/v1/table/test/query/":
content_len = int(request.headers.get("Content-Length", 0))
body = json.loads(request.rfile.read(content_len))
assert body["with_row_id"] is True
if body["columns"] == ["image"]:
# fetch_blob_files sizes its handles from a descriptor take.
assert body["filter"].startswith("_rowid IN")
response_table = blob_descriptor_take_table()
columns = body.get("columns")
assert columns in (None, ["id", "image"], ["id"], ["image"])
if columns == ["id"]:
response_table = blob_query_response_table().select(["id"])
else:
assert body["columns"] == ["id", "image"]
response_table = blob_query_response_table()
assert body["with_row_id"] is True
if columns == ["image"]:
# fetch_blob_files sizes its handles from a descriptor take.
assert body["filter"].startswith("_rowid IN")
response_table = blob_descriptor_take_table()
else:
response_table = blob_query_response_table()
request.send_response(200)
request.send_header("Content-Type", "application/vnd.apache.arrow.file")
request.end_headers()
@@ -2919,6 +3395,86 @@ def test_remote_blob_columns_and_fetch():
assert blobs.to_pylist() == [b"alpha", None, b"gamma"]
@pytest.mark.asyncio
async def test_async_remote_to_lance_rejects_storage_access(monkeypatch):
with blob_remote_table() as remote_table:
table = remote_table._table
async def fail_uri():
raise AssertionError("remote table URI must not be requested")
monkeypatch.setattr(table, "uri", fail_uri)
with pytest.raises(NotImplementedError, match="remote tables"):
await table.to_lance()
@pytest.mark.asyncio
async def test_async_remote_blob_descriptions_to_pandas_uses_server(monkeypatch):
with blob_remote_table() as remote_table:
table = remote_table._table
async def fail_uri():
raise AssertionError("remote table URI must not be requested")
async def fail_to_lance():
raise AssertionError("remote table storage must not be opened")
monkeypatch.setattr(table, "uri", fail_uri)
monkeypatch.setattr(table, "to_lance", fail_to_lance)
query_df = await (
table.query().select(["id", "image"]).to_pandas(blob_mode="descriptions")
)
table_df = await table.to_pandas(blob_mode="descriptions")
for df in (query_df, table_df):
assert df["id"].tolist() == [1, 2, 3]
assert "_rowid" not in df.columns
images = df["image"].tolist()
assert images[1] is None
assert images[0]["size"] == 5
assert "_lance_row_id" not in images[0]
@pytest.mark.asyncio
@pytest.mark.parametrize("blob_mode", ["bytes", "lazy"])
async def test_async_remote_blob_pandas_rejects_unpinned_fetch(monkeypatch, blob_mode):
with blob_remote_table() as remote_table:
table = remote_table._table
async def fail_uri():
raise AssertionError("remote table URI must not be requested")
async def fail_to_lance():
raise AssertionError("remote table storage must not be opened")
async def fail_fetch(*args, **kwargs):
raise AssertionError("blob fetch must not use an unpinned remote table")
monkeypatch.setattr(table, "uri", fail_uri)
monkeypatch.setattr(table, "to_lance", fail_to_lance)
monkeypatch.setattr(table, "fetch_blobs", fail_fetch)
monkeypatch.setattr(table, "fetch_blob_files", fail_fetch)
pandas_kwargs = {} if blob_mode == "lazy" else {"blob_mode": blob_mode}
with pytest.raises(NotImplementedError, match="stable table snapshot"):
await table.query().select(["id", "image"]).to_pandas(**pandas_kwargs)
with pytest.raises(NotImplementedError, match="stable table snapshot"):
await table.to_pandas(**pandas_kwargs)
@pytest.mark.asyncio
@pytest.mark.parametrize("blob_mode", ["bytes", "lazy"])
async def test_async_remote_nonblob_projection_to_pandas(blob_mode):
with blob_remote_table() as remote_table:
df = await (
remote_table._table.query().select(["id"]).to_pandas(blob_mode=blob_mode)
)
assert df["id"].tolist() == [1, 2, 3]
assert list(df.columns) == ["id"]
def test_remote_blob_files_are_lazy_seekable_handles():
with blob_remote_table() as table:
files = table.fetch_blob_files("image", [10, 20, 30, 40])
@@ -2995,9 +3551,15 @@ def test_remote_blob_query_survives_a_server_that_ignores_the_row_id_request():
def test_remote_blob_byte_apis_not_supported_on_old_server():
with blob_remote_table(server_version=Version("0.1.0")) as table:
assert table.blob_columns() == ["image"]
with pytest.raises(NotImplementedError, match="not supported"):
with pytest.raises(
NotImplementedError,
match="fetch_blobs is not supported by this LanceDB server",
):
table.fetch_blobs("image", [1])
with pytest.raises(NotImplementedError, match="not supported"):
with pytest.raises(
NotImplementedError,
match="fetch_blob_files requires LanceDB server 0.5.0 or newer",
):
table.fetch_blob_files("image", [1])
python/python/tests/test_sql.py
+110 -3
View File
@@ -1,8 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import threading
from uuid import UUID
import numpy as np
import pytest
import pyarrow as pa
@@ -11,7 +13,7 @@ from lancedb import _lancedb
from lancedb.arrow import AsyncRecordBatchReader
from lancedb.db import AsyncConnection
from lancedb.remote.db import RemoteDBConnection
from lancedb.sql import AsyncQuery, Query
from lancedb.sql import AsyncQuery, Query, to_parameter_batch
NIL_QUERY_ID = UUID(int=0)
@@ -24,12 +26,20 @@ class FakeNativeQuery:
class FakeNativeConnection:
async def execute_query_async(self, query, *, default_namespace_path=None):
def __init__(self):
self.parameters = []
async def execute_query_async(
self, query, *, default_namespace_path=None, parameters=None
):
self.parameters.append(parameters)
return FakeNativeQuery()
class FakeAsyncConnection:
async def execute_query_async(self, query, *, default_namespace_path=None):
async def execute_query_async(
self, query, *, default_namespace_path=None, parameters=None
):
return AsyncQuery(FakeNativeQuery())
@@ -160,3 +170,100 @@ def test_execute_query_async_rejects_invalid_namespace_components(
remote_connection().execute_query_async(
"SELECT 1", default_namespace_path=default_namespace_path
)
def test_positional_parameters_keep_their_types():
vector = np.array([0.1, 0.2, 0.3], dtype=np.float32)
batch = to_parameter_batch(
[42, 1.5, "a", np.float32(0.25), vector, pa.scalar(7, pa.int16())]
)
assert batch.num_rows == 1
assert batch.schema.names == ["1", "2", "3", "4", "5", "6"]
assert batch.schema.types == [
pa.int64(),
pa.float64(),
pa.string(),
pa.float32(),
pa.list_(pa.float32(), 3),
pa.int16(),
]
sent = batch.column(4).flatten().to_numpy()
assert sent.dtype == np.float32
assert sent.tobytes() == vector.tobytes()
def test_named_parameters_and_batches():
half = pa.array(np.array([1.0, 2.0], dtype=np.float16))
batch = to_parameter_batch({"vector": half, "k": 3})
assert batch.schema.names == ["vector", "k"]
assert batch.schema.field("vector").type == pa.list_(pa.float16(), 2)
row = pa.record_batch({"id": [1]})
assert to_parameter_batch(row) is row
assert to_parameter_batch(pa.table({"id": [1]})).equals(row)
assert to_parameter_batch(None) is None
@pytest.mark.parametrize(
"parameters",
[pa.record_batch({"id": [1, 2]}), pa.table({"id": pa.array([], pa.int64())})],
)
def test_parameters_are_exactly_one_row(parameters):
with pytest.raises(ValueError, match="exactly one row"):
to_parameter_batch(parameters)
def test_parameters_must_be_a_collection():
with pytest.raises(TypeError, match="query parameters"):
to_parameter_batch("SELECT 1")
@pytest.mark.asyncio
async def test_async_execute_query_sends_parameters_as_a_batch():
native = FakeNativeConnection()
connection = AsyncConnection(native)
await connection.execute_query("SELECT $1", parameters=[1])
await connection.execute_query_async("SELECT 1")
sent, unparameterized = native.parameters
assert sent.equals(pa.record_batch({"1": [1]}))
assert unparameterized is None
def test_parameters_reach_the_server_as_the_values_that_were_sent():
"""A round trip through the bindings against a Flight server that answers
each exchange with the parameter row it received."""
flight = pytest.importorskip("pyarrow.flight")
class EchoServer(flight.FlightServerBase):
def __init__(self):
super().__init__("grpc://127.0.0.1:0")
self.commands = []
self.lock = threading.Lock()
def do_exchange(self, context, descriptor, reader, writer):
parameters = reader.read_all()
with self.lock:
self.commands.append(descriptor.command)
writer.begin(parameters.schema)
writer.write_table(parameters)
writer.close()
vector = np.array([0.1, 1e-45, -0.0], dtype=np.float32)
with EchoServer() as server:
db = lancedb.connect(
"db://analytics",
api_key="test-key",
host_override="http://localhost:10024",
sql_host_override=f"grpc://127.0.0.1:{server.port}",
)
echoed = db.execute_query(
"SELECT echo", parameters={"vector": vector, "k": 10}
).read_all()
(command,) = server.commands
assert b"CommandStatementQuery" in command
assert b"SELECT echo" in command
assert echoed.schema.field("vector").type == pa.list_(pa.float32(), 3)
received = echoed.column("vector").combine_chunks().flatten().to_numpy()
assert received.view(np.uint32).tolist() == vector.view(np.uint32).tolist()
assert echoed.column("k").to_pylist() == [10]
python/src/connection.rs
+8 -3
View File
@@ -14,9 +14,10 @@ use crate::{
table::Table,
};
use arrow::{
array::RecordBatch,
datatypes::Schema,
ffi_stream::ArrowArrayStreamReader,
pyarrow::{FromPyArrow, ToPyArrow},
pyarrow::{FromPyArrow, PyArrowType, ToPyArrow},
};
use lancedb::{
connection::Connection as LanceConnection,
@@ -148,18 +149,22 @@ impl Connection {
self.get_inner().map(|inner| inner.uri().to_string())
}
#[pyo3(signature = (query, *, default_namespace_path=None))]
#[pyo3(signature = (query, *, default_namespace_path=None, parameters=None))]
pub fn execute_query_async<'a>(
self_: PyRef<'a, Self>,
query: String,
default_namespace_path: Option<Bound<'_, PyAny>>,
parameters: Option<PyArrowType<RecordBatch>>,
) -> PyResult<Bound<'a, PyAny>> {
let inner = self_.get_inner()?.clone();
let default_namespace_path = parse_default_namespace_path(default_namespace_path)?;
future_into_py(self_.py(), async move {
let operation = inner
let mut operation = inner
.execute_query_async(query)
.default_namespace_path(default_namespace_path);
if let Some(PyArrowType(parameters)) = parameters {
operation = operation.parameters(parameters);
}
operation
.execute()
.await
python/src/index.rs
+8 -4
View File
@@ -54,8 +54,6 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
.with_position(params.with_position)
.lower_case(params.lower_case)
.max_token_length(params.max_token_length)
.remove_stop_words(params.remove_stop_words)
.stem(params.stem)
.ascii_folding(params.ascii_folding)
.ngram_min_length(params.ngram_min_length)
.ngram_max_length(params.ngram_max_length)
@@ -65,6 +63,12 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
DocumentGranularity::try_from(params.document_granularity.as_str())
.map_err(|err| PyValueError::new_err(err.to_string()))?,
);
if let Some(stem) = params.stem {
inner_opts = inner_opts.stem(stem);
}
if let Some(remove_stop_words) = params.remove_stop_words {
inner_opts = inner_opts.remove_stop_words(remove_stop_words);
}
if let Some(memory_limit) = params.memory_limit {
inner_opts = inner_opts.memory_limit_mb(memory_limit);
}
@@ -215,8 +219,8 @@ struct FtsParams {
language: String,
max_token_length: Option<usize>,
lower_case: bool,
stem: bool,
remove_stop_words: bool,
stem: Option<bool>,
remove_stop_words: Option<bool>,
custom_stop_words: Option<Vec<String>>,
ascii_folding: bool,
ngram_min_length: u32,
python/src/query.rs
+5 -7
View File
@@ -336,9 +336,8 @@ pub struct PyQueryRequest {
pub use_lsm: Option<bool>,
pub column: Option<String>,
pub query_vector: Option<PyQueryVectors>,
pub nprobes: Option<usize>,
pub minimum_nprobes: Option<usize>,
// None means user did not set it and default should be used (currently 20)
// Some(0) means user set it to None and there is no limit
pub maximum_nprobes: Option<usize>,
pub lower_bound: Option<f32>,
pub upper_bound: Option<f32>,
@@ -369,6 +368,7 @@ impl From<AnyQuery> for PyQueryRequest {
use_lsm: query_request.use_lsm,
column: None,
query_vector: None,
nprobes: None,
minimum_nprobes: None,
maximum_nprobes: None,
lower_bound: None,
@@ -396,11 +396,9 @@ impl From<AnyQuery> for PyQueryRequest {
use_lsm: vector_query.base.use_lsm,
column: vector_query.column,
query_vector: Some(PyQueryVectors(vector_query.query_vector)),
minimum_nprobes: Some(vector_query.minimum_nprobes),
maximum_nprobes: match vector_query.maximum_nprobes {
None => Some(0),
Some(value) => Some(value),
},
nprobes: vector_query.nprobes,
minimum_nprobes: vector_query.minimum_nprobes,
maximum_nprobes: vector_query.maximum_nprobes,
lower_bound: vector_query.lower_bound,
upper_bound: vector_query.upper_bound,
ef: vector_query.ef,
python/src/table.rs
+24 -9
View File
@@ -823,8 +823,8 @@ impl From<LanceDbFtsToken> for FtsToken {
language = "English".to_string(),
max_token_length = Some(40),
lower_case = true,
stem = true,
remove_stop_words = true,
stem = None,
remove_stop_words = None,
custom_stop_words = None,
ascii_folding = true,
ngram_min_length = 3,
@@ -838,15 +838,15 @@ pub fn tokenize(
language: String,
max_token_length: Option<u32>,
lower_case: bool,
stem: bool,
remove_stop_words: bool,
stem: Option<bool>,
remove_stop_words: Option<bool>,
custom_stop_words: Option<Vec<String>>,
ascii_folding: bool,
ngram_min_length: u32,
ngram_max_length: u32,
prefix_only: bool,
) -> PyResult<Vec<FtsToken>> {
let params = FtsIndexBuilder::default()
let mut params = FtsIndexBuilder::default()
.base_tokenizer(base_tokenizer)
.language(&language)
.map_err(|_| {
@@ -857,13 +857,17 @@ pub fn tokenize(
})?
.max_token_length(max_token_length.map(|value| value as usize))
.lower_case(lower_case)
.stem(stem)
.remove_stop_words(remove_stop_words)
.ascii_folding(ascii_folding)
.ngram_min_length(ngram_min_length)
.ngram_max_length(ngram_max_length)
.ngram_prefix_only(prefix_only)
.custom_stop_words(custom_stop_words);
if let Some(stem) = stem {
params = params.stem(stem);
}
if let Some(remove_stop_words) = remove_stop_words {
params = params.remove_stop_words(remove_stop_words);
}
let tokens = lancedb_tokenize(&query, &params).infer_error()?;
Ok(tokens.into_iter().map(FtsToken::from).collect())
}
@@ -913,6 +917,11 @@ impl Table {
self.inner.is_some()
}
/// Whether this table has a local Lance dataset that the client may open.
pub fn _is_native(&self) -> PyResult<bool> {
Ok(self.inner_ref()?.as_native().is_some())
}
/// Closes the table, releasing any resources associated with it.
pub fn close(&mut self) {
self.inner.take();
@@ -1708,7 +1717,10 @@ impl Table {
future_into_py(self_.py(), async move {
inner
.as_native()
.ok_or_else(|| PyValueError::new_err("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "uses_v2_manifest_paths is not supported for remote tables.".into(),
})
.infer_error()?
.uses_v2_manifest_paths()
.await
.infer_error()
@@ -1720,7 +1732,10 @@ impl Table {
future_into_py(self_.py(), async move {
inner
.as_native()
.ok_or_else(|| PyValueError::new_err("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "migrate_manifest_paths_v2 is not supported for remote tables.".into(),
})
.infer_error()?
.migrate_manifest_paths_v2()
.await
.infer_error()
python/uv.lock
Generated
+15 -15
View File
@@ -1849,19 +1849,19 @@ wheels = [
[[package]]
name = "lance-namespace"
version = "0.11.1"
version = "0.13.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "lance-namespace-urllib3-client" },
]
sdist = { url = "https://files.pythonhosted.org/packages/bf/93/da5f7fcac690db9b282a3439ed9e34960c147619a0d6e1f4eb8cd240e7a5/lance_namespace-0.11.1.tar.gz", hash = "sha256:f67cfbbe0647b7cb42f23b673e7edf8a75b7d8a047265a916492f8d247ee1bc2", size = 11631, upload-time = "2026-08-18T17:40:06.294Z" }
sdist = { url = "https://files.pythonhosted.org/packages/c0/e7/d5d46594678ee479c0eda830c47b2f5c46133bd100e84e9aa6e01306eca7/lance_namespace-0.13.0.tar.gz", hash = "sha256:24554a0997bdb39595c6e4cb3ac6722069f6cd3bd1a74e7acbba7bfaf774a40d", size = 11636, upload-time = "2026-09-14T02:47:03.871523Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fa/bc/601f2b3cc4cfa0070d858a33223bc823fffdd7981a25c45984a5216ca952/lance_namespace-0.11.1-py3-none-any.whl", hash = "sha256:07643fce9a42ad4d58cc8bf91e3f592bc7f4cbd8d0ad5233223506debf67551c", size = 13507, upload-time = "2026-08-18T17:40:03.561Z" },
{ url = "https://files.pythonhosted.org/packages/e7/47/cfa33cca1ba7c749fd2918cfc5c8ded788f378cc6424d23e4fead5a14125/lance_namespace-0.13.0-py3-none-any.whl", hash = "sha256:438c7b17aef421c21c138196e715f2510d62f07e865372047f87c1e75e618c7a", size = 13507, upload-time = "2026-09-14T02:47:01.765655Z" },
]
[[package]]
name = "lance-namespace-urllib3-client"
version = "0.11.1"
version = "0.13.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pydantic" },
@@ -1869,9 +1869,9 @@ dependencies = [
{ name = "typing-extensions" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e4/c5/2bdd0ff98b469894c8a73be809d26ffdad5402517b0e5f9e758026cba29e/lance_namespace_urllib3_client-0.11.1.tar.gz", hash = "sha256:145a9e9424d7597487249b5b95ee274423bf2910e1a9160b6a07b676b61ea46a", size = 237345, upload-time = "2026-08-18T17:40:07.308Z" }
sdist = { url = "https://files.pythonhosted.org/packages/69/25/9aaa4a5e8999693fb0f227c2c0c4b97bd8f0539066408cdff0b89d41b2d5/lance_namespace_urllib3_client-0.13.0.tar.gz", hash = "sha256:1e8a79c6e4e6277033597fd76aa0e2f33d909ca1436c935936e9e569221f43ef", size = 238348, upload-time = "2026-09-14T02:47:04.521878Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/2a/2a/eaaefd55d1190291207049fedc6b3eb22b506e57d6de91bae46bbaaa9c60/lance_namespace_urllib3_client-0.11.1-py3-none-any.whl", hash = "sha256:36537f529294da6d884ba0fe783704483f0a75463497c7705fd083a4d0257990", size = 406311, upload-time = "2026-08-18T17:40:04.842Z" },
{ url = "https://files.pythonhosted.org/packages/60/4c/7b8f0712a7fe1b8655b711342bc4989696635a79e7027ee56ba3cae23e99/lance_namespace_urllib3_client-0.13.0-py3-none-any.whl", hash = "sha256:fb361eb4f6c7f2d1f9e92809609657b73e38241393e9c4516e12d6b6ee643a0a", size = 406848, upload-time = "2026-09-14T02:47:02.715954Z" },
]
[[package]]
@@ -1980,7 +1980,7 @@ requires-dist = [
{ name = "huggingface-hub", marker = "extra == 'embeddings'", specifier = ">=0.19.0" },
{ name = "ibm-watsonx-ai", marker = "python_full_version >= '3.10' and extra == 'embeddings'", specifier = ">=1.1.2" },
{ name = "instructorembedding", marker = "extra == 'embeddings'", specifier = ">=1.0.1" },
{ name = "lance-namespace", specifier = ">=0.3.2" },
{ name = "lance-namespace", specifier = ">=0.13.0" },
{ name = "mkdocs", marker = "extra == 'docs'" },
{ name = "mkdocs-jupyter", marker = "extra == 'docs'" },
{ name = "mkdocs-material", marker = "extra == 'docs'" },
@@ -2005,7 +2005,7 @@ requires-dist = [
{ name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" },
{ name = "pydantic", specifier = ">=2.7.4,<3" },
{ name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b6" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b10" },
{ name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },
{ name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" },
{ name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" },
@@ -3912,7 +3912,7 @@ crypto = [
[[package]]
name = "pylance"
version = "14.0.0b6"
version = "14.0.0b10"
source = { registry = "https://pypi.fury.io/lance-format/" }
dependencies = [
{ name = "lance-namespace" },
@@ -3922,12 +3922,12 @@ dependencies = [
{ name = "pyarrow" },
]
wheels = [
{ url = "https://pypi.fury.io/lance-format/-/ver_DuQEn/pylance-14.0.0b6-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:3eade86fa30d9ff3248ee7ffb0b27ce29495394d12327da4d22b5c54bc2b59fe" },
{ url = "https://pypi.fury.io/lance-format/-/ver_K5sqj/pylance-14.0.0b6-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10e291974ee738a1d5d7e9dad17a634394a91cb60080b5ceac209077fbaec43c" },
{ url = "https://pypi.fury.io/lance-format/-/ver_EnbkT/pylance-14.0.0b6-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61bac10f5081c1210edd017f81c8c784d800b89446bd0ed05d4a24e7001965bc" },
{ url = "https://pypi.fury.io/lance-format/-/ver_19ffYg/pylance-14.0.0b6-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3c627ad4a9870611becf84409d8e5ecddc6be64e1dd53bee4bd5fb10ab561e5c" },
{ url = "https://pypi.fury.io/lance-format/-/ver_2eLLY5/pylance-14.0.0b6-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:bc603a7dd690233f1fe000600171e1c092f153bc9060cdf198825c4bbe1418aa" },
{ url = "https://pypi.fury.io/lance-format/-/ver_t8ggc/pylance-14.0.0b6-cp310-abi3-win_amd64.whl", hash = "sha256:1d32df32a3d157611450def07eae8ace3a7401286eac68cad9a24e0d5deed2db" },
{ url = "https://pypi.fury.io/lance-format/-/ver_4hvE/pylance-14.0.0b10-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:415b6b1b9fe735fbae7c755eb9774ae2fe0feddc90bf80bee24087cbfbfdf8c1" },
{ url = "https://pypi.fury.io/lance-format/-/ver_pM1km/pylance-14.0.0b10-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87317cad22f4d35fea11eb70aa5c0d71a0e1ea3b96de38f3deca933daa749fff" },
{ url = "https://pypi.fury.io/lance-format/-/ver_24jdyj/pylance-14.0.0b10-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6da554a2fc2197ba254676b12824d8120b48d65937697e0661a61302458a4740" },
{ url = "https://pypi.fury.io/lance-format/-/ver_XclaZ/pylance-14.0.0b10-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:36c80a801675bb9dcbd681fa99d0cdd5ca642dfb3ca08d71ac980278a2f7748b" },
{ url = "https://pypi.fury.io/lance-format/-/ver_L0yh/pylance-14.0.0b10-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:e50ad0b866faefb2fa3ae87328b93fa0d06b2bdd5bb5d0c1a6c74b907cc2fd93" },
{ url = "https://pypi.fury.io/lance-format/-/ver_2gbNr8/pylance-14.0.0b10-cp310-abi3-win_amd64.whl", hash = "sha256:3d86451fe00f71510005bb700737fb1a4f9361554541cbf802f3c5072554d014" },
]
[[package]]
python/vendor/lance-arrow/Cargo.toml
Vendored
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "lance-arrow"
version = "14.0.0-beta.7"
version = "14.0.0-beta.10"
edition = "2024"
authors = ["Lance Devs <dev@lance.org>"]
license = "Apache-2.0"
@@ -25,7 +25,7 @@ bytemuck = { version = "1", default-features = false, features = ["extern_crate_
bytes.workspace = true
futures.workspace = true
half = { workspace = true, features = ["std", "bytemuck"] }
jsonb = { version = "0.5.3", default-features = false, features = ["databend"] }
lance-jsonb.workspace = true
num-traits.workspace = true
rand = { workspace = true, features = ["small_rng"] }
python/vendor/lance-arrow/README.lancedb.md
Vendored
+2 -2
View File
@@ -1,5 +1,5 @@
This directory contains `rust/lance-arrow` from Lance tag `v14.0.0-beta.7`
(commit `0fd26effc79c62c244a5baf92cdda06e301baf21`), licensed under Apache-2.0.
This directory contains `rust/lance-arrow` from Lance tag `v14.0.0-beta.10`
(commit `fa3638fd816ab2ff9a2ecbba7e51465d8879958f`), licensed under Apache-2.0.
LanceDB carries a local fix in `src/lib.rs` for sliced child validity bitmaps
until a Lance release includes the fix. The patch is selected in the root
python/vendor/lance-arrow/src/json.rs
Vendored
+8 -8
View File
@@ -14,8 +14,8 @@ use arrow_array::{
LargeStringArray, ListArray, MapArray, OffsetSizeTrait, RecordBatch, StringArray, StructArray,
};
use arrow_schema::{ArrowError, DataType, Field as ArrowField, Fields, Schema};
use jsonb::OwnedJsonb;
use jsonb::jsonpath::{JsonPath, Selector};
use lance_jsonb::OwnedJsonb;
use lance_jsonb::jsonpath::{JsonPath, Selector};
use crate::ARROW_EXT_NAME_KEY;
@@ -336,19 +336,19 @@ impl TryFrom<ArrayRef> for JsonArray {
/// Encode JSON string to JSONB format
pub fn encode_json(json_str: &str) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
let value = jsonb::parse_value(json_str.as_bytes())?;
let value = lance_jsonb::parse_value(json_str.as_bytes())?;
Ok(value.to_vec())
}
/// Decode JSONB bytes to JSON string
pub fn decode_json(jsonb_bytes: &[u8]) -> String {
let raw_jsonb = jsonb::RawJsonb::new(jsonb_bytes);
let raw_jsonb = lance_jsonb::RawJsonb::new(jsonb_bytes);
raw_jsonb.to_string()
}
/// Parse a JSONPath expression.
pub fn parse_json_path(path: &str) -> Result<JsonPath<'_>, ArrowError> {
jsonb::jsonpath::parse_json_path(path.as_bytes())
lance_jsonb::jsonpath::parse_json_path(path.as_bytes())
.map_err(|e| ArrowError::InvalidArgumentError(format!("Invalid JSONPath '{path}': {e}")))
}
@@ -360,7 +360,7 @@ pub fn select_json_path(
jsonb_bytes: &[u8],
path: &JsonPath<'_>,
) -> Result<Option<OwnedJsonb>, ArrowError> {
Selector::new(jsonb::RawJsonb::new(jsonb_bytes))
Selector::new(lance_jsonb::RawJsonb::new(jsonb_bytes))
.select_value(path)
.map_err(|e| {
ArrowError::InvalidArgumentError(format!("Failed to select JSONPath {path}: {e}"))
@@ -369,7 +369,7 @@ pub fn select_json_path(
/// Whether `path` matches any value in a JSONB value.
pub fn json_path_exists(jsonb_bytes: &[u8], path: &JsonPath<'_>) -> Result<bool, ArrowError> {
Selector::new(jsonb::RawJsonb::new(jsonb_bytes))
Selector::new(lance_jsonb::RawJsonb::new(jsonb_bytes))
.exists(path)
.map_err(|e| {
ArrowError::InvalidArgumentError(format!("Failed to match JSONPath {path}: {e}"))
@@ -381,7 +381,7 @@ pub fn select_json_path_values(
jsonb_bytes: &[u8],
path: &JsonPath<'_>,
) -> Result<Vec<OwnedJsonb>, ArrowError> {
Selector::new(jsonb::RawJsonb::new(jsonb_bytes))
Selector::new(lance_jsonb::RawJsonb::new(jsonb_bytes))
.select_values(path)
.map_err(|e| {
ArrowError::InvalidArgumentError(format!("Failed to select JSONPath {path}: {e}"))