mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat(python): expose FTS operators and match counts (#5042)
String full-text searches now accept `operator="AND"` or
`operator="OR"`, preserving the existing OR default. Sync and async FTS
builders expose `count_rows()` to count every match satisfying the
filter, independently of pagination, projection, ordering, and
reranking. Counting streams matching scores without reading document
columns; it can scan all matches.
```python
query = table.search("gripper closed", query_type="fts", operator="AND")
query = query.where("active = true")
matching_rows = query.count_rows()
page = query.limit(20).to_list()
query = await async_table.search("gripper closed", query_type="fts", operator="AND")
matching_rows = await query.where("active = true").count_rows()
```
Phrase searches keep the existing index default and now explain how to
rebuild an index with `FTS(with_position=True)`. AND requests retain
their operator through namespace pushdown and fail explicitly on older
remote servers that cannot represent it. Regression coverage includes
indexed and unindexed queries, filtered counts above the default limit,
pagination preservation, remote serialization, and phrase indexes with
and without positions.
This commit is contained in:
@@ -943,6 +943,7 @@ class FTSQuery:
|
||||
def add_query_vector(self, query_vec: pa.Array) -> None: ...
|
||||
def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ...
|
||||
def order_by(self, ordering: Optional[List[ColumnOrdering]]): ...
|
||||
async def count_rows(self) -> int: ...
|
||||
async def output_schema(self) -> pa.Schema: ...
|
||||
async def execute(
|
||||
self, max_batch_length: Optional[int], timeout: Optional[timedelta]
|
||||
|
||||
@@ -10,6 +10,7 @@ through a namespace abstraction.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import warnings
|
||||
from typing import Any, Dict, Iterable, List, Optional, Union
|
||||
@@ -39,6 +40,9 @@ from lance_namespace_urllib3_client.models.query_table_request_vector import (
|
||||
QueryTableRequestVector,
|
||||
)
|
||||
from lance_namespace_urllib3_client.models.string_fts_query import StringFtsQuery
|
||||
from lance_namespace_urllib3_client.models.structured_fts_query import (
|
||||
StructuredFtsQuery,
|
||||
)
|
||||
from lance_namespace.errors import (
|
||||
NamespaceNotEmptyError,
|
||||
NamespaceNotFoundError,
|
||||
@@ -78,6 +82,7 @@ from lancedb.materialized_view import (
|
||||
SelectArg,
|
||||
)
|
||||
from lancedb.table import AsyncTable, LanceTable, Table
|
||||
from lancedb.query import MatchQuery, MultiMatchQuery, PhraseQuery
|
||||
from lancedb.util import validate_table_name
|
||||
from lancedb.common import DATA
|
||||
from lancedb.pydantic import LanceModel
|
||||
@@ -178,6 +183,26 @@ def _query_to_namespace_request(
|
||||
full_text_query_request = QueryTableRequestFullTextQuery(
|
||||
string_query=StringFtsQuery(query=fts_query)
|
||||
)
|
||||
elif getattr(fts_query, "operator", None) is not None:
|
||||
columns = fts_query.columns or []
|
||||
text = fts_query.query
|
||||
if len(text) >= 2 and text.startswith('"') and text.endswith('"'):
|
||||
if len(columns) > 1:
|
||||
raise ValueError(
|
||||
"Phrase queries cannot be used with multiple columns."
|
||||
)
|
||||
structured = PhraseQuery(text[1:-1], columns[0] if columns else None)
|
||||
elif len(columns) > 1:
|
||||
structured = MultiMatchQuery(text, columns, operator=fts_query.operator)
|
||||
else:
|
||||
structured = MatchQuery(
|
||||
text, columns[0] if columns else None, operator=fts_query.operator
|
||||
)
|
||||
full_text_query_request = QueryTableRequestFullTextQuery(
|
||||
structured_query=StructuredFtsQuery.from_dict(
|
||||
{"query": json.loads(structured.to_json())}
|
||||
)
|
||||
)
|
||||
elif hasattr(fts_query, "query"):
|
||||
full_text_query_request = QueryTableRequestFullTextQuery(
|
||||
string_query=StringFtsQuery(
|
||||
|
||||
+116
-11
@@ -42,7 +42,7 @@ from .rerankers.base import Reranker
|
||||
from .rerankers.rrf import RRFReranker
|
||||
from .rerankers.util import check_reranker_result
|
||||
from .schema import blob_column_paths, is_blob_like_field, schema_has_blob_field
|
||||
from .util import _validate_query_vector, flatten_columns, get_uri_scheme
|
||||
from .util import _validate_query_vector, add_note, flatten_columns, get_uri_scheme
|
||||
from . import _wal_hybrid # WAL-PK-FUSION: delete.
|
||||
from ._blob import (
|
||||
BLOB_MODE_TO_HANDLING,
|
||||
@@ -873,10 +873,29 @@ class FullTextSearchQuery(pydantic.BaseModel):
|
||||
query: str | FullTextQuery
|
||||
If a string, it is treated as a MatchQuery.
|
||||
If a FullTextQuery object, it is used directly.
|
||||
operator: FullTextOperator, optional
|
||||
Combine terms in a string query using AND or OR (the default).
|
||||
Quoted strings keep phrase semantics regardless of the operator.
|
||||
"""
|
||||
|
||||
columns: Optional[List[str]] = None
|
||||
query: Union[str, FullTextQuery]
|
||||
operator: Optional[FullTextOperator] = None
|
||||
|
||||
@pydantic.model_validator(mode="after")
|
||||
def _validate_operator(self) -> Self:
|
||||
if self.operator is not None and not isinstance(self.query, str):
|
||||
raise ValueError("operator requires a string full-text query")
|
||||
return self
|
||||
|
||||
|
||||
def _add_fts_position_error_note(error: Exception) -> None:
|
||||
if "position is not found but required for phrase queries" in str(error):
|
||||
add_note(
|
||||
error,
|
||||
"Phrase queries require an index built with positions. Rebuild it with "
|
||||
"table.create_index(column, config=FTS(with_position=True), replace=True).",
|
||||
)
|
||||
|
||||
|
||||
class Query(pydantic.BaseModel):
|
||||
@@ -1147,6 +1166,7 @@ class LanceQueryBuilder(ABC):
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
fast_search: bool = None,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> Self:
|
||||
"""
|
||||
Create a query builder based on the given query and query type.
|
||||
@@ -1171,6 +1191,10 @@ class LanceQueryBuilder(ABC):
|
||||
fast_search: bool
|
||||
Skip flat search of unindexed data.
|
||||
"""
|
||||
if operator is not None:
|
||||
operator = FullTextOperator(operator)
|
||||
if not isinstance(query, str) or query_type not in ("auto", "fts"):
|
||||
raise ValueError("operator requires a string full-text query")
|
||||
if query_type != "fts":
|
||||
_validate_query_vector(query)
|
||||
|
||||
@@ -1201,6 +1225,9 @@ class LanceQueryBuilder(ABC):
|
||||
table, query, query_type, vector_column_name
|
||||
)
|
||||
|
||||
if operator is not None and query_type != "fts":
|
||||
raise ValueError("operator requires a string full-text query")
|
||||
|
||||
if query_type == "hybrid":
|
||||
return LanceHybridQueryBuilder(
|
||||
table, query, vector_column_name, fts_columns=fts_columns
|
||||
@@ -1214,7 +1241,7 @@ class LanceQueryBuilder(ABC):
|
||||
ordering_field_name=ordering_field_name,
|
||||
fts_columns=fts_columns,
|
||||
fast_search=fast_search,
|
||||
)
|
||||
).operator(operator)
|
||||
|
||||
if isinstance(query, (list, tuple)):
|
||||
query = np.array(query, dtype=np.float32)
|
||||
@@ -2563,6 +2590,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
|
||||
super().__init__(table)
|
||||
self._query = query
|
||||
self._phrase_query = False
|
||||
self._operator: Optional[FullTextOperator] = None
|
||||
# Deprecated compatibility parameter. Native FTS ordering is now
|
||||
# configured through order_by(); LanceQueryBuilder.create emits the warning.
|
||||
_ = ordering_field_name
|
||||
@@ -2572,6 +2600,36 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
|
||||
fts_columns = [fts_columns]
|
||||
self._fts_columns = fts_columns
|
||||
|
||||
def operator(
|
||||
self, operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]]
|
||||
) -> LanceFtsQueryBuilder:
|
||||
"""Combine terms in a string query with ``"AND"`` or ``"OR"``.
|
||||
|
||||
The default is OR. AND requires every term to match. For example,
|
||||
``table.search("gripper closed", query_type="fts").operator("AND")``.
|
||||
Structured queries specify their own operators. Quoted strings and
|
||||
``phrase_query()`` retain their phrase semantics.
|
||||
"""
|
||||
if operator is not None and not isinstance(self._query, str):
|
||||
raise ValueError("operator requires a string full-text query")
|
||||
self._operator = FullTextOperator(operator) if operator is not None else None
|
||||
return self
|
||||
|
||||
def count_rows(self) -> int:
|
||||
"""Count all full-text matches satisfying the query's filter.
|
||||
|
||||
Ignores limit, offset, selection, ordering, and reranking. ``fast_search``
|
||||
still restricts the count to indexed data. This streams matching scores
|
||||
without loading document columns and may scan every match.
|
||||
|
||||
For example, ``table.search("gripper closed", query_type="fts",``
|
||||
``operator="AND").where("active = true").count_rows()``.
|
||||
"""
|
||||
self._table._ensure_no_legacy_fts_index()
|
||||
query = self._table._table._sync_query_to_async(self.to_query_object())
|
||||
assert isinstance(query, AsyncFTSQuery)
|
||||
return LOOP.run(query.count_rows())
|
||||
|
||||
def phrase_query(self, phrase_query: bool = True) -> LanceFtsQueryBuilder:
|
||||
"""Set whether to use phrase query.
|
||||
|
||||
@@ -2628,7 +2686,9 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
|
||||
fragment_ids=self._fragment_ids,
|
||||
use_rte=self._use_rte,
|
||||
full_text_query=FullTextSearchQuery(
|
||||
query=self._query_with_phrase_semantics(), columns=self._fts_columns
|
||||
query=self._query_with_phrase_semantics(),
|
||||
columns=self._fts_columns,
|
||||
operator=self._operator,
|
||||
),
|
||||
offset=self._offset,
|
||||
fast_search=self._fast_search,
|
||||
@@ -4443,7 +4503,11 @@ class AsyncQuery(AsyncStandardQuery):
|
||||
)
|
||||
|
||||
def nearest_to_text(
|
||||
self, query: str | FullTextQuery, columns: Union[str, List[str], None] = None
|
||||
self,
|
||||
query: str | FullTextQuery,
|
||||
columns: Union[str, List[str], None] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> AsyncFTSQuery:
|
||||
"""
|
||||
Find the documents that are most relevant to the given text query.
|
||||
@@ -4464,7 +4528,11 @@ class AsyncQuery(AsyncStandardQuery):
|
||||
columns: str or list of str, default None
|
||||
The columns to search in. If None, all indexed columns are searched.
|
||||
For now only one column can be searched at a time.
|
||||
operator: "AND" or "OR", optional
|
||||
Combine terms in a string query. Defaults to OR. Quoted strings use
|
||||
phrase semantics and require an index built with positions.
|
||||
"""
|
||||
fts_query = FullTextSearchQuery(query=query, operator=operator)
|
||||
if isinstance(columns, str):
|
||||
columns = [columns]
|
||||
if columns is None:
|
||||
@@ -4472,7 +4540,13 @@ class AsyncQuery(AsyncStandardQuery):
|
||||
|
||||
if isinstance(query, str):
|
||||
return AsyncFTSQuery(
|
||||
self._inner.nearest_to_text({"query": query, "columns": columns}),
|
||||
self._inner.nearest_to_text(
|
||||
{
|
||||
"query": query,
|
||||
"columns": columns,
|
||||
"operator": fts_query.operator,
|
||||
}
|
||||
),
|
||||
self._table,
|
||||
)
|
||||
# FullTextQuery object
|
||||
@@ -4487,6 +4561,22 @@ class AsyncFTSQuery(AsyncStandardQuery):
|
||||
self._inner = inner
|
||||
self._reranker = None
|
||||
|
||||
async def count_rows(self) -> int:
|
||||
"""Count all full-text matches satisfying the query's filter.
|
||||
|
||||
Ignores limit, offset, selection, ordering, and reranking. ``fast_search``
|
||||
still restricts the count to indexed data. Matching scores are streamed
|
||||
without loading document columns, so counting may scan every match.
|
||||
|
||||
For example, ``await table.query().nearest_to_text("gripper closed",``
|
||||
``operator="AND").where("active = true").count_rows()``.
|
||||
"""
|
||||
try:
|
||||
return await self._inner.count_rows()
|
||||
except Exception as error:
|
||||
_add_fts_position_error_note(error)
|
||||
raise
|
||||
|
||||
def get_query(self) -> str:
|
||||
return self._inner.get_query()
|
||||
|
||||
@@ -4580,10 +4670,16 @@ class AsyncFTSQuery(AsyncStandardQuery):
|
||||
timeout: Optional[timedelta] = None,
|
||||
blob_sources: Optional[dict[str, str]] = None,
|
||||
) -> AsyncRecordBatchReader:
|
||||
reader = await super()._execute_batches(
|
||||
timeout=timeout, blob_sources=blob_sources
|
||||
)
|
||||
results = pa.Table.from_batches(await reader.read_all(), reader.schema)
|
||||
try:
|
||||
reader = await super()._execute_batches(
|
||||
max_batch_length=max_batch_length,
|
||||
timeout=timeout,
|
||||
blob_sources=blob_sources,
|
||||
)
|
||||
results = pa.Table.from_batches(await reader.read_all(), reader.schema)
|
||||
except Exception as error:
|
||||
_add_fts_position_error_note(error)
|
||||
raise
|
||||
if self._reranker:
|
||||
source = results
|
||||
results = self._reranker.rerank_fts(self.get_query(), results)
|
||||
@@ -4861,7 +4957,11 @@ class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase):
|
||||
return self
|
||||
|
||||
def nearest_to_text(
|
||||
self, query: str | FullTextQuery, columns: Union[str, List[str], None] = None
|
||||
self,
|
||||
query: str | FullTextQuery,
|
||||
columns: Union[str, List[str], None] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> AsyncHybridQuery:
|
||||
"""
|
||||
Find the documents that are most relevant to the given text query,
|
||||
@@ -4886,7 +4986,10 @@ class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase):
|
||||
columns: str or list of str, default None
|
||||
The columns to search in. If None, all indexed columns are searched.
|
||||
For now only one column can be searched at a time.
|
||||
operator: "AND" or "OR", optional
|
||||
Combine terms in a string query. Defaults to OR.
|
||||
"""
|
||||
fts_query = FullTextSearchQuery(query=query, operator=operator)
|
||||
if isinstance(columns, str):
|
||||
columns = [columns]
|
||||
if columns is None:
|
||||
@@ -4894,7 +4997,9 @@ class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase):
|
||||
|
||||
if isinstance(query, str):
|
||||
return AsyncHybridQuery(
|
||||
self._inner.nearest_to_text({"query": query, "columns": columns}),
|
||||
self._inner.nearest_to_text(
|
||||
{"query": query, "columns": columns, "operator": fts_query.operator}
|
||||
),
|
||||
self._table,
|
||||
)
|
||||
# FullTextQuery object
|
||||
|
||||
@@ -65,6 +65,7 @@ from lancedb.embeddings import EmbeddingFunctionRegistry
|
||||
from lancedb.table import _normalize_progress
|
||||
|
||||
from ..query import (
|
||||
FullTextOperator,
|
||||
AnalyzePlanDistributedMetrics,
|
||||
DocumentGranularity,
|
||||
FullTextQuery,
|
||||
@@ -799,6 +800,8 @@ class RemoteTable(Table):
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
fast_search: Optional[bool] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> LanceQueryBuilder:
|
||||
"""Create a search query to find the nearest neighbors
|
||||
of the given query vector. We currently support
|
||||
@@ -857,6 +860,10 @@ class RemoteTable(Table):
|
||||
|
||||
- *default None*, which leaves fast search disabled.
|
||||
|
||||
operator: "AND" or "OR", optional
|
||||
Combine terms in a string full-text query; the default is OR.
|
||||
Use ``query_type="fts"`` when the table also has embeddings.
|
||||
|
||||
Returns
|
||||
-------
|
||||
LanceQueryBuilder
|
||||
@@ -893,6 +900,7 @@ class RemoteTable(Table):
|
||||
ordering_field_name=ordering_field_name,
|
||||
fts_columns=fts_columns or [],
|
||||
fast_search=fast_search,
|
||||
operator=operator,
|
||||
)
|
||||
|
||||
def _execute_query(
|
||||
|
||||
@@ -91,6 +91,7 @@ from .functions import (
|
||||
from .merge import LanceMergeInsertBuilder
|
||||
from .pydantic import LanceModel, model_to_dict
|
||||
from .query import (
|
||||
FullTextOperator,
|
||||
AnalyzePlanDistributedMetrics,
|
||||
AsyncFTSQuery,
|
||||
AsyncHybridQuery,
|
||||
@@ -138,6 +139,11 @@ def _requires_local_namespace_execution(query: Query) -> bool:
|
||||
or query.take_row_ids is not None
|
||||
or query.search_effort is not None
|
||||
or query.approx_mode is not None
|
||||
or (
|
||||
query.full_text_query is not None
|
||||
and query.fast_search is True
|
||||
and query.vector is None
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -2443,6 +2449,8 @@ class Table(ABC):
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
fast_search: Optional[bool] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> LanceQueryBuilder:
|
||||
"""Create a search query to find the nearest neighbors
|
||||
of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/)
|
||||
@@ -2506,6 +2514,10 @@ class Table(ABC):
|
||||
- If `query` is a string, then the query type is "vector" if the
|
||||
table has embedding functions else the query type is "fts"
|
||||
|
||||
operator: "AND" or "OR", optional
|
||||
Combine terms in a string full-text query; the default is OR.
|
||||
Use ``query_type="fts"`` when the table also has embeddings.
|
||||
|
||||
Returns
|
||||
-------
|
||||
LanceQueryBuilder
|
||||
@@ -5061,6 +5073,8 @@ class LanceTable(Table):
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
fast_search: Optional[bool] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> LanceVectorQueryBuilder: ...
|
||||
|
||||
@overload
|
||||
@@ -5072,6 +5086,8 @@ class LanceTable(Table):
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
fast_search: Optional[bool] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> LanceFtsQueryBuilder: ...
|
||||
|
||||
@overload
|
||||
@@ -5085,6 +5101,8 @@ class LanceTable(Table):
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
fast_search: Optional[bool] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> LanceHybridQueryBuilder: ...
|
||||
|
||||
@overload
|
||||
@@ -5096,6 +5114,8 @@ class LanceTable(Table):
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
fast_search: Optional[bool] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> LanceEmptyQueryBuilder: ...
|
||||
|
||||
def search(
|
||||
@@ -5108,6 +5128,8 @@ class LanceTable(Table):
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
fast_search: Optional[bool] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> LanceQueryBuilder:
|
||||
"""Create a search query to find the nearest neighbors
|
||||
of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/)
|
||||
@@ -5168,6 +5190,10 @@ class LanceTable(Table):
|
||||
If None then the search is performed on all indexed columns.
|
||||
For now, only one column can be searched at a time.
|
||||
|
||||
operator: "AND" or "OR", optional
|
||||
Combine terms in a string full-text query; the default is OR.
|
||||
Use ``query_type="fts"`` when the table also has embeddings.
|
||||
|
||||
Returns
|
||||
-------
|
||||
LanceQueryBuilder
|
||||
@@ -5193,6 +5219,7 @@ class LanceTable(Table):
|
||||
ordering_field_name=ordering_field_name,
|
||||
fts_columns=fts_columns or [],
|
||||
fast_search=fast_search,
|
||||
operator=operator,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -7376,6 +7403,8 @@ class AsyncTable:
|
||||
query_type: QueryType = "auto",
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> AsyncQuery: ...
|
||||
|
||||
@overload
|
||||
@@ -7386,6 +7415,8 @@ class AsyncTable:
|
||||
query_type: Literal["auto"] = ...,
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> Union[AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]: ...
|
||||
|
||||
@overload
|
||||
@@ -7396,6 +7427,8 @@ class AsyncTable:
|
||||
query_type: Literal["hybrid"] = ...,
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> AsyncHybridQuery: ...
|
||||
|
||||
@overload
|
||||
@@ -7406,6 +7439,8 @@ class AsyncTable:
|
||||
query_type: Literal["auto"] = ...,
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> AsyncVectorQuery: ...
|
||||
|
||||
@overload
|
||||
@@ -7416,6 +7451,8 @@ class AsyncTable:
|
||||
query_type: Literal["fts"] = ...,
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> AsyncFTSQuery: ...
|
||||
|
||||
@overload
|
||||
@@ -7426,6 +7463,8 @@ class AsyncTable:
|
||||
query_type: Literal["vector"] = ...,
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> AsyncVectorQuery: ...
|
||||
|
||||
async def search(
|
||||
@@ -7437,6 +7476,8 @@ class AsyncTable:
|
||||
query_type: QueryType = "auto",
|
||||
ordering_field_name: Optional[str] = None,
|
||||
fts_columns: Optional[Union[str, List[str]]] = None,
|
||||
*,
|
||||
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
|
||||
) -> Union[AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]:
|
||||
"""Create a search query to find the nearest neighbors
|
||||
of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/)
|
||||
@@ -7480,12 +7521,20 @@ class AsyncTable:
|
||||
- If `query` is a string, then the query type is "vector" if the
|
||||
table has embedding functions else the query type is "fts"
|
||||
|
||||
operator: "AND" or "OR", optional
|
||||
Combine terms in a string full-text query; the default is OR.
|
||||
Use ``query_type="fts"`` when the table also has embeddings.
|
||||
|
||||
Returns
|
||||
-------
|
||||
AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, or AsyncVectorQuery
|
||||
A query builder object representing the query.
|
||||
"""
|
||||
|
||||
if operator is not None:
|
||||
operator = FullTextOperator(operator)
|
||||
if not isinstance(query, str) or query_type not in ("auto", "fts"):
|
||||
raise ValueError("operator requires a string full-text query")
|
||||
if query is None:
|
||||
return self.query()
|
||||
|
||||
@@ -7604,13 +7653,17 @@ class AsyncTable:
|
||||
)
|
||||
vector_query = await make_embedding(embedding_conf, query)
|
||||
|
||||
if operator is not None and query_type != "fts":
|
||||
raise ValueError("operator requires a string full-text query")
|
||||
if query_type == "vector":
|
||||
builder = self.query().nearest_to(vector_query)
|
||||
if vector_column_name:
|
||||
builder = builder.column(vector_column_name)
|
||||
return builder
|
||||
elif query_type == "fts":
|
||||
return self.query().nearest_to_text(query, columns=fts_columns)
|
||||
return self.query().nearest_to_text(
|
||||
query, columns=fts_columns, operator=operator
|
||||
)
|
||||
elif query_type == "hybrid":
|
||||
builder = self.query().nearest_to(vector_query)
|
||||
if vector_column_name:
|
||||
@@ -7720,7 +7773,9 @@ class AsyncTable:
|
||||
|
||||
if query.full_text_query:
|
||||
async_query = async_query.nearest_to_text(
|
||||
query.full_text_query.query, query.full_text_query.columns
|
||||
query.full_text_query.query,
|
||||
query.full_text_query.columns,
|
||||
operator=query.full_text_query.operator,
|
||||
)
|
||||
|
||||
return async_query
|
||||
|
||||
+172
-12
@@ -29,6 +29,7 @@ from lancedb.query import (
|
||||
CombinedFieldsQuery,
|
||||
DocumentGranularity,
|
||||
FullTextOperator,
|
||||
FullTextSearchQuery,
|
||||
MatchQuery,
|
||||
MultiMatchQuery,
|
||||
PhraseQuery,
|
||||
@@ -530,11 +531,9 @@ async def test_fts_select_async(async_table):
|
||||
|
||||
def test_search_fts_phrase_query(table):
|
||||
table.create_fts_index("text", with_position=False)
|
||||
try:
|
||||
phrase_results = table.search('"puppy runs"').limit(100).to_list()
|
||||
assert False
|
||||
except Exception:
|
||||
pass
|
||||
with pytest.raises(Exception) as error:
|
||||
table.search('"puppy runs"').limit(100).to_list()
|
||||
assert "FTS(with_position=True)" in exception_output(error)
|
||||
table.create_fts_index("text", with_position=True, replace=True)
|
||||
results = table.search("puppy").limit(100).to_list()
|
||||
|
||||
@@ -559,13 +558,9 @@ def test_search_fts_phrase_query(table):
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_fts_phrase_query_async(async_table):
|
||||
await async_table.create_index("text", config=FTS(with_position=False))
|
||||
try:
|
||||
phrase_results = (
|
||||
await async_table.query().nearest_to_text("puppy runs").limit(100).to_list()
|
||||
)
|
||||
assert False
|
||||
except Exception:
|
||||
pass
|
||||
with pytest.raises(Exception) as error:
|
||||
await async_table.query().nearest_to_text('"puppy runs"').to_list()
|
||||
assert "FTS(with_position=True)" in exception_output(error)
|
||||
await async_table.create_index("text", config=FTS(with_position=True))
|
||||
results = await async_table.query().nearest_to_text("puppy").limit(100).to_list()
|
||||
phrase_results = (
|
||||
@@ -1948,3 +1943,168 @@ async def test_async_fts_code_analyzer(mem_db_async):
|
||||
|
||||
rows = await table.query().nearest_to_text("http").to_list()
|
||||
assert len(rows) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("indexed", [False, True])
|
||||
@pytest.mark.parametrize("operator,expected", [(None, 25), ("OR", 25), ("AND", 15)])
|
||||
def test_simple_fts_operator_and_count(tmp_path, indexed, operator, expected):
|
||||
db = ldb.connect(tmp_path)
|
||||
table = db.create_table(
|
||||
"matches",
|
||||
pa.table(
|
||||
{
|
||||
"id": list(range(30)),
|
||||
"text": ["gripper closed"] * 15
|
||||
+ ["gripper open"] * 10
|
||||
+ ["camera ready"] * 5,
|
||||
}
|
||||
),
|
||||
)
|
||||
if indexed:
|
||||
table.create_index("text", config=FTS())
|
||||
kwargs = {"fts_columns": "text"}
|
||||
if operator is not None:
|
||||
kwargs["operator"] = operator
|
||||
query = table.search("gripper closed", query_type="fts", **kwargs)
|
||||
assert query.count_rows() == expected
|
||||
assert len(query.limit(100).to_list()) == expected
|
||||
assert (
|
||||
table.search("gripper closed", query_type="fts", fts_columns="text")
|
||||
.operator("AND")
|
||||
.count_rows()
|
||||
== 15
|
||||
)
|
||||
if operator is not None:
|
||||
assert query.to_query_object().full_text_query.operator == operator
|
||||
query.where("id >= 5", prefilter=False).select(["id"]).limit(2).offset(1)
|
||||
before = query.to_query_object()
|
||||
assert query.count_rows() == expected - 5
|
||||
assert query.to_query_object() == before
|
||||
assert len(query.to_list()) <= 2
|
||||
assert (
|
||||
table.search("absent", query_type="fts", fts_columns="text").count_rows() == 0
|
||||
)
|
||||
assert (
|
||||
table.search(MatchQuery("gripper closed", "text", operator="AND")).count_rows()
|
||||
== 15
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("indexed", [False, True])
|
||||
@pytest.mark.parametrize("operator,expected", [(None, 25), ("OR", 25), ("AND", 15)])
|
||||
async def test_simple_fts_operator_and_count_async(
|
||||
tmp_path, indexed, operator, expected
|
||||
):
|
||||
db = await ldb.connect_async(tmp_path)
|
||||
table = await db.create_table(
|
||||
"matches",
|
||||
pa.table(
|
||||
{
|
||||
"id": list(range(30)),
|
||||
"text": ["gripper closed"] * 15
|
||||
+ ["gripper open"] * 10
|
||||
+ ["camera ready"] * 5,
|
||||
}
|
||||
),
|
||||
)
|
||||
if indexed:
|
||||
await table.create_index("text", config=FTS())
|
||||
query = await table.search(
|
||||
"gripper closed", query_type="fts", fts_columns="text", operator=operator
|
||||
)
|
||||
assert await query.count_rows() == expected
|
||||
assert len(await query.limit(100).to_list()) == expected
|
||||
query.where("id >= 5").postfilter().select(["id"]).limit(2).offset(1)
|
||||
before = query.to_query_object()
|
||||
assert await query.count_rows() == expected - 5
|
||||
assert query.to_query_object() == before
|
||||
direct = table.query().nearest_to_text(
|
||||
"gripper closed", columns="text", operator="AND"
|
||||
)
|
||||
assert await direct.count_rows() == 15
|
||||
assert direct.to_query_object().full_text_query.query.operator == "AND"
|
||||
assert (
|
||||
await table.query().nearest_to_text("absent", columns="text").count_rows() == 0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("with_position", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"query", ['"gripper closed"', PhraseQuery("gripper closed", "text")]
|
||||
)
|
||||
def test_fts_phrase_count_positions(tmp_path, with_position, query):
|
||||
table = ldb.connect(tmp_path).create_table(
|
||||
"phrases", pa.table({"text": ["gripper closed", "gripper often closed"]})
|
||||
)
|
||||
table.create_index("text", config=FTS(with_position=with_position))
|
||||
search = table.search(query, query_type="fts")
|
||||
if with_position:
|
||||
assert search.count_rows() == 1
|
||||
assert len(search.to_list()) == 1
|
||||
assert (
|
||||
table.search("gripper closed", query_type="fts", operator="AND")
|
||||
.phrase_query()
|
||||
.count_rows()
|
||||
== 1
|
||||
)
|
||||
else:
|
||||
with pytest.raises(Exception) as error:
|
||||
search.count_rows()
|
||||
assert "FTS(with_position=True)" in exception_output(error)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("with_position", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"query", ['"gripper closed"', PhraseQuery("gripper closed", "text")]
|
||||
)
|
||||
async def test_fts_phrase_count_positions_async(tmp_path, with_position, query):
|
||||
db = await ldb.connect_async(tmp_path)
|
||||
table = await db.create_table(
|
||||
"phrases", pa.table({"text": ["gripper closed", "gripper often closed"]})
|
||||
)
|
||||
await table.create_index("text", config=FTS(with_position=with_position))
|
||||
search = table.query().nearest_to_text(query)
|
||||
if with_position:
|
||||
assert await search.count_rows() == 1
|
||||
assert len(await search.to_list()) == 1
|
||||
else:
|
||||
with pytest.raises(Exception) as error:
|
||||
await search.count_rows()
|
||||
assert "FTS(with_position=True)" in exception_output(error)
|
||||
|
||||
|
||||
def test_fts_operator_rejects_invalid_queries(table):
|
||||
with pytest.raises(ValueError):
|
||||
table.search("gripper", query_type="fts", operator="XOR")
|
||||
for query in [None, [1.0, 2.0], MatchQuery("gripper", "text")]:
|
||||
with pytest.raises(
|
||||
ValueError, match="operator requires a string full-text query"
|
||||
):
|
||||
table.search(query, operator="AND")
|
||||
with pytest.raises(ValueError, match="operator requires a string full-text query"):
|
||||
FullTextSearchQuery(query=MatchQuery("gripper"), operator="AND")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fts_operator_rejects_invalid_queries_async(async_table):
|
||||
with pytest.raises(ValueError):
|
||||
await async_table.search("gripper", query_type="fts", operator="XOR")
|
||||
for query in [None, [1.0, 2.0], MatchQuery("gripper", "text")]:
|
||||
with pytest.raises(
|
||||
ValueError, match="operator requires a string full-text query"
|
||||
):
|
||||
await async_table.search(query, operator="AND")
|
||||
with pytest.raises(ValueError):
|
||||
async_table.query().nearest_to_text(query, operator="AND")
|
||||
|
||||
|
||||
def test_fts_count_fast_search(tmp_path):
|
||||
table = ldb.connect(tmp_path).create_table(
|
||||
"matches", pa.table({"text": ["gripper closed"] * 15})
|
||||
)
|
||||
table.create_index("text", config=FTS())
|
||||
table.add(pa.table({"text": ["gripper closed"] * 5}))
|
||||
assert table.search("gripper", query_type="fts").count_rows() == 20
|
||||
assert table.search("gripper", query_type="fts").fast_search().count_rows() == 15
|
||||
|
||||
@@ -18,6 +18,7 @@ import lancedb
|
||||
from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
|
||||
from utils import start_server_thread
|
||||
from lancedb.listing import AsyncListing, Listing
|
||||
from lancedb.index import FTS
|
||||
from lancedb.materialized_view import DEFINITION_FORMAT, DEFINITION_META_KEY
|
||||
from lancedb.namespace import (
|
||||
_MAX_QUERY_K,
|
||||
@@ -25,7 +26,7 @@ from lancedb.namespace import (
|
||||
LanceNamespaceDBConnection,
|
||||
_query_to_namespace_request,
|
||||
)
|
||||
from lancedb.query import Query
|
||||
from lancedb.query import FullTextSearchQuery, Query
|
||||
from lancedb.table import AsyncTable, LanceTable
|
||||
|
||||
|
||||
@@ -1524,3 +1525,42 @@ def test_python_namespace_attribute_access(tmp_path):
|
||||
# Missing dunder lookups still raise, so copy/hasattr probes work.
|
||||
assert not hasattr(request, "__missing_dunder__")
|
||||
assert copy.deepcopy(request) == request
|
||||
|
||||
|
||||
@pytest.mark.parametrize("columns", [None, ["text"], ["title", "body"]])
|
||||
def test_namespace_fts_operator_is_preserved(columns):
|
||||
request = _query_to_namespace_request(
|
||||
["matches"],
|
||||
Query(
|
||||
full_text_query=FullTextSearchQuery(
|
||||
query="gripper closed", columns=columns, operator="AND"
|
||||
)
|
||||
),
|
||||
)
|
||||
structured = request.to_dict()["full_text_query"]["structured_query"]["query"]
|
||||
if columns and len(columns) > 1:
|
||||
assert all(
|
||||
query["operator"] == "And"
|
||||
for query in structured["multi_match"]["match_queries"]
|
||||
)
|
||||
else:
|
||||
assert structured["match"]["operator"] == "And"
|
||||
assert structured["match"].get("column") == (columns[0] if columns else None)
|
||||
|
||||
|
||||
def test_namespace_fts_count_fast_search(tmp_path):
|
||||
properties = {"root": str(tmp_path)}
|
||||
db = lancedb.connect_namespace("dir", properties)
|
||||
table = db.create_table("matches", pa.table({"text": ["gripper"] * 15}))
|
||||
table.create_index("text", config=FTS())
|
||||
table.add(pa.table({"text": ["gripper"] * 5}))
|
||||
assert table.search("gripper", query_type="fts").fast_search().count_rows() == 15
|
||||
pushed = lancedb.connect_namespace(
|
||||
"dir", properties, namespace_client_pushdown_operations=["QueryTable"]
|
||||
).open_table("matches")
|
||||
assert pushed.search("gripper", query_type="fts").count_rows() == 20
|
||||
query = pushed.search("gripper", query_type="fts").fast_search()
|
||||
assert query.use_rte(False).count_rows() == 15
|
||||
query = pushed.search("gripper", query_type="fts").fast_search()
|
||||
assert query.count_rows() == 15
|
||||
assert len(query.limit(100).to_list()) == 15
|
||||
|
||||
@@ -7802,3 +7802,46 @@ async def test_remote_index_exist_ok_wire(is_async, background, kwargs):
|
||||
assert body["replace"] is False
|
||||
else:
|
||||
assert "replace" not in body
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operator", ["AND", "OR"])
|
||||
def test_remote_fts_operator_and_count(operator):
|
||||
def handler(body):
|
||||
query = body["full_text_query"]["query"]["match"]
|
||||
assert query["terms"] == "gripper closed"
|
||||
assert query["operator"] == operator.title()
|
||||
assert query["column"] == "text"
|
||||
assert body["k"] == 2**63 - 1
|
||||
assert "offset" not in body
|
||||
assert body["columns"] == ["_score"]
|
||||
assert body["filter"] == "text IS NOT NULL"
|
||||
assert body["prefilter"] is False
|
||||
return pa.table({"_score": [1.0] * 25})
|
||||
|
||||
with query_test_table(
|
||||
handler, server_version=Version("0.6.0"), vector_columns={}
|
||||
) as table:
|
||||
query = (
|
||||
table.search(
|
||||
"gripper closed",
|
||||
query_type="fts",
|
||||
fts_columns="text",
|
||||
operator=operator,
|
||||
)
|
||||
.where("text IS NOT NULL", prefilter=False)
|
||||
.select(["text"])
|
||||
.limit(1)
|
||||
.offset(3)
|
||||
)
|
||||
assert query.count_rows() == 25
|
||||
assert query.to_query_object().limit == 1
|
||||
assert query.to_query_object().offset == 3
|
||||
|
||||
|
||||
def test_remote_fts_and_rejects_legacy_server():
|
||||
def handler(body):
|
||||
pytest.fail("AND must not silently become OR on a legacy server")
|
||||
|
||||
with query_test_table(handler, vector_columns={}) as table:
|
||||
with pytest.raises(NotImplementedError, match="server version 0.3.0"):
|
||||
table.search("gripper closed", query_type="fts", operator="AND").to_list()
|
||||
|
||||
+35
-2
@@ -16,19 +16,20 @@ use arrow::array::make_array;
|
||||
use arrow::pyarrow::FromPyArrow;
|
||||
use arrow::pyarrow::IntoPyArrow;
|
||||
use arrow::pyarrow::ToPyArrow;
|
||||
use futures::TryStreamExt;
|
||||
use lancedb::ApproxMode;
|
||||
use lancedb::index::scalar::{
|
||||
BooleanQuery, BoostQuery, CombinedFieldsQuery, DocumentGranularity, FtsQuery,
|
||||
FullTextSearchQuery, MatchQuery, MultiMatchQuery, Occur, Operator, PhraseQuery,
|
||||
};
|
||||
use lancedb::query::AnalyzePlanDistributedMetrics;
|
||||
use lancedb::query::QueryBase;
|
||||
use lancedb::query::QueryExecutionOptions;
|
||||
use lancedb::query::QueryFilter;
|
||||
use lancedb::query::{
|
||||
ColumnOrdering, ExecutableQuery, Query as LanceDbQuery, Select, TakeQuery as LanceDbTakeQuery,
|
||||
VectorQuery as LanceDbVectorQuery,
|
||||
};
|
||||
use lancedb::query::{HasQuery, QueryBase};
|
||||
use lancedb::table::AnyQuery;
|
||||
use pyo3::Bound;
|
||||
use pyo3::IntoPyObject;
|
||||
@@ -616,8 +617,18 @@ impl Query {
|
||||
query_text = query_text[1..query_text.len() - 1].to_string();
|
||||
}
|
||||
|
||||
let operator = query
|
||||
.get_item("operator")?
|
||||
.filter(|operator| !operator.is_none())
|
||||
.map(|operator| {
|
||||
let operator = operator.extract::<String>()?;
|
||||
Operator::try_from(operator.as_str())
|
||||
.map_err(|err| PyValueError::new_err(err.to_string()))
|
||||
})
|
||||
.transpose()?
|
||||
.unwrap_or(Operator::Or);
|
||||
let query: FtsQuery = match (is_phrase, is_multi_match) {
|
||||
(false, _) => MatchQuery::new(query_text).into(),
|
||||
(false, _) => MatchQuery::new(query_text).with_operator(operator).into(),
|
||||
(true, false) => PhraseQuery::new(query_text).into(),
|
||||
(true, true) => {
|
||||
return Err(PyValueError::new_err(
|
||||
@@ -898,6 +909,28 @@ impl FTSQuery {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn count_rows(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let mut inner = self_
|
||||
.inner
|
||||
.clone()
|
||||
.full_text_search(self_.fts_query.clone());
|
||||
let request = inner.mut_query();
|
||||
request.limit = None;
|
||||
request.offset = None;
|
||||
request.select = Select::columns(&["_score"]);
|
||||
request.order_by = None;
|
||||
request.with_row_id = false;
|
||||
request.reranker = None;
|
||||
future_into_py(self_.py(), async move {
|
||||
let mut stream = inner.execute().await.infer_error()?;
|
||||
let mut count = 0_u64;
|
||||
while let Some(batch) = stream.try_next().await.infer_error()? {
|
||||
count += batch.num_rows() as u64;
|
||||
}
|
||||
Ok(count)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn nearest_to(&mut self, vector: Bound<'_, PyAny>) -> PyResult<HybridQuery> {
|
||||
let vector_query = Query::new(self.inner.clone()).nearest_to(vector)?;
|
||||
Ok(HybridQuery {
|
||||
|
||||
Reference in New Issue
Block a user