feat(python): expose FTS operators and match counts (#5042)

String full-text searches now accept `operator="AND"` or
`operator="OR"`, preserving the existing OR default. Sync and async FTS
builders expose `count_rows()` to count every match satisfying the
filter, independently of pagination, projection, ordering, and
reranking. Counting streams matching scores without reading document
columns; it can scan all matches.

```python
query = table.search("gripper closed", query_type="fts", operator="AND")
query = query.where("active = true")
matching_rows = query.count_rows()
page = query.limit(20).to_list()

query = await async_table.search("gripper closed", query_type="fts", operator="AND")
matching_rows = await query.where("active = true").count_rows()
```

Phrase searches keep the existing index default and now explain how to
rebuild an index with `FTS(with_position=True)`. AND requests retain
their operator through namespace pushdown and fail explicitly on older
remote servers that cannot represent it. Regression coverage includes
indexed and unindexed queries, filtered counts above the default limit,
pagination preservation, remote serialization, and phrase indexes with
and without positions.
This commit is contained in:
Jack Ye
2026-11-15 01:21:28 -08:00
committed by GitHub
parent f17a07a2cd
commit 1feec84d2b
11 changed files with 599 additions and 61 deletions
+1
View File
@@ -943,6 +943,7 @@ class FTSQuery:
def add_query_vector(self, query_vec: pa.Array) -> None: ...
def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ...
def order_by(self, ordering: Optional[List[ColumnOrdering]]): ...
async def count_rows(self) -> int: ...
async def output_schema(self) -> pa.Schema: ...
async def execute(
self, max_batch_length: Optional[int], timeout: Optional[timedelta]
+25
View File
@@ -10,6 +10,7 @@ through a namespace abstraction.
from __future__ import annotations
import json
import sys
import warnings
from typing import Any, Dict, Iterable, List, Optional, Union
@@ -39,6 +40,9 @@ from lance_namespace_urllib3_client.models.query_table_request_vector import (
QueryTableRequestVector,
)
from lance_namespace_urllib3_client.models.string_fts_query import StringFtsQuery
from lance_namespace_urllib3_client.models.structured_fts_query import (
StructuredFtsQuery,
)
from lance_namespace.errors import (
NamespaceNotEmptyError,
NamespaceNotFoundError,
@@ -78,6 +82,7 @@ from lancedb.materialized_view import (
SelectArg,
)
from lancedb.table import AsyncTable, LanceTable, Table
from lancedb.query import MatchQuery, MultiMatchQuery, PhraseQuery
from lancedb.util import validate_table_name
from lancedb.common import DATA
from lancedb.pydantic import LanceModel
@@ -178,6 +183,26 @@ def _query_to_namespace_request(
full_text_query_request = QueryTableRequestFullTextQuery(
string_query=StringFtsQuery(query=fts_query)
)
elif getattr(fts_query, "operator", None) is not None:
columns = fts_query.columns or []
text = fts_query.query
if len(text) >= 2 and text.startswith('"') and text.endswith('"'):
if len(columns) > 1:
raise ValueError(
"Phrase queries cannot be used with multiple columns."
)
structured = PhraseQuery(text[1:-1], columns[0] if columns else None)
elif len(columns) > 1:
structured = MultiMatchQuery(text, columns, operator=fts_query.operator)
else:
structured = MatchQuery(
text, columns[0] if columns else None, operator=fts_query.operator
)
full_text_query_request = QueryTableRequestFullTextQuery(
structured_query=StructuredFtsQuery.from_dict(
{"query": json.loads(structured.to_json())}
)
)
elif hasattr(fts_query, "query"):
full_text_query_request = QueryTableRequestFullTextQuery(
string_query=StringFtsQuery(
+116 -11
View File
@@ -42,7 +42,7 @@ from .rerankers.base import Reranker
from .rerankers.rrf import RRFReranker
from .rerankers.util import check_reranker_result
from .schema import blob_column_paths, is_blob_like_field, schema_has_blob_field
from .util import _validate_query_vector, flatten_columns, get_uri_scheme
from .util import _validate_query_vector, add_note, flatten_columns, get_uri_scheme
from . import _wal_hybrid # WAL-PK-FUSION: delete.
from ._blob import (
BLOB_MODE_TO_HANDLING,
@@ -873,10 +873,29 @@ class FullTextSearchQuery(pydantic.BaseModel):
query: str | FullTextQuery
If a string, it is treated as a MatchQuery.
If a FullTextQuery object, it is used directly.
operator: FullTextOperator, optional
Combine terms in a string query using AND or OR (the default).
Quoted strings keep phrase semantics regardless of the operator.
"""
columns: Optional[List[str]] = None
query: Union[str, FullTextQuery]
operator: Optional[FullTextOperator] = None
@pydantic.model_validator(mode="after")
def _validate_operator(self) -> Self:
if self.operator is not None and not isinstance(self.query, str):
raise ValueError("operator requires a string full-text query")
return self
def _add_fts_position_error_note(error: Exception) -> None:
if "position is not found but required for phrase queries" in str(error):
add_note(
error,
"Phrase queries require an index built with positions. Rebuild it with "
"table.create_index(column, config=FTS(with_position=True), replace=True).",
)
class Query(pydantic.BaseModel):
@@ -1147,6 +1166,7 @@ class LanceQueryBuilder(ABC):
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
fast_search: bool = None,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> Self:
"""
Create a query builder based on the given query and query type.
@@ -1171,6 +1191,10 @@ class LanceQueryBuilder(ABC):
fast_search: bool
Skip flat search of unindexed data.
"""
if operator is not None:
operator = FullTextOperator(operator)
if not isinstance(query, str) or query_type not in ("auto", "fts"):
raise ValueError("operator requires a string full-text query")
if query_type != "fts":
_validate_query_vector(query)
@@ -1201,6 +1225,9 @@ class LanceQueryBuilder(ABC):
table, query, query_type, vector_column_name
)
if operator is not None and query_type != "fts":
raise ValueError("operator requires a string full-text query")
if query_type == "hybrid":
return LanceHybridQueryBuilder(
table, query, vector_column_name, fts_columns=fts_columns
@@ -1214,7 +1241,7 @@ class LanceQueryBuilder(ABC):
ordering_field_name=ordering_field_name,
fts_columns=fts_columns,
fast_search=fast_search,
)
).operator(operator)
if isinstance(query, (list, tuple)):
query = np.array(query, dtype=np.float32)
@@ -2563,6 +2590,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
super().__init__(table)
self._query = query
self._phrase_query = False
self._operator: Optional[FullTextOperator] = None
# Deprecated compatibility parameter. Native FTS ordering is now
# configured through order_by(); LanceQueryBuilder.create emits the warning.
_ = ordering_field_name
@@ -2572,6 +2600,36 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
fts_columns = [fts_columns]
self._fts_columns = fts_columns
def operator(
self, operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]]
) -> LanceFtsQueryBuilder:
"""Combine terms in a string query with ``"AND"`` or ``"OR"``.
The default is OR. AND requires every term to match. For example,
``table.search("gripper closed", query_type="fts").operator("AND")``.
Structured queries specify their own operators. Quoted strings and
``phrase_query()`` retain their phrase semantics.
"""
if operator is not None and not isinstance(self._query, str):
raise ValueError("operator requires a string full-text query")
self._operator = FullTextOperator(operator) if operator is not None else None
return self
def count_rows(self) -> int:
"""Count all full-text matches satisfying the query's filter.
Ignores limit, offset, selection, ordering, and reranking. ``fast_search``
still restricts the count to indexed data. This streams matching scores
without loading document columns and may scan every match.
For example, ``table.search("gripper closed", query_type="fts",``
``operator="AND").where("active = true").count_rows()``.
"""
self._table._ensure_no_legacy_fts_index()
query = self._table._table._sync_query_to_async(self.to_query_object())
assert isinstance(query, AsyncFTSQuery)
return LOOP.run(query.count_rows())
def phrase_query(self, phrase_query: bool = True) -> LanceFtsQueryBuilder:
"""Set whether to use phrase query.
@@ -2628,7 +2686,9 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
fragment_ids=self._fragment_ids,
use_rte=self._use_rte,
full_text_query=FullTextSearchQuery(
query=self._query_with_phrase_semantics(), columns=self._fts_columns
query=self._query_with_phrase_semantics(),
columns=self._fts_columns,
operator=self._operator,
),
offset=self._offset,
fast_search=self._fast_search,
@@ -4443,7 +4503,11 @@ class AsyncQuery(AsyncStandardQuery):
)
def nearest_to_text(
self, query: str | FullTextQuery, columns: Union[str, List[str], None] = None
self,
query: str | FullTextQuery,
columns: Union[str, List[str], None] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> AsyncFTSQuery:
"""
Find the documents that are most relevant to the given text query.
@@ -4464,7 +4528,11 @@ class AsyncQuery(AsyncStandardQuery):
columns: str or list of str, default None
The columns to search in. If None, all indexed columns are searched.
For now only one column can be searched at a time.
operator: "AND" or "OR", optional
Combine terms in a string query. Defaults to OR. Quoted strings use
phrase semantics and require an index built with positions.
"""
fts_query = FullTextSearchQuery(query=query, operator=operator)
if isinstance(columns, str):
columns = [columns]
if columns is None:
@@ -4472,7 +4540,13 @@ class AsyncQuery(AsyncStandardQuery):
if isinstance(query, str):
return AsyncFTSQuery(
self._inner.nearest_to_text({"query": query, "columns": columns}),
self._inner.nearest_to_text(
{
"query": query,
"columns": columns,
"operator": fts_query.operator,
}
),
self._table,
)
# FullTextQuery object
@@ -4487,6 +4561,22 @@ class AsyncFTSQuery(AsyncStandardQuery):
self._inner = inner
self._reranker = None
async def count_rows(self) -> int:
"""Count all full-text matches satisfying the query's filter.
Ignores limit, offset, selection, ordering, and reranking. ``fast_search``
still restricts the count to indexed data. Matching scores are streamed
without loading document columns, so counting may scan every match.
For example, ``await table.query().nearest_to_text("gripper closed",``
``operator="AND").where("active = true").count_rows()``.
"""
try:
return await self._inner.count_rows()
except Exception as error:
_add_fts_position_error_note(error)
raise
def get_query(self) -> str:
return self._inner.get_query()
@@ -4580,10 +4670,16 @@ class AsyncFTSQuery(AsyncStandardQuery):
timeout: Optional[timedelta] = None,
blob_sources: Optional[dict[str, str]] = None,
) -> AsyncRecordBatchReader:
reader = await super()._execute_batches(
timeout=timeout, blob_sources=blob_sources
)
results = pa.Table.from_batches(await reader.read_all(), reader.schema)
try:
reader = await super()._execute_batches(
max_batch_length=max_batch_length,
timeout=timeout,
blob_sources=blob_sources,
)
results = pa.Table.from_batches(await reader.read_all(), reader.schema)
except Exception as error:
_add_fts_position_error_note(error)
raise
if self._reranker:
source = results
results = self._reranker.rerank_fts(self.get_query(), results)
@@ -4861,7 +4957,11 @@ class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase):
return self
def nearest_to_text(
self, query: str | FullTextQuery, columns: Union[str, List[str], None] = None
self,
query: str | FullTextQuery,
columns: Union[str, List[str], None] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> AsyncHybridQuery:
"""
Find the documents that are most relevant to the given text query,
@@ -4886,7 +4986,10 @@ class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase):
columns: str or list of str, default None
The columns to search in. If None, all indexed columns are searched.
For now only one column can be searched at a time.
operator: "AND" or "OR", optional
Combine terms in a string query. Defaults to OR.
"""
fts_query = FullTextSearchQuery(query=query, operator=operator)
if isinstance(columns, str):
columns = [columns]
if columns is None:
@@ -4894,7 +4997,9 @@ class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase):
if isinstance(query, str):
return AsyncHybridQuery(
self._inner.nearest_to_text({"query": query, "columns": columns}),
self._inner.nearest_to_text(
{"query": query, "columns": columns, "operator": fts_query.operator}
),
self._table,
)
# FullTextQuery object
+8
View File
@@ -65,6 +65,7 @@ from lancedb.embeddings import EmbeddingFunctionRegistry
from lancedb.table import _normalize_progress
from ..query import (
FullTextOperator,
AnalyzePlanDistributedMetrics,
DocumentGranularity,
FullTextQuery,
@@ -799,6 +800,8 @@ class RemoteTable(Table):
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
fast_search: Optional[bool] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> LanceQueryBuilder:
"""Create a search query to find the nearest neighbors
of the given query vector. We currently support
@@ -857,6 +860,10 @@ class RemoteTable(Table):
- *default None*, which leaves fast search disabled.
operator: "AND" or "OR", optional
Combine terms in a string full-text query; the default is OR.
Use ``query_type="fts"`` when the table also has embeddings.
Returns
-------
LanceQueryBuilder
@@ -893,6 +900,7 @@ class RemoteTable(Table):
ordering_field_name=ordering_field_name,
fts_columns=fts_columns or [],
fast_search=fast_search,
operator=operator,
)
def _execute_query(
+57 -2
View File
@@ -91,6 +91,7 @@ from .functions import (
from .merge import LanceMergeInsertBuilder
from .pydantic import LanceModel, model_to_dict
from .query import (
FullTextOperator,
AnalyzePlanDistributedMetrics,
AsyncFTSQuery,
AsyncHybridQuery,
@@ -138,6 +139,11 @@ def _requires_local_namespace_execution(query: Query) -> bool:
or query.take_row_ids is not None
or query.search_effort is not None
or query.approx_mode is not None
or (
query.full_text_query is not None
and query.fast_search is True
and query.vector is None
)
)
@@ -2443,6 +2449,8 @@ class Table(ABC):
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
fast_search: Optional[bool] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> LanceQueryBuilder:
"""Create a search query to find the nearest neighbors
of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/)
@@ -2506,6 +2514,10 @@ class Table(ABC):
- If `query` is a string, then the query type is "vector" if the
table has embedding functions else the query type is "fts"
operator: "AND" or "OR", optional
Combine terms in a string full-text query; the default is OR.
Use ``query_type="fts"`` when the table also has embeddings.
Returns
-------
LanceQueryBuilder
@@ -5061,6 +5073,8 @@ class LanceTable(Table):
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
fast_search: Optional[bool] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> LanceVectorQueryBuilder: ...
@overload
@@ -5072,6 +5086,8 @@ class LanceTable(Table):
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
fast_search: Optional[bool] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> LanceFtsQueryBuilder: ...
@overload
@@ -5085,6 +5101,8 @@ class LanceTable(Table):
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
fast_search: Optional[bool] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> LanceHybridQueryBuilder: ...
@overload
@@ -5096,6 +5114,8 @@ class LanceTable(Table):
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
fast_search: Optional[bool] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> LanceEmptyQueryBuilder: ...
def search(
@@ -5108,6 +5128,8 @@ class LanceTable(Table):
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
fast_search: Optional[bool] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> LanceQueryBuilder:
"""Create a search query to find the nearest neighbors
of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/)
@@ -5168,6 +5190,10 @@ class LanceTable(Table):
If None then the search is performed on all indexed columns.
For now, only one column can be searched at a time.
operator: "AND" or "OR", optional
Combine terms in a string full-text query; the default is OR.
Use ``query_type="fts"`` when the table also has embeddings.
Returns
-------
LanceQueryBuilder
@@ -5193,6 +5219,7 @@ class LanceTable(Table):
ordering_field_name=ordering_field_name,
fts_columns=fts_columns or [],
fast_search=fast_search,
operator=operator,
)
@classmethod
@@ -7376,6 +7403,8 @@ class AsyncTable:
query_type: QueryType = "auto",
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> AsyncQuery: ...
@overload
@@ -7386,6 +7415,8 @@ class AsyncTable:
query_type: Literal["auto"] = ...,
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> Union[AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]: ...
@overload
@@ -7396,6 +7427,8 @@ class AsyncTable:
query_type: Literal["hybrid"] = ...,
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> AsyncHybridQuery: ...
@overload
@@ -7406,6 +7439,8 @@ class AsyncTable:
query_type: Literal["auto"] = ...,
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> AsyncVectorQuery: ...
@overload
@@ -7416,6 +7451,8 @@ class AsyncTable:
query_type: Literal["fts"] = ...,
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> AsyncFTSQuery: ...
@overload
@@ -7426,6 +7463,8 @@ class AsyncTable:
query_type: Literal["vector"] = ...,
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> AsyncVectorQuery: ...
async def search(
@@ -7437,6 +7476,8 @@ class AsyncTable:
query_type: QueryType = "auto",
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
*,
operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None,
) -> Union[AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]:
"""Create a search query to find the nearest neighbors
of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/)
@@ -7480,12 +7521,20 @@ class AsyncTable:
- If `query` is a string, then the query type is "vector" if the
table has embedding functions else the query type is "fts"
operator: "AND" or "OR", optional
Combine terms in a string full-text query; the default is OR.
Use ``query_type="fts"`` when the table also has embeddings.
Returns
-------
AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, or AsyncVectorQuery
A query builder object representing the query.
"""
if operator is not None:
operator = FullTextOperator(operator)
if not isinstance(query, str) or query_type not in ("auto", "fts"):
raise ValueError("operator requires a string full-text query")
if query is None:
return self.query()
@@ -7604,13 +7653,17 @@ class AsyncTable:
)
vector_query = await make_embedding(embedding_conf, query)
if operator is not None and query_type != "fts":
raise ValueError("operator requires a string full-text query")
if query_type == "vector":
builder = self.query().nearest_to(vector_query)
if vector_column_name:
builder = builder.column(vector_column_name)
return builder
elif query_type == "fts":
return self.query().nearest_to_text(query, columns=fts_columns)
return self.query().nearest_to_text(
query, columns=fts_columns, operator=operator
)
elif query_type == "hybrid":
builder = self.query().nearest_to(vector_query)
if vector_column_name:
@@ -7720,7 +7773,9 @@ class AsyncTable:
if query.full_text_query:
async_query = async_query.nearest_to_text(
query.full_text_query.query, query.full_text_query.columns
query.full_text_query.query,
query.full_text_query.columns,
operator=query.full_text_query.operator,
)
return async_query
+172 -12
View File
@@ -29,6 +29,7 @@ from lancedb.query import (
CombinedFieldsQuery,
DocumentGranularity,
FullTextOperator,
FullTextSearchQuery,
MatchQuery,
MultiMatchQuery,
PhraseQuery,
@@ -530,11 +531,9 @@ async def test_fts_select_async(async_table):
def test_search_fts_phrase_query(table):
table.create_fts_index("text", with_position=False)
try:
phrase_results = table.search('"puppy runs"').limit(100).to_list()
assert False
except Exception:
pass
with pytest.raises(Exception) as error:
table.search('"puppy runs"').limit(100).to_list()
assert "FTS(with_position=True)" in exception_output(error)
table.create_fts_index("text", with_position=True, replace=True)
results = table.search("puppy").limit(100).to_list()
@@ -559,13 +558,9 @@ def test_search_fts_phrase_query(table):
@pytest.mark.asyncio
async def test_search_fts_phrase_query_async(async_table):
await async_table.create_index("text", config=FTS(with_position=False))
try:
phrase_results = (
await async_table.query().nearest_to_text("puppy runs").limit(100).to_list()
)
assert False
except Exception:
pass
with pytest.raises(Exception) as error:
await async_table.query().nearest_to_text('"puppy runs"').to_list()
assert "FTS(with_position=True)" in exception_output(error)
await async_table.create_index("text", config=FTS(with_position=True))
results = await async_table.query().nearest_to_text("puppy").limit(100).to_list()
phrase_results = (
@@ -1948,3 +1943,168 @@ async def test_async_fts_code_analyzer(mem_db_async):
rows = await table.query().nearest_to_text("http").to_list()
assert len(rows) == 1
@pytest.mark.parametrize("indexed", [False, True])
@pytest.mark.parametrize("operator,expected", [(None, 25), ("OR", 25), ("AND", 15)])
def test_simple_fts_operator_and_count(tmp_path, indexed, operator, expected):
db = ldb.connect(tmp_path)
table = db.create_table(
"matches",
pa.table(
{
"id": list(range(30)),
"text": ["gripper closed"] * 15
+ ["gripper open"] * 10
+ ["camera ready"] * 5,
}
),
)
if indexed:
table.create_index("text", config=FTS())
kwargs = {"fts_columns": "text"}
if operator is not None:
kwargs["operator"] = operator
query = table.search("gripper closed", query_type="fts", **kwargs)
assert query.count_rows() == expected
assert len(query.limit(100).to_list()) == expected
assert (
table.search("gripper closed", query_type="fts", fts_columns="text")
.operator("AND")
.count_rows()
== 15
)
if operator is not None:
assert query.to_query_object().full_text_query.operator == operator
query.where("id >= 5", prefilter=False).select(["id"]).limit(2).offset(1)
before = query.to_query_object()
assert query.count_rows() == expected - 5
assert query.to_query_object() == before
assert len(query.to_list()) <= 2
assert (
table.search("absent", query_type="fts", fts_columns="text").count_rows() == 0
)
assert (
table.search(MatchQuery("gripper closed", "text", operator="AND")).count_rows()
== 15
)
@pytest.mark.asyncio
@pytest.mark.parametrize("indexed", [False, True])
@pytest.mark.parametrize("operator,expected", [(None, 25), ("OR", 25), ("AND", 15)])
async def test_simple_fts_operator_and_count_async(
tmp_path, indexed, operator, expected
):
db = await ldb.connect_async(tmp_path)
table = await db.create_table(
"matches",
pa.table(
{
"id": list(range(30)),
"text": ["gripper closed"] * 15
+ ["gripper open"] * 10
+ ["camera ready"] * 5,
}
),
)
if indexed:
await table.create_index("text", config=FTS())
query = await table.search(
"gripper closed", query_type="fts", fts_columns="text", operator=operator
)
assert await query.count_rows() == expected
assert len(await query.limit(100).to_list()) == expected
query.where("id >= 5").postfilter().select(["id"]).limit(2).offset(1)
before = query.to_query_object()
assert await query.count_rows() == expected - 5
assert query.to_query_object() == before
direct = table.query().nearest_to_text(
"gripper closed", columns="text", operator="AND"
)
assert await direct.count_rows() == 15
assert direct.to_query_object().full_text_query.query.operator == "AND"
assert (
await table.query().nearest_to_text("absent", columns="text").count_rows() == 0
)
@pytest.mark.parametrize("with_position", [False, True])
@pytest.mark.parametrize(
"query", ['"gripper closed"', PhraseQuery("gripper closed", "text")]
)
def test_fts_phrase_count_positions(tmp_path, with_position, query):
table = ldb.connect(tmp_path).create_table(
"phrases", pa.table({"text": ["gripper closed", "gripper often closed"]})
)
table.create_index("text", config=FTS(with_position=with_position))
search = table.search(query, query_type="fts")
if with_position:
assert search.count_rows() == 1
assert len(search.to_list()) == 1
assert (
table.search("gripper closed", query_type="fts", operator="AND")
.phrase_query()
.count_rows()
== 1
)
else:
with pytest.raises(Exception) as error:
search.count_rows()
assert "FTS(with_position=True)" in exception_output(error)
@pytest.mark.asyncio
@pytest.mark.parametrize("with_position", [False, True])
@pytest.mark.parametrize(
"query", ['"gripper closed"', PhraseQuery("gripper closed", "text")]
)
async def test_fts_phrase_count_positions_async(tmp_path, with_position, query):
db = await ldb.connect_async(tmp_path)
table = await db.create_table(
"phrases", pa.table({"text": ["gripper closed", "gripper often closed"]})
)
await table.create_index("text", config=FTS(with_position=with_position))
search = table.query().nearest_to_text(query)
if with_position:
assert await search.count_rows() == 1
assert len(await search.to_list()) == 1
else:
with pytest.raises(Exception) as error:
await search.count_rows()
assert "FTS(with_position=True)" in exception_output(error)
def test_fts_operator_rejects_invalid_queries(table):
with pytest.raises(ValueError):
table.search("gripper", query_type="fts", operator="XOR")
for query in [None, [1.0, 2.0], MatchQuery("gripper", "text")]:
with pytest.raises(
ValueError, match="operator requires a string full-text query"
):
table.search(query, operator="AND")
with pytest.raises(ValueError, match="operator requires a string full-text query"):
FullTextSearchQuery(query=MatchQuery("gripper"), operator="AND")
@pytest.mark.asyncio
async def test_fts_operator_rejects_invalid_queries_async(async_table):
with pytest.raises(ValueError):
await async_table.search("gripper", query_type="fts", operator="XOR")
for query in [None, [1.0, 2.0], MatchQuery("gripper", "text")]:
with pytest.raises(
ValueError, match="operator requires a string full-text query"
):
await async_table.search(query, operator="AND")
with pytest.raises(ValueError):
async_table.query().nearest_to_text(query, operator="AND")
def test_fts_count_fast_search(tmp_path):
table = ldb.connect(tmp_path).create_table(
"matches", pa.table({"text": ["gripper closed"] * 15})
)
table.create_index("text", config=FTS())
table.add(pa.table({"text": ["gripper closed"] * 5}))
assert table.search("gripper", query_type="fts").count_rows() == 20
assert table.search("gripper", query_type="fts").fast_search().count_rows() == 15
+41 -1
View File
@@ -18,6 +18,7 @@ import lancedb
from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
from utils import start_server_thread
from lancedb.listing import AsyncListing, Listing
from lancedb.index import FTS
from lancedb.materialized_view import DEFINITION_FORMAT, DEFINITION_META_KEY
from lancedb.namespace import (
_MAX_QUERY_K,
@@ -25,7 +26,7 @@ from lancedb.namespace import (
LanceNamespaceDBConnection,
_query_to_namespace_request,
)
from lancedb.query import Query
from lancedb.query import FullTextSearchQuery, Query
from lancedb.table import AsyncTable, LanceTable
@@ -1524,3 +1525,42 @@ def test_python_namespace_attribute_access(tmp_path):
# Missing dunder lookups still raise, so copy/hasattr probes work.
assert not hasattr(request, "__missing_dunder__")
assert copy.deepcopy(request) == request
@pytest.mark.parametrize("columns", [None, ["text"], ["title", "body"]])
def test_namespace_fts_operator_is_preserved(columns):
request = _query_to_namespace_request(
["matches"],
Query(
full_text_query=FullTextSearchQuery(
query="gripper closed", columns=columns, operator="AND"
)
),
)
structured = request.to_dict()["full_text_query"]["structured_query"]["query"]
if columns and len(columns) > 1:
assert all(
query["operator"] == "And"
for query in structured["multi_match"]["match_queries"]
)
else:
assert structured["match"]["operator"] == "And"
assert structured["match"].get("column") == (columns[0] if columns else None)
def test_namespace_fts_count_fast_search(tmp_path):
properties = {"root": str(tmp_path)}
db = lancedb.connect_namespace("dir", properties)
table = db.create_table("matches", pa.table({"text": ["gripper"] * 15}))
table.create_index("text", config=FTS())
table.add(pa.table({"text": ["gripper"] * 5}))
assert table.search("gripper", query_type="fts").fast_search().count_rows() == 15
pushed = lancedb.connect_namespace(
"dir", properties, namespace_client_pushdown_operations=["QueryTable"]
).open_table("matches")
assert pushed.search("gripper", query_type="fts").count_rows() == 20
query = pushed.search("gripper", query_type="fts").fast_search()
assert query.use_rte(False).count_rows() == 15
query = pushed.search("gripper", query_type="fts").fast_search()
assert query.count_rows() == 15
assert len(query.limit(100).to_list()) == 15
+43
View File
@@ -7802,3 +7802,46 @@ async def test_remote_index_exist_ok_wire(is_async, background, kwargs):
assert body["replace"] is False
else:
assert "replace" not in body
@pytest.mark.parametrize("operator", ["AND", "OR"])
def test_remote_fts_operator_and_count(operator):
def handler(body):
query = body["full_text_query"]["query"]["match"]
assert query["terms"] == "gripper closed"
assert query["operator"] == operator.title()
assert query["column"] == "text"
assert body["k"] == 2**63 - 1
assert "offset" not in body
assert body["columns"] == ["_score"]
assert body["filter"] == "text IS NOT NULL"
assert body["prefilter"] is False
return pa.table({"_score": [1.0] * 25})
with query_test_table(
handler, server_version=Version("0.6.0"), vector_columns={}
) as table:
query = (
table.search(
"gripper closed",
query_type="fts",
fts_columns="text",
operator=operator,
)
.where("text IS NOT NULL", prefilter=False)
.select(["text"])
.limit(1)
.offset(3)
)
assert query.count_rows() == 25
assert query.to_query_object().limit == 1
assert query.to_query_object().offset == 3
def test_remote_fts_and_rejects_legacy_server():
def handler(body):
pytest.fail("AND must not silently become OR on a legacy server")
with query_test_table(handler, vector_columns={}) as table:
with pytest.raises(NotImplementedError, match="server version 0.3.0"):
table.search("gripper closed", query_type="fts", operator="AND").to_list()
+35 -2
View File
@@ -16,19 +16,20 @@ use arrow::array::make_array;
use arrow::pyarrow::FromPyArrow;
use arrow::pyarrow::IntoPyArrow;
use arrow::pyarrow::ToPyArrow;
use futures::TryStreamExt;
use lancedb::ApproxMode;
use lancedb::index::scalar::{
BooleanQuery, BoostQuery, CombinedFieldsQuery, DocumentGranularity, FtsQuery,
FullTextSearchQuery, MatchQuery, MultiMatchQuery, Occur, Operator, PhraseQuery,
};
use lancedb::query::AnalyzePlanDistributedMetrics;
use lancedb::query::QueryBase;
use lancedb::query::QueryExecutionOptions;
use lancedb::query::QueryFilter;
use lancedb::query::{
ColumnOrdering, ExecutableQuery, Query as LanceDbQuery, Select, TakeQuery as LanceDbTakeQuery,
VectorQuery as LanceDbVectorQuery,
};
use lancedb::query::{HasQuery, QueryBase};
use lancedb::table::AnyQuery;
use pyo3::Bound;
use pyo3::IntoPyObject;
@@ -616,8 +617,18 @@ impl Query {
query_text = query_text[1..query_text.len() - 1].to_string();
}
let operator = query
.get_item("operator")?
.filter(|operator| !operator.is_none())
.map(|operator| {
let operator = operator.extract::<String>()?;
Operator::try_from(operator.as_str())
.map_err(|err| PyValueError::new_err(err.to_string()))
})
.transpose()?
.unwrap_or(Operator::Or);
let query: FtsQuery = match (is_phrase, is_multi_match) {
(false, _) => MatchQuery::new(query_text).into(),
(false, _) => MatchQuery::new(query_text).with_operator(operator).into(),
(true, false) => PhraseQuery::new(query_text).into(),
(true, true) => {
return Err(PyValueError::new_err(
@@ -898,6 +909,28 @@ impl FTSQuery {
})
}
pub fn count_rows(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let mut inner = self_
.inner
.clone()
.full_text_search(self_.fts_query.clone());
let request = inner.mut_query();
request.limit = None;
request.offset = None;
request.select = Select::columns(&["_score"]);
request.order_by = None;
request.with_row_id = false;
request.reranker = None;
future_into_py(self_.py(), async move {
let mut stream = inner.execute().await.infer_error()?;
let mut count = 0_u64;
while let Some(batch) = stream.try_next().await.infer_error()? {
count += batch.num_rows() as u64;
}
Ok(count)
})
}
pub fn nearest_to(&mut self, vector: Bound<'_, PyAny>) -> PyResult<HybridQuery> {
let vector_query = Query::new(self.inner.clone()).nearest_to(vector)?;
Ok(HybridQuery {