From 1feec84d2bb8166919a9c52640a7ad64d6f6e0e2 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Sun, 15 Nov 2026 01:21:28 -0800 Subject: [PATCH] feat(python): expose FTS operators and match counts (#5042) String full-text searches now accept `operator="AND"` or `operator="OR"`, preserving the existing OR default. Sync and async FTS builders expose `count_rows()` to count every match satisfying the filter, independently of pagination, projection, ordering, and reranking. Counting streams matching scores without reading document columns; it can scan all matches. ```python query = table.search("gripper closed", query_type="fts", operator="AND") query = query.where("active = true") matching_rows = query.count_rows() page = query.limit(20).to_list() query = await async_table.search("gripper closed", query_type="fts", operator="AND") matching_rows = await query.where("active = true").count_rows() ``` Phrase searches keep the existing index default and now explain how to rebuild an index with `FTS(with_position=True)`. AND requests retain their operator through namespace pushdown and fail explicitly on older remote servers that cannot represent it. Regression coverage includes indexed and unindexed queries, filtered counts above the default limit, pagination preservation, remote serialization, and phrase indexes with and without positions. --- python/python/lancedb/_lancedb.pyi | 1 + python/python/lancedb/namespace.py | 25 ++++ python/python/lancedb/query.py | 127 ++++++++++++++++-- python/python/lancedb/remote/table.py | 8 ++ python/python/lancedb/table.py | 59 ++++++++- python/python/tests/test_fts.py | 184 ++++++++++++++++++++++++-- python/python/tests/test_namespace.py | 42 +++++- python/python/tests/test_remote_db.py | 43 ++++++ python/src/query.rs | 37 +++++- rust/lancedb/src/remote/table.rs | 16 ++- rust/lancedb/src/table/query.rs | 118 ++++++++++++----- 11 files changed, 599 insertions(+), 61 deletions(-) diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 0cce2085e..63d442703 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -943,6 +943,7 @@ class FTSQuery: def add_query_vector(self, query_vec: pa.Array) -> None: ... def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ... def order_by(self, ordering: Optional[List[ColumnOrdering]]): ... + async def count_rows(self) -> int: ... async def output_schema(self) -> pa.Schema: ... async def execute( self, max_batch_length: Optional[int], timeout: Optional[timedelta] diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index 67fcc61f7..463536619 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -10,6 +10,7 @@ through a namespace abstraction. from __future__ import annotations +import json import sys import warnings from typing import Any, Dict, Iterable, List, Optional, Union @@ -39,6 +40,9 @@ from lance_namespace_urllib3_client.models.query_table_request_vector import ( QueryTableRequestVector, ) from lance_namespace_urllib3_client.models.string_fts_query import StringFtsQuery +from lance_namespace_urllib3_client.models.structured_fts_query import ( + StructuredFtsQuery, +) from lance_namespace.errors import ( NamespaceNotEmptyError, NamespaceNotFoundError, @@ -78,6 +82,7 @@ from lancedb.materialized_view import ( SelectArg, ) from lancedb.table import AsyncTable, LanceTable, Table +from lancedb.query import MatchQuery, MultiMatchQuery, PhraseQuery from lancedb.util import validate_table_name from lancedb.common import DATA from lancedb.pydantic import LanceModel @@ -178,6 +183,26 @@ def _query_to_namespace_request( full_text_query_request = QueryTableRequestFullTextQuery( string_query=StringFtsQuery(query=fts_query) ) + elif getattr(fts_query, "operator", None) is not None: + columns = fts_query.columns or [] + text = fts_query.query + if len(text) >= 2 and text.startswith('"') and text.endswith('"'): + if len(columns) > 1: + raise ValueError( + "Phrase queries cannot be used with multiple columns." + ) + structured = PhraseQuery(text[1:-1], columns[0] if columns else None) + elif len(columns) > 1: + structured = MultiMatchQuery(text, columns, operator=fts_query.operator) + else: + structured = MatchQuery( + text, columns[0] if columns else None, operator=fts_query.operator + ) + full_text_query_request = QueryTableRequestFullTextQuery( + structured_query=StructuredFtsQuery.from_dict( + {"query": json.loads(structured.to_json())} + ) + ) elif hasattr(fts_query, "query"): full_text_query_request = QueryTableRequestFullTextQuery( string_query=StringFtsQuery( diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index f99aaeb73..214d2255f 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -42,7 +42,7 @@ from .rerankers.base import Reranker from .rerankers.rrf import RRFReranker from .rerankers.util import check_reranker_result from .schema import blob_column_paths, is_blob_like_field, schema_has_blob_field -from .util import _validate_query_vector, flatten_columns, get_uri_scheme +from .util import _validate_query_vector, add_note, flatten_columns, get_uri_scheme from . import _wal_hybrid # WAL-PK-FUSION: delete. from ._blob import ( BLOB_MODE_TO_HANDLING, @@ -873,10 +873,29 @@ class FullTextSearchQuery(pydantic.BaseModel): query: str | FullTextQuery If a string, it is treated as a MatchQuery. If a FullTextQuery object, it is used directly. + operator: FullTextOperator, optional + Combine terms in a string query using AND or OR (the default). + Quoted strings keep phrase semantics regardless of the operator. """ columns: Optional[List[str]] = None query: Union[str, FullTextQuery] + operator: Optional[FullTextOperator] = None + + @pydantic.model_validator(mode="after") + def _validate_operator(self) -> Self: + if self.operator is not None and not isinstance(self.query, str): + raise ValueError("operator requires a string full-text query") + return self + + +def _add_fts_position_error_note(error: Exception) -> None: + if "position is not found but required for phrase queries" in str(error): + add_note( + error, + "Phrase queries require an index built with positions. Rebuild it with " + "table.create_index(column, config=FTS(with_position=True), replace=True).", + ) class Query(pydantic.BaseModel): @@ -1147,6 +1166,7 @@ class LanceQueryBuilder(ABC): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: bool = None, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> Self: """ Create a query builder based on the given query and query type. @@ -1171,6 +1191,10 @@ class LanceQueryBuilder(ABC): fast_search: bool Skip flat search of unindexed data. """ + if operator is not None: + operator = FullTextOperator(operator) + if not isinstance(query, str) or query_type not in ("auto", "fts"): + raise ValueError("operator requires a string full-text query") if query_type != "fts": _validate_query_vector(query) @@ -1201,6 +1225,9 @@ class LanceQueryBuilder(ABC): table, query, query_type, vector_column_name ) + if operator is not None and query_type != "fts": + raise ValueError("operator requires a string full-text query") + if query_type == "hybrid": return LanceHybridQueryBuilder( table, query, vector_column_name, fts_columns=fts_columns @@ -1214,7 +1241,7 @@ class LanceQueryBuilder(ABC): ordering_field_name=ordering_field_name, fts_columns=fts_columns, fast_search=fast_search, - ) + ).operator(operator) if isinstance(query, (list, tuple)): query = np.array(query, dtype=np.float32) @@ -2563,6 +2590,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder): super().__init__(table) self._query = query self._phrase_query = False + self._operator: Optional[FullTextOperator] = None # Deprecated compatibility parameter. Native FTS ordering is now # configured through order_by(); LanceQueryBuilder.create emits the warning. _ = ordering_field_name @@ -2572,6 +2600,36 @@ class LanceFtsQueryBuilder(LanceQueryBuilder): fts_columns = [fts_columns] self._fts_columns = fts_columns + def operator( + self, operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] + ) -> LanceFtsQueryBuilder: + """Combine terms in a string query with ``"AND"`` or ``"OR"``. + + The default is OR. AND requires every term to match. For example, + ``table.search("gripper closed", query_type="fts").operator("AND")``. + Structured queries specify their own operators. Quoted strings and + ``phrase_query()`` retain their phrase semantics. + """ + if operator is not None and not isinstance(self._query, str): + raise ValueError("operator requires a string full-text query") + self._operator = FullTextOperator(operator) if operator is not None else None + return self + + def count_rows(self) -> int: + """Count all full-text matches satisfying the query's filter. + + Ignores limit, offset, selection, ordering, and reranking. ``fast_search`` + still restricts the count to indexed data. This streams matching scores + without loading document columns and may scan every match. + + For example, ``table.search("gripper closed", query_type="fts",`` + ``operator="AND").where("active = true").count_rows()``. + """ + self._table._ensure_no_legacy_fts_index() + query = self._table._table._sync_query_to_async(self.to_query_object()) + assert isinstance(query, AsyncFTSQuery) + return LOOP.run(query.count_rows()) + def phrase_query(self, phrase_query: bool = True) -> LanceFtsQueryBuilder: """Set whether to use phrase query. @@ -2628,7 +2686,9 @@ class LanceFtsQueryBuilder(LanceQueryBuilder): fragment_ids=self._fragment_ids, use_rte=self._use_rte, full_text_query=FullTextSearchQuery( - query=self._query_with_phrase_semantics(), columns=self._fts_columns + query=self._query_with_phrase_semantics(), + columns=self._fts_columns, + operator=self._operator, ), offset=self._offset, fast_search=self._fast_search, @@ -4443,7 +4503,11 @@ class AsyncQuery(AsyncStandardQuery): ) def nearest_to_text( - self, query: str | FullTextQuery, columns: Union[str, List[str], None] = None + self, + query: str | FullTextQuery, + columns: Union[str, List[str], None] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncFTSQuery: """ Find the documents that are most relevant to the given text query. @@ -4464,7 +4528,11 @@ class AsyncQuery(AsyncStandardQuery): columns: str or list of str, default None The columns to search in. If None, all indexed columns are searched. For now only one column can be searched at a time. + operator: "AND" or "OR", optional + Combine terms in a string query. Defaults to OR. Quoted strings use + phrase semantics and require an index built with positions. """ + fts_query = FullTextSearchQuery(query=query, operator=operator) if isinstance(columns, str): columns = [columns] if columns is None: @@ -4472,7 +4540,13 @@ class AsyncQuery(AsyncStandardQuery): if isinstance(query, str): return AsyncFTSQuery( - self._inner.nearest_to_text({"query": query, "columns": columns}), + self._inner.nearest_to_text( + { + "query": query, + "columns": columns, + "operator": fts_query.operator, + } + ), self._table, ) # FullTextQuery object @@ -4487,6 +4561,22 @@ class AsyncFTSQuery(AsyncStandardQuery): self._inner = inner self._reranker = None + async def count_rows(self) -> int: + """Count all full-text matches satisfying the query's filter. + + Ignores limit, offset, selection, ordering, and reranking. ``fast_search`` + still restricts the count to indexed data. Matching scores are streamed + without loading document columns, so counting may scan every match. + + For example, ``await table.query().nearest_to_text("gripper closed",`` + ``operator="AND").where("active = true").count_rows()``. + """ + try: + return await self._inner.count_rows() + except Exception as error: + _add_fts_position_error_note(error) + raise + def get_query(self) -> str: return self._inner.get_query() @@ -4580,10 +4670,16 @@ class AsyncFTSQuery(AsyncStandardQuery): timeout: Optional[timedelta] = None, blob_sources: Optional[dict[str, str]] = None, ) -> AsyncRecordBatchReader: - reader = await super()._execute_batches( - timeout=timeout, blob_sources=blob_sources - ) - results = pa.Table.from_batches(await reader.read_all(), reader.schema) + try: + reader = await super()._execute_batches( + max_batch_length=max_batch_length, + timeout=timeout, + blob_sources=blob_sources, + ) + results = pa.Table.from_batches(await reader.read_all(), reader.schema) + except Exception as error: + _add_fts_position_error_note(error) + raise if self._reranker: source = results results = self._reranker.rerank_fts(self.get_query(), results) @@ -4861,7 +4957,11 @@ class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase): return self def nearest_to_text( - self, query: str | FullTextQuery, columns: Union[str, List[str], None] = None + self, + query: str | FullTextQuery, + columns: Union[str, List[str], None] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncHybridQuery: """ Find the documents that are most relevant to the given text query, @@ -4886,7 +4986,10 @@ class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase): columns: str or list of str, default None The columns to search in. If None, all indexed columns are searched. For now only one column can be searched at a time. + operator: "AND" or "OR", optional + Combine terms in a string query. Defaults to OR. """ + fts_query = FullTextSearchQuery(query=query, operator=operator) if isinstance(columns, str): columns = [columns] if columns is None: @@ -4894,7 +4997,9 @@ class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase): if isinstance(query, str): return AsyncHybridQuery( - self._inner.nearest_to_text({"query": query, "columns": columns}), + self._inner.nearest_to_text( + {"query": query, "columns": columns, "operator": fts_query.operator} + ), self._table, ) # FullTextQuery object diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 818d2a257..2d068536e 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -65,6 +65,7 @@ from lancedb.embeddings import EmbeddingFunctionRegistry from lancedb.table import _normalize_progress from ..query import ( + FullTextOperator, AnalyzePlanDistributedMetrics, DocumentGranularity, FullTextQuery, @@ -799,6 +800,8 @@ class RemoteTable(Table): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceQueryBuilder: """Create a search query to find the nearest neighbors of the given query vector. We currently support @@ -857,6 +860,10 @@ class RemoteTable(Table): - *default None*, which leaves fast search disabled. + operator: "AND" or "OR", optional + Combine terms in a string full-text query; the default is OR. + Use ``query_type="fts"`` when the table also has embeddings. + Returns ------- LanceQueryBuilder @@ -893,6 +900,7 @@ class RemoteTable(Table): ordering_field_name=ordering_field_name, fts_columns=fts_columns or [], fast_search=fast_search, + operator=operator, ) def _execute_query( diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 4a713da06..807edbc46 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -91,6 +91,7 @@ from .functions import ( from .merge import LanceMergeInsertBuilder from .pydantic import LanceModel, model_to_dict from .query import ( + FullTextOperator, AnalyzePlanDistributedMetrics, AsyncFTSQuery, AsyncHybridQuery, @@ -138,6 +139,11 @@ def _requires_local_namespace_execution(query: Query) -> bool: or query.take_row_ids is not None or query.search_effort is not None or query.approx_mode is not None + or ( + query.full_text_query is not None + and query.fast_search is True + and query.vector is None + ) ) @@ -2443,6 +2449,8 @@ class Table(ABC): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceQueryBuilder: """Create a search query to find the nearest neighbors of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/) @@ -2506,6 +2514,10 @@ class Table(ABC): - If `query` is a string, then the query type is "vector" if the table has embedding functions else the query type is "fts" + operator: "AND" or "OR", optional + Combine terms in a string full-text query; the default is OR. + Use ``query_type="fts"`` when the table also has embeddings. + Returns ------- LanceQueryBuilder @@ -5061,6 +5073,8 @@ class LanceTable(Table): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceVectorQueryBuilder: ... @overload @@ -5072,6 +5086,8 @@ class LanceTable(Table): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceFtsQueryBuilder: ... @overload @@ -5085,6 +5101,8 @@ class LanceTable(Table): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceHybridQueryBuilder: ... @overload @@ -5096,6 +5114,8 @@ class LanceTable(Table): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceEmptyQueryBuilder: ... def search( @@ -5108,6 +5128,8 @@ class LanceTable(Table): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceQueryBuilder: """Create a search query to find the nearest neighbors of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/) @@ -5168,6 +5190,10 @@ class LanceTable(Table): If None then the search is performed on all indexed columns. For now, only one column can be searched at a time. + operator: "AND" or "OR", optional + Combine terms in a string full-text query; the default is OR. + Use ``query_type="fts"`` when the table also has embeddings. + Returns ------- LanceQueryBuilder @@ -5193,6 +5219,7 @@ class LanceTable(Table): ordering_field_name=ordering_field_name, fts_columns=fts_columns or [], fast_search=fast_search, + operator=operator, ) @classmethod @@ -7376,6 +7403,8 @@ class AsyncTable: query_type: QueryType = "auto", ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncQuery: ... @overload @@ -7386,6 +7415,8 @@ class AsyncTable: query_type: Literal["auto"] = ..., ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> Union[AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]: ... @overload @@ -7396,6 +7427,8 @@ class AsyncTable: query_type: Literal["hybrid"] = ..., ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncHybridQuery: ... @overload @@ -7406,6 +7439,8 @@ class AsyncTable: query_type: Literal["auto"] = ..., ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncVectorQuery: ... @overload @@ -7416,6 +7451,8 @@ class AsyncTable: query_type: Literal["fts"] = ..., ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncFTSQuery: ... @overload @@ -7426,6 +7463,8 @@ class AsyncTable: query_type: Literal["vector"] = ..., ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncVectorQuery: ... async def search( @@ -7437,6 +7476,8 @@ class AsyncTable: query_type: QueryType = "auto", ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> Union[AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]: """Create a search query to find the nearest neighbors of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/) @@ -7480,12 +7521,20 @@ class AsyncTable: - If `query` is a string, then the query type is "vector" if the table has embedding functions else the query type is "fts" + operator: "AND" or "OR", optional + Combine terms in a string full-text query; the default is OR. + Use ``query_type="fts"`` when the table also has embeddings. + Returns ------- AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, or AsyncVectorQuery A query builder object representing the query. """ + if operator is not None: + operator = FullTextOperator(operator) + if not isinstance(query, str) or query_type not in ("auto", "fts"): + raise ValueError("operator requires a string full-text query") if query is None: return self.query() @@ -7604,13 +7653,17 @@ class AsyncTable: ) vector_query = await make_embedding(embedding_conf, query) + if operator is not None and query_type != "fts": + raise ValueError("operator requires a string full-text query") if query_type == "vector": builder = self.query().nearest_to(vector_query) if vector_column_name: builder = builder.column(vector_column_name) return builder elif query_type == "fts": - return self.query().nearest_to_text(query, columns=fts_columns) + return self.query().nearest_to_text( + query, columns=fts_columns, operator=operator + ) elif query_type == "hybrid": builder = self.query().nearest_to(vector_query) if vector_column_name: @@ -7720,7 +7773,9 @@ class AsyncTable: if query.full_text_query: async_query = async_query.nearest_to_text( - query.full_text_query.query, query.full_text_query.columns + query.full_text_query.query, + query.full_text_query.columns, + operator=query.full_text_query.operator, ) return async_query diff --git a/python/python/tests/test_fts.py b/python/python/tests/test_fts.py index f2fcc970b..26ce5e528 100644 --- a/python/python/tests/test_fts.py +++ b/python/python/tests/test_fts.py @@ -29,6 +29,7 @@ from lancedb.query import ( CombinedFieldsQuery, DocumentGranularity, FullTextOperator, + FullTextSearchQuery, MatchQuery, MultiMatchQuery, PhraseQuery, @@ -530,11 +531,9 @@ async def test_fts_select_async(async_table): def test_search_fts_phrase_query(table): table.create_fts_index("text", with_position=False) - try: - phrase_results = table.search('"puppy runs"').limit(100).to_list() - assert False - except Exception: - pass + with pytest.raises(Exception) as error: + table.search('"puppy runs"').limit(100).to_list() + assert "FTS(with_position=True)" in exception_output(error) table.create_fts_index("text", with_position=True, replace=True) results = table.search("puppy").limit(100).to_list() @@ -559,13 +558,9 @@ def test_search_fts_phrase_query(table): @pytest.mark.asyncio async def test_search_fts_phrase_query_async(async_table): await async_table.create_index("text", config=FTS(with_position=False)) - try: - phrase_results = ( - await async_table.query().nearest_to_text("puppy runs").limit(100).to_list() - ) - assert False - except Exception: - pass + with pytest.raises(Exception) as error: + await async_table.query().nearest_to_text('"puppy runs"').to_list() + assert "FTS(with_position=True)" in exception_output(error) await async_table.create_index("text", config=FTS(with_position=True)) results = await async_table.query().nearest_to_text("puppy").limit(100).to_list() phrase_results = ( @@ -1948,3 +1943,168 @@ async def test_async_fts_code_analyzer(mem_db_async): rows = await table.query().nearest_to_text("http").to_list() assert len(rows) == 1 + + +@pytest.mark.parametrize("indexed", [False, True]) +@pytest.mark.parametrize("operator,expected", [(None, 25), ("OR", 25), ("AND", 15)]) +def test_simple_fts_operator_and_count(tmp_path, indexed, operator, expected): + db = ldb.connect(tmp_path) + table = db.create_table( + "matches", + pa.table( + { + "id": list(range(30)), + "text": ["gripper closed"] * 15 + + ["gripper open"] * 10 + + ["camera ready"] * 5, + } + ), + ) + if indexed: + table.create_index("text", config=FTS()) + kwargs = {"fts_columns": "text"} + if operator is not None: + kwargs["operator"] = operator + query = table.search("gripper closed", query_type="fts", **kwargs) + assert query.count_rows() == expected + assert len(query.limit(100).to_list()) == expected + assert ( + table.search("gripper closed", query_type="fts", fts_columns="text") + .operator("AND") + .count_rows() + == 15 + ) + if operator is not None: + assert query.to_query_object().full_text_query.operator == operator + query.where("id >= 5", prefilter=False).select(["id"]).limit(2).offset(1) + before = query.to_query_object() + assert query.count_rows() == expected - 5 + assert query.to_query_object() == before + assert len(query.to_list()) <= 2 + assert ( + table.search("absent", query_type="fts", fts_columns="text").count_rows() == 0 + ) + assert ( + table.search(MatchQuery("gripper closed", "text", operator="AND")).count_rows() + == 15 + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("indexed", [False, True]) +@pytest.mark.parametrize("operator,expected", [(None, 25), ("OR", 25), ("AND", 15)]) +async def test_simple_fts_operator_and_count_async( + tmp_path, indexed, operator, expected +): + db = await ldb.connect_async(tmp_path) + table = await db.create_table( + "matches", + pa.table( + { + "id": list(range(30)), + "text": ["gripper closed"] * 15 + + ["gripper open"] * 10 + + ["camera ready"] * 5, + } + ), + ) + if indexed: + await table.create_index("text", config=FTS()) + query = await table.search( + "gripper closed", query_type="fts", fts_columns="text", operator=operator + ) + assert await query.count_rows() == expected + assert len(await query.limit(100).to_list()) == expected + query.where("id >= 5").postfilter().select(["id"]).limit(2).offset(1) + before = query.to_query_object() + assert await query.count_rows() == expected - 5 + assert query.to_query_object() == before + direct = table.query().nearest_to_text( + "gripper closed", columns="text", operator="AND" + ) + assert await direct.count_rows() == 15 + assert direct.to_query_object().full_text_query.query.operator == "AND" + assert ( + await table.query().nearest_to_text("absent", columns="text").count_rows() == 0 + ) + + +@pytest.mark.parametrize("with_position", [False, True]) +@pytest.mark.parametrize( + "query", ['"gripper closed"', PhraseQuery("gripper closed", "text")] +) +def test_fts_phrase_count_positions(tmp_path, with_position, query): + table = ldb.connect(tmp_path).create_table( + "phrases", pa.table({"text": ["gripper closed", "gripper often closed"]}) + ) + table.create_index("text", config=FTS(with_position=with_position)) + search = table.search(query, query_type="fts") + if with_position: + assert search.count_rows() == 1 + assert len(search.to_list()) == 1 + assert ( + table.search("gripper closed", query_type="fts", operator="AND") + .phrase_query() + .count_rows() + == 1 + ) + else: + with pytest.raises(Exception) as error: + search.count_rows() + assert "FTS(with_position=True)" in exception_output(error) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("with_position", [False, True]) +@pytest.mark.parametrize( + "query", ['"gripper closed"', PhraseQuery("gripper closed", "text")] +) +async def test_fts_phrase_count_positions_async(tmp_path, with_position, query): + db = await ldb.connect_async(tmp_path) + table = await db.create_table( + "phrases", pa.table({"text": ["gripper closed", "gripper often closed"]}) + ) + await table.create_index("text", config=FTS(with_position=with_position)) + search = table.query().nearest_to_text(query) + if with_position: + assert await search.count_rows() == 1 + assert len(await search.to_list()) == 1 + else: + with pytest.raises(Exception) as error: + await search.count_rows() + assert "FTS(with_position=True)" in exception_output(error) + + +def test_fts_operator_rejects_invalid_queries(table): + with pytest.raises(ValueError): + table.search("gripper", query_type="fts", operator="XOR") + for query in [None, [1.0, 2.0], MatchQuery("gripper", "text")]: + with pytest.raises( + ValueError, match="operator requires a string full-text query" + ): + table.search(query, operator="AND") + with pytest.raises(ValueError, match="operator requires a string full-text query"): + FullTextSearchQuery(query=MatchQuery("gripper"), operator="AND") + + +@pytest.mark.asyncio +async def test_fts_operator_rejects_invalid_queries_async(async_table): + with pytest.raises(ValueError): + await async_table.search("gripper", query_type="fts", operator="XOR") + for query in [None, [1.0, 2.0], MatchQuery("gripper", "text")]: + with pytest.raises( + ValueError, match="operator requires a string full-text query" + ): + await async_table.search(query, operator="AND") + with pytest.raises(ValueError): + async_table.query().nearest_to_text(query, operator="AND") + + +def test_fts_count_fast_search(tmp_path): + table = ldb.connect(tmp_path).create_table( + "matches", pa.table({"text": ["gripper closed"] * 15}) + ) + table.create_index("text", config=FTS()) + table.add(pa.table({"text": ["gripper closed"] * 5})) + assert table.search("gripper", query_type="fts").count_rows() == 20 + assert table.search("gripper", query_type="fts").fast_search().count_rows() == 15 diff --git a/python/python/tests/test_namespace.py b/python/python/tests/test_namespace.py index 9bce1059d..b4ca746b0 100644 --- a/python/python/tests/test_namespace.py +++ b/python/python/tests/test_namespace.py @@ -18,6 +18,7 @@ import lancedb from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError from utils import start_server_thread from lancedb.listing import AsyncListing, Listing +from lancedb.index import FTS from lancedb.materialized_view import DEFINITION_FORMAT, DEFINITION_META_KEY from lancedb.namespace import ( _MAX_QUERY_K, @@ -25,7 +26,7 @@ from lancedb.namespace import ( LanceNamespaceDBConnection, _query_to_namespace_request, ) -from lancedb.query import Query +from lancedb.query import FullTextSearchQuery, Query from lancedb.table import AsyncTable, LanceTable @@ -1524,3 +1525,42 @@ def test_python_namespace_attribute_access(tmp_path): # Missing dunder lookups still raise, so copy/hasattr probes work. assert not hasattr(request, "__missing_dunder__") assert copy.deepcopy(request) == request + + +@pytest.mark.parametrize("columns", [None, ["text"], ["title", "body"]]) +def test_namespace_fts_operator_is_preserved(columns): + request = _query_to_namespace_request( + ["matches"], + Query( + full_text_query=FullTextSearchQuery( + query="gripper closed", columns=columns, operator="AND" + ) + ), + ) + structured = request.to_dict()["full_text_query"]["structured_query"]["query"] + if columns and len(columns) > 1: + assert all( + query["operator"] == "And" + for query in structured["multi_match"]["match_queries"] + ) + else: + assert structured["match"]["operator"] == "And" + assert structured["match"].get("column") == (columns[0] if columns else None) + + +def test_namespace_fts_count_fast_search(tmp_path): + properties = {"root": str(tmp_path)} + db = lancedb.connect_namespace("dir", properties) + table = db.create_table("matches", pa.table({"text": ["gripper"] * 15})) + table.create_index("text", config=FTS()) + table.add(pa.table({"text": ["gripper"] * 5})) + assert table.search("gripper", query_type="fts").fast_search().count_rows() == 15 + pushed = lancedb.connect_namespace( + "dir", properties, namespace_client_pushdown_operations=["QueryTable"] + ).open_table("matches") + assert pushed.search("gripper", query_type="fts").count_rows() == 20 + query = pushed.search("gripper", query_type="fts").fast_search() + assert query.use_rte(False).count_rows() == 15 + query = pushed.search("gripper", query_type="fts").fast_search() + assert query.count_rows() == 15 + assert len(query.limit(100).to_list()) == 15 diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index f404e36f7..587e2d587 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -7802,3 +7802,46 @@ async def test_remote_index_exist_ok_wire(is_async, background, kwargs): assert body["replace"] is False else: assert "replace" not in body + + +@pytest.mark.parametrize("operator", ["AND", "OR"]) +def test_remote_fts_operator_and_count(operator): + def handler(body): + query = body["full_text_query"]["query"]["match"] + assert query["terms"] == "gripper closed" + assert query["operator"] == operator.title() + assert query["column"] == "text" + assert body["k"] == 2**63 - 1 + assert "offset" not in body + assert body["columns"] == ["_score"] + assert body["filter"] == "text IS NOT NULL" + assert body["prefilter"] is False + return pa.table({"_score": [1.0] * 25}) + + with query_test_table( + handler, server_version=Version("0.6.0"), vector_columns={} + ) as table: + query = ( + table.search( + "gripper closed", + query_type="fts", + fts_columns="text", + operator=operator, + ) + .where("text IS NOT NULL", prefilter=False) + .select(["text"]) + .limit(1) + .offset(3) + ) + assert query.count_rows() == 25 + assert query.to_query_object().limit == 1 + assert query.to_query_object().offset == 3 + + +def test_remote_fts_and_rejects_legacy_server(): + def handler(body): + pytest.fail("AND must not silently become OR on a legacy server") + + with query_test_table(handler, vector_columns={}) as table: + with pytest.raises(NotImplementedError, match="server version 0.3.0"): + table.search("gripper closed", query_type="fts", operator="AND").to_list() diff --git a/python/src/query.rs b/python/src/query.rs index 2097dac69..9864d7d74 100644 --- a/python/src/query.rs +++ b/python/src/query.rs @@ -16,19 +16,20 @@ use arrow::array::make_array; use arrow::pyarrow::FromPyArrow; use arrow::pyarrow::IntoPyArrow; use arrow::pyarrow::ToPyArrow; +use futures::TryStreamExt; use lancedb::ApproxMode; use lancedb::index::scalar::{ BooleanQuery, BoostQuery, CombinedFieldsQuery, DocumentGranularity, FtsQuery, FullTextSearchQuery, MatchQuery, MultiMatchQuery, Occur, Operator, PhraseQuery, }; use lancedb::query::AnalyzePlanDistributedMetrics; -use lancedb::query::QueryBase; use lancedb::query::QueryExecutionOptions; use lancedb::query::QueryFilter; use lancedb::query::{ ColumnOrdering, ExecutableQuery, Query as LanceDbQuery, Select, TakeQuery as LanceDbTakeQuery, VectorQuery as LanceDbVectorQuery, }; +use lancedb::query::{HasQuery, QueryBase}; use lancedb::table::AnyQuery; use pyo3::Bound; use pyo3::IntoPyObject; @@ -616,8 +617,18 @@ impl Query { query_text = query_text[1..query_text.len() - 1].to_string(); } + let operator = query + .get_item("operator")? + .filter(|operator| !operator.is_none()) + .map(|operator| { + let operator = operator.extract::()?; + Operator::try_from(operator.as_str()) + .map_err(|err| PyValueError::new_err(err.to_string())) + }) + .transpose()? + .unwrap_or(Operator::Or); let query: FtsQuery = match (is_phrase, is_multi_match) { - (false, _) => MatchQuery::new(query_text).into(), + (false, _) => MatchQuery::new(query_text).with_operator(operator).into(), (true, false) => PhraseQuery::new(query_text).into(), (true, true) => { return Err(PyValueError::new_err( @@ -898,6 +909,28 @@ impl FTSQuery { }) } + pub fn count_rows(self_: PyRef<'_, Self>) -> PyResult> { + let mut inner = self_ + .inner + .clone() + .full_text_search(self_.fts_query.clone()); + let request = inner.mut_query(); + request.limit = None; + request.offset = None; + request.select = Select::columns(&["_score"]); + request.order_by = None; + request.with_row_id = false; + request.reranker = None; + future_into_py(self_.py(), async move { + let mut stream = inner.execute().await.infer_error()?; + let mut count = 0_u64; + while let Some(batch) = stream.try_next().await.infer_error()? { + count += batch.num_rows() as u64; + } + Ok(count) + }) + } + pub fn nearest_to(&mut self, vector: Bound<'_, PyAny>) -> PyResult { let vector_query = Query::new(self.inner.clone()).nearest_to(vector)?; Ok(HybridQuery { diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 2224203cf..1394f1ab0 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -20,7 +20,7 @@ use crate::error::{HttpErrorSource, RemoteErrorMetadata}; use crate::expr::expr_to_sql_string; use crate::index::Index; use crate::index::IndexStatistics; -use crate::index::scalar::FtsQuery; +use crate::index::scalar::{FtsQuery, Operator}; use crate::index::waiter::{MAX_WAIT, wait_for_index}; use crate::job::Job; use crate::materialized_view::{MaterializedViewInfo, RefreshMaterializedViewResult}; @@ -1374,6 +1374,20 @@ impl RemoteTable { "query": full_text_search.query.clone(), }); } else { + let has_and = match &full_text_search.query { + FtsQuery::Match(query) => query.operator == Operator::And, + FtsQuery::MultiMatch(query) => query + .match_queries + .iter() + .any(|query| query.operator == Operator::And), + _ => false, + }; + if has_and { + return Err(Error::NotSupported { + message: "AND full-text queries require server version 0.3.0 or later" + .into(), + }); + } body["full_text_query"] = serde_json::json!({ "columns": full_text_search.columns().into_iter().collect::>(), "query": full_text_search.query.query(), diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index e9bcd380d..7d98dc001 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -12,6 +12,7 @@ use super::NativeTable; use crate::connection::NamespaceClientPushdownOperation; use crate::error::{Error, Result}; use crate::expr::expr_to_sql_string; +use crate::index::scalar::{FtsQuery, Operator as FullTextOperator}; use crate::query::{ DEFAULT_TOP_K, QueryExecutionOptions, QueryFilter, QueryRequest, Select, VectorQueryRequest, }; @@ -36,12 +37,13 @@ use lance::io::exec::{ANNIvfBatchExec, ANNIvfSubIndexExec}; use lance_datafusion::exec::{analyze_plan as lance_analyze_plan, execute_plan}; use lance_index::IndexType; use lance_index::metrics::NoOpMetricsCollector; +use lance_index::scalar::FullTextSearchQuery; use lance_index::vector::DIST_COL; use lance_linalg::distance::DistanceType as LanceDistanceType; use lance_namespace::LanceNamespace; use lance_namespace::models::{ QueryTableRequest as NsQueryTableRequest, QueryTableRequestColumns, - QueryTableRequestFullTextQuery, QueryTableRequestVector, StringFtsQuery, + QueryTableRequestFullTextQuery, QueryTableRequestVector, StringFtsQuery, StructuredFtsQuery, }; #[derive(Debug, Clone)] @@ -124,6 +126,10 @@ fn requires_local_namespace_execution(query: &AnyQuery) -> bool { if query.base().use_rte.is_some() || query.base().has_take() { return true; } + if matches!(query, AnyQuery::Query(query) if query.fast_search && query.full_text_search.is_some()) + { + return true; + } matches!( query, AnyQuery::VectorQuery(vector_query) @@ -858,6 +864,33 @@ async fn execute_namespace_query( parse_arrow_ipc_response(response_bytes).await } +fn namespace_fts_query(fts: &FullTextSearchQuery) -> Result> { + if !matches!(&fts.query, FtsQuery::Match(query) if query.operator == FullTextOperator::Or) { + let query = serde_json::to_value(&fts.query) + .and_then(serde_json::from_value) + .map_err(|error| Error::InvalidInput { + message: format!("Cannot serialize namespace full-text query: {error}"), + })?; + Ok(Box::new(QueryTableRequestFullTextQuery { + string_query: None, + structured_query: Some(Box::new(StructuredFtsQuery::new(query))), + })) + } else { + let columns = fts.columns(); + Ok(Box::new(QueryTableRequestFullTextQuery { + string_query: Some(Box::new(StringFtsQuery { + query: fts.query.query().clone(), + columns: if columns.is_empty() { + None + } else { + Some(columns.into_iter().collect()) + }, + })), + structured_query: None, + })) + } +} + /// Convert an AnyQuery to the namespace QueryTableRequest format. fn convert_to_namespace_query(query: &AnyQuery) -> Result { query.base().check_filter()?; @@ -912,21 +945,12 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result { } // Convert FTS query if present - let full_text_query = vq.base.full_text_search.as_ref().map(|fts| { - let columns = fts.columns(); - let columns_vec = if columns.is_empty() { - None - } else { - Some(columns.into_iter().collect()) - }; - Box::new(QueryTableRequestFullTextQuery { - string_query: Some(Box::new(StringFtsQuery { - query: fts.query.to_string(), - columns: columns_vec, - })), - structured_query: None, - }) - }); + let full_text_query = vq + .base + .full_text_search + .as_ref() + .map(namespace_fts_query) + .transpose()?; Ok(NsQueryTableRequest { id: None, // Will be set in namespace_query @@ -963,7 +987,6 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result { let k = match q.limit { Some(limit) => limit as i32, - None if q.full_text_search.is_some() => DEFAULT_TOP_K as i32, None => i32::MAX, }; @@ -995,20 +1018,11 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result { }; // Handle full text search if present - let full_text_query = q.full_text_search.as_ref().map(|fts| { - let columns_vec = if fts.columns().is_empty() { - None - } else { - Some(fts.columns().iter().cloned().collect()) - }; - Box::new(QueryTableRequestFullTextQuery { - string_query: Some(Box::new(StringFtsQuery { - query: fts.query.to_string(), - columns: columns_vec, - })), - structured_query: None, - }) - }); + let full_text_query = q + .full_text_search + .as_ref() + .map(namespace_fts_query) + .transpose()?; // Empty vector for non-vector queries let vector = Box::new(QueryTableRequestVector { @@ -1023,6 +1037,7 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result { filter, columns, prefilter: Some(q.prefilter), + fast_search: Some(q.fast_search), offset: q.offset.map(|o| o as i32), vector_column: None, // No vector column for plain queries with_row_id: Some(q.with_row_id), @@ -1147,6 +1162,7 @@ async fn parse_arrow_ipc_response(bytes: bytes::Bytes) -> Result