diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 0cce2085e..63d442703 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -943,6 +943,7 @@ class FTSQuery: def add_query_vector(self, query_vec: pa.Array) -> None: ... def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ... def order_by(self, ordering: Optional[List[ColumnOrdering]]): ... + async def count_rows(self) -> int: ... async def output_schema(self) -> pa.Schema: ... async def execute( self, max_batch_length: Optional[int], timeout: Optional[timedelta] diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index 67fcc61f7..463536619 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -10,6 +10,7 @@ through a namespace abstraction. from __future__ import annotations +import json import sys import warnings from typing import Any, Dict, Iterable, List, Optional, Union @@ -39,6 +40,9 @@ from lance_namespace_urllib3_client.models.query_table_request_vector import ( QueryTableRequestVector, ) from lance_namespace_urllib3_client.models.string_fts_query import StringFtsQuery +from lance_namespace_urllib3_client.models.structured_fts_query import ( + StructuredFtsQuery, +) from lance_namespace.errors import ( NamespaceNotEmptyError, NamespaceNotFoundError, @@ -78,6 +82,7 @@ from lancedb.materialized_view import ( SelectArg, ) from lancedb.table import AsyncTable, LanceTable, Table +from lancedb.query import MatchQuery, MultiMatchQuery, PhraseQuery from lancedb.util import validate_table_name from lancedb.common import DATA from lancedb.pydantic import LanceModel @@ -178,6 +183,26 @@ def _query_to_namespace_request( full_text_query_request = QueryTableRequestFullTextQuery( string_query=StringFtsQuery(query=fts_query) ) + elif getattr(fts_query, "operator", None) is not None: + columns = fts_query.columns or [] + text = fts_query.query + if len(text) >= 2 and text.startswith('"') and text.endswith('"'): + if len(columns) > 1: + raise ValueError( + "Phrase queries cannot be used with multiple columns." + ) + structured = PhraseQuery(text[1:-1], columns[0] if columns else None) + elif len(columns) > 1: + structured = MultiMatchQuery(text, columns, operator=fts_query.operator) + else: + structured = MatchQuery( + text, columns[0] if columns else None, operator=fts_query.operator + ) + full_text_query_request = QueryTableRequestFullTextQuery( + structured_query=StructuredFtsQuery.from_dict( + {"query": json.loads(structured.to_json())} + ) + ) elif hasattr(fts_query, "query"): full_text_query_request = QueryTableRequestFullTextQuery( string_query=StringFtsQuery( diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index f99aaeb73..214d2255f 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -42,7 +42,7 @@ from .rerankers.base import Reranker from .rerankers.rrf import RRFReranker from .rerankers.util import check_reranker_result from .schema import blob_column_paths, is_blob_like_field, schema_has_blob_field -from .util import _validate_query_vector, flatten_columns, get_uri_scheme +from .util import _validate_query_vector, add_note, flatten_columns, get_uri_scheme from . import _wal_hybrid # WAL-PK-FUSION: delete. from ._blob import ( BLOB_MODE_TO_HANDLING, @@ -873,10 +873,29 @@ class FullTextSearchQuery(pydantic.BaseModel): query: str | FullTextQuery If a string, it is treated as a MatchQuery. If a FullTextQuery object, it is used directly. + operator: FullTextOperator, optional + Combine terms in a string query using AND or OR (the default). + Quoted strings keep phrase semantics regardless of the operator. """ columns: Optional[List[str]] = None query: Union[str, FullTextQuery] + operator: Optional[FullTextOperator] = None + + @pydantic.model_validator(mode="after") + def _validate_operator(self) -> Self: + if self.operator is not None and not isinstance(self.query, str): + raise ValueError("operator requires a string full-text query") + return self + + +def _add_fts_position_error_note(error: Exception) -> None: + if "position is not found but required for phrase queries" in str(error): + add_note( + error, + "Phrase queries require an index built with positions. Rebuild it with " + "table.create_index(column, config=FTS(with_position=True), replace=True).", + ) class Query(pydantic.BaseModel): @@ -1147,6 +1166,7 @@ class LanceQueryBuilder(ABC): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: bool = None, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> Self: """ Create a query builder based on the given query and query type. @@ -1171,6 +1191,10 @@ class LanceQueryBuilder(ABC): fast_search: bool Skip flat search of unindexed data. """ + if operator is not None: + operator = FullTextOperator(operator) + if not isinstance(query, str) or query_type not in ("auto", "fts"): + raise ValueError("operator requires a string full-text query") if query_type != "fts": _validate_query_vector(query) @@ -1201,6 +1225,9 @@ class LanceQueryBuilder(ABC): table, query, query_type, vector_column_name ) + if operator is not None and query_type != "fts": + raise ValueError("operator requires a string full-text query") + if query_type == "hybrid": return LanceHybridQueryBuilder( table, query, vector_column_name, fts_columns=fts_columns @@ -1214,7 +1241,7 @@ class LanceQueryBuilder(ABC): ordering_field_name=ordering_field_name, fts_columns=fts_columns, fast_search=fast_search, - ) + ).operator(operator) if isinstance(query, (list, tuple)): query = np.array(query, dtype=np.float32) @@ -2563,6 +2590,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder): super().__init__(table) self._query = query self._phrase_query = False + self._operator: Optional[FullTextOperator] = None # Deprecated compatibility parameter. Native FTS ordering is now # configured through order_by(); LanceQueryBuilder.create emits the warning. _ = ordering_field_name @@ -2572,6 +2600,36 @@ class LanceFtsQueryBuilder(LanceQueryBuilder): fts_columns = [fts_columns] self._fts_columns = fts_columns + def operator( + self, operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] + ) -> LanceFtsQueryBuilder: + """Combine terms in a string query with ``"AND"`` or ``"OR"``. + + The default is OR. AND requires every term to match. For example, + ``table.search("gripper closed", query_type="fts").operator("AND")``. + Structured queries specify their own operators. Quoted strings and + ``phrase_query()`` retain their phrase semantics. + """ + if operator is not None and not isinstance(self._query, str): + raise ValueError("operator requires a string full-text query") + self._operator = FullTextOperator(operator) if operator is not None else None + return self + + def count_rows(self) -> int: + """Count all full-text matches satisfying the query's filter. + + Ignores limit, offset, selection, ordering, and reranking. ``fast_search`` + still restricts the count to indexed data. This streams matching scores + without loading document columns and may scan every match. + + For example, ``table.search("gripper closed", query_type="fts",`` + ``operator="AND").where("active = true").count_rows()``. + """ + self._table._ensure_no_legacy_fts_index() + query = self._table._table._sync_query_to_async(self.to_query_object()) + assert isinstance(query, AsyncFTSQuery) + return LOOP.run(query.count_rows()) + def phrase_query(self, phrase_query: bool = True) -> LanceFtsQueryBuilder: """Set whether to use phrase query. @@ -2628,7 +2686,9 @@ class LanceFtsQueryBuilder(LanceQueryBuilder): fragment_ids=self._fragment_ids, use_rte=self._use_rte, full_text_query=FullTextSearchQuery( - query=self._query_with_phrase_semantics(), columns=self._fts_columns + query=self._query_with_phrase_semantics(), + columns=self._fts_columns, + operator=self._operator, ), offset=self._offset, fast_search=self._fast_search, @@ -4443,7 +4503,11 @@ class AsyncQuery(AsyncStandardQuery): ) def nearest_to_text( - self, query: str | FullTextQuery, columns: Union[str, List[str], None] = None + self, + query: str | FullTextQuery, + columns: Union[str, List[str], None] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncFTSQuery: """ Find the documents that are most relevant to the given text query. @@ -4464,7 +4528,11 @@ class AsyncQuery(AsyncStandardQuery): columns: str or list of str, default None The columns to search in. If None, all indexed columns are searched. For now only one column can be searched at a time. + operator: "AND" or "OR", optional + Combine terms in a string query. Defaults to OR. Quoted strings use + phrase semantics and require an index built with positions. """ + fts_query = FullTextSearchQuery(query=query, operator=operator) if isinstance(columns, str): columns = [columns] if columns is None: @@ -4472,7 +4540,13 @@ class AsyncQuery(AsyncStandardQuery): if isinstance(query, str): return AsyncFTSQuery( - self._inner.nearest_to_text({"query": query, "columns": columns}), + self._inner.nearest_to_text( + { + "query": query, + "columns": columns, + "operator": fts_query.operator, + } + ), self._table, ) # FullTextQuery object @@ -4487,6 +4561,22 @@ class AsyncFTSQuery(AsyncStandardQuery): self._inner = inner self._reranker = None + async def count_rows(self) -> int: + """Count all full-text matches satisfying the query's filter. + + Ignores limit, offset, selection, ordering, and reranking. ``fast_search`` + still restricts the count to indexed data. Matching scores are streamed + without loading document columns, so counting may scan every match. + + For example, ``await table.query().nearest_to_text("gripper closed",`` + ``operator="AND").where("active = true").count_rows()``. + """ + try: + return await self._inner.count_rows() + except Exception as error: + _add_fts_position_error_note(error) + raise + def get_query(self) -> str: return self._inner.get_query() @@ -4580,10 +4670,16 @@ class AsyncFTSQuery(AsyncStandardQuery): timeout: Optional[timedelta] = None, blob_sources: Optional[dict[str, str]] = None, ) -> AsyncRecordBatchReader: - reader = await super()._execute_batches( - timeout=timeout, blob_sources=blob_sources - ) - results = pa.Table.from_batches(await reader.read_all(), reader.schema) + try: + reader = await super()._execute_batches( + max_batch_length=max_batch_length, + timeout=timeout, + blob_sources=blob_sources, + ) + results = pa.Table.from_batches(await reader.read_all(), reader.schema) + except Exception as error: + _add_fts_position_error_note(error) + raise if self._reranker: source = results results = self._reranker.rerank_fts(self.get_query(), results) @@ -4861,7 +4957,11 @@ class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase): return self def nearest_to_text( - self, query: str | FullTextQuery, columns: Union[str, List[str], None] = None + self, + query: str | FullTextQuery, + columns: Union[str, List[str], None] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncHybridQuery: """ Find the documents that are most relevant to the given text query, @@ -4886,7 +4986,10 @@ class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase): columns: str or list of str, default None The columns to search in. If None, all indexed columns are searched. For now only one column can be searched at a time. + operator: "AND" or "OR", optional + Combine terms in a string query. Defaults to OR. """ + fts_query = FullTextSearchQuery(query=query, operator=operator) if isinstance(columns, str): columns = [columns] if columns is None: @@ -4894,7 +4997,9 @@ class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase): if isinstance(query, str): return AsyncHybridQuery( - self._inner.nearest_to_text({"query": query, "columns": columns}), + self._inner.nearest_to_text( + {"query": query, "columns": columns, "operator": fts_query.operator} + ), self._table, ) # FullTextQuery object diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 818d2a257..2d068536e 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -65,6 +65,7 @@ from lancedb.embeddings import EmbeddingFunctionRegistry from lancedb.table import _normalize_progress from ..query import ( + FullTextOperator, AnalyzePlanDistributedMetrics, DocumentGranularity, FullTextQuery, @@ -799,6 +800,8 @@ class RemoteTable(Table): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceQueryBuilder: """Create a search query to find the nearest neighbors of the given query vector. We currently support @@ -857,6 +860,10 @@ class RemoteTable(Table): - *default None*, which leaves fast search disabled. + operator: "AND" or "OR", optional + Combine terms in a string full-text query; the default is OR. + Use ``query_type="fts"`` when the table also has embeddings. + Returns ------- LanceQueryBuilder @@ -893,6 +900,7 @@ class RemoteTable(Table): ordering_field_name=ordering_field_name, fts_columns=fts_columns or [], fast_search=fast_search, + operator=operator, ) def _execute_query( diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 4a713da06..807edbc46 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -91,6 +91,7 @@ from .functions import ( from .merge import LanceMergeInsertBuilder from .pydantic import LanceModel, model_to_dict from .query import ( + FullTextOperator, AnalyzePlanDistributedMetrics, AsyncFTSQuery, AsyncHybridQuery, @@ -138,6 +139,11 @@ def _requires_local_namespace_execution(query: Query) -> bool: or query.take_row_ids is not None or query.search_effort is not None or query.approx_mode is not None + or ( + query.full_text_query is not None + and query.fast_search is True + and query.vector is None + ) ) @@ -2443,6 +2449,8 @@ class Table(ABC): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceQueryBuilder: """Create a search query to find the nearest neighbors of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/) @@ -2506,6 +2514,10 @@ class Table(ABC): - If `query` is a string, then the query type is "vector" if the table has embedding functions else the query type is "fts" + operator: "AND" or "OR", optional + Combine terms in a string full-text query; the default is OR. + Use ``query_type="fts"`` when the table also has embeddings. + Returns ------- LanceQueryBuilder @@ -5061,6 +5073,8 @@ class LanceTable(Table): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceVectorQueryBuilder: ... @overload @@ -5072,6 +5086,8 @@ class LanceTable(Table): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceFtsQueryBuilder: ... @overload @@ -5085,6 +5101,8 @@ class LanceTable(Table): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceHybridQueryBuilder: ... @overload @@ -5096,6 +5114,8 @@ class LanceTable(Table): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceEmptyQueryBuilder: ... def search( @@ -5108,6 +5128,8 @@ class LanceTable(Table): ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> LanceQueryBuilder: """Create a search query to find the nearest neighbors of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/) @@ -5168,6 +5190,10 @@ class LanceTable(Table): If None then the search is performed on all indexed columns. For now, only one column can be searched at a time. + operator: "AND" or "OR", optional + Combine terms in a string full-text query; the default is OR. + Use ``query_type="fts"`` when the table also has embeddings. + Returns ------- LanceQueryBuilder @@ -5193,6 +5219,7 @@ class LanceTable(Table): ordering_field_name=ordering_field_name, fts_columns=fts_columns or [], fast_search=fast_search, + operator=operator, ) @classmethod @@ -7376,6 +7403,8 @@ class AsyncTable: query_type: QueryType = "auto", ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncQuery: ... @overload @@ -7386,6 +7415,8 @@ class AsyncTable: query_type: Literal["auto"] = ..., ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> Union[AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]: ... @overload @@ -7396,6 +7427,8 @@ class AsyncTable: query_type: Literal["hybrid"] = ..., ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncHybridQuery: ... @overload @@ -7406,6 +7439,8 @@ class AsyncTable: query_type: Literal["auto"] = ..., ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncVectorQuery: ... @overload @@ -7416,6 +7451,8 @@ class AsyncTable: query_type: Literal["fts"] = ..., ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncFTSQuery: ... @overload @@ -7426,6 +7463,8 @@ class AsyncTable: query_type: Literal["vector"] = ..., ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> AsyncVectorQuery: ... async def search( @@ -7437,6 +7476,8 @@ class AsyncTable: query_type: QueryType = "auto", ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + *, + operator: Optional[Union[FullTextOperator, Literal["AND", "OR"]]] = None, ) -> Union[AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]: """Create a search query to find the nearest neighbors of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/) @@ -7480,12 +7521,20 @@ class AsyncTable: - If `query` is a string, then the query type is "vector" if the table has embedding functions else the query type is "fts" + operator: "AND" or "OR", optional + Combine terms in a string full-text query; the default is OR. + Use ``query_type="fts"`` when the table also has embeddings. + Returns ------- AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, or AsyncVectorQuery A query builder object representing the query. """ + if operator is not None: + operator = FullTextOperator(operator) + if not isinstance(query, str) or query_type not in ("auto", "fts"): + raise ValueError("operator requires a string full-text query") if query is None: return self.query() @@ -7604,13 +7653,17 @@ class AsyncTable: ) vector_query = await make_embedding(embedding_conf, query) + if operator is not None and query_type != "fts": + raise ValueError("operator requires a string full-text query") if query_type == "vector": builder = self.query().nearest_to(vector_query) if vector_column_name: builder = builder.column(vector_column_name) return builder elif query_type == "fts": - return self.query().nearest_to_text(query, columns=fts_columns) + return self.query().nearest_to_text( + query, columns=fts_columns, operator=operator + ) elif query_type == "hybrid": builder = self.query().nearest_to(vector_query) if vector_column_name: @@ -7720,7 +7773,9 @@ class AsyncTable: if query.full_text_query: async_query = async_query.nearest_to_text( - query.full_text_query.query, query.full_text_query.columns + query.full_text_query.query, + query.full_text_query.columns, + operator=query.full_text_query.operator, ) return async_query diff --git a/python/python/tests/test_fts.py b/python/python/tests/test_fts.py index f2fcc970b..26ce5e528 100644 --- a/python/python/tests/test_fts.py +++ b/python/python/tests/test_fts.py @@ -29,6 +29,7 @@ from lancedb.query import ( CombinedFieldsQuery, DocumentGranularity, FullTextOperator, + FullTextSearchQuery, MatchQuery, MultiMatchQuery, PhraseQuery, @@ -530,11 +531,9 @@ async def test_fts_select_async(async_table): def test_search_fts_phrase_query(table): table.create_fts_index("text", with_position=False) - try: - phrase_results = table.search('"puppy runs"').limit(100).to_list() - assert False - except Exception: - pass + with pytest.raises(Exception) as error: + table.search('"puppy runs"').limit(100).to_list() + assert "FTS(with_position=True)" in exception_output(error) table.create_fts_index("text", with_position=True, replace=True) results = table.search("puppy").limit(100).to_list() @@ -559,13 +558,9 @@ def test_search_fts_phrase_query(table): @pytest.mark.asyncio async def test_search_fts_phrase_query_async(async_table): await async_table.create_index("text", config=FTS(with_position=False)) - try: - phrase_results = ( - await async_table.query().nearest_to_text("puppy runs").limit(100).to_list() - ) - assert False - except Exception: - pass + with pytest.raises(Exception) as error: + await async_table.query().nearest_to_text('"puppy runs"').to_list() + assert "FTS(with_position=True)" in exception_output(error) await async_table.create_index("text", config=FTS(with_position=True)) results = await async_table.query().nearest_to_text("puppy").limit(100).to_list() phrase_results = ( @@ -1948,3 +1943,168 @@ async def test_async_fts_code_analyzer(mem_db_async): rows = await table.query().nearest_to_text("http").to_list() assert len(rows) == 1 + + +@pytest.mark.parametrize("indexed", [False, True]) +@pytest.mark.parametrize("operator,expected", [(None, 25), ("OR", 25), ("AND", 15)]) +def test_simple_fts_operator_and_count(tmp_path, indexed, operator, expected): + db = ldb.connect(tmp_path) + table = db.create_table( + "matches", + pa.table( + { + "id": list(range(30)), + "text": ["gripper closed"] * 15 + + ["gripper open"] * 10 + + ["camera ready"] * 5, + } + ), + ) + if indexed: + table.create_index("text", config=FTS()) + kwargs = {"fts_columns": "text"} + if operator is not None: + kwargs["operator"] = operator + query = table.search("gripper closed", query_type="fts", **kwargs) + assert query.count_rows() == expected + assert len(query.limit(100).to_list()) == expected + assert ( + table.search("gripper closed", query_type="fts", fts_columns="text") + .operator("AND") + .count_rows() + == 15 + ) + if operator is not None: + assert query.to_query_object().full_text_query.operator == operator + query.where("id >= 5", prefilter=False).select(["id"]).limit(2).offset(1) + before = query.to_query_object() + assert query.count_rows() == expected - 5 + assert query.to_query_object() == before + assert len(query.to_list()) <= 2 + assert ( + table.search("absent", query_type="fts", fts_columns="text").count_rows() == 0 + ) + assert ( + table.search(MatchQuery("gripper closed", "text", operator="AND")).count_rows() + == 15 + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("indexed", [False, True]) +@pytest.mark.parametrize("operator,expected", [(None, 25), ("OR", 25), ("AND", 15)]) +async def test_simple_fts_operator_and_count_async( + tmp_path, indexed, operator, expected +): + db = await ldb.connect_async(tmp_path) + table = await db.create_table( + "matches", + pa.table( + { + "id": list(range(30)), + "text": ["gripper closed"] * 15 + + ["gripper open"] * 10 + + ["camera ready"] * 5, + } + ), + ) + if indexed: + await table.create_index("text", config=FTS()) + query = await table.search( + "gripper closed", query_type="fts", fts_columns="text", operator=operator + ) + assert await query.count_rows() == expected + assert len(await query.limit(100).to_list()) == expected + query.where("id >= 5").postfilter().select(["id"]).limit(2).offset(1) + before = query.to_query_object() + assert await query.count_rows() == expected - 5 + assert query.to_query_object() == before + direct = table.query().nearest_to_text( + "gripper closed", columns="text", operator="AND" + ) + assert await direct.count_rows() == 15 + assert direct.to_query_object().full_text_query.query.operator == "AND" + assert ( + await table.query().nearest_to_text("absent", columns="text").count_rows() == 0 + ) + + +@pytest.mark.parametrize("with_position", [False, True]) +@pytest.mark.parametrize( + "query", ['"gripper closed"', PhraseQuery("gripper closed", "text")] +) +def test_fts_phrase_count_positions(tmp_path, with_position, query): + table = ldb.connect(tmp_path).create_table( + "phrases", pa.table({"text": ["gripper closed", "gripper often closed"]}) + ) + table.create_index("text", config=FTS(with_position=with_position)) + search = table.search(query, query_type="fts") + if with_position: + assert search.count_rows() == 1 + assert len(search.to_list()) == 1 + assert ( + table.search("gripper closed", query_type="fts", operator="AND") + .phrase_query() + .count_rows() + == 1 + ) + else: + with pytest.raises(Exception) as error: + search.count_rows() + assert "FTS(with_position=True)" in exception_output(error) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("with_position", [False, True]) +@pytest.mark.parametrize( + "query", ['"gripper closed"', PhraseQuery("gripper closed", "text")] +) +async def test_fts_phrase_count_positions_async(tmp_path, with_position, query): + db = await ldb.connect_async(tmp_path) + table = await db.create_table( + "phrases", pa.table({"text": ["gripper closed", "gripper often closed"]}) + ) + await table.create_index("text", config=FTS(with_position=with_position)) + search = table.query().nearest_to_text(query) + if with_position: + assert await search.count_rows() == 1 + assert len(await search.to_list()) == 1 + else: + with pytest.raises(Exception) as error: + await search.count_rows() + assert "FTS(with_position=True)" in exception_output(error) + + +def test_fts_operator_rejects_invalid_queries(table): + with pytest.raises(ValueError): + table.search("gripper", query_type="fts", operator="XOR") + for query in [None, [1.0, 2.0], MatchQuery("gripper", "text")]: + with pytest.raises( + ValueError, match="operator requires a string full-text query" + ): + table.search(query, operator="AND") + with pytest.raises(ValueError, match="operator requires a string full-text query"): + FullTextSearchQuery(query=MatchQuery("gripper"), operator="AND") + + +@pytest.mark.asyncio +async def test_fts_operator_rejects_invalid_queries_async(async_table): + with pytest.raises(ValueError): + await async_table.search("gripper", query_type="fts", operator="XOR") + for query in [None, [1.0, 2.0], MatchQuery("gripper", "text")]: + with pytest.raises( + ValueError, match="operator requires a string full-text query" + ): + await async_table.search(query, operator="AND") + with pytest.raises(ValueError): + async_table.query().nearest_to_text(query, operator="AND") + + +def test_fts_count_fast_search(tmp_path): + table = ldb.connect(tmp_path).create_table( + "matches", pa.table({"text": ["gripper closed"] * 15}) + ) + table.create_index("text", config=FTS()) + table.add(pa.table({"text": ["gripper closed"] * 5})) + assert table.search("gripper", query_type="fts").count_rows() == 20 + assert table.search("gripper", query_type="fts").fast_search().count_rows() == 15 diff --git a/python/python/tests/test_namespace.py b/python/python/tests/test_namespace.py index 9bce1059d..b4ca746b0 100644 --- a/python/python/tests/test_namespace.py +++ b/python/python/tests/test_namespace.py @@ -18,6 +18,7 @@ import lancedb from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError from utils import start_server_thread from lancedb.listing import AsyncListing, Listing +from lancedb.index import FTS from lancedb.materialized_view import DEFINITION_FORMAT, DEFINITION_META_KEY from lancedb.namespace import ( _MAX_QUERY_K, @@ -25,7 +26,7 @@ from lancedb.namespace import ( LanceNamespaceDBConnection, _query_to_namespace_request, ) -from lancedb.query import Query +from lancedb.query import FullTextSearchQuery, Query from lancedb.table import AsyncTable, LanceTable @@ -1524,3 +1525,42 @@ def test_python_namespace_attribute_access(tmp_path): # Missing dunder lookups still raise, so copy/hasattr probes work. assert not hasattr(request, "__missing_dunder__") assert copy.deepcopy(request) == request + + +@pytest.mark.parametrize("columns", [None, ["text"], ["title", "body"]]) +def test_namespace_fts_operator_is_preserved(columns): + request = _query_to_namespace_request( + ["matches"], + Query( + full_text_query=FullTextSearchQuery( + query="gripper closed", columns=columns, operator="AND" + ) + ), + ) + structured = request.to_dict()["full_text_query"]["structured_query"]["query"] + if columns and len(columns) > 1: + assert all( + query["operator"] == "And" + for query in structured["multi_match"]["match_queries"] + ) + else: + assert structured["match"]["operator"] == "And" + assert structured["match"].get("column") == (columns[0] if columns else None) + + +def test_namespace_fts_count_fast_search(tmp_path): + properties = {"root": str(tmp_path)} + db = lancedb.connect_namespace("dir", properties) + table = db.create_table("matches", pa.table({"text": ["gripper"] * 15})) + table.create_index("text", config=FTS()) + table.add(pa.table({"text": ["gripper"] * 5})) + assert table.search("gripper", query_type="fts").fast_search().count_rows() == 15 + pushed = lancedb.connect_namespace( + "dir", properties, namespace_client_pushdown_operations=["QueryTable"] + ).open_table("matches") + assert pushed.search("gripper", query_type="fts").count_rows() == 20 + query = pushed.search("gripper", query_type="fts").fast_search() + assert query.use_rte(False).count_rows() == 15 + query = pushed.search("gripper", query_type="fts").fast_search() + assert query.count_rows() == 15 + assert len(query.limit(100).to_list()) == 15 diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index f404e36f7..587e2d587 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -7802,3 +7802,46 @@ async def test_remote_index_exist_ok_wire(is_async, background, kwargs): assert body["replace"] is False else: assert "replace" not in body + + +@pytest.mark.parametrize("operator", ["AND", "OR"]) +def test_remote_fts_operator_and_count(operator): + def handler(body): + query = body["full_text_query"]["query"]["match"] + assert query["terms"] == "gripper closed" + assert query["operator"] == operator.title() + assert query["column"] == "text" + assert body["k"] == 2**63 - 1 + assert "offset" not in body + assert body["columns"] == ["_score"] + assert body["filter"] == "text IS NOT NULL" + assert body["prefilter"] is False + return pa.table({"_score": [1.0] * 25}) + + with query_test_table( + handler, server_version=Version("0.6.0"), vector_columns={} + ) as table: + query = ( + table.search( + "gripper closed", + query_type="fts", + fts_columns="text", + operator=operator, + ) + .where("text IS NOT NULL", prefilter=False) + .select(["text"]) + .limit(1) + .offset(3) + ) + assert query.count_rows() == 25 + assert query.to_query_object().limit == 1 + assert query.to_query_object().offset == 3 + + +def test_remote_fts_and_rejects_legacy_server(): + def handler(body): + pytest.fail("AND must not silently become OR on a legacy server") + + with query_test_table(handler, vector_columns={}) as table: + with pytest.raises(NotImplementedError, match="server version 0.3.0"): + table.search("gripper closed", query_type="fts", operator="AND").to_list() diff --git a/python/src/query.rs b/python/src/query.rs index 2097dac69..9864d7d74 100644 --- a/python/src/query.rs +++ b/python/src/query.rs @@ -16,19 +16,20 @@ use arrow::array::make_array; use arrow::pyarrow::FromPyArrow; use arrow::pyarrow::IntoPyArrow; use arrow::pyarrow::ToPyArrow; +use futures::TryStreamExt; use lancedb::ApproxMode; use lancedb::index::scalar::{ BooleanQuery, BoostQuery, CombinedFieldsQuery, DocumentGranularity, FtsQuery, FullTextSearchQuery, MatchQuery, MultiMatchQuery, Occur, Operator, PhraseQuery, }; use lancedb::query::AnalyzePlanDistributedMetrics; -use lancedb::query::QueryBase; use lancedb::query::QueryExecutionOptions; use lancedb::query::QueryFilter; use lancedb::query::{ ColumnOrdering, ExecutableQuery, Query as LanceDbQuery, Select, TakeQuery as LanceDbTakeQuery, VectorQuery as LanceDbVectorQuery, }; +use lancedb::query::{HasQuery, QueryBase}; use lancedb::table::AnyQuery; use pyo3::Bound; use pyo3::IntoPyObject; @@ -616,8 +617,18 @@ impl Query { query_text = query_text[1..query_text.len() - 1].to_string(); } + let operator = query + .get_item("operator")? + .filter(|operator| !operator.is_none()) + .map(|operator| { + let operator = operator.extract::()?; + Operator::try_from(operator.as_str()) + .map_err(|err| PyValueError::new_err(err.to_string())) + }) + .transpose()? + .unwrap_or(Operator::Or); let query: FtsQuery = match (is_phrase, is_multi_match) { - (false, _) => MatchQuery::new(query_text).into(), + (false, _) => MatchQuery::new(query_text).with_operator(operator).into(), (true, false) => PhraseQuery::new(query_text).into(), (true, true) => { return Err(PyValueError::new_err( @@ -898,6 +909,28 @@ impl FTSQuery { }) } + pub fn count_rows(self_: PyRef<'_, Self>) -> PyResult> { + let mut inner = self_ + .inner + .clone() + .full_text_search(self_.fts_query.clone()); + let request = inner.mut_query(); + request.limit = None; + request.offset = None; + request.select = Select::columns(&["_score"]); + request.order_by = None; + request.with_row_id = false; + request.reranker = None; + future_into_py(self_.py(), async move { + let mut stream = inner.execute().await.infer_error()?; + let mut count = 0_u64; + while let Some(batch) = stream.try_next().await.infer_error()? { + count += batch.num_rows() as u64; + } + Ok(count) + }) + } + pub fn nearest_to(&mut self, vector: Bound<'_, PyAny>) -> PyResult { let vector_query = Query::new(self.inner.clone()).nearest_to(vector)?; Ok(HybridQuery { diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 2224203cf..1394f1ab0 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -20,7 +20,7 @@ use crate::error::{HttpErrorSource, RemoteErrorMetadata}; use crate::expr::expr_to_sql_string; use crate::index::Index; use crate::index::IndexStatistics; -use crate::index::scalar::FtsQuery; +use crate::index::scalar::{FtsQuery, Operator}; use crate::index::waiter::{MAX_WAIT, wait_for_index}; use crate::job::Job; use crate::materialized_view::{MaterializedViewInfo, RefreshMaterializedViewResult}; @@ -1374,6 +1374,20 @@ impl RemoteTable { "query": full_text_search.query.clone(), }); } else { + let has_and = match &full_text_search.query { + FtsQuery::Match(query) => query.operator == Operator::And, + FtsQuery::MultiMatch(query) => query + .match_queries + .iter() + .any(|query| query.operator == Operator::And), + _ => false, + }; + if has_and { + return Err(Error::NotSupported { + message: "AND full-text queries require server version 0.3.0 or later" + .into(), + }); + } body["full_text_query"] = serde_json::json!({ "columns": full_text_search.columns().into_iter().collect::>(), "query": full_text_search.query.query(), diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index e9bcd380d..7d98dc001 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -12,6 +12,7 @@ use super::NativeTable; use crate::connection::NamespaceClientPushdownOperation; use crate::error::{Error, Result}; use crate::expr::expr_to_sql_string; +use crate::index::scalar::{FtsQuery, Operator as FullTextOperator}; use crate::query::{ DEFAULT_TOP_K, QueryExecutionOptions, QueryFilter, QueryRequest, Select, VectorQueryRequest, }; @@ -36,12 +37,13 @@ use lance::io::exec::{ANNIvfBatchExec, ANNIvfSubIndexExec}; use lance_datafusion::exec::{analyze_plan as lance_analyze_plan, execute_plan}; use lance_index::IndexType; use lance_index::metrics::NoOpMetricsCollector; +use lance_index::scalar::FullTextSearchQuery; use lance_index::vector::DIST_COL; use lance_linalg::distance::DistanceType as LanceDistanceType; use lance_namespace::LanceNamespace; use lance_namespace::models::{ QueryTableRequest as NsQueryTableRequest, QueryTableRequestColumns, - QueryTableRequestFullTextQuery, QueryTableRequestVector, StringFtsQuery, + QueryTableRequestFullTextQuery, QueryTableRequestVector, StringFtsQuery, StructuredFtsQuery, }; #[derive(Debug, Clone)] @@ -124,6 +126,10 @@ fn requires_local_namespace_execution(query: &AnyQuery) -> bool { if query.base().use_rte.is_some() || query.base().has_take() { return true; } + if matches!(query, AnyQuery::Query(query) if query.fast_search && query.full_text_search.is_some()) + { + return true; + } matches!( query, AnyQuery::VectorQuery(vector_query) @@ -858,6 +864,33 @@ async fn execute_namespace_query( parse_arrow_ipc_response(response_bytes).await } +fn namespace_fts_query(fts: &FullTextSearchQuery) -> Result> { + if !matches!(&fts.query, FtsQuery::Match(query) if query.operator == FullTextOperator::Or) { + let query = serde_json::to_value(&fts.query) + .and_then(serde_json::from_value) + .map_err(|error| Error::InvalidInput { + message: format!("Cannot serialize namespace full-text query: {error}"), + })?; + Ok(Box::new(QueryTableRequestFullTextQuery { + string_query: None, + structured_query: Some(Box::new(StructuredFtsQuery::new(query))), + })) + } else { + let columns = fts.columns(); + Ok(Box::new(QueryTableRequestFullTextQuery { + string_query: Some(Box::new(StringFtsQuery { + query: fts.query.query().clone(), + columns: if columns.is_empty() { + None + } else { + Some(columns.into_iter().collect()) + }, + })), + structured_query: None, + })) + } +} + /// Convert an AnyQuery to the namespace QueryTableRequest format. fn convert_to_namespace_query(query: &AnyQuery) -> Result { query.base().check_filter()?; @@ -912,21 +945,12 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result { } // Convert FTS query if present - let full_text_query = vq.base.full_text_search.as_ref().map(|fts| { - let columns = fts.columns(); - let columns_vec = if columns.is_empty() { - None - } else { - Some(columns.into_iter().collect()) - }; - Box::new(QueryTableRequestFullTextQuery { - string_query: Some(Box::new(StringFtsQuery { - query: fts.query.to_string(), - columns: columns_vec, - })), - structured_query: None, - }) - }); + let full_text_query = vq + .base + .full_text_search + .as_ref() + .map(namespace_fts_query) + .transpose()?; Ok(NsQueryTableRequest { id: None, // Will be set in namespace_query @@ -963,7 +987,6 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result { let k = match q.limit { Some(limit) => limit as i32, - None if q.full_text_search.is_some() => DEFAULT_TOP_K as i32, None => i32::MAX, }; @@ -995,20 +1018,11 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result { }; // Handle full text search if present - let full_text_query = q.full_text_search.as_ref().map(|fts| { - let columns_vec = if fts.columns().is_empty() { - None - } else { - Some(fts.columns().iter().cloned().collect()) - }; - Box::new(QueryTableRequestFullTextQuery { - string_query: Some(Box::new(StringFtsQuery { - query: fts.query.to_string(), - columns: columns_vec, - })), - structured_query: None, - }) - }); + let full_text_query = q + .full_text_search + .as_ref() + .map(namespace_fts_query) + .transpose()?; // Empty vector for non-vector queries let vector = Box::new(QueryTableRequestVector { @@ -1023,6 +1037,7 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result { filter, columns, prefilter: Some(q.prefilter), + fast_search: Some(q.fast_search), offset: q.offset.map(|o| o as i32), vector_column: None, // No vector column for plain queries with_row_id: Some(q.with_row_id), @@ -1147,6 +1162,7 @@ async fn parse_arrow_ipc_response(bytes: bytes::Bytes) -> Result