rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-14 14:46:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(fts): complete custom stop-word snapshot support

Browse Source
This commit is contained in:
Yang Cen
2026-07-29 01:22:26 +08:00
parent 659c64b523
commit 34d5c1175a
50 changed files with 5461 additions and 1095 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python/lancedb/__init__.py
+30 -15
View File
@@ -19,10 +19,16 @@ from .db import AsyncConnection, DBConnection, LanceDBConnection
from .remote import ClientConfig
from .remote.db import RemoteDBConnection
from .expr import Expr, col, lit, func
from .index import (
CustomStopWords,
FtsStopWordsFile as FtsStopWordsFile,
FtsStopWordsTable as FtsStopWordsTable,
)
from .schema import blob, vector, BlobType
from .table import AsyncTable, Table
from .types import BaseTokenizerType
from ._lancedb import Session
from .background_loop import LOOP
from .namespace import (
connect_namespace,
connect_namespace_async,
@@ -258,6 +264,7 @@ def tokenize(
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
custom_stop_words: CustomStopWords = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
ngram_max_length: int = 3,
@@ -265,22 +272,30 @@ def tokenize(
) -> Iterable[FtsToken]:
"""Tokenize a full-text search query using an explicit tokenizer.
This does not require a table or FTS index. The tokenizer options match
:class:`lancedb.index.FTS`.
This does not require an FTS index. The tokenizer options match
:class:`lancedb.index.FTS`. ``custom_stop_words`` accepts an inline
sequence, :class:`lancedb.index.FtsStopWordsFile`, or
:class:`lancedb.index.FtsStopWordsTable`. Table sources require an open
local/native table handle; remote table sources are rejected.
"""
return _tokenize(
query,
base_tokenizer=base_tokenizer,
language=language,
max_token_length=max_token_length,
lower_case=lower_case,
stem=stem,
remove_stop_words=remove_stop_words,
ascii_folding=ascii_folding,
ngram_min_length=ngram_min_length,
ngram_max_length=ngram_max_length,
prefix_only=prefix_only,
)
async def tokenize_async():
return await _tokenize(
query,
base_tokenizer=base_tokenizer,
language=language,
max_token_length=max_token_length,
lower_case=lower_case,
stem=stem,
remove_stop_words=remove_stop_words,
custom_stop_words=custom_stop_words,
ascii_folding=ascii_folding,
ngram_min_length=ngram_min_length,
ngram_max_length=ngram_max_length,
prefix_only=prefix_only,
)
return LOOP.run(tokenize_async())
WORKER_PROPERTY_PREFIX = "_lancedb_worker_"
python/python/lancedb/_lancedb.pyi
+3 -1
View File
@@ -16,6 +16,7 @@ from .index import (
HnswSq,
HnswFlat,
FTS,
CustomStopWords,
)
from lance_namespace import (
ListNamespacesResponse,
@@ -50,7 +51,7 @@ class MetricDescription:
def register_lancedb_metrics_recorder() -> bool: ...
def lancedb_metrics_catalog() -> List[MetricDescription]: ...
def snapshot_lancedb_metrics() -> List[MetricPoint]: ...
def tokenize(
async def tokenize(
query: str,
*,
base_tokenizer: BaseTokenizerType = "simple",
@@ -59,6 +60,7 @@ def tokenize(
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
custom_stop_words: CustomStopWords = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
ngram_max_length: int = 3,
python/python/lancedb/index.py
+102 -46
View File
@@ -1,14 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from dataclasses import dataclass, field
from typing import List, Literal, Optional, Union
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Literal, Optional, Sequence, Union
from ._lancedb import (
IndexConfig,
)
from .types import BaseTokenizerType
if TYPE_CHECKING:
from .table import AsyncTable, Table
lang_mapping = {
"ar": "Arabic",
"da": "Danish",
@@ -32,35 +38,86 @@ lang_mapping = {
@dataclass(frozen=True)
class InlineStopWordsSource:
"""Inline custom stop words for a remote FTS create-index request."""
class FtsStopWordsFile:
"""Use newline-delimited custom FTS stop words from a UTF-8 file.
words: List[str]
type: Literal["inline"] = field(default="inline", init=False)
The file is read when the index is created (or when
:func:`lancedb.tokenize` is called). Its contents are resolved to a stable
in-memory snapshot before any remote request is sent.
Parameters
----------
path : str or os.PathLike
Non-empty path to a strictly UTF-8 encoded, newline-delimited
stop-word file.
"""
path: Union[str, os.PathLike[str]]
@property
def _path(self) -> str:
path = os.fspath(self.path)
if not isinstance(path, str):
raise TypeError("FtsStopWordsFile.path must be a string path")
if not path:
raise ValueError("FtsStopWordsFile.path must be a non-empty string path")
return path
@dataclass(frozen=True)
class FileStopWordsSource:
"""A UTF-8, newline-delimited custom stop-word object for a remote request."""
class FtsStopWordsTable:
"""Use custom FTS stop words from a local LanceDB table string column.
uri: str
type: Literal["file"] = field(default="file", init=False)
The column is read when the index is created (or when
:func:`lancedb.tokenize` is called). Its values are resolved to a stable
in-memory snapshot before any remote request is sent.
Parameters
----------
table : lancedb.Table or lancedb.AsyncTable
An open local/native LanceDB table handle. Remote table sources are
rejected because LanceDB cannot guarantee an untruncated snapshot;
materialize the column locally first.
column : str
Non-empty name of the string column containing stop words.
"""
@dataclass(frozen=True)
class TableStopWordsSource:
"""A table string column used by a remote FTS create-index request."""
table: str
table: Union["Table", "AsyncTable"]
column: str
type: Literal["table"] = field(default="table", init=False)
@property
def _column(self) -> str:
if not isinstance(self.column, str):
raise TypeError("FtsStopWordsTable.column must be a string")
if not self.column:
raise ValueError("FtsStopWordsTable.column must be a non-empty string")
return self.column
@property
def _native_table(self) -> Any:
# Synchronous RemoteTable lazily reopens through LOOP.run when its
# private `_table` property is accessed. This property is itself read
# from the LOOP thread by PyO3, so touching `_table` for a stale
# forked/unpickled handle would deadlock. Remote sources are unsupported
# anyway, so reject them without attempting a reopen.
if hasattr(self.table, "_table_handle"):
raise ValueError(
"remote table sources cannot guarantee a complete snapshot; "
"materialize the stop-word column into a local LanceDB table, "
"or use an inline list or UTF-8 file"
)
native_table = getattr(self.table, "_inner", None)
if native_table is None:
async_table = getattr(self.table, "_table", None)
native_table = getattr(async_table, "_inner", None)
if native_table is None:
raise TypeError(
"FtsStopWordsTable.table must be a LanceDB Table or AsyncTable"
)
return native_table
CustomStopWordsSource = Union[
InlineStopWordsSource,
FileStopWordsSource,
TableStopWordsSource,
]
CustomStopWords = Optional[Union[Sequence[str], FtsStopWordsFile, FtsStopWordsTable]]
@dataclass
@@ -183,15 +240,17 @@ class FTS:
remove_stop_words : bool, default True
Whether to remove stop words. Stop words are common words that are often
removed from text before indexing. For example, in English "the" and "and".
custom_stop_words : list of str, optional
A custom stop-word list that replaces the built-in language list. ``None``
uses the built-in list, while ``[]`` explicitly selects no stop words.
This only affects tokenization when ``remove_stop_words`` is True.
custom_stop_words_source : CustomStopWordsSource, optional
A request-only inline, UTF-8 file, or table-column source resolved by a
remote LanceDB service. It is mutually exclusive with
``custom_stop_words``. Local native tables reject source descriptors;
load the source explicitly and pass ``custom_stop_words`` instead.
custom_stop_words : sequence of str, file source, or table source, optional
A custom stop-word source. A sequence is used inline;
:class:`FtsStopWordsFile` reads a strictly UTF-8 newline-delimited file;
and :class:`FtsStopWordsTable` reads a string column from an open local
LanceDB table. Remote tables cannot be used as sources. Custom words
replace the built-in language stop words
and only take effect when ``remove_stop_words`` is True. ``None`` uses
the built-in language list, while an empty sequence explicitly uses no
stop words. Exact empty strings are ignored and exact duplicates are
removed while preserving first-seen order. Values are not trimmed,
lowercased, or otherwise normalized.
ascii_folding : bool, default True
Whether to fold ASCII characters. This converts accented characters to
their ASCII equivalent. For example, "café" would be converted to "cafe".
@@ -206,6 +265,18 @@ class FTS:
require tokenizer models in Lance's language model home. Set
``LANCE_LANGUAGE_MODEL_HOME`` to override the default platform data
directory under ``lance/language_models``.
Local FTS queries with active custom stop words reject an explicit
``fuzziness`` greater than zero because upstream fuzzy matching does not
reuse the index's complete tokenizer configuration. An omitted fuzziness
value and explicit zero continue to use the configured stop-word snapshot.
Remote tables temporarily reject every explicitly positive ``fuzziness``
before sending a query request, regardless of stop-word configuration,
because the service does not yet expose a capability that atomically binds
a query to its tokenizer snapshot. Native namespace pushdown routes
explicitly fuzzy queries to local execution, where the local rule above
applies.
"""
with_position: bool = False
@@ -220,18 +291,7 @@ class FTS:
ngram_max_length: int = 3
prefix_only: bool = False
block_size: int = 128
custom_stop_words: Optional[List[str]] = None
custom_stop_words_source: Optional[CustomStopWordsSource] = None
def __post_init__(self):
if (
self.custom_stop_words is not None
and self.custom_stop_words_source is not None
):
raise ValueError(
"custom_stop_words and custom_stop_words_source are mutually "
"exclusive; choose one"
)
custom_stop_words: CustomStopWords = None
@dataclass
@@ -906,10 +966,6 @@ __all__ = [
"HnswFlat",
"IndexConfig",
"FTS",
"CustomStopWordsSource",
"InlineStopWordsSource",
"FileStopWordsSource",
"TableStopWordsSource",
"Bitmap",
"LabelList",
"Fm",
python/python/lancedb/remote/table.py
+2 -4
View File
@@ -39,7 +39,7 @@ from lancedb.index import (
FTS,
BTree,
Bitmap,
CustomStopWordsSource,
CustomStopWords,
HnswFlat,
HnswSq,
IvfFlat,
@@ -341,8 +341,7 @@ class RemoteTable(Table):
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
custom_stop_words: Optional[List[str]] = None,
custom_stop_words_source: Optional[CustomStopWordsSource] = None,
custom_stop_words: CustomStopWords = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
ngram_max_length: int = 3,
@@ -365,7 +364,6 @@ class RemoteTable(Table):
stem=stem,
remove_stop_words=remove_stop_words,
custom_stop_words=custom_stop_words,
custom_stop_words_source=custom_stop_words_source,
ascii_folding=ascii_folding,
ngram_min_length=ngram_min_length,
ngram_max_length=ngram_max_length,
python/python/lancedb/table.py
+26 -14
View File
@@ -69,7 +69,7 @@ from .index import (
HnswSq,
HnswFlat,
FTS,
CustomStopWordsSource,
CustomStopWords,
)
from .expr import Expr
from .merge import LanceMergeInsertBuilder
@@ -1104,8 +1104,7 @@ class Table(ABC):
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
custom_stop_words: Optional[List[str]] = None,
custom_stop_words_source: Optional[CustomStopWordsSource] = None,
custom_stop_words: CustomStopWords = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
ngram_max_length: int = 3,
@@ -1140,7 +1139,11 @@ class Table(ABC):
for english it would be "en_stem". For new native FTS indexes, use
``base_tokenizer`` directly; ``tokenizer_name`` is a legacy
compatibility alias and does not expose model-backed tokenizer names
such as ``jieba/default`` or ``lindera/ipadic``.
such as ``jieba/default`` or ``lindera/ipadic``. Its historical
``remove_stop_words=False`` setting is preserved. A custom
stop-word snapshot is persisted when both options are supplied,
but remains inactive; use the modern ``FTS`` config with
``remove_stop_words=True`` to enable it.
use_tantivy: bool, default False
Deprecated legacy Tantivy parameter. Setting this to True raises an
error.
@@ -1173,12 +1176,12 @@ class Table(ABC):
remove_stop_words : bool, default True
Whether to remove stop words. Stop words are common words that are often
removed from text before indexing. For example, in English "the" and "and".
custom_stop_words : list of str, optional
Custom words that replace the built-in language stop-word list. ``None``
uses the built-in list; ``[]`` selects no stop words.
custom_stop_words_source : CustomStopWordsSource, optional
Request-only remote source for custom stop words. Mutually exclusive
with ``custom_stop_words`` and rejected by local native tables.
custom_stop_words : sequence of str, file source, or table source, optional
Custom words that replace the built-in language stop words. ``None``
uses the built-in list; an empty sequence explicitly uses no stop
words. File and table sources are resolved to a stable snapshot
before the index is created. Table sources must be local/native;
remote table sources are rejected.
ascii_folding : bool, default True
Whether to fold ASCII characters. This converts accented characters to
their ASCII equivalent. For example, "café" would be converted to "cafe".
@@ -1867,6 +1870,8 @@ class Table(ABC):
Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
rebuilt in the client process from index metadata. For remote tables,
this means the same tokenizer model files must also exist locally.
Custom stop-word sources are not re-read; this uses the persisted
snapshot stored in the index configuration.
"""
@abstractmethod
@@ -3064,8 +3069,7 @@ class LanceTable(Table):
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
custom_stop_words: Optional[List[str]] = None,
custom_stop_words_source: Optional[CustomStopWordsSource] = None,
custom_stop_words: CustomStopWords = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
ngram_max_length: int = 3,
@@ -3078,6 +3082,12 @@ class LanceTable(Table):
.. deprecated:: 0.25.0
Use :meth:`create_index` with an FTS config instead.
Example: ``table.create_index("text_column", config=FTS())``
When ``tokenizer_name`` is supplied, its historical
``remove_stop_words=False`` setting is preserved. ``custom_stop_words``
is still snapshotted and persisted, but remains inactive. Use the modern
:class:`lancedb.index.FTS` configuration with
``remove_stop_words=True`` to enable custom stop-word removal.
"""
self._ensure_no_legacy_fts_index()
@@ -3113,7 +3123,6 @@ class LanceTable(Table):
"stem": stem,
"remove_stop_words": remove_stop_words,
"custom_stop_words": custom_stop_words,
"custom_stop_words_source": custom_stop_words_source,
"ascii_folding": ascii_folding,
"ngram_min_length": ngram_min_length,
"ngram_max_length": ngram_max_length,
@@ -3122,7 +3131,6 @@ class LanceTable(Table):
else:
tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
tokenizer_configs["custom_stop_words"] = custom_stop_words
tokenizer_configs["custom_stop_words_source"] = custom_stop_words_source
config = FTS(block_size=block_size, **tokenizer_configs)
@@ -3856,6 +3864,8 @@ class LanceTable(Table):
Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
rebuilt in the client process from index metadata. For remote tables,
this means the same tokenizer model files must also exist locally.
Custom stop-word sources are not re-read; this uses the persisted
snapshot stored in the index configuration.
"""
return LOOP.run(
self._table.tokenize(query, column=column, index_name=index_name)
@@ -6003,6 +6013,8 @@ class AsyncTable:
Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
rebuilt in the client process from index metadata. For remote tables,
this means the same tokenizer model files must also exist locally.
Custom stop-word sources are not re-read; this uses the persisted
snapshot stored in the index configuration.
"""
return await self._inner.tokenize(query, column=column, index_name=index_name)
python/python/tests/docs/test_search.py
+46 -1
View File
@@ -24,7 +24,7 @@ from pydantic import BaseModel
# --8<-- [end:import-pydantic-base-model]
# --8<-- [start:import-lancedb-fts]
from lancedb.index import FTS
from lancedb.index import FTS, FtsStopWordsFile, FtsStopWordsTable
# --8<-- [end:import-lancedb-fts]
# --8<-- [start:import-os]
@@ -351,6 +351,51 @@ def test_fts_native():
# --8<-- [end:fts_incremental_index]
def test_fts_custom_stop_word_sources(tmp_path):
# --8<-- [start:fts_custom_stop_words]
db = lancedb.connect(tmp_path / "db")
documents = db.create_table(
"documents",
data=[{"text": "the lance database"}, {"text": "searchable data"}],
)
# Inline values
documents.create_index(
"text",
config=FTS(stem=False, custom_stop_words=["lance", "searchable"]),
)
# A strict UTF-8 file with one stop word per line
stop_words_path = tmp_path / "stop-words.txt"
stop_words_path.write_text("lance\nsearchable\n", encoding="utf-8")
documents.create_index(
"text",
config=FTS(
stem=False,
custom_stop_words=FtsStopWordsFile(stop_words_path),
),
replace=True,
)
# A string column from another open LanceDB table
stop_words = db.create_table(
"stop_words",
data=[{"word": "lance"}, {"word": "searchable"}],
)
documents.create_index(
"text",
config=FTS(
stem=False,
custom_stop_words=FtsStopWordsTable(stop_words, "word"),
),
replace=True,
)
# --8<-- [end:fts_custom_stop_words]
tokens = documents.tokenize("the lance database", column="text")
assert [token.text for token in tokens] == ["the", "database"]
@pytest.mark.skipif(
os.name == "nt", reason="Need to fix https://github.com/lancedb/lance/issues/3905"
)
python/python/tests/test_fts.py
+297 -48
View File
@@ -22,7 +22,7 @@ import zipfile
import lancedb as ldb
from lancedb.db import DBConnection
from lancedb.index import FTS, FileStopWordsSource
from lancedb.index import FTS, FtsStopWordsFile, FtsStopWordsTable
from lancedb.query import (
BoostQuery,
MatchQuery,
@@ -243,47 +243,46 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
table.create_index("text", config=FTS(block_size=129))
def test_custom_stop_words_preserve_none_and_empty(mem_db: DBConnection):
built_in = mem_db.create_table(
"built_in_stop_words", [{"text": "the lance database"}]
@pytest.mark.parametrize(
("custom_stop_words", "expected_the", "expected_lance"),
[
(None, 0, 2),
([], 1, 2),
(["lance"], 1, 0),
],
ids=["builtin", "explicit-empty", "custom"],
)
def test_custom_stop_words_inline_none_vs_empty(
tmp_path, custom_stop_words, expected_the, expected_lance
):
db = ldb.connect(tmp_path)
table = db.create_table(
"documents",
data=pa.table({"text": ["the lance database", "lance data"]}),
)
built_in.create_index(
table.create_index(
"text",
config=FTS(stem=False, custom_stop_words=None),
config=FTS(stem=False, custom_stop_words=custom_stop_words),
)
assert [token.text for token in built_in.tokenize("the lance", column="text")] == [
"lance"
]
empty = mem_db.create_table(
"empty_custom_stop_words", [{"text": "the lance database"}]
assert (
len(table.search("the", query_type="fts").limit(10).to_list()) == expected_the
)
empty.create_index(
"text",
config=FTS(stem=False, custom_stop_words=[]),
assert (
len(table.search("lance", query_type="fts").limit(10).to_list())
== expected_lance
)
assert [token.text for token in empty.tokenize("the lance", column="text")] == [
"the",
"lance",
]
details = table.list_indices()[0].index_details
assert details["custom_stop_words"] == custom_stop_words
custom = mem_db.create_table(
"non_empty_custom_stop_words", [{"text": "the lance database"}]
)
custom.create_index(
"text",
config=FTS(stem=False, custom_stop_words=["lance"]),
)
assert [token.text for token in custom.tokenize("the lance", column="text")] == [
"the"
]
assert len(custom.search("the", query_type="fts").to_list()) == 1
assert custom.search("lance", query_type="fts").to_list() == []
disabled = mem_db.create_table(
"disabled_custom_stop_words", [{"text": "the lance database"}]
def test_custom_stop_words_are_inactive_when_removal_is_disabled(tmp_path):
db = ldb.connect(tmp_path)
table = db.create_table(
"documents",
data=pa.table({"text": ["the lance database", "lance data"]}),
)
disabled.create_index(
table.create_index(
"text",
config=FTS(
stem=False,
@@ -291,31 +290,227 @@ def test_custom_stop_words_preserve_none_and_empty(mem_db: DBConnection):
custom_stop_words=["lance"],
),
)
assert [token.text for token in disabled.tokenize("the lance", column="text")] == [
"the",
assert len(table.search("lance", query_type="fts").limit(10).to_list()) == 2
tokens = table.tokenize("the lance data", column="text")
assert [token.text for token in tokens] == ["the", "lance", "data"]
def test_custom_stop_words_file_is_snapshotted_and_reopened(tmp_path):
stop_words_path = tmp_path / "stop-words.txt"
stop_words_path.write_bytes(
"lance\r\n\r\nlance\r\nKeep Case\r\n spaced \r\n".encode()
)
db = ldb.connect(tmp_path / "db")
table = db.create_table(
"documents",
data=pa.table({"text": ["the lance database", "lance data"]}),
)
table.create_index(
"text",
config=FTS(
stem=False,
custom_stop_words=FtsStopWordsFile(stop_words_path),
),
)
stop_words_path.write_text("the\n", encoding="utf-8")
reopened = db.open_table("documents")
details = reopened.list_indices()[0].index_details
assert details["custom_stop_words"] == ["lance", "Keep Case", " spaced "]
tokens = reopened.tokenize("the lance data", column="text")
assert [token.text for token in tokens] == ["the", "data"]
assert len(reopened.search("the", query_type="fts").limit(10).to_list()) == 1
assert len(reopened.search("lance", query_type="fts").limit(10).to_list()) == 0
def test_custom_stop_words_table_is_snapshotted(tmp_path):
db = ldb.connect(tmp_path)
stop_words = db.create_table(
"stop_words",
data=pa.table({"word": ["lance", "", "lance", "data"]}),
)
documents = db.create_table(
"documents",
data=pa.table({"text": ["the lance database", "lance data"]}),
)
documents.create_index(
"text",
config=FTS(
stem=False,
custom_stop_words=FtsStopWordsTable(stop_words, "word"),
),
)
db.drop_table("stop_words")
reopened = db.open_table("documents")
assert reopened.list_indices()[0].index_details["custom_stop_words"] == [
"lance",
"data",
]
assert len(disabled.search("lance", query_type="fts").to_list()) == 1
tokens = reopened.tokenize("the lance data", column="text")
assert [token.text for token in tokens] == ["the"]
def test_local_fts_rejects_custom_stop_words_source(mem_db: DBConnection):
table = mem_db.create_table("source_stop_words", [{"text": "lance database"}])
with pytest.raises(RuntimeError, match="only supported for remote tables"):
table.create_index(
@pytest.mark.asyncio
async def test_custom_stop_words_table_async(tmp_path):
db = await ldb.connect_async(tmp_path)
stop_words = await db.create_table(
"stop_words",
data=pa.table({"word": ["lance", "", "lance"]}),
)
documents = await db.create_table(
"documents",
data=pa.table({"text": ["the lance database", "lance data"]}),
)
await documents.create_index(
"text",
config=FTS(
stem=False,
custom_stop_words=FtsStopWordsTable(stop_words, "word"),
),
)
tokens = await documents.tokenize("the lance data", column="text")
assert [token.text for token in tokens] == ["the", "data"]
def test_custom_stop_words_deprecated_create_fts_index(table):
with pytest.warns(DeprecationWarning, match="create_fts_index"):
table.create_fts_index(
"text",
stem=False,
custom_stop_words=["puppy"],
)
assert len(table.search("puppy").limit(5).to_list()) == 0
def test_custom_stop_words_tokenizer_name_compatibility_is_persisted(table):
with pytest.warns(DeprecationWarning, match="create_fts_index"):
table.create_fts_index(
"text",
tokenizer_name="default",
custom_stop_words=["puppy"],
)
details = table.list_indices()[0].index_details
assert details["custom_stop_words"] == ["puppy"]
assert details["remove_stop_words"] is False
assert len(table.search("puppy").limit(5).to_list()) == 5
assert [token.text for token in table.tokenize("puppy runs", column="text")] == [
"puppy",
"runs",
]
@pytest.mark.parametrize(
("source_factory", "match"),
[
(lambda _tmp, _table: "not-a-sequence", "not a string"),
(lambda _tmp, _table: ["valid", 42], r"custom_stop_words\[1\]"),
(
lambda tmp, _table: FtsStopWordsFile(tmp / "missing.txt"),
"failed to read custom stop words file",
),
(
lambda _tmp, _table: FtsStopWordsFile(""),
"path must be a non-empty string path",
),
(
lambda _tmp, table: FtsStopWordsTable(table, "missing"),
"failed to resolve custom stop words column",
),
(
lambda _tmp, table: FtsStopWordsTable(table, ""),
"column must be a non-empty string",
),
(
lambda _tmp, table: FtsStopWordsTable(table, "number"),
"must have type Utf8",
),
],
)
def test_custom_stop_words_validation_errors(tmp_path, source_factory, match):
db = ldb.connect(tmp_path / "db")
documents = db.create_table(
"documents",
data=pa.table(
{
"text": ["the lance database"],
"number": [1],
}
),
)
with pytest.raises((TypeError, ValueError), match=match):
documents.create_index(
"text",
config=FTS(custom_stop_words=source_factory(tmp_path, documents)),
)
def test_custom_stop_words_file_rejects_invalid_utf8(tmp_path):
stop_words_path = tmp_path / "stop-words.txt"
stop_words_path.write_bytes(b"valid\n\xff")
db = ldb.connect(tmp_path / "db")
documents = db.create_table(
"documents",
data=pa.table({"text": ["the lance database"]}),
)
with pytest.raises(ValueError, match="not valid UTF-8"):
documents.create_index(
"text",
config=FTS(custom_stop_words=FtsStopWordsFile(stop_words_path)),
)
@pytest.mark.parametrize(
("values", "match"),
[
(pa.array(["valid", None]), "contains NULL"),
(pa.array([1, 2]), "must have type Utf8"),
],
)
def test_custom_stop_words_table_rejects_invalid_values(tmp_path, values, match):
db = ldb.connect(tmp_path)
stop_words = db.create_table("stop_words", data=pa.table({"word": values}))
documents = db.create_table(
"documents",
data=pa.table({"text": ["the lance database"]}),
)
with pytest.raises(ValueError, match=match):
documents.create_index(
"text",
config=FTS(
custom_stop_words_source=FileStopWordsSource(
"file:///tmp/stop-words.txt"
)
custom_stop_words=FtsStopWordsTable(stop_words, "word"),
),
)
def test_fts_rejects_both_custom_stop_words_forms():
with pytest.raises(ValueError, match="mutually exclusive"):
FTS(
custom_stop_words=[],
custom_stop_words_source=FileStopWordsSource("s3://bucket/stop-words.txt"),
def test_custom_stop_words_table_rejects_closed_table(tmp_path):
db = ldb.connect(tmp_path)
stop_words = db.create_table(
"stop_words",
data=pa.table({"word": ["lance"]}),
)
documents = db.create_table(
"documents",
data=pa.table({"text": ["the lance database"]}),
)
stop_words._table.close()
with pytest.raises(RuntimeError, match="Table stop_words is closed"):
documents.create_index(
"text",
config=FTS(
custom_stop_words=FtsStopWordsTable(stop_words, "word"),
),
)
@@ -970,6 +1165,60 @@ def test_tokenize_uses_explicit_icu_tokenizer():
]
@pytest.mark.parametrize(
("custom_stop_words", "expected"),
[
(None, ["lance", "data"]),
([], ["the", "lance", "data"]),
(["lance"], ["the", "data"]),
],
ids=["builtin", "explicit-empty", "custom"],
)
def test_tokenize_custom_stop_words_none_vs_empty(custom_stop_words, expected):
tokens = ldb.tokenize(
"the lance data",
stem=False,
custom_stop_words=custom_stop_words,
)
assert [token.text for token in tokens] == expected
def test_tokenize_custom_stop_words_file_and_table(tmp_path):
stop_words_path = tmp_path / "stop-words.txt"
stop_words_path.write_text("lance\nlance\n\n", encoding="utf-8")
file_tokens = ldb.tokenize(
"the lance data",
stem=False,
custom_stop_words=FtsStopWordsFile(stop_words_path),
)
db = ldb.connect(tmp_path / "db")
stop_words = db.create_table(
"stop_words",
data=pa.table({"word": ["lance", "", "lance"]}),
)
table_tokens = ldb.tokenize(
"the lance data",
stem=False,
custom_stop_words=FtsStopWordsTable(stop_words, "word"),
)
assert [token.text for token in file_tokens] == ["the", "data"]
assert [token.text for token in table_tokens] == ["the", "data"]
def test_tokenize_custom_stop_words_disabled():
tokens = ldb.tokenize(
"the lance data",
stem=False,
remove_stop_words=False,
custom_stop_words=["lance"],
)
assert [token.text for token in tokens] == ["the", "lance", "data"]
def test_fts_on_list(mem_db: DBConnection):
data = pa.table(
{
python/python/tests/test_remote_db.py
+89 -31
View File
@@ -17,7 +17,7 @@ from packaging.version import Version
import lancedb
from lancedb.conftest import MockTextEmbeddingFunction
from lancedb.query import ColumnOrdering
from lancedb.query import ColumnOrdering, MatchQuery
from lancedb.remote import ClientConfig
from lancedb.remote.errors import HttpError, RetryError
import pytest
@@ -771,6 +771,7 @@ def test_table_create_indices():
"text",
wait_timeout=timedelta(seconds=2),
block_size=256,
custom_stop_words=["cloud", "", "cloud"],
name="custom_fts_idx",
)
@@ -795,6 +796,7 @@ def test_table_create_indices():
assert "name" in fts_req
assert fts_req["name"] == "custom_fts_idx"
assert fts_req["block_size"] == 256
assert fts_req["custom_stop_words"] == ["cloud"]
# Check vector index request has custom name
vector_req = received_requests[2]
@@ -810,7 +812,7 @@ def test_table_create_indices():
table.drop_index("custom_fts_idx")
def test_remote_create_index_new_api():
def test_remote_create_index_new_api(tmp_path):
received_requests = []
def handler(request):
@@ -871,16 +873,19 @@ def test_remote_create_index_new_api():
from lancedb.index import (
BTree,
FTS,
FileStopWordsSource,
InlineStopWordsSource,
FtsStopWordsFile,
FtsStopWordsTable,
IvfPq,
IvfRq,
TableStopWordsSource,
)
with mock_lancedb_connection(handler) as db:
table = db.create_table("test", [{"id": 1}])
local_db = lancedb.connect(tmp_path / "local-source")
local_stop_words = local_db.create_table(
"stop_words",
data=[{"word": "table-word"}, {"word": ""}, {"word": "table-word"}],
)
# New API: column-first, config= kwarg. Should NOT emit DeprecationWarning.
import warnings as _warnings
@@ -890,25 +895,24 @@ def test_remote_create_index_new_api():
table.create_index("category", config=BTree())
table.create_index("text", config=FTS(block_size=256))
table.create_index("text", config=FTS(custom_stop_words=[]))
table.create_index(
"text", config=FTS(custom_stop_words=["lance", "database"])
)
table.create_index(
"text",
config=FTS(custom_stop_words_source=InlineStopWordsSource(["lance"])),
config=FTS(
custom_stop_words=["cloud", "", "cloud", "Keep Case"],
),
)
stop_words_path = tmp_path / "stop-words.txt"
stop_words_path.write_text("file-word\n\nfile-word\n", encoding="utf-8")
table.create_index(
"text",
config=FTS(
custom_stop_words_source=FileStopWordsSource(
"s3://bucket/stop-words.txt"
)
custom_stop_words=FtsStopWordsFile(stop_words_path),
),
)
table.create_index(
"text",
config=FTS(
custom_stop_words_source=TableStopWordsSource("stop_words", "word")
custom_stop_words=FtsStopWordsTable(local_stop_words, "word"),
),
)
# IvfRq via new API
@@ -922,7 +926,26 @@ def test_remote_create_index_new_api():
num_partitions=8,
)
assert len(received_requests) == 10
with pytest.raises(
ValueError,
match="remote table sources cannot guarantee a complete snapshot",
):
table.create_index(
"text",
config=FTS(
custom_stop_words=FtsStopWordsTable(table, "text"),
),
)
with pytest.raises(
ValueError,
match="remote table sources cannot guarantee a complete snapshot",
):
lancedb.tokenize(
"the cloud",
custom_stop_words=FtsStopWordsTable(table, "text"),
)
assert len(received_requests) == 9
assert [req["column"] for req in received_requests] == [
"vector",
"category",
@@ -931,27 +954,33 @@ def test_remote_create_index_new_api():
"text",
"text",
"text",
"text",
"vector",
"vector",
]
assert received_requests[2]["block_size"] == 256
assert received_requests[2]["custom_stop_words"] is None
assert received_requests[2].get("custom_stop_words") is None
assert received_requests[3]["custom_stop_words"] == []
assert received_requests[4]["custom_stop_words"] == ["lance", "database"]
assert received_requests[5]["custom_stop_words_source"] == {
"type": "inline",
"words": ["lance"],
}
assert received_requests[6]["custom_stop_words_source"] == {
"type": "file",
"uri": "s3://bucket/stop-words.txt",
}
assert received_requests[7]["custom_stop_words_source"] == {
"type": "table",
"table": "stop_words",
"column": "word",
}
assert received_requests[4]["custom_stop_words"] == ["cloud", "Keep Case"]
assert received_requests[5]["custom_stop_words"] == ["file-word"]
assert received_requests[6]["custom_stop_words"] == ["table-word"]
assert "path" not in json.dumps(received_requests[5])
assert "table" not in json.dumps(received_requests[5])
assert "table-word" in json.dumps(received_requests[6])
assert not {"source", "table", "path"} & received_requests[6].keys()
# A forked/unpickled RemoteTable may have no live handle. Stop-word
# validation must fail without trying to reopen through LOOP.run from
# the LOOP thread (which would deadlock).
table._table_handle = None
with pytest.raises(
ValueError,
match="remote table sources cannot guarantee a complete snapshot",
):
lancedb.tokenize(
"the cloud",
custom_stop_words=FtsStopWordsTable(table, "text"),
)
assert table._table_handle is None
def test_table_wait_for_index_timeout():
@@ -1348,6 +1377,35 @@ def test_query_sync_fts():
)
def test_query_sync_fts_fuzziness_boundary():
query_requests = 0
def handler(body):
nonlocal query_requests
query_requests += 1
assert body["full_text_query"]["query"]["match"]["fuzziness"] == 0
return pa.table({"id": [1]})
with query_test_table(handler, server_version=Version("0.3.0")) as table:
table.search(
MatchQuery("puppy", "text", fuzziness=0),
query_type="fts",
).to_list()
assert query_requests == 1
with pytest.raises(
ValueError,
match="explicit fuzzy FTS queries are not supported.*fuzziness=0",
):
table.search(
MatchQuery("puppy", "text", fuzziness=1),
query_type="fts",
).to_list()
# Positive fuzziness fails before issuing another remote query.
assert query_requests == 1
def test_query_sync_hybrid():
def handler(body):
if "full_text_query" in body:
python/src/index.rs
+93 -43
View File
@@ -8,15 +8,15 @@ use lancedb::index::vector::{
};
use lancedb::index::{
Index as LanceDbIndex,
scalar::{BTreeIndexBuilder, CustomStopWordsSource, FmIndexBuilder, FtsIndexBuilder},
scalar::{BTreeIndexBuilder, FmIndexBuilder, FtsIndexBuilder, FtsStopWordsSource},
};
use pyo3::IntoPyObject;
use pyo3::types::PyStringMethods;
use pyo3::{
Bound, FromPyObject, Py, PyAny, PyResult, Python,
exceptions::{PyKeyError, PyValueError},
Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
exceptions::{PyKeyError, PyTypeError, PyValueError},
intern, pyclass, pymethods,
types::{PyAnyMethods, PyString},
types::{PyAnyMethods, PySequence, PyString, PyTypeMethods},
};
use crate::util::parse_distance_type;
@@ -33,42 +33,94 @@ pub fn class_name(ob: &'_ Bound<'_, PyAny>) -> PyResult<String> {
}
}
fn extract_custom_stop_words_source(
source: &Bound<'_, PyAny>,
) -> PyResult<Option<CustomStopWordsSource>> {
let source = source.getattr("custom_stop_words_source")?;
if source.is_none() {
pub(crate) fn extract_fts_stop_words_source(
source: &Option<Bound<'_, PyAny>>,
) -> PyResult<Option<FtsStopWordsSource>> {
let Some(config) = source else {
return Ok(None);
};
if class_name(config)? != "FTS" {
return Ok(None);
}
let source_type: String = source.getattr("type")?.extract()?;
match source_type.as_str() {
"inline" => Ok(Some(CustomStopWordsSource::Inline {
words: source.getattr("words")?.extract()?,
})),
"file" => Ok(Some(CustomStopWordsSource::File {
uri: source.getattr("uri")?.extract()?,
})),
"table" => Ok(Some(CustomStopWordsSource::Table {
table: source.getattr("table")?.extract()?,
column: source.getattr("column")?.extract()?,
})),
other => Err(PyValueError::new_err(format!(
"invalid custom_stop_words_source type '{other}'; expected inline, file, or table"
))),
let value = config.getattr(intern!(config.py(), "custom_stop_words"))?;
extract_fts_stop_words_value(&value)
}
pub(crate) fn extract_fts_stop_words_value(
value: &Bound<'_, PyAny>,
) -> PyResult<Option<FtsStopWordsSource>> {
if value.is_none() {
return Ok(None);
}
match class_name(value)?.as_str() {
"FtsStopWordsFile" => {
let path = value
.getattr(intern!(value.py(), "_path"))?
.extract::<String>()
.map_err(|_| {
PyTypeError::new_err(
"FtsStopWordsFile.path must be a string or os.PathLike[str]",
)
})?;
Ok(Some(FtsStopWordsSource::file(path)))
}
"FtsStopWordsTable" => {
let column = value
.getattr(intern!(value.py(), "_column"))?
.extract::<String>()
.map_err(|_| PyTypeError::new_err("FtsStopWordsTable.column must be a string"))?;
let native_table = value.getattr(intern!(value.py(), "_native_table"))?;
let native_table = native_table
.extract::<PyRef<'_, crate::table::Table>>()
.map_err(|_| {
PyTypeError::new_err(
"FtsStopWordsTable.table must be a LanceDB Table or AsyncTable",
)
})?;
let table = native_table.inner_ref()?.clone();
Ok(Some(FtsStopWordsSource::table(table, column)))
}
_ => {
if value.cast::<PyString>().is_ok() {
return Err(PyTypeError::new_err(
"custom_stop_words must be a sequence of strings, not a string",
));
}
value.cast::<PySequence>().map_err(|_| {
PyTypeError::new_err(
"custom_stop_words must be a sequence of strings, \
FtsStopWordsFile, FtsStopWordsTable, or None",
)
})?;
let mut words = Vec::new();
for (idx, word) in value.try_iter()?.enumerate() {
let word = word?;
words.push(word.extract::<String>().map_err(|_| {
PyTypeError::new_err(format!(
"custom_stop_words[{idx}] must be a string, got {}",
word.get_type()
.name()
.map(|name| name.to_string_lossy().into_owned())
.unwrap_or_else(|_| "unknown".to_string())
))
})?);
}
Ok(Some(FtsStopWordsSource::inline(words)))
}
}
}
pub fn extract_index_params(
source: &Option<Bound<'_, PyAny>>,
) -> PyResult<(LanceDbIndex, Option<CustomStopWordsSource>)> {
pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<LanceDbIndex> {
if let Some(source) = source {
match class_name(source)?.as_str() {
"BTree" => Ok((LanceDbIndex::BTree(BTreeIndexBuilder::default()), None)),
"Bitmap" => Ok((LanceDbIndex::Bitmap(Default::default()), None)),
"LabelList" => Ok((LanceDbIndex::LabelList(Default::default()), None)),
"Fm" => Ok((LanceDbIndex::Fm(FmIndexBuilder::default()), None)),
"BTree" => Ok(LanceDbIndex::BTree(BTreeIndexBuilder::default())),
"Bitmap" => Ok(LanceDbIndex::Bitmap(Default::default())),
"LabelList" => Ok(LanceDbIndex::LabelList(Default::default())),
"Fm" => Ok(LanceDbIndex::Fm(FmIndexBuilder::default())),
"FTS" => {
let custom_stop_words_source = extract_custom_stop_words_source(source)?;
let params = source.extract::<FtsParams>()?;
let inner_opts = FtsIndexBuilder::default()
.base_tokenizer(params.base_tokenizer)
@@ -83,7 +135,6 @@ pub fn extract_index_params(
.lower_case(params.lower_case)
.max_token_length(params.max_token_length)
.remove_stop_words(params.remove_stop_words)
.custom_stop_words(params.custom_stop_words)
.stem(params.stem)
.ascii_folding(params.ascii_folding)
.ngram_min_length(params.ngram_min_length)
@@ -92,7 +143,7 @@ pub fn extract_index_params(
let inner_opts = inner_opts
.block_size(params.block_size)
.map_err(|err| PyValueError::new_err(err.to_string()))?;
Ok((LanceDbIndex::FTS(inner_opts), custom_stop_words_source))
Ok(LanceDbIndex::FTS(inner_opts))
}
"IvfFlat" => {
let params = source.extract::<IvfFlatParams>()?;
@@ -108,7 +159,7 @@ pub fn extract_index_params(
ivf_flat_builder =
ivf_flat_builder.target_partition_size(target_partition_size);
}
Ok((LanceDbIndex::IvfFlat(ivf_flat_builder), None))
Ok(LanceDbIndex::IvfFlat(ivf_flat_builder))
}
"IvfPq" => {
let params = source.extract::<IvfPqParams>()?;
@@ -127,7 +178,7 @@ pub fn extract_index_params(
if let Some(num_sub_vectors) = params.num_sub_vectors {
ivf_pq_builder = ivf_pq_builder.num_sub_vectors(num_sub_vectors);
}
Ok((LanceDbIndex::IvfPq(ivf_pq_builder), None))
Ok(LanceDbIndex::IvfPq(ivf_pq_builder))
}
"IvfSq" => {
let params = source.extract::<IvfSqParams>()?;
@@ -142,7 +193,7 @@ pub fn extract_index_params(
if let Some(target_partition_size) = params.target_partition_size {
ivf_sq_builder = ivf_sq_builder.target_partition_size(target_partition_size);
}
Ok((LanceDbIndex::IvfSq(ivf_sq_builder), None))
Ok(LanceDbIndex::IvfSq(ivf_sq_builder))
}
"IvfRq" => {
let params = source.extract::<IvfRqParams>()?;
@@ -158,7 +209,7 @@ pub fn extract_index_params(
if let Some(target_partition_size) = params.target_partition_size {
ivf_rq_builder = ivf_rq_builder.target_partition_size(target_partition_size);
}
Ok((LanceDbIndex::IvfRq(ivf_rq_builder), None))
Ok(LanceDbIndex::IvfRq(ivf_rq_builder))
}
"HnswPq" => {
let params = source.extract::<IvfHnswPqParams>()?;
@@ -179,7 +230,7 @@ pub fn extract_index_params(
if let Some(num_sub_vectors) = params.num_sub_vectors {
hnsw_pq_builder = hnsw_pq_builder.num_sub_vectors(num_sub_vectors);
}
Ok((LanceDbIndex::IvfHnswPq(hnsw_pq_builder), None))
Ok(LanceDbIndex::IvfHnswPq(hnsw_pq_builder))
}
"HnswSq" => {
let params = source.extract::<IvfHnswSqParams>()?;
@@ -196,7 +247,7 @@ pub fn extract_index_params(
if let Some(target_partition_size) = params.target_partition_size {
hnsw_sq_builder = hnsw_sq_builder.target_partition_size(target_partition_size);
}
Ok((LanceDbIndex::IvfHnswSq(hnsw_sq_builder), None))
Ok(LanceDbIndex::IvfHnswSq(hnsw_sq_builder))
}
"HnswFlat" => {
let params = source.extract::<IvfHnswFlatParams>()?;
@@ -214,7 +265,7 @@ pub fn extract_index_params(
hnsw_flat_builder =
hnsw_flat_builder.target_partition_size(target_partition_size);
}
Ok((LanceDbIndex::IvfHnswFlat(hnsw_flat_builder), None))
Ok(LanceDbIndex::IvfHnswFlat(hnsw_flat_builder))
}
not_supported => Err(PyValueError::new_err(format!(
"Invalid index type '{}'. Must be one of BTree, Bitmap, LabelList, Fm, FTS, IvfPq, IvfSq, IvfHnswPq, IvfHnswSq, or IvfHnswFlat",
@@ -222,7 +273,7 @@ pub fn extract_index_params(
))),
}
} else {
Ok((LanceDbIndex::Auto, None))
Ok(LanceDbIndex::Auto)
}
}
@@ -235,7 +286,6 @@ struct FtsParams {
lower_case: bool,
stem: bool,
remove_stop_words: bool,
custom_stop_words: Option<Vec<String>>,
ascii_folding: bool,
ngram_min_length: u32,
ngram_max_length: u32,
python/src/table.rs
+27 -8
View File
@@ -7,7 +7,10 @@ use crate::{
connection::Connection,
error::PythonErrorExt,
expr::PyExpr,
index::{IndexConfig, extract_index_params},
index::{
IndexConfig, extract_fts_stop_words_source, extract_fts_stop_words_value,
extract_index_params,
},
query::{Query, TakeQuery},
table::scannable::PyScannable,
};
@@ -520,13 +523,15 @@ impl From<LanceDbFtsToken> for FtsToken {
lower_case = true,
stem = true,
remove_stop_words = true,
custom_stop_words = None,
ascii_folding = true,
ngram_min_length = 3,
ngram_max_length = 3,
prefix_only = false
))]
#[allow(clippy::too_many_arguments)]
pub fn tokenize(
pub fn tokenize<'py>(
py: Python<'py>,
query: String,
base_tokenizer: String,
language: String,
@@ -534,11 +539,17 @@ pub fn tokenize(
lower_case: bool,
stem: bool,
remove_stop_words: bool,
custom_stop_words: Option<Bound<'py, PyAny>>,
ascii_folding: bool,
ngram_min_length: u32,
ngram_max_length: u32,
prefix_only: bool,
) -> PyResult<Vec<FtsToken>> {
) -> PyResult<Bound<'py, PyAny>> {
let custom_stop_words = custom_stop_words
.as_ref()
.map(extract_fts_stop_words_value)
.transpose()?
.flatten();
let params = FtsIndexBuilder::default()
.base_tokenizer(base_tokenizer)
.language(&language)
@@ -556,8 +567,15 @@ pub fn tokenize(
.ngram_min_length(ngram_min_length)
.ngram_max_length(ngram_max_length)
.ngram_prefix_only(prefix_only);
let tokens = lancedb_tokenize(&query, &params).infer_error()?;
Ok(tokens.into_iter().map(FtsToken::from).collect())
future_into_py(py, async move {
let params = if let Some(source) = custom_stop_words {
params.custom_stop_words(Some(source.resolve().await.infer_error()?))
} else {
params
};
let tokens = lancedb_tokenize(&query, &params).infer_error()?;
Ok(tokens.into_iter().map(FtsToken::from).collect::<Vec<_>>())
})
}
#[pyclass]
@@ -781,7 +799,8 @@ impl Table {
name: Option<String>,
train: Option<bool>,
) -> PyResult<Bound<'a, PyAny>> {
let (index, custom_stop_words_source) = extract_index_params(&index)?;
let custom_stop_words = extract_fts_stop_words_source(&index)?;
let index = extract_index_params(&index)?;
let timeout = wait_timeout.map(|t| t.extract::<std::time::Duration>().unwrap());
let mut op = self_
.inner_ref()?
@@ -795,8 +814,8 @@ impl Table {
if let Some(train) = train {
op = op.train(train);
}
if let Some(source) = custom_stop_words_source {
op = op.custom_stop_words_source(source);
if let Some(custom_stop_words) = custom_stop_words {
op = op.custom_stop_words(custom_stop_words).infer_error()?;
}
future_into_py(self_.py(), async move {