feat(fts): support custom stop-word lists (#3734)

## What

Expose custom FTS stop-word lists in the Python and TypeScript public
APIs, including their standalone tokenize helpers and remote index
creation.

This PR supports concrete string lists only. It does not add file or
LanceDB-table stop-word sources.

## Why

Rust already exposes Lance's custom stop-word list option. The Python
and TypeScript APIs did not pass it through, and local index details did
not retain the full tokenizer parameters needed by index-backed
tokenization after reopening a table.

## How

- Add `custom_stop_words` / `customStopWords` to the Python and
TypeScript FTS and tokenize options.
- Preserve `None` / `undefined`, empty lists, and list contents without
normalization.
- Load the persisted FTS segment parameters when returning local index
details.
- Serialize the concrete list in remote create-index requests.
- Keep Python and TypeScript tests thin; behavior, persistence, query
tokenization, and remote JSON coverage live primarily in Rust.

## Validation

- `cargo check --quiet --features remote --tests --examples`
- `cargo clippy --quiet --features remote --tests --examples`
- `cargo test --quiet --features remote --tests`
- Python extension rebuild with `uv` and `maturin`
- Targeted Python tests: 4 passed
- Python `ruff format --check` and `ruff check`
- TypeScript build, typecheck, Biome lint, generated docs, and targeted
tests

---------

Co-authored-by: Yang Cen <yangcen@Yangs-Mac-mini.local>

python/python/lancedb/__init__.py

@@ -258,6 +258,7 @@ def tokenize(
     lower_case: bool = True,
     stem: bool = True,
     remove_stop_words: bool = True,
+    custom_stop_words: Optional[List[str]] = None,
     ascii_folding: bool = True,
     ngram_min_length: int = 3,
     ngram_max_length: int = 3,
@@ -265,9 +266,10 @@ def tokenize(
 ) -> Iterable[FtsToken]:
     """Tokenize a full-text search query using an explicit tokenizer.
 
-    This does not require a table or FTS index. The tokenizer options match
-    :class:`lancedb.index.FTS`.
+    This does not require an FTS index. The tokenizer options match
+    :class:`lancedb.index.FTS`. ``custom_stop_words`` accepts a list of strings.
     """
+
     return _tokenize(
         query,
         base_tokenizer=base_tokenizer,
@@ -276,6 +278,7 @@ def tokenize(
         lower_case=lower_case,
         stem=stem,
         remove_stop_words=remove_stop_words,
+        custom_stop_words=custom_stop_words,
         ascii_folding=ascii_folding,
         ngram_min_length=ngram_min_length,
         ngram_max_length=ngram_max_length,

python/python/lancedb/_lancedb.pyi

@@ -59,6 +59,7 @@ def tokenize(
     lower_case: bool = True,
     stem: bool = True,
     remove_stop_words: bool = True,
+    custom_stop_words: Optional[List[str]] = None,
     ascii_folding: bool = True,
     ngram_min_length: int = 3,
     ngram_max_length: int = 3,

python/python/lancedb/index.py

@@ -2,7 +2,7 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 from dataclasses import dataclass
-from typing import Literal, Optional
+from typing import List, Literal, Optional
 
 from ._lancedb import (
     IndexConfig,
@@ -151,6 +151,11 @@ class FTS:
     remove_stop_words : bool, default True
         Whether to remove stop words. Stop words are common words that are often
         removed from text before indexing. For example, in English "the" and "and".
+    custom_stop_words : list of str, optional
+        Custom words replace the built-in language stop words
+        and only take effect when ``remove_stop_words`` is True. ``None`` uses
+        the built-in language list, while an empty list explicitly uses no
+        stop words.
     ascii_folding : bool, default True
         Whether to fold ASCII characters. This converts accented characters to
         their ASCII equivalent. For example, "café" would be converted to "cafe".
@@ -179,6 +184,7 @@ class FTS:
     ngram_max_length: int = 3
     prefix_only: bool = False
     block_size: int = 128
+    custom_stop_words: Optional[List[str]] = None
 
 
 @dataclass

python/python/lancedb/remote/table.py

@@ -340,6 +340,7 @@ class RemoteTable(Table):
         lower_case: bool = True,
         stem: bool = True,
         remove_stop_words: bool = True,
+        custom_stop_words: Optional[List[str]] = None,
         ascii_folding: bool = True,
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
@@ -361,6 +362,7 @@ class RemoteTable(Table):
             lower_case=lower_case,
             stem=stem,
             remove_stop_words=remove_stop_words,
+            custom_stop_words=custom_stop_words,
             ascii_folding=ascii_folding,
             ngram_min_length=ngram_min_length,
             ngram_max_length=ngram_max_length,

python/python/lancedb/table.py

@@ -1103,6 +1103,7 @@ class Table(ABC):
         lower_case: bool = True,
         stem: bool = True,
         remove_stop_words: bool = True,
+        custom_stop_words: Optional[List[str]] = None,
         ascii_folding: bool = True,
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
@@ -1170,6 +1171,9 @@ class Table(ABC):
         remove_stop_words : bool, default True
             Whether to remove stop words. Stop words are common words that are often
             removed from text before indexing. For example, in English "the" and "and".
+        custom_stop_words : list of str, optional
+            Custom words that replace the built-in language stop words. ``None``
+            uses the built-in list; an empty list explicitly uses no stop words.
         ascii_folding : bool, default True
             Whether to fold ASCII characters. This converts accented characters to
             their ASCII equivalent. For example, "café" would be converted to "cafe".
@@ -3055,6 +3059,7 @@ class LanceTable(Table):
         lower_case: bool = True,
         stem: bool = True,
         remove_stop_words: bool = True,
+        custom_stop_words: Optional[List[str]] = None,
         ascii_folding: bool = True,
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
@@ -3101,6 +3106,7 @@ class LanceTable(Table):
                 "lower_case": lower_case,
                 "stem": stem,
                 "remove_stop_words": remove_stop_words,
+                "custom_stop_words": custom_stop_words,
                 "ascii_folding": ascii_folding,
                 "ngram_min_length": ngram_min_length,
                 "ngram_max_length": ngram_max_length,
@@ -3108,6 +3114,7 @@ class LanceTable(Table):
             }
         else:
             tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
+            tokenizer_configs["custom_stop_words"] = custom_stop_words
 
         config = FTS(block_size=block_size, **tokenizer_configs)
 

python/python/tests/test_fts.py

@@ -219,11 +219,13 @@ def test_create_inverted_index(table, with_position):
         table.create_fts_index(
             "text",
             with_position=with_position,
+            custom_stop_words=["puppy"],
             name="custom_fts_index",
         )
     indices = table.list_indices()
     fts_indices = [i for i in indices if i.index_type == "FTS"]
     assert any(i.name == "custom_fts_index" for i in fts_indices)
+    assert fts_indices[0].index_details["custom_stop_words"] == ["puppy"]
 
 
 @pytest.mark.parametrize("block_size", [128, 256])
@@ -243,6 +245,24 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
         table.create_index("text", config=FTS(block_size=129))
 
 
+def test_custom_stop_words_list(table):
+    table.create_index(
+        "text",
+        config=FTS(stem=False, custom_stop_words=["lance"]),
+    )
+
+    assert table.list_indices()[0].index_details["custom_stop_words"] == ["lance"]
+    tokens = table.tokenize("the lance data", column="text")
+    assert [token.text for token in tokens] == ["the", "data"]
+    empty_tokens = ldb.tokenize("the lance data", stem=False, custom_stop_words=[])
+    assert [token.text for token in empty_tokens] == ["the", "lance", "data"]
+    with pytest.raises(TypeError, match=r"custom_stop_words.*int"):
+        ldb.tokenize(
+            "the lance data",
+            custom_stop_words=["lance", 42],
+        )
+
+
 def test_search_fts(table):
     table.create_fts_index("text")
     results = table.search("puppy").select(["id", "text"]).limit(5).to_list()

python/python/tests/test_remote_db.py

@@ -771,6 +771,7 @@ def test_table_create_indices():
                 "text",
                 wait_timeout=timedelta(seconds=2),
                 block_size=256,
+                custom_stop_words=["cloud"],
                 name="custom_fts_idx",
             )
 
@@ -795,6 +796,7 @@ def test_table_create_indices():
         assert "name" in fts_req
         assert fts_req["name"] == "custom_fts_idx"
         assert fts_req["block_size"] == 256
+        assert fts_req["custom_stop_words"] == ["cloud"]
 
         # Check vector index request has custom name
         vector_req = received_requests[2]

python/src/index.rs

@@ -59,7 +59,8 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     .ascii_folding(params.ascii_folding)
                     .ngram_min_length(params.ngram_min_length)
                     .ngram_max_length(params.ngram_max_length)
-                    .ngram_prefix_only(params.prefix_only);
+                    .ngram_prefix_only(params.prefix_only)
+                    .custom_stop_words(params.custom_stop_words);
                 let inner_opts = inner_opts
                     .block_size(params.block_size)
                     .map_err(|err| PyValueError::new_err(err.to_string()))?;
@@ -206,6 +207,7 @@ struct FtsParams {
     lower_case: bool,
     stem: bool,
     remove_stop_words: bool,
+    custom_stop_words: Option<Vec<String>>,
     ascii_folding: bool,
     ngram_min_length: u32,
     ngram_max_length: u32,

python/src/table.rs

@@ -520,6 +520,7 @@ impl From<LanceDbFtsToken> for FtsToken {
     lower_case = true,
     stem = true,
     remove_stop_words = true,
+    custom_stop_words = None,
     ascii_folding = true,
     ngram_min_length = 3,
     ngram_max_length = 3,
@@ -534,6 +535,7 @@ pub fn tokenize(
     lower_case: bool,
     stem: bool,
     remove_stop_words: bool,
+    custom_stop_words: Option<Vec<String>>,
     ascii_folding: bool,
     ngram_min_length: u32,
     ngram_max_length: u32,
@@ -555,7 +557,8 @@ pub fn tokenize(
         .ascii_folding(ascii_folding)
         .ngram_min_length(ngram_min_length)
         .ngram_max_length(ngram_max_length)
-        .ngram_prefix_only(prefix_only);
+        .ngram_prefix_only(prefix_only)
+        .custom_stop_words(custom_stop_words);
     let tokens = lancedb_tokenize(&query, &params).infer_error()?;
     Ok(tokens.into_iter().map(FtsToken::from).collect())
 }