feat(fts): add custom stop-word sources

python/python/lancedb/index.py

@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-from dataclasses import dataclass
-from typing import Literal, Optional
+from dataclasses import dataclass, field
+from typing import List, Literal, Optional, Union
 
 from ._lancedb import (
     IndexConfig,
@@ -31,6 +31,38 @@ lang_mapping = {
 }
 
 
+@dataclass(frozen=True)
+class InlineStopWordsSource:
+    """Inline custom stop words for a remote FTS create-index request."""
+
+    words: List[str]
+    type: Literal["inline"] = field(default="inline", init=False)
+
+
+@dataclass(frozen=True)
+class FileStopWordsSource:
+    """A UTF-8, newline-delimited custom stop-word object for a remote request."""
+
+    uri: str
+    type: Literal["file"] = field(default="file", init=False)
+
+
+@dataclass(frozen=True)
+class TableStopWordsSource:
+    """A table string column used by a remote FTS create-index request."""
+
+    table: str
+    column: str
+    type: Literal["table"] = field(default="table", init=False)
+
+
+CustomStopWordsSource = Union[
+    InlineStopWordsSource,
+    FileStopWordsSource,
+    TableStopWordsSource,
+]
+
+
 @dataclass
 class BTree:
     """Describes a btree index configuration
@@ -151,6 +183,15 @@ class FTS:
     remove_stop_words : bool, default True
         Whether to remove stop words. Stop words are common words that are often
         removed from text before indexing. For example, in English "the" and "and".
+    custom_stop_words : list of str, optional
+        A custom stop-word list that replaces the built-in language list. ``None``
+        uses the built-in list, while ``[]`` explicitly selects no stop words.
+        This only affects tokenization when ``remove_stop_words`` is True.
+    custom_stop_words_source : CustomStopWordsSource, optional
+        A request-only inline, UTF-8 file, or table-column source resolved by a
+        remote LanceDB service. It is mutually exclusive with
+        ``custom_stop_words``. Local native tables reject source descriptors;
+        load the source explicitly and pass ``custom_stop_words`` instead.
     ascii_folding : bool, default True
         Whether to fold ASCII characters. This converts accented characters to
         their ASCII equivalent. For example, "café" would be converted to "cafe".
@@ -179,6 +220,18 @@ class FTS:
     ngram_max_length: int = 3
     prefix_only: bool = False
     block_size: int = 128
+    custom_stop_words: Optional[List[str]] = None
+    custom_stop_words_source: Optional[CustomStopWordsSource] = None
+
+    def __post_init__(self):
+        if (
+            self.custom_stop_words is not None
+            and self.custom_stop_words_source is not None
+        ):
+            raise ValueError(
+                "custom_stop_words and custom_stop_words_source are mutually "
+                "exclusive; choose one"
+            )
 
 
 @dataclass
@@ -853,6 +906,10 @@ __all__ = [
     "HnswFlat",
     "IndexConfig",
     "FTS",
+    "CustomStopWordsSource",
+    "InlineStopWordsSource",
+    "FileStopWordsSource",
+    "TableStopWordsSource",
     "Bitmap",
     "LabelList",
     "Fm",

python/python/lancedb/remote/table.py

@@ -39,6 +39,7 @@ from lancedb.index import (
     FTS,
     BTree,
     Bitmap,
+    CustomStopWordsSource,
     HnswFlat,
     HnswSq,
     IvfFlat,
@@ -340,6 +341,8 @@ class RemoteTable(Table):
         lower_case: bool = True,
         stem: bool = True,
         remove_stop_words: bool = True,
+        custom_stop_words: Optional[List[str]] = None,
+        custom_stop_words_source: Optional[CustomStopWordsSource] = None,
         ascii_folding: bool = True,
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
@@ -361,6 +364,8 @@ class RemoteTable(Table):
             lower_case=lower_case,
             stem=stem,
             remove_stop_words=remove_stop_words,
+            custom_stop_words=custom_stop_words,
+            custom_stop_words_source=custom_stop_words_source,
             ascii_folding=ascii_folding,
             ngram_min_length=ngram_min_length,
             ngram_max_length=ngram_max_length,

python/python/lancedb/table.py

@@ -69,6 +69,7 @@ from .index import (
     HnswSq,
     HnswFlat,
     FTS,
+    CustomStopWordsSource,
 )
 from .expr import Expr
 from .merge import LanceMergeInsertBuilder
@@ -1103,6 +1104,8 @@ class Table(ABC):
         lower_case: bool = True,
         stem: bool = True,
         remove_stop_words: bool = True,
+        custom_stop_words: Optional[List[str]] = None,
+        custom_stop_words_source: Optional[CustomStopWordsSource] = None,
         ascii_folding: bool = True,
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
@@ -1170,6 +1173,12 @@ class Table(ABC):
         remove_stop_words : bool, default True
             Whether to remove stop words. Stop words are common words that are often
             removed from text before indexing. For example, in English "the" and "and".
+        custom_stop_words : list of str, optional
+            Custom words that replace the built-in language stop-word list. ``None``
+            uses the built-in list; ``[]`` selects no stop words.
+        custom_stop_words_source : CustomStopWordsSource, optional
+            Request-only remote source for custom stop words. Mutually exclusive
+            with ``custom_stop_words`` and rejected by local native tables.
         ascii_folding : bool, default True
             Whether to fold ASCII characters. This converts accented characters to
             their ASCII equivalent. For example, "café" would be converted to "cafe".
@@ -3055,6 +3064,8 @@ class LanceTable(Table):
         lower_case: bool = True,
         stem: bool = True,
         remove_stop_words: bool = True,
+        custom_stop_words: Optional[List[str]] = None,
+        custom_stop_words_source: Optional[CustomStopWordsSource] = None,
         ascii_folding: bool = True,
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
@@ -3101,6 +3112,8 @@ class LanceTable(Table):
                 "lower_case": lower_case,
                 "stem": stem,
                 "remove_stop_words": remove_stop_words,
+                "custom_stop_words": custom_stop_words,
+                "custom_stop_words_source": custom_stop_words_source,
                 "ascii_folding": ascii_folding,
                 "ngram_min_length": ngram_min_length,
                 "ngram_max_length": ngram_max_length,
@@ -3108,6 +3121,8 @@ class LanceTable(Table):
             }
         else:
             tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
+            tokenizer_configs["custom_stop_words"] = custom_stop_words
+            tokenizer_configs["custom_stop_words_source"] = custom_stop_words_source
 
         config = FTS(block_size=block_size, **tokenizer_configs)
 

python/python/tests/test_fts.py

@@ -22,7 +22,7 @@ import zipfile
 
 import lancedb as ldb
 from lancedb.db import DBConnection
-from lancedb.index import FTS
+from lancedb.index import FTS, FileStopWordsSource
 from lancedb.query import (
     BoostQuery,
     MatchQuery,
@@ -243,6 +243,82 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
         table.create_index("text", config=FTS(block_size=129))
 
 
+def test_custom_stop_words_preserve_none_and_empty(mem_db: DBConnection):
+    built_in = mem_db.create_table(
+        "built_in_stop_words", [{"text": "the lance database"}]
+    )
+    built_in.create_index(
+        "text",
+        config=FTS(stem=False, custom_stop_words=None),
+    )
+    assert [token.text for token in built_in.tokenize("the lance", column="text")] == [
+        "lance"
+    ]
+
+    empty = mem_db.create_table(
+        "empty_custom_stop_words", [{"text": "the lance database"}]
+    )
+    empty.create_index(
+        "text",
+        config=FTS(stem=False, custom_stop_words=[]),
+    )
+    assert [token.text for token in empty.tokenize("the lance", column="text")] == [
+        "the",
+        "lance",
+    ]
+
+    custom = mem_db.create_table(
+        "non_empty_custom_stop_words", [{"text": "the lance database"}]
+    )
+    custom.create_index(
+        "text",
+        config=FTS(stem=False, custom_stop_words=["lance"]),
+    )
+    assert [token.text for token in custom.tokenize("the lance", column="text")] == [
+        "the"
+    ]
+    assert len(custom.search("the", query_type="fts").to_list()) == 1
+    assert custom.search("lance", query_type="fts").to_list() == []
+
+    disabled = mem_db.create_table(
+        "disabled_custom_stop_words", [{"text": "the lance database"}]
+    )
+    disabled.create_index(
+        "text",
+        config=FTS(
+            stem=False,
+            remove_stop_words=False,
+            custom_stop_words=["lance"],
+        ),
+    )
+    assert [token.text for token in disabled.tokenize("the lance", column="text")] == [
+        "the",
+        "lance",
+    ]
+    assert len(disabled.search("lance", query_type="fts").to_list()) == 1
+
+
+def test_local_fts_rejects_custom_stop_words_source(mem_db: DBConnection):
+    table = mem_db.create_table("source_stop_words", [{"text": "lance database"}])
+    with pytest.raises(RuntimeError, match="only supported for remote tables"):
+        table.create_index(
+            "text",
+            config=FTS(
+                custom_stop_words_source=FileStopWordsSource(
+                    "file:///tmp/stop-words.txt"
+                )
+            ),
+        )
+
+
+def test_fts_rejects_both_custom_stop_words_forms():
+    with pytest.raises(ValueError, match="mutually exclusive"):
+        FTS(
+            custom_stop_words=[],
+            custom_stop_words_source=FileStopWordsSource("s3://bucket/stop-words.txt"),
+        )
+
+
 def test_search_fts(table):
     table.create_fts_index("text")
     results = table.search("puppy").select(["id", "text"]).limit(5).to_list()

python/python/tests/test_remote_db.py

@@ -868,7 +868,15 @@ def test_remote_create_index_new_api():
             request.send_response(404)
             request.end_headers()
 
-    from lancedb.index import BTree, FTS, IvfPq, IvfRq
+    from lancedb.index import (
+        BTree,
+        FTS,
+        FileStopWordsSource,
+        InlineStopWordsSource,
+        IvfPq,
+        IvfRq,
+        TableStopWordsSource,
+    )
 
     with mock_lancedb_connection(handler) as db:
         table = db.create_table("test", [{"id": 1}])
@@ -881,6 +889,28 @@ def test_remote_create_index_new_api():
             table.create_index("vector", config=IvfPq(distance_type="l2"))
             table.create_index("category", config=BTree())
             table.create_index("text", config=FTS(block_size=256))
+            table.create_index("text", config=FTS(custom_stop_words=[]))
+            table.create_index(
+                "text", config=FTS(custom_stop_words=["lance", "database"])
+            )
+            table.create_index(
+                "text",
+                config=FTS(custom_stop_words_source=InlineStopWordsSource(["lance"])),
+            )
+            table.create_index(
+                "text",
+                config=FTS(
+                    custom_stop_words_source=FileStopWordsSource(
+                        "s3://bucket/stop-words.txt"
+                    )
+                ),
+            )
+            table.create_index(
+                "text",
+                config=FTS(
+                    custom_stop_words_source=TableStopWordsSource("stop_words", "word")
+                ),
+            )
             # IvfRq via new API
             table.create_index("vector", config=IvfRq(distance_type="l2"))
 
@@ -892,15 +922,36 @@ def test_remote_create_index_new_api():
                 num_partitions=8,
             )
 
-        assert len(received_requests) == 5
+        assert len(received_requests) == 10
         assert [req["column"] for req in received_requests] == [
             "vector",
             "category",
             "text",
+            "text",
+            "text",
+            "text",
+            "text",
+            "text",
             "vector",
             "vector",
         ]
         assert received_requests[2]["block_size"] == 256
+        assert received_requests[2]["custom_stop_words"] is None
+        assert received_requests[3]["custom_stop_words"] == []
+        assert received_requests[4]["custom_stop_words"] == ["lance", "database"]
+        assert received_requests[5]["custom_stop_words_source"] == {
+            "type": "inline",
+            "words": ["lance"],
+        }
+        assert received_requests[6]["custom_stop_words_source"] == {
+            "type": "file",
+            "uri": "s3://bucket/stop-words.txt",
+        }
+        assert received_requests[7]["custom_stop_words_source"] == {
+            "type": "table",
+            "table": "stop_words",
+            "column": "word",
+        }
 
 
 def test_table_wait_for_index_timeout():

python/src/index.rs

@@ -8,7 +8,7 @@ use lancedb::index::vector::{
 };
 use lancedb::index::{
     Index as LanceDbIndex,
-    scalar::{BTreeIndexBuilder, FmIndexBuilder, FtsIndexBuilder},
+    scalar::{BTreeIndexBuilder, CustomStopWordsSource, FmIndexBuilder, FtsIndexBuilder},
 };
 use pyo3::IntoPyObject;
 use pyo3::types::PyStringMethods;
@@ -33,14 +33,42 @@ pub fn class_name(ob: &'_ Bound<'_, PyAny>) -> PyResult<String> {
     }
 }
 
-pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<LanceDbIndex> {
+fn extract_custom_stop_words_source(
+    source: &Bound<'_, PyAny>,
+) -> PyResult<Option<CustomStopWordsSource>> {
+    let source = source.getattr("custom_stop_words_source")?;
+    if source.is_none() {
+        return Ok(None);
+    }
+    let source_type: String = source.getattr("type")?.extract()?;
+    match source_type.as_str() {
+        "inline" => Ok(Some(CustomStopWordsSource::Inline {
+            words: source.getattr("words")?.extract()?,
+        })),
+        "file" => Ok(Some(CustomStopWordsSource::File {
+            uri: source.getattr("uri")?.extract()?,
+        })),
+        "table" => Ok(Some(CustomStopWordsSource::Table {
+            table: source.getattr("table")?.extract()?,
+            column: source.getattr("column")?.extract()?,
+        })),
+        other => Err(PyValueError::new_err(format!(
+            "invalid custom_stop_words_source type '{other}'; expected inline, file, or table"
+        ))),
+    }
+}
+
+pub fn extract_index_params(
+    source: &Option<Bound<'_, PyAny>>,
+) -> PyResult<(LanceDbIndex, Option<CustomStopWordsSource>)> {
     if let Some(source) = source {
         match class_name(source)?.as_str() {
-            "BTree" => Ok(LanceDbIndex::BTree(BTreeIndexBuilder::default())),
-            "Bitmap" => Ok(LanceDbIndex::Bitmap(Default::default())),
-            "LabelList" => Ok(LanceDbIndex::LabelList(Default::default())),
-            "Fm" => Ok(LanceDbIndex::Fm(FmIndexBuilder::default())),
+            "BTree" => Ok((LanceDbIndex::BTree(BTreeIndexBuilder::default()), None)),
+            "Bitmap" => Ok((LanceDbIndex::Bitmap(Default::default()), None)),
+            "LabelList" => Ok((LanceDbIndex::LabelList(Default::default()), None)),
+            "Fm" => Ok((LanceDbIndex::Fm(FmIndexBuilder::default()), None)),
             "FTS" => {
+                let custom_stop_words_source = extract_custom_stop_words_source(source)?;
                 let params = source.extract::<FtsParams>()?;
                 let inner_opts = FtsIndexBuilder::default()
                     .base_tokenizer(params.base_tokenizer)
@@ -55,6 +83,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     .lower_case(params.lower_case)
                     .max_token_length(params.max_token_length)
                     .remove_stop_words(params.remove_stop_words)
+                    .custom_stop_words(params.custom_stop_words)
                     .stem(params.stem)
                     .ascii_folding(params.ascii_folding)
                     .ngram_min_length(params.ngram_min_length)
@@ -63,7 +92,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                 let inner_opts = inner_opts
                     .block_size(params.block_size)
                     .map_err(|err| PyValueError::new_err(err.to_string()))?;
-                Ok(LanceDbIndex::FTS(inner_opts))
+                Ok((LanceDbIndex::FTS(inner_opts), custom_stop_words_source))
             }
             "IvfFlat" => {
                 let params = source.extract::<IvfFlatParams>()?;
@@ -79,7 +108,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     ivf_flat_builder =
                         ivf_flat_builder.target_partition_size(target_partition_size);
                 }
-                Ok(LanceDbIndex::IvfFlat(ivf_flat_builder))
+                Ok((LanceDbIndex::IvfFlat(ivf_flat_builder), None))
             }
             "IvfPq" => {
                 let params = source.extract::<IvfPqParams>()?;
@@ -98,7 +127,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                 if let Some(num_sub_vectors) = params.num_sub_vectors {
                     ivf_pq_builder = ivf_pq_builder.num_sub_vectors(num_sub_vectors);
                 }
-                Ok(LanceDbIndex::IvfPq(ivf_pq_builder))
+                Ok((LanceDbIndex::IvfPq(ivf_pq_builder), None))
             }
             "IvfSq" => {
                 let params = source.extract::<IvfSqParams>()?;
@@ -113,7 +142,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                 if let Some(target_partition_size) = params.target_partition_size {
                     ivf_sq_builder = ivf_sq_builder.target_partition_size(target_partition_size);
                 }
-                Ok(LanceDbIndex::IvfSq(ivf_sq_builder))
+                Ok((LanceDbIndex::IvfSq(ivf_sq_builder), None))
             }
             "IvfRq" => {
                 let params = source.extract::<IvfRqParams>()?;
@@ -129,7 +158,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                 if let Some(target_partition_size) = params.target_partition_size {
                     ivf_rq_builder = ivf_rq_builder.target_partition_size(target_partition_size);
                 }
-                Ok(LanceDbIndex::IvfRq(ivf_rq_builder))
+                Ok((LanceDbIndex::IvfRq(ivf_rq_builder), None))
             }
             "HnswPq" => {
                 let params = source.extract::<IvfHnswPqParams>()?;
@@ -150,7 +179,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                 if let Some(num_sub_vectors) = params.num_sub_vectors {
                     hnsw_pq_builder = hnsw_pq_builder.num_sub_vectors(num_sub_vectors);
                 }
-                Ok(LanceDbIndex::IvfHnswPq(hnsw_pq_builder))
+                Ok((LanceDbIndex::IvfHnswPq(hnsw_pq_builder), None))
             }
             "HnswSq" => {
                 let params = source.extract::<IvfHnswSqParams>()?;
@@ -167,7 +196,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                 if let Some(target_partition_size) = params.target_partition_size {
                     hnsw_sq_builder = hnsw_sq_builder.target_partition_size(target_partition_size);
                 }
-                Ok(LanceDbIndex::IvfHnswSq(hnsw_sq_builder))
+                Ok((LanceDbIndex::IvfHnswSq(hnsw_sq_builder), None))
             }
             "HnswFlat" => {
                 let params = source.extract::<IvfHnswFlatParams>()?;
@@ -185,7 +214,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     hnsw_flat_builder =
                         hnsw_flat_builder.target_partition_size(target_partition_size);
                 }
-                Ok(LanceDbIndex::IvfHnswFlat(hnsw_flat_builder))
+                Ok((LanceDbIndex::IvfHnswFlat(hnsw_flat_builder), None))
             }
             not_supported => Err(PyValueError::new_err(format!(
                 "Invalid index type '{}'.  Must be one of BTree, Bitmap, LabelList, Fm, FTS, IvfPq, IvfSq, IvfHnswPq, IvfHnswSq, or IvfHnswFlat",
@@ -193,7 +222,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
             ))),
         }
     } else {
-        Ok(LanceDbIndex::Auto)
+        Ok((LanceDbIndex::Auto, None))
     }
 }
 
@@ -206,6 +235,7 @@ struct FtsParams {
     lower_case: bool,
     stem: bool,
     remove_stop_words: bool,
+    custom_stop_words: Option<Vec<String>>,
     ascii_folding: bool,
     ngram_min_length: u32,
     ngram_max_length: u32,

python/src/table.rs

@@ -781,7 +781,7 @@ impl Table {
         name: Option<String>,
         train: Option<bool>,
     ) -> PyResult<Bound<'a, PyAny>> {
-        let index = extract_index_params(&index)?;
+        let (index, custom_stop_words_source) = extract_index_params(&index)?;
         let timeout = wait_timeout.map(|t| t.extract::<std::time::Duration>().unwrap());
         let mut op = self_
             .inner_ref()?
@@ -795,6 +795,9 @@ impl Table {
         if let Some(train) = train {
             op = op.train(train);
         }
+        if let Some(source) = custom_stop_words_source {
+            op = op.custom_stop_words_source(source);
+        }
 
         future_into_py(self_.py(), async move {
             op.execute().await.infer_error()?;