feat(fts): support custom stop-word lists (#3734)

## What

Expose custom FTS stop-word lists in the Python and TypeScript public
APIs, including their standalone tokenize helpers and remote index
creation.

This PR supports concrete string lists only. It does not add file or
LanceDB-table stop-word sources.

## Why

Rust already exposes Lance's custom stop-word list option. The Python
and TypeScript APIs did not pass it through, and local index details did
not retain the full tokenizer parameters needed by index-backed
tokenization after reopening a table.

## How

- Add `custom_stop_words` / `customStopWords` to the Python and
TypeScript FTS and tokenize options.
- Preserve `None` / `undefined`, empty lists, and list contents without
normalization.
- Load the persisted FTS segment parameters when returning local index
details.
- Serialize the concrete list in remote create-index requests.
- Keep Python and TypeScript tests thin; behavior, persistence, query
tokenization, and remote JSON coverage live primarily in Rust.

## Validation

- `cargo check --quiet --features remote --tests --examples`
- `cargo clippy --quiet --features remote --tests --examples`
- `cargo test --quiet --features remote --tests`
- Python extension rebuild with `uv` and `maturin`
- Targeted Python tests: 4 passed
- Python `ruff format --check` and `ruff check`
- TypeScript build, typecheck, Biome lint, generated docs, and targeted
tests

---------

Co-authored-by: Yang Cen <yangcen@Yangs-Mac-mini.local>

docs/openapi.yml

@@ -453,6 +453,16 @@ paths:
                     The metric type to use for the index. l2, Cosine, Dot are supported.
                 index_type:
                   type: string
+                custom_stop_words:
+                  type: [array, "null"]
+                  items:
+                    type: string
+                  description: |
+                    The custom stop-word list for an FTS index. A non-null
+                    array replaces the language's built-in stop-word list and is only
+                    applied when remove_stop_words is enabled. Null uses the built-in
+                    language list, while an empty array explicitly replaces it with no
+                    stop words.
       responses:
         "200":
           description: Index successfully created
@@ -510,4 +520,4 @@ paths:
         "401":
           $ref: "#/components/responses/unauthorized"
         "404":
-          $ref: "#/components/responses/not_found"
+          $ref: "#/components/responses/not_found"

docs/src/js/interfaces/FtsOptions.md

@@ -56,6 +56,21 @@ the experimental FTS V3 format and may introduce breaking changes.
 
 ***
 
+### customStopWords?
+
+```ts
+optional customStopWords: string[];
+```
+
+Custom stop words that replace the built-in list for `language`.
+
+This option only affects tokenization when `removeStopWords` is true.
+
+`undefined` keeps the built-in language list. An empty array explicitly
+replaces it with no stop words.
+
+***
+
 ### language?
 
 ```ts

docs/src/js/interfaces/TokenizeOptions.md

@@ -30,6 +30,21 @@ The tokenizer to use. The default is "simple".
 
 ***
 
+### customStopWords?
+
+```ts
+optional customStopWords: string[];
+```
+
+Custom stop words that replace the built-in list for `language`.
+
+This option only affects tokenization when `removeStopWords` is true.
+
+`undefined` keeps the built-in language list. An empty array explicitly
+replaces it with no stop words.
+
+***
+
 ### language?
 
 ```ts

docs/src/python/python.md

@@ -94,9 +94,33 @@ of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and
 
 ## Full text search
 
-Use [lancedb.table.Table.create_fts_index][] for the synchronous API or
-[lancedb.table.AsyncTable.create_index][] with [lancedb.index.FTS][] for the
-asynchronous API.
+Pass `custom_stop_words` to [lancedb.index.FTS][]:
+
+```python
+from lancedb.index import FTS
+
+table.create_index(
+    "text",
+    config=FTS(remove_stop_words=True, custom_stop_words=["acme", "internal"]),
+)
+```
+
+The list replaces the built-in stop words and is used only when
+`remove_stop_words=True`:
+
+- `custom_stop_words=None` uses the built-in list for `language`.
+- `custom_stop_words=[]` removes no words.
+- Values are passed through without trimming, lowercasing, or other rewriting.
+
+The same option is available on `lancedb.tokenize(...)` and the deprecated
+[lancedb.table.Table.create_fts_index][] compatibility helper:
+
+```python
+import lancedb
+
+tokens = list(lancedb.tokenize("acme makes searchable data",
+                               custom_stop_words=["acme"]))
+```
 
 ::: lancedb.index.FTS
 

nodejs/__test__/remote.test.ts

@@ -226,7 +226,7 @@ describe("remote connection", () => {
     );
   });
 
-  it("sends the FTS posting block size to remote tables", async () => {
+  it("sends FTS options to remote tables", async () => {
     let createIndexBody: Record<string, unknown> | undefined;
 
     await withMockDatabase(
@@ -264,7 +264,11 @@ describe("remote connection", () => {
       async (db) => {
         const table = await db.openTable("t");
         await table.createIndex("text", {
-          config: Index.fts({ blockSize: 256 }),
+          config: Index.fts({
+            blockSize: 256,
+            removeStopWords: true,
+            customStopWords: ["the"],
+          }),
         });
       },
     );
@@ -272,6 +276,7 @@ describe("remote connection", () => {
     expect(createIndexBody?.["column"]).toBe("text");
     expect(createIndexBody?.["index_type"]).toBe("FTS");
     expect(createIndexBody?.["block_size"]).toBe(256);
+    expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]);
   });
 
   it("diffs and merges remote branches", async () => {

nodejs/__test__/table.test.ts

@@ -2769,6 +2769,15 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
   },
 );
 
+test("tokenize supports custom stop words", async () => {
+  const tokens = await tokenize("the lance data", {
+    stem: false,
+    removeStopWords: true,
+    customStopWords: ["lance"],
+  });
+  expect(tokens.map((token) => token.text)).toEqual(["the", "data"]);
+});
+
 describe("when calling explainPlan", () => {
   let tmpDir: tmp.DirResult;
   let table: Table;

nodejs/examples/full_text_search.test.ts

@@ -29,8 +29,14 @@ test("full text search", async () => {
     const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
 
     await tbl.createIndex("doc", {
-      config: lancedb.Index.fts(),
+      config: lancedb.Index.fts({
+        stem: false,
+        removeStopWords: true,
+        customStopWords: ["banana"],
+      }),
     });
+    const tokens = await tbl.tokenize("apple banana", { column: "doc" });
+    expect(tokens.map((token) => token.text)).toEqual(["apple"]);
 
     // --8<-- [start:full_text_search]
     const result = await tbl

nodejs/lancedb/index.ts

@@ -194,6 +194,16 @@ export interface TokenizeOptions {
   /** Whether to remove stop words. */
   removeStopWords?: boolean;
 
+  /**
+   * Custom stop words that replace the built-in list for `language`.
+   *
+   * This option only affects tokenization when `removeStopWords` is true.
+   *
+   * `undefined` keeps the built-in language list. An empty array explicitly
+   * replaces it with no stop words.
+   */
+  customStopWords?: string[];
+
   /** Whether to fold ASCII characters. */
   asciiFolding?: boolean;
 
@@ -225,6 +235,7 @@ export async function tokenize(
     options?.lowercase,
     options?.stem,
     options?.removeStopWords,
+    options?.customStopWords,
     options?.asciiFolding,
     options?.ngramMinLength,
     options?.ngramMaxLength,

nodejs/lancedb/indices.ts

@@ -553,6 +553,16 @@ export interface FtsOptions {
    */
   removeStopWords?: boolean;
 
+  /**
+   * Custom stop words that replace the built-in list for `language`.
+   *
+   * This option only affects tokenization when `removeStopWords` is true.
+   *
+   * `undefined` keeps the built-in language list. An empty array explicitly
+   * replaces it with no stop words.
+   */
+  customStopWords?: string[];
+
   /**
    * whether to remove punctuation
    */
@@ -755,6 +765,7 @@ export class Index {
         options?.lowercase,
         options?.stem,
         options?.removeStopWords,
+        options?.customStopWords,
         options?.asciiFolding,
         options?.ngramMinLength,
         options?.ngramMaxLength,

nodejs/src/index.rs

@@ -43,6 +43,7 @@ pub fn tokenize(
     lower_case: Option<bool>,
     stem: Option<bool>,
     remove_stop_words: Option<bool>,
+    custom_stop_words: Option<Vec<String>>,
     ascii_folding: Option<bool>,
     ngram_min_length: Option<u32>,
     ngram_max_length: Option<u32>,
@@ -72,6 +73,7 @@ pub fn tokenize(
     if let Some(remove_stop_words) = remove_stop_words {
         opts = opts.remove_stop_words(remove_stop_words);
     }
+    opts = opts.custom_stop_words(custom_stop_words);
     if let Some(ascii_folding) = ascii_folding {
         opts = opts.ascii_folding(ascii_folding);
     }
@@ -222,6 +224,7 @@ impl Index {
         lower_case: Option<bool>,
         stem: Option<bool>,
         remove_stop_words: Option<bool>,
+        custom_stop_words: Option<Vec<String>>,
         ascii_folding: Option<bool>,
         ngram_min_length: Option<u32>,
         ngram_max_length: Option<u32>,
@@ -250,6 +253,7 @@ impl Index {
         if let Some(remove_stop_words) = remove_stop_words {
             opts = opts.remove_stop_words(remove_stop_words);
         }
+        opts = opts.custom_stop_words(custom_stop_words);
         if let Some(ascii_folding) = ascii_folding {
             opts = opts.ascii_folding(ascii_folding);
         }

python/python/lancedb/__init__.py

@@ -258,6 +258,7 @@ def tokenize(
     lower_case: bool = True,
     stem: bool = True,
     remove_stop_words: bool = True,
+    custom_stop_words: Optional[List[str]] = None,
     ascii_folding: bool = True,
     ngram_min_length: int = 3,
     ngram_max_length: int = 3,
@@ -265,9 +266,10 @@ def tokenize(
 ) -> Iterable[FtsToken]:
     """Tokenize a full-text search query using an explicit tokenizer.
 
-    This does not require a table or FTS index. The tokenizer options match
-    :class:`lancedb.index.FTS`.
+    This does not require an FTS index. The tokenizer options match
+    :class:`lancedb.index.FTS`. ``custom_stop_words`` accepts a list of strings.
     """
+
     return _tokenize(
         query,
         base_tokenizer=base_tokenizer,
@@ -276,6 +278,7 @@ def tokenize(
         lower_case=lower_case,
         stem=stem,
         remove_stop_words=remove_stop_words,
+        custom_stop_words=custom_stop_words,
         ascii_folding=ascii_folding,
         ngram_min_length=ngram_min_length,
         ngram_max_length=ngram_max_length,

python/python/lancedb/_lancedb.pyi

@@ -59,6 +59,7 @@ def tokenize(
     lower_case: bool = True,
     stem: bool = True,
     remove_stop_words: bool = True,
+    custom_stop_words: Optional[List[str]] = None,
     ascii_folding: bool = True,
     ngram_min_length: int = 3,
     ngram_max_length: int = 3,

python/python/lancedb/index.py

@@ -2,7 +2,7 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 from dataclasses import dataclass
-from typing import Literal, Optional
+from typing import List, Literal, Optional
 
 from ._lancedb import (
     IndexConfig,
@@ -151,6 +151,11 @@ class FTS:
     remove_stop_words : bool, default True
         Whether to remove stop words. Stop words are common words that are often
         removed from text before indexing. For example, in English "the" and "and".
+    custom_stop_words : list of str, optional
+        Custom words replace the built-in language stop words
+        and only take effect when ``remove_stop_words`` is True. ``None`` uses
+        the built-in language list, while an empty list explicitly uses no
+        stop words.
     ascii_folding : bool, default True
         Whether to fold ASCII characters. This converts accented characters to
         their ASCII equivalent. For example, "café" would be converted to "cafe".
@@ -179,6 +184,7 @@ class FTS:
     ngram_max_length: int = 3
     prefix_only: bool = False
     block_size: int = 128
+    custom_stop_words: Optional[List[str]] = None
 
 
 @dataclass

python/python/lancedb/remote/table.py

@@ -340,6 +340,7 @@ class RemoteTable(Table):
         lower_case: bool = True,
         stem: bool = True,
         remove_stop_words: bool = True,
+        custom_stop_words: Optional[List[str]] = None,
         ascii_folding: bool = True,
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
@@ -361,6 +362,7 @@ class RemoteTable(Table):
             lower_case=lower_case,
             stem=stem,
             remove_stop_words=remove_stop_words,
+            custom_stop_words=custom_stop_words,
             ascii_folding=ascii_folding,
             ngram_min_length=ngram_min_length,
             ngram_max_length=ngram_max_length,

python/python/lancedb/table.py

@@ -1103,6 +1103,7 @@ class Table(ABC):
         lower_case: bool = True,
         stem: bool = True,
         remove_stop_words: bool = True,
+        custom_stop_words: Optional[List[str]] = None,
         ascii_folding: bool = True,
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
@@ -1170,6 +1171,9 @@ class Table(ABC):
         remove_stop_words : bool, default True
             Whether to remove stop words. Stop words are common words that are often
             removed from text before indexing. For example, in English "the" and "and".
+        custom_stop_words : list of str, optional
+            Custom words that replace the built-in language stop words. ``None``
+            uses the built-in list; an empty list explicitly uses no stop words.
         ascii_folding : bool, default True
             Whether to fold ASCII characters. This converts accented characters to
             their ASCII equivalent. For example, "café" would be converted to "cafe".
@@ -3055,6 +3059,7 @@ class LanceTable(Table):
         lower_case: bool = True,
         stem: bool = True,
         remove_stop_words: bool = True,
+        custom_stop_words: Optional[List[str]] = None,
         ascii_folding: bool = True,
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
@@ -3101,6 +3106,7 @@ class LanceTable(Table):
                 "lower_case": lower_case,
                 "stem": stem,
                 "remove_stop_words": remove_stop_words,
+                "custom_stop_words": custom_stop_words,
                 "ascii_folding": ascii_folding,
                 "ngram_min_length": ngram_min_length,
                 "ngram_max_length": ngram_max_length,
@@ -3108,6 +3114,7 @@ class LanceTable(Table):
             }
         else:
             tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
+            tokenizer_configs["custom_stop_words"] = custom_stop_words
 
         config = FTS(block_size=block_size, **tokenizer_configs)
 

python/python/tests/test_fts.py

@@ -219,11 +219,13 @@ def test_create_inverted_index(table, with_position):
         table.create_fts_index(
             "text",
             with_position=with_position,
+            custom_stop_words=["puppy"],
             name="custom_fts_index",
         )
     indices = table.list_indices()
     fts_indices = [i for i in indices if i.index_type == "FTS"]
     assert any(i.name == "custom_fts_index" for i in fts_indices)
+    assert fts_indices[0].index_details["custom_stop_words"] == ["puppy"]
 
 
 @pytest.mark.parametrize("block_size", [128, 256])
@@ -243,6 +245,24 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
         table.create_index("text", config=FTS(block_size=129))
 
 
+def test_custom_stop_words_list(table):
+    table.create_index(
+        "text",
+        config=FTS(stem=False, custom_stop_words=["lance"]),
+    )
+
+    assert table.list_indices()[0].index_details["custom_stop_words"] == ["lance"]
+    tokens = table.tokenize("the lance data", column="text")
+    assert [token.text for token in tokens] == ["the", "data"]
+    empty_tokens = ldb.tokenize("the lance data", stem=False, custom_stop_words=[])
+    assert [token.text for token in empty_tokens] == ["the", "lance", "data"]
+    with pytest.raises(TypeError, match=r"custom_stop_words.*int"):
+        ldb.tokenize(
+            "the lance data",
+            custom_stop_words=["lance", 42],
+        )
+
+
 def test_search_fts(table):
     table.create_fts_index("text")
     results = table.search("puppy").select(["id", "text"]).limit(5).to_list()

python/python/tests/test_remote_db.py

@@ -771,6 +771,7 @@ def test_table_create_indices():
                 "text",
                 wait_timeout=timedelta(seconds=2),
                 block_size=256,
+                custom_stop_words=["cloud"],
                 name="custom_fts_idx",
             )
 
@@ -795,6 +796,7 @@ def test_table_create_indices():
         assert "name" in fts_req
         assert fts_req["name"] == "custom_fts_idx"
         assert fts_req["block_size"] == 256
+        assert fts_req["custom_stop_words"] == ["cloud"]
 
         # Check vector index request has custom name
         vector_req = received_requests[2]

python/src/index.rs

@@ -59,7 +59,8 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     .ascii_folding(params.ascii_folding)
                     .ngram_min_length(params.ngram_min_length)
                     .ngram_max_length(params.ngram_max_length)
-                    .ngram_prefix_only(params.prefix_only);
+                    .ngram_prefix_only(params.prefix_only)
+                    .custom_stop_words(params.custom_stop_words);
                 let inner_opts = inner_opts
                     .block_size(params.block_size)
                     .map_err(|err| PyValueError::new_err(err.to_string()))?;
@@ -206,6 +207,7 @@ struct FtsParams {
     lower_case: bool,
     stem: bool,
     remove_stop_words: bool,
+    custom_stop_words: Option<Vec<String>>,
     ascii_folding: bool,
     ngram_min_length: u32,
     ngram_max_length: u32,

python/src/table.rs

@@ -520,6 +520,7 @@ impl From<LanceDbFtsToken> for FtsToken {
     lower_case = true,
     stem = true,
     remove_stop_words = true,
+    custom_stop_words = None,
     ascii_folding = true,
     ngram_min_length = 3,
     ngram_max_length = 3,
@@ -534,6 +535,7 @@ pub fn tokenize(
     lower_case: bool,
     stem: bool,
     remove_stop_words: bool,
+    custom_stop_words: Option<Vec<String>>,
     ascii_folding: bool,
     ngram_min_length: u32,
     ngram_max_length: u32,
@@ -555,7 +557,8 @@ pub fn tokenize(
         .ascii_folding(ascii_folding)
         .ngram_min_length(ngram_min_length)
         .ngram_max_length(ngram_max_length)
-        .ngram_prefix_only(prefix_only);
+        .ngram_prefix_only(prefix_only)
+        .custom_stop_words(custom_stop_words);
     let tokens = lancedb_tokenize(&query, &params).infer_error()?;
     Ok(tokens.into_iter().map(FtsToken::from).collect())
 }

rust/lancedb/examples/full_text_search.rs

@@ -76,7 +76,12 @@ async fn create_table(db: &Connection) -> Result<Table> {
 
 async fn create_index(table: &Table) -> Result<()> {
     table
-        .create_index(&["doc"], Index::FTS(FtsIndexBuilder::default()))
+        .create_index(
+            &["doc"],
+            Index::FTS(
+                FtsIndexBuilder::default().custom_stop_words(Some(vec!["example".to_owned()])),
+            ),
+        )
         .execute()
         .await?;
     Ok(())

rust/lancedb/src/remote/table.rs

@@ -4553,6 +4553,19 @@ mod tests {
                 },
                 Index::FTS(InvertedIndexParams::default().block_size(256).unwrap()),
             ),
+            (
+                "FTS",
+                {
+                    let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
+                    body["custom_stop_words"] = json!(["cat", " cat ", "CAT"]);
+                    body
+                },
+                Index::FTS(InvertedIndexParams::default().custom_stop_words(Some(vec![
+                    "cat".to_string(),
+                    " cat ".to_string(),
+                    "CAT".to_string(),
+                ]))),
+            ),
         ];
 
         for (index_type, expected_body, index) in cases {
@@ -5084,8 +5097,9 @@ mod tests {
             "max_token_length": 40,
             "lower_case": true,
             "stem": false,
-            "remove_stop_words": false,
+            "remove_stop_words": true,
             "ascii_folding": true,
+            "custom_stop_words": ["hello"],
         })
         .to_string();
         let table = Table::new_with_handler("my_table", move |request| {
@@ -5123,10 +5137,6 @@ mod tests {
         assert_eq!(
             tokens,
             vec![
-                FtsToken {
-                    text: "hello".to_string(),
-                    position: 0,
-                },
                 FtsToken {
                     text: "こんにちは".to_string(),
                     position: 1,

rust/lancedb/src/table.rs

@@ -21,6 +21,7 @@ use lance::dataset::WriteMode;
 use lance::dataset::builder::DatasetBuilder;
 use lance::dataset::{InsertBuilder, WriteParams};
 use lance::index::DatasetIndexExt;
+use lance::index::scalar::load_segment_params;
 use lance::io::{ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_index::IndexCriteria;
@@ -3193,10 +3194,9 @@ impl BaseTable for NativeTable {
     async fn list_indices(&self) -> Result<Vec<IndexConfig>> {
         let dataset = self.dataset.get().await?;
         let total_rows = dataset.count_rows(None).await? as u64;
-        let indices = dataset
-            .describe_indices(None)
-            .await?
-            .into_iter()
+        let descriptions = dataset.describe_indices(None).await?;
+        let mut indices: Vec<IndexConfig> = descriptions
+            .iter()
             .filter_map(|idx_desc| {
                 let index_type: crate::index::IndexType = idx_desc
                     .index_type()
@@ -3254,6 +3254,31 @@ impl BaseTable for NativeTable {
                 })
             })
             .collect();
+
+        for index in indices
+            .iter_mut()
+            .filter(|index| index.index_type == crate::index::IndexType::FTS)
+        {
+            let Some(description) = descriptions
+                .iter()
+                .find(|description| description.name() == index.name)
+            else {
+                continue;
+            };
+            let segments = description.segments();
+            let Some(segment) = segments.first() else {
+                continue;
+            };
+            let params = load_segment_params(&dataset, segment).await?;
+            let details = serde_json::to_string(&params).map_err(|source| Error::Other {
+                message: format!(
+                    "Failed to serialize full text search configuration for index '{}'",
+                    index.name
+                ),
+                source: Some(Box::new(source)),
+            })?;
+            index.index_details = Some(details);
+        }
         Ok(indices)
     }
 

rust/lancedb/src/table/create_index.rs

@@ -1366,11 +1366,19 @@ mod tests {
         table
             .create_index(
                 &["text"],
-                Index::FTS(FtsIndexBuilder::default().block_size(256).unwrap()),
+                Index::FTS(
+                    FtsIndexBuilder::default()
+                        .stem(false)
+                        .custom_stop_words(Some(vec!["cat".to_string()]))
+                        .block_size(256)
+                        .unwrap(),
+                ),
             )
             .execute()
             .await
             .unwrap();
+        drop(table);
+        let table = conn.open_table("test_bitmap").execute().await.unwrap();
         let index_configs = table.list_indices().await.unwrap();
         assert_eq!(index_configs.len(), 1);
         let index = index_configs.into_iter().next().unwrap();
@@ -1381,6 +1389,32 @@ mod tests {
         let index_params: FtsIndexBuilder =
             serde_json::from_str(index.index_details.as_deref().unwrap()).unwrap();
         assert_eq!(index_params.posting_block_size(), 256);
+        assert_eq!(
+            serde_json::to_value(&index_params).unwrap()["custom_stop_words"],
+            serde_json::json!(["cat"])
+        );
+        assert_eq!(
+            table
+                .tokenize("cat dog", "text_idx")
+                .await
+                .unwrap()
+                .into_iter()
+                .map(|token| token.text)
+                .collect::<Vec<_>>(),
+            vec!["dog"]
+        );
+
+        let batches = table
+            .query()
+            .full_text_search(FullTextSearchQuery::new("cat dog".to_string()))
+            .limit(120)
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 40);
 
         let num_rows = 120;
         let stats = table.index_stats("text_idx").await.unwrap().unwrap();