From f7feed48c31e1683a77f7ba9cd073975521d460b Mon Sep 17 00:00:00 2001 From: Yang Cen Date: Wed, 29 Jul 2026 17:40:12 +0800 Subject: [PATCH] feat(fts): support custom stop-word lists (#3734) ## What Expose custom FTS stop-word lists in the Python and TypeScript public APIs, including their standalone tokenize helpers and remote index creation. This PR supports concrete string lists only. It does not add file or LanceDB-table stop-word sources. ## Why Rust already exposes Lance's custom stop-word list option. The Python and TypeScript APIs did not pass it through, and local index details did not retain the full tokenizer parameters needed by index-backed tokenization after reopening a table. ## How - Add `custom_stop_words` / `customStopWords` to the Python and TypeScript FTS and tokenize options. - Preserve `None` / `undefined`, empty lists, and list contents without normalization. - Load the persisted FTS segment parameters when returning local index details. - Serialize the concrete list in remote create-index requests. - Keep Python and TypeScript tests thin; behavior, persistence, query tokenization, and remote JSON coverage live primarily in Rust. ## Validation - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` - `cargo test --quiet --features remote --tests` - Python extension rebuild with `uv` and `maturin` - Targeted Python tests: 4 passed - Python `ruff format --check` and `ruff check` - TypeScript build, typecheck, Biome lint, generated docs, and targeted tests --------- Co-authored-by: Yang Cen --- docs/openapi.yml | 12 +++++++- docs/src/js/interfaces/FtsOptions.md | 15 ++++++++++ docs/src/js/interfaces/TokenizeOptions.md | 15 ++++++++++ docs/src/python/python.md | 30 +++++++++++++++++-- nodejs/__test__/remote.test.ts | 9 ++++-- nodejs/__test__/table.test.ts | 9 ++++++ nodejs/examples/full_text_search.test.ts | 8 ++++- nodejs/lancedb/index.ts | 11 +++++++ nodejs/lancedb/indices.ts | 11 +++++++ nodejs/src/index.rs | 4 +++ python/python/lancedb/__init__.py | 7 +++-- python/python/lancedb/_lancedb.pyi | 1 + python/python/lancedb/index.py | 8 ++++- python/python/lancedb/remote/table.py | 2 ++ python/python/lancedb/table.py | 7 +++++ python/python/tests/test_fts.py | 20 +++++++++++++ python/python/tests/test_remote_db.py | 2 ++ python/src/index.rs | 4 ++- python/src/table.rs | 5 +++- rust/lancedb/examples/full_text_search.rs | 7 ++++- rust/lancedb/src/remote/table.rs | 20 +++++++++---- rust/lancedb/src/table.rs | 33 ++++++++++++++++++--- rust/lancedb/src/table/create_index.rs | 36 ++++++++++++++++++++++- 23 files changed, 253 insertions(+), 23 deletions(-) diff --git a/docs/openapi.yml b/docs/openapi.yml index b07e47e04..2f9ae7d99 100644 --- a/docs/openapi.yml +++ b/docs/openapi.yml @@ -453,6 +453,16 @@ paths: The metric type to use for the index. l2, Cosine, Dot are supported. index_type: type: string + custom_stop_words: + type: [array, "null"] + items: + type: string + description: | + The custom stop-word list for an FTS index. A non-null + array replaces the language's built-in stop-word list and is only + applied when remove_stop_words is enabled. Null uses the built-in + language list, while an empty array explicitly replaces it with no + stop words. responses: "200": description: Index successfully created @@ -510,4 +520,4 @@ paths: "401": $ref: "#/components/responses/unauthorized" "404": - $ref: "#/components/responses/not_found" \ No newline at end of file + $ref: "#/components/responses/not_found" diff --git a/docs/src/js/interfaces/FtsOptions.md b/docs/src/js/interfaces/FtsOptions.md index b8b264fed..1d3c9ff82 100644 --- a/docs/src/js/interfaces/FtsOptions.md +++ b/docs/src/js/interfaces/FtsOptions.md @@ -56,6 +56,21 @@ the experimental FTS V3 format and may introduce breaking changes. *** +### customStopWords? + +```ts +optional customStopWords: string[]; +``` + +Custom stop words that replace the built-in list for `language`. + +This option only affects tokenization when `removeStopWords` is true. + +`undefined` keeps the built-in language list. An empty array explicitly +replaces it with no stop words. + +*** + ### language? ```ts diff --git a/docs/src/js/interfaces/TokenizeOptions.md b/docs/src/js/interfaces/TokenizeOptions.md index f061356d1..8ef15d03d 100644 --- a/docs/src/js/interfaces/TokenizeOptions.md +++ b/docs/src/js/interfaces/TokenizeOptions.md @@ -30,6 +30,21 @@ The tokenizer to use. The default is "simple". *** +### customStopWords? + +```ts +optional customStopWords: string[]; +``` + +Custom stop words that replace the built-in list for `language`. + +This option only affects tokenization when `removeStopWords` is true. + +`undefined` keeps the built-in language list. An empty array explicitly +replaces it with no stop words. + +*** + ### language? ```ts diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 1e928be54..dd60451cc 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -94,9 +94,33 @@ of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and ## Full text search -Use [lancedb.table.Table.create_fts_index][] for the synchronous API or -[lancedb.table.AsyncTable.create_index][] with [lancedb.index.FTS][] for the -asynchronous API. +Pass `custom_stop_words` to [lancedb.index.FTS][]: + +```python +from lancedb.index import FTS + +table.create_index( + "text", + config=FTS(remove_stop_words=True, custom_stop_words=["acme", "internal"]), +) +``` + +The list replaces the built-in stop words and is used only when +`remove_stop_words=True`: + +- `custom_stop_words=None` uses the built-in list for `language`. +- `custom_stop_words=[]` removes no words. +- Values are passed through without trimming, lowercasing, or other rewriting. + +The same option is available on `lancedb.tokenize(...)` and the deprecated +[lancedb.table.Table.create_fts_index][] compatibility helper: + +```python +import lancedb + +tokens = list(lancedb.tokenize("acme makes searchable data", + custom_stop_words=["acme"])) +``` ::: lancedb.index.FTS diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 22562b0bc..75c5540eb 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -226,7 +226,7 @@ describe("remote connection", () => { ); }); - it("sends the FTS posting block size to remote tables", async () => { + it("sends FTS options to remote tables", async () => { let createIndexBody: Record | undefined; await withMockDatabase( @@ -264,7 +264,11 @@ describe("remote connection", () => { async (db) => { const table = await db.openTable("t"); await table.createIndex("text", { - config: Index.fts({ blockSize: 256 }), + config: Index.fts({ + blockSize: 256, + removeStopWords: true, + customStopWords: ["the"], + }), }); }, ); @@ -272,6 +276,7 @@ describe("remote connection", () => { expect(createIndexBody?.["column"]).toBe("text"); expect(createIndexBody?.["index_type"]).toBe("FTS"); expect(createIndexBody?.["block_size"]).toBe(256); + expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]); }); it("diffs and merges remote branches", async () => { diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 15248f5cf..41025047e 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2769,6 +2769,15 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( }, ); +test("tokenize supports custom stop words", async () => { + const tokens = await tokenize("the lance data", { + stem: false, + removeStopWords: true, + customStopWords: ["lance"], + }); + expect(tokens.map((token) => token.text)).toEqual(["the", "data"]); +}); + describe("when calling explainPlan", () => { let tmpDir: tmp.DirResult; let table: Table; diff --git a/nodejs/examples/full_text_search.test.ts b/nodejs/examples/full_text_search.test.ts index 9777bda7f..d1567dfbb 100644 --- a/nodejs/examples/full_text_search.test.ts +++ b/nodejs/examples/full_text_search.test.ts @@ -29,8 +29,14 @@ test("full text search", async () => { const tbl = await db.createTable("myVectors", data, { mode: "overwrite" }); await tbl.createIndex("doc", { - config: lancedb.Index.fts(), + config: lancedb.Index.fts({ + stem: false, + removeStopWords: true, + customStopWords: ["banana"], + }), }); + const tokens = await tbl.tokenize("apple banana", { column: "doc" }); + expect(tokens.map((token) => token.text)).toEqual(["apple"]); // --8<-- [start:full_text_search] const result = await tbl diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index cb91c30aa..63a4b08d0 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -194,6 +194,16 @@ export interface TokenizeOptions { /** Whether to remove stop words. */ removeStopWords?: boolean; + /** + * Custom stop words that replace the built-in list for `language`. + * + * This option only affects tokenization when `removeStopWords` is true. + * + * `undefined` keeps the built-in language list. An empty array explicitly + * replaces it with no stop words. + */ + customStopWords?: string[]; + /** Whether to fold ASCII characters. */ asciiFolding?: boolean; @@ -225,6 +235,7 @@ export async function tokenize( options?.lowercase, options?.stem, options?.removeStopWords, + options?.customStopWords, options?.asciiFolding, options?.ngramMinLength, options?.ngramMaxLength, diff --git a/nodejs/lancedb/indices.ts b/nodejs/lancedb/indices.ts index 9b438ef0f..dbeebf433 100644 --- a/nodejs/lancedb/indices.ts +++ b/nodejs/lancedb/indices.ts @@ -553,6 +553,16 @@ export interface FtsOptions { */ removeStopWords?: boolean; + /** + * Custom stop words that replace the built-in list for `language`. + * + * This option only affects tokenization when `removeStopWords` is true. + * + * `undefined` keeps the built-in language list. An empty array explicitly + * replaces it with no stop words. + */ + customStopWords?: string[]; + /** * whether to remove punctuation */ @@ -755,6 +765,7 @@ export class Index { options?.lowercase, options?.stem, options?.removeStopWords, + options?.customStopWords, options?.asciiFolding, options?.ngramMinLength, options?.ngramMaxLength, diff --git a/nodejs/src/index.rs b/nodejs/src/index.rs index 66dee7913..762f74870 100644 --- a/nodejs/src/index.rs +++ b/nodejs/src/index.rs @@ -43,6 +43,7 @@ pub fn tokenize( lower_case: Option, stem: Option, remove_stop_words: Option, + custom_stop_words: Option>, ascii_folding: Option, ngram_min_length: Option, ngram_max_length: Option, @@ -72,6 +73,7 @@ pub fn tokenize( if let Some(remove_stop_words) = remove_stop_words { opts = opts.remove_stop_words(remove_stop_words); } + opts = opts.custom_stop_words(custom_stop_words); if let Some(ascii_folding) = ascii_folding { opts = opts.ascii_folding(ascii_folding); } @@ -222,6 +224,7 @@ impl Index { lower_case: Option, stem: Option, remove_stop_words: Option, + custom_stop_words: Option>, ascii_folding: Option, ngram_min_length: Option, ngram_max_length: Option, @@ -250,6 +253,7 @@ impl Index { if let Some(remove_stop_words) = remove_stop_words { opts = opts.remove_stop_words(remove_stop_words); } + opts = opts.custom_stop_words(custom_stop_words); if let Some(ascii_folding) = ascii_folding { opts = opts.ascii_folding(ascii_folding); } diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 6f6468d9c..71349b8a5 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -258,6 +258,7 @@ def tokenize( lower_case: bool = True, stem: bool = True, remove_stop_words: bool = True, + custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, ngram_max_length: int = 3, @@ -265,9 +266,10 @@ def tokenize( ) -> Iterable[FtsToken]: """Tokenize a full-text search query using an explicit tokenizer. - This does not require a table or FTS index. The tokenizer options match - :class:`lancedb.index.FTS`. + This does not require an FTS index. The tokenizer options match + :class:`lancedb.index.FTS`. ``custom_stop_words`` accepts a list of strings. """ + return _tokenize( query, base_tokenizer=base_tokenizer, @@ -276,6 +278,7 @@ def tokenize( lower_case=lower_case, stem=stem, remove_stop_words=remove_stop_words, + custom_stop_words=custom_stop_words, ascii_folding=ascii_folding, ngram_min_length=ngram_min_length, ngram_max_length=ngram_max_length, diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 448947a29..67325e10b 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -59,6 +59,7 @@ def tokenize( lower_case: bool = True, stem: bool = True, remove_stop_words: bool = True, + custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, ngram_max_length: int = 3, diff --git a/python/python/lancedb/index.py b/python/python/lancedb/index.py index c06bb9903..5ced0600f 100644 --- a/python/python/lancedb/index.py +++ b/python/python/lancedb/index.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors from dataclasses import dataclass -from typing import Literal, Optional +from typing import List, Literal, Optional from ._lancedb import ( IndexConfig, @@ -151,6 +151,11 @@ class FTS: remove_stop_words : bool, default True Whether to remove stop words. Stop words are common words that are often removed from text before indexing. For example, in English "the" and "and". + custom_stop_words : list of str, optional + Custom words replace the built-in language stop words + and only take effect when ``remove_stop_words`` is True. ``None`` uses + the built-in language list, while an empty list explicitly uses no + stop words. ascii_folding : bool, default True Whether to fold ASCII characters. This converts accented characters to their ASCII equivalent. For example, "café" would be converted to "cafe". @@ -179,6 +184,7 @@ class FTS: ngram_max_length: int = 3 prefix_only: bool = False block_size: int = 128 + custom_stop_words: Optional[List[str]] = None @dataclass diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 682b8533c..01b90c019 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -340,6 +340,7 @@ class RemoteTable(Table): lower_case: bool = True, stem: bool = True, remove_stop_words: bool = True, + custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, ngram_max_length: int = 3, @@ -361,6 +362,7 @@ class RemoteTable(Table): lower_case=lower_case, stem=stem, remove_stop_words=remove_stop_words, + custom_stop_words=custom_stop_words, ascii_folding=ascii_folding, ngram_min_length=ngram_min_length, ngram_max_length=ngram_max_length, diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 1da366a78..032c6dce7 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -1103,6 +1103,7 @@ class Table(ABC): lower_case: bool = True, stem: bool = True, remove_stop_words: bool = True, + custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, ngram_max_length: int = 3, @@ -1170,6 +1171,9 @@ class Table(ABC): remove_stop_words : bool, default True Whether to remove stop words. Stop words are common words that are often removed from text before indexing. For example, in English "the" and "and". + custom_stop_words : list of str, optional + Custom words that replace the built-in language stop words. ``None`` + uses the built-in list; an empty list explicitly uses no stop words. ascii_folding : bool, default True Whether to fold ASCII characters. This converts accented characters to their ASCII equivalent. For example, "café" would be converted to "cafe". @@ -3055,6 +3059,7 @@ class LanceTable(Table): lower_case: bool = True, stem: bool = True, remove_stop_words: bool = True, + custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, ngram_max_length: int = 3, @@ -3101,6 +3106,7 @@ class LanceTable(Table): "lower_case": lower_case, "stem": stem, "remove_stop_words": remove_stop_words, + "custom_stop_words": custom_stop_words, "ascii_folding": ascii_folding, "ngram_min_length": ngram_min_length, "ngram_max_length": ngram_max_length, @@ -3108,6 +3114,7 @@ class LanceTable(Table): } else: tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name) + tokenizer_configs["custom_stop_words"] = custom_stop_words config = FTS(block_size=block_size, **tokenizer_configs) diff --git a/python/python/tests/test_fts.py b/python/python/tests/test_fts.py index 127510bc2..f791f9886 100644 --- a/python/python/tests/test_fts.py +++ b/python/python/tests/test_fts.py @@ -219,11 +219,13 @@ def test_create_inverted_index(table, with_position): table.create_fts_index( "text", with_position=with_position, + custom_stop_words=["puppy"], name="custom_fts_index", ) indices = table.list_indices() fts_indices = [i for i in indices if i.index_type == "FTS"] assert any(i.name == "custom_fts_index" for i in fts_indices) + assert fts_indices[0].index_details["custom_stop_words"] == ["puppy"] @pytest.mark.parametrize("block_size", [128, 256]) @@ -243,6 +245,24 @@ def test_create_inverted_index_rejects_invalid_block_size(table): table.create_index("text", config=FTS(block_size=129)) +def test_custom_stop_words_list(table): + table.create_index( + "text", + config=FTS(stem=False, custom_stop_words=["lance"]), + ) + + assert table.list_indices()[0].index_details["custom_stop_words"] == ["lance"] + tokens = table.tokenize("the lance data", column="text") + assert [token.text for token in tokens] == ["the", "data"] + empty_tokens = ldb.tokenize("the lance data", stem=False, custom_stop_words=[]) + assert [token.text for token in empty_tokens] == ["the", "lance", "data"] + with pytest.raises(TypeError, match=r"custom_stop_words.*int"): + ldb.tokenize( + "the lance data", + custom_stop_words=["lance", 42], + ) + + def test_search_fts(table): table.create_fts_index("text") results = table.search("puppy").select(["id", "text"]).limit(5).to_list() diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 648a52b7a..55718f516 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -771,6 +771,7 @@ def test_table_create_indices(): "text", wait_timeout=timedelta(seconds=2), block_size=256, + custom_stop_words=["cloud"], name="custom_fts_idx", ) @@ -795,6 +796,7 @@ def test_table_create_indices(): assert "name" in fts_req assert fts_req["name"] == "custom_fts_idx" assert fts_req["block_size"] == 256 + assert fts_req["custom_stop_words"] == ["cloud"] # Check vector index request has custom name vector_req = received_requests[2] diff --git a/python/src/index.rs b/python/src/index.rs index a98b90cf9..8c81dcecf 100644 --- a/python/src/index.rs +++ b/python/src/index.rs @@ -59,7 +59,8 @@ pub fn extract_index_params(source: &Option>) -> PyResult>, ascii_folding: bool, ngram_min_length: u32, ngram_max_length: u32, diff --git a/python/src/table.rs b/python/src/table.rs index bb44fc023..84ba91f39 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -520,6 +520,7 @@ impl From for FtsToken { lower_case = true, stem = true, remove_stop_words = true, + custom_stop_words = None, ascii_folding = true, ngram_min_length = 3, ngram_max_length = 3, @@ -534,6 +535,7 @@ pub fn tokenize( lower_case: bool, stem: bool, remove_stop_words: bool, + custom_stop_words: Option>, ascii_folding: bool, ngram_min_length: u32, ngram_max_length: u32, @@ -555,7 +557,8 @@ pub fn tokenize( .ascii_folding(ascii_folding) .ngram_min_length(ngram_min_length) .ngram_max_length(ngram_max_length) - .ngram_prefix_only(prefix_only); + .ngram_prefix_only(prefix_only) + .custom_stop_words(custom_stop_words); let tokens = lancedb_tokenize(&query, ¶ms).infer_error()?; Ok(tokens.into_iter().map(FtsToken::from).collect()) } diff --git a/rust/lancedb/examples/full_text_search.rs b/rust/lancedb/examples/full_text_search.rs index 54d4a38f5..8d4b7557b 100644 --- a/rust/lancedb/examples/full_text_search.rs +++ b/rust/lancedb/examples/full_text_search.rs @@ -76,7 +76,12 @@ async fn create_table(db: &Connection) -> Result { async fn create_index(table: &Table) -> Result<()> { table - .create_index(&["doc"], Index::FTS(FtsIndexBuilder::default())) + .create_index( + &["doc"], + Index::FTS( + FtsIndexBuilder::default().custom_stop_words(Some(vec!["example".to_owned()])), + ), + ) .execute() .await?; Ok(()) diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 4c3c1bc6f..685e2b54e 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -4553,6 +4553,19 @@ mod tests { }, Index::FTS(InvertedIndexParams::default().block_size(256).unwrap()), ), + ( + "FTS", + { + let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap(); + body["custom_stop_words"] = json!(["cat", " cat ", "CAT"]); + body + }, + Index::FTS(InvertedIndexParams::default().custom_stop_words(Some(vec![ + "cat".to_string(), + " cat ".to_string(), + "CAT".to_string(), + ]))), + ), ]; for (index_type, expected_body, index) in cases { @@ -5084,8 +5097,9 @@ mod tests { "max_token_length": 40, "lower_case": true, "stem": false, - "remove_stop_words": false, + "remove_stop_words": true, "ascii_folding": true, + "custom_stop_words": ["hello"], }) .to_string(); let table = Table::new_with_handler("my_table", move |request| { @@ -5123,10 +5137,6 @@ mod tests { assert_eq!( tokens, vec![ - FtsToken { - text: "hello".to_string(), - position: 0, - }, FtsToken { text: "こんにちは".to_string(), position: 1, diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 0e9354ffc..00efd1f40 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -21,6 +21,7 @@ use lance::dataset::WriteMode; use lance::dataset::builder::DatasetBuilder; use lance::dataset::{InsertBuilder, WriteParams}; use lance::index::DatasetIndexExt; +use lance::index::scalar::load_segment_params; use lance::io::{ObjectStoreParams, WrappingObjectStore}; use lance_datafusion::utils::StreamingWriteSource; use lance_index::IndexCriteria; @@ -3193,10 +3194,9 @@ impl BaseTable for NativeTable { async fn list_indices(&self) -> Result> { let dataset = self.dataset.get().await?; let total_rows = dataset.count_rows(None).await? as u64; - let indices = dataset - .describe_indices(None) - .await? - .into_iter() + let descriptions = dataset.describe_indices(None).await?; + let mut indices: Vec = descriptions + .iter() .filter_map(|idx_desc| { let index_type: crate::index::IndexType = idx_desc .index_type() @@ -3254,6 +3254,31 @@ impl BaseTable for NativeTable { }) }) .collect(); + + for index in indices + .iter_mut() + .filter(|index| index.index_type == crate::index::IndexType::FTS) + { + let Some(description) = descriptions + .iter() + .find(|description| description.name() == index.name) + else { + continue; + }; + let segments = description.segments(); + let Some(segment) = segments.first() else { + continue; + }; + let params = load_segment_params(&dataset, segment).await?; + let details = serde_json::to_string(¶ms).map_err(|source| Error::Other { + message: format!( + "Failed to serialize full text search configuration for index '{}'", + index.name + ), + source: Some(Box::new(source)), + })?; + index.index_details = Some(details); + } Ok(indices) } diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 6be9de0e6..ab412a502 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -1366,11 +1366,19 @@ mod tests { table .create_index( &["text"], - Index::FTS(FtsIndexBuilder::default().block_size(256).unwrap()), + Index::FTS( + FtsIndexBuilder::default() + .stem(false) + .custom_stop_words(Some(vec!["cat".to_string()])) + .block_size(256) + .unwrap(), + ), ) .execute() .await .unwrap(); + drop(table); + let table = conn.open_table("test_bitmap").execute().await.unwrap(); let index_configs = table.list_indices().await.unwrap(); assert_eq!(index_configs.len(), 1); let index = index_configs.into_iter().next().unwrap(); @@ -1381,6 +1389,32 @@ mod tests { let index_params: FtsIndexBuilder = serde_json::from_str(index.index_details.as_deref().unwrap()).unwrap(); assert_eq!(index_params.posting_block_size(), 256); + assert_eq!( + serde_json::to_value(&index_params).unwrap()["custom_stop_words"], + serde_json::json!(["cat"]) + ); + assert_eq!( + table + .tokenize("cat dog", "text_idx") + .await + .unwrap() + .into_iter() + .map(|token| token.text) + .collect::>(), + vec!["dog"] + ); + + let batches = table + .query() + .full_text_search(FullTextSearchQuery::new("cat dog".to_string())) + .limit(120) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 40); let num_rows = 120; let stats = table.index_stats("text_idx").await.unwrap().unwrap();