diff --git a/docs/openapi.yml b/docs/openapi.yml index b07e47e04..2f9ae7d99 100644 --- a/docs/openapi.yml +++ b/docs/openapi.yml @@ -453,6 +453,16 @@ paths: The metric type to use for the index. l2, Cosine, Dot are supported. index_type: type: string + custom_stop_words: + type: [array, "null"] + items: + type: string + description: | + The custom stop-word list for an FTS index. A non-null + array replaces the language's built-in stop-word list and is only + applied when remove_stop_words is enabled. Null uses the built-in + language list, while an empty array explicitly replaces it with no + stop words. responses: "200": description: Index successfully created @@ -510,4 +520,4 @@ paths: "401": $ref: "#/components/responses/unauthorized" "404": - $ref: "#/components/responses/not_found" \ No newline at end of file + $ref: "#/components/responses/not_found" diff --git a/docs/src/js/interfaces/FtsOptions.md b/docs/src/js/interfaces/FtsOptions.md index b8b264fed..1d3c9ff82 100644 --- a/docs/src/js/interfaces/FtsOptions.md +++ b/docs/src/js/interfaces/FtsOptions.md @@ -56,6 +56,21 @@ the experimental FTS V3 format and may introduce breaking changes. *** +### customStopWords? + +```ts +optional customStopWords: string[]; +``` + +Custom stop words that replace the built-in list for `language`. + +This option only affects tokenization when `removeStopWords` is true. + +`undefined` keeps the built-in language list. An empty array explicitly +replaces it with no stop words. + +*** + ### language? ```ts diff --git a/docs/src/js/interfaces/TokenizeOptions.md b/docs/src/js/interfaces/TokenizeOptions.md index f061356d1..8ef15d03d 100644 --- a/docs/src/js/interfaces/TokenizeOptions.md +++ b/docs/src/js/interfaces/TokenizeOptions.md @@ -30,6 +30,21 @@ The tokenizer to use. The default is "simple". *** +### customStopWords? + +```ts +optional customStopWords: string[]; +``` + +Custom stop words that replace the built-in list for `language`. + +This option only affects tokenization when `removeStopWords` is true. + +`undefined` keeps the built-in language list. An empty array explicitly +replaces it with no stop words. + +*** + ### language? ```ts diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 1e928be54..dd60451cc 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -94,9 +94,33 @@ of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and ## Full text search -Use [lancedb.table.Table.create_fts_index][] for the synchronous API or -[lancedb.table.AsyncTable.create_index][] with [lancedb.index.FTS][] for the -asynchronous API. +Pass `custom_stop_words` to [lancedb.index.FTS][]: + +```python +from lancedb.index import FTS + +table.create_index( + "text", + config=FTS(remove_stop_words=True, custom_stop_words=["acme", "internal"]), +) +``` + +The list replaces the built-in stop words and is used only when +`remove_stop_words=True`: + +- `custom_stop_words=None` uses the built-in list for `language`. +- `custom_stop_words=[]` removes no words. +- Values are passed through without trimming, lowercasing, or other rewriting. + +The same option is available on `lancedb.tokenize(...)` and the deprecated +[lancedb.table.Table.create_fts_index][] compatibility helper: + +```python +import lancedb + +tokens = list(lancedb.tokenize("acme makes searchable data", + custom_stop_words=["acme"])) +``` ::: lancedb.index.FTS diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 22562b0bc..75c5540eb 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -226,7 +226,7 @@ describe("remote connection", () => { ); }); - it("sends the FTS posting block size to remote tables", async () => { + it("sends FTS options to remote tables", async () => { let createIndexBody: Record | undefined; await withMockDatabase( @@ -264,7 +264,11 @@ describe("remote connection", () => { async (db) => { const table = await db.openTable("t"); await table.createIndex("text", { - config: Index.fts({ blockSize: 256 }), + config: Index.fts({ + blockSize: 256, + removeStopWords: true, + customStopWords: ["the"], + }), }); }, ); @@ -272,6 +276,7 @@ describe("remote connection", () => { expect(createIndexBody?.["column"]).toBe("text"); expect(createIndexBody?.["index_type"]).toBe("FTS"); expect(createIndexBody?.["block_size"]).toBe(256); + expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]); }); it("diffs and merges remote branches", async () => { diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 15248f5cf..41025047e 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2769,6 +2769,15 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( }, ); +test("tokenize supports custom stop words", async () => { + const tokens = await tokenize("the lance data", { + stem: false, + removeStopWords: true, + customStopWords: ["lance"], + }); + expect(tokens.map((token) => token.text)).toEqual(["the", "data"]); +}); + describe("when calling explainPlan", () => { let tmpDir: tmp.DirResult; let table: Table; diff --git a/nodejs/examples/full_text_search.test.ts b/nodejs/examples/full_text_search.test.ts index 9777bda7f..d1567dfbb 100644 --- a/nodejs/examples/full_text_search.test.ts +++ b/nodejs/examples/full_text_search.test.ts @@ -29,8 +29,14 @@ test("full text search", async () => { const tbl = await db.createTable("myVectors", data, { mode: "overwrite" }); await tbl.createIndex("doc", { - config: lancedb.Index.fts(), + config: lancedb.Index.fts({ + stem: false, + removeStopWords: true, + customStopWords: ["banana"], + }), }); + const tokens = await tbl.tokenize("apple banana", { column: "doc" }); + expect(tokens.map((token) => token.text)).toEqual(["apple"]); // --8<-- [start:full_text_search] const result = await tbl diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index cb91c30aa..63a4b08d0 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -194,6 +194,16 @@ export interface TokenizeOptions { /** Whether to remove stop words. */ removeStopWords?: boolean; + /** + * Custom stop words that replace the built-in list for `language`. + * + * This option only affects tokenization when `removeStopWords` is true. + * + * `undefined` keeps the built-in language list. An empty array explicitly + * replaces it with no stop words. + */ + customStopWords?: string[]; + /** Whether to fold ASCII characters. */ asciiFolding?: boolean; @@ -225,6 +235,7 @@ export async function tokenize( options?.lowercase, options?.stem, options?.removeStopWords, + options?.customStopWords, options?.asciiFolding, options?.ngramMinLength, options?.ngramMaxLength, diff --git a/nodejs/lancedb/indices.ts b/nodejs/lancedb/indices.ts index 9b438ef0f..dbeebf433 100644 --- a/nodejs/lancedb/indices.ts +++ b/nodejs/lancedb/indices.ts @@ -553,6 +553,16 @@ export interface FtsOptions { */ removeStopWords?: boolean; + /** + * Custom stop words that replace the built-in list for `language`. + * + * This option only affects tokenization when `removeStopWords` is true. + * + * `undefined` keeps the built-in language list. An empty array explicitly + * replaces it with no stop words. + */ + customStopWords?: string[]; + /** * whether to remove punctuation */ @@ -755,6 +765,7 @@ export class Index { options?.lowercase, options?.stem, options?.removeStopWords, + options?.customStopWords, options?.asciiFolding, options?.ngramMinLength, options?.ngramMaxLength, diff --git a/nodejs/src/index.rs b/nodejs/src/index.rs index 66dee7913..762f74870 100644 --- a/nodejs/src/index.rs +++ b/nodejs/src/index.rs @@ -43,6 +43,7 @@ pub fn tokenize( lower_case: Option, stem: Option, remove_stop_words: Option, + custom_stop_words: Option>, ascii_folding: Option, ngram_min_length: Option, ngram_max_length: Option, @@ -72,6 +73,7 @@ pub fn tokenize( if let Some(remove_stop_words) = remove_stop_words { opts = opts.remove_stop_words(remove_stop_words); } + opts = opts.custom_stop_words(custom_stop_words); if let Some(ascii_folding) = ascii_folding { opts = opts.ascii_folding(ascii_folding); } @@ -222,6 +224,7 @@ impl Index { lower_case: Option, stem: Option, remove_stop_words: Option, + custom_stop_words: Option>, ascii_folding: Option, ngram_min_length: Option, ngram_max_length: Option, @@ -250,6 +253,7 @@ impl Index { if let Some(remove_stop_words) = remove_stop_words { opts = opts.remove_stop_words(remove_stop_words); } + opts = opts.custom_stop_words(custom_stop_words); if let Some(ascii_folding) = ascii_folding { opts = opts.ascii_folding(ascii_folding); } diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 6f6468d9c..71349b8a5 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -258,6 +258,7 @@ def tokenize( lower_case: bool = True, stem: bool = True, remove_stop_words: bool = True, + custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, ngram_max_length: int = 3, @@ -265,9 +266,10 @@ def tokenize( ) -> Iterable[FtsToken]: """Tokenize a full-text search query using an explicit tokenizer. - This does not require a table or FTS index. The tokenizer options match - :class:`lancedb.index.FTS`. + This does not require an FTS index. The tokenizer options match + :class:`lancedb.index.FTS`. ``custom_stop_words`` accepts a list of strings. """ + return _tokenize( query, base_tokenizer=base_tokenizer, @@ -276,6 +278,7 @@ def tokenize( lower_case=lower_case, stem=stem, remove_stop_words=remove_stop_words, + custom_stop_words=custom_stop_words, ascii_folding=ascii_folding, ngram_min_length=ngram_min_length, ngram_max_length=ngram_max_length, diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 448947a29..67325e10b 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -59,6 +59,7 @@ def tokenize( lower_case: bool = True, stem: bool = True, remove_stop_words: bool = True, + custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, ngram_max_length: int = 3, diff --git a/python/python/lancedb/index.py b/python/python/lancedb/index.py index c06bb9903..5ced0600f 100644 --- a/python/python/lancedb/index.py +++ b/python/python/lancedb/index.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors from dataclasses import dataclass -from typing import Literal, Optional +from typing import List, Literal, Optional from ._lancedb import ( IndexConfig, @@ -151,6 +151,11 @@ class FTS: remove_stop_words : bool, default True Whether to remove stop words. Stop words are common words that are often removed from text before indexing. For example, in English "the" and "and". + custom_stop_words : list of str, optional + Custom words replace the built-in language stop words + and only take effect when ``remove_stop_words`` is True. ``None`` uses + the built-in language list, while an empty list explicitly uses no + stop words. ascii_folding : bool, default True Whether to fold ASCII characters. This converts accented characters to their ASCII equivalent. For example, "café" would be converted to "cafe". @@ -179,6 +184,7 @@ class FTS: ngram_max_length: int = 3 prefix_only: bool = False block_size: int = 128 + custom_stop_words: Optional[List[str]] = None @dataclass diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 682b8533c..01b90c019 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -340,6 +340,7 @@ class RemoteTable(Table): lower_case: bool = True, stem: bool = True, remove_stop_words: bool = True, + custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, ngram_max_length: int = 3, @@ -361,6 +362,7 @@ class RemoteTable(Table): lower_case=lower_case, stem=stem, remove_stop_words=remove_stop_words, + custom_stop_words=custom_stop_words, ascii_folding=ascii_folding, ngram_min_length=ngram_min_length, ngram_max_length=ngram_max_length, diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 1da366a78..032c6dce7 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -1103,6 +1103,7 @@ class Table(ABC): lower_case: bool = True, stem: bool = True, remove_stop_words: bool = True, + custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, ngram_max_length: int = 3, @@ -1170,6 +1171,9 @@ class Table(ABC): remove_stop_words : bool, default True Whether to remove stop words. Stop words are common words that are often removed from text before indexing. For example, in English "the" and "and". + custom_stop_words : list of str, optional + Custom words that replace the built-in language stop words. ``None`` + uses the built-in list; an empty list explicitly uses no stop words. ascii_folding : bool, default True Whether to fold ASCII characters. This converts accented characters to their ASCII equivalent. For example, "café" would be converted to "cafe". @@ -3055,6 +3059,7 @@ class LanceTable(Table): lower_case: bool = True, stem: bool = True, remove_stop_words: bool = True, + custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, ngram_max_length: int = 3, @@ -3101,6 +3106,7 @@ class LanceTable(Table): "lower_case": lower_case, "stem": stem, "remove_stop_words": remove_stop_words, + "custom_stop_words": custom_stop_words, "ascii_folding": ascii_folding, "ngram_min_length": ngram_min_length, "ngram_max_length": ngram_max_length, @@ -3108,6 +3114,7 @@ class LanceTable(Table): } else: tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name) + tokenizer_configs["custom_stop_words"] = custom_stop_words config = FTS(block_size=block_size, **tokenizer_configs) diff --git a/python/python/tests/test_fts.py b/python/python/tests/test_fts.py index 127510bc2..f791f9886 100644 --- a/python/python/tests/test_fts.py +++ b/python/python/tests/test_fts.py @@ -219,11 +219,13 @@ def test_create_inverted_index(table, with_position): table.create_fts_index( "text", with_position=with_position, + custom_stop_words=["puppy"], name="custom_fts_index", ) indices = table.list_indices() fts_indices = [i for i in indices if i.index_type == "FTS"] assert any(i.name == "custom_fts_index" for i in fts_indices) + assert fts_indices[0].index_details["custom_stop_words"] == ["puppy"] @pytest.mark.parametrize("block_size", [128, 256]) @@ -243,6 +245,24 @@ def test_create_inverted_index_rejects_invalid_block_size(table): table.create_index("text", config=FTS(block_size=129)) +def test_custom_stop_words_list(table): + table.create_index( + "text", + config=FTS(stem=False, custom_stop_words=["lance"]), + ) + + assert table.list_indices()[0].index_details["custom_stop_words"] == ["lance"] + tokens = table.tokenize("the lance data", column="text") + assert [token.text for token in tokens] == ["the", "data"] + empty_tokens = ldb.tokenize("the lance data", stem=False, custom_stop_words=[]) + assert [token.text for token in empty_tokens] == ["the", "lance", "data"] + with pytest.raises(TypeError, match=r"custom_stop_words.*int"): + ldb.tokenize( + "the lance data", + custom_stop_words=["lance", 42], + ) + + def test_search_fts(table): table.create_fts_index("text") results = table.search("puppy").select(["id", "text"]).limit(5).to_list() diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 648a52b7a..55718f516 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -771,6 +771,7 @@ def test_table_create_indices(): "text", wait_timeout=timedelta(seconds=2), block_size=256, + custom_stop_words=["cloud"], name="custom_fts_idx", ) @@ -795,6 +796,7 @@ def test_table_create_indices(): assert "name" in fts_req assert fts_req["name"] == "custom_fts_idx" assert fts_req["block_size"] == 256 + assert fts_req["custom_stop_words"] == ["cloud"] # Check vector index request has custom name vector_req = received_requests[2] diff --git a/python/src/index.rs b/python/src/index.rs index a98b90cf9..8c81dcecf 100644 --- a/python/src/index.rs +++ b/python/src/index.rs @@ -59,7 +59,8 @@ pub fn extract_index_params(source: &Option>) -> PyResult>, ascii_folding: bool, ngram_min_length: u32, ngram_max_length: u32, diff --git a/python/src/table.rs b/python/src/table.rs index bb44fc023..84ba91f39 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -520,6 +520,7 @@ impl From for FtsToken { lower_case = true, stem = true, remove_stop_words = true, + custom_stop_words = None, ascii_folding = true, ngram_min_length = 3, ngram_max_length = 3, @@ -534,6 +535,7 @@ pub fn tokenize( lower_case: bool, stem: bool, remove_stop_words: bool, + custom_stop_words: Option>, ascii_folding: bool, ngram_min_length: u32, ngram_max_length: u32, @@ -555,7 +557,8 @@ pub fn tokenize( .ascii_folding(ascii_folding) .ngram_min_length(ngram_min_length) .ngram_max_length(ngram_max_length) - .ngram_prefix_only(prefix_only); + .ngram_prefix_only(prefix_only) + .custom_stop_words(custom_stop_words); let tokens = lancedb_tokenize(&query, ¶ms).infer_error()?; Ok(tokens.into_iter().map(FtsToken::from).collect()) } diff --git a/rust/lancedb/examples/full_text_search.rs b/rust/lancedb/examples/full_text_search.rs index 54d4a38f5..8d4b7557b 100644 --- a/rust/lancedb/examples/full_text_search.rs +++ b/rust/lancedb/examples/full_text_search.rs @@ -76,7 +76,12 @@ async fn create_table(db: &Connection) -> Result { async fn create_index(table: &Table) -> Result<()> { table - .create_index(&["doc"], Index::FTS(FtsIndexBuilder::default())) + .create_index( + &["doc"], + Index::FTS( + FtsIndexBuilder::default().custom_stop_words(Some(vec!["example".to_owned()])), + ), + ) .execute() .await?; Ok(()) diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 4c3c1bc6f..685e2b54e 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -4553,6 +4553,19 @@ mod tests { }, Index::FTS(InvertedIndexParams::default().block_size(256).unwrap()), ), + ( + "FTS", + { + let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap(); + body["custom_stop_words"] = json!(["cat", " cat ", "CAT"]); + body + }, + Index::FTS(InvertedIndexParams::default().custom_stop_words(Some(vec![ + "cat".to_string(), + " cat ".to_string(), + "CAT".to_string(), + ]))), + ), ]; for (index_type, expected_body, index) in cases { @@ -5084,8 +5097,9 @@ mod tests { "max_token_length": 40, "lower_case": true, "stem": false, - "remove_stop_words": false, + "remove_stop_words": true, "ascii_folding": true, + "custom_stop_words": ["hello"], }) .to_string(); let table = Table::new_with_handler("my_table", move |request| { @@ -5123,10 +5137,6 @@ mod tests { assert_eq!( tokens, vec![ - FtsToken { - text: "hello".to_string(), - position: 0, - }, FtsToken { text: "こんにちは".to_string(), position: 1, diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 0e9354ffc..00efd1f40 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -21,6 +21,7 @@ use lance::dataset::WriteMode; use lance::dataset::builder::DatasetBuilder; use lance::dataset::{InsertBuilder, WriteParams}; use lance::index::DatasetIndexExt; +use lance::index::scalar::load_segment_params; use lance::io::{ObjectStoreParams, WrappingObjectStore}; use lance_datafusion::utils::StreamingWriteSource; use lance_index::IndexCriteria; @@ -3193,10 +3194,9 @@ impl BaseTable for NativeTable { async fn list_indices(&self) -> Result> { let dataset = self.dataset.get().await?; let total_rows = dataset.count_rows(None).await? as u64; - let indices = dataset - .describe_indices(None) - .await? - .into_iter() + let descriptions = dataset.describe_indices(None).await?; + let mut indices: Vec = descriptions + .iter() .filter_map(|idx_desc| { let index_type: crate::index::IndexType = idx_desc .index_type() @@ -3254,6 +3254,31 @@ impl BaseTable for NativeTable { }) }) .collect(); + + for index in indices + .iter_mut() + .filter(|index| index.index_type == crate::index::IndexType::FTS) + { + let Some(description) = descriptions + .iter() + .find(|description| description.name() == index.name) + else { + continue; + }; + let segments = description.segments(); + let Some(segment) = segments.first() else { + continue; + }; + let params = load_segment_params(&dataset, segment).await?; + let details = serde_json::to_string(¶ms).map_err(|source| Error::Other { + message: format!( + "Failed to serialize full text search configuration for index '{}'", + index.name + ), + source: Some(Box::new(source)), + })?; + index.index_details = Some(details); + } Ok(indices) } diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 6be9de0e6..ab412a502 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -1366,11 +1366,19 @@ mod tests { table .create_index( &["text"], - Index::FTS(FtsIndexBuilder::default().block_size(256).unwrap()), + Index::FTS( + FtsIndexBuilder::default() + .stem(false) + .custom_stop_words(Some(vec!["cat".to_string()])) + .block_size(256) + .unwrap(), + ), ) .execute() .await .unwrap(); + drop(table); + let table = conn.open_table("test_bitmap").execute().await.unwrap(); let index_configs = table.list_indices().await.unwrap(); assert_eq!(index_configs.len(), 1); let index = index_configs.into_iter().next().unwrap(); @@ -1381,6 +1389,32 @@ mod tests { let index_params: FtsIndexBuilder = serde_json::from_str(index.index_details.as_deref().unwrap()).unwrap(); assert_eq!(index_params.posting_block_size(), 256); + assert_eq!( + serde_json::to_value(&index_params).unwrap()["custom_stop_words"], + serde_json::json!(["cat"]) + ); + assert_eq!( + table + .tokenize("cat dog", "text_idx") + .await + .unwrap() + .into_iter() + .map(|token| token.text) + .collect::>(), + vec!["dog"] + ); + + let batches = table + .query() + .full_text_search(FullTextSearchQuery::new("cat dog".to_string())) + .limit(120) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 40); let num_rows = 120; let stats = table.index_stats("text_idx").await.unwrap().unwrap();