mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-18 12:08:35 +00:00
feat(fts): support custom stop-word lists (#3734)
## What Expose custom FTS stop-word lists in the Python and TypeScript public APIs, including their standalone tokenize helpers and remote index creation. This PR supports concrete string lists only. It does not add file or LanceDB-table stop-word sources. ## Why Rust already exposes Lance's custom stop-word list option. The Python and TypeScript APIs did not pass it through, and local index details did not retain the full tokenizer parameters needed by index-backed tokenization after reopening a table. ## How - Add `custom_stop_words` / `customStopWords` to the Python and TypeScript FTS and tokenize options. - Preserve `None` / `undefined`, empty lists, and list contents without normalization. - Load the persisted FTS segment parameters when returning local index details. - Serialize the concrete list in remote create-index requests. - Keep Python and TypeScript tests thin; behavior, persistence, query tokenization, and remote JSON coverage live primarily in Rust. ## Validation - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` - `cargo test --quiet --features remote --tests` - Python extension rebuild with `uv` and `maturin` - Targeted Python tests: 4 passed - Python `ruff format --check` and `ruff check` - TypeScript build, typecheck, Biome lint, generated docs, and targeted tests --------- Co-authored-by: Yang Cen <yangcen@Yangs-Mac-mini.local>
This commit is contained in:
+4
-1
@@ -520,6 +520,7 @@ impl From<LanceDbFtsToken> for FtsToken {
|
||||
lower_case = true,
|
||||
stem = true,
|
||||
remove_stop_words = true,
|
||||
custom_stop_words = None,
|
||||
ascii_folding = true,
|
||||
ngram_min_length = 3,
|
||||
ngram_max_length = 3,
|
||||
@@ -534,6 +535,7 @@ pub fn tokenize(
|
||||
lower_case: bool,
|
||||
stem: bool,
|
||||
remove_stop_words: bool,
|
||||
custom_stop_words: Option<Vec<String>>,
|
||||
ascii_folding: bool,
|
||||
ngram_min_length: u32,
|
||||
ngram_max_length: u32,
|
||||
@@ -555,7 +557,8 @@ pub fn tokenize(
|
||||
.ascii_folding(ascii_folding)
|
||||
.ngram_min_length(ngram_min_length)
|
||||
.ngram_max_length(ngram_max_length)
|
||||
.ngram_prefix_only(prefix_only);
|
||||
.ngram_prefix_only(prefix_only)
|
||||
.custom_stop_words(custom_stop_words);
|
||||
let tokens = lancedb_tokenize(&query, ¶ms).infer_error()?;
|
||||
Ok(tokens.into_iter().map(FtsToken::from).collect())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user