From bcf5ee263f6d17e0e8fadae59ef1c68fbdcde513 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:53:16 -0700 Subject: [PATCH] fix(python): preserve code tokenizer filter defaults (#4370) Python's FTS bindings always forwarded `stem=True` and `remove_stop_words=True`, overriding Lance's code tokenizer defaults. As a result, code indexes stemmed identifiers and discarded stop words, and `lancedb.tokenize` reproduced the same behavior. Use `None` for omitted filter options and only apply explicit booleans in the Rust bindings. This preserves tokenizer defaults across local and remote index creation, the deprecated `create_fts_index` helpers, and standalone tokenization. Update the signatures and documentation to describe the inherited defaults. Regression tests cover stored local settings, token output, stop-word searches, remote request payloads, explicit overrides, and the existing simple tokenizer behavior. The regression suite produced 16 failures before the fix and passes all 28 cases after it. Validation: - Rebuilt the Python extension using the uv-managed development environment. - Full Python FTS and index test files plus relevant remote index tests: 105 passed. - `cargo clippy --quiet --features remote --tests --examples -p lancedb-python` - `cargo fmt --all`, Ruff format/lint with CI's pinned version 0.9.9, and Pyright (zero errors). Fixes #4365 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/lancedb/__init__.py | 6 ++- python/python/lancedb/_lancedb.pyi | 4 +- python/python/lancedb/index.py | 13 +++-- python/python/lancedb/remote/table.py | 4 +- python/python/lancedb/table.py | 17 ++++--- python/python/tests/test_fts.py | 72 +++++++++++++++++++++++++++ python/python/tests/test_remote_db.py | 71 ++++++++++++++++++++++++++ python/src/index.rs | 12 +++-- python/src/table.rs | 18 ++++--- 9 files changed, 190 insertions(+), 27 deletions(-) diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 213794ef3..39353f270 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -321,8 +321,8 @@ def tokenize( language: str = "English", max_token_length: Optional[int] = 40, lower_case: bool = True, - stem: bool = True, - remove_stop_words: bool = True, + stem: Optional[bool] = None, + remove_stop_words: Optional[bool] = None, custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, @@ -333,6 +333,8 @@ def tokenize( This does not require an FTS index. The tokenizer options match :class:`lancedb.index.FTS`. ``custom_stop_words`` accepts a list of strings. + When ``stem`` or ``remove_stop_words`` is None, the base tokenizer's default + applies: False for ``code`` and ``ngram``, True otherwise. """ return _tokenize( diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index e4679382f..af964d459 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -58,8 +58,8 @@ def tokenize( language: str = "English", max_token_length: Optional[int] = 40, lower_case: bool = True, - stem: bool = True, - remove_stop_words: bool = True, + stem: Optional[bool] = None, + remove_stop_words: Optional[bool] = None, custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, diff --git a/python/python/lancedb/index.py b/python/python/lancedb/index.py index 78ee72e57..2a7731be3 100644 --- a/python/python/lancedb/index.py +++ b/python/python/lancedb/index.py @@ -138,6 +138,7 @@ class FTS: - "simple": Splits text by whitespace and punctuation. - "whitespace": Split text by whitespace, but not punctuation. - "raw": No tokenization. The entire text is treated as a single token. + - "code": Tokenizes source code and identifiers. - "ngram": N-gram tokenizer for substring-style matching. - "icu": ICU dictionary-based word segmentation. - "icu/split": ICU segmentation with simple-style delimiter splitting. @@ -151,12 +152,16 @@ class FTS: ignored. lower_case : bool, default True Whether to convert the token to lower case. This makes queries case-insensitive. - stem : bool, default True + stem : bool, optional Whether to stem the token. Stemming reduces words to their root form. For example, in English "running" and "runs" would both be reduced to "run". - remove_stop_words : bool, default True + ``None`` uses the base tokenizer's default: False for ``code`` and + ``ngram``, True otherwise. + remove_stop_words : bool, optional Whether to remove stop words. Stop words are common words that are often removed from text before indexing. For example, in English "the" and "and". + ``None`` uses the base tokenizer's default: False for ``code`` and + ``ngram``, True otherwise. custom_stop_words : list of str, optional Custom words replace the built-in language stop words and only take effect when ``remove_stop_words`` is True. ``None`` uses @@ -197,8 +202,8 @@ class FTS: language: str = "English" max_token_length: Optional[int] = 40 lower_case: bool = True - stem: bool = True - remove_stop_words: bool = True + stem: Optional[bool] = None + remove_stop_words: Optional[bool] = None ascii_folding: bool = True ngram_min_length: int = 3 ngram_max_length: int = 3 diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index ef4d4009a..7f03f8fa9 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -354,8 +354,8 @@ class RemoteTable(Table): language: str = "English", max_token_length: Optional[int] = 40, lower_case: bool = True, - stem: bool = True, - remove_stop_words: bool = True, + stem: Optional[bool] = None, + remove_stop_words: Optional[bool] = None, custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 2275b0a93..186d38ad6 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -1602,8 +1602,8 @@ class Table(ABC): language: str = "English", max_token_length: Optional[int] = 40, lower_case: bool = True, - stem: bool = True, - remove_stop_words: bool = True, + stem: Optional[bool] = None, + remove_stop_words: Optional[bool] = None, custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, @@ -1653,6 +1653,7 @@ class Table(ABC): - "simple": Splits text by whitespace and punctuation. - "whitespace": Split text by whitespace, but not punctuation. - "raw": No tokenization. The entire text is treated as a single token. + - "code": Tokenizes source code and identifiers. - "ngram": N-Gram tokenizer. - "icu": ICU dictionary-based word segmentation. - "icu/split": ICU segmentation with simple-style delimiter splitting. @@ -1667,12 +1668,16 @@ class Table(ABC): lower_case : bool, default True Whether to convert the token to lower case. This makes queries case-insensitive. - stem : bool, default True + stem : bool, optional Whether to stem the token. Stemming reduces words to their root form. For example, in English "running" and "runs" would both be reduced to "run". - remove_stop_words : bool, default True + ``None`` uses the base tokenizer's default: False for ``code`` and + ``ngram``, True otherwise. + remove_stop_words : bool, optional Whether to remove stop words. Stop words are common words that are often removed from text before indexing. For example, in English "the" and "and". + ``None`` uses the base tokenizer's default: False for ``code`` and + ``ngram``, True otherwise. custom_stop_words : list of str, optional Custom words that replace the built-in language stop words. ``None`` uses the built-in list; an empty list explicitly uses no stop words. @@ -3808,8 +3813,8 @@ class LanceTable(Table): language: str = "English", max_token_length: Optional[int] = 40, lower_case: bool = True, - stem: bool = True, - remove_stop_words: bool = True, + stem: Optional[bool] = None, + remove_stop_words: Optional[bool] = None, custom_stop_words: Optional[List[str]] = None, ascii_folding: bool = True, ngram_min_length: int = 3, diff --git a/python/python/tests/test_fts.py b/python/python/tests/test_fts.py index a867d8405..e3963559d 100644 --- a/python/python/tests/test_fts.py +++ b/python/python/tests/test_fts.py @@ -895,6 +895,78 @@ def test_tokenize_uses_simple_index_tokenizer(mem_db: DBConnection): ] +@pytest.mark.parametrize("legacy", [False, True]) +@pytest.mark.parametrize( + ("base_tokenizer", "options", "expected_filters", "expected_tokens"), + [ + ( + "code", + {}, + (False, False), + [("running", 0), ("in", 1), ("cafe", 2)], + ), + ( + "code", + {"stem": None, "remove_stop_words": None}, + (False, False), + [("running", 0), ("in", 1), ("cafe", 2)], + ), + ( + "code", + {"stem": True}, + (True, False), + [("run", 0), ("in", 1), ("cafe", 2)], + ), + ( + "code", + {"remove_stop_words": True}, + (False, True), + [("running", 0), ("cafe", 2)], + ), + ( + "code", + {"stem": True, "remove_stop_words": True}, + (True, True), + [("run", 0), ("cafe", 2)], + ), + ( + "simple", + {}, + (True, True), + [("run", 0), ("cafe", 2)], + ), + ( + "simple", + {"stem": False, "remove_stop_words": False}, + (False, False), + [("running", 0), ("in", 1), ("cafe", 2)], + ), + ], +) +def test_fts_tokenizer_filter_defaults( + mem_db, legacy, base_tokenizer, options, expected_filters, expected_tokens +): + text = "Running in café" + table = mem_db.create_table("test_filter_defaults", data=[{"text": text}]) + if legacy: + with pytest.warns(DeprecationWarning, match="create_fts_index"): + table.create_fts_index("text", base_tokenizer=base_tokenizer, **options) + else: + table.create_index("text", config=FTS(base_tokenizer=base_tokenizer, **options)) + + details = table.list_indices()[0].index_details + assert (details["stem"], details["remove_stop_words"]) == expected_filters + assert [ + (token.text, token.position) for token in table.tokenize(text, column="text") + ] == expected_tokens + assert [ + (token.text, token.position) + for token in ldb.tokenize(text, base_tokenizer=base_tokenizer, **options) + ] == expected_tokens + if base_tokenizer == "code" and not expected_filters[1]: + assert table.search("in", query_type="fts").to_list()[0]["text"] == text + + def test_tokenize_uses_icu_index_tokenizer_by_name(mem_db: DBConnection): data = pa.table({"text": ["Hello, こんにちは世界!"]}) table = mem_db.create_table("test_tokenize_icu", data=data) diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 038eb8d16..2494d7a80 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1023,6 +1023,77 @@ def test_table_create_indices(): table.drop_index("custom_fts_idx") +@pytest.mark.parametrize("legacy", [False, True]) +@pytest.mark.parametrize( + ("base_tokenizer", "options", "expected_filters"), + [ + ("code", {}, (False, False)), + ("code", {"stem": None, "remove_stop_words": None}, (False, False)), + ("code", {"stem": True}, (True, False)), + ("code", {"remove_stop_words": True}, (False, True)), + ("code", {"stem": True, "remove_stop_words": True}, (True, True)), + ("simple", {}, (True, True)), + ("simple", {"stem": False, "remove_stop_words": False}, (False, False)), + ], +) +def test_remote_fts_tokenizer_filter_defaults( + legacy, base_tokenizer, options, expected_filters +): + from lancedb.index import FTS + + received_requests = [] + + def handler(request): + if request.path == "/v1/table/test/create_index/": + content_len = int(request.headers["Content-Length"]) + received_requests.append(json.loads(request.rfile.read(content_len))) + request.send_response(200) + request.end_headers() + elif request.path == "/v1/table/test/create/?mode=create": + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b"{}") + elif request.path == "/v1/table/test/describe/": + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write( + json.dumps( + { + "version": 1, + "schema": { + "fields": [ + { + "name": "text", + "type": {"type": "string"}, + "nullable": False, + } + ] + }, + } + ).encode() + ) + else: + request.send_response(404) + request.end_headers() + + with mock_lancedb_connection(handler) as db: + table = db.create_table("test", [{"text": "Running in cafes"}]) + if legacy: + with pytest.warns(DeprecationWarning, match="create_fts_index"): + table.create_fts_index("text", base_tokenizer=base_tokenizer, **options) + else: + table.create_index( + "text", config=FTS(base_tokenizer=base_tokenizer, **options) + ) + + assert len(received_requests) == 1 + params = received_requests[0] + assert params["base_tokenizer"] == base_tokenizer + assert (params["stem"], params["remove_stop_words"]) == expected_filters + + def test_remote_create_index_async_returns_job(): from lancedb.index import BTree diff --git a/python/src/index.rs b/python/src/index.rs index 54b15f55e..d4e16a67e 100644 --- a/python/src/index.rs +++ b/python/src/index.rs @@ -54,8 +54,6 @@ pub fn extract_index_params(source: &Option>) -> PyResult>) -> PyResult, lower_case: bool, - stem: bool, - remove_stop_words: bool, + stem: Option, + remove_stop_words: Option, custom_stop_words: Option>, ascii_folding: bool, ngram_min_length: u32, diff --git a/python/src/table.rs b/python/src/table.rs index 59e96619c..7102a8aa7 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -823,8 +823,8 @@ impl From for FtsToken { language = "English".to_string(), max_token_length = Some(40), lower_case = true, - stem = true, - remove_stop_words = true, + stem = None, + remove_stop_words = None, custom_stop_words = None, ascii_folding = true, ngram_min_length = 3, @@ -838,15 +838,15 @@ pub fn tokenize( language: String, max_token_length: Option, lower_case: bool, - stem: bool, - remove_stop_words: bool, + stem: Option, + remove_stop_words: Option, custom_stop_words: Option>, ascii_folding: bool, ngram_min_length: u32, ngram_max_length: u32, prefix_only: bool, ) -> PyResult> { - let params = FtsIndexBuilder::default() + let mut params = FtsIndexBuilder::default() .base_tokenizer(base_tokenizer) .language(&language) .map_err(|_| { @@ -857,13 +857,17 @@ pub fn tokenize( })? .max_token_length(max_token_length.map(|value| value as usize)) .lower_case(lower_case) - .stem(stem) - .remove_stop_words(remove_stop_words) .ascii_folding(ascii_folding) .ngram_min_length(ngram_min_length) .ngram_max_length(ngram_max_length) .ngram_prefix_only(prefix_only) .custom_stop_words(custom_stop_words); + if let Some(stem) = stem { + params = params.stem(stem); + } + if let Some(remove_stop_words) = remove_stop_words { + params = params.remove_stop_words(remove_stop_words); + } let tokens = lancedb_tokenize(&query, ¶ms).infer_error()?; Ok(tokens.into_iter().map(FtsToken::from).collect()) }