fix(python): preserve code tokenizer filter defaults (#4370)

Python's FTS bindings always forwarded `stem=True` and
`remove_stop_words=True`, overriding Lance's code tokenizer defaults. As
a result, code indexes stemmed identifiers and discarded stop words, and
`lancedb.tokenize` reproduced the same behavior.

Use `None` for omitted filter options and only apply explicit booleans
in the Rust bindings. This preserves tokenizer defaults across local and
remote index creation, the deprecated `create_fts_index` helpers, and
standalone tokenization. Update the signatures and documentation to
describe the inherited defaults.

Regression tests cover stored local settings, token output, stop-word
searches, remote request payloads, explicit overrides, and the existing
simple tokenizer behavior. The regression suite produced 16 failures
before the fix and passes all 28 cases after it.

Validation:

- Rebuilt the Python extension using the uv-managed development
environment.
- Full Python FTS and index test files plus relevant remote index tests:
105 passed.
- `cargo clippy --quiet --features remote --tests --examples -p
lancedb-python`
- `cargo fmt --all`, Ruff format/lint with CI's pinned version 0.9.9,
and Pyright (zero errors).

Fixes #4365

<!-- lance-gatekeeper-fix:v1 agent=9c1db2d3fc25c49c5b340f8debf43a0f
generation=1 -->

Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>

python/python/tests/test_remote_db.py

@@ -1023,6 +1023,77 @@ def test_table_create_indices():
         table.drop_index("custom_fts_idx")
 
 
+@pytest.mark.parametrize("legacy", [False, True])
+@pytest.mark.parametrize(
+    ("base_tokenizer", "options", "expected_filters"),
+    [
+        ("code", {}, (False, False)),
+        ("code", {"stem": None, "remove_stop_words": None}, (False, False)),
+        ("code", {"stem": True}, (True, False)),
+        ("code", {"remove_stop_words": True}, (False, True)),
+        ("code", {"stem": True, "remove_stop_words": True}, (True, True)),
+        ("simple", {}, (True, True)),
+        ("simple", {"stem": False, "remove_stop_words": False}, (False, False)),
+    ],
+)
+def test_remote_fts_tokenizer_filter_defaults(
+    legacy, base_tokenizer, options, expected_filters
+):
+    from lancedb.index import FTS
+
+    received_requests = []
+
+    def handler(request):
+        if request.path == "/v1/table/test/create_index/":
+            content_len = int(request.headers["Content-Length"])
+            received_requests.append(json.loads(request.rfile.read(content_len)))
+            request.send_response(200)
+            request.end_headers()
+        elif request.path == "/v1/table/test/create/?mode=create":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b"{}")
+        elif request.path == "/v1/table/test/describe/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(
+                json.dumps(
+                    {
+                        "version": 1,
+                        "schema": {
+                            "fields": [
+                                {
+                                    "name": "text",
+                                    "type": {"type": "string"},
+                                    "nullable": False,
+                                }
+                            ]
+                        },
+                    }
+                ).encode()
+            )
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        table = db.create_table("test", [{"text": "Running in cafes"}])
+        if legacy:
+            with pytest.warns(DeprecationWarning, match="create_fts_index"):
+                table.create_fts_index("text", base_tokenizer=base_tokenizer, **options)
+        else:
+            table.create_index(
+                "text", config=FTS(base_tokenizer=base_tokenizer, **options)
+            )
+
+    assert len(received_requests) == 1
+    params = received_requests[0]
+    assert params["base_tokenizer"] == base_tokenizer
+    assert (params["stem"], params["remove_stop_words"]) == expected_filters
+
+
 def test_remote_create_index_async_returns_job():
     from lancedb.index import BTree