feat(fts): support custom stop-word lists (#3734 )

## What Expose custom FTS stop-word lists in the Python and TypeScript public APIs, including their standalone tokenize helpers and remote index creation. This PR supports concrete string lists only. It does not add file or LanceDB-table stop-word sources. ## Why Rust already exposes Lance's custom stop-word list option. The Python and TypeScript APIs did not pass it through, and local index details did not retain the full tokenizer parameters needed by index-backed tokenization after reopening a table. ## How - Add `custom_stop_words` / `customStopWords` to the Python and TypeScript FTS and tokenize options. - Preserve `None` / `undefined`, empty lists, and list contents without normalization. - Load the persisted FTS segment parameters when returning local index details. - Serialize the concrete list in remote create-index requests. - Keep Python and TypeScript tests thin; behavior, persistence, query tokenization, and remote JSON coverage live primarily in Rust. ## Validation - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` - `cargo test --quiet --features remote --tests` - Python extension rebuild with `uv` and `maturin` - Targeted Python tests: 4 passed - Python `ruff format --check` and `ruff check` - TypeScript build, typecheck, Biome lint, generated docs, and targeted tests --------- Co-authored-by: Yang Cen <yangcen@Yangs-Mac-mini.local>
chore: update lance dependency to v10.0.0-beta.5
2026-08-26 07:58:31 +00:00 · 2026-07-29 17:48:15 +08:00 · 2026-07-25 04:17:17 +00:00 · 2026-07-24 22:04:02 +00:00
40 changed files with 342 additions and 99 deletions
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.32.0-beta.2"
+current_version = "0.32.0-beta.3"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
@@ -3421,8 +3421,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"

 [[package]]
 name = "fsst"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "rand 0.9.5",
@@ -4777,8 +4777,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"

 [[package]]
 name = "lance"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -4852,8 +4852,8 @@ dependencies = [

 [[package]]
 name = "lance-arrow"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4875,7 +4875,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4889,7 +4889,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -4898,8 +4898,8 @@ dependencies = [

 [[package]]
 name = "lance-bitpacking"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrayref",
 "crunchy",
@@ -4909,8 +4909,8 @@ dependencies = [

 [[package]]
 name = "lance-core"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4933,6 +4933,7 @@ dependencies = [
 "object_store",
 "pin-project",
 "prost",
+ "quick_cache",
 "rand 0.9.5",
 "roaring",
 "serde_json",
@@ -4948,8 +4949,8 @@ dependencies = [

 [[package]]
 name = "lance-datafusion"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4979,8 +4980,8 @@ dependencies = [

 [[package]]
 name = "lance-datagen"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4997,8 +4998,8 @@ dependencies = [

 [[package]]
 name = "lance-derive"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5007,8 +5008,8 @@ dependencies = [

 [[package]]
 name = "lance-encoding"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5043,8 +5044,8 @@ dependencies = [

 [[package]]
 name = "lance-file"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5074,8 +5075,8 @@ dependencies = [

 [[package]]
 name = "lance-index"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -5142,8 +5143,8 @@ dependencies = [

 [[package]]
 name = "lance-index-core"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5165,8 +5166,8 @@ dependencies = [

 [[package]]
 name = "lance-io"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow",
 "arrow-arith",
@@ -5209,8 +5210,8 @@ dependencies = [

 [[package]]
 name = "lance-linalg"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5226,8 +5227,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5239,8 +5240,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace-impls"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -5294,8 +5295,8 @@ dependencies = [

 [[package]]
 name = "lance-select"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5310,8 +5311,8 @@ dependencies = [

 [[package]]
 name = "lance-table"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5350,8 +5351,8 @@ dependencies = [

 [[package]]
 name = "lance-testing"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5364,8 +5365,8 @@ dependencies = [

 [[package]]
 name = "lance-tokenizer"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "icu_segmenter",
 "jieba-rs",
@@ -5378,7 +5379,7 @@ dependencies = [

 [[package]]
 name = "lancedb"
-version = "0.32.0-beta.2"
+version = "0.32.0-beta.3"
 dependencies = [
 "ahash",
 "anyhow",
@@ -5466,7 +5467,7 @@ dependencies = [

 [[package]]
 name = "lancedb-nodejs"
-version = "0.32.0-beta.2"
+version = "0.32.0-beta.3"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5491,7 +5492,7 @@ dependencies = [

 [[package]]
 name = "lancedb-python"
-version = "0.35.0-beta.2"
+version = "0.35.0-beta.3"
 dependencies = [
 "arrow",
 "async-trait",
@@ -7802,6 +7803,18 @@ dependencies = [
 "memchr",
 ]

+[[package]]
+name = "quick_cache"
+version = "0.6.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9"
+dependencies = [
+ "ahash",
+ "equivalent",
+ "hashbrown 0.16.1",
+ "parking_lot",
+]
+
 [[package]]
 name = "quinn"
 version = "0.11.9"
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=10.0.0-beta.3", default-features = false, "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=10.0.0-beta.3", default-features = false, "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=10.0.0-beta.3", default-features = false, "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }
@@ -453,6 +453,16 @@ paths:
                    The metric type to use for the index. l2, Cosine, Dot are supported.
                index_type:
                  type: string
+                custom_stop_words:
+                  type: [array, "null"]
+                  items:
+                    type: string
+                  description: |
+                    The custom stop-word list for an FTS index. A non-null
+                    array replaces the language's built-in stop-word list and is only
+                    applied when remove_stop_words is enabled. Null uses the built-in
+                    language list, while an empty array explicitly replaces it with no
+                    stop words.
      responses:
        "200":
          description: Index successfully created
@@ -510,4 +520,4 @@ paths:
        "401":
          $ref: "#/components/responses/unauthorized"
        "404":
-          $ref: "#/components/responses/not_found"
+          $ref: "#/components/responses/not_found"
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-core</artifactId>
-    <version>0.32.0-beta.2</version>
+    <version>0.32.0-beta.3</version>
 </dependency>
 ```

@@ -56,6 +56,21 @@ the experimental FTS V3 format and may introduce breaking changes.

 ***

+### customStopWords?
+
+```ts
+optional customStopWords: string[];
+```
+
+Custom stop words that replace the built-in list for `language`.
+
+This option only affects tokenization when `removeStopWords` is true.
+
+`undefined` keeps the built-in language list. An empty array explicitly
+replaces it with no stop words.
+
+***
+
 ### language?

 ```ts
@@ -30,6 +30,21 @@ The tokenizer to use. The default is "simple".

 ***

+### customStopWords?
+
+```ts
+optional customStopWords: string[];
+```
+
+Custom stop words that replace the built-in list for `language`.
+
+This option only affects tokenization when `removeStopWords` is true.
+
+`undefined` keeps the built-in language list. An empty array explicitly
+replaces it with no stop words.
+
+***
+
 ### language?

 ```ts
@@ -94,9 +94,33 @@ of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and

 ## Full text search

-Use [lancedb.table.Table.create_fts_index][] for the synchronous API or
-[lancedb.table.AsyncTable.create_index][] with [lancedb.index.FTS][] for the
-asynchronous API.
+Pass `custom_stop_words` to [lancedb.index.FTS][]:
+
+```python
+from lancedb.index import FTS
+
+table.create_index(
+    "text",
+    config=FTS(remove_stop_words=True, custom_stop_words=["acme", "internal"]),
+)
+```
+
+The list replaces the built-in stop words and is used only when
+`remove_stop_words=True`:
+
+- `custom_stop_words=None` uses the built-in list for `language`.
+- `custom_stop_words=[]` removes no words.
+- Values are passed through without trimming, lowercasing, or other rewriting.
+
+The same option is available on `lancedb.tokenize(...)` and the deprecated
+[lancedb.table.Table.create_fts_index][] compatibility helper:
+
+```python
+import lancedb
+
+tokens = list(lancedb.tokenize("acme makes searchable data",
+                               custom_stop_words=["acme"]))
+```

 ::: lancedb.index.FTS

@@ -8,7 +8,7 @@
    <parent>
      <groupId>com.lancedb</groupId>
      <artifactId>lancedb-parent</artifactId>
-      <version>0.32.0-beta.2</version>
+      <version>0.32.0-beta.3</version>
      <relativePath>../pom.xml</relativePath>
    </parent>

@@ -6,7 +6,7 @@

    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-parent</artifactId>
-    <version>0.32.0-beta.2</version>
+    <version>0.32.0-beta.3</version>
    <packaging>pom</packaging>
    <name>${project.artifactId}</name>
    <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>10.0.0-beta.3</lance-core.version>
+        <lance-core.version>10.0.0-beta.5</lance-core.version>
        <spotless.skip>false</spotless.skip>
        <spotless.version>2.30.0</spotless.version>
        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.32.0-beta.2"
+version = "0.32.0-beta.3"
 publish = false
 license.workspace = true
 description.workspace = true
@@ -226,7 +226,7 @@ describe("remote connection", () => {
    );
  });

-  it("sends the FTS posting block size to remote tables", async () => {
+  it("sends FTS options to remote tables", async () => {
    let createIndexBody: Record<string, unknown> | undefined;

    await withMockDatabase(
@@ -264,7 +264,11 @@ describe("remote connection", () => {
      async (db) => {
        const table = await db.openTable("t");
        await table.createIndex("text", {
-          config: Index.fts({ blockSize: 256 }),
+          config: Index.fts({
+            blockSize: 256,
+            removeStopWords: true,
+            customStopWords: ["the"],
+          }),
        });
      },
    );
@@ -272,6 +276,7 @@ describe("remote connection", () => {
    expect(createIndexBody?.["column"]).toBe("text");
    expect(createIndexBody?.["index_type"]).toBe("FTS");
    expect(createIndexBody?.["block_size"]).toBe(256);
+    expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]);
  });

  it("diffs and merges remote branches", async () => {
@@ -2761,6 +2761,15 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
  },
 );

+test("tokenize supports custom stop words", async () => {
+  const tokens = await tokenize("the lance data", {
+    stem: false,
+    removeStopWords: true,
+    customStopWords: ["lance"],
+  });
+  expect(tokens.map((token) => token.text)).toEqual(["the", "data"]);
+});
+
 describe("when calling explainPlan", () => {
  let tmpDir: tmp.DirResult;
  let table: Table;
@@ -29,8 +29,14 @@ test("full text search", async () => {
    const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });

    await tbl.createIndex("doc", {
-      config: lancedb.Index.fts(),
+      config: lancedb.Index.fts({
+        stem: false,
+        removeStopWords: true,
+        customStopWords: ["banana"],
+      }),
    });
+    const tokens = await tbl.tokenize("apple banana", { column: "doc" });
+    expect(tokens.map((token) => token.text)).toEqual(["apple"]);

    // --8<-- [start:full_text_search]
    const result = await tbl
@@ -194,6 +194,16 @@ export interface TokenizeOptions {
  /** Whether to remove stop words. */
  removeStopWords?: boolean;

+  /**
+   * Custom stop words that replace the built-in list for `language`.
+   *
+   * This option only affects tokenization when `removeStopWords` is true.
+   *
+   * `undefined` keeps the built-in language list. An empty array explicitly
+   * replaces it with no stop words.
+   */
+  customStopWords?: string[];
+
  /** Whether to fold ASCII characters. */
  asciiFolding?: boolean;

@@ -225,6 +235,7 @@ export async function tokenize(
    options?.lowercase,
    options?.stem,
    options?.removeStopWords,
+    options?.customStopWords,
    options?.asciiFolding,
    options?.ngramMinLength,
    options?.ngramMaxLength,
@@ -553,6 +553,16 @@ export interface FtsOptions {
   */
  removeStopWords?: boolean;

+  /**
+   * Custom stop words that replace the built-in list for `language`.
+   *
+   * This option only affects tokenization when `removeStopWords` is true.
+   *
+   * `undefined` keeps the built-in language list. An empty array explicitly
+   * replaces it with no stop words.
+   */
+  customStopWords?: string[];
+
  /**
   * whether to remove punctuation
   */
@@ -755,6 +765,7 @@ export class Index {
        options?.lowercase,
        options?.stem,
        options?.removeStopWords,
+        options?.customStopWords,
        options?.asciiFolding,
        options?.ngramMinLength,
        options?.ngramMaxLength,
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.32.0-beta.2",
+	"version": "0.32.0-beta.3",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.32.0-beta.2",
+	"version": "0.32.0-beta.3",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.32.0-beta.2",
+	"version": "0.32.0-beta.3",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.32.0-beta.2",
+	"version": "0.32.0-beta.3",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.32.0-beta.2",
+	"version": "0.32.0-beta.3",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",
@@ -1,6 +1,6 @@
 {
  "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.32.0-beta.2",
+  "version": "0.32.0-beta.3",
  "os": [
    "win32"
  ],
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.32.0-beta.2",
+	"version": "0.32.0-beta.3",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
@@ -1,12 +1,12 @@
 {
  "name": "@lancedb/lancedb",
-  "version": "0.32.0-beta.2",
+  "version": "0.32.0-beta.3",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "@lancedb/lancedb",
-      "version": "0.32.0-beta.2",
+      "version": "0.32.0-beta.3",
      "cpu": [
        "x64",
        "arm64"
@@ -11,7 +11,7 @@
    "ann"
  ],
  "private": false,
-  "version": "0.32.0-beta.2",
+  "version": "0.32.0-beta.3",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
@@ -43,6 +43,7 @@ pub fn tokenize(
    lower_case: Option<bool>,
    stem: Option<bool>,
    remove_stop_words: Option<bool>,
+    custom_stop_words: Option<Vec<String>>,
    ascii_folding: Option<bool>,
    ngram_min_length: Option<u32>,
    ngram_max_length: Option<u32>,
@@ -72,6 +73,7 @@ pub fn tokenize(
    if let Some(remove_stop_words) = remove_stop_words {
        opts = opts.remove_stop_words(remove_stop_words);
    }
+    opts = opts.custom_stop_words(custom_stop_words);
    if let Some(ascii_folding) = ascii_folding {
        opts = opts.ascii_folding(ascii_folding);
    }
@@ -222,6 +224,7 @@ impl Index {
        lower_case: Option<bool>,
        stem: Option<bool>,
        remove_stop_words: Option<bool>,
+        custom_stop_words: Option<Vec<String>>,
        ascii_folding: Option<bool>,
        ngram_min_length: Option<u32>,
        ngram_max_length: Option<u32>,
@@ -250,6 +253,7 @@ impl Index {
        if let Some(remove_stop_words) = remove_stop_words {
            opts = opts.remove_stop_words(remove_stop_words);
        }
+        opts = opts.custom_stop_words(custom_stop_words);
        if let Some(ascii_folding) = ascii_folding {
            opts = opts.ascii_folding(ascii_folding);
        }
@@ -258,6 +258,7 @@ def tokenize(
    lower_case: bool = True,
    stem: bool = True,
    remove_stop_words: bool = True,
+    custom_stop_words: Optional[List[str]] = None,
    ascii_folding: bool = True,
    ngram_min_length: int = 3,
    ngram_max_length: int = 3,
@@ -265,9 +266,10 @@ def tokenize(
 ) -> Iterable[FtsToken]:
    """Tokenize a full-text search query using an explicit tokenizer.

-    This does not require a table or FTS index. The tokenizer options match
-    :class:`lancedb.index.FTS`.
+    This does not require an FTS index. The tokenizer options match
+    :class:`lancedb.index.FTS`. ``custom_stop_words`` accepts a list of strings.
    """
+
    return _tokenize(
        query,
        base_tokenizer=base_tokenizer,
@@ -276,6 +278,7 @@ def tokenize(
        lower_case=lower_case,
        stem=stem,
        remove_stop_words=remove_stop_words,
+        custom_stop_words=custom_stop_words,
        ascii_folding=ascii_folding,
        ngram_min_length=ngram_min_length,
        ngram_max_length=ngram_max_length,
@@ -59,6 +59,7 @@ def tokenize(
    lower_case: bool = True,
    stem: bool = True,
    remove_stop_words: bool = True,
+    custom_stop_words: Optional[List[str]] = None,
    ascii_folding: bool = True,
    ngram_min_length: int = 3,
    ngram_max_length: int = 3,
@@ -2,7 +2,7 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors

 from dataclasses import dataclass
-from typing import Literal, Optional
+from typing import List, Literal, Optional

 from ._lancedb import (
    IndexConfig,
@@ -151,6 +151,11 @@ class FTS:
    remove_stop_words : bool, default True
        Whether to remove stop words. Stop words are common words that are often
        removed from text before indexing. For example, in English "the" and "and".
+    custom_stop_words : list of str, optional
+        Custom words replace the built-in language stop words
+        and only take effect when ``remove_stop_words`` is True. ``None`` uses
+        the built-in language list, while an empty list explicitly uses no
+        stop words.
    ascii_folding : bool, default True
        Whether to fold ASCII characters. This converts accented characters to
        their ASCII equivalent. For example, "café" would be converted to "cafe".
@@ -179,6 +184,7 @@ class FTS:
    ngram_max_length: int = 3
    prefix_only: bool = False
    block_size: int = 128
+    custom_stop_words: Optional[List[str]] = None


@dataclass
@@ -340,6 +340,7 @@ class RemoteTable(Table):
        lower_case: bool = True,
        stem: bool = True,
        remove_stop_words: bool = True,
+        custom_stop_words: Optional[List[str]] = None,
        ascii_folding: bool = True,
        ngram_min_length: int = 3,
        ngram_max_length: int = 3,
@@ -361,6 +362,7 @@ class RemoteTable(Table):
            lower_case=lower_case,
            stem=stem,
            remove_stop_words=remove_stop_words,
+            custom_stop_words=custom_stop_words,
            ascii_folding=ascii_folding,
            ngram_min_length=ngram_min_length,
            ngram_max_length=ngram_max_length,
@@ -1102,6 +1102,7 @@ class Table(ABC):
        lower_case: bool = True,
        stem: bool = True,
        remove_stop_words: bool = True,
+        custom_stop_words: Optional[List[str]] = None,
        ascii_folding: bool = True,
        ngram_min_length: int = 3,
        ngram_max_length: int = 3,
@@ -1169,6 +1170,9 @@ class Table(ABC):
        remove_stop_words : bool, default True
            Whether to remove stop words. Stop words are common words that are often
            removed from text before indexing. For example, in English "the" and "and".
+        custom_stop_words : list of str, optional
+            Custom words that replace the built-in language stop words. ``None``
+            uses the built-in list; an empty list explicitly uses no stop words.
        ascii_folding : bool, default True
            Whether to fold ASCII characters. This converts accented characters to
            their ASCII equivalent. For example, "café" would be converted to "cafe".
@@ -3027,6 +3031,7 @@ class LanceTable(Table):
        lower_case: bool = True,
        stem: bool = True,
        remove_stop_words: bool = True,
+        custom_stop_words: Optional[List[str]] = None,
        ascii_folding: bool = True,
        ngram_min_length: int = 3,
        ngram_max_length: int = 3,
@@ -3073,6 +3078,7 @@ class LanceTable(Table):
                "lower_case": lower_case,
                "stem": stem,
                "remove_stop_words": remove_stop_words,
+                "custom_stop_words": custom_stop_words,
                "ascii_folding": ascii_folding,
                "ngram_min_length": ngram_min_length,
                "ngram_max_length": ngram_max_length,
@@ -3080,6 +3086,7 @@ class LanceTable(Table):
            }
        else:
            tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
+            tokenizer_configs["custom_stop_words"] = custom_stop_words

        config = FTS(block_size=block_size, **tokenizer_configs)

@@ -219,11 +219,13 @@ def test_create_inverted_index(table, with_position):
        table.create_fts_index(
            "text",
            with_position=with_position,
+            custom_stop_words=["puppy"],
            name="custom_fts_index",
        )
    indices = table.list_indices()
    fts_indices = [i for i in indices if i.index_type == "FTS"]
    assert any(i.name == "custom_fts_index" for i in fts_indices)
+    assert fts_indices[0].index_details["custom_stop_words"] == ["puppy"]


@pytest.mark.parametrize("block_size", [128, 256])
@@ -243,6 +245,24 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
        table.create_index("text", config=FTS(block_size=129))


+def test_custom_stop_words_list(table):
+    table.create_index(
+        "text",
+        config=FTS(stem=False, custom_stop_words=["lance"]),
+    )
+
+    assert table.list_indices()[0].index_details["custom_stop_words"] == ["lance"]
+    tokens = table.tokenize("the lance data", column="text")
+    assert [token.text for token in tokens] == ["the", "data"]
+    empty_tokens = ldb.tokenize("the lance data", stem=False, custom_stop_words=[])
+    assert [token.text for token in empty_tokens] == ["the", "lance", "data"]
+    with pytest.raises(TypeError, match=r"custom_stop_words.*int"):
+        ldb.tokenize(
+            "the lance data",
+            custom_stop_words=["lance", 42],
+        )
+
+
 def test_search_fts(table):
    table.create_fts_index("text")
    results = table.search("puppy").select(["id", "text"]).limit(5).to_list()
@@ -771,6 +771,7 @@ def test_table_create_indices():
                "text",
                wait_timeout=timedelta(seconds=2),
                block_size=256,
+                custom_stop_words=["cloud"],
                name="custom_fts_idx",
            )

@@ -795,6 +796,7 @@ def test_table_create_indices():
        assert "name" in fts_req
        assert fts_req["name"] == "custom_fts_idx"
        assert fts_req["block_size"] == 256
+        assert fts_req["custom_stop_words"] == ["cloud"]

        # Check vector index request has custom name
        vector_req = received_requests[2]
@@ -59,7 +59,8 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                    .ascii_folding(params.ascii_folding)
                    .ngram_min_length(params.ngram_min_length)
                    .ngram_max_length(params.ngram_max_length)
-                    .ngram_prefix_only(params.prefix_only);
+                    .ngram_prefix_only(params.prefix_only)
+                    .custom_stop_words(params.custom_stop_words);
                let inner_opts = inner_opts
                    .block_size(params.block_size)
                    .map_err(|err| PyValueError::new_err(err.to_string()))?;
@@ -206,6 +207,7 @@ struct FtsParams {
    lower_case: bool,
    stem: bool,
    remove_stop_words: bool,
+    custom_stop_words: Option<Vec<String>>,
    ascii_folding: bool,
    ngram_min_length: u32,
    ngram_max_length: u32,
@@ -520,6 +520,7 @@ impl From<LanceDbFtsToken> for FtsToken {
    lower_case = true,
    stem = true,
    remove_stop_words = true,
+    custom_stop_words = None,
    ascii_folding = true,
    ngram_min_length = 3,
    ngram_max_length = 3,
@@ -534,6 +535,7 @@ pub fn tokenize(
    lower_case: bool,
    stem: bool,
    remove_stop_words: bool,
+    custom_stop_words: Option<Vec<String>>,
    ascii_folding: bool,
    ngram_min_length: u32,
    ngram_max_length: u32,
@@ -555,7 +557,8 @@ pub fn tokenize(
        .ascii_folding(ascii_folding)
        .ngram_min_length(ngram_min_length)
        .ngram_max_length(ngram_max_length)
-        .ngram_prefix_only(prefix_only);
+        .ngram_prefix_only(prefix_only)
+        .custom_stop_words(custom_stop_words);
    let tokens = lancedb_tokenize(&query, &params).infer_error()?;
    Ok(tokens.into_iter().map(FtsToken::from).collect())
 }
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.32.0-beta.2"
+version = "0.32.0-beta.3"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -76,7 +76,12 @@ async fn create_table(db: &Connection) -> Result<Table> {

 async fn create_index(table: &Table) -> Result<()> {
    table
-        .create_index(&["doc"], Index::FTS(FtsIndexBuilder::default()))
+        .create_index(
+            &["doc"],
+            Index::FTS(
+                FtsIndexBuilder::default().custom_stop_words(Some(vec!["example".to_owned()])),
+            ),
+        )
        .execute()
        .await?;
    Ok(())
@@ -4505,6 +4505,19 @@ mod tests {
                },
                Index::FTS(InvertedIndexParams::default().block_size(256).unwrap()),
            ),
+            (
+                "FTS",
+                {
+                    let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
+                    body["custom_stop_words"] = json!(["cat", " cat ", "CAT"]);
+                    body
+                },
+                Index::FTS(InvertedIndexParams::default().custom_stop_words(Some(vec![
+                    "cat".to_string(),
+                    " cat ".to_string(),
+                    "CAT".to_string(),
+                ]))),
+            ),
        ];

        for (index_type, expected_body, index) in cases {
@@ -5036,8 +5049,9 @@ mod tests {
            "max_token_length": 40,
            "lower_case": true,
            "stem": false,
-            "remove_stop_words": false,
+            "remove_stop_words": true,
            "ascii_folding": true,
+            "custom_stop_words": ["hello"],
        })
        .to_string();
        let table = Table::new_with_handler("my_table", move |request| {
@@ -5075,10 +5089,6 @@ mod tests {
        assert_eq!(
            tokens,
            vec![
-                FtsToken {
-                    text: "hello".to_string(),
-                    position: 0,
-                },
                FtsToken {
                    text: "こんにちは".to_string(),
                    position: 1,
@@ -21,6 +21,7 @@ use lance::dataset::WriteMode;
 use lance::dataset::builder::DatasetBuilder;
 use lance::dataset::{InsertBuilder, WriteParams};
 use lance::index::DatasetIndexExt;
+use lance::index::scalar::load_segment_params;
 use lance::io::{ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_index::IndexCriteria;
@@ -3131,10 +3132,9 @@ impl BaseTable for NativeTable {
    async fn list_indices(&self) -> Result<Vec<IndexConfig>> {
        let dataset = self.dataset.get().await?;
        let total_rows = dataset.count_rows(None).await? as u64;
-        let indices = dataset
-            .describe_indices(None)
-            .await?
-            .into_iter()
+        let descriptions = dataset.describe_indices(None).await?;
+        let mut indices: Vec<IndexConfig> = descriptions
+            .iter()
            .filter_map(|idx_desc| {
                let index_type: crate::index::IndexType = idx_desc
                    .index_type()
@@ -3192,6 +3192,31 @@ impl BaseTable for NativeTable {
                })
            })
            .collect();
+
+        for index in indices
+            .iter_mut()
+            .filter(|index| index.index_type == crate::index::IndexType::FTS)
+        {
+            let Some(description) = descriptions
+                .iter()
+                .find(|description| description.name() == index.name)
+            else {
+                continue;
+            };
+            let segments = description.segments();
+            let Some(segment) = segments.first() else {
+                continue;
+            };
+            let params = load_segment_params(&dataset, segment).await?;
+            let details = serde_json::to_string(&params).map_err(|source| Error::Other {
+                message: format!(
+                    "Failed to serialize full text search configuration for index '{}'",
+                    index.name
+                ),
+                source: Some(Box::new(source)),
+            })?;
+            index.index_details = Some(details);
+        }
        Ok(indices)
    }

@@ -1366,11 +1366,19 @@ mod tests {
        table
            .create_index(
                &["text"],
-                Index::FTS(FtsIndexBuilder::default().block_size(256).unwrap()),
+                Index::FTS(
+                    FtsIndexBuilder::default()
+                        .stem(false)
+                        .custom_stop_words(Some(vec!["cat".to_string()]))
+                        .block_size(256)
+                        .unwrap(),
+                ),
            )
            .execute()
            .await
            .unwrap();
+        drop(table);
+        let table = conn.open_table("test_bitmap").execute().await.unwrap();
        let index_configs = table.list_indices().await.unwrap();
        assert_eq!(index_configs.len(), 1);
        let index = index_configs.into_iter().next().unwrap();
@@ -1381,6 +1389,32 @@ mod tests {
        let index_params: FtsIndexBuilder =
            serde_json::from_str(index.index_details.as_deref().unwrap()).unwrap();
        assert_eq!(index_params.posting_block_size(), 256);
+        assert_eq!(
+            serde_json::to_value(&index_params).unwrap()["custom_stop_words"],
+            serde_json::json!(["cat"])
+        );
+        assert_eq!(
+            table
+                .tokenize("cat dog", "text_idx")
+                .await
+                .unwrap()
+                .into_iter()
+                .map(|token| token.text)
+                .collect::<Vec<_>>(),
+            vec!["dog"]
+        );
+
+        let batches = table
+            .query()
+            .full_text_search(FullTextSearchQuery::new("cat dog".to_string()))
+            .limit(120)
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 40);

        let num_rows = 120;
        let stats = table.index_stats("text_idx").await.unwrap().unwrap();