fix(python): expose FTS build memory limits (#3796)

## Summary

- expose `memory_limit` and `num_workers` on the Python FTS
configuration for local builds
- forward both build-only settings to the Lance inverted-index builder
- add an end-to-end regression proving the configured memory budget
reaches the native build

## Root cause

LanceDB 0.26.1 pinned Lance 1.0.1. That Lance version used an FTS
partition-merge path whose retained data made memory grow with merge
progress on very large indexes. Upstream Lance
[#5754](https://github.com/lance-format/lance/pull/5754) changed
partition merging to stream its inputs, reducing peak memory by about
25%. Lance [#6174](https://github.com/lance-format/lance/pull/6174) then
removed the old merge phase, compressed posting lists during
construction, reduced indexing memory by about 60%, and introduced a
total build `memory_limit` for bounded workers.

Current `main` pins Lance 11.0.0-beta.3, which contains those
architectural fixes. This PR does not duplicate or claim the upstream
leak fix; it addresses the remaining Python API gap.

## This repair

LanceDB Python did not expose the native FTS builder resource controls.
`memory_limit` now sets the total local-build budget in MiB, divided
among effective workers, and `num_workers` controls build parallelism.
Both are build-only settings and do not affect remote builds or
persisted index configuration.

## Validation

- `cargo check --quiet --features remote --tests --examples`
- `cargo fmt --all`
- `uv run --project python --extra tests --extra dev ruff check .`
- `uv run --project python --extra tests --extra dev ruff format --check
python/python/lancedb/index.py python/python/tests/test_fts.py`
- `uv run --project python --extra tests pytest python/tests/test_fts.py
-q` (51 passed)

Fixes #2923

<!-- lance-gatekeeper-fix:v1 agent=a1ceedf74531e0212cb6f1ebf9390a26
generation=1 -->

---------

Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>

python/Cargo.toml

@@ -26,7 +26,9 @@ lance-namespace-impls.workspace = true
 lance-io.workspace = true
 env_logger.workspace = true
 log.workspace = true
-pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "chrono"] }
+# Maturin enables extension-module mode for Python builds. Keeping it out of
+# Cargo features lets Rust unit tests link against libpython.
+pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] }
 chrono.workspace = true
 pyo3-async-runtimes = { version = "0.28", features = [
     "attributes",
@@ -41,10 +43,7 @@ tokio.workspace = true
 libc = "0.2"
 
 [build-dependencies]
-pyo3-build-config = { version = "0.28", features = [
-    "extension-module",
-    "abi3-py310",
-] }
+pyo3-build-config = { version = "0.28", features = ["abi3-py310"] }
 
 [features]
 default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"]

python/pyproject.toml

@@ -103,7 +103,7 @@ python-source = "python"
 module-name = "lancedb._lancedb"
 
 [build-system]
-requires = ["maturin>=1.4"]
+requires = ["maturin>=1.9.4"]
 build-backend = "maturin"
 
 [tool.ruff.lint]

python/python/lancedb/index.py

@@ -163,6 +163,15 @@ class FTS:
         The number of documents per compressed posting block. Supported values
         are 128 and 256. A value of 256 uses the experimental FTS V3 format
         and may introduce breaking changes.
+    memory_limit : int, optional
+        The total memory limit in MiB for the local FTS build stage. The limit
+        is divided evenly among indexing workers. This build-only setting is
+        not persisted with the index and does not apply to remote tables.
+    num_workers : int, optional
+        The number of workers for a local FTS build. By default Lance uses
+        roughly half of the available CPU cores. The effective value is
+        limited by the available compute capacity. This build-only setting is
+        not persisted with the index and does not apply to remote tables.
 
     Notes
     -----
@@ -185,6 +194,8 @@ class FTS:
     prefix_only: bool = False
     block_size: int = 128
     custom_stop_words: Optional[List[str]] = None
+    memory_limit: Optional[int] = None
+    num_workers: Optional[int] = None
 
 
 @dataclass

python/python/tests/test_fts.py

@@ -245,6 +245,14 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
         table.create_index("text", config=FTS(block_size=129))
 
 
+def test_create_inverted_index_respects_build_memory_limit(table):
+    with pytest.raises(ValueError, match="exceeds worker memory limit"):
+        table.create_index(
+            "text",
+            config=FTS(memory_limit=0, num_workers=1),
+        )
+
+
 def test_custom_stop_words_list(table):
     table.create_index(
         "text",

python/src/index.rs

@@ -42,7 +42,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
             "Fm" => Ok(LanceDbIndex::Fm(FmIndexBuilder::default())),
             "FTS" => {
                 let params = source.extract::<FtsParams>()?;
-                let inner_opts = FtsIndexBuilder::default()
+                let mut inner_opts = FtsIndexBuilder::default()
                     .base_tokenizer(params.base_tokenizer)
                     .language(&params.language)
                     .map_err(|_| {
@@ -61,6 +61,12 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     .ngram_max_length(params.ngram_max_length)
                     .ngram_prefix_only(params.prefix_only)
                     .custom_stop_words(params.custom_stop_words);
+                if let Some(memory_limit) = params.memory_limit {
+                    inner_opts = inner_opts.memory_limit_mb(memory_limit);
+                }
+                if let Some(num_workers) = params.num_workers {
+                    inner_opts = inner_opts.num_workers(num_workers);
+                }
                 let inner_opts = inner_opts
                     .block_size(params.block_size)
                     .map_err(|err| PyValueError::new_err(err.to_string()))?;
@@ -213,6 +219,8 @@ struct FtsParams {
     ngram_max_length: u32,
     prefix_only: bool,
     block_size: usize,
+    memory_limit: Option<u64>,
+    num_workers: Option<usize>,
 }
 
 #[derive(FromPyObject)]
@@ -444,3 +452,51 @@ impl IndexConfig {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pyo3::types::{PyDict, PyDictMethods};
+    use serde_json::json;
+
+    #[test]
+    fn fts_build_controls_are_forwarded() {
+        Python::initialize();
+        Python::attach(|py| {
+            let locals = PyDict::new(py);
+            py.run(
+                c"class FTS:
+    with_position = True
+    base_tokenizer = 'simple'
+    language = 'English'
+    max_token_length = None
+    lower_case = True
+    stem = False
+    remove_stop_words = False
+    custom_stop_words = None
+    ascii_folding = False
+    ngram_min_length = 3
+    ngram_max_length = 3
+    prefix_only = False
+    block_size = 128
+    memory_limit = 2048
+    num_workers = 7
+
+config = FTS()",
+                None,
+                Some(&locals),
+            )
+            .unwrap();
+
+            let config = locals.get_item("config").unwrap().unwrap();
+            let index = extract_index_params(&Some(config)).unwrap();
+            let LanceDbIndex::FTS(params) = index else {
+                panic!("expected FTS index parameters");
+            };
+            let training_json = params.to_training_json().unwrap();
+
+            assert_eq!(training_json.get("memory_limit"), Some(&json!(2048)));
+            assert_eq!(training_json.get("num_workers"), Some(&json!(7)));
+        });
+    }
+}