Merge origin/main into gatekeeper/fix-2820-1

python/Cargo.toml

@@ -26,7 +26,9 @@ lance-namespace-impls.workspace = true
 lance-io.workspace = true
 env_logger.workspace = true
 log.workspace = true
-pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "chrono"] }
+# Maturin enables extension-module mode for Python builds. Keeping it out of
+# Cargo features lets Rust unit tests link against libpython.
+pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] }
 chrono.workspace = true
 pyo3-async-runtimes = { version = "0.28", features = [
     "attributes",
@@ -41,10 +43,7 @@ tokio.workspace = true
 libc = "0.2"
 
 [build-dependencies]
-pyo3-build-config = { version = "0.28", features = [
-    "extension-module",
-    "abi3-py310",
-] }
+pyo3-build-config = { version = "0.28", features = ["abi3-py310"] }
 
 [features]
 default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"]

python/README.md

@@ -38,6 +38,25 @@ Stable releases are created about every 2 weeks. For the latest features and bug
 pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
 ```
 
+### Threading in CPU-limited containers
+
+LanceDB uses separate pools for compute work and storage I/O. On a container with
+two visible CPUs, current releases intentionally use one compute worker by default;
+no manual configuration is needed. If every query logs an I/O core reservation
+warning on a two-CPU container, upgrade from LanceDB 0.21.1 or earlier.
+
+The two commonly tuned environment variables control different resources:
+
+- `LANCE_CPU_THREADS` overrides the number of compute workers. One worker is the
+  appropriate setting for a two-CPU container when an explicit override is needed.
+- `LANCE_IO_THREADS` controls concurrent storage operations, not reserved CPU
+  cores. Its default can be greater than the number of CPUs because I/O workers
+  spend much of their time waiting for storage.
+
+Keep the defaults unless measurements show that the workload benefits from an
+override. See the [Lance threading model](https://lance.org/guide/performance/#threading-model)
+for the current defaults and tuning guidance.
+
 ## Usage
 
 ### Basic Example

python/pyproject.toml

@@ -103,7 +103,7 @@ python-source = "python"
 module-name = "lancedb._lancedb"
 
 [build-system]
-requires = ["maturin>=1.4"]
+requires = ["maturin>=1.9.4"]
 build-backend = "maturin"
 
 [tool.ruff.lint]

python/python/lancedb/index.py

@@ -163,6 +163,15 @@ class FTS:
         The number of documents per compressed posting block. Supported values
         are 128 and 256. A value of 256 uses the experimental FTS V3 format
         and may introduce breaking changes.
+    memory_limit : int, optional
+        The total memory limit in MiB for the local FTS build stage. The limit
+        is divided evenly among indexing workers. This build-only setting is
+        not persisted with the index and does not apply to remote tables.
+    num_workers : int, optional
+        The number of workers for a local FTS build. By default Lance uses
+        roughly half of the available CPU cores. The effective value is
+        limited by the available compute capacity. This build-only setting is
+        not persisted with the index and does not apply to remote tables.
 
     Notes
     -----
@@ -185,6 +194,8 @@ class FTS:
     prefix_only: bool = False
     block_size: int = 128
     custom_stop_words: Optional[List[str]] = None
+    memory_limit: Optional[int] = None
+    num_workers: Optional[int] = None
 
 
 @dataclass

python/python/tests/test_fts.py

@@ -245,6 +245,14 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
         table.create_index("text", config=FTS(block_size=129))
 
 
+def test_create_inverted_index_respects_build_memory_limit(table):
+    with pytest.raises(ValueError, match="exceeds worker memory limit"):
+        table.create_index(
+            "text",
+            config=FTS(memory_limit=0, num_workers=1),
+        )
+
+
 def test_custom_stop_words_list(table):
     table.create_index(
         "text",

python/src/index.rs

@@ -42,7 +42,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
             "Fm" => Ok(LanceDbIndex::Fm(FmIndexBuilder::default())),
             "FTS" => {
                 let params = source.extract::<FtsParams>()?;
-                let inner_opts = FtsIndexBuilder::default()
+                let mut inner_opts = FtsIndexBuilder::default()
                     .base_tokenizer(params.base_tokenizer)
                     .language(&params.language)
                     .map_err(|_| {
@@ -61,6 +61,12 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     .ngram_max_length(params.ngram_max_length)
                     .ngram_prefix_only(params.prefix_only)
                     .custom_stop_words(params.custom_stop_words);
+                if let Some(memory_limit) = params.memory_limit {
+                    inner_opts = inner_opts.memory_limit_mb(memory_limit);
+                }
+                if let Some(num_workers) = params.num_workers {
+                    inner_opts = inner_opts.num_workers(num_workers);
+                }
                 let inner_opts = inner_opts
                     .block_size(params.block_size)
                     .map_err(|err| PyValueError::new_err(err.to_string()))?;
@@ -213,6 +219,8 @@ struct FtsParams {
     ngram_max_length: u32,
     prefix_only: bool,
     block_size: usize,
+    memory_limit: Option<u64>,
+    num_workers: Option<usize>,
 }
 
 #[derive(FromPyObject)]
@@ -444,3 +452,51 @@ impl IndexConfig {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pyo3::types::{PyDict, PyDictMethods};
+    use serde_json::json;
+
+    #[test]
+    fn fts_build_controls_are_forwarded() {
+        Python::initialize();
+        Python::attach(|py| {
+            let locals = PyDict::new(py);
+            py.run(
+                c"class FTS:
+    with_position = True
+    base_tokenizer = 'simple'
+    language = 'English'
+    max_token_length = None
+    lower_case = True
+    stem = False
+    remove_stop_words = False
+    custom_stop_words = None
+    ascii_folding = False
+    ngram_min_length = 3
+    ngram_max_length = 3
+    prefix_only = False
+    block_size = 128
+    memory_limit = 2048
+    num_workers = 7
+
+config = FTS()",
+                None,
+                Some(&locals),
+            )
+            .unwrap();
+
+            let config = locals.get_item("config").unwrap().unwrap();
+            let index = extract_index_params(&Some(config)).unwrap();
+            let LanceDbIndex::FTS(params) = index else {
+                panic!("expected FTS index parameters");
+            };
+            let training_json = params.to_training_json().unwrap();
+
+            assert_eq!(training_json.get("memory_limit"), Some(&json!(2048)));
+            assert_eq!(training_json.get("num_workers"), Some(&json!(7)));
+        });
+    }
+}