From 0aa842b17f13d1f8ffd2851bc57c222bc5a376ba Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 9 Oct 2026 12:48:29 +0800 Subject: [PATCH 01/19] feat: support FTS indexes on JSON fields (#4219) `FTS()` rejects JSON columns because validation only considers their `LargeBinary` storage type. Validate the complete Arrow field in embedded and remote creation so `lance.json` columns reach Lance's existing JSON-aware inverted index, while ordinary binary fields remain rejected. JSON searches use path-qualified triples such as `meta.tag,str,nature`; the API documentation describes this syntax and its field-scoping behavior. Remote creation preserves the supplied FTS configuration. Two integration limits remain: malformed triples can still panic in the pinned Lance dependency (tracked by lance-format/lance#8812 and the pending fix in lance-format/lance#8813), and the opt-in server test has not run against a real deployment because no remote database is configured. That test covers index completion, statistics, search results, and index use; HTTP mock coverage only verifies the client request. Refs #3891. --- python/python/lancedb/index.py | 37 +++++++++++- python/python/tests/test_e2e_remote_db.py | 65 +++++++++++++++++++++ python/python/tests/test_fts.py | 69 +++++++++++++++++++++++ python/python/tests/test_remote_db.py | 66 ++++++++++++++++++++++ rust/lancedb/src/index.rs | 11 ++++ rust/lancedb/src/remote/db.rs | 2 +- rust/lancedb/src/remote/table.rs | 21 ++++--- rust/lancedb/src/table/create_index.rs | 6 +- rust/lancedb/src/utils/mod.rs | 57 +++++++++++++++++++ 9 files changed, 321 insertions(+), 13 deletions(-) diff --git a/python/python/lancedb/index.py b/python/python/lancedb/index.py index 2a7731be3..cf3bf8e4f 100644 --- a/python/python/lancedb/index.py +++ b/python/python/lancedb/index.py @@ -112,9 +112,42 @@ class Fm: class FTS: """Describe a FTS index configuration. - `FTS` is a full-text search index that can be used on `String` columns + `FTS` supports string columns, lists of strings, and JSON columns created + from PyArrow ``pa.json_()``. JSON columns use Lance's JSON-aware tokenizer + and are stored as ``LargeBinary`` with the ``lance.json`` logical type. + Arbitrary binary columns are not supported. - For example, it works with `title`, `description`, `content`, etc. + JSON queries use ``path,str,value`` triples. Paths are relative to the JSON + column and use dots for nested object keys. For example, + ``meta.tag,str,nature`` matches the string value at ``$.meta.tag``; + it does not search other paths or JSON key names. The value is processed + with the configured text tokenizer. Use ``str``, not ``string``. + + Create and query a JSON index through the modern API:: + + import pyarrow as pa + from lancedb.index import FTS + + table = db.create_table( + "documents", + pa.table({"doc": pa.array( + ['{"meta":{"tag":"nature"}}'], type=pa.json_() + )}), + ) + table.create_index("doc", config=FTS()) + table.search( + "meta.tag,str,nature", query_type="fts", fts_columns="doc" + ).to_list() + + Create the index on the JSON column itself, not a JSON subpath such as + ``doc.meta.tag``. JSON-path scalar indexes do not provide FTS path scoping. + Plain text such as ``nature`` is not a valid JSON FTS query. The current + Lance tokenizer can panic on malformed triples; see + https://github.com/lance-format/lance/issues/8812. + + Remote tables use the same ``FTS()`` configuration and require a server + that supports JSON FTS. Wait for ``table.wait_for_index(["doc_idx"])`` + before searching a newly created remote index with its default name. Examples -------- diff --git a/python/python/tests/test_e2e_remote_db.py b/python/python/tests/test_e2e_remote_db.py index d4364bc43..74d5cd7de 100644 --- a/python/python/tests/test_e2e_remote_db.py +++ b/python/python/tests/test_e2e_remote_db.py @@ -2,13 +2,78 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import os +from datetime import timedelta +from uuid import uuid4 + +import lancedb import numpy as np +import pyarrow as pa import pytest from lancedb import LanceDBConnection +from lancedb.index import FTS # TODO: setup integ test mark and script +@pytest.mark.skipif( + not os.environ.get("LANCEDB_URI"), reason="Set LANCEDB_URI to a JSON FTS server" +) +def test_json_fts_against_server(): + """Requires a writable database and capacity to run the distributed indexer. + + Configure LANCEDB_URI, LANCEDB_API_KEY, and optionally LANCEDB_REGION and + LANCEDB_HOST_OVERRIDE before running this test. + """ + uri = os.environ["LANCEDB_URI"] + assert uri.startswith("db://"), "LANCEDB_URI must identify a remote database" + conn = lancedb.connect( + uri, + api_key=os.environ.get("LANCEDB_API_KEY"), + region=os.environ.get("LANCEDB_REGION", "us-east-1"), + host_override=os.environ.get("LANCEDB_HOST_OVERRIDE"), + ) + name = f"json_fts_{uuid4().hex}" + table = conn.create_table( + name, + pa.table( + { + "id": [0, 1, 2], + "doc": pa.array( + [ + '{"meta":{"tag":"nature"},"author":"alice"}', + '{"meta":{"tag":"city"},"author":"nature"}', + '{"meta":{"tag":"nature"},"author":"bob"}', + ], + type=pa.json_(), + ), + "raw": pa.array([b"nature"] * 3, type=pa.large_binary()), + } + ), + ) + try: + table.create_index("doc", config=FTS(), name="doc_fts") + table.wait_for_index(["doc_fts"], timeout=timedelta(minutes=5)) + stats = table.index_stats("doc_fts") + assert stats.num_indexed_rows == 3 + assert stats.num_unindexed_rows == 0 + assert stats.index_type == "FTS" + for query, expected in [ + ("meta.tag,str,nature", [0, 2]), + ("meta.tag,str,alice", []), + ("meta.tag,str,tag", []), + ]: + search = table.search(query, query_type="fts", fts_columns="doc") + assert sorted(row["id"] for row in search.to_list()) == expected + plan = search.explain_plan(True) + assert "MatchQuery" in plan, plan + assert "FlatMatchQuery" not in plan, plan + with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"): + table.create_index("raw", config=FTS()) + finally: + conn.drop_table(name) + + @pytest.mark.skip(reason="Need to set up a local server") def test_against_local_server(): conn = LanceDBConnection("lancedb+http://localhost:10024") diff --git a/python/python/tests/test_fts.py b/python/python/tests/test_fts.py index e3963559d..d7fa2e0ec 100644 --- a/python/python/tests/test_fts.py +++ b/python/python/tests/test_fts.py @@ -45,6 +45,75 @@ from utils import exception_output TEST_LANGUAGE_MODEL_HOME = Path(__file__).parent / "models" +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +@pytest.mark.parametrize("block_size", [128, 256]) +def test_json_fts(tmp_path, block_size): + db = ldb.connect(tmp_path) + data = pa.table( + { + "id": [0, 1, 2, 3], + "doc": pa.array( + [ + '{"title":"brown fox","meta":{"tag":"nature"},"author":"alice"}', + '{"title":"blue car","meta":{"tag":"city"}}', + '{"title":"brown bear","meta":{"tag":"nature"}}', + '{"title":"red fox","author":"alice"}', + ], + type=pa.json_(), + ), + } + ) + table = db.create_table("json_fts", data) + table.create_index("doc", config=FTS(block_size=block_size), name="doc_fts") + table = db.open_table("json_fts") + index = table.list_indices()[0] + assert index.name == "doc_fts" + assert index.index_type == "FTS" + assert index.columns == ["doc"] + stats = table.index_stats("doc_fts") + assert stats.num_indexed_rows == 4 + assert stats.num_unindexed_rows == 0 + + for query, expected in [ + ("title,str,brown", [0, 2]), + ("meta.tag,str,nature", [0, 2]), + ("author,str,alice", [0, 3]), + ("title,str,alice", []), + ("title,str,title", []), + ]: + search = table.search(query, query_type="fts", fts_columns="doc") + assert sorted(row["id"] for row in search.to_list()) == expected + plan = search.explain_plan(True) + assert "MatchQuery" in plan, plan + assert "FlatMatchQuery" not in plan, plan + + +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +@pytest.mark.asyncio +async def test_json_fts_async(tmp_path): + db = await ldb.connect_async(tmp_path) + table = await db.create_table( + "json_fts", + pa.table({"doc": pa.array(['{"meta":{"tag":"nature"}}'], type=pa.json_())}), + ) + await table.create_index("doc", config=FTS()) + search = table.query().nearest_to_text(MatchQuery("meta.tag,str,nature", "doc")) + assert len(await search.to_list()) == 1 + plan = await search.explain_plan(True) + assert "MatchQuery" in plan, plan + assert "FlatMatchQuery" not in plan, plan + + +@pytest.mark.parametrize("dtype", [pa.binary(), pa.large_binary()]) +def test_fts_rejects_raw_binary(tmp_path, dtype): + table = ldb.connect(tmp_path).create_table( + "binary_fts", pa.table({"doc": pa.array([b'{"title":"brown"}'], type=dtype)}) + ) + with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"): + table.create_index("doc", config=FTS()) + assert table.list_indices() == [] + + @pytest.fixture def table(tmp_path) -> ldb.table.LanceTable: # Use local random state to avoid affecting other tests diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 00e575e46..81fb3e10e 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1410,6 +1410,72 @@ def test_remote_job_wait_raises_on_failure(): job.wait() +@pytest.mark.parametrize("server_version", ["0.2.0", "0.6.0"]) +def test_remote_json_fts_config(server_version): + from lancedb.index import FTS + + received_requests = [] + + def handler(request): + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.send_header("phalanx-version", server_version) + request.end_headers() + if request.path == "/v1/table/test/describe/": + fields = [ + { + "name": "doc", + "type": {"type": "large_binary"}, + "nullable": True, + "metadata": {"ARROW:extension:name": "lance.json"}, + }, + { + "name": "raw", + "type": {"type": "large_binary"}, + "nullable": True, + }, + ] + request.wfile.write( + json.dumps({"version": 1, "schema": {"fields": fields}}).encode() + ) + elif request.path == "/v1/table/test/create_index/": + received_requests.append(read_json_body(request)) + request.wfile.write(b"{}") + else: + raise AssertionError(f"Unexpected request: {request.path}") + + with mock_lancedb_connection(handler) as db: + table = db.open_table("test") + table.create_index( + "doc", + config=FTS( + with_position=True, + base_tokenizer="whitespace", + stem=False, + remove_stop_words=False, + lower_case=False, + block_size=256, + ), + name="doc_fts", + replace=False, + ) + with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"): + table.create_index("raw", config=FTS()) + + assert len(received_requests) == 1 + request = received_requests[0] + assert request["column"] == "doc" + assert request["index_type"] == "FTS" + assert request["name"] == "doc_fts" + assert request["replace"] is False + assert request["with_position"] is True + assert request["base_tokenizer"] == "whitespace" + assert request["stem"] is False + assert request["remove_stop_words"] is False + assert request["lower_case"] is False + assert request["block_size"] == 256 + + def test_remote_create_index_new_api(caplog): received_requests = [] diff --git a/rust/lancedb/src/index.rs b/rust/lancedb/src/index.rs index cdc973044..82f8ad845 100644 --- a/rust/lancedb/src/index.rs +++ b/rust/lancedb/src/index.rs @@ -75,6 +75,17 @@ pub enum Index { /// Full text search index using BM25. /// + /// Supports strings, lists of strings, and `lance.json` logical fields + /// stored as `LargeBinary`. Raw binary fields are not supported. Lance + /// selects its JSON-aware tokenizer from the field metadata. + /// + /// JSON queries use `path,str,value` triples, where the dotted path is + /// relative to the indexed column. For example, `meta.tag,str,nature` + /// searches the string value at `$.meta.tag`, not other paths or key names. + /// Create the index on the JSON column, not on a JSON subpath. Plain text + /// queries are invalid for this tokenizer; malformed triples can panic in + /// the current Lance dependency (see [lance#8812](https://github.com/lance-format/lance/issues/8812)). + /// /// The posting block size defaults to 128. Supported values are 128 and 256; /// a value of 256 uses the experimental FTS V3 format and may introduce /// breaking changes. diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 1ea11a23d..c6462a834 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use async_trait::async_trait; diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index a17aadc9f..f0d1e554d 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -47,6 +47,7 @@ use crate::utils::background_cache::BackgroundCache; use crate::utils::{ MaxBatchLengthStream, TimeoutStream, public_fts_field_path_by_id, resolve_arrow_field_path, resolve_arrow_fts_field_path, supported_btree_data_type, supported_vector_data_type, + validate_fts_field, }; use crate::{DistanceType, Error}; use crate::{ @@ -597,6 +598,7 @@ impl RemoteTable { Index::BloomFilter(p) => ("BLOOM_FILTER", Some(to_json(p)?)), Index::RTree(p) => ("RTREE", Some(to_json(p)?)), Index::FTS(p) => { + validate_fts_field(&field)?; let mut params = to_json(p)?; if p.get_document_granularity().is_list_element() { params["document_granularity"] = "list_element".into(); @@ -7083,19 +7085,24 @@ mod tests { ]; for (index_type, expected_body, index) in cases { + let data_type = match &index { + Index::FTS(params) if params.get_document_granularity().is_list_element() => { + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))) + } + Index::FTS(_) => DataType::Utf8, + _ => DataType::Int32, + }; + let schema = Schema::new(vec![Field::new("a", data_type, false)]); let table = Table::new_with_handler_version( "my_table", semver::Version::new(0, 6, 0), move |request| { assert_eq!(request.method(), "POST"); match request.url().path() { - "/v1/table/my_table/describe/" => { - let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]); - http::Response::builder() - .status(200) - .body(describe_response(&schema)) - .unwrap() - } + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(describe_response(&schema)) + .unwrap(), "/v1/table/my_table/create_index/" => { assert_eq!( request.headers().get("Content-Type").unwrap(), diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 916b28ed1..3ac759bdd 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -29,8 +29,8 @@ use crate::index::Index; use crate::index::vector::{VectorIndex, suggested_num_sub_vectors}; use crate::utils::{ resolve_lance_fts_field_path, supported_bitmap_data_type, supported_btree_data_type, - supported_fm_data_type, supported_fts_data_type, supported_label_list_data_type, - supported_vector_data_type, supported_zonemap_data_type, + supported_fm_data_type, supported_label_list_data_type, supported_vector_data_type, + supported_zonemap_data_type, validate_fts_field, }; use super::NativeTable; @@ -292,7 +292,7 @@ impl NativeTable { )) } Index::FTS(fts_opts) => { - Self::validate_index_type(field, "FTS", supported_fts_data_type)?; + validate_fts_field(field)?; Ok(Box::new(fts_opts)) } Index::IvfFlat(index) => { diff --git a/rust/lancedb/src/utils/mod.rs b/rust/lancedb/src/utils/mod.rs index 52b7bd5e8..7692f9bb5 100644 --- a/rust/lancedb/src/utils/mod.rs +++ b/rust/lancedb/src/utils/mod.rs @@ -494,6 +494,26 @@ pub fn supported_fts_data_type(dtype: &DataType) -> bool { supported_fts_data_type_impl(dtype, false) } +/// Validate FTS input without losing the logical type in field metadata. +pub(crate) fn validate_fts_field(field: &Field) -> Result<()> { + if lance_arrow::json::is_json_field(field) || supported_fts_data_type(field.data_type()) { + return Ok(()); + } + let logical_type = field + .metadata() + .get("ARROW:extension:name") + .map(|name| format!(" (logical type {name})")) + .unwrap_or_default(); + Err(Error::Schema { + message: format!( + "A FTS index cannot be created on the field `{}` which has data type {}{}. FTS supports strings, lists of strings, and lance.json fields stored as LargeBinary; raw binary is not supported", + field.name(), + field.data_type(), + logical_type, + ), + }) +} + fn supported_fts_data_type_impl(dtype: &DataType, in_list: bool) -> bool { match (dtype, in_list) { (DataType::Utf8 | DataType::LargeUtf8, _) => true, @@ -707,6 +727,43 @@ mod tests { use super::*; + #[test] + fn test_fts_field_logical_type_validation() { + let json = lance_arrow::json::json_field("doc", true); + assert!(validate_fts_field(&json).is_ok()); + + for dtype in [DataType::Binary, DataType::LargeBinary] { + let raw = Field::new("doc", dtype.clone(), true); + let err = validate_fts_field(&raw).unwrap_err().to_string(); + assert!(err.contains("raw binary is not supported"), "{err}"); + assert!(err.contains("lance.json"), "{err}"); + + let other_extension = raw.with_metadata(std::collections::HashMap::from([( + "ARROW:extension:name".into(), + "other.json".into(), + )])); + let err = validate_fts_field(&other_extension) + .unwrap_err() + .to_string(); + assert!(err.contains("logical type other.json"), "{err}"); + } + + // A JSON marker on the wrong binary storage type is insufficient. + let wrong_storage = + Field::new("doc", DataType::Binary, true).with_metadata(json.metadata().clone()); + assert!(validate_fts_field(&wrong_storage).is_err()); + for dtype in [DataType::Utf8, DataType::LargeUtf8] { + let text = Field::new("text", dtype, true); + assert!(validate_fts_field(&text).is_ok()); + for list_type in [ + DataType::List(Arc::new(text.clone())), + DataType::LargeList(Arc::new(text.clone())), + ] { + assert!(validate_fts_field(&Field::new("texts", list_type, true)).is_ok()); + } + } + } + #[test] fn test_public_fts_field_path_prefers_exact_case() { let text_list = || { From df29f7f6022a10abc85294594c4f6611013f302c Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 9 Oct 2026 12:22:50 -0400 Subject: [PATCH 02/19] test: run lance's top-k late materialization over the table adapter (#4470) Covers lance-format/lance#9750's `TopKLateMaterialization` rule through lancedb's DataFusion `TableProvider`, now that main is on lance v14.0.0-beta.7: SQL `ORDER BY ... LIMIT` over `BaseTableAdapter` with the rule registered returns the same rows as without it, the plan contains a `Take`, and the output schema metadata stays erased. No certification of `MetadataEraserExec` is needed because DataFusion pushes the top-k sort below it. ENT-2777 --- rust/lancedb/src/table/datafusion.rs | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/rust/lancedb/src/table/datafusion.rs b/rust/lancedb/src/table/datafusion.rs index ea5222b15..26c423497 100644 --- a/rust/lancedb/src/table/datafusion.rs +++ b/rust/lancedb/src/table/datafusion.rs @@ -699,6 +699,44 @@ pub mod tests { ); } + #[tokio::test] + async fn test_topk_late_materialization_over_table_adapter() { + use lance::io::exec::topk_late_materialization::TopKLateMaterialization; + + let fixture = TestFixture::new().await; + let run = |rule: Option| { + let adapter = fixture.adapter2.clone(); + async move { + let mut state = + datafusion::execution::SessionStateBuilder::new().with_default_features(); + if let Some(rule) = rule { + state = state.with_physical_optimizer_rule(Arc::new(rule)); + } + let ctx = SessionContext::new_with_state(state.build()); + ctx.register_table("tbl2", adapter).unwrap(); + let df = ctx + .sql("SELECT ints, strings, jsons FROM tbl2 WHERE floats > 10 ORDER BY ints DESC LIMIT 5") + .await + .unwrap(); + let plan = df.clone().create_physical_plan().await.unwrap(); + let plan = datafusion_physical_plan::displayable(plan.as_ref()) + .indent(true) + .to_string(); + (plan, df.collect().await.unwrap()) + } + }; + + let (plan, expected) = run(None).await; + assert!(!plan.contains("Take"), "{plan}"); + + // The sort lands below `MetadataEraserExec`, so the rule needs no + // certification for it. + let (plan, batches) = run(Some(TopKLateMaterialization::new())).await; + assert!(plan.contains("Take"), "{plan}"); + assert_eq!(batches, expected); + assert!(batches[0].schema().metadata().is_empty()); + } + #[tokio::test] async fn test_metadata_eraser_with_new_children_takes_child_schema() { let fixture = TestFixture::new().await; From ab2ae49200ad6220e9b6b92feed600791aea6f32 Mon Sep 17 00:00:00 2001 From: Jonathan Hsieh Date: Fri, 9 Oct 2026 11:17:53 -0700 Subject: [PATCH 03/19] feat!: carry a FunctionVersion's creation time as epoch millis (#4482) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moves Function timestamps on the wire from RFC 3339 strings to epoch milliseconds. ## What - `FunctionVersion.created_at` (an RFC 3339 string) becomes `created_at_millis` (milliseconds since the epoch, as an integer). That's the shape Secrets, jobs and `FunctionErrorRecord` already use. - Rust: `FunctionVersion::created_at() -> &str` becomes `created_at_millis() -> i64`. - Python: `FunctionVersion.created_at: str` becomes `created_at_millis: int`. - Fixtures and the mock server now send `1787270400000`, the old `2026-08-21T00:00:00Z`. Node.js doesn't expose `FunctionVersion`, so it doesn't change. ## Compatibility Breaking, and it must ship together with the matching server change: a server that still sends `created_at` fails to deserialize on this client. Functions aren't deployed yet, so nothing is migrated. ## Tests - `cargo test --features remote --test first_class_function_slice1`: 10 passed - `cargo test --features remote --lib function`: 51 passed - `pytest python/tests/test_first_class_function*.py`: 100 passed - clippy, `cargo fmt --check`, ruff check and ruff format are clean 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01BY5xcEGEAa1E11VG55Kd1w Co-authored-by: Claude Opus 5.5 --- python/python/lancedb/functions.py | 2 +- python/python/tests/test_first_class_function_slice2.py | 2 +- rust/lancedb/src/function.rs | 8 +++++--- .../first_class_functions/v1/remote_function_job.json | 2 +- .../v1/remote_function_version.canonical.json | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index c170c7375..445d31027 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -372,7 +372,7 @@ class FunctionVersion(_RemoteValue): image: FunctionImage signature: FunctionSignature secret_bindings: tuple[SecretBinding, ...] = () - created_at: str + created_at_millis: int metadata: Mapping[str, str] disabled: bool diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index c18893f1b..975310ed4 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -1555,7 +1555,7 @@ def _mock_remote_function_catalog(): )["image"], "signature": body["signature"], "secret_bindings": body.get("secret_bindings", []), - "created_at": "2026-08-21T00:00:00Z", + "created_at_millis": 1787270400000, # 2026-08-21T00:00:00Z } response = {"job_id": "job-register"} status = 202 diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index f05633281..98d915118 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -453,7 +453,8 @@ pub struct FunctionVersion { signature: FunctionSignature, #[serde(default, skip_serializing_if = "Vec::is_empty")] secret_bindings: Vec, - created_at: String, + /// When this version was created, in milliseconds since the epoch. + created_at_millis: i64, metadata: BTreeMap, disabled: bool, } @@ -507,8 +508,9 @@ impl FunctionVersion { &self.secret_bindings } - pub fn created_at(&self) -> &str { - &self.created_at + /// When this version was created, in milliseconds since the epoch. + pub fn created_at_millis(&self) -> i64 { + self.created_at_millis } } diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json index c6bbd83e3..e07c29f98 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json @@ -23,7 +23,7 @@ "nullable": false } }, - "created_at": "2026-08-21T00:00:00Z", + "created_at_millis": 1787270400000, "image": { "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6", "descriptor": { diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json index 63f3c824e..44a84b17c 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json @@ -1 +1 @@ -{"created_at":"2026-08-21T00:00:00Z","disabled":false,"image":{"descriptor":{"behavior":{"result_stability":"input_and_context","side_effects":"non_idempotent"},"entrypoint":"app.function:create","format_version":1,"interface":{"type":"lance.scalar","version":1},"python":{"abi_tag":"cp312","executable":"/usr/local/bin/python3","implementation":"cpython","import_paths":["/opt/function/code"],"version":"3.12.14"},"python_api":1,"requires":{"capabilities":[],"kernel_min":"4.18.0"},"schemas":{"initialization":"/opt/function/schemas/initialization.arrow","input":"/opt/function/schemas/input.arrow","output":"/opt/function/schemas/output.arrow"}},"manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","source":false},"location":"memory:///fixture","metadata":{},"name":"embed","object_id":"fixture","secret_bindings":[{"kind":"env","secret_ref":{"name":"hf-prod"},"variable":"HF_TOKEN"}],"signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list","kind":"scalar","nullable":false}},"version":"1"} +{"created_at_millis":1787270400000,"disabled":false,"image":{"descriptor":{"behavior":{"result_stability":"input_and_context","side_effects":"non_idempotent"},"entrypoint":"app.function:create","format_version":1,"interface":{"type":"lance.scalar","version":1},"python":{"abi_tag":"cp312","executable":"/usr/local/bin/python3","implementation":"cpython","import_paths":["/opt/function/code"],"version":"3.12.14"},"python_api":1,"requires":{"capabilities":[],"kernel_min":"4.18.0"},"schemas":{"initialization":"/opt/function/schemas/initialization.arrow","input":"/opt/function/schemas/input.arrow","output":"/opt/function/schemas/output.arrow"}},"manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","source":false},"location":"memory:///fixture","metadata":{},"name":"embed","object_id":"fixture","secret_bindings":[{"kind":"env","secret_ref":{"name":"hf-prod"},"variable":"HF_TOKEN"}],"signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list","kind":"scalar","nullable":false}},"version":"1"} From f85c235d4879ffd38baff4a6c4aab5e0c35c34ba Mon Sep 17 00:00:00 2001 From: Lance Release Date: Fri, 9 Oct 2026 18:20:53 +0000 Subject: [PATCH 04/19] =?UTF-8?q?Bump=20version:=200.41.0-beta.1=20?= =?UTF-8?q?=E2=86=92=200.41.0-beta.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index f14c9b1cb..70718edbb 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.41.0-beta.1" +current_version = "0.41.0-beta.2" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index a3d865be8..eecf3d0d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5762,7 +5762,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.41.0-beta.1" +version = "0.41.0-beta.2" dependencies = [ "ahash", "anyhow", @@ -5859,7 +5859,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.41.0-beta.1" +version = "0.41.0-beta.2" dependencies = [ "arrow-array", "arrow-buffer", @@ -5884,7 +5884,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.41.0-beta.1" +version = "0.41.0-beta.2" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 20db07d56..88cfc4a30 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.41.0-beta.1 + 0.41.0-beta.2 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index c5f30e13d..987d51345 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.1 + 0.41.0-beta.2 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index d01e30465..59efeb627 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.1 + 0.41.0-beta.2 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 20363ec85..c81fdb459 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.41.0-beta.1" +version = "0.41.0-beta.2" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 9ff0cd5e8..90106b6d4 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.41.0-beta.1", + "version": "0.41.0-beta.2", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index a04377631..73bafc124 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.41.0-beta.1", + "version": "0.41.0-beta.2", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index da117d8a5..dea0f49a2 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.41.0-beta.1", + "version": "0.41.0-beta.2", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 46189293d..0ae15f6d5 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.41.0-beta.1", + "version": "0.41.0-beta.2", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index c576aa720..b3a17754b 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.41.0-beta.1", + "version": "0.41.0-beta.2", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index b73302811..4681f63c9 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.41.0-beta.1", + "version": "0.41.0-beta.2", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index bb686b5bb..bcf00b687 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.41.0-beta.1", + "version": "0.41.0-beta.2", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index 71232b770..b4b64eec1 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.41.0-beta.1", + "version": "0.41.0-beta.2", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 94fe5fddf..4f0dc603e 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.41.0-beta.1" +version = "0.41.0-beta.2" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 2d7299a0e..b15c24ca5 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.41.0-beta.1" +version = "0.41.0-beta.2" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From ca389248821ac04c32a7eaa34ff11ab87c7856d4 Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Fri, 9 Oct 2026 12:17:56 -0700 Subject: [PATCH 05/19] fix!: centralize entity name validation for remote APIs (#4490) Centralize the entity name validation for tables, views, functions, namespaces, secrets, and databases. This helps in several ways. One is that percent-encoding is no longer needed in most cases (the exception is database names, which can contain slashes). Another good thing is that LLMs looking at this repository can now understand what the naming rules are, since they will be the same here as on the server side. Finally, this will help with the upcoming authZ code, since it also needs to serialize these names, and needs to know that they don't contain (among other things) colons. --- rust/lancedb/src/connection.rs | 9 +- rust/lancedb/src/remote/catalog.rs | 49 ++-- rust/lancedb/src/remote/client.rs | 25 ++ rust/lancedb/src/remote/db.rs | 265 +++++++++++---------- rust/lancedb/src/utils/mod.rs | 354 +++++++++++++++++++---------- 5 files changed, 428 insertions(+), 274 deletions(-) diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index ff01f4109..8a0e489c2 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -38,10 +38,7 @@ use crate::remote::{ }, }; use crate::secrets::SecretInfo; -use crate::utils::{ - validate_namespace, validate_secret_component, validate_secret_reference, - validate_view_reference, -}; +use crate::utils::{validate_namespace, validate_secret_reference, validate_view_reference}; use crate::view::ViewDescription; use lance::io::ObjectStoreParams; pub use lance_file::version::LanceFileVersion; @@ -811,9 +808,7 @@ impl Connection { let database = database.clone(); let namespace_path = namespace_path.clone(); async move { - for segment in &namespace_path { - validate_secret_component("Secret namespace path segment", segment)?; - } + validate_namespace(&namespace_path)?; database.list_secrets(&namespace_path, options).await } }) diff --git a/rust/lancedb/src/remote/catalog.rs b/rust/lancedb/src/remote/catalog.rs index 72b77a87e..fa6b185cc 100644 --- a/rust/lancedb/src/remote/catalog.rs +++ b/rust/lancedb/src/remote/catalog.rs @@ -19,6 +19,7 @@ use crate::catalog::{ ListDatabasesResponse, }; use crate::database::Database; +use crate::utils::validate_database_name; use crate::{Error, Result}; /// Authentication and client settings shared by a catalog and its databases. @@ -154,29 +155,6 @@ impl RemoteCatalog { }) } - fn validate_name(&self, name: &str) -> Result<()> { - let delimiter = self - .options - .client_config - .id_delimiter - .as_deref() - .unwrap_or("$"); - if name.is_empty() - || name.trim() != name - || !name.is_ascii() - || name.chars().any(char::is_control) - || name.contains(delimiter) - || matches!(name, "." | "..") - { - return Err(Error::InvalidInput { - message: format!( - "Invalid database name '{name}': expected a nonempty ASCII name without surrounding whitespace, control characters, or namespace delimiter '{delimiter}'" - ), - }); - } - Ok(()) - } - fn database(&self, name: &str) -> Result> { Ok(Arc::new(RemoteDatabase::for_catalog( &self.endpoint, @@ -203,7 +181,7 @@ impl Catalog for RemoteCatalog { } async fn create_database(&self, request: CreateDatabaseRequest) -> Result> { - self.validate_name(&request.name)?; + validate_database_name(&request.name)?; self.root .create_namespace(CreateNamespaceRequest { id: Some(vec![request.name.clone()]), @@ -231,7 +209,7 @@ impl Catalog for RemoteCatalog { } async fn drop_database(&self, request: DropDatabaseRequest) -> Result<()> { - self.validate_name(&request.name)?; + validate_database_name(&request.name)?; let result = self .root .drop_namespace(DropNamespaceRequest { @@ -286,7 +264,7 @@ impl Catalog for RemoteCatalog { } async fn connect_database(&self, name: &str) -> Result> { - self.validate_name(name)?; + validate_database_name(name)?; self.root .describe_namespace(DescribeNamespaceRequest { id: Some(vec![name.into()]), @@ -676,7 +654,24 @@ mod tests { } let catalog = RemoteCatalog::try_new("http://127.0.0.1:1", RemoteCatalogOptions::default()).unwrap(); - for name in ["", "a$b", "\r\ninjected", "..", "café", " padded "] { + for name in [ + "", + "/", + "//", + "a$b", + "\r\ninjected", + "..", + "café", + " padded ", + "a b", + "a?b", + "/db", + "db/", + "a//b", + "a/../b", + "a/./b", + &"a".repeat(65), + ] { assert!(matches!( catalog.create_database(name.into()).await, Err(Error::InvalidInput { .. }) diff --git a/rust/lancedb/src/remote/client.rs b/rust/lancedb/src/remote/client.rs index 705506705..5f24d24c7 100644 --- a/rust/lancedb/src/remote/client.rs +++ b/rust/lancedb/src/remote/client.rs @@ -12,6 +12,7 @@ use std::{collections::HashMap, future::Future, str::FromStr, sync::Arc, time::D use crate::error::{Error, Result}; use crate::remote::db::RemoteOptions; use crate::remote::retry::{ResolvedRetryConfig, RetryCounter}; +use crate::utils::validate_database_name; const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id"); @@ -383,6 +384,7 @@ pub fn parse_db_url(db_url: &str) -> Result { message: format!("Invalid encoded database name: {err}"), })? .into_owned(); + validate_database_name(&db_name)?; let db_prefix = { let prefix = parsed_url.path().trim_start_matches('/'); if prefix.is_empty() { @@ -1204,6 +1206,29 @@ mod tests { assert_eq!(parsed.db_prefix.as_deref(), Some("prefix")); } + #[test] + fn test_parse_database_uri_validates_decoded_name() { + for name in [ + "a b", + "a$b", + "a?b", + "café", + "/db", + "db/", + "a//b", + "a/../b", + "a/./b", + &"a".repeat(65), + ] { + let uri = format!("db://{}", urlencoding::encode(name)); + assert!( + matches!(parse_db_url(&uri), Err(Error::InvalidInput { .. })), + "accepted {name:?}" + ); + } + assert!(parse_db_url(&format!("db://{}", "a".repeat(64))).is_ok()); + } + #[test] fn test_timeout_config_default() { let config = TimeoutConfig::default(); diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index c6462a834..28582cdc3 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -36,7 +36,10 @@ use crate::remote::util::stream_as_body; use crate::secrets::SecretBinding; use crate::secrets::SecretInfo; use crate::table::BaseTable; -use crate::utils::{reject_relative_segment, validate_table_name}; +use crate::utils::{ + validate_database_name, validate_function_name, validate_namespace, validate_secret_name, + validate_table_name, validate_view_name, +}; use crate::view::ViewDescription; use super::client::{ @@ -230,6 +233,7 @@ pub struct RemoteDatabase { /// TLS configuration for mTLS support tls_config: Option, sql_client: Option, + is_catalog_root: bool, } #[derive(Clone)] @@ -320,6 +324,9 @@ impl RemoteDatabase { name: Option<&str>, options: &super::catalog::RemoteCatalogOptions, ) -> Result { + if let Some(name) = name { + validate_database_name(name)?; + } let scope = super::catalog::ScopedHeaderProvider { provider: options.client_config.header_provider.clone(), database: name.map(str::to_string), @@ -348,6 +355,7 @@ impl RemoteDatabase { )?; if name.is_none() { db.sql_client = None; + db.is_catalog_root = true; } Ok(db) } @@ -422,11 +430,38 @@ impl RemoteDatabase { namespace_context_provider, tls_config: client_config.tls_config, sql_client: Some(sql_client), + is_catalog_root: false, }) } } impl RemoteDatabase { + // A catalog's root namespaces are database names, which may contain slashes. + fn namespace_identifier(&self, namespace: &[String]) -> Result { + match namespace.first() { + None => Ok(ID_DELIMITER.to_string()), + Some(first) => { + if self.is_catalog_root { + // When dealing with the catalog root, the first component is a + // database name. Those need to be validated differently because + // they can contain slashes. Also, we need to use percent encoding + // for the slashes. + validate_database_name(first)?; + validate_namespace(&namespace[1..])?; + Ok(namespace + .iter() + .map(|component| urlencoding::encode(component).into_owned()) + .collect::>() + .join(ID_DELIMITER)) + } else { + // If we're not dealing with the catalog root, build the namespace identifier in the + // usual way. + build_namespace_identifier(namespace) + } + } + } + } + /// Post a request whose body carries a credential. /// /// Shared by the create and alter verbs, which declare their own request @@ -537,6 +572,7 @@ mod test_utils { namespace_context_provider: None, tls_config: None, sql_client: None, + is_catalog_root: false, } } @@ -560,6 +596,7 @@ mod test_utils { namespace_context_provider, tls_config: config.tls_config.clone(), sql_client: None, + is_catalog_root: false, } } } @@ -581,93 +618,60 @@ impl From<&CreateTableMode> for &'static str { } } -/// The path segment addressing one object: its namespace path and its name. -/// -/// One builder for tables, Secrets, Functions and materialized views: the -/// identifier grammar belongs to the namespace spec, not to an object type. An -/// empty path addresses an object with no namespace. -/// -/// Components are checked for addressability, not a character set. The name's -/// grammar is the caller's, so a table reports [`Error::InvalidTableName`], a -/// Function admits names a table may not, and a catalog database carries the -/// `/` that [`RemoteCatalog`] allows. -/// -/// [`RemoteCatalog`]: super::catalog::RemoteCatalog -fn build_object_identifier(what: &str, name: &str, namespace: &[String]) -> Result { - for segment in namespace { - reject_unaddressable_component("namespace segment", segment)?; +fn build_compound_identifier(name: &str, namespace: &[String]) -> Result { + validate_namespace(namespace)?; + let mut identifier = namespace.join("$"); + if !namespace.is_empty() { + identifier.push('$'); } - reject_unaddressable_component(what, name)?; - Ok(join_identifier( - namespace.iter().map(String::as_str).chain([name]), - )) + identifier.push_str(name); + Ok(identifier) } -/// What a component may not be if the join is to survive being split back -/// apart: empty, a segment URL parsing resolves away, or the delimiter itself. -/// -/// Each erases a boundary no encoding of the joined form recovers. `["prod", -/// ""]` joins to `prod$`, which reads back as `["prod"]`, so a drop reaches the -/// parent of the namespace the caller named. -/// -/// Not a character set: per-component percent-encoding makes the wider set -/// safe, since a `/` in a name reaches the service as `%2F`, still one -/// segment. -fn reject_unaddressable_component(what: &str, value: &str) -> Result<()> { - if value.is_empty() { - return Err(Error::InvalidInput { - message: format!( - "{what} must not be empty: the identifier would carry two delimiters in a row, \ - and splitting it back apart would name a different object" - ), - }); - } - reject_relative_segment(what, value)?; - if value.contains(ID_DELIMITER) { - return Err(Error::InvalidInput { - message: format!( - "{what} '{value}' contains the identifier delimiter '{ID_DELIMITER}', so the \ - namespace path and the name it joins could not be told apart" - ), - }); - } - Ok(()) -} - -/// The path segment addressing one table. A wrapper for the error type: -/// callers match on [`Error::InvalidTableName`]. +/// Build a full path to a table. Note that we do not need to percent-encode +/// this when using it in a URL because the validation functions ensure that +/// only URL-safe characters are present. fn build_table_identifier(name: &str, namespace: &[String]) -> Result { validate_table_name(name)?; - build_object_identifier("table name", name, namespace) + build_compound_identifier(name, namespace) } -/// Join components into the `{id}` a route addresses: each percent-encoded, -/// then joined by the delimiter. -/// -/// Per component rather than over the joined string, so the delimiter stays a -/// delimiter and nothing inside a component can end the path segment. -/// -/// A second line, not the first: a component from the object charset is all -/// unreserved and encodes to itself, so the route reads as the caller wrote it. -/// It does not cover `.` and `..`, which are unreserved too and resolve away -/// after decoding -- [`build_object_identifier`] refuses those. -fn join_identifier<'a>(components: impl Iterator) -> String { - components - .map(|component| urlencoding::encode(component).into_owned()) - .collect::>() - .join(ID_DELIMITER) -} - -/// The path segment addressing one namespace. +/// Translate an array of namespace components into a namespace string. +/// Note that we do not need to percent-encode this when using it in a URL +/// because validate_namespace ensures that only URL-safe characters are +/// present. fn build_namespace_identifier(namespace: &[String]) -> Result { - for segment in namespace { - reject_unaddressable_component("namespace segment", segment)?; - } + validate_namespace(namespace)?; if namespace.is_empty() { // According to the namespace spec, use delimiter to represent root namespace - return Ok(ID_DELIMITER.to_string()); + Ok(ID_DELIMITER.to_string()) + } else { + Ok(namespace.join(ID_DELIMITER)) } - Ok(join_identifier(namespace.iter().map(String::as_str))) +} + +/// Build a full path to a function. Note that we do not need to percent-encode +/// this when using it in a URL because the validation functions ensure that +/// only URL-safe characters are present. +fn build_function_identifier(name: &str, namespace: &[String]) -> Result { + validate_function_name(name)?; + build_compound_identifier(name, namespace) +} + +/// Build a full path to a secret. Note that we do not need to percent-encode +/// this when using it in a URL because the validation functions ensure that +/// only URL-safe characters are present. +fn build_secret_identifier(name: &str, namespace: &[String]) -> Result { + validate_secret_name(name)?; + build_compound_identifier(name, namespace) +} + +/// Build a full path to a view. Note that we do not need to percent-encode +/// this when using it in a URL because the validation functions ensure that +/// only URL-safe characters are present. +fn build_view_identifier(name: &str, namespace: &[String]) -> Result { + validate_view_name(name)?; + build_compound_identifier(name, namespace) } /// Build a secure cache key using length prefixes. @@ -945,7 +949,7 @@ impl Database for RemoteDatabase { request: FunctionRegistrationRequest, namespace_path: &[String], ) -> Result> { - let function_id = build_object_identifier("Function name", &request.name, namespace_path)?; + let function_id = build_function_identifier(&request.name, namespace_path)?; let req = self .client .post(&format!("/v1/function/{function_id}/create")) @@ -976,7 +980,7 @@ impl Database for RemoteDatabase { version: &str, namespace_path: &[String], ) -> Result { - let function_id = build_object_identifier("Function name", name, namespace_path)?; + let function_id = build_function_identifier(name, namespace_path)?; let req = self .client .post(&format!("/v1/function/{function_id}/describe")) @@ -1036,7 +1040,7 @@ impl Database for RemoteDatabase { version: &str, namespace_path: &[String], ) -> Result<(bool, Job)> { - let function_id = build_object_identifier("Function name", name, namespace_path)?; + let function_id = build_function_identifier(name, namespace_path)?; let req = self .client .post(&format!("/v1/function/{function_id}/drop")) @@ -1081,7 +1085,7 @@ impl Database for RemoteDatabase { value: &str, namespace_path: &[String], ) -> Result<()> { - let secret_id = build_object_identifier("Secret name", name, namespace_path)?; + let secret_id = build_secret_identifier(name, namespace_path)?; self.post_secret_write( &format!("/v1/secret/{secret_id}/create"), &RemoteCreateSecretRequest { value }, @@ -1090,7 +1094,7 @@ impl Database for RemoteDatabase { } async fn alter_secret(&self, name: &str, value: &str, namespace_path: &[String]) -> Result<()> { - let secret_id = build_object_identifier("Secret name", name, namespace_path)?; + let secret_id = build_secret_identifier(name, namespace_path)?; self.post_secret_write( &format!("/v1/secret/{secret_id}/alter"), &RemoteAlterSecretRequest { value }, @@ -1127,7 +1131,7 @@ impl Database for RemoteDatabase { } async fn drop_secret(&self, name: &str, namespace_path: &[String]) -> Result<()> { - let secret_id = build_object_identifier("Secret name", name, namespace_path)?; + let secret_id = build_secret_identifier(name, namespace_path)?; let req = self.client.post(&format!("/v1/secret/{secret_id}/drop")); let (request_id, response) = self.client.send(req).await?; self.client.check_response(&request_id, response).await?; @@ -1135,7 +1139,7 @@ impl Database for RemoteDatabase { } async fn describe_secret(&self, name: &str, namespace_path: &[String]) -> Result { - let secret_id = build_object_identifier("Secret name", name, namespace_path)?; + let secret_id = build_secret_identifier(name, namespace_path)?; let req = self .client .post(&format!("/v1/secret/{secret_id}/describe")); @@ -1150,7 +1154,7 @@ impl Database for RemoteDatabase { query: &str, namespace_path: &[String], ) -> Result { - let view_id = build_object_identifier("View name", name, namespace_path)?; + let view_id = build_view_identifier(name, namespace_path)?; let req = self .client .post(&format!("/v1/view/{view_id}/create")) @@ -1167,7 +1171,7 @@ impl Database for RemoteDatabase { name: &str, namespace_path: &[String], ) -> Result { - let view_id = build_object_identifier("View name", name, namespace_path)?; + let view_id = build_view_identifier(name, namespace_path)?; let req = self.client.post(&format!("/v1/view/{view_id}/describe")); let (request_id, response) = self.client.send(req).await?; let response = self.client.check_response(&request_id, response).await?; @@ -1184,7 +1188,7 @@ impl Database for RemoteDatabase { } async fn drop_view_async(&self, name: &str, namespace_path: &[String]) -> Result { - let view_id = build_object_identifier("View name", name, namespace_path)?; + let view_id = build_view_identifier(name, namespace_path)?; let req = self.client.post(&format!("/v1/view/{view_id}/drop")); let (request_id, response) = self.client.send(req).await?; let response = self.client.check_response(&request_id, response).await?; @@ -1407,7 +1411,7 @@ impl Database for RemoteDatabase { async fn list_tables(&self, request: ListTablesRequest) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts)?; + let namespace_id = self.namespace_identifier(namespace_parts)?; let mut req = self .client .get(&format!("/v1/namespace/{}/table/list", namespace_id)); @@ -1674,7 +1678,7 @@ impl Database for RemoteDatabase { request: ListNamespacesRequest, ) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts)?; + let namespace_id = self.namespace_identifier(namespace_parts)?; let mut req = self .client .get(&format!("/v1/namespace/{}/list", namespace_id)); @@ -1696,7 +1700,7 @@ impl Database for RemoteDatabase { request: CreateNamespaceRequest, ) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts)?; + let namespace_id = self.namespace_identifier(namespace_parts)?; let mut req = self .client .post(&format!("/v1/namespace/{}/create", namespace_id)); @@ -1727,7 +1731,7 @@ impl Database for RemoteDatabase { async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts)?; + let namespace_id = self.namespace_identifier(namespace_parts)?; let mut req = self .client .post(&format!("/v1/namespace/{}/drop", namespace_id)); @@ -1761,7 +1765,7 @@ impl Database for RemoteDatabase { request: DescribeNamespaceRequest, ) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts)?; + let namespace_id = self.namespace_identifier(namespace_parts)?; let req = self .client .post(&format!("/v1/namespace/{}/describe", namespace_id)) @@ -1887,6 +1891,40 @@ mod tests { remote::{ARROW_STREAM_CONTENT_TYPE, ClientConfig, HeaderProvider, JSON_CONTENT_TYPE}, }; + #[tokio::test] + async fn test_catalog_namespace_identifier_validates_all_namespace_components() { + let mut db = super::RemoteDatabase::new_mock(|_| -> http::Response { + panic!("identifier validation must not send a request") + }); + db.is_catalog_root = true; + + for (path, expected) in [ + (vec![], "$"), + (vec!["team/search"], "team%2Fsearch"), + (vec!["team/search", "ns"], "team%2Fsearch$ns"), + (vec!["team/search", "ns", "child"], "team%2Fsearch$ns$child"), + ] { + let path = path.into_iter().map(String::from).collect::>(); + assert_eq!(db.namespace_identifier(&path).unwrap(), expected); + } + + for path in [ + vec!["../search"], + vec!["team/search", "bad/name"], + vec!["team/search", "ns", "bad/name"], + vec!["team/search", "public", "child"], + ] { + let path = path.into_iter().map(String::from).collect::>(); + assert!( + matches!( + db.namespace_identifier(&path), + Err(Error::InvalidInput { .. }) + ), + "accepted {path:?}" + ); + } + } + #[test] fn test_cache_key_security() { // Test that cache keys are unique regardless of delimiter manipulation @@ -4045,7 +4083,7 @@ mod tests { .expect_err("an illegal component must be refused"); assert!(!*reached.lock().unwrap(), "{name:?} reached the transport"); assert!( - error.to_string().contains("Secret name"), + error.to_string().contains("secret name"), "{name:?}: {error}" ); } @@ -4076,7 +4114,7 @@ mod tests { .await .expect_err("a dot-only name must be refused"); assert!( - by_name.to_string().contains("relative path segments"), + by_name.to_string().contains("Illegal secret name"), "{by_name}" ); @@ -4086,7 +4124,9 @@ mod tests { .await .expect_err("a dot-only namespace segment must be refused"); assert!( - by_segment.to_string().contains("relative path segments"), + by_segment + .to_string() + .contains("Illegal namespace component"), "{by_segment}" ); assert!( @@ -4165,28 +4205,20 @@ mod tests { } } - /// A segment outside the table charset still addresses one segment: the - /// service decides whether it may exist, and percent-encoding is what keeps - /// the question reaching the right route. A catalog database is named this - /// way. + /// Slashes belong to catalog database names, not table namespace segments. #[tokio::test] - async fn test_a_namespace_segment_outside_the_charset_is_encoded_not_refused() { - use std::sync::{Arc, Mutex}; - let seen = Arc::new(Mutex::new(String::new())); - let path = seen.clone(); - let conn = Connection::new_with_handler(move |request| { - *path.lock().unwrap() = request.url().path().to_string(); - http::Response::builder().status(200).body("{}").unwrap() + async fn test_a_namespace_segment_outside_the_charset_is_refused() { + let conn = Connection::new_with_handler(|_| -> http::Response { + panic!("invalid namespace reached the transport") }); - conn.drop_table("t", &["team/search".to_string()]) - .await - .unwrap(); - assert_eq!(*seen.lock().unwrap(), "/v1/table/team%2Fsearch$t/drop/"); + assert!(matches!( + conn.drop_table("t", &["team/search".to_string()]).await, + Err(Error::InvalidInput { .. }) + )); } - /// A Function name is percent-encoded, which covers everything but the - /// relative segment: `..` is unreserved, so it survives encoding and is - /// then resolved away, posting a registration body to `/v1/create`. + /// Reject relative segments and their encoded forms before sending a request, + /// so a function name cannot change the route through URL normalization. #[tokio::test] async fn test_a_relative_segment_function_name_is_refused() { use std::sync::{Arc, Mutex}; @@ -4201,10 +4233,7 @@ mod tests { .drop_function(name, "fv_1", &[]) .await .expect_err("a dot-only Function name must be refused"); - assert!( - error.to_string().contains("relative path segments"), - "{error}" - ); + assert!(matches!(error, Error::InvalidInput { .. }), "{error}"); assert!(!*reached.lock().unwrap(), "{name:?} reached the transport"); } } @@ -4585,7 +4614,7 @@ mod tests { }; // A view is not a table, so the refusal says so. assert!( - error.contains("view name") || error.contains("view namespace path segment"), + error.contains("view name") || error.contains("namespace component"), "{error}" ); } diff --git a/rust/lancedb/src/utils/mod.rs b/rust/lancedb/src/utils/mod.rs index 7692f9bb5..20f209f2e 100644 --- a/rust/lancedb/src/utils/mod.rs +++ b/rust/lancedb/src/utils/mod.rs @@ -19,15 +19,27 @@ use std::pin::Pin; use crate::error::{Error, Result}; use datafusion_physical_plan::SendableRecordBatchStream; -/// The characters any object name may contain: a table, a namespace segment, a -/// Secret, a materialized view. -/// -/// No positional rule on top of it -- a name may begin with `_`, `-` or `.`, -/// as LanceDB namespaces already do. `.` and `..` are excluded separately, by -/// [`reject_relative_segment`]: that is a property of where a name sits in a -/// URL, not of the name. Length is the service's to bound. -static OBJECT_NAME_REGEX: std::sync::LazyLock = - std::sync::LazyLock::new(|| regex::Regex::new(r"^[A-Za-z0-9_.\-]+$").unwrap()); +/// The maximum length of a table name. Table names are used in URLs and in the keys used with the +/// object store. +const MAX_TABLE_NAME_CHARS: usize = 128; + +/// The maximum length of a component of a namespace. +const MAX_NAMESPACE_NAME_CHARS: usize = 64; + +/// The maximum number of namespace components that are allowed. +const MAX_NAMESPACE_COMPONENTS: usize = 10; + +/// The maximum length of a view name. +const MAX_VIEW_NAME_CHARS: usize = 128; + +/// The maximum length of a secret name. +const MAX_SECRET_NAME_CHARS: usize = 128; + +/// The maximum length of a function name. +const MAX_FUNCTION_NAME_CHARS: usize = 128; + +/// The maximum length of a database name. +const MAX_DB_NAME_CHARS: usize = 64; pub trait PatchStoreParam { fn patch_with_store_wrapper( @@ -86,129 +98,190 @@ impl PatchReadParam for ReadParams { } } -/// The reason `.` and `..` are refused wherever a name becomes a path segment. -const RELATIVE_SEGMENT_REASON: &str = - "'.' and '..' are read as relative path segments and cannot address an object"; - -/// Whether URL parsing would resolve this component away rather than keep it. -/// -/// Exactly `.` and `..`, and their percent-encoded spellings -- resolution -/// happens after decoding, so `%2E%2E` collapses as surely as `..` does, and -/// `drop_table("..")` would reach `/v1/drop/`. No wider than that: `...` is an -/// ordinary segment that addresses fine. -fn is_relative_segment(value: &str) -> bool { - let decoded = value.replace("%2e", ".").replace("%2E", "."); - decoded == "." || decoded == ".." +/// Translate a string into something that can be safely logged. This involves removing control +/// characters that could be used in an exploit like CVE-2003-0063. +fn sanitize(input: &str) -> String { + input.chars().filter(|c| !c.is_control()).collect() } -/// Refuse a path component that URL parsing resolves as a relative segment. -/// -/// Reachable on its own for an identifier with no other validator: a Function -/// name has no client-side grammar, so this is the only rule that applies. -pub(crate) fn reject_relative_segment(what: &str, value: &str) -> Result<()> { - if is_relative_segment(value) { - return Err(Error::InvalidInput { - message: format!("invalid {what} '{value}': {RELATIVE_SEGMENT_REASON}"), +pub fn validate_table_name(name: &str) -> Result<()> { + if name.is_empty() || name == "." || name == ".." { + return Err(Error::InvalidTableName { + name: name.to_string(), + reason: "Illegal table name.".to_string(), + }); + } + if name.len() > MAX_TABLE_NAME_CHARS { + return Err(Error::InvalidTableName { + name: sanitize(name), + reason: format!( + "Table name cannot be more than {MAX_TABLE_NAME_CHARS} characters long." + ), + }); + } + if !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.') + { + return Err(Error::InvalidTableName { + name: sanitize(name), + reason: "Only alphanumeric characters, dashes, underscores, and\ + periods are allowed in table names." + .to_string(), }); } Ok(()) } -/// Every rule an object name obeys: non-empty, inside [`OBJECT_NAME_REGEX`], -/// and addressable as a path segment. -/// -/// Returns the reason rather than an [`Error`], because the error type is each -/// API's own -- a table reports [`Error::InvalidTableName`], the rest -/// [`Error::InvalidInput`]. Sharing the rules but not the error keeps a table, -/// a namespace segment and a Secret from drifting apart. -fn check_object_name(name: &str) -> std::result::Result<(), &'static str> { - if name.is_empty() { - return Err("it must not be empty"); +/// Validate that an input string contains alphanumerics, or dashes, underscores, and periods. +fn validate_view_or_namespace_or_secret(input: &str, kind: &str, max_len: usize) -> Result<()> { + if input.is_empty() || input == "." || input == ".." { + return Err(Error::InvalidInput { + message: format!("Illegal {} '{}'.", kind, input), + }); } - if !OBJECT_NAME_REGEX.is_match(name) { - return Err( - "it may contain only alphanumeric characters, underscores, hyphens and periods", - ); + if input.len() > max_len { + return Err(Error::InvalidInput { + message: format!( + "Invalid {} '{}' is more than {} characters long.", + kind, + sanitize(input), + max_len + ), + }); } - if is_relative_segment(name) { - return Err(RELATIVE_SEGMENT_REASON); + if !input + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.') + { + return Err(Error::InvalidInput { + message: format!( + "Invalid {} '{}': only alphanumeric characters, \ + dashes, underscores, and periods are allowed.", + kind, + sanitize(input) + ), + }); } Ok(()) } -/// Validate a table name. -pub fn validate_table_name(name: &str) -> Result<()> { - check_object_name(name).map_err(|reason| Error::InvalidTableName { - name: name.to_string(), - reason: reason.to_string(), - }) -} - /// Validate one component of a namespace path -- a single segment, not the /// whole path. [`validate_namespace`] covers a path. -pub fn validate_namespace_name(name: &str) -> Result<()> { - check_object_name(name).map_err(|reason| Error::InvalidInput { - message: format!("invalid namespace name '{name}': {reason}"), - }) +pub fn validate_namespace_component(name: &str) -> Result<()> { + validate_view_or_namespace_or_secret(name, "namespace component", MAX_NAMESPACE_NAME_CHARS) } -/// Validate one component of a Secret identifier: a Secret name, or one segment -/// of the namespace path holding it. -/// -/// The join decides identity, and the service only sees what the split -/// produced. `"a$b"` is not a name the service accepts, but joined and split it -/// reads as the namespace `a` and the name `b` -- a different Secret that may -/// already exist. This is not a second opinion on the name; it is what lets the -/// service have one. -pub fn validate_secret_component(what: &str, value: &str) -> Result<()> { - check_object_name(value).map_err(|reason| Error::InvalidInput { - message: format!("invalid {what} '{value}': {reason}"), - }) +/// Validate all components of a namespace +pub fn validate_namespace(namespace: &[String]) -> Result<()> { + if namespace.len() > 1 && namespace.first().map(String::as_str) == Some("public") { + return Err(Error::InvalidInput { + message: "Namespaces that begin with public cannot have multiple namespace components." + .to_string(), + }); + } + if namespace.len() > MAX_NAMESPACE_COMPONENTS { + return Err(Error::InvalidInput { + message: format!( + "Cannot have a namespace with more than {MAX_NAMESPACE_COMPONENTS} components." + ), + }); + } + for component in namespace { + validate_namespace_component(component)?; + } + Ok(()) +} + +/// Validate a secret name. +pub fn validate_secret_name(name: &str) -> Result<()> { + validate_view_or_namespace_or_secret(name, "secret name", MAX_SECRET_NAME_CHARS) } /// Validate a Secret name and every segment of the namespace path holding it. pub fn validate_secret_reference(name: &str, namespace_path: &[String]) -> Result<()> { - for segment in namespace_path { - validate_secret_component("Secret namespace path segment", segment)?; - } - validate_secret_component("Secret name", name) + validate_namespace(namespace_path)?; + validate_secret_name(name) +} + +pub fn validate_view_name(name: &str) -> Result<()> { + validate_view_or_namespace_or_secret(name, "view name", MAX_VIEW_NAME_CHARS) } /// Validate a view name and every segment of the namespace path holding it. -/// -/// Worded for a view rather than deferring to [`validate_table_name`]: a view -/// is not a table, and a caller who mistypes one should not be told their -/// table name is invalid. The rule is the same one every object name follows, -/// which is what lets the `$`-joined identifier split back apart. pub fn validate_view_reference(name: &str, namespace_path: &[String]) -> Result<()> { - for segment in namespace_path { - validate_view_component("view namespace path segment", segment)?; + validate_namespace(namespace_path)?; + validate_view_name(name) +} + +/// Validate a function name. These do not allow dashes or periods. +pub fn validate_function_name(name: &str) -> Result<()> { + if name.is_empty() { + return Err(Error::InvalidInput { + message: format!("Invalid function name '{}'.", name), + }); } - validate_view_component("view name", name) + if name.len() > MAX_FUNCTION_NAME_CHARS { + return Err(Error::InvalidInput { + message: format!( + "Invalid function name '{}' is more than {} characters long.", + sanitize(name), + MAX_FUNCTION_NAME_CHARS + ), + }); + } + if !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Err(Error::InvalidInput { + message: format!( + "Invalid function name '{}': only alphanumeric \ + characters and underscores are allowed.", + sanitize(name) + ), + }); + } + Ok(()) } -/// Validate one component of a view identifier: a view name, or one segment of -/// the namespace path holding it. -pub fn validate_view_component(what: &str, value: &str) -> Result<()> { - check_object_name(value).map_err(|reason| Error::InvalidInput { - message: format!("invalid {what} '{value}': {reason}"), - }) -} - -/// Validate all components of a namespace -/// -/// Iterates through all namespace components and validates each one. -/// Returns an error if any component is invalid. -/// -/// # Arguments -/// * `namespace` - The namespace components to validate -/// -/// # Returns -/// * `Ok(())` if all namespace components are valid -/// * `Err(Error)` if any component is invalid -pub fn validate_namespace(namespace: &[String]) -> Result<()> { - for component in namespace { - validate_namespace_name(component)?; +/// Validate a database name, including slash-separated components. +pub fn validate_database_name(name: &str) -> Result<()> { + if name.len() > MAX_DB_NAME_CHARS { + return Err(Error::InvalidInput { + message: format!( + "Invalid database name '{}' is more than {} characters long.", + sanitize(name), + MAX_DB_NAME_CHARS + ), + }); + } + if !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' || c == '/') + { + return Err(Error::InvalidInput { + message: format!( + "Invalid database name '{}': only alphanumeric characters, dashes, \ + underscores, periods, and slashes are allowed.", + sanitize(name) + ), + }); + } + for component in name.split('/') { + if component.is_empty() { + return Err(Error::InvalidInput { + message: format!( + "Invalid database name '{}': invalid empty path component.", + sanitize(name) + ), + }); + } + if component == "." || component == ".." { + return Err(Error::InvalidInput { + message: format!( + "Invalid database name '{}': illegal path component.", + sanitize(name) + ), + }); + } } Ok(()) } @@ -927,6 +1000,37 @@ mod tests { assert!(err.contains("text.embedding")); } + #[test] + fn test_validate_database_name() { + for name in [ + "db", + "team/search", + "Team_1/db-2.3", + &"a".repeat(MAX_DB_NAME_CHARS), + ] { + assert!(validate_database_name(name).is_ok(), "rejected {name:?}"); + } + for name in [ + "", + "/", + "//", + ".", + "..", + "/db", + "db/", + "a//b", + "a/./b", + "a/../b", + "a$b", + "a b", + "a?b", + "café", + &"a".repeat(MAX_DB_NAME_CHARS + 1), + ] { + assert!(validate_database_name(name).is_err(), "accepted {name:?}"); + } + } + #[test] fn test_validate_table_name() { assert!(validate_table_name("my_table").is_ok()); @@ -947,27 +1051,27 @@ mod tests { } #[test] - fn test_validate_namespace_name() { + fn test_validate_namespace_component() { // Valid namespace names - assert!(validate_namespace_name("ns1").is_ok()); - assert!(validate_namespace_name("namespace_123").is_ok()); - assert!(validate_namespace_name("my-namespace").is_ok()); - assert!(validate_namespace_name("my.namespace").is_ok()); - assert!(validate_namespace_name("NS_1.2.3").is_ok()); - assert!(validate_namespace_name("a").is_ok()); - assert!(validate_namespace_name("123").is_ok()); - assert!(validate_namespace_name("_underscore").is_ok()); - assert!(validate_namespace_name("-hyphen").is_ok()); - assert!(validate_namespace_name(".period").is_ok()); + assert!(validate_namespace_component("ns1").is_ok()); + assert!(validate_namespace_component("namespace_123").is_ok()); + assert!(validate_namespace_component("my-namespace").is_ok()); + assert!(validate_namespace_component("my.namespace").is_ok()); + assert!(validate_namespace_component("NS_1.2.3").is_ok()); + assert!(validate_namespace_component("a").is_ok()); + assert!(validate_namespace_component("123").is_ok()); + assert!(validate_namespace_component("_underscore").is_ok()); + assert!(validate_namespace_component("-hyphen").is_ok()); + assert!(validate_namespace_component(".period").is_ok()); // Invalid namespace names - assert!(validate_namespace_name("").is_err()); - assert!(validate_namespace_name("namespace with spaces").is_err()); - assert!(validate_namespace_name("namespace/with/slashes").is_err()); - assert!(validate_namespace_name("namespace\\with\\backslashes").is_err()); - assert!(validate_namespace_name("namespace$with$delimiter").is_err()); - assert!(validate_namespace_name("namespace@special").is_err()); - assert!(validate_namespace_name("namespace#hash").is_err()); + assert!(validate_namespace_component("").is_err()); + assert!(validate_namespace_component("namespace with spaces").is_err()); + assert!(validate_namespace_component("namespace/with/slashes").is_err()); + assert!(validate_namespace_component("namespace\\with\\backslashes").is_err()); + assert!(validate_namespace_component("namespace$with$delimiter").is_err()); + assert!(validate_namespace_component("namespace@special").is_err()); + assert!(validate_namespace_component("namespace#hash").is_err()); } #[test] @@ -983,6 +1087,9 @@ mod tests { // Empty namespace (root) is valid assert!(validate_namespace(&[]).is_ok()); + // Valid public namespace (which is equivalent to the root namespace.) + assert!(validate_namespace(&["public".to_string()]).is_ok()); + // Invalid: contains empty component assert!(validate_namespace(&["ns1".to_string(), "".to_string()]).is_err()); @@ -994,6 +1101,9 @@ mod tests { assert!(validate_namespace(&["ns1".to_string(), "ns/2".to_string()]).is_err()); assert!(validate_namespace(&["ns1".to_string(), "ns$2".to_string()]).is_err()); + // Invalid: starts with public but contains other components. + assert!(validate_namespace(&["public".to_string(), "foo".to_string()]).is_err()); + // Valid: underscores, hyphens, and periods are allowed assert!( validate_namespace(&["ns_1".to_string(), "ns-2".to_string(), "ns.3".to_string()]) From f305a05fd4e792bdd0d404bbbf8131786d088006 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Fri, 9 Oct 2026 12:45:02 -0700 Subject: [PATCH 06/19] fix: preserve adaptive IVF probe defaults (#4168) Keep IVF probe fields unset by default so Lance supplies adaptive behavior. Preserve `nprobes`, `minimum_nprobes`, and `maximum_nprobes` independently through Rust, Python, namespace, and remote request construction, with compatibility fields serialized only when explicitly configured. The final Rust namespace and LSM propagation update depends on the Lance release containing lance-format/lance#9184. --- Cargo.lock | 132 +++++++++--------- Cargo.toml | 28 ++-- docs/src/js/classes/VectorQuery.md | 11 +- docs/web/indexing/vector-index.mdx | 10 +- java/pom.xml | 2 +- nodejs/__test__/table.test.ts | 4 +- nodejs/lancedb/query.ts | 11 +- python/pyproject.toml | 4 +- python/python/lancedb/_lancedb.pyi | 1 + python/python/lancedb/namespace.py | 13 +- python/python/lancedb/query.py | 52 ++++--- python/python/lancedb/table.py | 5 +- python/python/tests/test_hybrid_query.py | 14 ++ python/python/tests/test_namespace.py | 26 +++- python/python/tests/test_query.py | 116 +++++++++++---- python/python/tests/test_remote_db.py | 37 +++-- python/src/query.rs | 12 +- python/uv.lock | 30 ++-- rust/lancedb/src/query.rs | 99 +++++++++---- rust/lancedb/src/remote/table.rs | 71 +++++++--- .../src/table/datafusion/udtf/vector_query.rs | 4 +- rust/lancedb/src/table/merge.rs | 51 +++++-- rust/lancedb/src/table/query.rs | 92 ++++++++++-- rust/lancedb/src/table/query/lsm.rs | 19 ++- 24 files changed, 576 insertions(+), 268 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eecf3d0d1..18cb1a98e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3588,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5095,26 +5095,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "jsonb" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb98fb29636087c40ad0d1274d9a30c0c1e83e03ae93f6e7e89247b37fcc6953" -dependencies = [ - "byteorder", - "ethnum", - "fast-float2", - "itoa", - "jiff", - "nom 8.0.0", - "num-traits", - "ordered-float 5.3.0", - "rand 0.9.5", - "serde", - "serde_json", - "zmij", -] - [[package]] name = "kanaria" version = "0.2.0" @@ -5143,8 +5123,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arc-swap", "arrow", @@ -5217,8 +5197,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-buffer", @@ -5232,7 +5212,7 @@ dependencies = [ "futures", "getrandom 0.2.17", "half", - "jsonb", + "lance-jsonb", "num-traits", "rand 0.9.5", ] @@ -5240,7 +5220,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-buffer", @@ -5254,7 +5234,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-schema", @@ -5263,8 +5243,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrayref", "crunchy", @@ -5274,8 +5254,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-buffer", @@ -5312,8 +5292,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "arrow-array", @@ -5330,10 +5310,10 @@ dependencies = [ "datafusion-physical-expr", "futures", "half", - "jsonb", "lance-arrow", "lance-core", "lance-geo", + "lance-jsonb", "log", "pin-project", "prost", @@ -5344,8 +5324,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "arrow-array", @@ -5362,8 +5342,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "proc-macro2", "quote", @@ -5372,8 +5352,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-arith", "arrow-array", @@ -5406,8 +5386,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-arith", "arrow-array", @@ -5439,8 +5419,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "datafusion", "geo-traits", @@ -5454,8 +5434,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arc-swap", "arrow", @@ -5486,7 +5466,6 @@ dependencies = [ "hex", "itertools 0.14.0", "jieba-rs", - "jsonb", "lance-arrow", "lance-arrow-stats", "lance-bitpacking", @@ -5497,6 +5476,7 @@ dependencies = [ "lance-geo", "lance-index-core", "lance-io", + "lance-jsonb", "lance-linalg", "lance-select", "lance-table", @@ -5526,8 +5506,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-schema", @@ -5550,8 +5530,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "arrow-array", @@ -5591,10 +5571,28 @@ dependencies = [ "uuid", ] +[[package]] +name = "lance-jsonb" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +dependencies = [ + "byteorder", + "ethnum", + "fast-float2", + "itoa", + "jiff", + "nom 8.0.0", + "num-traits", + "ordered-float 5.3.0", + "serde", + "serde_json", + "zmij", +] + [[package]] name = "lance-linalg" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-schema", @@ -5608,8 +5606,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "async-trait", @@ -5623,8 +5621,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arc-swap", "arrow", @@ -5678,8 +5676,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-buffer", @@ -5693,8 +5691,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "arrow-array", @@ -5734,8 +5732,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-schema", @@ -5748,8 +5746,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index ad7c00477..6c05f297a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/docs/src/js/classes/VectorQuery.md b/docs/src/js/classes/VectorQuery.md index a0a77fd7c..3f56bb600 100644 --- a/docs/src/js/classes/VectorQuery.md +++ b/docs/src/js/classes/VectorQuery.md @@ -460,17 +460,16 @@ exhaustiely searched to find matches. This parameter controls how many partitions should be searched. Increasing this value will increase the recall of your query but will -also increase the latency of your query. The default value is 20. This -default is good for many cases but the best value to use will depend on -your data and the recall that you need to achieve. +also increase the latency of your query. If this method is not called, +Lance's adaptive probe defaults are used. For best results we recommend tuning this parameter with a benchmark against your actual data to find the smallest possible value that will still give you the desired recall. -For more fine grained control over behavior when you have a very narrow filter -you can use `minimumNprobes` and `maximumNprobes`. This method sets both -the minimum and maximum to the same value. +The value is retained as `nprobes` through client and server request +construction. Lance sets both probe bounds to this value. Explicit minimum +or maximum settings can override their respective bounds. #### Parameters diff --git a/docs/web/indexing/vector-index.mdx b/docs/web/indexing/vector-index.mdx index 3cd3a970a..b10da3f6f 100644 --- a/docs/web/indexing/vector-index.mdx +++ b/docs/web/indexing/vector-index.mdx @@ -139,9 +139,9 @@ Learn how to configure, build, and search LanceDB vector indexes, including buil | Parameter | Description | | :--- | :--- | | `limit` | Number of results to return (the `k` in `k-ANN`). | -| `nprobes` | Shorthand that sets both `minimum_nprobes` and `maximum_nprobes` to the same value. LanceDB defaults this to 20 which is a good default for many cases. At very high scales you will likely need to experiment to find your preferred recall / latency balance. | -| `minimum_nprobes` | Minimum number of partitions scanned. | -| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results — useful for narrow filters. Set to `0` to remove the cap. | +| `nprobes` | Sets both `minimum_nprobes` and `maximum_nprobes` to the same value in Lance. Leave unset to use Lance's adaptive defaults. At very high scales you will likely need to experiment to find your preferred recall / latency balance. | +| `minimum_nprobes` | Minimum number of partitions scanned. Leave unset to use Lance's default. | +| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results — useful for narrow filters. Leave unset, or set to `0`, to remove the cap. | | `ef` | HNSW search-time exploration factor. Start around `1.5 * k` and increase up to `10 * k` for higher recall. | | `refine_factor` | Reads and reranks additional candidates in memory to recover recall lost to quantization. Start between `5` and `50` (a factor of `25` reads and reranks the top `25 * k` candidates) and measure the latency cost, typically a few milliseconds, against the recall gained. | | `distance_range(lower_bound, upper_bound)` | Return only rows whose distance falls within `[lower_bound, upper_bound)`. Either bound is optional. Useful for near-duplicate detection or "close-enough" matching. | @@ -165,8 +165,8 @@ older client. | Index type | Guidance | | :--- | :--- | -| `IVF_HNSW_FLAT`, `IVF_HNSW_SQ` | Keep the default `nprobes`, then tune `ef` first. Expect higher latency variance under filtered search. | -| `IVF_RQ`, `IVF_PQ` | Keep the default `nprobes` and raise when recall is insufficient. If using a selective filter you may need to increase `maximum_nprobes` if you do not received enough results. | +| `IVF_HNSW_FLAT`, `IVF_HNSW_SQ` | Leave the probe settings unset, then tune `ef` first. Expect higher latency variance under filtered search. | +| `IVF_RQ`, `IVF_PQ` | Leave the probe settings unset and configure them only when recall is insufficient. If using a selective filter you may need to increase `maximum_nprobes` if you do not receive enough results. | diff --git a/java/pom.xml b/java/pom.xml index 59efeb627..964dfed0c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 14.0.0-beta.7 + 14.0.0-beta.10 false 2.30.0 1.7 diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index eee40fb9c..d2244b48c 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -1231,7 +1231,9 @@ describe("When creating an index", () => { expect(() => tbl.query().nearestTo(queryVec).minimumNprobes(0)).toThrow( "Invalid input, minimum_nprobes must be greater than 0", ); - expect(() => tbl.query().nearestTo(queryVec).maximumNprobes(5)).toThrow( + expect(() => + tbl.query().nearestTo(queryVec).minimumNprobes(10).maximumNprobes(5), + ).toThrow( "Invalid input, maximum_nprobes must be greater than or equal to minimum_nprobes", ); diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index f370e1b31..0f0eaf610 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -532,17 +532,16 @@ export class VectorQuery extends StandardQueryBase { * partitions should be searched. * * Increasing this value will increase the recall of your query but will - * also increase the latency of your query. The default value is 20. This - * default is good for many cases but the best value to use will depend on - * your data and the recall that you need to achieve. + * also increase the latency of your query. If this method is not called, + * Lance's adaptive probe defaults are used. * * For best results we recommend tuning this parameter with a benchmark against * your actual data to find the smallest possible value that will still give * you the desired recall. * - * For more fine grained control over behavior when you have a very narrow filter - * you can use `minimumNprobes` and `maximumNprobes`. This method sets both - * the minimum and maximum to the same value. + * The value is retained as `nprobes` through client and server request + * construction. Lance sets both probe bounds to this value. Explicit minimum + * or maximum settings can override their respective bounds. */ nprobes(nprobes: number): VectorQuery { this.doVectorCall((inner) => inner.nprobes(nprobes)); diff --git a/python/pyproject.toml b/python/pyproject.toml index 2d753c139..cde7247f3 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "pyarrow>=16", "pydantic>=2.7.4,<3", "tqdm>=4.27.0", - "lance-namespace>=0.3.2" + "lance-namespace>=0.13.0" ] description = "lancedb" authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }] @@ -63,7 +63,7 @@ tests = [ "polars>=0.19, <=1.32.3", "pyarrow<25", "pyarrow-stubs>=16.0", - "pylance==14.0.0b6", + "pylance==14.0.0b10", "requests>=2.31.0", "datafusion>=54,<55", "opentelemetry-sdk>=1.30.0", diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index a8f02e458..cfaa8c68a 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -778,6 +778,7 @@ class PyQueryRequest: use_lsm: Optional[bool] column: Optional[str] query_vector: Optional[List[pa.Array]] + nprobes: Optional[int] minimum_nprobes: Optional[int] maximum_nprobes: Optional[int] lower_bound: Optional[float] diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index 195c59e04..ed87e441f 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -190,8 +190,12 @@ def _query_to_namespace_request( kwargs["with_row_id"] = query.with_row_id if query.ef is not None: kwargs["ef"] = query.ef + if query.nprobes is not None: + kwargs["nprobes"] = query.nprobes if query.minimum_nprobes is not None: - kwargs["nprobes"] = query.minimum_nprobes + kwargs["minimum_nprobes"] = query.minimum_nprobes + if query.maximum_nprobes is not None: + kwargs["maximum_nprobes"] = query.maximum_nprobes if query.refine_factor is not None: kwargs["refine_factor"] = query.refine_factor if query.lower_bound is not None: @@ -205,6 +209,13 @@ def _query_to_namespace_request( if query.distance_type is not None: kwargs["distance_type"] = query.distance_type + if kwargs.get("maximum_nprobes") == 0: + # The generated 0.13 model rejects LanceDB's unbounded sentinel; + # model_copy preserves direct wire pass-through to the Lance receiver. + maximum_nprobes = kwargs.pop("maximum_nprobes") + return QueryTableRequest(**kwargs).model_copy( + update={"maximum_nprobes": maximum_nprobes} + ) return QueryTableRequest(**kwargs) diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index 6f68692a2..c7664ab78 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -693,8 +693,9 @@ class Query(pydantic.BaseModel): if True then apply the filter after vector / FTS search. This is ignored for plain SQL filtering. nprobes : Optional[int] - The number of IVF partitions to search. If this is None then a default - number of partitions will be used. + The legacy number of IVF partitions to search. Lance sets both probe + bounds to this value. If this is None then Lance's default probe settings + will be used. - A higher number makes search more accurate but also slower. @@ -773,20 +774,23 @@ class Query(pydantic.BaseModel): # distance type to use for vector search distance_type: Optional[str] = None + # legacy number of IVF partitions to search + # + # Lance sets both probe bounds to this value. Explicit bounds can override it. + nprobes: Optional[int] = None + # which columns to return in the results (dict values may be str or Expr) columns: QueryProjection = None # minimum number of IVF partitions to search # - # If None then a default value (20) will be used. + # If None then Lance's default will be used. minimum_nprobes: Optional[int] = None # maximum number of IVF partitions to search # - # If None then a default value (20) will be used. - # - # If 0 then no limit will be applied and all partitions could be searched - # if needed to satisfy the limit. + # If None then Lance's default will be used. If 0 then no limit will be applied + # and all partitions could be searched if needed to satisfy the limit. maximum_nprobes: Optional[int] = None # lower bound for distance search @@ -847,6 +851,7 @@ class Query(pydantic.BaseModel): query.vector_column = req.column query.vector = req.query_vector query.distance_type = req.distance_type + query.nprobes = req.nprobes query.minimum_nprobes = req.minimum_nprobes query.maximum_nprobes = req.maximum_nprobes query.lower_bound = req.lower_bound @@ -1627,6 +1632,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): super().__init__(table) self._query = query self._distance_type = None + self._nprobes = None self._minimum_nprobes = None self._maximum_nprobes = None self._lower_bound = None @@ -1691,9 +1697,9 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): See discussion in [Querying an ANN Index](https://lancedb.com/docs/indexing/) for tuning advice. - This method sets both the minimum and maximum number of probes to the same - value. See `minimum_nprobes` and `maximum_nprobes` for more fine-grained - control. + The value is retained as `nprobes` through client and server request + construction. Lance sets both probe bounds to this value. Explicit minimum + or maximum settings can override their respective bounds. Parameters ---------- @@ -1707,8 +1713,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): """ if nprobes <= 0: raise ValueError("Invalid input, nprobes must be greater than 0") - self._minimum_nprobes = nprobes - self._maximum_nprobes = nprobes + self._nprobes = nprobes return self def minimum_nprobes(self, minimum_nprobes: int) -> LanceVectorQueryBuilder: @@ -1850,6 +1855,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): limit=self._limit, distance_type=self._distance_type, columns=self._columns, + nprobes=self._nprobes, minimum_nprobes=self._minimum_nprobes, maximum_nprobes=self._maximum_nprobes, lower_bound=self._lower_bound, @@ -2208,6 +2214,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): self._fts_columns = fts_columns self._norm = None self._reranker = None + self._nprobes = None self._minimum_nprobes = None self._maximum_nprobes = None self._refine_factor = None @@ -2499,6 +2506,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): Higher values will yield better recall (more likely to find vectors if they exist) at the expense of latency. + The value is retained as `nprobes` until Lance sets both probe bounds to it. + Explicit minimum or maximum settings can override their respective bounds. + Parameters ---------- nprobes: int @@ -2511,8 +2521,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): """ if nprobes <= 0: raise ValueError("Invalid input, nprobes must be greater than 0") - self._minimum_nprobes = nprobes - self._maximum_nprobes = nprobes + self._nprobes = nprobes return self def minimum_nprobes(self, minimum_nprobes: int) -> LanceHybridQueryBuilder: @@ -2791,6 +2800,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): self._fts_query.phrase_query(True) if self._distance_type: self._vector_query.metric(self._distance_type) + if self._nprobes is not None: + self._vector_query.nprobes(self._nprobes) if self._minimum_nprobes is not None: self._vector_query.minimum_nprobes(self._minimum_nprobes) if self._maximum_nprobes is not None: @@ -3690,7 +3701,7 @@ class AsyncVectorQueryBase: def nprobes(self, nprobes: int) -> Self: """ - Set the number of partitions to search (probe) + Set the legacy IVF probe parameter The number of probes must be greater than 0. @@ -3702,16 +3713,19 @@ class AsyncVectorQueryBase: The partition whose centroids are closest to the query vector will be exhaustiely searched to find matches. This parameter controls how many - partitions should be searched. + partitions will be searched. Increasing this value will increase the recall of your query but will - also increase the latency of your query. The default value is 20. This - default is good for many cases but the best value to use will depend on - your data and the recall that you need to achieve. + also increase the latency of your query. If this method is not called, + Lance's adaptive probe defaults are used. For best results we recommend tuning this parameter with a benchmark against your actual data to find the smallest possible value that will still give you the desired recall. + + LanceDB retains this as `nprobes` through local and remote request + construction. Lance sets both probe bounds to this value. Explicit minimum + or maximum settings can override their respective bounds. """ self._inner.nprobes(nprobes) return self diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 2970cb902..21838ecef 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -6439,9 +6439,10 @@ class AsyncTable: ) if query.distance_type is not None: async_query = async_query.distance_type(query.distance_type) + if query.nprobes is not None: + async_query = async_query.nprobes(query.nprobes) if query.minimum_nprobes is not None and query.maximum_nprobes is not None: - # Set both to the minimum first to avoid min > max error. - async_query = async_query.nprobes( + async_query = async_query.minimum_nprobes( query.minimum_nprobes ).maximum_nprobes(query.maximum_nprobes) elif query.minimum_nprobes is not None: diff --git a/python/python/tests/test_hybrid_query.py b/python/python/tests/test_hybrid_query.py index c5f58308d..8d670a136 100644 --- a/python/python/tests/test_hybrid_query.py +++ b/python/python/tests/test_hybrid_query.py @@ -355,6 +355,20 @@ def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table): ) +def test_hybrid_query_nprobes_remains_independent(sync_table: Table): + query = sync_table.search(query_type="hybrid").nprobes(20) + + assert query._nprobes == 20 + assert query._minimum_nprobes is None + assert query._maximum_nprobes is None + + query = query.minimum_nprobes(5).nprobes(20) + + assert query._nprobes == 20 + assert query._minimum_nprobes == 5 + assert query._maximum_nprobes is None + + def test_hybrid_query_distance_range(sync_table: Table): reranker = RRFReranker(return_score="all") result = ( diff --git a/python/python/tests/test_namespace.py b/python/python/tests/test_namespace.py index 2a537bbf9..76aa5fc55 100644 --- a/python/python/tests/test_namespace.py +++ b/python/python/tests/test_namespace.py @@ -15,7 +15,8 @@ import pyarrow as pa import lancedb from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError from lancedb.listing import AsyncListing, Listing -from lancedb.namespace import _MAX_QUERY_K +from lancedb.namespace import _MAX_QUERY_K, _query_to_namespace_request +from lancedb.query import Query from lancedb.table import AsyncTable, LanceTable @@ -1111,6 +1112,29 @@ class TestPushdownOperations: ] assert all(r.k <= 2**31 - 1 for r in namespace_client.requests) + def test_probe_fields_pass_through_independently(self): + request = _query_to_namespace_request( + ["geneva", "hist"], + Query( + vector=[1.0, 2.0], + nprobes=20, + minimum_nprobes=3, + maximum_nprobes=10, + ), + ) + + assert request.nprobes == 20 + assert request.minimum_nprobes == 3 + assert request.maximum_nprobes == 10 + + request = _query_to_namespace_request( + ["geneva", "hist"], + Query(vector=[1.0, 2.0], nprobes=5, maximum_nprobes=0), + ) + assert request.nprobes == 5 + assert request.maximum_nprobes == 0 + assert request.to_dict()["maximum_nprobes"] == 0 + @pytest.mark.asyncio class TestAsyncPushdownOperations: diff --git a/python/python/tests/test_query.py b/python/python/tests/test_query.py index d3fedbcc8..daee0dd07 100644 --- a/python/python/tests/test_query.py +++ b/python/python/tests/test_query.py @@ -995,12 +995,22 @@ def test_invalid_nprobes_sync(table): ValueError, match="maximum_nprobes must be greater than or equal to minimum_nprobes", ): - LanceVectorQueryBuilder(table, [0, 0], "vector").maximum_nprobes(5).to_list() + ( + LanceVectorQueryBuilder(table, [0, 0], "vector") + .minimum_nprobes(10) + .maximum_nprobes(5) + .to_list() + ) with pytest.raises( ValueError, - match="minimum_nprobes must be less than or equal to maximum_nprobes", + match="maximum_nprobes must be greater than or equal to minimum_nprobes", ): - LanceVectorQueryBuilder(table, [0, 0], "vector").minimum_nprobes(100).to_list() + ( + LanceVectorQueryBuilder(table, [0, 0], "vector") + .maximum_nprobes(5) + .minimum_nprobes(100) + .to_list() + ) def test_nprobes_works_sync(table): @@ -1039,12 +1049,22 @@ async def test_invalid_nprobes_async(table_async: AsyncTable): ValueError, match="maximum_nprobes must be greater than or equal to minimum_nprobes", ): - await table_async.vector_search([0, 0]).maximum_nprobes(5).to_list() + await ( + table_async.vector_search([0, 0]) + .minimum_nprobes(10) + .maximum_nprobes(5) + .to_list() + ) with pytest.raises( ValueError, match="minimum_nprobes must be less than or equal to maximum_nprobes", ): - await table_async.vector_search([0, 0]).minimum_nprobes(100).to_list() + await ( + table_async.vector_search([0, 0]) + .maximum_nprobes(5) + .minimum_nprobes(100) + .to_list() + ) def test_query_builder_with_prefilter(table): @@ -1635,8 +1655,9 @@ def test_query_serialization_sync(table: lancedb.table.Table): q, vector_column="vector", vector=[5.0, 6.0], - minimum_nprobes=10, - maximum_nprobes=10, + nprobes=10, + minimum_nprobes=None, + maximum_nprobes=None, refine_factor=5, ) @@ -1654,8 +1675,19 @@ def test_query_serialization_sync(table: lancedb.table.Table): q, vector_column="vector", vector=[5.0, 6.0], - minimum_nprobes=50, - maximum_nprobes=50, + nprobes=50, + minimum_nprobes=None, + maximum_nprobes=None, + ) + + q = table.search([5.0, 6.0]).minimum_nprobes(5).nprobes(50).to_query_object() + check_set_props( + q, + vector_column="vector", + vector=[5.0, 6.0], + nprobes=50, + minimum_nprobes=5, + maximum_nprobes=None, ) q = table.search([5.0, 6.0]).maximum_nprobes(10).to_query_object() @@ -1667,6 +1699,15 @@ def test_query_serialization_sync(table: lancedb.table.Table): minimum_nprobes=None, ) + q = table.search([5.0, 6.0]).nprobes(5).maximum_nprobes(0).to_query_object() + check_set_props( + q, + vector_column="vector", + vector=[5.0, 6.0], + nprobes=5, + maximum_nprobes=0, + ) + q = table.search([5.0, 6.0]).distance_range(0.0, 1.0).to_query_object() check_set_props( q, vector_column="vector", vector=[5.0, 6.0], lower_bound=0.0, upper_bound=1.0 @@ -1716,8 +1757,8 @@ async def test_query_serialization_async(table_async: AsyncTable): limit=10, vector=sample_vector, postfilter=False, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, ) @@ -1727,8 +1768,8 @@ async def test_query_serialization_async(table_async: AsyncTable): q, vector=sample_vector, postfilter=False, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, limit=10, @@ -1739,8 +1780,9 @@ async def test_query_serialization_async(table_async: AsyncTable): q, vector=sample_vector, postfilter=False, - minimum_nprobes=50, - maximum_nprobes=50, + nprobes=50, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, limit=10, @@ -1759,8 +1801,8 @@ async def test_query_serialization_async(table_async: AsyncTable): filter="id = 1", postfilter=True, vector=sample_vector, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, ) @@ -1774,8 +1816,9 @@ async def test_query_serialization_async(table_async: AsyncTable): check_set_props( q, vector=sample_vector, - minimum_nprobes=10, - maximum_nprobes=10, + nprobes=10, + minimum_nprobes=None, + maximum_nprobes=None, refine_factor=5, postfilter=False, with_row_id=False, @@ -1788,7 +1831,24 @@ async def test_query_serialization_async(table_async: AsyncTable): q, vector=sample_vector, minimum_nprobes=5, - maximum_nprobes=20, + maximum_nprobes=None, + postfilter=False, + with_row_id=False, + bypass_vector_index=False, + limit=10, + ) + + q = ( + (await table_async.search([5.0, 6.0])) + .nprobes(5) + .maximum_nprobes(0) + .to_query_object() + ) + check_set_props( + q, + vector=sample_vector, + nprobes=5, + maximum_nprobes=0, postfilter=False, with_row_id=False, bypass_vector_index=False, @@ -1806,8 +1866,8 @@ async def test_query_serialization_async(table_async: AsyncTable): lower_bound=0.0, upper_bound=1.0, postfilter=False, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, limit=10, @@ -1819,8 +1879,8 @@ async def test_query_serialization_async(table_async: AsyncTable): distance_type="cosine", vector=sample_vector, postfilter=False, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, limit=10, @@ -1832,8 +1892,8 @@ async def test_query_serialization_async(table_async: AsyncTable): ef=7, vector=sample_vector, postfilter=False, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, limit=10, @@ -1845,8 +1905,8 @@ async def test_query_serialization_async(table_async: AsyncTable): bypass_vector_index=True, vector=sample_vector, postfilter=False, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, limit=10, ) diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 81fb3e10e..3e66ac0e0 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -2164,9 +2164,6 @@ def test_query_sync_minimal(): "ef": None, "vector": [1.0, 2.0, 3.0], "vector_column": "vector", - "nprobes": 20, - "minimum_nprobes": 20, - "maximum_nprobes": 20, "version": None, } @@ -2338,8 +2335,6 @@ def test_query_sync_maximal(): "refine_factor": 10, "vector": [1.0, 2.0, 3.0], "nprobes": 5, - "minimum_nprobes": 5, - "maximum_nprobes": 5, "lower_bound": None, "upper_bound": None, "ef": None, @@ -2435,7 +2430,7 @@ async def test_query_async_nprobes_zero(hybrid): assert requests == ["/v1/table/test/describe/"] -def test_query_sync_nprobes(): +def test_query_sync_probe_bounds(): def handler(body): assert body == { "k": 10, @@ -2447,7 +2442,6 @@ def test_query_sync_nprobes(): "upper_bound": None, "ef": None, "vector": [1.0, 2.0, 3.0], - "nprobes": 5, "minimum_nprobes": 5, "maximum_nprobes": 15, "version": None, @@ -2464,7 +2458,28 @@ def test_query_sync_nprobes(): ) -def test_query_sync_no_max_nprobes(): +def test_query_sync_nprobes_passthrough(): + def handler(body): + assert body["nprobes"] == 20 + assert "minimum_nprobes" not in body + assert "maximum_nprobes" not in body + return pa.table({"id": [1]}) + + with query_test_table(handler) as table: + table.search([1, 2, 3]).nprobes(20).to_list() + + +def test_query_sync_nprobes_with_unbounded_maximum(): + def handler(body): + assert body["nprobes"] == 20 + assert body["maximum_nprobes"] == 0 + return pa.table({"id": [1]}) + + with query_test_table(handler) as table: + table.search([1, 2, 3]).nprobes(20).maximum_nprobes(0).to_list() + + +def test_query_sync_minimum_nprobes_only(): def handler(body): assert body == { "k": 10, @@ -2476,9 +2491,7 @@ def test_query_sync_no_max_nprobes(): "upper_bound": None, "ef": None, "vector": [1.0, 2.0, 3.0], - "nprobes": 5, "minimum_nprobes": 5, - "maximum_nprobes": 0, "version": None, } @@ -2488,7 +2501,6 @@ def test_query_sync_no_max_nprobes(): ( table.search([1, 2, 3], vector_column_name="vector2", fast_search=True) .minimum_nprobes(5) - .maximum_nprobes(0) .to_list() ) @@ -2643,9 +2655,6 @@ def test_query_sync_hybrid(): "refine_factor": None, "vector": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "vector_column": "vector", - "nprobes": 20, - "minimum_nprobes": 20, - "maximum_nprobes": 20, "lower_bound": None, "upper_bound": None, "ef": None, diff --git a/python/src/query.rs b/python/src/query.rs index 1079fb314..88b7243d4 100644 --- a/python/src/query.rs +++ b/python/src/query.rs @@ -336,9 +336,8 @@ pub struct PyQueryRequest { pub use_lsm: Option, pub column: Option, pub query_vector: Option, + pub nprobes: Option, pub minimum_nprobes: Option, - // None means user did not set it and default should be used (currently 20) - // Some(0) means user set it to None and there is no limit pub maximum_nprobes: Option, pub lower_bound: Option, pub upper_bound: Option, @@ -369,6 +368,7 @@ impl From for PyQueryRequest { use_lsm: query_request.use_lsm, column: None, query_vector: None, + nprobes: None, minimum_nprobes: None, maximum_nprobes: None, lower_bound: None, @@ -396,11 +396,9 @@ impl From for PyQueryRequest { use_lsm: vector_query.base.use_lsm, column: vector_query.column, query_vector: Some(PyQueryVectors(vector_query.query_vector)), - minimum_nprobes: Some(vector_query.minimum_nprobes), - maximum_nprobes: match vector_query.maximum_nprobes { - None => Some(0), - Some(value) => Some(value), - }, + nprobes: vector_query.nprobes, + minimum_nprobes: vector_query.minimum_nprobes, + maximum_nprobes: vector_query.maximum_nprobes, lower_bound: vector_query.lower_bound, upper_bound: vector_query.upper_bound, ef: vector_query.ef, diff --git a/python/uv.lock b/python/uv.lock index f80541711..b013e5982 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1849,19 +1849,19 @@ wheels = [ [[package]] name = "lance-namespace" -version = "0.11.1" +version = "0.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lance-namespace-urllib3-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bf/93/da5f7fcac690db9b282a3439ed9e34960c147619a0d6e1f4eb8cd240e7a5/lance_namespace-0.11.1.tar.gz", hash = "sha256:f67cfbbe0647b7cb42f23b673e7edf8a75b7d8a047265a916492f8d247ee1bc2", size = 11631, upload-time = "2026-08-18T17:40:06.294Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/e7/d5d46594678ee479c0eda830c47b2f5c46133bd100e84e9aa6e01306eca7/lance_namespace-0.13.0.tar.gz", hash = "sha256:24554a0997bdb39595c6e4cb3ac6722069f6cd3bd1a74e7acbba7bfaf774a40d", size = 11636, upload-time = "2026-09-14T02:47:03.871523Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/bc/601f2b3cc4cfa0070d858a33223bc823fffdd7981a25c45984a5216ca952/lance_namespace-0.11.1-py3-none-any.whl", hash = "sha256:07643fce9a42ad4d58cc8bf91e3f592bc7f4cbd8d0ad5233223506debf67551c", size = 13507, upload-time = "2026-08-18T17:40:03.561Z" }, + { url = "https://files.pythonhosted.org/packages/e7/47/cfa33cca1ba7c749fd2918cfc5c8ded788f378cc6424d23e4fead5a14125/lance_namespace-0.13.0-py3-none-any.whl", hash = "sha256:438c7b17aef421c21c138196e715f2510d62f07e865372047f87c1e75e618c7a", size = 13507, upload-time = "2026-09-14T02:47:01.765655Z" }, ] [[package]] name = "lance-namespace-urllib3-client" -version = "0.11.1" +version = "0.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -1869,9 +1869,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e4/c5/2bdd0ff98b469894c8a73be809d26ffdad5402517b0e5f9e758026cba29e/lance_namespace_urllib3_client-0.11.1.tar.gz", hash = "sha256:145a9e9424d7597487249b5b95ee274423bf2910e1a9160b6a07b676b61ea46a", size = 237345, upload-time = "2026-08-18T17:40:07.308Z" } +sdist = { url = "https://files.pythonhosted.org/packages/69/25/9aaa4a5e8999693fb0f227c2c0c4b97bd8f0539066408cdff0b89d41b2d5/lance_namespace_urllib3_client-0.13.0.tar.gz", hash = "sha256:1e8a79c6e4e6277033597fd76aa0e2f33d909ca1436c935936e9e569221f43ef", size = 238348, upload-time = "2026-09-14T02:47:04.521878Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2a/eaaefd55d1190291207049fedc6b3eb22b506e57d6de91bae46bbaaa9c60/lance_namespace_urllib3_client-0.11.1-py3-none-any.whl", hash = "sha256:36537f529294da6d884ba0fe783704483f0a75463497c7705fd083a4d0257990", size = 406311, upload-time = "2026-08-18T17:40:04.842Z" }, + { url = "https://files.pythonhosted.org/packages/60/4c/7b8f0712a7fe1b8655b711342bc4989696635a79e7027ee56ba3cae23e99/lance_namespace_urllib3_client-0.13.0-py3-none-any.whl", hash = "sha256:fb361eb4f6c7f2d1f9e92809609657b73e38241393e9c4516e12d6b6ee643a0a", size = 406848, upload-time = "2026-09-14T02:47:02.715954Z" }, ] [[package]] @@ -1980,7 +1980,7 @@ requires-dist = [ { name = "huggingface-hub", marker = "extra == 'embeddings'", specifier = ">=0.19.0" }, { name = "ibm-watsonx-ai", marker = "python_full_version >= '3.10' and extra == 'embeddings'", specifier = ">=1.1.2" }, { name = "instructorembedding", marker = "extra == 'embeddings'", specifier = ">=1.0.1" }, - { name = "lance-namespace", specifier = ">=0.3.2" }, + { name = "lance-namespace", specifier = ">=0.13.0" }, { name = "mkdocs", marker = "extra == 'docs'" }, { name = "mkdocs-jupyter", marker = "extra == 'docs'" }, { name = "mkdocs-material", marker = "extra == 'docs'" }, @@ -2005,7 +2005,7 @@ requires-dist = [ { name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" }, { name = "pydantic", specifier = ">=2.7.4,<3" }, { name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" }, - { name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b6" }, + { name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b10" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" }, { name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" }, { name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" }, @@ -3912,7 +3912,7 @@ crypto = [ [[package]] name = "pylance" -version = "14.0.0b6" +version = "14.0.0b10" source = { registry = "https://pypi.fury.io/lance-format/" } dependencies = [ { name = "lance-namespace" }, @@ -3922,12 +3922,12 @@ dependencies = [ { name = "pyarrow" }, ] wheels = [ - { url = "https://pypi.fury.io/lance-format/-/ver_DuQEn/pylance-14.0.0b6-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:3eade86fa30d9ff3248ee7ffb0b27ce29495394d12327da4d22b5c54bc2b59fe" }, - { url = "https://pypi.fury.io/lance-format/-/ver_K5sqj/pylance-14.0.0b6-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10e291974ee738a1d5d7e9dad17a634394a91cb60080b5ceac209077fbaec43c" }, - { url = "https://pypi.fury.io/lance-format/-/ver_EnbkT/pylance-14.0.0b6-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61bac10f5081c1210edd017f81c8c784d800b89446bd0ed05d4a24e7001965bc" }, - { url = "https://pypi.fury.io/lance-format/-/ver_19ffYg/pylance-14.0.0b6-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3c627ad4a9870611becf84409d8e5ecddc6be64e1dd53bee4bd5fb10ab561e5c" }, - { url = "https://pypi.fury.io/lance-format/-/ver_2eLLY5/pylance-14.0.0b6-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:bc603a7dd690233f1fe000600171e1c092f153bc9060cdf198825c4bbe1418aa" }, - { url = "https://pypi.fury.io/lance-format/-/ver_t8ggc/pylance-14.0.0b6-cp310-abi3-win_amd64.whl", hash = "sha256:1d32df32a3d157611450def07eae8ace3a7401286eac68cad9a24e0d5deed2db" }, + { url = "https://pypi.fury.io/lance-format/-/ver_4hvE/pylance-14.0.0b10-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:415b6b1b9fe735fbae7c755eb9774ae2fe0feddc90bf80bee24087cbfbfdf8c1" }, + { url = "https://pypi.fury.io/lance-format/-/ver_pM1km/pylance-14.0.0b10-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87317cad22f4d35fea11eb70aa5c0d71a0e1ea3b96de38f3deca933daa749fff" }, + { url = "https://pypi.fury.io/lance-format/-/ver_24jdyj/pylance-14.0.0b10-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6da554a2fc2197ba254676b12824d8120b48d65937697e0661a61302458a4740" }, + { url = "https://pypi.fury.io/lance-format/-/ver_XclaZ/pylance-14.0.0b10-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:36c80a801675bb9dcbd681fa99d0cdd5ca642dfb3ca08d71ac980278a2f7748b" }, + { url = "https://pypi.fury.io/lance-format/-/ver_L0yh/pylance-14.0.0b10-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:e50ad0b866faefb2fa3ae87328b93fa0d06b2bdd5bb5d0c1a6c74b907cc2fd93" }, + { url = "https://pypi.fury.io/lance-format/-/ver_2gbNr8/pylance-14.0.0b10-cp310-abi3-win_amd64.whl", hash = "sha256:3d86451fe00f71510005bb700737fb1a4f9361554541cbf802f3c5072554d014" }, ] [[package]] diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index 6b20fb123..8fb56ceca 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -1114,9 +1114,18 @@ pub struct VectorQueryRequest { pub column: Option, /// The vector(s) to search for pub query_vector: Vec>, - /// The minimum number of partitions to search - pub minimum_nprobes: usize, - /// The maximum number of partitions to search + /// The legacy number of partitions to search. + /// + /// Lance sets both probe bounds to this value. Explicit bounds can override it. + pub nprobes: Option, + /// The minimum number of partitions to search. + /// + /// If unset, Lance's default is used. + pub minimum_nprobes: Option, + /// The maximum number of partitions to search. + /// + /// If unset, Lance's default is used. Zero represents an explicitly unbounded + /// maximum on request and wire representations. pub maximum_nprobes: Option, /// The lower bound (inclusive) of the distance to search for. pub lower_bound: Option, @@ -1141,8 +1150,9 @@ impl Default for VectorQueryRequest { base: QueryRequest::default(), column: None, query_vector: Vec::new(), - minimum_nprobes: 20, - maximum_nprobes: Some(20), + nprobes: None, + minimum_nprobes: None, + maximum_nprobes: None, lower_bound: None, upper_bound: None, ef: None, @@ -1248,7 +1258,7 @@ impl VectorQuery { Ok(self) } - /// Set the number of partitions to search (probe) + /// Set the legacy IVF probe parameter /// /// This argument is only used when the vector column has an IVF PQ index. /// If there is no index then this value is ignored. @@ -1258,19 +1268,19 @@ impl VectorQuery { /// /// The partition whose centroids are closest to the query vector will be /// exhaustiely searched to find matches. This parameter controls how many - /// partitions should be searched. + /// partitions may be searched. /// /// Increasing this value will increase the recall of your query but will - /// also increase the latency of your query. The default value is 20. This - /// default is good for many cases but the best value to use will depend on - /// your data and the recall that you need to achieve. + /// also increase the latency of your query. If this method is not called, + /// Lance's adaptive probe defaults are used. /// /// For best results we recommend tuning this parameter with a benchmark against /// your actual data to find the smallest possible value that will still give /// you the desired recall. /// - /// This method sets both the minimum and maximum number of partitions to search. - /// For more fine-grained control see [`VectorQuery::minimum_nprobes`] and + /// LanceDB retains this as `nprobes` through local and remote request + /// construction. Lance sets both probe bounds to this value. For more + /// fine-grained control see [`VectorQuery::minimum_nprobes`] and /// [`VectorQuery::maximum_nprobes`]. /// /// Returns an error if `nprobes` is not greater than 0. @@ -1280,8 +1290,7 @@ impl VectorQuery { message: "nprobes must be greater than 0".to_string(), }); } - self.request.minimum_nprobes = nprobes; - self.request.maximum_nprobes = Some(nprobes); + self.request.nprobes = Some(nprobes); Ok(self) } @@ -1303,6 +1312,7 @@ impl VectorQuery { }); } if let Some(maximum_nprobes) = self.request.maximum_nprobes + && maximum_nprobes != 0 && minimum_nprobes > maximum_nprobes { return Err(Error::InvalidInput { @@ -1310,7 +1320,7 @@ impl VectorQuery { .to_string(), }); } - self.request.minimum_nprobes = minimum_nprobes; + self.request.minimum_nprobes = Some(minimum_nprobes); Ok(self) } @@ -1327,7 +1337,7 @@ impl VectorQuery { /// This can be useful when there is a narrow filter to allow these queries to /// spend more time searching and avoid potential false negatives. /// - /// Set to None to search all partitions, if needed, to satisfy the limit + /// Set to None to use Lance's default maximum probe behavior. pub fn maximum_nprobes(mut self, maximum_nprobes: Option) -> Result { if let Some(maximum_nprobes) = maximum_nprobes { if maximum_nprobes == 0 { @@ -1335,14 +1345,18 @@ impl VectorQuery { message: "maximum_nprobes must be greater than 0".to_string(), }); } - if maximum_nprobes < self.request.minimum_nprobes { + if let Some(minimum_nprobes) = self.request.minimum_nprobes + && maximum_nprobes < minimum_nprobes + { return Err(Error::InvalidInput { message: "maximum_nprobes must be greater than or equal to minimum_nprobes" .to_string(), }); } } - self.request.maximum_nprobes = maximum_nprobes; + // Preserve an explicit reset separately from an untouched maximum. The + // request protocols already use zero to represent an unbounded maximum. + self.request.maximum_nprobes = Some(maximum_nprobes.unwrap_or(0)); Ok(self) } @@ -2431,6 +2445,9 @@ mod tests { let vector = Float32Array::from_iter_values([0.1, 0.2]); let query = table.query().nearest_to(&[0.1, 0.2]).unwrap(); + assert_eq!(query.request.nprobes, None); + assert_eq!(query.request.minimum_nprobes, None); + assert_eq!(query.request.maximum_nprobes, None); assert_eq!( *query .request @@ -2442,6 +2459,37 @@ mod tests { vector ); + let minimum_only_query = query.clone().minimum_nprobes(20).unwrap(); + assert_eq!(minimum_only_query.request.minimum_nprobes, Some(20)); + assert_eq!(minimum_only_query.request.maximum_nprobes, None); + + let maximum_only_query = query.maximum_nprobes(Some(20)).unwrap(); + assert_eq!(maximum_only_query.request.minimum_nprobes, None); + assert_eq!(maximum_only_query.request.maximum_nprobes, Some(20)); + + let explicitly_unbounded_query = table + .query() + .nearest_to(&[0.1, 0.2]) + .unwrap() + .nprobes(20) + .unwrap() + .maximum_nprobes(None) + .unwrap(); + assert_eq!(explicitly_unbounded_query.request.nprobes, Some(20)); + assert_eq!(explicitly_unbounded_query.request.maximum_nprobes, Some(0)); + + let bounded_query = table + .query() + .nearest_to(&[0.1, 0.2]) + .unwrap() + .minimum_nprobes(5) + .unwrap() + .nprobes(20) + .unwrap(); + assert_eq!(bounded_query.request.nprobes, Some(20)); + assert_eq!(bounded_query.request.minimum_nprobes, Some(5)); + assert_eq!(bounded_query.request.maximum_nprobes, None); + let new_vector = Float32Array::from_iter_values([9.8, 8.7]); let query = table @@ -2450,7 +2498,7 @@ mod tests { .offset(1) .nearest_to(&[9.8, 8.7]) .unwrap() - .nprobes(1000) + .nprobes(20) .unwrap() .postfilter() .distance_type(DistanceType::Cosine) @@ -2469,8 +2517,9 @@ mod tests { ); assert_eq!(query.request.base.limit.unwrap(), 100); assert_eq!(query.request.base.offset.unwrap(), 1); - assert_eq!(query.request.minimum_nprobes, 1000); - assert_eq!(query.request.maximum_nprobes, Some(1000)); + assert_eq!(query.request.nprobes, Some(20)); + assert_eq!(query.request.minimum_nprobes, None); + assert_eq!(query.request.maximum_nprobes, None); assert!(query.request.use_index); assert_eq!(query.request.distance_type, Some(DistanceType::Cosine)); assert_eq!(query.request.approx_mode, Some(ApproxMode::Accurate)); @@ -2492,13 +2541,13 @@ mod tests { Err(Error::InvalidInput { message }) if message == "nprobes must be greater than 0" )); - // Fixed probe counts replace both bounds, even when moving below or - // above the previous range. + // Legacy probe counts remain independent until Lance resolves them. let mut query = query; for nprobes in [30, 1, 50] { query = query.nprobes(nprobes).unwrap(); - assert_eq!(query.request.minimum_nprobes, nprobes); - assert_eq!(query.request.maximum_nprobes, Some(nprobes)); + assert_eq!(query.request.nprobes, Some(nprobes)); + assert_eq!(query.request.minimum_nprobes, None); + assert_eq!(query.request.maximum_nprobes, None); } } diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index f0d1e554d..119577dc5 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -77,7 +77,6 @@ use lance::dataset::{ColumnAlteration, NewColumnTransform, Version}; use lance_datafusion::exec::{OneShotExec, execute_plan}; use reqwest::{RequestBuilder, Response}; use serde::{Deserialize, Serialize}; -use serde_json::Number; use std::collections::{HashMap, HashSet}; use std::io::Cursor; use std::pin::Pin; @@ -1147,17 +1146,14 @@ impl RemoteTable { if let Some(approx_mode) = query.approx_mode { body["approx_mode"] = serde_json::json!(approx_mode); } - // In 0.23.1 we migrated from `nprobes` to `minimum_nprobes` and `maximum_nprobes`. - // Old client / new server: since minimum_nprobes is missing, fallback to nprobes - // New client / old server: old server will only see nprobes, make sure to set both - // nprobes and minimum_nprobes - // New client / new server: since minimum_nprobes is present, server can ignore nprobes - body["nprobes"] = query.minimum_nprobes.into(); - body["minimum_nprobes"] = query.minimum_nprobes.into(); + if let Some(nprobes) = query.nprobes { + body["nprobes"] = nprobes.into(); + } + if let Some(minimum_nprobes) = query.minimum_nprobes { + body["minimum_nprobes"] = minimum_nprobes.into(); + } if let Some(maximum_nprobes) = query.maximum_nprobes { body["maximum_nprobes"] = maximum_nprobes.into(); - } else { - body["maximum_nprobes"] = serde_json::Value::Number(Number::from_u128(0).unwrap()) } body["lower_bound"] = query.lower_bound.into(); body["upper_bound"] = query.upper_bound.into(); @@ -6090,9 +6086,6 @@ mod tests { let body: serde_json::Value = serde_json::from_slice(body).unwrap(); let mut expected_body = serde_json::json!({ "prefilter": true, - "nprobes": 20, - "minimum_nprobes": 20, - "maximum_nprobes": 20, "lower_bound": Option::::None, "upper_bound": Option::::None, "k": 10, @@ -6124,7 +6117,7 @@ mod tests { } #[tokio::test] - async fn test_query_vector_approx_mode_sent_when_set() { + async fn test_query_vector_minimum_only_and_approx_mode() { let expected_data = RecordBatch::try_new( Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], @@ -6144,9 +6137,7 @@ mod tests { let body: serde_json::Value = serde_json::from_slice(body).unwrap(); let mut expected_body = serde_json::json!({ "prefilter": true, - "nprobes": 20, - "minimum_nprobes": 20, - "maximum_nprobes": 20, + "minimum_nprobes": 5, "approx_mode": "accurate", "lower_bound": Option::::None, "upper_bound": Option::::None, @@ -6170,6 +6161,8 @@ mod tests { .query() .nearest_to(vec![0.1, 0.2, 0.3]) .unwrap() + .minimum_nprobes(5) + .unwrap() .approx_mode(crate::ApproxMode::Accurate) .execute() .await; @@ -6178,6 +6171,39 @@ mod tests { assert_eq!(data[0].as_ref().unwrap(), &expected_data); } + #[tokio::test] + async fn test_query_vector_explicitly_unbounded_maximum() { + let table = Table::new_with_handler("my_table", |request| { + let body = request.body().unwrap().as_bytes().unwrap(); + let body: serde_json::Value = serde_json::from_slice(body).unwrap(); + assert_eq!(body["nprobes"], 5); + assert_eq!(body["maximum_nprobes"], 0); + + let data = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), + vec![Arc::new(Int32Array::from(vec![1]))], + ) + .unwrap(); + http::Response::builder() + .status(200) + .header(CONTENT_TYPE, ARROW_FILE_CONTENT_TYPE) + .body(write_ipc_file(&data)) + .unwrap() + }); + + table + .query() + .nearest_to(vec![0.1, 0.2, 0.3]) + .unwrap() + .nprobes(5) + .unwrap() + .maximum_nprobes(None) + .unwrap() + .execute() + .await + .unwrap(); + } + #[tokio::test] async fn test_query_fts_default_values() { let expected_data = RecordBatch::try_new( @@ -6276,8 +6302,8 @@ mod tests { } ], "nprobes": 12, - "minimum_nprobes": 12, - "maximum_nprobes": 12, + "minimum_nprobes": 3, + "maximum_nprobes": 10, "lower_bound": Option::::None, "upper_bound": Option::::None, "ef": Option::::None, @@ -6317,6 +6343,10 @@ mod tests { .distance_type(crate::DistanceType::Cosine) .nprobes(12) .unwrap() + .minimum_nprobes(3) + .unwrap() + .maximum_nprobes(Some(10)) + .unwrap() .refine_factor(2) .bypass_vector_index() .execute() @@ -6347,9 +6377,6 @@ mod tests { "vector_column": "image.embedding", "prefilter": true, "k": 10, - "nprobes": 20, - "minimum_nprobes": 20, - "maximum_nprobes": 20, "lower_bound": Option::::None, "upper_bound": Option::::None, "ef": Option::::None, diff --git a/rust/lancedb/src/table/datafusion/udtf/vector_query.rs b/rust/lancedb/src/table/datafusion/udtf/vector_query.rs index 000addbe1..543a59589 100644 --- a/rust/lancedb/src/table/datafusion/udtf/vector_query.rs +++ b/rust/lancedb/src/table/datafusion/udtf/vector_query.rs @@ -411,9 +411,7 @@ impl ExecutionPlan for VectorDedupExec { .map_err(|e| datafusion_common::DataFusionError::External(Box::new(e)))?; let mut scanner = dataset.scan(); scanner - .with_row_addr_prefilter(RowAddrMask::from_block(RowAddrTreeMap::from_iter( - removed, - ))) + .with_row_id_prefilter(RowAddrMask::from_block(RowAddrTreeMap::from_iter(removed))) .blob_handling(BlobHandling::AllBinary) .batch_size(MATERIALIZE_BATCH_ROWS) .strict_batch_size(true); diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 354eef5a5..1ad084e22 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -1296,6 +1296,21 @@ mod lsm_tests { rows } + async fn collect_ids(stream: SendableRecordBatchStream) -> Vec { + let batches: Vec<_> = stream.try_collect().await.unwrap(); + batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_primitive::() + .values() + .to_vec() + }) + .collect() + } + /// Upsert `ids` (value = 0..n) through the LSM `merge_insert` path. async fn lsm_upsert(table: &Table, ids: Vec) { let mut builder = table.merge_insert(&[]); @@ -1772,7 +1787,6 @@ mod lsm_tests { use crate::index::Index; use crate::index::vector::IvfPqIndexBuilder; use arrow::array::{FixedSizeListBuilder, Float32Builder}; - use arrow::datatypes::Int64Type; const DIM: i32 = 8; const N: i64 = 256; @@ -1817,7 +1831,7 @@ mod lsm_tests { &["vec"], Index::IvfPq( IvfPqIndexBuilder::default() - .num_partitions(1) + .num_partitions(8) .num_sub_vectors(2), ), ) @@ -1851,21 +1865,32 @@ mod lsm_tests { .execute() .await .unwrap(); - let batches: Vec<_> = stream.try_collect().await.unwrap(); - let ids: Vec = batches - .iter() - .flat_map(|b| { - b.column_by_name("id") - .unwrap() - .as_primitive::() - .values() - .to_vec() - }) - .collect(); + let ids = collect_ids(stream).await; assert_eq!( ids, vec![9999], "LSM vector search must rank the memtable row first" ); + + let base_query = table + .query() + .nearest_to(&[0.0_f32; 8]) + .unwrap() + .only_if("id = 255") + .limit(1); + for query in [ + base_query.clone(), + base_query.clone().nprobes(8).unwrap(), + base_query.clone().minimum_nprobes(8).unwrap(), + base_query.clone().maximum_nprobes(Some(8)).unwrap(), + base_query + .nprobes(1) + .unwrap() + .maximum_nprobes(None) + .unwrap(), + ] { + let ids = collect_ids(query.execute().await.unwrap()).await; + assert_eq!(ids, vec![255]); + } } } diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index d611fe8a5..77c4ca1a5 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -106,19 +106,16 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R } fn requires_local_namespace_execution(query: &AnyQuery) -> bool { - // The namespace QueryTable request has no approx_mode or use_lsm field yet, so - // pushing these down would silently ignore the user's setting. For use_lsm that - // is worse than a tuning miss: MemWAL read routing lives only in `create_plan`, - // so a pushed-down query would return stale base-only data with no error. + // The namespace QueryTable request cannot represent approx_mode or use_lsm. + // For use_lsm that is worse than a tuning miss: MemWAL read routing lives only + // in `create_plan`, so a pushed-down query would return stale base-only data. if query.base().use_lsm.is_some() || query.base().take_offsets.is_some() { return true; } matches!( query, - AnyQuery::VectorQuery(VectorQueryRequest { - approx_mode: Some(_), - .. - }) + AnyQuery::VectorQuery(vector_query) + if vector_query.approx_mode.is_some() ) } @@ -296,8 +293,17 @@ pub async fn create_plan( scanner.approx_mode(approx_mode.into()); } - scanner.minimum_nprobes(query.minimum_nprobes); - if let Some(maximum_nprobes) = query.maximum_nprobes { + if let Some(nprobes) = query.nprobes { + if query.maximum_nprobes == Some(0) { + scanner.minimum_nprobes(nprobes); + } else { + scanner.nprobes(nprobes); + } + } + if let Some(minimum_nprobes) = query.minimum_nprobes { + scanner.minimum_nprobes(minimum_nprobes); + } + if let Some(maximum_nprobes) = query.maximum_nprobes.filter(|nprobes| *nprobes != 0) { scanner.maximum_nprobes(maximum_nprobes); } } @@ -630,7 +636,9 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result { columns, offset: vq.base.offset.map(|o| o as i32), distance_type: vq.distance_type.map(|dt| dt.to_string()), - nprobes: Some(vq.minimum_nprobes as i32), + nprobes: vq.nprobes.map(|nprobes| nprobes as i32), + minimum_nprobes: vq.minimum_nprobes.map(|nprobes| nprobes as i32), + maximum_nprobes: vq.maximum_nprobes.map(|nprobes| nprobes as i32), ef: vq.ef.map(|e| e as i32), refine_factor: vq.refine_factor.map(|r| r as i32), lower_bound: vq.lower_bound, @@ -919,7 +927,6 @@ mod tests { column: Some("vector".to_string()), // We cast here to satisfy the struct definition query_vector: vec![query_vector as Arc], - minimum_nprobes: 20, distance_type: Some(crate::DistanceType::L2), ..Default::default() }; @@ -940,6 +947,7 @@ mod tests { ); assert_eq!(ns_request.vector_column, Some("vector".to_string())); assert_eq!(ns_request.distance_type, Some("l2".to_string())); + assert_eq!(ns_request.nprobes, None); // Verify the vector data was extracted correctly assert!(ns_request.vector.single_vector.is_some()); @@ -949,6 +957,66 @@ mod tests { ); } + #[test] + fn test_namespace_query_probe_pushdown_compatibility() { + let default_query = AnyQuery::VectorQuery(VectorQueryRequest::default()); + assert!(!requires_local_namespace_execution(&default_query)); + + let nprobes_query = AnyQuery::VectorQuery(VectorQueryRequest { + query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], + nprobes: Some(20), + ..Default::default() + }); + assert!(!requires_local_namespace_execution(&nprobes_query)); + assert_eq!( + convert_to_namespace_query(&nprobes_query).unwrap().nprobes, + Some(20) + ); + + let minimum_query = AnyQuery::VectorQuery(VectorQueryRequest { + query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], + minimum_nprobes: Some(5), + ..Default::default() + }); + assert!(!requires_local_namespace_execution(&minimum_query)); + let request = convert_to_namespace_query(&minimum_query).unwrap(); + assert_eq!(request.minimum_nprobes, Some(5)); + assert_eq!(request.maximum_nprobes, None); + + let maximum_query = AnyQuery::VectorQuery(VectorQueryRequest { + query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], + maximum_nprobes: Some(10), + ..Default::default() + }); + assert!(!requires_local_namespace_execution(&maximum_query)); + let request = convert_to_namespace_query(&maximum_query).unwrap(); + assert_eq!(request.minimum_nprobes, None); + assert_eq!(request.maximum_nprobes, Some(10)); + + let exact_query = AnyQuery::VectorQuery(VectorQueryRequest { + query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], + nprobes: Some(8), + minimum_nprobes: Some(20), + maximum_nprobes: Some(20), + ..Default::default() + }); + assert!(!requires_local_namespace_execution(&exact_query)); + let request = convert_to_namespace_query(&exact_query).unwrap(); + assert_eq!(request.nprobes, Some(8)); + assert_eq!(request.minimum_nprobes, Some(20)); + assert_eq!(request.maximum_nprobes, Some(20)); + + let explicitly_unbounded_query = AnyQuery::VectorQuery(VectorQueryRequest { + query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], + nprobes: Some(8), + maximum_nprobes: Some(0), + ..Default::default() + }); + let request = convert_to_namespace_query(&explicitly_unbounded_query).unwrap(); + assert_eq!(request.nprobes, Some(8)); + assert_eq!(request.maximum_nprobes, Some(0)); + } + #[test] fn test_convert_to_namespace_query_plain_query() { let q = QueryRequest { diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index f3e40fc7b..5ec053483 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -170,9 +170,8 @@ fn reject_unsupported(query: &VectorQueryRequest) -> Result<()> { // Vector-only knobs the LSM scanner cannot honor. Both change results rather // than just recall, so error instead of silently ignoring them: distance_range // would return rows outside the bound, and use_index(false) asks for a - // brute-force search the index-only base arm can't do. (ef / approx_mode / - // maximum_nprobes are recall/speed knobs and are left to no-op — and - // maximum_nprobes defaults to Some, so it cannot be rejected on presence.) + // brute-force search the index-only base arm can't do. (ef / approx_mode are + // recall/speed knobs and are left to no-op.) if !query.query_vector.is_empty() { if query.lower_bound.is_some() || query.upper_bound.is_some() { return unsupported("distance_range on vector search"); @@ -740,8 +739,20 @@ async fn vector_plan( let mut scanner = base_scanner(dataset, query, pk_columns, snapshots, in_memory)? .with_overfetch_factor(LSM_OVERFETCH_FACTOR) .nearest(&column, query_vector.as_ref(), k)? - .nprobes(query.minimum_nprobes) .distance_metric(distance_type.into()); + if let Some(nprobes) = query.nprobes { + scanner = if query.maximum_nprobes == Some(0) { + scanner.minimum_nprobes(nprobes) + } else { + scanner.nprobes(nprobes) + }; + } + if let Some(minimum_nprobes) = query.minimum_nprobes { + scanner = scanner.minimum_nprobes(minimum_nprobes); + } + if let Some(maximum_nprobes) = query.maximum_nprobes.filter(|nprobes| *nprobes != 0) { + scanner = scanner.maximum_nprobes(maximum_nprobes); + } if let Some(refine_factor) = query.refine_factor { scanner = scanner.refine(refine_factor); } From 55ca56d4e2250c8121e62bd36c2e75a2e17a8552 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Fri, 9 Oct 2026 21:13:04 +0000 Subject: [PATCH 07/19] =?UTF-8?q?Bump=20version:=200.41.0-beta.2=20?= =?UTF-8?q?=E2=86=92=200.41.0-beta.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 70718edbb..e74604e0c 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.41.0-beta.2" +current_version = "0.41.0-beta.3" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 18cb1a98e..3180e6ad7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5760,7 +5760,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.41.0-beta.2" +version = "0.41.0-beta.3" dependencies = [ "ahash", "anyhow", @@ -5857,7 +5857,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.41.0-beta.2" +version = "0.41.0-beta.3" dependencies = [ "arrow-array", "arrow-buffer", @@ -5882,7 +5882,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.41.0-beta.2" +version = "0.41.0-beta.3" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 88cfc4a30..81b4afa42 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.41.0-beta.2 + 0.41.0-beta.3 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 987d51345..df20bc7f0 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.2 + 0.41.0-beta.3 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 964dfed0c..e6308140d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.2 + 0.41.0-beta.3 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index c81fdb459..0fd2434c1 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.41.0-beta.2" +version = "0.41.0-beta.3" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 90106b6d4..486d9692a 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.41.0-beta.2", + "version": "0.41.0-beta.3", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 73bafc124..aeeee36c1 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.41.0-beta.2", + "version": "0.41.0-beta.3", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index dea0f49a2..f6cc198c4 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.41.0-beta.2", + "version": "0.41.0-beta.3", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 0ae15f6d5..37c83e4ee 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.41.0-beta.2", + "version": "0.41.0-beta.3", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index b3a17754b..c25486f7c 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.41.0-beta.2", + "version": "0.41.0-beta.3", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 4681f63c9..c963f9422 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.41.0-beta.2", + "version": "0.41.0-beta.3", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index bcf00b687..7220ca9f1 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.41.0-beta.2", + "version": "0.41.0-beta.3", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index b4b64eec1..acecfc075 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.41.0-beta.2", + "version": "0.41.0-beta.3", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 4f0dc603e..4bd3cf357 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.41.0-beta.2" +version = "0.41.0-beta.3" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index b15c24ca5..f42c05bed 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.41.0-beta.2" +version = "0.41.0-beta.3" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 1f378c1b4999d753facc82f5b829c664069948d1 Mon Sep 17 00:00:00 2001 From: Kevin Dong <2970752254@qq.com> Date: Fri, 9 Oct 2026 16:51:56 -0700 Subject: [PATCH 08/19] fix(python): preserve Decimal precision in table updates (#4431) Python `Decimal` values and Arrow decimal scalars raise `NotImplementedError` in `table.update(values=...)`. Register a Decimal converter that emits quoted fixed-point text so the native cast to the target column preserves every digit, including decimal128 and decimal256 values. Non-finite Decimal values raise `ValueError` with a static message. Closes #4360. Validation with the native extension built from the current source at 7f593359, on macOS arm64 and Python 3.12.14: - All 33 new regression cases failed before the fix and passed afterward. They cover Python and Arrow scalar updates, signs, zero, exponents, full 38/76-digit precision, and a Decimal context precision of 6. - 16 existing converter, update, and Decimal expression tests passed. - Repository Ruff formatting and lint passed; formatting left all 156 files unchanged. - A native round-trip check confirmed that an unquoted SQL number rounds a high-precision value while the quoted Decimal update preserves it exactly. The full suite, other Python versions, and remote API were not exercised. AI assistance: Codex helped implement and validate the Decimal conversion and its regression tests. --- python/python/lancedb/util.py | 9 ++++++ python/python/tests/test_table.py | 47 ++++++++++++++++++++++++++++++- python/python/tests/test_util.py | 36 +++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/python/python/lancedb/util.py b/python/python/lancedb/util.py index 3a6a8a102..fefc234ce 100644 --- a/python/python/lancedb/util.py +++ b/python/python/lancedb/util.py @@ -10,6 +10,7 @@ import os import pathlib import warnings from datetime import date, datetime +from decimal import Decimal from functools import singledispatch from typing import Tuple, Union, Optional, Any, List from urllib.parse import urlparse @@ -373,6 +374,14 @@ def _(value: float): return str(value) +@value_to_sql.register(Decimal) +def _(value: Decimal): + if not value.is_finite(): + raise ValueError("Non-finite Decimal values cannot be converted to SQL") + # Quote exact decimal text so the target-column cast avoids float rounding. + return value_to_sql(format(value, "f")) + + @value_to_sql.register(bool) def _(value: bool): return str(value).upper() diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index c579f3037..a963a0d19 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -12,7 +12,7 @@ import warnings import weakref from concurrent.futures import ThreadPoolExecutor from datetime import date, datetime, timedelta -from decimal import Decimal +from decimal import Decimal, localcontext from time import sleep from typing import List from unittest.mock import patch @@ -2612,6 +2612,51 @@ def test_update_with_arrow_scalar(mem_db: DBConnection): assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0, 3.0, 4.0]] +@pytest.mark.parametrize("use_arrow_scalar", [False, True]) +@pytest.mark.parametrize( + "decimal_type, value", + [ + (pa.decimal128(10, 2), "9.99"), + (pa.decimal128(10, 2), "-9.99"), + (pa.decimal128(10, 2), "0.00"), + (pa.decimal128(10, 2), "1.23E+5"), + (pa.decimal128(38, 18), "1E-18"), + (pa.decimal128(38, 18), "12345678901234567.123456789012345678"), + (pa.decimal128(38, 18), "12345678901234567890.123456789012345678"), + (pa.decimal128(38, 18), "-12345678901234567890.123456789012345678"), + ( + pa.decimal256(76, 38), + "12345678901234567890123456789012345678." + "12345678901234567890123456789012345678", + ), + ( + pa.decimal256(76, 38), + "-12345678901234567890123456789012345678." + "12345678901234567890123456789012345678", + ), + ], +) +def test_update_decimal(mem_db: DBConnection, decimal_type, value, use_arrow_scalar): + expected = Decimal(value) + table = mem_db.create_table( + "my_table", + pa.table( + { + "id": [1, 2], + "price": pa.array([Decimal("1"), expected], type=decimal_type), + } + ), + ) + update = table.to_arrow()["price"][1] if use_arrow_scalar else expected + with localcontext() as context: + context.prec = 6 + result = table.update(where="id = 1", values={"price": update}) + assert result.rows_updated == 1 + actual = table.to_arrow() + assert actual.schema.field("price").type == decimal_type + assert actual["price"].to_pylist() == [expected, expected] + + def test_update_types(mem_db: DBConnection): table = mem_db.create_table( "my_table", diff --git a/python/python/tests/test_util.py b/python/python/tests/test_util.py index d9c574eb9..fe5187604 100644 --- a/python/python/tests/test_util.py +++ b/python/python/tests/test_util.py @@ -4,6 +4,7 @@ import os import pathlib +from decimal import Decimal, localcontext from typing import Optional import lance @@ -157,6 +158,41 @@ def test_value_to_sql_string(tmp_path): assert table.to_pandas().query("search == @value")["replace"].item() == value +@pytest.mark.parametrize( + "value, expected", + [ + ("9.99", "'9.99'"), + ("-9.99", "'-9.99'"), + ("0", "'0'"), + ("-0.00", "'-0.00'"), + ("1.23E+5", "'123000'"), + ("1E-18", "'0.000000000000000001'"), + ( + "12345678901234567890.123456789012345678", + "'12345678901234567890.123456789012345678'", + ), + ( + "12345678901234567890123456789012345678." + "12345678901234567890123456789012345678", + "'12345678901234567890123456789012345678." + "12345678901234567890123456789012345678'", + ), + ], +) +def test_value_to_sql_decimal(value, expected): + with localcontext() as context: + context.prec = 6 + assert value_to_sql(Decimal(value)) == expected + + +@pytest.mark.parametrize("value", ["NaN", "sNaN", "NaN123456", "Infinity", "-Infinity"]) +def test_value_to_sql_decimal_non_finite(value): + with pytest.raises( + ValueError, match="^Non-finite Decimal values cannot be converted to SQL$" + ): + value_to_sql(Decimal(value)) + + def test_value_to_sql_dict(): # Simple flat struct assert value_to_sql({"a": 1, "b": "hello"}) == "named_struct('a', 1, 'b', 'hello')" From b1429ffa68ea9efb16c366db5c8a37655538241e Mon Sep 17 00:00:00 2001 From: Will Jones Date: Fri, 9 Oct 2026 19:36:49 -0700 Subject: [PATCH 09/19] fix: use aws-lc-rs instead of ring for remote HTTP and gRPC TLS (#4504) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary LanceDB's remote HTTP client (reqwest) and gRPC client (tonic) used rustls with the `ring` crypto provider, while the AWS SDK and object_store use aws-lc-rs. This moves both clients to aws-lc-rs: - `tonic`: `tls-ring` → `tls-aws-lc`. - `reqwest`: 0.12 → 0.13. reqwest 0.12 has no aws-lc-rs feature; 0.13's `rustls` feature uses it. 0.13 was already in the lockfile through object_store. No code changes were needed. - `macos-system-configuration` is replaced by its 0.13 name, `system-proxy`, which also reads the system proxy on Windows. - `query` is now an opt-in feature in 0.13; it's used for request query parameters. - Certificate verification moves from `rustls-native-certs` to `rustls-platform-verifier`, which uses the OS verifier on macOS and Windows. `ring` is still compiled in, because `lance-namespace-impls` and `lance-namespace-reqwest-client` enable it through reqwest 0.12 (lance-format/lance#9855, lance-format/lance-namespace#369). Both clients here pick aws-lc-rs explicitly, so they don't depend on which providers are compiled in. ## Evidence A temporary example that connects the remote client to real HTTPS hosts: ``` https://example.com: Http error: ... 404 Not Found: ... # TLS handshake OK https://expired.badssl.com: Hit retry limit ... connect_failures=3/3 # bad cert rejected ``` ## Merge Risk **Door:** two-way; dependency feature changes only. **Blast Radius:** `remote` TLS and certificate verification (Python, Node, and Rust LanceDB Cloud clients). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 5 ++++- Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 8 +++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3180e6ad7..b0860f72f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5833,7 +5833,7 @@ dependencies = [ "rand 0.9.5", "random_word", "regex", - "reqwest 0.12.28", + "reqwest 0.13.4", "roaring", "rstest", "semver", @@ -8972,6 +8972,7 @@ checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64 0.22.1", "bytes", + "encoding_rs", "futures-core", "futures-util", "h2 0.4.16", @@ -8983,6 +8984,7 @@ dependencies = [ "hyper-util", "js-sys", "log", + "mime", "percent-encoding", "pin-project-lite", "quinn", @@ -8991,6 +8993,7 @@ dependencies = [ "rustls-platform-verifier", "serde", "serde_json", + "serde_urlencoded", "sync_wrapper", "tokio", "tokio-rustls 0.26.4", diff --git a/Cargo.toml b/Cargo.toml index 6c05f297a..8c9f5a91e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,7 +75,7 @@ serde = "1" serde_json = "1" tempfile = "3.5.0" tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] } -tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] } +tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] } uuid = { version = "1.7.0", features = ["v4", "v7"] } chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index f42c05bed..d148d19f5 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -70,15 +70,17 @@ tempfile = { workspace = true } aws-smithy-types = { workspace = true, optional = true } aws-sdk-bedrockruntime = { version = "1.27.0", optional = true } # For remote feature -reqwest = { version = "0.12.0", default-features = false, features = [ +reqwest = { version = "0.13.0", default-features = false, features = [ "charset", "gzip", "http2", "json", - "macos-system-configuration", + "query", # Avoid linking OpenSSL into Python wheels, which breaks on FIPS hosts. - "rustls-tls-native-roots", + # This uses the aws-lc-rs crypto provider. + "rustls", "stream", + "system-proxy", ], optional = true } tonic = { workspace = true, optional = true } http = { version = "1", optional = true } # Matching what is in reqwest From 4b20906eb39e4a4e030f06099c94bd3b31861ea2 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Fri, 9 Oct 2026 19:53:40 -0700 Subject: [PATCH 10/19] fix: bump vulnerable Rust deps and drop legacy rustls 0.21 (#4502) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes the Dependabot alerts for Rust crates that have a patched release. - `Cargo.lock`: aws-smithy-json 0.62.5 → 0.62.7, quinn-proto 0.11.14 → 0.11.19, xxhash-rust 0.8.15 → 0.8.19. - `docs/web-tests/rs/Cargo.lock`: rustls 0.23.35 → 0.23.45, rustls-webpki 0.103.13 → 0.103.15, serde_with 3.16.1 → 3.22.0, rand 0.8.5 → 0.8.8 and 0.9.2 → 0.9.5, xxhash-rust 0.8.15 → 0.8.19. - `rust/lancedb/Cargo.toml`: the AWS SDK crates' default features include the legacy `rustls` feature, which pulls in rustls 0.21 and the vulnerable rustls-webpki 0.101.7 via hyper 0.14. `aws-sdk-bedrockruntime` (behind the optional `bedrock` feature) and the `aws-sdk-s3`/`dynamodb`/`kms` dev-dependencies now use `default-features = false` with `default-https-client` + `rt-tokio`, the same rustls 0.23 client `aws-config` already uses. Minimum versions are raised to the first releases that have `default-https-client`. This removes rustls 0.21, rustls-webpki 0.101, hyper 0.14 and h2 0.3 from the lockfile. Not fixed: fast-float 0.2.0 has no patched release. It comes from polars-arrow via the optional `polars` feature, pinned to `<0.40`; removing it needs a polars upgrade. ## Merge Risk **Door:** two-way; lockfile and dependency-feature changes only. **Blast Radius:** `bedrock` embedding function's HTTP client, and S3/DynamoDB/KMS integration tests. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 248 ++++++++++------------------------- docs/web-tests/rs/Cargo.lock | 149 +++++++++------------ rust/lancedb/Cargo.toml | 10 +- 3 files changed, 141 insertions(+), 266 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0860f72f..12aab95c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -986,40 +986,36 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.12" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a2f165a7feee6f263028b899d0a181987f4fa7179a6411a32a439fba7c5f769" +checksum = "5c3ef8931ad1c98aa6a55b4256f847f3116090819844e0dd41ea682cac5dd2d3" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.3.27", - "h2 0.4.16", - "http 0.2.12", + "h2", "http 1.5.0", - "http-body 0.4.6", - "hyper 0.14.32", - "hyper 1.9.0", - "hyper-rustls 0.24.2", - "hyper-rustls 0.27.9", + "hyper", + "hyper-rustls", "hyper-util", "pin-project-lite", - "rustls 0.21.12", - "rustls 0.23.45", + "rustls", "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tower", "tracing", ] [[package]] name = "aws-smithy-json" -version = "0.62.5" +version = "0.62.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a" +checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", ] @@ -1070,9 +1066,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.12.1" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc117c179ecf39a62a0a3f49f600e9ac26a7ad7dd172177999f83933af776c32" +checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -1170,7 +1166,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.9.0", + "hyper", "hyper-util", "itoa", "matchit 0.7.3", @@ -1819,7 +1815,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3315,7 +3311,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4000,11 +3996,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if 1.0.4", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -4110,25 +4104,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "h2" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.14.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.16" @@ -4430,30 +4405,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.27", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.10", - "tokio", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "1.9.0" @@ -4464,7 +4415,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.16", + "h2", "http 1.5.0", "http-body 1.1.0", "httparse", @@ -4476,21 +4427,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http 0.2.12", - "hyper 0.14.32", - "log", - "rustls 0.21.12", - "tokio", - "tokio-rustls 0.24.1", -] - [[package]] name = "hyper-rustls" version = "0.27.9" @@ -4498,12 +4434,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.5.0", - "hyper 1.9.0", + "hyper", "hyper-util", - "rustls 0.23.45", + "rustls", "rustls-native-certs", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tower-service", "webpki-roots 1.0.7", ] @@ -4514,7 +4450,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.9.0", + "hyper", "hyper-util", "pin-project-lite", "tokio", @@ -4533,12 +4469,12 @@ dependencies = [ "futures-util", "http 1.5.0", "http-body 1.1.0", - "hyper 1.9.0", + "hyper", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2", "system-configuration", "tokio", "tower-service", @@ -4880,7 +4816,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6899,7 +6835,7 @@ dependencies = [ "http-body-util", "httparse", "humantime", - "hyper 1.9.0", + "hyper", "itertools 0.15.0", "md-5", "nix 0.31.3", @@ -8379,8 +8315,8 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.45", - "socket2 0.6.3", + "rustls", + "socket2", "thiserror 2.0.18", "tokio", "tracing", @@ -8389,18 +8325,19 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.2", "lru-slab", - "rand 0.9.5", + "rand 0.10.1", + "rand_pcg", "ring", "rustc-hash", - "rustls 0.23.45", + "rustls", "rustls-pki-types", "slab", "thiserror 2.0.18", @@ -8418,9 +8355,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -8565,6 +8502,15 @@ dependencies = [ "rand 0.9.5", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rand_xoshiro" version = "0.7.0" @@ -8929,12 +8875,12 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.16", + "h2", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.9.0", - "hyper-rustls 0.27.9", + "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", @@ -8943,7 +8889,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.45", + "rustls", "rustls-native-certs", "rustls-pki-types", "serde", @@ -8951,7 +8897,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tokio-util", "tower", "tower-http 0.6.10", @@ -8975,12 +8921,12 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.16", + "h2", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.9.0", - "hyper-rustls 0.27.9", + "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", @@ -8988,7 +8934,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.45", + "rustls", "rustls-pki-types", "rustls-platform-verifier", "serde", @@ -8996,7 +8942,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tokio-util", "tower", "tower-http 0.6.10", @@ -9221,19 +9167,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", + "windows-sys 0.52.0", ] [[package]] @@ -9247,7 +9181,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.15", + "rustls-webpki", "subtle", "zeroize", ] @@ -9285,14 +9219,14 @@ dependencies = [ "jni", "log", "once_cell", - "rustls 0.23.45", + "rustls", "rustls-native-certs", "rustls-platform-verifier-android", - "rustls-webpki 0.103.15", + "rustls-webpki", "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -9301,16 +9235,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted 0.9.0", -] - [[package]] name = "rustls-webpki" version = "0.103.15" @@ -9426,16 +9350,6 @@ dependencies = [ "sha2 0.10.9", ] -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted 0.9.0", -] - [[package]] name = "sec1" version = "0.3.0" @@ -9857,16 +9771,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.3" @@ -10285,7 +10189,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -10498,7 +10402,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.3", + "socket2", "tokio-macros", "windows-sys 0.61.2", ] @@ -10525,23 +10429,13 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.12", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.45", + "rustls", "tokio", ] @@ -10634,20 +10528,20 @@ dependencies = [ "axum 0.8.9", "base64 0.22.1", "bytes", - "h2 0.4.16", + "h2", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.9.0", + "hyper", "hyper-timeout", "hyper-util", "percent-encoding", "pin-project", "rustls-native-certs", - "socket2 0.6.3", + "socket2", "sync_wrapper", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tokio-stream", "tower", "tower-layer", @@ -10955,7 +10849,7 @@ dependencies = [ "flate2", "log", "once_cell", - "rustls 0.23.45", + "rustls", "rustls-pki-types", "serde", "serde_json", @@ -11306,7 +11200,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -11776,7 +11670,7 @@ dependencies = [ "crc32fast", "futures", "http 1.5.0", - "hyper 1.9.0", + "hyper", "more-asserts", "rand 0.10.1", "redb", @@ -11912,9 +11806,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7" [[package]] name = "yoke" diff --git a/docs/web-tests/rs/Cargo.lock b/docs/web-tests/rs/Cargo.lock index 0dff8ddfc..599633c5e 100644 --- a/docs/web-tests/rs/Cargo.lock +++ b/docs/web-tests/rs/Cargo.lock @@ -556,6 +556,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bumpalo" version = "3.19.0" @@ -1065,12 +1074,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core 0.21.3", - "darling_macro 0.21.3", + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] @@ -1089,11 +1098,10 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", @@ -1114,11 +1122,11 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core 0.21.3", + "darling_core 0.23.0", "quote", "syn 2.0.117", ] @@ -1177,7 +1185,7 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.9.2", + "rand 0.9.5", "regex", "sqlparser 0.61.0", "tempfile", @@ -1292,7 +1300,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "rand 0.9.2", + "rand 0.9.5", "tokio", "url", ] @@ -1392,7 +1400,7 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.9.2", + "rand 0.9.5", "tempfile", "url", ] @@ -1457,7 +1465,7 @@ dependencies = [ "md-5", "memchr", "num-traits", - "rand 0.9.2", + "rand 0.9.5", "regex", "sha2", "unicode-segmentation", @@ -2078,7 +2086,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7af6e24ed12cf382082d5a7f365df2b8e3d6b1518f615d54c85165e9b9718250" dependencies = [ "arrow-array", - "rand 0.9.2", + "rand 0.9.5", ] [[package]] @@ -2498,7 +2506,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core 0.61.2", ] [[package]] @@ -2877,7 +2885,7 @@ dependencies = [ "nom", "num-traits", "ordered-float", - "rand 0.9.2", + "rand 0.9.5", "ryu", "serde", "serde_json", @@ -2950,7 +2958,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.2", + "rand 0.9.5", "rayon", "roaring", "rustc-hash", @@ -2985,7 +2993,7 @@ dependencies = [ "half", "jsonb", "num-traits", - "rand 0.9.2", + "rand 0.9.5", ] [[package]] @@ -3007,7 +3015,7 @@ dependencies = [ "half", "jsonb", "num-traits", - "rand 0.9.2", + "rand 0.9.5", ] [[package]] @@ -3071,7 +3079,7 @@ dependencies = [ "object_store", "pin-project", "prost", - "rand 0.9.2", + "rand 0.9.5", "roaring", "serde_json", "snafu 0.9.1", @@ -3110,7 +3118,7 @@ dependencies = [ "object_store", "pin-project", "prost", - "rand 0.9.2", + "rand 0.9.5", "roaring", "serde_json", "snafu 0.9.1", @@ -3169,7 +3177,7 @@ dependencies = [ "futures", "half", "hex", - "rand 0.9.2", + "rand 0.9.5", "rand_distr 0.5.1", "rand_xoshiro", ] @@ -3214,7 +3222,7 @@ dependencies = [ "num-traits", "prost", "prost-build", - "rand 0.9.2", + "rand 0.9.5", "strum 0.26.3", "tokio", "tracing", @@ -3306,7 +3314,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.2", + "rand 0.9.5", "rand_distr 0.5.1", "rangemap", "rayon", @@ -3352,7 +3360,7 @@ dependencies = [ "path_abs", "pin-project", "prost", - "rand 0.9.2", + "rand 0.9.5", "serde", "tempfile", "tokio", @@ -3374,7 +3382,7 @@ dependencies = [ "lance-arrow 8.0.0", "lance-core 8.0.0", "num-traits", - "rand 0.9.2", + "rand 0.9.5", ] [[package]] @@ -3428,7 +3436,7 @@ dependencies = [ "lance-table", "log", "object_store", - "rand 0.9.2", + "rand 0.9.5", "roaring", "serde_json", "time", @@ -3508,7 +3516,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.2", + "rand 0.9.5", "rangemap", "roaring", "semver", @@ -3533,7 +3541,7 @@ dependencies = [ "lance-arrow 8.0.0", "num-traits", "pprof", - "rand 0.9.2", + "rand 0.9.5", ] [[package]] @@ -3603,7 +3611,7 @@ dependencies = [ "pin-project", "polars", "polars-arrow", - "rand 0.9.2", + "rand 0.9.5", "regex", "semver", "serde", @@ -4366,7 +4374,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared 0.11.3", - "rand 0.8.5", + "rand 0.8.8", ] [[package]] @@ -4584,7 +4592,7 @@ dependencies = [ "polars-error", "polars-row", "polars-utils", - "rand 0.8.5", + "rand 0.8.8", "rand_distr 0.4.3", "rayon", "regex", @@ -4783,7 +4791,7 @@ dependencies = [ "polars-error", "polars-lazy", "polars-plan", - "rand 0.8.5", + "rand 0.8.8", "serde", "serde_json", "sqlparser 0.39.0", @@ -5096,9 +5104,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -5107,9 +5115,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", @@ -5177,7 +5185,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.8", ] [[package]] @@ -5187,7 +5195,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.2", + "rand 0.9.5", ] [[package]] @@ -5533,9 +5541,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.35" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "once_cell", "ring", @@ -5569,9 +5577,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", @@ -5747,15 +5755,17 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.16.1" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" +checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" dependencies = [ "base64 0.22.1", + "bs58", "chrono", "hex", "indexmap 1.9.3", "indexmap 2.14.0", + "jiff", "schemars 0.9.0", "schemars 1.1.0", "serde_core", @@ -5766,11 +5776,11 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.16.1" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" +checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "proc-macro2", "quote", "syn 2.0.117", @@ -6544,7 +6554,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" dependencies = [ - "rand 0.9.2", + "rand 0.9.5", ] [[package]] @@ -6917,21 +6927,8 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link 0.1.3", - "windows-result 0.3.4", - "windows-strings 0.4.2", -] - -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", + "windows-result", + "windows-strings", ] [[package]] @@ -6998,15 +6995,6 @@ dependencies = [ "windows-link 0.1.3", ] -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link 0.2.1", -] - [[package]] name = "windows-strings" version = "0.4.2" @@ -7016,15 +7004,6 @@ dependencies = [ "windows-link 0.1.3", ] -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link 0.2.1", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -7310,9 +7289,9 @@ dependencies = [ [[package]] name = "xxhash-rust" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7" [[package]] name = "yoke" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index d148d19f5..6743ed1a1 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -68,7 +68,8 @@ async-openai = { version = "0.20.0", optional = true } serde_with = { version = "3.8.1" } tempfile = { workspace = true } aws-smithy-types = { workspace = true, optional = true } -aws-sdk-bedrockruntime = { version = "1.27.0", optional = true } +# Default features include the legacy `rustls` feature, which pulls in rustls 0.21. +aws-sdk-bedrockruntime = { version = "1.77.0", optional = true, default-features = false, features = ["default-https-client", "rt-tokio"] } # For remote feature reqwest = { version = "0.13.0", default-features = false, features = [ "charset", @@ -116,9 +117,10 @@ random_word = { version = "0.4.3", features = ["en"] } tokio = { workspace = true, features = ["io-util", "macros", "net", "test-util"] } uuid = { workspace = true } walkdir = "2" -aws-sdk-dynamodb = { version = "1.55.0" } -aws-sdk-s3 = { version = "1.55.0" } -aws-sdk-kms = { version = "1.48.0" } +# Default features include the legacy `rustls` feature, which pulls in rustls 0.21. +aws-sdk-dynamodb = { version = "1.68.0", default-features = false, features = ["default-https-client", "rt-tokio"] } +aws-sdk-s3 = { version = "1.79.0", default-features = false, features = ["default-https-client", "rt-tokio", "sigv4a", "http-1x"] } +aws-sdk-kms = { version = "1.63.0", default-features = false, features = ["default-https-client", "rt-tokio"] } aws-config = { version = "1.5.10" } aws-smithy-runtime = { version = "1.9.1" } aws-smithy-types.workspace = true From c2c8269fc83b5f1b09ed26d04408bfe980f45b4f Mon Sep 17 00:00:00 2001 From: Will Jones Date: Fri, 9 Oct 2026 19:55:19 -0700 Subject: [PATCH 11/19] fix(java): upgrade Jackson to 2.18.11 to address CVEs (#4500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The Java SDK pinned `jackson-databind` 2.17.1, and `jackson-core` and the other Jackson modules came in transitively via `arrow-vector` and `lance-namespace-apache-client` at 2.16.x–2.17.x, all affected by known CVEs. This imports `jackson-bom` 2.18.11 in the parent POM, and declares `jackson-core`, `jackson-annotations`, `jackson-datatype-jsr310`, and `jackson-jaxrs-json-provider` as direct (BOM-managed) dependencies of `lancedb-core`. The direct declarations are needed because the parent's dependency management doesn't apply to a consumer's transitive resolution; without them, Maven's nearest-wins mediation still hands downstream apps the older versions from `arrow-vector`. ## Evidence `mvn dependency:list` in a separate consumer project depending only on `lancedb-core`: | Artifact | Before | After | |---|---|---| | jackson-core | 2.16.0 | 2.18.11 | | jackson-annotations | 2.16.0 | 2.18.11 | | jackson-databind | 2.17.1 | 2.18.11 | | jackson-datatype-jsr310 | 2.16.0 | 2.18.11 | | jackson-jaxrs-base / json-provider | 2.17.1 | 2.18.11 | | jackson-module-jaxb-annotations | 2.17.1 | 2.18.11 | ## Merge Risk **Door:** Two-way. Dependency versions only, and they stay within Jackson 2.x. **Blast Radius:** `java` dependencies. Downstream users of `lancedb-core` now get Jackson 2.18.11 transitively. Fixes OSS-2360, OSS-2337 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- java/lancedb-core/pom.xml | 25 ++++++++++++++++++++++++- java/pom.xml | 10 ++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index df20bc7f0..9affef0ef 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -44,7 +44,30 @@ com.fasterxml.jackson.core jackson-databind - 2.17.1 + + + + + com.fasterxml.jackson.core + jackson-core + + + + com.fasterxml.jackson.core + jackson-annotations + + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider diff --git a/java/pom.xml b/java/pom.xml index e6308140d..be828e21a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -29,6 +29,7 @@ UTF-8 15.0.0 14.0.0-beta.10 + 2.18.11 false 2.30.0 1.7 @@ -62,6 +63,15 @@ + + + com.fasterxml.jackson + jackson-bom + ${jackson.version} + pom + import + org.lance lance-core From f0bfb89d9cd3ae47248f90345573b0a390c2f678 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Sat, 10 Oct 2026 23:33:56 -0700 Subject: [PATCH 12/19] chore: update lance dependency to v14.0.0-beta.11 (#4518) Updates the Rust workspace Lance dependencies, Cargo lockfile, and Java lance-core to [v14.0.0-beta.11](https://github.com/lance-format/lance/releases/tag/v14.0.0-beta.11). Adapts the LSM resident-index guards to Lance's query-based index API and adds regression coverage for index presence and type checks. Validation: workspace Clippy with all features and warnings denied, Rust formatting, and targeted LSM query unit tests. --- Cargo.lock | 92 ++++++++++++++--------------- Cargo.toml | 28 ++++----- java/pom.xml | 2 +- rust/lancedb/src/table/query/lsm.rs | 60 ++++++++++++++++++- 4 files changed, 119 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 12aab95c3..f2f4984d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arc-swap", "arrow", @@ -5133,8 +5133,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-buffer", @@ -5156,7 +5156,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-buffer", @@ -5170,7 +5170,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-schema", @@ -5179,8 +5179,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrayref", "crunchy", @@ -5190,8 +5190,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-buffer", @@ -5228,8 +5228,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow", "arrow-array", @@ -5260,8 +5260,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow", "arrow-array", @@ -5278,8 +5278,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "proc-macro2", "quote", @@ -5288,8 +5288,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-arith", "arrow-array", @@ -5322,8 +5322,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-arith", "arrow-array", @@ -5355,8 +5355,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "datafusion", "geo-traits", @@ -5370,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arc-swap", "arrow", @@ -5442,8 +5442,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-schema", @@ -5466,8 +5466,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow", "arrow-array", @@ -5509,8 +5509,8 @@ dependencies = [ [[package]] name = "lance-jsonb" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "byteorder", "ethnum", @@ -5527,8 +5527,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-schema", @@ -5542,8 +5542,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow", "async-trait", @@ -5557,8 +5557,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arc-swap", "arrow", @@ -5612,8 +5612,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-buffer", @@ -5627,8 +5627,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow", "arrow-array", @@ -5668,8 +5668,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-schema", @@ -5682,8 +5682,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 8c9f5a91e..49d1792de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index be828e21a..40da4e36a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 14.0.0-beta.10 + 14.0.0-beta.11 2.18.11 false 2.30.0 diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 5ec053483..698b9c736 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -33,6 +33,7 @@ use datafusion_physical_plan::expressions::Column; use datafusion_physical_plan::projection::ProjectionExec; use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr}; use lance::Dataset; +use lance::dataset::mem_wal::index::{IndexStore, VectorMemQuery}; use lance::dataset::mem_wal::scanner::{InMemoryMemTableRef, InMemoryMemTables}; use lance::dataset::mem_wal::{ DatasetMemWalExt, LsmScanner, ShardManifestStore, ShardSnapshot, ShardWriterConfig, @@ -474,7 +475,7 @@ async fn fts_plan( // omits un-compacted documents. Reject rather than mislead. if !index_maintained(dataset, column, details, "InvertedIndexDetails").await? || !resident_memtables_carry(&in_memory, |memtable| { - memtable.index_store.get_fts_by_column(column).is_some() + !memtable.index_store.fts_granularities_on(column).is_empty() }) { return Err(Error::NotSupported { @@ -712,7 +713,7 @@ async fn vector_plan( // Reject rather than silently omit rows, mirroring the FTS arm. if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? || !resident_memtables_carry(&in_memory, |memtable| { - memtable.index_store.get_hnsw_by_column(&column).is_some() + carries_vector_index(&memtable.index_store, &column) }) { return Err(Error::NotSupported { @@ -760,6 +761,22 @@ async fn vector_plan( Ok(scanner.create_plan().await?) } +/// Probe for a resident vector index without imposing a distance metric, matching +/// the presence check used before Lance introduced query-based index lookup. +fn carries_vector_index(store: &IndexStore, column: &str) -> bool { + let probe = VectorMemQuery { + vector: arrow_array::FixedSizeListArray::new_null( + Arc::new(arrow_schema::Field::new("item", DataType::Float32, true)), + 1, + 1, + ), + k: 1, + ef: None, + distance_type: None, + }; + store.index_answering(column, &probe).is_some() +} + /// Whether `column` stores binary (uint8) vectors, which the LSM vector planner /// does not support. fn is_binary_vector_column(schema: &ArrowSchema, column: &str) -> bool { @@ -804,6 +821,45 @@ mod tests { use super::*; use lance_index::mem_wal::{CompactedSsTable, IndexCatchupProgress}; + #[test] + fn resident_index_checks_distinguish_columns_and_query_types() { + use arrow_schema::Field; + use lance::dataset::mem_wal::index::MemIndexSpec; + + let schema = ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("text", DataType::Utf8, true), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), + true, + ), + ]); + let schema = lance_core::datatypes::Schema::try_from(&schema).unwrap(); + let specs = vec![ + MemIndexSpec::btree("id_idx", 0, "id"), + MemIndexSpec::fts("text_idx", 1, "text"), + MemIndexSpec::hnsw( + "vector_idx", + 2, + "vector", + lance_linalg::distance::DistanceType::Cosine, + ), + ]; + let store = IndexStore::from_specs(&specs, &schema, 1_000, 16).unwrap(); + assert!(carries_vector_index(&store, "vector")); + for column in ["id", "text", "missing"] { + assert!(!carries_vector_index(&store, column)); + } + assert!(!store.fts_granularities_on("text").is_empty()); + for column in ["id", "vector", "missing"] { + assert!(store.fts_granularities_on(column).is_empty()); + } + let empty = IndexStore::from_specs(&[], &schema, 1_000, 16).unwrap(); + assert!(!carries_vector_index(&empty, "vector")); + assert!(empty.fts_granularities_on("text").is_empty()); + } + #[test] fn exclusion_watermark_gates_on_lagging_index_catchup() { let shard = Uuid::from_u128(1); From 142d032cf1674db4748f0095771041c1c5b1e428 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Sun, 11 Oct 2026 06:34:40 +0000 Subject: [PATCH 13/19] =?UTF-8?q?Bump=20version:=200.41.0-beta.3=20?= =?UTF-8?q?=E2=86=92=200.41.0-beta.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index e74604e0c..487948f7d 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.41.0-beta.3" +current_version = "0.41.0-beta.4" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index f2f4984d3..45b286d85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5696,7 +5696,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.41.0-beta.3" +version = "0.41.0-beta.4" dependencies = [ "ahash", "anyhow", @@ -5793,7 +5793,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.41.0-beta.3" +version = "0.41.0-beta.4" dependencies = [ "arrow-array", "arrow-buffer", @@ -5818,7 +5818,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.41.0-beta.3" +version = "0.41.0-beta.4" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 81b4afa42..93b051ffb 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.41.0-beta.3 + 0.41.0-beta.4 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 9affef0ef..89672e7c5 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.3 + 0.41.0-beta.4 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 40da4e36a..9fa79452e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.3 + 0.41.0-beta.4 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 0fd2434c1..4b3431ab1 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.41.0-beta.3" +version = "0.41.0-beta.4" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 486d9692a..b10fe483a 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.41.0-beta.3", + "version": "0.41.0-beta.4", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index aeeee36c1..5eab9429f 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.41.0-beta.3", + "version": "0.41.0-beta.4", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index f6cc198c4..9a6443e24 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.41.0-beta.3", + "version": "0.41.0-beta.4", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 37c83e4ee..d1870228e 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.41.0-beta.3", + "version": "0.41.0-beta.4", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index c25486f7c..3ee0ad870 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.41.0-beta.3", + "version": "0.41.0-beta.4", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index c963f9422..6789b433c 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.41.0-beta.3", + "version": "0.41.0-beta.4", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 7220ca9f1..94859f899 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.41.0-beta.3", + "version": "0.41.0-beta.4", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index acecfc075..ff43a9f47 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.41.0-beta.3", + "version": "0.41.0-beta.4", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 4bd3cf357..456103ade 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.41.0-beta.3" +version = "0.41.0-beta.4" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 6743ed1a1..768d55c4e 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.41.0-beta.3" +version = "0.41.0-beta.4" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From e8b4d95938e6e22f573e066d96d0975bafee6a20 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Sun, 11 Oct 2026 23:02:42 -0700 Subject: [PATCH 14/19] chore: update lance dependency to v14.0.0-beta.13 (#4523) Updates Rust workspace Lance dependencies and Java lance-core from v14.0.0-beta.11 to [v14.0.0-beta.13](https://github.com/lance-format/lance/releases/tag/v14.0.0-beta.13), with the Cargo lockfile refreshed. No compatibility fixes were required. Validation: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and `git diff --check` passed. --- Cargo.lock | 92 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 28 ++++++++-------- java/pom.xml | 2 +- 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 45b286d85..7a1d75332 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arc-swap", "arrow", @@ -5133,8 +5133,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow-array", "arrow-buffer", @@ -5156,7 +5156,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow-array", "arrow-buffer", @@ -5170,7 +5170,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow-array", "arrow-schema", @@ -5179,8 +5179,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrayref", "crunchy", @@ -5190,8 +5190,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow-array", "arrow-buffer", @@ -5228,8 +5228,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow", "arrow-array", @@ -5260,8 +5260,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow", "arrow-array", @@ -5278,8 +5278,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "proc-macro2", "quote", @@ -5288,8 +5288,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow-arith", "arrow-array", @@ -5322,8 +5322,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow-arith", "arrow-array", @@ -5355,8 +5355,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "datafusion", "geo-traits", @@ -5370,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arc-swap", "arrow", @@ -5442,8 +5442,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow-array", "arrow-schema", @@ -5466,8 +5466,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow", "arrow-array", @@ -5509,8 +5509,8 @@ dependencies = [ [[package]] name = "lance-jsonb" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "byteorder", "ethnum", @@ -5527,8 +5527,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow-array", "arrow-schema", @@ -5542,8 +5542,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow", "async-trait", @@ -5557,8 +5557,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arc-swap", "arrow", @@ -5612,8 +5612,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow-array", "arrow-buffer", @@ -5627,8 +5627,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow", "arrow-array", @@ -5668,8 +5668,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "arrow-array", "arrow-schema", @@ -5682,8 +5682,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" +version = "14.0.0-beta.13" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 49d1792de..d5821d993 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 9fa79452e..8d8a3f535 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 14.0.0-beta.11 + 14.0.0-beta.13 2.18.11 false 2.30.0 From f32e9ab26a98fd4e4496b4b063c6c8572259f218 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Mon, 12 Oct 2026 06:03:27 +0000 Subject: [PATCH 15/19] =?UTF-8?q?Bump=20version:=200.41.0-beta.4=20?= =?UTF-8?q?=E2=86=92=200.41.0-beta.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 487948f7d..cb1b6ebd7 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.41.0-beta.4" +current_version = "0.41.0-beta.5" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 7a1d75332..dc2e67b08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5696,7 +5696,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.41.0-beta.4" +version = "0.41.0-beta.5" dependencies = [ "ahash", "anyhow", @@ -5793,7 +5793,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.41.0-beta.4" +version = "0.41.0-beta.5" dependencies = [ "arrow-array", "arrow-buffer", @@ -5818,7 +5818,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.41.0-beta.4" +version = "0.41.0-beta.5" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 93b051ffb..1fb657175 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.41.0-beta.4 + 0.41.0-beta.5 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 89672e7c5..ba0d1c8f2 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.4 + 0.41.0-beta.5 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 8d8a3f535..7949e9c79 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.4 + 0.41.0-beta.5 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 4b3431ab1..2f38dd299 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.41.0-beta.4" +version = "0.41.0-beta.5" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index b10fe483a..6b8fcf993 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.41.0-beta.4", + "version": "0.41.0-beta.5", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 5eab9429f..cbb0462e7 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.41.0-beta.4", + "version": "0.41.0-beta.5", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 9a6443e24..e752f03a8 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.41.0-beta.4", + "version": "0.41.0-beta.5", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index d1870228e..43d9d2fc3 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.41.0-beta.4", + "version": "0.41.0-beta.5", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 3ee0ad870..109e469a5 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.41.0-beta.4", + "version": "0.41.0-beta.5", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 6789b433c..5c1723bed 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.41.0-beta.4", + "version": "0.41.0-beta.5", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 94859f899..928dfc5bb 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.41.0-beta.4", + "version": "0.41.0-beta.5", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index ff43a9f47..121896f41 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.41.0-beta.4", + "version": "0.41.0-beta.5", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 456103ade..eeb39a672 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.41.0-beta.4" +version = "0.41.0-beta.5" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 768d55c4e..419577d05 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.41.0-beta.4" +version = "0.41.0-beta.5" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 0bceaaf3f07ac3c533164c104d34804e8531dae1 Mon Sep 17 00:00:00 2001 From: Yang Cen Date: Mon, 12 Oct 2026 19:32:38 +0800 Subject: [PATCH 16/19] feat(search): expose search effort (#4520) Lance supports `search_effort`, but LanceDB cannot pass it through its query APIs. Expose the parameter in Rust, Python sync/async and TypeScript, including remote requests and hybrid query conversion, using the [Lance v14.0.0-beta.13](https://github.com/lance-format/lance/releases/tag/v14.0.0-beta.13) dependency already on main. The default is `None` (Auto, internally 0.5). Any explicit effort, including 0.5, overrides fixed `nprobes` in either setter order; explicit minimum/maximum bounds remain effective. Invalid values are rejected in the Rust core. Hybrid conversion now retains the vector probe parameters and handles an inferred FTS column without panicking. Namespace queries with explicit effort execute locally because the namespace wire model cannot carry it. MemWAL LSM queries reject it explicitly because Lance's LSM scanner does not yet expose this option. Maturin is included in the dev extra so the documented build command works in the project environment. BREAKING CHANGE: The public Rust `VectorQueryRequest` struct now includes `search_effort: Option`. Downstream code that initializes every field explicitly must add `search_effort: None` or use `..Default::default()`. Exhaustive destructuring patterns must include the new field or `..`. Query-builder calls and struct literals that already use `..Default::default()` require no source changes. The new effort setters in Python and TypeScript are additive. This PR does not change the persisted file format. Using explicit effort with remote tables requires server support for `search_effort` and its precedence over `nprobes`; the client wire tests do not establish that a deployed server supports it. Server-side support is tracked in lancedb/sophon#8607. Validation on AWS after merging current main (Lance beta.13 / SDK beta.5): - 14 Rust regressions and the API doctest pass. - 60 parameter regressions, the full 47-test namespace suite, and three API doctests pass. The parameter selection includes some namespace cases. - 40 TypeScript query tests and 54 remote tests pass; build, lint and generated docs pass. - Rust Clippy, Python package lint/format, Pyright and lockfile checks pass. Repository-wide Ruff reports 49 lint errors and 11 formatting failures in unchanged `docs/web-tests` files; those files are byte-identical to main and outside this change. All 1,103 Git-tracked files match the AWS validation checkout. No benchmark scripts or result artifacts are included, and no performance improvement is claimed. Uses lance-format/lance#9798. Server-side support is in lancedb/sophon#8607. --- docs/src/js/classes/VectorQuery.md | 32 +++++++ nodejs/__test__/query.test.ts | 103 +++++++++++++++++++++ nodejs/lancedb/query.ts | 21 +++++ nodejs/src/query.rs | 10 +++ python/pyproject.toml | 3 +- python/python/lancedb/_lancedb.pyi | 3 + python/python/lancedb/namespace.py | 5 ++ python/python/lancedb/query.py | 88 +++++++++++++++++- python/python/lancedb/table.py | 4 + python/python/tests/test_namespace.py | 12 +++ python/python/tests/test_query.py | 124 ++++++++++++++++++++++++++ python/src/query.rs | 34 ++++++- python/uv.lock | 42 +++++++-- rust/lancedb/src/query.rs | 67 +++++++++++++- rust/lancedb/src/remote/table.rs | 75 ++++++++++++++++ rust/lancedb/src/table/query.rs | 30 ++++++- rust/lancedb/src/table/query/lsm.rs | 16 ++++ 17 files changed, 650 insertions(+), 19 deletions(-) diff --git a/docs/src/js/classes/VectorQuery.md b/docs/src/js/classes/VectorQuery.md index 3f56bb600..4809ebefd 100644 --- a/docs/src/js/classes/VectorQuery.md +++ b/docs/src/js/classes/VectorQuery.md @@ -646,6 +646,38 @@ rerank(reranker): VectorQuery *** +### searchEffort() + +```ts +searchEffort(searchEffort): VectorQuery +``` + +Set the initial IVF search effort to a finite value in [0, 1]. + +The default, null, uses Auto (internally 0.5). Zero starts at the caller +minimum; one searches all partitions within the caller maximum. Later +probing may continue when too few candidates are found. +Any explicit effort, including 0.5, overrides nprobes in either call order. +Explicit minimum and maximum bounds still apply. This is not a recall +guarantee and does not change refinement or HNSW ef. The MemWAL LSM +scanner does not yet support this option. + +#### Parameters + +* **searchEffort**: `null` \| `number` + +#### Returns + +[`VectorQuery`](VectorQuery.md) + +#### Example + +```ts +table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10); +``` + +*** + ### select() ```ts diff --git a/nodejs/__test__/query.test.ts b/nodejs/__test__/query.test.ts index 16a595002..786f04015 100644 --- a/nodejs/__test__/query.test.ts +++ b/nodejs/__test__/query.test.ts @@ -527,3 +527,106 @@ describe("Query orderBy", () => { ]); }); }); + +describe("IVF search effort", () => { + let directory: tmp.DirResult; + let table: Table; + const vector = Array.from({ length: 16 }, (_, i) => (i === 0 ? 1 : 0)); + + beforeEach(async () => { + directory = tmp.dirSync({ unsafeCleanup: true }); + const db = await connect(directory.name); + const rows = Array.from({ length: 512 }, (_, id) => ({ + id, + vector: Array.from( + { length: 16 }, + (_, j) => + (Math.floor(id / 32) === j ? 1 : 0) + Math.sin(id * 17 + j) * 0.001, + ), + })); + table = await db.createTable("effort", rows.slice(0, 256)); + await table.add(rows.slice(256)); + await table.createIndex("vector", { + config: Index.ivfFlat({ numPartitions: 16 }), + }); + }); + + afterEach(() => { + directory.removeCallback(); + }); + + it.each([ + [null, 6], + [0, 1], + [0.25, 3], + [0.5, 6], + [0.75, 10], + [1, 16], + ])( + "preserves effort %s through execution and probe precedence", + async (effort, expected) => { + const partitions = async (query: ReturnType) => { + const plan = await query.limit(8).analyzePlan(); + const matches = [...plan.matchAll(/partitions_searched=(\d+)/g)]; + expect(matches.length).toBeGreaterThan(0); + return matches.reduce((total, match) => total + Number(match[1]), 0); + }; + expect( + await partitions(table.vectorSearch(vector).searchEffort(effort)), + ).toBe(expected); + expect( + await partitions( + table.vectorSearch(vector).nprobes(1).searchEffort(effort), + ), + ).toBe(effort === null ? 1 : expected); + expect( + await partitions( + table.vectorSearch(vector).searchEffort(effort).nprobes(1), + ), + ).toBe(effort === null ? 1 : expected); + expect( + await partitions( + table + .vectorSearch(vector) + .searchEffort(effort) + .nprobes(1) + .searchEffort(null), + ), + ).toBe(1); + expect( + await partitions( + table.vectorSearch(vector).searchEffort(effort).maximumNprobes(2), + ), + ).toBeLessThanOrEqual(2); + const actual = await table + .vectorSearch(vector) + .searchEffort(effort) + .limit(8) + .toArray(); + const exact = await table + .vectorSearch(vector) + .bypassVectorIndex() + .limit(8) + .toArray(); + const ids = new Set(exact.map((row) => row.id)); + expect( + actual.filter((row) => ids.has(row.id)).length / 8, + ).toBeGreaterThanOrEqual(0.5); + }, + ); + + it.each([ + -0.1, + 1.1, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + ])("rejects invalid effort %s before execution", (effort) => { + expect(() => table.vectorSearch(vector).searchEffort(effort)).toThrow( + "search_effort must be finite and in [0, 1]", + ); + expect(() => + table.vectorSearch(vector).fullTextSearch("dog").searchEffort(effort), + ).toThrow("search_effort must be finite and in [0, 1]"); + }); +}); diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index 0f0eaf610..cce505c97 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -549,6 +549,27 @@ export class VectorQuery extends StandardQueryBase { return this; } + /** + * Set the initial IVF search effort to a finite value in [0, 1]. + * + * The default, null, uses Auto (internally 0.5). Zero starts at the caller + * minimum; one searches all partitions within the caller maximum. Later + * probing may continue when too few candidates are found. + * Any explicit effort, including 0.5, overrides nprobes in either call order. + * Explicit minimum and maximum bounds still apply. This is not a recall + * guarantee and does not change refinement or HNSW ef. The MemWAL LSM + * scanner does not yet support this option. + * + * @example + * ```ts + * table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10); + * ``` + */ + searchEffort(searchEffort: number | null): VectorQuery { + this.doVectorCall((inner) => inner.searchEffort(searchEffort)); + return this; + } + /** * Set the minimum number of probes used. * diff --git a/nodejs/src/query.rs b/nodejs/src/query.rs index 712569b2f..480426411 100644 --- a/nodejs/src/query.rs +++ b/nodejs/src/query.rs @@ -298,6 +298,16 @@ impl VectorQuery { Ok(()) } + #[napi] + pub fn search_effort(&mut self, search_effort: Option) -> napi::Result<()> { + self.inner = self + .inner + .clone() + .search_effort(search_effort) + .default_error()?; + Ok(()) + } + #[napi] pub fn minimum_nprobes(&mut self, minimum_nprobe: u32) -> napi::Result<()> { self.inner = self diff --git a/python/pyproject.toml b/python/pyproject.toml index cde7247f3..ed54ae536 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -63,7 +63,7 @@ tests = [ "polars>=0.19, <=1.32.3", "pyarrow<25", "pyarrow-stubs>=16.0", - "pylance==14.0.0b10", + "pylance==14.0.0b12", "requests>=2.31.0", "datafusion>=54,<55", "opentelemetry-sdk>=1.30.0", @@ -73,6 +73,7 @@ dev = [ "pre-commit>=3.5.0", "pyright>=1.1.350", 'typing-extensions>=4.0.0; python_version < "3.11"', + "maturin>=1.10", ] docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings-python"] clip = ["torch", "pillow>=12.1.1", "open-clip-torch"] diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index cfaa8c68a..8d0a86e4a 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -727,6 +727,7 @@ class VectorQuery: def use_lsm(self, enable: bool): ... def refine_factor(self, refine_factor: int): ... def nprobes(self, nprobes: int): ... + def search_effort(self, search_effort: Optional[float]): ... def minimum_nprobes(self, minimum_nprobes: int): ... def maximum_nprobes(self, maximum_nprobes: int): ... def bypass_vector_index(self): ... @@ -752,6 +753,7 @@ class HybridQuery: def distance_type(self, distance_type: str): ... def refine_factor(self, refine_factor: int): ... def nprobes(self, nprobes: int): ... + def search_effort(self, search_effort: Optional[float]): ... def minimum_nprobes(self, minimum_nprobes: int): ... def maximum_nprobes(self, maximum_nprobes: int): ... def bypass_vector_index(self): ... @@ -781,6 +783,7 @@ class PyQueryRequest: nprobes: Optional[int] minimum_nprobes: Optional[int] maximum_nprobes: Optional[int] + search_effort: Optional[float] lower_bound: Optional[float] upper_bound: Optional[float] ef: Optional[int] diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index ed87e441f..26c62925f 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -105,6 +105,11 @@ def _query_to_namespace_request( if not isinstance(query, Query): raise TypeError(f"Expected Query, got {type(query)}") + if query.search_effort is not None: + raise ValueError( + "namespace QueryTable does not support search_effort; " + "execute the query locally" + ) vector_request = None if query.vector is not None: diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index c7664ab78..eece390c2 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -475,8 +475,8 @@ class MatchQuery(FullTextQuery): ---------- query : str The query string to match against. - column : str - The name of the column to match against. + column : str, optional + The column to match against. None leaves column selection to query planning. boost : float, default 1.0 The boost factor for the query. The score of each matching document is multiplied by this value. @@ -505,7 +505,7 @@ class MatchQuery(FullTextQuery): """ query: str - column: str + column: Optional[str] = None boost: float = pydantic.Field(1.0, kw_only=True) fuzziness: Optional[int] = pydantic.Field(0, kw_only=True) max_expansions: int = pydantic.Field(50, kw_only=True) @@ -793,6 +793,9 @@ class Query(pydantic.BaseModel): # and all partitions could be searched if needed to satisfy the limit. maximum_nprobes: Optional[int] = None + # Optional initial IVF effort; None preserves Auto and explicit nprobes. + search_effort: Optional[float] = None + # lower bound for distance search lower_bound: Optional[float] = None @@ -854,6 +857,7 @@ class Query(pydantic.BaseModel): query.nprobes = req.nprobes query.minimum_nprobes = req.minimum_nprobes query.maximum_nprobes = req.maximum_nprobes + query.search_effort = req.search_effort query.lower_bound = req.lower_bound query.upper_bound = req.upper_bound query.ef = req.ef @@ -1635,6 +1639,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): self._nprobes = None self._minimum_nprobes = None self._maximum_nprobes = None + self._search_effort = None self._lower_bound = None self._upper_bound = None self._refine_factor = None @@ -1716,6 +1721,27 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): self._nprobes = nprobes return self + def search_effort(self, search_effort: Optional[float]) -> LanceVectorQueryBuilder: + """Set the initial IVF search effort to a finite value in [0, 1]. + + None (the default) uses Auto, internally 0.5. Zero starts at the caller + minimum; one searches all partitions within the caller maximum. Later + probing may continue when too few candidates are found. Any explicit + effort, including 0.5, overrides nprobes in either call order. Explicit + minimum and maximum bounds still apply. This is not a recall guarantee + and does not change refinement or HNSW ef. MemWAL LSM queries do not + yet support this option. + + Examples + -------- + >>> import lancedb + >>> db = lancedb.connect("memory://") + >>> table = db.create_table("effort_vectors", [{"vector": [0.1, 0.2]}]) + >>> query = table.search([0.1, 0.2]).search_effort(0.75) + """ + self._search_effort = search_effort + return self + def minimum_nprobes(self, minimum_nprobes: int) -> LanceVectorQueryBuilder: """Set the minimum number of probes to use. @@ -1858,6 +1884,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): nprobes=self._nprobes, minimum_nprobes=self._minimum_nprobes, maximum_nprobes=self._maximum_nprobes, + search_effort=self._search_effort, lower_bound=self._lower_bound, upper_bound=self._upper_bound, refine_factor=self._refine_factor, @@ -2217,6 +2244,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): self._nprobes = None self._minimum_nprobes = None self._maximum_nprobes = None + self._search_effort = None self._refine_factor = None self._distance_type = None self._phrase_query = None @@ -2524,6 +2552,32 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): self._nprobes = nprobes return self + def search_effort(self, search_effort: Optional[float]) -> LanceHybridQueryBuilder: + """Set the initial IVF search effort to a finite value in [0, 1]. + + None (the default) uses Auto, internally 0.5. Zero starts at the caller + minimum; one searches all partitions within the caller maximum. Later + probing may continue when too few candidates are found. Any explicit + effort, including 0.5, overrides nprobes in either call order. Explicit + minimum and maximum bounds still apply. This is not a recall guarantee + and does not change refinement or HNSW ef. MemWAL LSM queries do not + yet support this option. + + Examples + -------- + >>> import lancedb + >>> db = lancedb.connect("memory://") + >>> table = db.create_table( + ... "effort_hybrid", [{"vector": [0.1, 0.2], "text": "dog"}] + ... ) + >>> query = ( + ... table.search(query_type="hybrid").vector([0.1, 0.2]) + ... .text("dog").search_effort(0.75) + ... ) + """ + self._search_effort = search_effort + return self + def minimum_nprobes(self, minimum_nprobes: int) -> LanceHybridQueryBuilder: """Set the minimum number of probes to use. @@ -2802,6 +2856,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): self._vector_query.metric(self._distance_type) if self._nprobes is not None: self._vector_query.nprobes(self._nprobes) + if self._search_effort is not None: + self._vector_query.search_effort(self._search_effort) if self._minimum_nprobes is not None: self._vector_query.minimum_nprobes(self._minimum_nprobes) if self._maximum_nprobes is not None: @@ -3730,6 +3786,32 @@ class AsyncVectorQueryBase: self._inner.nprobes(nprobes) return self + def search_effort(self, search_effort: Optional[float]) -> Self: + """Set the initial IVF search effort to a finite value in [0, 1]. + + None preserves Auto (internally 0.5) and fixed nprobes. Any explicit + effort, including 0.5, overrides nprobes in either call order; explicit + minimum and maximum bounds remain effective. Zero starts at the caller + minimum and one searches all partitions within the maximum. Later + probing may continue to obtain enough candidates. This is not a recall + guarantee and does not change refinement or HNSW ef. MemWAL LSM queries + do not yet support this option. + + Examples + -------- + >>> import asyncio + >>> import lancedb + >>> async def build_query(): + ... db = await lancedb.connect_async("memory://") + ... table = await db.create_table( + ... "effort_async", [{"vector": [0.1, 0.2]}] + ... ) + ... return table.vector_search([0.1, 0.2]).search_effort(0.75) + >>> query = asyncio.run(build_query()) + """ + self._inner.search_effort(search_effort) + return self + def minimum_nprobes(self, minimum_nprobes: int) -> Self: """Set the minimum number of probes to use. diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 21838ecef..81b376ebd 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -4440,6 +4440,7 @@ class LanceTable(Table): and not self._route_pushdown_to_rust and self.current_branch() is None and query.take_offsets is None + and query.search_effort is None ): from lancedb.namespace import _execute_server_side_query @@ -6441,6 +6442,8 @@ class AsyncTable: async_query = async_query.distance_type(query.distance_type) if query.nprobes is not None: async_query = async_query.nprobes(query.nprobes) + if query.search_effort is not None: + async_query = async_query.search_effort(query.search_effort) if query.minimum_nprobes is not None and query.maximum_nprobes is not None: async_query = async_query.minimum_nprobes( query.minimum_nprobes @@ -6481,6 +6484,7 @@ class AsyncTable: ) and not self._route_pushdown_to_rust and query.take_offsets is None + and query.search_effort is None ): from lancedb.namespace import _execute_server_side_query diff --git a/python/python/tests/test_namespace.py b/python/python/tests/test_namespace.py index 76aa5fc55..520a6992d 100644 --- a/python/python/tests/test_namespace.py +++ b/python/python/tests/test_namespace.py @@ -1209,3 +1209,15 @@ def test_local_table_to_arrow_and_to_pandas_are_unchanged(tmp_path): assert table.to_arrow().column("id").to_pylist() == [1, 2] assert table.to_pandas()["id"].tolist() == [1, 2] + + +@pytest.mark.parametrize("effort", [0.0, 0.5, 1.0]) +def test_namespace_request_rejects_search_effort(effort): + from lancedb.query import Query + + with pytest.raises( + ValueError, match="namespace QueryTable does not support search_effort" + ): + _query_to_namespace_request( + ["test"], Query(vector=[1.0, 0.0], search_effort=effort) + ) diff --git a/python/python/tests/test_query.py b/python/python/tests/test_query.py index daee0dd07..74a67b909 100644 --- a/python/python/tests/test_query.py +++ b/python/python/tests/test_query.py @@ -2265,3 +2265,127 @@ def test_blob_v2_to_batches_row_id(tmp_db): assert "_rowid" in hits.column_names blobs = table.fetch_blobs("blob", hits) assert [blobs[i].as_py() for i in range(len(blobs))] == [b"one", b"two"] + + +@pytest.mark.parametrize("metric", ["l2", "cosine", "dot"]) +@pytest.mark.parametrize( + "effort,probes", [(None, 4), (0, 1), (0.25, 2), (0.5, 4), (0.75, 8), (1, 16)] +) +def test_search_effort_probe_budget(tmp_path, monkeypatch, metric, effort, probes): + import re + from lancedb.index import IvfFlat + + monkeypatch.setenv("LANCE_AUTO_PROBE_MARGIN", "0") + monkeypatch.setenv("LANCE_AUTO_MIN_INITIAL_NPROBES", "4") + monkeypatch.setenv("LANCE_AUTO_MAX_INITIAL_NPROBES", "4") + rng = np.random.default_rng(42) + vectors = np.repeat(np.eye(16, dtype=np.float32), 32, axis=0) + vectors += rng.normal(0, 0.001, vectors.shape).astype(np.float32) + data = pa.table( + { + "id": np.arange(512), + "text": ["dog"] * 512, + "vector": pa.FixedSizeListArray.from_arrays(pa.array(vectors.ravel()), 16), + } + ) + table = lancedb.connect(tmp_path).create_table("effort", data.slice(0, 256)) + table.add(data.slice(256)) + table.create_index( + "vector", config=IvfFlat(distance_type=metric, num_partitions=16) + ) + vector = np.eye(16, dtype=np.float32)[0] + + def search(value=effort): + return table.search(vector).distance_type(metric).search_effort(value).limit(8) + + def partitions(query): + vector_plan = query.analyze_plan().split("FTS Search Plan:", 1)[0] + values = re.findall(r"partitions_searched=(\d+)", vector_plan) + assert values + return sum(map(int, values)) + + assert partitions(search()) == probes + exact = set( + table.search(vector) + .distance_type(metric) + .bypass_vector_index() + .limit(8) + .to_arrow()["id"] + .to_pylist() + ) + actual = set(search().to_arrow()["id"].to_pylist()) + assert len(actual & exact) / 8 >= 0.5 + expected = 1 if effort is None else probes + assert partitions(search().nprobes(1)) == expected + assert ( + partitions( + table.search(vector) + .distance_type(metric) + .nprobes(1) + .search_effort(effort) + .limit(8) + ) + == expected + ) + assert partitions(search().nprobes(1).search_effort(None)) == 1 + assert partitions(search().maximum_nprobes(2)) <= 2 + assert partitions(search().minimum_nprobes(3).maximum_nprobes(3)) == 3 + assert partitions(search(0).nprobes(16).maximum_nprobes(0)) == 1 + if effort is None: + assert search().to_arrow() == search(0.5).to_arrow() + filtered = search(0).maximum_nprobes(2).where("id % 8 = 0", prefilter=True) + assert partitions(filtered) <= 2 + assert all(row["id"] % 8 == 0 for row in filtered.to_list()) + + if metric == "l2" and effort == 0.5: + table.create_index("text", config=FTS()) + hybrid = ( + table.search(query_type="hybrid") + .vector(vector) + .text("dog") + .nprobes(1) + .search_effort(0.5) + .limit(8) + ) + assert partitions(hybrid) == 4 + assert partitions(hybrid.search_effort(None)) == 1 + + +@pytest.mark.parametrize( + "effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")] +) +def test_search_effort_invalid_sync(table, effort): + with pytest.raises(ValueError, match="search_effort must be finite and in"): + table.search([0, 0]).search_effort(effort).to_list() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("hybrid", [False, True]) +@pytest.mark.parametrize("effort", [None, 0, 0.25, 0.5, 0.75, 1]) +async def test_search_effort_async_roundtrip(table_async, hybrid, effort): + query = table_async.vector_search([0, 0]) + if hybrid: + query = query.nearest_to_text("dog") + query = query.nprobes(5).search_effort(effort).minimum_nprobes(2).maximum_nprobes(3) + request = query.to_query_object() + assert request.search_effort == effort + assert request.nprobes == 5 + assert request.minimum_nprobes == 2 + assert request.maximum_nprobes == 3 + assert query.search_effort(None).to_query_object().search_effort is None + query.search_effort(effort) + if not hybrid: + assert len(await query.limit(1).to_list()) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("hybrid", [False, True]) +@pytest.mark.parametrize( + "effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")] +) +async def test_search_effort_invalid_async(table_async, hybrid, effort): + query = table_async.vector_search([0, 0]) + if hybrid: + query = query.nearest_to_text("dog") + with pytest.raises(ValueError, match="search_effort must be finite and in"): + query.search_effort(effort) diff --git a/python/src/query.rs b/python/src/query.rs index 88b7243d4..e5547881a 100644 --- a/python/src/query.rs +++ b/python/src/query.rs @@ -71,7 +71,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB { match class_name(&ob)?.as_str() { "MatchQuery" => { let query = ob.getattr("query")?.extract()?; - let column = ob.getattr("column")?.extract()?; + let column = ob.getattr("column")?.extract::>()?; let boost = ob.getattr("boost")?.extract()?; let fuzziness = ob.getattr("fuzziness")?.extract()?; let max_expansions = ob.getattr("max_expansions")?.extract()?; @@ -88,7 +88,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB { let mut query = MatchQuery::new(query) - .with_column(Some(column)) + .with_column(column) .with_boost(boost) .with_fuzziness(fuzziness) .with_max_expansions(max_expansions) @@ -199,7 +199,7 @@ impl<'py> IntoPyObject<'py> for PyLanceDB { } namespace .getattr(intern!(py, "MatchQuery"))? - .call((query.terms, query.column.unwrap()), Some(&kwargs)) + .call((query.terms, query.column), Some(&kwargs)) } FtsQuery::Phrase(query) => { let kwargs = PyDict::new(py); @@ -339,6 +339,7 @@ pub struct PyQueryRequest { pub nprobes: Option, pub minimum_nprobes: Option, pub maximum_nprobes: Option, + pub search_effort: Option, pub lower_bound: Option, pub upper_bound: Option, pub ef: Option, @@ -371,6 +372,7 @@ impl From for PyQueryRequest { nprobes: None, minimum_nprobes: None, maximum_nprobes: None, + search_effort: None, lower_bound: None, upper_bound: None, ef: None, @@ -399,6 +401,7 @@ impl From for PyQueryRequest { nprobes: vector_query.nprobes, minimum_nprobes: vector_query.minimum_nprobes, maximum_nprobes: vector_query.maximum_nprobes, + search_effort: vector_query.search_effort, lower_bound: vector_query.lower_bound, upper_bound: vector_query.upper_bound, ef: vector_query.ef, @@ -998,6 +1001,15 @@ impl VectorQuery { Ok(()) } + pub fn search_effort(&mut self, search_effort: Option) -> PyResult<()> { + self.inner = self + .inner + .clone() + .search_effort(search_effort) + .infer_error()?; + Ok(()) + } + pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> { self.inner = self .inner @@ -1191,6 +1203,18 @@ impl HybridQuery { self.inner_vec.nprobes(nprobe) } + pub fn search_effort(&mut self, search_effort: Option) -> PyResult<()> { + self.inner_vec.search_effort(search_effort) + } + + pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> { + self.inner_vec.minimum_nprobes(minimum_nprobes) + } + + pub fn maximum_nprobes(&mut self, maximum_nprobes: u32) -> PyResult<()> { + self.inner_vec.maximum_nprobes(maximum_nprobes) + } + pub fn ef(&mut self, ef: u32) { self.inner_vec.ef(ef); } @@ -1235,6 +1259,10 @@ impl HybridQuery { req.query_vector = vec_req.query_vector; req.column = vec_req.column; req.distance_type = vec_req.distance_type; + req.nprobes = vec_req.nprobes; + req.minimum_nprobes = vec_req.minimum_nprobes; + req.maximum_nprobes = vec_req.maximum_nprobes; + req.search_effort = vec_req.search_effort; req.ef = vec_req.ef; req.refine_factor = vec_req.refine_factor; req.lower_bound = vec_req.lower_bound; diff --git a/python/uv.lock b/python/uv.lock index b013e5982..ace8b98b9 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1900,6 +1900,7 @@ clip = [ { name = "torch" }, ] dev = [ + { name = "maturin" }, { name = "pre-commit" }, { name = "pyright" }, { name = "ruff" }, @@ -1981,6 +1982,7 @@ requires-dist = [ { name = "ibm-watsonx-ai", marker = "python_full_version >= '3.10' and extra == 'embeddings'", specifier = ">=1.1.2" }, { name = "instructorembedding", marker = "extra == 'embeddings'", specifier = ">=1.0.1" }, { name = "lance-namespace", specifier = ">=0.13.0" }, + { name = "maturin", marker = "extra == 'dev'", specifier = ">=1.10" }, { name = "mkdocs", marker = "extra == 'docs'" }, { name = "mkdocs-jupyter", marker = "extra == 'docs'" }, { name = "mkdocs-material", marker = "extra == 'docs'" }, @@ -2005,7 +2007,7 @@ requires-dist = [ { name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" }, { name = "pydantic", specifier = ">=2.7.4,<3" }, { name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" }, - { name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b10" }, + { name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b12" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" }, { name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" }, { name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" }, @@ -2156,6 +2158,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, ] +[[package]] +name = "maturin" +version = "1.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/c8/22e5e21b2679c9bce6415ca578034ca2cc9316be0642ae21e051a2d5198c/maturin-1.15.0.tar.gz", hash = "sha256:94b26cc8e8aba61a5f2099715fe640e18c5f678e9a500408b38761263954228a", size = 385504, upload-time = "2026-08-24T12:11:22.665Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/69/5c01b461044eb1f45ddcce006706eb88110c793cdb11c7ae0b5e08492e94/maturin-1.15.0-py3-none-linux_armv6l.whl", hash = "sha256:6bf6dc62e22d4dcfd5a51244ff0d58975fa4979c48209fe84159617648956d82", size = 10206220, upload-time = "2026-08-24T12:10:53.327Z" }, + { url = "https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:cd35772633f489841132bc8e71d6fc7f842df30b9c05cd5cdf1ee1ddcb744cc7", size = 19416513, upload-time = "2026-08-24T12:10:56.126Z" }, + { url = "https://files.pythonhosted.org/packages/51/36/e23a21cb34a648b711036b9b2fe1d4f3f4ee24f8db54215d73f1a9a3a3ec/maturin-1.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c40b4eae7bf5ef1f4b1af8d623fe4105016f93578fb15b764e741d08ec3b92dd", size = 10014962, upload-time = "2026-08-24T12:10:58.486Z" }, + { url = "https://files.pythonhosted.org/packages/8c/80/33b15cb2d8f30f12c807955e8f2fd775027692904e30ec0784744ce8cd83/maturin-1.15.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:7eb066372f541f8eb4909c79c5d9bd0b9e8125980bdf1ec9e8aba23c6c8d6c55", size = 10196223, upload-time = "2026-08-24T12:11:00.696Z" }, + { url = "https://files.pythonhosted.org/packages/fe/91/b495e19e2f5c503b540452b2039115e7b2363867e8c5ad4179eb752fa92c/maturin-1.15.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:653020a63525bb224e5ab0adf02e17a2e08bc86dbea7fc1399c9a56d7529b99e", size = 10541186, upload-time = "2026-08-24T12:11:02.857Z" }, + { url = "https://files.pythonhosted.org/packages/5d/2b/2abff58037188d852b124871b1f0d720e1c2bfb3d4f1b03d87c52cd66488/maturin-1.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:0ebf9767892725083138e671c34482c660317a2f3d6a29fc0e0f34e9d8c99136", size = 10083468, upload-time = "2026-08-24T12:11:05.012Z" }, + { url = "https://files.pythonhosted.org/packages/3f/07/b7e9f8be99a6627849e81ac7b6694876bce8f50a92995fe17e3cf2610f0a/maturin-1.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7ab7eebffd7b8debca2265985de4eaeb332141276d24b9560b5ad484d4b3add1", size = 10047786, upload-time = "2026-08-24T12:11:07.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ab/167e3cb7accee11b507dbe53e0e87aeccb376d44ae66284c96ee4df3a9fd/maturin-1.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:126e12e618b4db42f68c779a56d41f82a390145ba36ac3f621d057eb34f5ad9d", size = 13315332, upload-time = "2026-08-24T12:11:09.433Z" }, + { url = "https://files.pythonhosted.org/packages/14/4d/801379f646cbc6b00998e5289b0630a886be3a4ee4c75b6bc9b87478a7f1/maturin-1.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f9d33e6c3f9615c8caceecbbbd440f8eb25a3ddeb687077682cd5eca2e9ae15", size = 10807183, upload-time = "2026-08-24T12:11:11.73Z" }, + { url = "https://files.pythonhosted.org/packages/89/27/2e612e1cbd1580e9e94d4722c227b5180dca27b32b955a34b79918aa1292/maturin-1.15.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:bf29beddd0c6708f112db51d5275fc28b28b9e9c9c5faae387eaef662918b176", size = 10413274, upload-time = "2026-08-24T12:11:14.04Z" }, + { url = "https://files.pythonhosted.org/packages/70/d8/202a7b4d75a51f20f84ec9ce3b7345b12b822207072164e2b1c6ef665125/maturin-1.15.0-py3-none-win32.whl", hash = "sha256:da649988be98e87e009e51b1bf0d301b6a301bc0cecbdd60d40d8ba60748d1ca", size = 8928744, upload-time = "2026-08-24T12:11:16.269Z" }, + { url = "https://files.pythonhosted.org/packages/40/dc/4e90da594986ba78dd3bc8a5921ecdcdb11085b22b02a412caab3b225601/maturin-1.15.0-py3-none-win_amd64.whl", hash = "sha256:552c2be4afd43fe8d5c9f3ec8d4c4756d973b8dcbe94c14084390301f50243e1", size = 10335085, upload-time = "2026-08-24T12:11:18.326Z" }, + { url = "https://files.pythonhosted.org/packages/8b/10/15d4314edf130955edf2dc237aa393a8a7c10f2b9b57b89fa2f61f915659/maturin-1.15.0-py3-none-win_arm64.whl", hash = "sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48", size = 9713795, upload-time = "2026-08-24T12:11:20.83Z" }, +] + [[package]] name = "mdit-py-plugins" version = "0.6.1" @@ -3912,7 +3938,7 @@ crypto = [ [[package]] name = "pylance" -version = "14.0.0b10" +version = "14.0.0b12" source = { registry = "https://pypi.fury.io/lance-format/" } dependencies = [ { name = "lance-namespace" }, @@ -3922,12 +3948,12 @@ dependencies = [ { name = "pyarrow" }, ] wheels = [ - { url = "https://pypi.fury.io/lance-format/-/ver_4hvE/pylance-14.0.0b10-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:415b6b1b9fe735fbae7c755eb9774ae2fe0feddc90bf80bee24087cbfbfdf8c1" }, - { url = "https://pypi.fury.io/lance-format/-/ver_pM1km/pylance-14.0.0b10-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87317cad22f4d35fea11eb70aa5c0d71a0e1ea3b96de38f3deca933daa749fff" }, - { url = "https://pypi.fury.io/lance-format/-/ver_24jdyj/pylance-14.0.0b10-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6da554a2fc2197ba254676b12824d8120b48d65937697e0661a61302458a4740" }, - { url = "https://pypi.fury.io/lance-format/-/ver_XclaZ/pylance-14.0.0b10-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:36c80a801675bb9dcbd681fa99d0cdd5ca642dfb3ca08d71ac980278a2f7748b" }, - { url = "https://pypi.fury.io/lance-format/-/ver_L0yh/pylance-14.0.0b10-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:e50ad0b866faefb2fa3ae87328b93fa0d06b2bdd5bb5d0c1a6c74b907cc2fd93" }, - { url = "https://pypi.fury.io/lance-format/-/ver_2gbNr8/pylance-14.0.0b10-cp310-abi3-win_amd64.whl", hash = "sha256:3d86451fe00f71510005bb700737fb1a4f9361554541cbf802f3c5072554d014" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1Ixasc/pylance-14.0.0b12-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:84350a3c5c1eea3398f4c4fad7cfe85b831d831a5e43b5ca2a783dbdb0e11969" }, + { url = "https://pypi.fury.io/lance-format/-/ver_sCEkG/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76b386c182a6a8e7b86fbef28f84d1dd39e4626ac8681a0cf36f40a45dd7cb87" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1vdT1C/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6aa548c7937fcdfa8f1943a2d59041a42f8b500199880dbb509c7195fa253381" }, + { url = "https://pypi.fury.io/lance-format/-/ver_2fXu1V/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:79f82d6a6c50f37aab338470e3733436e6a27fbb5972d29fcf453ece56d3b3c6" }, + { url = "https://pypi.fury.io/lance-format/-/ver_24UavN/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:de0305999257002785f60169e6926896584d66f1da050bb5afd1d48c6e3d9bb9" }, + { url = "https://pypi.fury.io/lance-format/-/ver_XNynX/pylance-14.0.0b12-cp310-abi3-win_amd64.whl", hash = "sha256:4d0cdb689e898ce71e8e434d00a3b97639cea8fb97da21f1c921543be363268f" }, ] [[package]] diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index 8fb56ceca..07db69bb3 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -1127,6 +1127,11 @@ pub struct VectorQueryRequest { /// If unset, Lance's default is used. Zero represents an explicitly unbounded /// maximum on request and wire representations. pub maximum_nprobes: Option, + /// Optional initial IVF search effort in [0, 1]. + /// + /// None preserves Auto and fixed `nprobes`; any explicit value overrides + /// `nprobes` while retaining explicit minimum and maximum bounds. + pub search_effort: Option, /// The lower bound (inclusive) of the distance to search for. pub lower_bound: Option, /// The upper bound (exclusive) of the distance to search for. @@ -1153,6 +1158,7 @@ impl Default for VectorQueryRequest { nprobes: None, minimum_nprobes: None, maximum_nprobes: None, + search_effort: None, lower_bound: None, upper_bound: None, ef: None, @@ -1165,6 +1171,17 @@ impl Default for VectorQueryRequest { } impl VectorQueryRequest { + pub(crate) fn validate_search_effort(&self) -> Result<()> { + if let Some(effort) = self.search_effort + && (!effort.is_finite() || !(0.0..=1.0).contains(&effort)) + { + return Err(Error::InvalidInput { + message: format!("search_effort must be finite and in [0, 1], got {effort}"), + }); + } + Ok(()) + } + fn add_query_vector(&mut self, vector: impl IntoQueryVector) -> Result<()> { let vector = vector.to_query_vector(&DataType::Float32, "default")?; let array = @@ -1279,7 +1296,8 @@ impl VectorQuery { /// you the desired recall. /// /// LanceDB retains this as `nprobes` through local and remote request - /// construction. Lance sets both probe bounds to this value. For more + /// construction. Lance sets both probe bounds to this value only when + /// [`Self::search_effort`] is unset. For more /// fine-grained control see [`VectorQuery::minimum_nprobes`] and /// [`VectorQuery::maximum_nprobes`]. /// @@ -1294,6 +1312,31 @@ impl VectorQuery { Ok(self) } + /// Set the initial IVF search effort to a finite value in [0, 1]. + /// + /// None (the default) resolves to 0.5, preserving Auto probing. At 0 the + /// search starts at the caller minimum; at 1 it searches all partitions + /// within the caller maximum. Intermediate values interpolate around Auto. + /// Later probing may continue when too few candidates are found. + /// Any explicit value, including 0.5, overrides [`Self::nprobes`] regardless + /// of setter order. Explicit minimum and maximum bounds still apply. + /// This does not change refinement or HNSW ef and is not a recall guarantee. + /// Lance's MemWAL LSM scanner does not yet support this option. + /// + /// ``` + /// # use lancedb::{Table, query::{QueryBase, ExecutableQuery}}; + /// # async fn example(table: &Table) -> lancedb::Result<()> { + /// let results = table.query().nearest_to([0.1, 0.2].as_slice())? + /// .search_effort(0.75)?.execute().await?; + /// # Ok(()) + /// # } + /// ``` + pub fn search_effort(mut self, search_effort: impl Into>) -> Result { + self.request.search_effort = search_effort.into(); + self.request.validate_search_effort()?; + Ok(self) + } + /// Set the minimum number of partitions to search /// /// This argument is only used when the vector column has an IVF PQ index. @@ -1333,6 +1376,8 @@ impl VectorQuery { /// /// If this value is greater than minimum_nprobes then the excess partitions will /// only be searched if the initial search does not return enough results. + /// The bound applies per index segment to initial and later probing without + /// disabling Auto probing. /// /// This can be useful when there is a narrow filter to allow these queries to /// spend more time searching and avoid potential false negatives. @@ -2448,6 +2493,26 @@ mod tests { assert_eq!(query.request.nprobes, None); assert_eq!(query.request.minimum_nprobes, None); assert_eq!(query.request.maximum_nprobes, None); + assert_eq!(query.request.search_effort, None); + + let effort_query = query + .clone() + .nprobes(20) + .unwrap() + .search_effort(0.5) + .unwrap(); + assert_eq!(effort_query.request.search_effort, Some(0.5)); + assert_eq!(effort_query.request.nprobes, Some(20)); + assert_eq!(effort_query.request.minimum_nprobes, None); + assert_eq!(effort_query.request.maximum_nprobes, None); + assert_eq!( + effort_query + .search_effort(None) + .unwrap() + .request + .search_effort, + None + ); assert_eq!( *query .request diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 119577dc5..356fb1e1a 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -1137,6 +1137,7 @@ impl RemoteTable { mut body: serde_json::Value, query: &VectorQueryRequest, ) -> Result> { + query.validate_search_effort()?; self.apply_query_params(&mut body, &query.base)?; // Apply general parameters, before we dispatch based on number of query vectors. @@ -1155,6 +1156,9 @@ impl RemoteTable { if let Some(maximum_nprobes) = query.maximum_nprobes { body["maximum_nprobes"] = maximum_nprobes.into(); } + if let Some(search_effort) = query.search_effort { + body["search_effort"] = search_effort.into(); + } body["lower_bound"] = query.lower_bound.into(); body["upper_bound"] = query.upper_bound.into(); body["ef"] = query.ef.into(); @@ -6269,6 +6273,77 @@ mod tests { )); } + #[rstest] + #[case::unset(None)] + #[case::minimum(Some(0.0))] + #[case::low(Some(0.25))] + #[case::auto(Some(0.5))] + #[case::high(Some(0.75))] + #[case::maximum(Some(1.0))] + #[tokio::test] + async fn test_query_vector_search_effort_wire(#[case] effort: Option) { + let table = Table::new_with_handler("my_table", move |request| { + let body = request_body_json(&request); + assert_eq!(body["nprobes"], 7); + assert_eq!(body["minimum_nprobes"], 2); + assert_eq!(body["maximum_nprobes"], 8); + if let Some(effort) = effort { + assert_eq!(body["search_effort"], effort); + } else { + assert!(body.get("search_effort").is_none()); + } + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])), + vec![Arc::new(Int32Array::from(vec![1]))], + ) + .unwrap(); + http::Response::builder() + .status(200) + .header(CONTENT_TYPE, ARROW_FILE_CONTENT_TYPE) + .body(write_ipc_file(&batch)) + .unwrap() + }); + let batches = table + .query() + .nearest_to(vec![0.1, 0.2]) + .unwrap() + .search_effort(effort) + .unwrap() + .nprobes(7) + .unwrap() + .minimum_nprobes(2) + .unwrap() + .maximum_nprobes(Some(8)) + .unwrap() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches[0].num_rows(), 1); + } + + #[rstest] + #[case::negative(-0.1)] + #[case::too_large(1.1)] + #[case::nan(f64::NAN)] + #[case::infinity(f64::INFINITY)] + #[case::negative_infinity(f64::NEG_INFINITY)] + #[tokio::test] + async fn test_query_vector_search_effort_invalid(#[case] effort: f64) { + let table = Table::new_with_handler::<&str>("my_table", |_| { + panic!("invalid search_effort must not send a request") + }); + let result = table + .query() + .nearest_to(vec![0.1, 0.2]) + .unwrap() + .search_effort(effort); + assert!(matches!(result, Err(Error::InvalidInput { message }) + if message.contains("search_effort must be finite and in [0, 1]"))); + } + #[tokio::test] async fn test_query_vector_all_params() { let table = Table::new_with_handler("my_table", |request| { diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index 77c4ca1a5..3ab54ae07 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -106,7 +106,7 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R } fn requires_local_namespace_execution(query: &AnyQuery) -> bool { - // The namespace QueryTable request cannot represent approx_mode or use_lsm. + // The namespace QueryTable request cannot represent approx_mode, search_effort or use_lsm. // For use_lsm that is worse than a tuning miss: MemWAL read routing lives only // in `create_plan`, so a pushed-down query would return stale base-only data. if query.base().use_lsm.is_some() || query.base().take_offsets.is_some() { @@ -115,7 +115,7 @@ fn requires_local_namespace_execution(query: &AnyQuery) -> bool { matches!( query, AnyQuery::VectorQuery(vector_query) - if vector_query.approx_mode.is_some() + if vector_query.approx_mode.is_some() || vector_query.search_effort.is_some() ) } @@ -163,6 +163,7 @@ pub async fn create_plan( AnyQuery::Query(query) => VectorQueryRequest::from_plain_query(query), }; query.base.check_filter()?; + query.validate_search_effort()?; let ds_ref = table.dataset.get().await?; @@ -293,7 +294,9 @@ pub async fn create_plan( scanner.approx_mode(approx_mode.into()); } - if let Some(nprobes) = query.nprobes { + if query.search_effort.is_none() + && let Some(nprobes) = query.nprobes + { if query.maximum_nprobes == Some(0) { scanner.minimum_nprobes(nprobes); } else { @@ -306,6 +309,9 @@ pub async fn create_plan( if let Some(maximum_nprobes) = query.maximum_nprobes.filter(|nprobes| *nprobes != 0) { scanner.maximum_nprobes(maximum_nprobes); } + if let Some(search_effort) = query.search_effort { + scanner.search_effort(search_effort)?; + } } // For a batch query, `nearest` already applies k to each query vector. @@ -604,6 +610,11 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result { }; // Check for unsupported features + if vq.search_effort.is_some() { + return Err(Error::NotSupported { + message: "namespace QueryTable does not support search_effort; execute the query locally".to_string(), + }); + } if vq.base.reranker.is_some() { return Err(Error::NotSupported { message: "Reranker is not supported for server-side queries".to_string(), @@ -962,6 +973,19 @@ mod tests { let default_query = AnyQuery::VectorQuery(VectorQueryRequest::default()); assert!(!requires_local_namespace_execution(&default_query)); + for effort in [0.0, 0.5, 1.0] { + let effort_query = AnyQuery::VectorQuery(VectorQueryRequest { + query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], + search_effort: Some(effort), + ..Default::default() + }); + assert!(requires_local_namespace_execution(&effort_query)); + assert!(matches!( + convert_to_namespace_query(&effort_query), + Err(Error::NotSupported { message }) if message.contains("search_effort") + )); + } + let nprobes_query = AnyQuery::VectorQuery(VectorQueryRequest { query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], nprobes: Some(20), diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 698b9c736..20e36e95d 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -174,6 +174,9 @@ fn reject_unsupported(query: &VectorQueryRequest) -> Result<()> { // brute-force search the index-only base arm can't do. (ef / approx_mode are // recall/speed knobs and are left to no-op.) if !query.query_vector.is_empty() { + if query.search_effort.is_some() { + return unsupported("search_effort on vector search"); + } if query.lower_bound.is_some() || query.upper_bound.is_some() { return unsupported("distance_range on vector search"); } @@ -821,6 +824,19 @@ mod tests { use super::*; use lance_index::mem_wal::{CompactedSsTable, IndexCatchupProgress}; + #[test] + fn search_effort_is_not_silently_ignored() { + let query = VectorQueryRequest { + query_vector: vec![Arc::new(arrow_array::Float32Array::from(vec![1.0, 0.0]))], + search_effort: Some(0.5), + ..Default::default() + }; + assert!( + matches!(reject_unsupported(&query), Err(Error::NotSupported { message }) + if message.contains("search_effort") && message.contains("un-compacted MemWAL")) + ); + } + #[test] fn resident_index_checks_distinguish_columns_and_query_types() { use arrow_schema::Field; From 0a988a40ac501baa7ea7e837631caefc5c38b9d7 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Mon, 12 Oct 2026 11:37:20 +0000 Subject: [PATCH 17/19] =?UTF-8?q?Bump=20version:=200.41.0-beta.5=20?= =?UTF-8?q?=E2=86=92=200.41.0-beta.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index cb1b6ebd7..12d7f0804 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.41.0-beta.5" +current_version = "0.41.0-beta.6" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index dc2e67b08..3f79185d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5696,7 +5696,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.41.0-beta.5" +version = "0.41.0-beta.6" dependencies = [ "ahash", "anyhow", @@ -5793,7 +5793,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.41.0-beta.5" +version = "0.41.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -5818,7 +5818,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.41.0-beta.5" +version = "0.41.0-beta.6" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 1fb657175..e2d46921e 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.41.0-beta.5 + 0.41.0-beta.6 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index ba0d1c8f2..b312ffebf 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.5 + 0.41.0-beta.6 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 7949e9c79..18bdfe413 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.5 + 0.41.0-beta.6 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 2f38dd299..9404a0913 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.41.0-beta.5" +version = "0.41.0-beta.6" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 6b8fcf993..08e5b3a41 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index cbb0462e7..5aaa172ac 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index e752f03a8..cb1881aaf 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 43d9d2fc3..9881b85d1 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 109e469a5..29f3e7986 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 5c1723bed..f70f4eacd 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 928dfc5bb..c06f89a36 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index 121896f41..4c9be04a7 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index eeb39a672..40b74520e 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.41.0-beta.5" +version = "0.41.0-beta.6" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 419577d05..3cc524bfc 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.41.0-beta.5" +version = "0.41.0-beta.6" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 25fcfc696a71ddda8a2add791671f3a3d89e22f8 Mon Sep 17 00:00:00 2001 From: XY Zhan Date: Mon, 12 Oct 2026 14:18:44 -0400 Subject: [PATCH 18/19] feat: maintain memtable index kinds from a table's registry (#4477) ## Why lance-format/lance#9803 makes the MemWAL's memtable index kinds pluggable: a `MemIndexRegistry` on the writer config decides which base-table index kinds are maintained in memory, so a kind Lance does not build in can be added by registering a plugin. LanceDB opens the MemWAL writer itself for `merge_insert`, and checks a named maintained set itself, both against Lance's built-ins only. So a kind registered outside Lance could not be named in `LsmWriteSpec::with_maintained_indexes`, and under "maintain every index" it was skipped. ## What changes - `NativeTable::set_mem_index_registry(registry)` sets the memtable index kinds a table maintains. The default is Lance's built-ins, so nothing changes unless a caller sets one. The registry checks a named maintained set (`validate_maintained_indexes_with`), is recorded on `initialize_mem_wal`, and goes on the writer config `merge_insert` opens. `lancedb::table` re-exports `MemIndexRegistry` and Lance's plugin traits (`mem_index`), so a caller needs no direct `lance` dependency. - The LSM vector read no longer checks resident memtables at all, and the `carries_vector_index` helper goes with it. Lance already compares every row of a memtable whose index can't answer the search, so no rows are missed. It still requires the base vector index to be maintained. ## Limits - The registry belongs to the table handle. Clones of a handle and its branch handles share it; a handle opened separately starts from the built-ins. Set it before the handle's first LSM write: a writer already open keeps the kinds it opened with. - The maintained set is stored with the table but the registry is not. Once a spec names an index only a registered plugin maintains, an LSM write from a handle without that registry is refused with Lance's "no registered plugin maintains" error rather than skipping the index. That includes the Python and Node bindings, which have no way to set a registry yet. - Rust only. A registry holds Rust plugin objects, which the Python, TypeScript and Java bindings cannot construct, so they get no setter; their documented behaviour (built-in kinds only) is unchanged. - Per-index plugin settings (`ShardWriterConfig::index_overrides`, from lance-format/lance#9805) are not exposed; writers use each plugin's defaults. ## Open decision Whether the registry should also be settable on the connection, so every table a connection opens carries it. This PR keeps it per table; a connection-level option would sit on top of `set_mem_index_registry` without changing it. ## Depends on Lance v14.0.0-beta.13, which `main` already uses. It includes lance-format/lance#9803 and #9805 (pluggable memtable index kinds) and #9857 (flushed generations keep the table's field ids). ## Testing - `lsm_maintains_an_index_kind_from_the_tables_registry`: a stand-in plugin maintains the base table's bitmap index as a B-tree. Naming the index is refused without the registry and accepted with it; a separately opened handle without the registry is refused an LSM write. Rows written through `merge_insert` are found by id before and after the flush. Each generation the shard manifests publish carries a bitmap index on `region`, and a filter on it is answered through that index. - `lsm_default_set_maintains_a_registry_kind_created_while_writing`: under "maintain every index", a bitmap index created after the writer opened is picked up by that writer and flushed. A clone of the table handle shares the registry. - `lsm_vector_search_compares_every_row_when_no_resident_index_answers`: with a vector plugin that answers no search, LSM vector search still returns the right rows, including a second page. - `cargo test -p lancedb --features remote --lib` passes; `cargo clippy -p lancedb --features remote --tests -- -D warnings` and `cargo fmt --all -- --check` are clean. --------- Co-authored-by: XYZhan --- rust/lancedb/src/table.rs | 55 +++- rust/lancedb/src/table/merge.rs | 458 ++++++++++++++++++++++++++-- rust/lancedb/src/table/merge/lsm.rs | 12 +- rust/lancedb/src/table/query/lsm.rs | 39 +-- 4 files changed, 506 insertions(+), 58 deletions(-) diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index b296d135d..15ca3702b 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -19,6 +19,9 @@ pub use lance::dataset::ReadParams; pub use lance::dataset::Version; use lance::dataset::WriteMode; use lance::dataset::builder::DatasetBuilder; +pub use lance::dataset::mem_wal::MemIndexRegistry; +/// The traits a memtable index plugin implements; see [`MemIndexRegistry`]. +pub use lance::dataset::mem_wal::index as mem_index; use lance::dataset::{InsertBuilder, WriteParams}; use lance::index::DatasetIndexExt; use lance::index::scalar::load_segment_params; @@ -424,7 +427,8 @@ impl LsmWriteSpec { /// changes, so an index created later is maintained too; one of a kind the /// MemWAL cannot mirror is skipped rather than failing the table. A list is /// verbatim: each name must already exist and be maintainable, and an empty - /// list maintains nothing. + /// list maintains nothing. The maintainable kinds are Lance's built-ins plus + /// any added with [`NativeTable::set_mem_index_registry`]. /// /// ``` /// # use lancedb::table::LsmWriteSpec; @@ -2590,6 +2594,9 @@ pub struct NativeTable { pub(crate) pushdown_operations: HashSet, // Read-freshness baseline; `Some` only for namespace-backed tables. freshness: Option, + // Memtable index kinds this handle's LSM writes maintain, shared with its + // clones and branch handles. + mem_index_registry: Arc>, } impl std::fmt::Debug for NativeTable { @@ -2756,9 +2763,51 @@ impl NativeTable { namespace_client, pushdown_operations, freshness: None, + mem_index_registry: Default::default(), }) } + /// Maintain the memtable index kinds in `registry` when this table writes + /// through its [`LsmWriteSpec`], and accept them when + /// [`LsmWriteSpec::with_maintained_indexes`] names one. + /// + /// The registry belongs to this handle, its clones and its branch handles, + /// and is not stored with the table: a handle opened separately starts from + /// Lance's built-ins. Set it before the handle's first LSM write; writers + /// already open keep the registry they were opened with. Local tables only, + /// reached through [`Table::as_native`]. + /// + /// ``` + /// # use std::sync::Arc; + /// # use lancedb::Table; + /// # use lancedb::table::{LsmWriteSpec, MemIndexRegistry, mem_index::MemIndexPlugin}; + /// # async fn example( + /// # table: &Table, + /// # plugin: Arc, + /// # ) -> Result<(), Box> { + /// let registry = MemIndexRegistry::default().with_plugin(plugin)?; + /// if let Some(native) = table.as_native() { + /// native.set_mem_index_registry(registry); + /// } + /// let spec = LsmWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]); + /// table.set_lsm_write_spec(spec).await?; + /// # Ok(()) + /// # } + /// ``` + pub fn set_mem_index_registry(&self, registry: MemIndexRegistry) { + *self + .mem_index_registry + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = registry; + } + + pub(crate) fn mem_index_registry(&self) -> MemIndexRegistry { + self.mem_index_registry + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + /// Set the namespace client for server-side query execution. /// /// When set, queries will be executed on the namespace server instead of locally. @@ -2786,6 +2835,7 @@ impl NativeTable { namespace_client: self.namespace_client.clone(), pushdown_operations: self.pushdown_operations.clone(), freshness: self.freshness.clone(), + mem_index_registry: self.mem_index_registry.clone(), } } @@ -2893,6 +2943,7 @@ impl NativeTable { namespace_client: stored_namespace_client, pushdown_operations, freshness: None, + mem_index_registry: Default::default(), }) } @@ -2984,6 +3035,7 @@ impl NativeTable { namespace_client, pushdown_operations, freshness: None, + mem_index_registry: Default::default(), }) } @@ -3117,6 +3169,7 @@ impl NativeTable { namespace_client: stored_namespace_client, pushdown_operations, freshness: None, + mem_index_registry: Default::default(), }) } diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 1ad084e22..28d4a0e13 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -1782,8 +1782,13 @@ mod lsm_tests { assert_eq!(rows, 1, "LSM FTS must still see base rows"); } - #[tokio::test] - async fn lsm_read_vector_search() { + /// A table whose vector index the LSM maintains: base rows filled with + /// their id, and one memtable row filled with 1000, nearest to `[1000; 8]`. + /// `registry` replaces the table's memtable kinds. + async fn lsm_vector_table( + dir: &std::path::Path, + registry: Option, + ) -> Table { use crate::index::Index; use crate::index::vector::IvfPqIndexBuilder; use arrow::array::{FixedSizeListBuilder, Float32Builder}; @@ -1815,12 +1820,7 @@ mod lsm_tests { .unwrap() }; - let dir = tempdir().unwrap(); - let conn = connect(dir.path().to_str().unwrap()) - .execute() - .await - .unwrap(); - // Base rows fill each vector with its own id (0..256); all far from 1000. + let conn = connect(dir.to_str().unwrap()).execute().await.unwrap(); let base = make_batch((0..N).map(|i| (i, i as f32)).collect()); let base_reader: Box = Box::new(RecordBatchIterator::new(vec![Ok(base)], schema.clone())); @@ -1838,13 +1838,15 @@ mod lsm_tests { .execute() .await .unwrap(); + if let Some(registry) = registry { + table.as_native().unwrap().set_mem_index_registry(registry); + } let vec_index = table.list_indices().await.unwrap()[0].name.clone(); table .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![vec_index])) .await .unwrap(); - // Insert a vector (filled with 1000) that is nearest to the query. let mut builder = table.merge_insert(&[]); builder .when_matched_update_all(None) @@ -1854,20 +1856,21 @@ mod lsm_tests { schema.clone(), )); builder.execute(insert_reader).await.unwrap(); + table + } - // KNN near [1000; DIM]: the default (auto-routed) read surfaces the - // memtable row. - let stream = table - .query() - .nearest_to(&[1000.0_f32; 8]) - .unwrap() - .limit(1) - .execute() - .await - .unwrap(); - let ids = collect_ids(stream).await; + /// The ids of the nearest row to `[1000; 8]` through the default read. + async fn nearest_ids(table: &Table) -> Vec { + let query = table.query().nearest_to(&[1000.0_f32; 8]).unwrap().limit(1); + collect_ids(query.execute().await.unwrap()).await + } + + #[tokio::test] + async fn lsm_read_vector_search() { + let dir = tempdir().unwrap(); + let table = lsm_vector_table(dir.path(), None).await; assert_eq!( - ids, + nearest_ids(&table).await, vec![9999], "LSM vector search must rank the memtable row first" ); @@ -1893,4 +1896,417 @@ mod lsm_tests { assert_eq!(ids, vec![255]); } } + + /// The built-in vector plugin, except that its index answers no search. + #[derive(Debug)] + struct AnswersNothing; + + #[derive(Debug)] + struct AnswersNothingIndex(Arc); + + fn vector_plugin() -> Arc { + Arc::new(lance::dataset::mem_wal::index::HnswMemIndexPlugin) + } + + #[async_trait::async_trait] + impl lance::dataset::mem_wal::index::MemIndexPlugin for AnswersNothing { + fn name(&self) -> &str { + "AnswersNothing" + } + fn details_message(&self) -> &str { + "VectorIndexDetails" + } + fn flush_index_type(&self) -> lance_index::IndexType { + vector_plugin().flush_index_type() + } + fn training_criteria(&self) -> lance_index::scalar::registry::TrainingCriteria { + vector_plugin().training_criteria() + } + async fn resolve( + &self, + ctx: &lance::dataset::mem_wal::index::ResolveContext<'_>, + ) -> lance_core::Result { + vector_plugin().resolve(ctx).await + } + fn validate( + &self, + ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>, + ) -> lance_core::Result<()> { + vector_plugin().validate(ctx) + } + fn create( + &self, + ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>, + ) -> lance_core::Result> { + Ok(Arc::new(AnswersNothingIndex(vector_plugin().create(ctx)?))) + } + } + + #[async_trait::async_trait] + impl lance::dataset::mem_wal::index::MemIndex for AnswersNothingIndex { + fn columns(&self) -> &[String] { + self.0.columns() + } + fn can_answer(&self, _query: &dyn lance::dataset::mem_wal::index::MemQuery) -> bool { + false + } + fn insert(&self, batch: &RecordBatch, row_offset: u64) -> lance_core::Result<()> { + self.0.insert(batch, row_offset) + } + fn insert_batches( + &self, + batches: &[lance::dataset::mem_wal::write::StoredBatch], + ) -> lance_core::Result<()> { + self.0.insert_batches(batches) + } + fn resident_bytes(&self) -> usize { + self.0.resident_bytes() + } + fn search( + &self, + _query: &dyn lance::dataset::mem_wal::index::MemQuery, + _ctx: &lance::dataset::mem_wal::index::SearchContext, + ) -> lance_core::Result> { + Ok(None) + } + async fn flush( + &self, + ctx: &lance::dataset::mem_wal::index::FlushContext<'_>, + ) -> lance_core::Result { + self.0.flush(ctx).await + } + } + + /// A resident memtable whose vector index answers nothing still serves an + /// LSM vector search: Lance compares every row instead. + #[tokio::test] + async fn lsm_vector_search_compares_every_row_when_no_resident_index_answers() { + let dir = tempdir().unwrap(); + let mut registry = lance::dataset::mem_wal::MemIndexRegistry::default(); + registry.replace_plugin(Arc::new(AnswersNothing)).unwrap(); + let table = lsm_vector_table(dir.path(), Some(registry)).await; + assert_eq!(nearest_ids(&table).await, vec![9999]); + + // The second page starts after the memtable row. + let page = table + .query() + .nearest_to(&[1000.0_f32; 8]) + .unwrap() + .limit(2) + .offset(1); + assert_eq!( + collect_ids(page.execute().await.unwrap()).await, + vec![255, 254] + ); + } + + /// Claims the base table's bitmap index and keeps it as a B-tree, whose + /// flush rows are in the shape the bitmap trainer takes. + #[derive(Debug)] + struct BitmapAsBTree; + + fn btree_plugin() -> Arc { + lance::dataset::mem_wal::MemIndexRegistry::default() + .plugin_for_details_url("/lance.table.BTreeIndexDetails") + .unwrap() + .clone() + } + + #[async_trait::async_trait] + impl lance::dataset::mem_wal::index::MemIndexPlugin for BitmapAsBTree { + fn name(&self) -> &str { + "BitmapAsBTree" + } + fn details_message(&self) -> &str { + "BitmapIndexDetails" + } + fn flush_index_type(&self) -> lance_index::IndexType { + lance_index::IndexType::Bitmap + } + fn training_criteria(&self) -> lance_index::scalar::registry::TrainingCriteria { + btree_plugin().training_criteria() + } + fn validate( + &self, + ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>, + ) -> lance_core::Result<()> { + btree_plugin().validate(ctx) + } + fn create( + &self, + ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>, + ) -> lance_core::Result> { + btree_plugin().create(ctx) + } + } + + /// A kind Lance has no memtable index for is refused when named until the + /// table's registry maintains it; then it is maintained, flushed with each + /// generation, and answers searches on the flushed generations. + #[tokio::test] + async fn lsm_maintains_an_index_kind_from_the_tables_registry() { + let dir = tempdir().unwrap(); + let conn = connect(dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let table = conn + .create_table("t", id_region_reader(vec![(1, "us"), (2, "eu"), (3, "us")])) + .execute() + .await + .unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); + table + .create_index(&["region"], crate::index::Index::Bitmap(Default::default())) + .name("region_bitmap".to_string()) + .execute() + .await + .unwrap(); + let spec = + || LsmWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]); + + let refused = table.set_lsm_write_spec(spec()).await.unwrap_err(); + assert!( + refused.to_string().contains("region_bitmap") + && refused + .to_string() + .contains("no registered plugin maintains"), + "{refused:?}" + ); + + let registry = lance::dataset::mem_wal::MemIndexRegistry::default() + .with_plugin(Arc::new(BitmapAsBTree)) + .unwrap(); + table.as_native().unwrap().set_mem_index_registry(registry); + table.set_lsm_write_spec(spec()).await.unwrap(); + + // The registry is not stored with the table. + let other = conn.open_table("t").execute().await.unwrap(); + let mut builder = other.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + let refused = builder + .execute(id_region_reader(vec![(20, "us")])) + .await + .unwrap_err(); + assert!( + refused + .to_string() + .contains("no registered plugin maintains"), + "{refused:?}" + ); + + let mut builder = table.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + builder + .execute(id_region_reader(vec![ + (10, "fresh"), + (11, "fresh"), + (12, "eu"), + ])) + .await + .unwrap(); + + let fresh = || async { + let query = table.query().only_if("region = 'fresh'"); + let mut ids = collect_ids(query.execute().await.unwrap()).await; + ids.sort_unstable(); + ids + }; + assert_eq!(fresh().await, vec![10, 11]); + let (_, _, memtables) = table + .as_native() + .unwrap() + .dataset + .shard_writer() + .read_snapshot() + .await + .unwrap() + .expect("the merge opened a writer"); + let active = memtables.expect("the writer keeps memtables").active; + assert!( + active + .index_store + .index_names() + .contains(&"region_bitmap".to_string()), + "the memtable maintains the plugin's index: {:?}", + active.index_store.index_names() + ); + + table.close_lsm_writers().await.unwrap(); + assert_eq!(fresh().await, vec![10, 11]); + assert_generations_answer_through_region_bitmap(&table, vec![10, 11]).await; + } + + /// The generations each shard's manifest publishes. + async fn published_generations(table: &Table) -> Vec { + use lance::dataset::mem_wal::ShardManifestStore; + + let uri = table.uri().await.unwrap(); + let (store, base) = lance_io::object_store::ObjectStore::from_uri(&uri) + .await + .unwrap(); + let mem_wal = std::path::Path::new(&uri).join("_mem_wal"); + let mut generations = Vec::new(); + for shard in std::fs::read_dir(&mem_wal).unwrap() { + let Ok(shard_id) = shard.unwrap().file_name().to_string_lossy().parse() else { + continue; + }; + let manifest = ShardManifestStore::new(store.clone(), &base, shard_id, 2) + .latest() + .await + .unwrap() + .expect("a shard directory has a manifest"); + for sstable in manifest.sstables { + let path = mem_wal.join(shard_id.to_string()).join(&sstable.path); + generations.push(lance::Dataset::open(path.to_str().unwrap()).await.unwrap()); + } + } + generations + } + + /// Every published generation carries `region_bitmap` as a bitmap index on + /// `region`, and a search for `region = 'fresh'` on it uses that index. + async fn assert_generations_answer_through_region_bitmap(table: &Table, fresh: Vec) { + use lance::index::DatasetIndexExt; + + let generations = published_generations(table).await; + assert!( + !generations.is_empty(), + "closing the writer flushed a generation" + ); + let mut found = Vec::new(); + for generation in generations { + let region = generation.schema().field("region").unwrap().id; + let indices = generation.load_indices().await.unwrap(); + let index = indices + .iter() + .find(|index| index.name == "region_bitmap") + .expect("the generation carries the index the plugin trained"); + assert_eq!(index.fields, vec![region]); + assert!( + index + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("BitmapIndexDetails")), + "{:?}", + index.index_details + ); + + let mut scan = generation.scan(); + scan.filter("region = 'fresh'").unwrap(); + let plan = scan.explain_plan(false).await.unwrap(); + assert!(plan.contains("ScalarIndexQuery"), "{plan}"); + let batch = scan.try_into_batch().await.unwrap(); + found.extend( + batch["id"] + .as_primitive::() + .values() + .iter() + .copied(), + ); + } + found.sort_unstable(); + assert_eq!(found, fresh); + } + + /// Under the default set, a kind only the registry maintains is picked up + /// when its index is created after the writer opened, and the table's + /// clones share the registry. + #[tokio::test] + async fn lsm_default_set_maintains_a_registry_kind_created_while_writing() { + let dir = tempdir().unwrap(); + let conn = connect(dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let table = conn + .create_table("t", id_region_reader(vec![(1, "us"), (2, "eu")])) + .execute() + .await + .unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); + let registry = lance::dataset::mem_wal::MemIndexRegistry::default() + .with_plugin(Arc::new(BitmapAsBTree)) + .unwrap(); + table.as_native().unwrap().set_mem_index_registry(registry); + let clone = table.clone(); + assert!( + clone + .as_native() + .unwrap() + .mem_index_registry() + .plugin_for_details_url("/lance.table.BitmapIndexDetails") + .is_some(), + "a clone shares the handle's registry" + ); + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + + let upsert = |rows: Vec<(i64, &'static str)>| { + let table = table.clone(); + async move { + let mut builder = table.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + builder.execute(id_region_reader(rows)).await.unwrap(); + } + }; + // Opens the writer before the table has a bitmap index. + upsert(vec![(3, "us")]).await; + table + .create_index(&["region"], crate::index::Index::Bitmap(Default::default())) + .name("region_bitmap".to_string()) + .execute() + .await + .unwrap(); + upsert(vec![(10, "fresh"), (11, "fresh")]).await; + + let (_, _, memtables) = table + .as_native() + .unwrap() + .dataset + .shard_writer() + .read_snapshot() + .await + .unwrap() + .expect("the merge opened a writer"); + let active = memtables.expect("the writer keeps memtables").active; + assert!( + active + .index_store + .index_names() + .contains(&"region_bitmap".to_string()), + "the open writer picked up the new index: {:?}", + active.index_store.index_names() + ); + + table.close_lsm_writers().await.unwrap(); + // Only the generation written after the index exists carries it. + let generations = published_generations(&table).await; + let mut carrying = 0; + for generation in &generations { + use lance::index::DatasetIndexExt; + if generation + .load_indices() + .await + .unwrap() + .iter() + .any(|index| index.name == "region_bitmap") + { + carrying += 1; + } + } + assert!( + carrying >= 1, + "{} generations, none carry the index", + generations.len() + ); + } } diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index 0f86fdcac..33bf54535 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -29,7 +29,7 @@ use arrow_schema::{DataType, Schema as ArrowSchema, SchemaRef}; use lance::Dataset; use lance::dataset::mem_wal::{ DatasetMemWalExt, ShardWriter, ShardWriterConfig, evaluate_sharding_spec, - validate_maintained_indexes, + validate_maintained_indexes_with, }; use lance::index::DatasetIndexExt; use lance_core::datatypes::Schema as LanceSchema; @@ -103,7 +103,8 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) } } let dataset = table.dataset.get().await?; - validate_maintained_indexes(&dataset, requested).await?; + validate_maintained_indexes_with(&dataset, requested, &table.mem_index_registry()) + .await?; Some(requested.to_vec()) } None => None, @@ -126,7 +127,9 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) .into(), }); } - let mut builder = dataset.initialize_mem_wal(); + let mut builder = dataset + .initialize_mem_wal() + .mem_index_registry(table.mem_index_registry()); let writer_config_defaults = match spec { LsmWriteSpec::Bucket { column, @@ -725,7 +728,8 @@ pub(crate) async fn execute_lsm_merge_insert( return Ok(lsm_merge_result(0)); }; - let config = shard_writer_config_from_defaults(&plan.writer_config_defaults); + let mut config = shard_writer_config_from_defaults(&plan.writer_config_defaults); + config.mem_index_registry = table.mem_index_registry(); let writer = table .dataset .shard_writer() diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 20e36e95d..4798da431 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -33,7 +33,6 @@ use datafusion_physical_plan::expressions::Column; use datafusion_physical_plan::projection::ProjectionExec; use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr}; use lance::Dataset; -use lance::dataset::mem_wal::index::{IndexStore, VectorMemQuery}; use lance::dataset::mem_wal::scanner::{InMemoryMemTableRef, InMemoryMemTables}; use lance::dataset::mem_wal::{ DatasetMemWalExt, LsmScanner, ShardManifestStore, ShardSnapshot, ShardWriterConfig, @@ -711,17 +710,14 @@ async fn vector_plan( // The base arm relies on the column's vector index (`fast_search`). Unmaintained, // its catch-up is untracked and exclusion falls back to the compaction watermark, - // dropping compacted SSTables the lagging base index has not re-indexed; and a - // resident MemTable built before the index joined the set carries none either. + // dropping compacted SSTables the lagging base index has not re-indexed. // Reject rather than silently omit rows, mirroring the FTS arm. - if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? - || !resident_memtables_carry(&in_memory, |memtable| { - carries_vector_index(&memtable.index_store, &column) - }) - { + // A resident MemTable needs no index of its own: one that cannot answer + // compares every row. + if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only" + "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only" ), }); } @@ -764,22 +760,6 @@ async fn vector_plan( Ok(scanner.create_plan().await?) } -/// Probe for a resident vector index without imposing a distance metric, matching -/// the presence check used before Lance introduced query-based index lookup. -fn carries_vector_index(store: &IndexStore, column: &str) -> bool { - let probe = VectorMemQuery { - vector: arrow_array::FixedSizeListArray::new_null( - Arc::new(arrow_schema::Field::new("item", DataType::Float32, true)), - 1, - 1, - ), - k: 1, - ef: None, - distance_type: None, - }; - store.index_answering(column, &probe).is_some() -} - /// Whether `column` stores binary (uint8) vectors, which the LSM vector planner /// does not support. fn is_binary_vector_column(schema: &ArrowSchema, column: &str) -> bool { @@ -838,9 +818,9 @@ mod tests { } #[test] - fn resident_index_checks_distinguish_columns_and_query_types() { + fn resident_fts_check_distinguishes_columns() { use arrow_schema::Field; - use lance::dataset::mem_wal::index::MemIndexSpec; + use lance::dataset::mem_wal::index::{IndexStore, MemIndexSpec}; let schema = ArrowSchema::new(vec![ Field::new("id", DataType::Int32, false), @@ -863,16 +843,11 @@ mod tests { ), ]; let store = IndexStore::from_specs(&specs, &schema, 1_000, 16).unwrap(); - assert!(carries_vector_index(&store, "vector")); - for column in ["id", "text", "missing"] { - assert!(!carries_vector_index(&store, column)); - } assert!(!store.fts_granularities_on("text").is_empty()); for column in ["id", "vector", "missing"] { assert!(store.fts_granularities_on(column).is_empty()); } let empty = IndexStore::from_specs(&[], &schema, 1_000, 16).unwrap(); - assert!(!carries_vector_index(&empty, "vector")); assert!(empty.fts_granularities_on("text").is_empty()); } From 792ffd108251e0b4818485f2c171bc33e5dec39d Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:56:39 -0700 Subject: [PATCH 19/19] fix(python): materialize nested blob bytes in pandas (#4305) --- python/python/lancedb/_blob.py | 63 ++++++++++++++--- python/python/tests/test_blob.py | 116 +++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 11 deletions(-) diff --git a/python/python/lancedb/_blob.py b/python/python/lancedb/_blob.py index dc4ed37df..ee78a7312 100644 --- a/python/python/lancedb/_blob.py +++ b/python/python/lancedb/_blob.py @@ -110,7 +110,9 @@ def projection_includes_blob_column( if projection is None: return True for output, source in _iter_projection_pairs(projection): - if output in columns or source in columns: + if output in columns or any( + column == source or column.startswith(f"{source}.") for column in columns + ): return True return False @@ -122,14 +124,16 @@ def blob_v2_projection_sources( blob_columns = row_addressable_blob_v2_paths(schema) if not blob_columns: return {} - columns = set(blob_columns) if projection is None: return {column: column for column in blob_columns} - return { - output: source - for output, source in _iter_projection_pairs(projection) - if source in columns - } + sources = {} + for output, source in _iter_projection_pairs(projection): + for column in blob_columns: + if column == source: + sources[output] = column + elif column.startswith(f"{source}."): + sources[f"{output}{column[len(source) :]}"] = column + return sources def v2_projection_needs_row_id( @@ -180,7 +184,7 @@ async def replace_v2_blob_columns_with_bytes( fetch_blobs: FetchBlobsAsync, ) -> pa.Table: for output_name, source_name in blob_sources.items(): - if output_name not in tbl.column_names: + if not _has_column_path(tbl.schema, output_name): continue blobs = await fetch_blobs(source_name, tbl) tbl = _set_blob_column(tbl, output_name, blobs) @@ -193,7 +197,7 @@ def replace_v2_blob_columns_with_bytes_sync( fetch_blobs: FetchBlobsSync, ) -> pa.Table: for output_name, source_name in blob_sources.items(): - if output_name not in tbl.column_names: + if not _has_column_path(tbl.schema, output_name): continue blobs = fetch_blobs(source_name, tbl) tbl = _set_blob_column(tbl, output_name, blobs) @@ -287,9 +291,46 @@ def _iter_projection_pairs( yield name, source if source is not None else expr.to_sql() -def _set_blob_column(tbl: pa.Table, output_name: str, blobs: pa.Array) -> pa.Table: +def _has_column_path(schema: pa.Schema, path: str) -> bool: + if schema.get_field_index(path) >= 0: + return True + top_name, *rest = path.split(".") + index = schema.get_field_index(top_name) + if index < 0: + return False + field = schema.field(index) + for name in rest: + if not pa.types.is_struct(field.type): + return False + index = field.type.get_field_index(name) + if index < 0: + return False + field = field.type.field(index) + return True + + +def _set_blob_column( + tbl: pa.Table, output_name: str, blobs: pa.Array | pa.ChunkedArray +) -> pa.Table: index = tbl.schema.get_field_index(output_name) - return tbl.set_column(index, pa.field(output_name, blobs.type), [blobs]) + if index >= 0: + return tbl.set_column(index, pa.field(output_name, blobs.type), [blobs]) + + parent_path, leaf_name = output_name.rsplit(".", 1) + if isinstance(blobs, pa.ChunkedArray): + blobs = blobs.combine_chunks() + + def replace_blob(children: list, child_fields: list) -> None: + for i, field in enumerate(child_fields): + if field.name == leaf_name: + children[i] = blobs + child_fields[i] = pa.field( + leaf_name, blobs.type, nullable=field.nullable + ) + return + raise KeyError(f"Blob field '{leaf_name}' not found in struct '{parent_path}'") + + return _transform_struct_column(tbl, parent_path, replace_blob) def _embed_row_id_in_column(tbl: pa.Table, path: str, row_ids: pa.Array) -> pa.Table: diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 72107364c..2bb5b6bea 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -15,6 +15,7 @@ from lance.blob import Blob, BlobType as LanceBlobType import lancedb from lancedb._blob import ( + _set_blob_column, blob_v2_projection_sources, read_row_ids_from_hits, stash_auto_row_ids, @@ -297,6 +298,121 @@ def test_blob_v2_projection_sources_use_typed_column_name(): } +def test_blob_v2_projection_sources_include_selected_struct(): + schema = pa.schema( + [ + pa.field( + "info", pa.struct([pa.field("name", pa.string()), lancedb.blob("blob")]) + ) + ] + ) + + assert blob_v2_projection_sources(schema, ["info"]) == {"info.blob": "info.blob"} + assert blob_v2_projection_sources(schema, {"renamed": "info"}) == { + "renamed.blob": "info.blob" + } + + +@pytest.mark.parametrize("parent_path", ["info", "outer.info"]) +def test_set_blob_column_missing_nested_field_raises(parent_path): + row = {"name": "first"} + for name in reversed(parent_path.split(".")): + row = {name: row} + table = pa.Table.from_pylist([row]) + blobs = pa.array([b"hello"], type=pa.large_binary()) + + with pytest.raises(KeyError) as exc_info: + _set_blob_column(table, f"{parent_path}.blob", blobs) + + assert parent_path in str(exc_info.value) + assert "'blob'" in str(exc_info.value) + + +def _nested_blob_table(db, name): + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field( + "info", pa.struct([pa.field("name", pa.string()), lancedb.blob("blob")]) + ), + ] + ) + table = db.create_table(name, schema=schema) + info = pa.StructArray.from_arrays( + [ + pa.array(["first", "second", "third"]), + pa.array([b"hello", None, b"hidden"], type=pa.large_binary()), + ], + names=["name", "blob"], + mask=pa.array([False, False, True]), + ) + table.add(pa.table({"id": pa.array([0, 1, 2]), "info": info})) + return table + + +def test_nested_blob_to_pandas_bytes(tmp_path): + table = _nested_blob_table(lancedb.connect(tmp_path), "nested_pandas_bytes") + expected = [ + {"name": "first", "blob": b"hello"}, + {"name": "second", "blob": None}, + None, + ] + + assert table.to_pandas(blob_mode="bytes")["info"].tolist() == expected + assert table.search().to_pandas(blob_mode="bytes")["info"].tolist() == expected + assert ( + table.search().select(["info"]).to_pandas(blob_mode="bytes")["info"].tolist() + == expected + ) + assert ( + table.search() + .select({"renamed": "info"}) + .to_pandas(blob_mode="bytes")["renamed"] + .tolist() + == expected + ) + assert table.search().select({"b": "info.blob"}).to_pandas(blob_mode="bytes")[ + "b" + ].tolist() == [b"hello", None, None] + + +@pytest.mark.asyncio +async def test_nested_blob_to_pandas_bytes_async(tmp_path): + db = await lancedb.connect_async(tmp_path) + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field( + "info", pa.struct([pa.field("name", pa.string()), lancedb.blob("blob")]) + ), + ] + ) + table = await db.create_table("nested_pandas_bytes_async", schema=schema) + await table.add( + pa.table( + { + "id": pa.array([0, 1]), + "info": pa.StructArray.from_arrays( + [ + pa.array(["first", "second"]), + pa.array([b"hello", None], type=pa.large_binary()), + ], + names=["name", "blob"], + ), + } + ) + ) + expected = [ + {"name": "first", "blob": b"hello"}, + {"name": "second", "blob": None}, + ] + + assert (await table.to_pandas(blob_mode="bytes"))["info"].tolist() == expected + assert (await table.query().to_pandas(blob_mode="bytes"))[ + "info" + ].tolist() == expected + + def _legacy_v1_table(name): # Legacy v1 blob columns are only writable at file version <= 2.1. db = lancedb.connect(