From 0bceaaf3f07ac3c533164c104d34804e8531dae1 Mon Sep 17 00:00:00 2001 From: Yang Cen Date: Mon, 12 Oct 2026 19:32:38 +0800 Subject: [PATCH 1/7] feat(search): expose search effort (#4520) Lance supports `search_effort`, but LanceDB cannot pass it through its query APIs. Expose the parameter in Rust, Python sync/async and TypeScript, including remote requests and hybrid query conversion, using the [Lance v14.0.0-beta.13](https://github.com/lance-format/lance/releases/tag/v14.0.0-beta.13) dependency already on main. The default is `None` (Auto, internally 0.5). Any explicit effort, including 0.5, overrides fixed `nprobes` in either setter order; explicit minimum/maximum bounds remain effective. Invalid values are rejected in the Rust core. Hybrid conversion now retains the vector probe parameters and handles an inferred FTS column without panicking. Namespace queries with explicit effort execute locally because the namespace wire model cannot carry it. MemWAL LSM queries reject it explicitly because Lance's LSM scanner does not yet expose this option. Maturin is included in the dev extra so the documented build command works in the project environment. BREAKING CHANGE: The public Rust `VectorQueryRequest` struct now includes `search_effort: Option`. Downstream code that initializes every field explicitly must add `search_effort: None` or use `..Default::default()`. Exhaustive destructuring patterns must include the new field or `..`. Query-builder calls and struct literals that already use `..Default::default()` require no source changes. The new effort setters in Python and TypeScript are additive. This PR does not change the persisted file format. Using explicit effort with remote tables requires server support for `search_effort` and its precedence over `nprobes`; the client wire tests do not establish that a deployed server supports it. Server-side support is tracked in lancedb/sophon#8607. Validation on AWS after merging current main (Lance beta.13 / SDK beta.5): - 14 Rust regressions and the API doctest pass. - 60 parameter regressions, the full 47-test namespace suite, and three API doctests pass. The parameter selection includes some namespace cases. - 40 TypeScript query tests and 54 remote tests pass; build, lint and generated docs pass. - Rust Clippy, Python package lint/format, Pyright and lockfile checks pass. Repository-wide Ruff reports 49 lint errors and 11 formatting failures in unchanged `docs/web-tests` files; those files are byte-identical to main and outside this change. All 1,103 Git-tracked files match the AWS validation checkout. No benchmark scripts or result artifacts are included, and no performance improvement is claimed. Uses lance-format/lance#9798. Server-side support is in lancedb/sophon#8607. --- docs/src/js/classes/VectorQuery.md | 32 +++++++ nodejs/__test__/query.test.ts | 103 +++++++++++++++++++++ nodejs/lancedb/query.ts | 21 +++++ nodejs/src/query.rs | 10 +++ python/pyproject.toml | 3 +- python/python/lancedb/_lancedb.pyi | 3 + python/python/lancedb/namespace.py | 5 ++ python/python/lancedb/query.py | 88 +++++++++++++++++- python/python/lancedb/table.py | 4 + python/python/tests/test_namespace.py | 12 +++ python/python/tests/test_query.py | 124 ++++++++++++++++++++++++++ python/src/query.rs | 34 ++++++- python/uv.lock | 42 +++++++-- rust/lancedb/src/query.rs | 67 +++++++++++++- rust/lancedb/src/remote/table.rs | 75 ++++++++++++++++ rust/lancedb/src/table/query.rs | 30 ++++++- rust/lancedb/src/table/query/lsm.rs | 16 ++++ 17 files changed, 650 insertions(+), 19 deletions(-) diff --git a/docs/src/js/classes/VectorQuery.md b/docs/src/js/classes/VectorQuery.md index 3f56bb600..4809ebefd 100644 --- a/docs/src/js/classes/VectorQuery.md +++ b/docs/src/js/classes/VectorQuery.md @@ -646,6 +646,38 @@ rerank(reranker): VectorQuery *** +### searchEffort() + +```ts +searchEffort(searchEffort): VectorQuery +``` + +Set the initial IVF search effort to a finite value in [0, 1]. + +The default, null, uses Auto (internally 0.5). Zero starts at the caller +minimum; one searches all partitions within the caller maximum. Later +probing may continue when too few candidates are found. +Any explicit effort, including 0.5, overrides nprobes in either call order. +Explicit minimum and maximum bounds still apply. This is not a recall +guarantee and does not change refinement or HNSW ef. The MemWAL LSM +scanner does not yet support this option. + +#### Parameters + +* **searchEffort**: `null` \| `number` + +#### Returns + +[`VectorQuery`](VectorQuery.md) + +#### Example + +```ts +table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10); +``` + +*** + ### select() ```ts diff --git a/nodejs/__test__/query.test.ts b/nodejs/__test__/query.test.ts index 16a595002..786f04015 100644 --- a/nodejs/__test__/query.test.ts +++ b/nodejs/__test__/query.test.ts @@ -527,3 +527,106 @@ describe("Query orderBy", () => { ]); }); }); + +describe("IVF search effort", () => { + let directory: tmp.DirResult; + let table: Table; + const vector = Array.from({ length: 16 }, (_, i) => (i === 0 ? 1 : 0)); + + beforeEach(async () => { + directory = tmp.dirSync({ unsafeCleanup: true }); + const db = await connect(directory.name); + const rows = Array.from({ length: 512 }, (_, id) => ({ + id, + vector: Array.from( + { length: 16 }, + (_, j) => + (Math.floor(id / 32) === j ? 1 : 0) + Math.sin(id * 17 + j) * 0.001, + ), + })); + table = await db.createTable("effort", rows.slice(0, 256)); + await table.add(rows.slice(256)); + await table.createIndex("vector", { + config: Index.ivfFlat({ numPartitions: 16 }), + }); + }); + + afterEach(() => { + directory.removeCallback(); + }); + + it.each([ + [null, 6], + [0, 1], + [0.25, 3], + [0.5, 6], + [0.75, 10], + [1, 16], + ])( + "preserves effort %s through execution and probe precedence", + async (effort, expected) => { + const partitions = async (query: ReturnType) => { + const plan = await query.limit(8).analyzePlan(); + const matches = [...plan.matchAll(/partitions_searched=(\d+)/g)]; + expect(matches.length).toBeGreaterThan(0); + return matches.reduce((total, match) => total + Number(match[1]), 0); + }; + expect( + await partitions(table.vectorSearch(vector).searchEffort(effort)), + ).toBe(expected); + expect( + await partitions( + table.vectorSearch(vector).nprobes(1).searchEffort(effort), + ), + ).toBe(effort === null ? 1 : expected); + expect( + await partitions( + table.vectorSearch(vector).searchEffort(effort).nprobes(1), + ), + ).toBe(effort === null ? 1 : expected); + expect( + await partitions( + table + .vectorSearch(vector) + .searchEffort(effort) + .nprobes(1) + .searchEffort(null), + ), + ).toBe(1); + expect( + await partitions( + table.vectorSearch(vector).searchEffort(effort).maximumNprobes(2), + ), + ).toBeLessThanOrEqual(2); + const actual = await table + .vectorSearch(vector) + .searchEffort(effort) + .limit(8) + .toArray(); + const exact = await table + .vectorSearch(vector) + .bypassVectorIndex() + .limit(8) + .toArray(); + const ids = new Set(exact.map((row) => row.id)); + expect( + actual.filter((row) => ids.has(row.id)).length / 8, + ).toBeGreaterThanOrEqual(0.5); + }, + ); + + it.each([ + -0.1, + 1.1, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + ])("rejects invalid effort %s before execution", (effort) => { + expect(() => table.vectorSearch(vector).searchEffort(effort)).toThrow( + "search_effort must be finite and in [0, 1]", + ); + expect(() => + table.vectorSearch(vector).fullTextSearch("dog").searchEffort(effort), + ).toThrow("search_effort must be finite and in [0, 1]"); + }); +}); diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index 0f0eaf610..cce505c97 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -549,6 +549,27 @@ export class VectorQuery extends StandardQueryBase { return this; } + /** + * Set the initial IVF search effort to a finite value in [0, 1]. + * + * The default, null, uses Auto (internally 0.5). Zero starts at the caller + * minimum; one searches all partitions within the caller maximum. Later + * probing may continue when too few candidates are found. + * Any explicit effort, including 0.5, overrides nprobes in either call order. + * Explicit minimum and maximum bounds still apply. This is not a recall + * guarantee and does not change refinement or HNSW ef. The MemWAL LSM + * scanner does not yet support this option. + * + * @example + * ```ts + * table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10); + * ``` + */ + searchEffort(searchEffort: number | null): VectorQuery { + this.doVectorCall((inner) => inner.searchEffort(searchEffort)); + return this; + } + /** * Set the minimum number of probes used. * diff --git a/nodejs/src/query.rs b/nodejs/src/query.rs index 712569b2f..480426411 100644 --- a/nodejs/src/query.rs +++ b/nodejs/src/query.rs @@ -298,6 +298,16 @@ impl VectorQuery { Ok(()) } + #[napi] + pub fn search_effort(&mut self, search_effort: Option) -> napi::Result<()> { + self.inner = self + .inner + .clone() + .search_effort(search_effort) + .default_error()?; + Ok(()) + } + #[napi] pub fn minimum_nprobes(&mut self, minimum_nprobe: u32) -> napi::Result<()> { self.inner = self diff --git a/python/pyproject.toml b/python/pyproject.toml index cde7247f3..ed54ae536 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -63,7 +63,7 @@ tests = [ "polars>=0.19, <=1.32.3", "pyarrow<25", "pyarrow-stubs>=16.0", - "pylance==14.0.0b10", + "pylance==14.0.0b12", "requests>=2.31.0", "datafusion>=54,<55", "opentelemetry-sdk>=1.30.0", @@ -73,6 +73,7 @@ dev = [ "pre-commit>=3.5.0", "pyright>=1.1.350", 'typing-extensions>=4.0.0; python_version < "3.11"', + "maturin>=1.10", ] docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings-python"] clip = ["torch", "pillow>=12.1.1", "open-clip-torch"] diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index cfaa8c68a..8d0a86e4a 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -727,6 +727,7 @@ class VectorQuery: def use_lsm(self, enable: bool): ... def refine_factor(self, refine_factor: int): ... def nprobes(self, nprobes: int): ... + def search_effort(self, search_effort: Optional[float]): ... def minimum_nprobes(self, minimum_nprobes: int): ... def maximum_nprobes(self, maximum_nprobes: int): ... def bypass_vector_index(self): ... @@ -752,6 +753,7 @@ class HybridQuery: def distance_type(self, distance_type: str): ... def refine_factor(self, refine_factor: int): ... def nprobes(self, nprobes: int): ... + def search_effort(self, search_effort: Optional[float]): ... def minimum_nprobes(self, minimum_nprobes: int): ... def maximum_nprobes(self, maximum_nprobes: int): ... def bypass_vector_index(self): ... @@ -781,6 +783,7 @@ class PyQueryRequest: nprobes: Optional[int] minimum_nprobes: Optional[int] maximum_nprobes: Optional[int] + search_effort: Optional[float] lower_bound: Optional[float] upper_bound: Optional[float] ef: Optional[int] diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index ed87e441f..26c62925f 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -105,6 +105,11 @@ def _query_to_namespace_request( if not isinstance(query, Query): raise TypeError(f"Expected Query, got {type(query)}") + if query.search_effort is not None: + raise ValueError( + "namespace QueryTable does not support search_effort; " + "execute the query locally" + ) vector_request = None if query.vector is not None: diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index c7664ab78..eece390c2 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -475,8 +475,8 @@ class MatchQuery(FullTextQuery): ---------- query : str The query string to match against. - column : str - The name of the column to match against. + column : str, optional + The column to match against. None leaves column selection to query planning. boost : float, default 1.0 The boost factor for the query. The score of each matching document is multiplied by this value. @@ -505,7 +505,7 @@ class MatchQuery(FullTextQuery): """ query: str - column: str + column: Optional[str] = None boost: float = pydantic.Field(1.0, kw_only=True) fuzziness: Optional[int] = pydantic.Field(0, kw_only=True) max_expansions: int = pydantic.Field(50, kw_only=True) @@ -793,6 +793,9 @@ class Query(pydantic.BaseModel): # and all partitions could be searched if needed to satisfy the limit. maximum_nprobes: Optional[int] = None + # Optional initial IVF effort; None preserves Auto and explicit nprobes. + search_effort: Optional[float] = None + # lower bound for distance search lower_bound: Optional[float] = None @@ -854,6 +857,7 @@ class Query(pydantic.BaseModel): query.nprobes = req.nprobes query.minimum_nprobes = req.minimum_nprobes query.maximum_nprobes = req.maximum_nprobes + query.search_effort = req.search_effort query.lower_bound = req.lower_bound query.upper_bound = req.upper_bound query.ef = req.ef @@ -1635,6 +1639,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): self._nprobes = None self._minimum_nprobes = None self._maximum_nprobes = None + self._search_effort = None self._lower_bound = None self._upper_bound = None self._refine_factor = None @@ -1716,6 +1721,27 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): self._nprobes = nprobes return self + def search_effort(self, search_effort: Optional[float]) -> LanceVectorQueryBuilder: + """Set the initial IVF search effort to a finite value in [0, 1]. + + None (the default) uses Auto, internally 0.5. Zero starts at the caller + minimum; one searches all partitions within the caller maximum. Later + probing may continue when too few candidates are found. Any explicit + effort, including 0.5, overrides nprobes in either call order. Explicit + minimum and maximum bounds still apply. This is not a recall guarantee + and does not change refinement or HNSW ef. MemWAL LSM queries do not + yet support this option. + + Examples + -------- + >>> import lancedb + >>> db = lancedb.connect("memory://") + >>> table = db.create_table("effort_vectors", [{"vector": [0.1, 0.2]}]) + >>> query = table.search([0.1, 0.2]).search_effort(0.75) + """ + self._search_effort = search_effort + return self + def minimum_nprobes(self, minimum_nprobes: int) -> LanceVectorQueryBuilder: """Set the minimum number of probes to use. @@ -1858,6 +1884,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): nprobes=self._nprobes, minimum_nprobes=self._minimum_nprobes, maximum_nprobes=self._maximum_nprobes, + search_effort=self._search_effort, lower_bound=self._lower_bound, upper_bound=self._upper_bound, refine_factor=self._refine_factor, @@ -2217,6 +2244,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): self._nprobes = None self._minimum_nprobes = None self._maximum_nprobes = None + self._search_effort = None self._refine_factor = None self._distance_type = None self._phrase_query = None @@ -2524,6 +2552,32 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): self._nprobes = nprobes return self + def search_effort(self, search_effort: Optional[float]) -> LanceHybridQueryBuilder: + """Set the initial IVF search effort to a finite value in [0, 1]. + + None (the default) uses Auto, internally 0.5. Zero starts at the caller + minimum; one searches all partitions within the caller maximum. Later + probing may continue when too few candidates are found. Any explicit + effort, including 0.5, overrides nprobes in either call order. Explicit + minimum and maximum bounds still apply. This is not a recall guarantee + and does not change refinement or HNSW ef. MemWAL LSM queries do not + yet support this option. + + Examples + -------- + >>> import lancedb + >>> db = lancedb.connect("memory://") + >>> table = db.create_table( + ... "effort_hybrid", [{"vector": [0.1, 0.2], "text": "dog"}] + ... ) + >>> query = ( + ... table.search(query_type="hybrid").vector([0.1, 0.2]) + ... .text("dog").search_effort(0.75) + ... ) + """ + self._search_effort = search_effort + return self + def minimum_nprobes(self, minimum_nprobes: int) -> LanceHybridQueryBuilder: """Set the minimum number of probes to use. @@ -2802,6 +2856,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): self._vector_query.metric(self._distance_type) if self._nprobes is not None: self._vector_query.nprobes(self._nprobes) + if self._search_effort is not None: + self._vector_query.search_effort(self._search_effort) if self._minimum_nprobes is not None: self._vector_query.minimum_nprobes(self._minimum_nprobes) if self._maximum_nprobes is not None: @@ -3730,6 +3786,32 @@ class AsyncVectorQueryBase: self._inner.nprobes(nprobes) return self + def search_effort(self, search_effort: Optional[float]) -> Self: + """Set the initial IVF search effort to a finite value in [0, 1]. + + None preserves Auto (internally 0.5) and fixed nprobes. Any explicit + effort, including 0.5, overrides nprobes in either call order; explicit + minimum and maximum bounds remain effective. Zero starts at the caller + minimum and one searches all partitions within the maximum. Later + probing may continue to obtain enough candidates. This is not a recall + guarantee and does not change refinement or HNSW ef. MemWAL LSM queries + do not yet support this option. + + Examples + -------- + >>> import asyncio + >>> import lancedb + >>> async def build_query(): + ... db = await lancedb.connect_async("memory://") + ... table = await db.create_table( + ... "effort_async", [{"vector": [0.1, 0.2]}] + ... ) + ... return table.vector_search([0.1, 0.2]).search_effort(0.75) + >>> query = asyncio.run(build_query()) + """ + self._inner.search_effort(search_effort) + return self + def minimum_nprobes(self, minimum_nprobes: int) -> Self: """Set the minimum number of probes to use. diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 21838ecef..81b376ebd 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -4440,6 +4440,7 @@ class LanceTable(Table): and not self._route_pushdown_to_rust and self.current_branch() is None and query.take_offsets is None + and query.search_effort is None ): from lancedb.namespace import _execute_server_side_query @@ -6441,6 +6442,8 @@ class AsyncTable: async_query = async_query.distance_type(query.distance_type) if query.nprobes is not None: async_query = async_query.nprobes(query.nprobes) + if query.search_effort is not None: + async_query = async_query.search_effort(query.search_effort) if query.minimum_nprobes is not None and query.maximum_nprobes is not None: async_query = async_query.minimum_nprobes( query.minimum_nprobes @@ -6481,6 +6484,7 @@ class AsyncTable: ) and not self._route_pushdown_to_rust and query.take_offsets is None + and query.search_effort is None ): from lancedb.namespace import _execute_server_side_query diff --git a/python/python/tests/test_namespace.py b/python/python/tests/test_namespace.py index 76aa5fc55..520a6992d 100644 --- a/python/python/tests/test_namespace.py +++ b/python/python/tests/test_namespace.py @@ -1209,3 +1209,15 @@ def test_local_table_to_arrow_and_to_pandas_are_unchanged(tmp_path): assert table.to_arrow().column("id").to_pylist() == [1, 2] assert table.to_pandas()["id"].tolist() == [1, 2] + + +@pytest.mark.parametrize("effort", [0.0, 0.5, 1.0]) +def test_namespace_request_rejects_search_effort(effort): + from lancedb.query import Query + + with pytest.raises( + ValueError, match="namespace QueryTable does not support search_effort" + ): + _query_to_namespace_request( + ["test"], Query(vector=[1.0, 0.0], search_effort=effort) + ) diff --git a/python/python/tests/test_query.py b/python/python/tests/test_query.py index daee0dd07..74a67b909 100644 --- a/python/python/tests/test_query.py +++ b/python/python/tests/test_query.py @@ -2265,3 +2265,127 @@ def test_blob_v2_to_batches_row_id(tmp_db): assert "_rowid" in hits.column_names blobs = table.fetch_blobs("blob", hits) assert [blobs[i].as_py() for i in range(len(blobs))] == [b"one", b"two"] + + +@pytest.mark.parametrize("metric", ["l2", "cosine", "dot"]) +@pytest.mark.parametrize( + "effort,probes", [(None, 4), (0, 1), (0.25, 2), (0.5, 4), (0.75, 8), (1, 16)] +) +def test_search_effort_probe_budget(tmp_path, monkeypatch, metric, effort, probes): + import re + from lancedb.index import IvfFlat + + monkeypatch.setenv("LANCE_AUTO_PROBE_MARGIN", "0") + monkeypatch.setenv("LANCE_AUTO_MIN_INITIAL_NPROBES", "4") + monkeypatch.setenv("LANCE_AUTO_MAX_INITIAL_NPROBES", "4") + rng = np.random.default_rng(42) + vectors = np.repeat(np.eye(16, dtype=np.float32), 32, axis=0) + vectors += rng.normal(0, 0.001, vectors.shape).astype(np.float32) + data = pa.table( + { + "id": np.arange(512), + "text": ["dog"] * 512, + "vector": pa.FixedSizeListArray.from_arrays(pa.array(vectors.ravel()), 16), + } + ) + table = lancedb.connect(tmp_path).create_table("effort", data.slice(0, 256)) + table.add(data.slice(256)) + table.create_index( + "vector", config=IvfFlat(distance_type=metric, num_partitions=16) + ) + vector = np.eye(16, dtype=np.float32)[0] + + def search(value=effort): + return table.search(vector).distance_type(metric).search_effort(value).limit(8) + + def partitions(query): + vector_plan = query.analyze_plan().split("FTS Search Plan:", 1)[0] + values = re.findall(r"partitions_searched=(\d+)", vector_plan) + assert values + return sum(map(int, values)) + + assert partitions(search()) == probes + exact = set( + table.search(vector) + .distance_type(metric) + .bypass_vector_index() + .limit(8) + .to_arrow()["id"] + .to_pylist() + ) + actual = set(search().to_arrow()["id"].to_pylist()) + assert len(actual & exact) / 8 >= 0.5 + expected = 1 if effort is None else probes + assert partitions(search().nprobes(1)) == expected + assert ( + partitions( + table.search(vector) + .distance_type(metric) + .nprobes(1) + .search_effort(effort) + .limit(8) + ) + == expected + ) + assert partitions(search().nprobes(1).search_effort(None)) == 1 + assert partitions(search().maximum_nprobes(2)) <= 2 + assert partitions(search().minimum_nprobes(3).maximum_nprobes(3)) == 3 + assert partitions(search(0).nprobes(16).maximum_nprobes(0)) == 1 + if effort is None: + assert search().to_arrow() == search(0.5).to_arrow() + filtered = search(0).maximum_nprobes(2).where("id % 8 = 0", prefilter=True) + assert partitions(filtered) <= 2 + assert all(row["id"] % 8 == 0 for row in filtered.to_list()) + + if metric == "l2" and effort == 0.5: + table.create_index("text", config=FTS()) + hybrid = ( + table.search(query_type="hybrid") + .vector(vector) + .text("dog") + .nprobes(1) + .search_effort(0.5) + .limit(8) + ) + assert partitions(hybrid) == 4 + assert partitions(hybrid.search_effort(None)) == 1 + + +@pytest.mark.parametrize( + "effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")] +) +def test_search_effort_invalid_sync(table, effort): + with pytest.raises(ValueError, match="search_effort must be finite and in"): + table.search([0, 0]).search_effort(effort).to_list() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("hybrid", [False, True]) +@pytest.mark.parametrize("effort", [None, 0, 0.25, 0.5, 0.75, 1]) +async def test_search_effort_async_roundtrip(table_async, hybrid, effort): + query = table_async.vector_search([0, 0]) + if hybrid: + query = query.nearest_to_text("dog") + query = query.nprobes(5).search_effort(effort).minimum_nprobes(2).maximum_nprobes(3) + request = query.to_query_object() + assert request.search_effort == effort + assert request.nprobes == 5 + assert request.minimum_nprobes == 2 + assert request.maximum_nprobes == 3 + assert query.search_effort(None).to_query_object().search_effort is None + query.search_effort(effort) + if not hybrid: + assert len(await query.limit(1).to_list()) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("hybrid", [False, True]) +@pytest.mark.parametrize( + "effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")] +) +async def test_search_effort_invalid_async(table_async, hybrid, effort): + query = table_async.vector_search([0, 0]) + if hybrid: + query = query.nearest_to_text("dog") + with pytest.raises(ValueError, match="search_effort must be finite and in"): + query.search_effort(effort) diff --git a/python/src/query.rs b/python/src/query.rs index 88b7243d4..e5547881a 100644 --- a/python/src/query.rs +++ b/python/src/query.rs @@ -71,7 +71,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB { match class_name(&ob)?.as_str() { "MatchQuery" => { let query = ob.getattr("query")?.extract()?; - let column = ob.getattr("column")?.extract()?; + let column = ob.getattr("column")?.extract::>()?; let boost = ob.getattr("boost")?.extract()?; let fuzziness = ob.getattr("fuzziness")?.extract()?; let max_expansions = ob.getattr("max_expansions")?.extract()?; @@ -88,7 +88,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB { let mut query = MatchQuery::new(query) - .with_column(Some(column)) + .with_column(column) .with_boost(boost) .with_fuzziness(fuzziness) .with_max_expansions(max_expansions) @@ -199,7 +199,7 @@ impl<'py> IntoPyObject<'py> for PyLanceDB { } namespace .getattr(intern!(py, "MatchQuery"))? - .call((query.terms, query.column.unwrap()), Some(&kwargs)) + .call((query.terms, query.column), Some(&kwargs)) } FtsQuery::Phrase(query) => { let kwargs = PyDict::new(py); @@ -339,6 +339,7 @@ pub struct PyQueryRequest { pub nprobes: Option, pub minimum_nprobes: Option, pub maximum_nprobes: Option, + pub search_effort: Option, pub lower_bound: Option, pub upper_bound: Option, pub ef: Option, @@ -371,6 +372,7 @@ impl From for PyQueryRequest { nprobes: None, minimum_nprobes: None, maximum_nprobes: None, + search_effort: None, lower_bound: None, upper_bound: None, ef: None, @@ -399,6 +401,7 @@ impl From for PyQueryRequest { nprobes: vector_query.nprobes, minimum_nprobes: vector_query.minimum_nprobes, maximum_nprobes: vector_query.maximum_nprobes, + search_effort: vector_query.search_effort, lower_bound: vector_query.lower_bound, upper_bound: vector_query.upper_bound, ef: vector_query.ef, @@ -998,6 +1001,15 @@ impl VectorQuery { Ok(()) } + pub fn search_effort(&mut self, search_effort: Option) -> PyResult<()> { + self.inner = self + .inner + .clone() + .search_effort(search_effort) + .infer_error()?; + Ok(()) + } + pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> { self.inner = self .inner @@ -1191,6 +1203,18 @@ impl HybridQuery { self.inner_vec.nprobes(nprobe) } + pub fn search_effort(&mut self, search_effort: Option) -> PyResult<()> { + self.inner_vec.search_effort(search_effort) + } + + pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> { + self.inner_vec.minimum_nprobes(minimum_nprobes) + } + + pub fn maximum_nprobes(&mut self, maximum_nprobes: u32) -> PyResult<()> { + self.inner_vec.maximum_nprobes(maximum_nprobes) + } + pub fn ef(&mut self, ef: u32) { self.inner_vec.ef(ef); } @@ -1235,6 +1259,10 @@ impl HybridQuery { req.query_vector = vec_req.query_vector; req.column = vec_req.column; req.distance_type = vec_req.distance_type; + req.nprobes = vec_req.nprobes; + req.minimum_nprobes = vec_req.minimum_nprobes; + req.maximum_nprobes = vec_req.maximum_nprobes; + req.search_effort = vec_req.search_effort; req.ef = vec_req.ef; req.refine_factor = vec_req.refine_factor; req.lower_bound = vec_req.lower_bound; diff --git a/python/uv.lock b/python/uv.lock index b013e5982..ace8b98b9 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1900,6 +1900,7 @@ clip = [ { name = "torch" }, ] dev = [ + { name = "maturin" }, { name = "pre-commit" }, { name = "pyright" }, { name = "ruff" }, @@ -1981,6 +1982,7 @@ requires-dist = [ { name = "ibm-watsonx-ai", marker = "python_full_version >= '3.10' and extra == 'embeddings'", specifier = ">=1.1.2" }, { name = "instructorembedding", marker = "extra == 'embeddings'", specifier = ">=1.0.1" }, { name = "lance-namespace", specifier = ">=0.13.0" }, + { name = "maturin", marker = "extra == 'dev'", specifier = ">=1.10" }, { name = "mkdocs", marker = "extra == 'docs'" }, { name = "mkdocs-jupyter", marker = "extra == 'docs'" }, { name = "mkdocs-material", marker = "extra == 'docs'" }, @@ -2005,7 +2007,7 @@ requires-dist = [ { name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" }, { name = "pydantic", specifier = ">=2.7.4,<3" }, { name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" }, - { name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b10" }, + { name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b12" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" }, { name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" }, { name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" }, @@ -2156,6 +2158,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, ] +[[package]] +name = "maturin" +version = "1.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/c8/22e5e21b2679c9bce6415ca578034ca2cc9316be0642ae21e051a2d5198c/maturin-1.15.0.tar.gz", hash = "sha256:94b26cc8e8aba61a5f2099715fe640e18c5f678e9a500408b38761263954228a", size = 385504, upload-time = "2026-08-24T12:11:22.665Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/69/5c01b461044eb1f45ddcce006706eb88110c793cdb11c7ae0b5e08492e94/maturin-1.15.0-py3-none-linux_armv6l.whl", hash = "sha256:6bf6dc62e22d4dcfd5a51244ff0d58975fa4979c48209fe84159617648956d82", size = 10206220, upload-time = "2026-08-24T12:10:53.327Z" }, + { url = "https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:cd35772633f489841132bc8e71d6fc7f842df30b9c05cd5cdf1ee1ddcb744cc7", size = 19416513, upload-time = "2026-08-24T12:10:56.126Z" }, + { url = "https://files.pythonhosted.org/packages/51/36/e23a21cb34a648b711036b9b2fe1d4f3f4ee24f8db54215d73f1a9a3a3ec/maturin-1.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c40b4eae7bf5ef1f4b1af8d623fe4105016f93578fb15b764e741d08ec3b92dd", size = 10014962, upload-time = "2026-08-24T12:10:58.486Z" }, + { url = "https://files.pythonhosted.org/packages/8c/80/33b15cb2d8f30f12c807955e8f2fd775027692904e30ec0784744ce8cd83/maturin-1.15.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:7eb066372f541f8eb4909c79c5d9bd0b9e8125980bdf1ec9e8aba23c6c8d6c55", size = 10196223, upload-time = "2026-08-24T12:11:00.696Z" }, + { url = "https://files.pythonhosted.org/packages/fe/91/b495e19e2f5c503b540452b2039115e7b2363867e8c5ad4179eb752fa92c/maturin-1.15.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:653020a63525bb224e5ab0adf02e17a2e08bc86dbea7fc1399c9a56d7529b99e", size = 10541186, upload-time = "2026-08-24T12:11:02.857Z" }, + { url = "https://files.pythonhosted.org/packages/5d/2b/2abff58037188d852b124871b1f0d720e1c2bfb3d4f1b03d87c52cd66488/maturin-1.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:0ebf9767892725083138e671c34482c660317a2f3d6a29fc0e0f34e9d8c99136", size = 10083468, upload-time = "2026-08-24T12:11:05.012Z" }, + { url = "https://files.pythonhosted.org/packages/3f/07/b7e9f8be99a6627849e81ac7b6694876bce8f50a92995fe17e3cf2610f0a/maturin-1.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7ab7eebffd7b8debca2265985de4eaeb332141276d24b9560b5ad484d4b3add1", size = 10047786, upload-time = "2026-08-24T12:11:07.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ab/167e3cb7accee11b507dbe53e0e87aeccb376d44ae66284c96ee4df3a9fd/maturin-1.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:126e12e618b4db42f68c779a56d41f82a390145ba36ac3f621d057eb34f5ad9d", size = 13315332, upload-time = "2026-08-24T12:11:09.433Z" }, + { url = "https://files.pythonhosted.org/packages/14/4d/801379f646cbc6b00998e5289b0630a886be3a4ee4c75b6bc9b87478a7f1/maturin-1.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f9d33e6c3f9615c8caceecbbbd440f8eb25a3ddeb687077682cd5eca2e9ae15", size = 10807183, upload-time = "2026-08-24T12:11:11.73Z" }, + { url = "https://files.pythonhosted.org/packages/89/27/2e612e1cbd1580e9e94d4722c227b5180dca27b32b955a34b79918aa1292/maturin-1.15.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:bf29beddd0c6708f112db51d5275fc28b28b9e9c9c5faae387eaef662918b176", size = 10413274, upload-time = "2026-08-24T12:11:14.04Z" }, + { url = "https://files.pythonhosted.org/packages/70/d8/202a7b4d75a51f20f84ec9ce3b7345b12b822207072164e2b1c6ef665125/maturin-1.15.0-py3-none-win32.whl", hash = "sha256:da649988be98e87e009e51b1bf0d301b6a301bc0cecbdd60d40d8ba60748d1ca", size = 8928744, upload-time = "2026-08-24T12:11:16.269Z" }, + { url = "https://files.pythonhosted.org/packages/40/dc/4e90da594986ba78dd3bc8a5921ecdcdb11085b22b02a412caab3b225601/maturin-1.15.0-py3-none-win_amd64.whl", hash = "sha256:552c2be4afd43fe8d5c9f3ec8d4c4756d973b8dcbe94c14084390301f50243e1", size = 10335085, upload-time = "2026-08-24T12:11:18.326Z" }, + { url = "https://files.pythonhosted.org/packages/8b/10/15d4314edf130955edf2dc237aa393a8a7c10f2b9b57b89fa2f61f915659/maturin-1.15.0-py3-none-win_arm64.whl", hash = "sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48", size = 9713795, upload-time = "2026-08-24T12:11:20.83Z" }, +] + [[package]] name = "mdit-py-plugins" version = "0.6.1" @@ -3912,7 +3938,7 @@ crypto = [ [[package]] name = "pylance" -version = "14.0.0b10" +version = "14.0.0b12" source = { registry = "https://pypi.fury.io/lance-format/" } dependencies = [ { name = "lance-namespace" }, @@ -3922,12 +3948,12 @@ dependencies = [ { name = "pyarrow" }, ] wheels = [ - { url = "https://pypi.fury.io/lance-format/-/ver_4hvE/pylance-14.0.0b10-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:415b6b1b9fe735fbae7c755eb9774ae2fe0feddc90bf80bee24087cbfbfdf8c1" }, - { url = "https://pypi.fury.io/lance-format/-/ver_pM1km/pylance-14.0.0b10-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87317cad22f4d35fea11eb70aa5c0d71a0e1ea3b96de38f3deca933daa749fff" }, - { url = "https://pypi.fury.io/lance-format/-/ver_24jdyj/pylance-14.0.0b10-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6da554a2fc2197ba254676b12824d8120b48d65937697e0661a61302458a4740" }, - { url = "https://pypi.fury.io/lance-format/-/ver_XclaZ/pylance-14.0.0b10-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:36c80a801675bb9dcbd681fa99d0cdd5ca642dfb3ca08d71ac980278a2f7748b" }, - { url = "https://pypi.fury.io/lance-format/-/ver_L0yh/pylance-14.0.0b10-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:e50ad0b866faefb2fa3ae87328b93fa0d06b2bdd5bb5d0c1a6c74b907cc2fd93" }, - { url = "https://pypi.fury.io/lance-format/-/ver_2gbNr8/pylance-14.0.0b10-cp310-abi3-win_amd64.whl", hash = "sha256:3d86451fe00f71510005bb700737fb1a4f9361554541cbf802f3c5072554d014" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1Ixasc/pylance-14.0.0b12-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:84350a3c5c1eea3398f4c4fad7cfe85b831d831a5e43b5ca2a783dbdb0e11969" }, + { url = "https://pypi.fury.io/lance-format/-/ver_sCEkG/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76b386c182a6a8e7b86fbef28f84d1dd39e4626ac8681a0cf36f40a45dd7cb87" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1vdT1C/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6aa548c7937fcdfa8f1943a2d59041a42f8b500199880dbb509c7195fa253381" }, + { url = "https://pypi.fury.io/lance-format/-/ver_2fXu1V/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:79f82d6a6c50f37aab338470e3733436e6a27fbb5972d29fcf453ece56d3b3c6" }, + { url = "https://pypi.fury.io/lance-format/-/ver_24UavN/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:de0305999257002785f60169e6926896584d66f1da050bb5afd1d48c6e3d9bb9" }, + { url = "https://pypi.fury.io/lance-format/-/ver_XNynX/pylance-14.0.0b12-cp310-abi3-win_amd64.whl", hash = "sha256:4d0cdb689e898ce71e8e434d00a3b97639cea8fb97da21f1c921543be363268f" }, ] [[package]] diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index 8fb56ceca..07db69bb3 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -1127,6 +1127,11 @@ pub struct VectorQueryRequest { /// If unset, Lance's default is used. Zero represents an explicitly unbounded /// maximum on request and wire representations. pub maximum_nprobes: Option, + /// Optional initial IVF search effort in [0, 1]. + /// + /// None preserves Auto and fixed `nprobes`; any explicit value overrides + /// `nprobes` while retaining explicit minimum and maximum bounds. + pub search_effort: Option, /// The lower bound (inclusive) of the distance to search for. pub lower_bound: Option, /// The upper bound (exclusive) of the distance to search for. @@ -1153,6 +1158,7 @@ impl Default for VectorQueryRequest { nprobes: None, minimum_nprobes: None, maximum_nprobes: None, + search_effort: None, lower_bound: None, upper_bound: None, ef: None, @@ -1165,6 +1171,17 @@ impl Default for VectorQueryRequest { } impl VectorQueryRequest { + pub(crate) fn validate_search_effort(&self) -> Result<()> { + if let Some(effort) = self.search_effort + && (!effort.is_finite() || !(0.0..=1.0).contains(&effort)) + { + return Err(Error::InvalidInput { + message: format!("search_effort must be finite and in [0, 1], got {effort}"), + }); + } + Ok(()) + } + fn add_query_vector(&mut self, vector: impl IntoQueryVector) -> Result<()> { let vector = vector.to_query_vector(&DataType::Float32, "default")?; let array = @@ -1279,7 +1296,8 @@ impl VectorQuery { /// you the desired recall. /// /// LanceDB retains this as `nprobes` through local and remote request - /// construction. Lance sets both probe bounds to this value. For more + /// construction. Lance sets both probe bounds to this value only when + /// [`Self::search_effort`] is unset. For more /// fine-grained control see [`VectorQuery::minimum_nprobes`] and /// [`VectorQuery::maximum_nprobes`]. /// @@ -1294,6 +1312,31 @@ impl VectorQuery { Ok(self) } + /// Set the initial IVF search effort to a finite value in [0, 1]. + /// + /// None (the default) resolves to 0.5, preserving Auto probing. At 0 the + /// search starts at the caller minimum; at 1 it searches all partitions + /// within the caller maximum. Intermediate values interpolate around Auto. + /// Later probing may continue when too few candidates are found. + /// Any explicit value, including 0.5, overrides [`Self::nprobes`] regardless + /// of setter order. Explicit minimum and maximum bounds still apply. + /// This does not change refinement or HNSW ef and is not a recall guarantee. + /// Lance's MemWAL LSM scanner does not yet support this option. + /// + /// ``` + /// # use lancedb::{Table, query::{QueryBase, ExecutableQuery}}; + /// # async fn example(table: &Table) -> lancedb::Result<()> { + /// let results = table.query().nearest_to([0.1, 0.2].as_slice())? + /// .search_effort(0.75)?.execute().await?; + /// # Ok(()) + /// # } + /// ``` + pub fn search_effort(mut self, search_effort: impl Into>) -> Result { + self.request.search_effort = search_effort.into(); + self.request.validate_search_effort()?; + Ok(self) + } + /// Set the minimum number of partitions to search /// /// This argument is only used when the vector column has an IVF PQ index. @@ -1333,6 +1376,8 @@ impl VectorQuery { /// /// If this value is greater than minimum_nprobes then the excess partitions will /// only be searched if the initial search does not return enough results. + /// The bound applies per index segment to initial and later probing without + /// disabling Auto probing. /// /// This can be useful when there is a narrow filter to allow these queries to /// spend more time searching and avoid potential false negatives. @@ -2448,6 +2493,26 @@ mod tests { assert_eq!(query.request.nprobes, None); assert_eq!(query.request.minimum_nprobes, None); assert_eq!(query.request.maximum_nprobes, None); + assert_eq!(query.request.search_effort, None); + + let effort_query = query + .clone() + .nprobes(20) + .unwrap() + .search_effort(0.5) + .unwrap(); + assert_eq!(effort_query.request.search_effort, Some(0.5)); + assert_eq!(effort_query.request.nprobes, Some(20)); + assert_eq!(effort_query.request.minimum_nprobes, None); + assert_eq!(effort_query.request.maximum_nprobes, None); + assert_eq!( + effort_query + .search_effort(None) + .unwrap() + .request + .search_effort, + None + ); assert_eq!( *query .request diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 119577dc5..356fb1e1a 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -1137,6 +1137,7 @@ impl RemoteTable { mut body: serde_json::Value, query: &VectorQueryRequest, ) -> Result> { + query.validate_search_effort()?; self.apply_query_params(&mut body, &query.base)?; // Apply general parameters, before we dispatch based on number of query vectors. @@ -1155,6 +1156,9 @@ impl RemoteTable { if let Some(maximum_nprobes) = query.maximum_nprobes { body["maximum_nprobes"] = maximum_nprobes.into(); } + if let Some(search_effort) = query.search_effort { + body["search_effort"] = search_effort.into(); + } body["lower_bound"] = query.lower_bound.into(); body["upper_bound"] = query.upper_bound.into(); body["ef"] = query.ef.into(); @@ -6269,6 +6273,77 @@ mod tests { )); } + #[rstest] + #[case::unset(None)] + #[case::minimum(Some(0.0))] + #[case::low(Some(0.25))] + #[case::auto(Some(0.5))] + #[case::high(Some(0.75))] + #[case::maximum(Some(1.0))] + #[tokio::test] + async fn test_query_vector_search_effort_wire(#[case] effort: Option) { + let table = Table::new_with_handler("my_table", move |request| { + let body = request_body_json(&request); + assert_eq!(body["nprobes"], 7); + assert_eq!(body["minimum_nprobes"], 2); + assert_eq!(body["maximum_nprobes"], 8); + if let Some(effort) = effort { + assert_eq!(body["search_effort"], effort); + } else { + assert!(body.get("search_effort").is_none()); + } + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])), + vec![Arc::new(Int32Array::from(vec![1]))], + ) + .unwrap(); + http::Response::builder() + .status(200) + .header(CONTENT_TYPE, ARROW_FILE_CONTENT_TYPE) + .body(write_ipc_file(&batch)) + .unwrap() + }); + let batches = table + .query() + .nearest_to(vec![0.1, 0.2]) + .unwrap() + .search_effort(effort) + .unwrap() + .nprobes(7) + .unwrap() + .minimum_nprobes(2) + .unwrap() + .maximum_nprobes(Some(8)) + .unwrap() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches[0].num_rows(), 1); + } + + #[rstest] + #[case::negative(-0.1)] + #[case::too_large(1.1)] + #[case::nan(f64::NAN)] + #[case::infinity(f64::INFINITY)] + #[case::negative_infinity(f64::NEG_INFINITY)] + #[tokio::test] + async fn test_query_vector_search_effort_invalid(#[case] effort: f64) { + let table = Table::new_with_handler::<&str>("my_table", |_| { + panic!("invalid search_effort must not send a request") + }); + let result = table + .query() + .nearest_to(vec![0.1, 0.2]) + .unwrap() + .search_effort(effort); + assert!(matches!(result, Err(Error::InvalidInput { message }) + if message.contains("search_effort must be finite and in [0, 1]"))); + } + #[tokio::test] async fn test_query_vector_all_params() { let table = Table::new_with_handler("my_table", |request| { diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index 77c4ca1a5..3ab54ae07 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -106,7 +106,7 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R } fn requires_local_namespace_execution(query: &AnyQuery) -> bool { - // The namespace QueryTable request cannot represent approx_mode or use_lsm. + // The namespace QueryTable request cannot represent approx_mode, search_effort or use_lsm. // For use_lsm that is worse than a tuning miss: MemWAL read routing lives only // in `create_plan`, so a pushed-down query would return stale base-only data. if query.base().use_lsm.is_some() || query.base().take_offsets.is_some() { @@ -115,7 +115,7 @@ fn requires_local_namespace_execution(query: &AnyQuery) -> bool { matches!( query, AnyQuery::VectorQuery(vector_query) - if vector_query.approx_mode.is_some() + if vector_query.approx_mode.is_some() || vector_query.search_effort.is_some() ) } @@ -163,6 +163,7 @@ pub async fn create_plan( AnyQuery::Query(query) => VectorQueryRequest::from_plain_query(query), }; query.base.check_filter()?; + query.validate_search_effort()?; let ds_ref = table.dataset.get().await?; @@ -293,7 +294,9 @@ pub async fn create_plan( scanner.approx_mode(approx_mode.into()); } - if let Some(nprobes) = query.nprobes { + if query.search_effort.is_none() + && let Some(nprobes) = query.nprobes + { if query.maximum_nprobes == Some(0) { scanner.minimum_nprobes(nprobes); } else { @@ -306,6 +309,9 @@ pub async fn create_plan( if let Some(maximum_nprobes) = query.maximum_nprobes.filter(|nprobes| *nprobes != 0) { scanner.maximum_nprobes(maximum_nprobes); } + if let Some(search_effort) = query.search_effort { + scanner.search_effort(search_effort)?; + } } // For a batch query, `nearest` already applies k to each query vector. @@ -604,6 +610,11 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result { }; // Check for unsupported features + if vq.search_effort.is_some() { + return Err(Error::NotSupported { + message: "namespace QueryTable does not support search_effort; execute the query locally".to_string(), + }); + } if vq.base.reranker.is_some() { return Err(Error::NotSupported { message: "Reranker is not supported for server-side queries".to_string(), @@ -962,6 +973,19 @@ mod tests { let default_query = AnyQuery::VectorQuery(VectorQueryRequest::default()); assert!(!requires_local_namespace_execution(&default_query)); + for effort in [0.0, 0.5, 1.0] { + let effort_query = AnyQuery::VectorQuery(VectorQueryRequest { + query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], + search_effort: Some(effort), + ..Default::default() + }); + assert!(requires_local_namespace_execution(&effort_query)); + assert!(matches!( + convert_to_namespace_query(&effort_query), + Err(Error::NotSupported { message }) if message.contains("search_effort") + )); + } + let nprobes_query = AnyQuery::VectorQuery(VectorQueryRequest { query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], nprobes: Some(20), diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 698b9c736..20e36e95d 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -174,6 +174,9 @@ fn reject_unsupported(query: &VectorQueryRequest) -> Result<()> { // brute-force search the index-only base arm can't do. (ef / approx_mode are // recall/speed knobs and are left to no-op.) if !query.query_vector.is_empty() { + if query.search_effort.is_some() { + return unsupported("search_effort on vector search"); + } if query.lower_bound.is_some() || query.upper_bound.is_some() { return unsupported("distance_range on vector search"); } @@ -821,6 +824,19 @@ mod tests { use super::*; use lance_index::mem_wal::{CompactedSsTable, IndexCatchupProgress}; + #[test] + fn search_effort_is_not_silently_ignored() { + let query = VectorQueryRequest { + query_vector: vec![Arc::new(arrow_array::Float32Array::from(vec![1.0, 0.0]))], + search_effort: Some(0.5), + ..Default::default() + }; + assert!( + matches!(reject_unsupported(&query), Err(Error::NotSupported { message }) + if message.contains("search_effort") && message.contains("un-compacted MemWAL")) + ); + } + #[test] fn resident_index_checks_distinguish_columns_and_query_types() { use arrow_schema::Field; From 0a988a40ac501baa7ea7e837631caefc5c38b9d7 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Mon, 12 Oct 2026 11:37:20 +0000 Subject: [PATCH 2/7] =?UTF-8?q?Bump=20version:=200.41.0-beta.5=20=E2=86=92?= =?UTF-8?q?=200.41.0-beta.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index cb1b6ebd7..12d7f0804 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.41.0-beta.5" +current_version = "0.41.0-beta.6" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index dc2e67b08..3f79185d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5696,7 +5696,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.41.0-beta.5" +version = "0.41.0-beta.6" dependencies = [ "ahash", "anyhow", @@ -5793,7 +5793,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.41.0-beta.5" +version = "0.41.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -5818,7 +5818,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.41.0-beta.5" +version = "0.41.0-beta.6" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 1fb657175..e2d46921e 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.41.0-beta.5 + 0.41.0-beta.6 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index ba0d1c8f2..b312ffebf 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.5 + 0.41.0-beta.6 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 7949e9c79..18bdfe413 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.5 + 0.41.0-beta.6 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 2f38dd299..9404a0913 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.41.0-beta.5" +version = "0.41.0-beta.6" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 6b8fcf993..08e5b3a41 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index cbb0462e7..5aaa172ac 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index e752f03a8..cb1881aaf 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 43d9d2fc3..9881b85d1 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 109e469a5..29f3e7986 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 5c1723bed..f70f4eacd 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 928dfc5bb..c06f89a36 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index 121896f41..4c9be04a7 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.41.0-beta.5", + "version": "0.41.0-beta.6", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index eeb39a672..40b74520e 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.41.0-beta.5" +version = "0.41.0-beta.6" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 419577d05..3cc524bfc 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.41.0-beta.5" +version = "0.41.0-beta.6" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 25fcfc696a71ddda8a2add791671f3a3d89e22f8 Mon Sep 17 00:00:00 2001 From: XY Zhan Date: Mon, 12 Oct 2026 14:18:44 -0400 Subject: [PATCH 3/7] feat: maintain memtable index kinds from a table's registry (#4477) ## Why lance-format/lance#9803 makes the MemWAL's memtable index kinds pluggable: a `MemIndexRegistry` on the writer config decides which base-table index kinds are maintained in memory, so a kind Lance does not build in can be added by registering a plugin. LanceDB opens the MemWAL writer itself for `merge_insert`, and checks a named maintained set itself, both against Lance's built-ins only. So a kind registered outside Lance could not be named in `LsmWriteSpec::with_maintained_indexes`, and under "maintain every index" it was skipped. ## What changes - `NativeTable::set_mem_index_registry(registry)` sets the memtable index kinds a table maintains. The default is Lance's built-ins, so nothing changes unless a caller sets one. The registry checks a named maintained set (`validate_maintained_indexes_with`), is recorded on `initialize_mem_wal`, and goes on the writer config `merge_insert` opens. `lancedb::table` re-exports `MemIndexRegistry` and Lance's plugin traits (`mem_index`), so a caller needs no direct `lance` dependency. - The LSM vector read no longer checks resident memtables at all, and the `carries_vector_index` helper goes with it. Lance already compares every row of a memtable whose index can't answer the search, so no rows are missed. It still requires the base vector index to be maintained. ## Limits - The registry belongs to the table handle. Clones of a handle and its branch handles share it; a handle opened separately starts from the built-ins. Set it before the handle's first LSM write: a writer already open keeps the kinds it opened with. - The maintained set is stored with the table but the registry is not. Once a spec names an index only a registered plugin maintains, an LSM write from a handle without that registry is refused with Lance's "no registered plugin maintains" error rather than skipping the index. That includes the Python and Node bindings, which have no way to set a registry yet. - Rust only. A registry holds Rust plugin objects, which the Python, TypeScript and Java bindings cannot construct, so they get no setter; their documented behaviour (built-in kinds only) is unchanged. - Per-index plugin settings (`ShardWriterConfig::index_overrides`, from lance-format/lance#9805) are not exposed; writers use each plugin's defaults. ## Open decision Whether the registry should also be settable on the connection, so every table a connection opens carries it. This PR keeps it per table; a connection-level option would sit on top of `set_mem_index_registry` without changing it. ## Depends on Lance v14.0.0-beta.13, which `main` already uses. It includes lance-format/lance#9803 and #9805 (pluggable memtable index kinds) and #9857 (flushed generations keep the table's field ids). ## Testing - `lsm_maintains_an_index_kind_from_the_tables_registry`: a stand-in plugin maintains the base table's bitmap index as a B-tree. Naming the index is refused without the registry and accepted with it; a separately opened handle without the registry is refused an LSM write. Rows written through `merge_insert` are found by id before and after the flush. Each generation the shard manifests publish carries a bitmap index on `region`, and a filter on it is answered through that index. - `lsm_default_set_maintains_a_registry_kind_created_while_writing`: under "maintain every index", a bitmap index created after the writer opened is picked up by that writer and flushed. A clone of the table handle shares the registry. - `lsm_vector_search_compares_every_row_when_no_resident_index_answers`: with a vector plugin that answers no search, LSM vector search still returns the right rows, including a second page. - `cargo test -p lancedb --features remote --lib` passes; `cargo clippy -p lancedb --features remote --tests -- -D warnings` and `cargo fmt --all -- --check` are clean. --------- Co-authored-by: XYZhan --- rust/lancedb/src/table.rs | 55 +++- rust/lancedb/src/table/merge.rs | 458 ++++++++++++++++++++++++++-- rust/lancedb/src/table/merge/lsm.rs | 12 +- rust/lancedb/src/table/query/lsm.rs | 39 +-- 4 files changed, 506 insertions(+), 58 deletions(-) diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index b296d135d..15ca3702b 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -19,6 +19,9 @@ pub use lance::dataset::ReadParams; pub use lance::dataset::Version; use lance::dataset::WriteMode; use lance::dataset::builder::DatasetBuilder; +pub use lance::dataset::mem_wal::MemIndexRegistry; +/// The traits a memtable index plugin implements; see [`MemIndexRegistry`]. +pub use lance::dataset::mem_wal::index as mem_index; use lance::dataset::{InsertBuilder, WriteParams}; use lance::index::DatasetIndexExt; use lance::index::scalar::load_segment_params; @@ -424,7 +427,8 @@ impl LsmWriteSpec { /// changes, so an index created later is maintained too; one of a kind the /// MemWAL cannot mirror is skipped rather than failing the table. A list is /// verbatim: each name must already exist and be maintainable, and an empty - /// list maintains nothing. + /// list maintains nothing. The maintainable kinds are Lance's built-ins plus + /// any added with [`NativeTable::set_mem_index_registry`]. /// /// ``` /// # use lancedb::table::LsmWriteSpec; @@ -2590,6 +2594,9 @@ pub struct NativeTable { pub(crate) pushdown_operations: HashSet, // Read-freshness baseline; `Some` only for namespace-backed tables. freshness: Option, + // Memtable index kinds this handle's LSM writes maintain, shared with its + // clones and branch handles. + mem_index_registry: Arc>, } impl std::fmt::Debug for NativeTable { @@ -2756,9 +2763,51 @@ impl NativeTable { namespace_client, pushdown_operations, freshness: None, + mem_index_registry: Default::default(), }) } + /// Maintain the memtable index kinds in `registry` when this table writes + /// through its [`LsmWriteSpec`], and accept them when + /// [`LsmWriteSpec::with_maintained_indexes`] names one. + /// + /// The registry belongs to this handle, its clones and its branch handles, + /// and is not stored with the table: a handle opened separately starts from + /// Lance's built-ins. Set it before the handle's first LSM write; writers + /// already open keep the registry they were opened with. Local tables only, + /// reached through [`Table::as_native`]. + /// + /// ``` + /// # use std::sync::Arc; + /// # use lancedb::Table; + /// # use lancedb::table::{LsmWriteSpec, MemIndexRegistry, mem_index::MemIndexPlugin}; + /// # async fn example( + /// # table: &Table, + /// # plugin: Arc, + /// # ) -> Result<(), Box> { + /// let registry = MemIndexRegistry::default().with_plugin(plugin)?; + /// if let Some(native) = table.as_native() { + /// native.set_mem_index_registry(registry); + /// } + /// let spec = LsmWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]); + /// table.set_lsm_write_spec(spec).await?; + /// # Ok(()) + /// # } + /// ``` + pub fn set_mem_index_registry(&self, registry: MemIndexRegistry) { + *self + .mem_index_registry + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = registry; + } + + pub(crate) fn mem_index_registry(&self) -> MemIndexRegistry { + self.mem_index_registry + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + /// Set the namespace client for server-side query execution. /// /// When set, queries will be executed on the namespace server instead of locally. @@ -2786,6 +2835,7 @@ impl NativeTable { namespace_client: self.namespace_client.clone(), pushdown_operations: self.pushdown_operations.clone(), freshness: self.freshness.clone(), + mem_index_registry: self.mem_index_registry.clone(), } } @@ -2893,6 +2943,7 @@ impl NativeTable { namespace_client: stored_namespace_client, pushdown_operations, freshness: None, + mem_index_registry: Default::default(), }) } @@ -2984,6 +3035,7 @@ impl NativeTable { namespace_client, pushdown_operations, freshness: None, + mem_index_registry: Default::default(), }) } @@ -3117,6 +3169,7 @@ impl NativeTable { namespace_client: stored_namespace_client, pushdown_operations, freshness: None, + mem_index_registry: Default::default(), }) } diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 1ad084e22..28d4a0e13 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -1782,8 +1782,13 @@ mod lsm_tests { assert_eq!(rows, 1, "LSM FTS must still see base rows"); } - #[tokio::test] - async fn lsm_read_vector_search() { + /// A table whose vector index the LSM maintains: base rows filled with + /// their id, and one memtable row filled with 1000, nearest to `[1000; 8]`. + /// `registry` replaces the table's memtable kinds. + async fn lsm_vector_table( + dir: &std::path::Path, + registry: Option, + ) -> Table { use crate::index::Index; use crate::index::vector::IvfPqIndexBuilder; use arrow::array::{FixedSizeListBuilder, Float32Builder}; @@ -1815,12 +1820,7 @@ mod lsm_tests { .unwrap() }; - let dir = tempdir().unwrap(); - let conn = connect(dir.path().to_str().unwrap()) - .execute() - .await - .unwrap(); - // Base rows fill each vector with its own id (0..256); all far from 1000. + let conn = connect(dir.to_str().unwrap()).execute().await.unwrap(); let base = make_batch((0..N).map(|i| (i, i as f32)).collect()); let base_reader: Box = Box::new(RecordBatchIterator::new(vec![Ok(base)], schema.clone())); @@ -1838,13 +1838,15 @@ mod lsm_tests { .execute() .await .unwrap(); + if let Some(registry) = registry { + table.as_native().unwrap().set_mem_index_registry(registry); + } let vec_index = table.list_indices().await.unwrap()[0].name.clone(); table .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![vec_index])) .await .unwrap(); - // Insert a vector (filled with 1000) that is nearest to the query. let mut builder = table.merge_insert(&[]); builder .when_matched_update_all(None) @@ -1854,20 +1856,21 @@ mod lsm_tests { schema.clone(), )); builder.execute(insert_reader).await.unwrap(); + table + } - // KNN near [1000; DIM]: the default (auto-routed) read surfaces the - // memtable row. - let stream = table - .query() - .nearest_to(&[1000.0_f32; 8]) - .unwrap() - .limit(1) - .execute() - .await - .unwrap(); - let ids = collect_ids(stream).await; + /// The ids of the nearest row to `[1000; 8]` through the default read. + async fn nearest_ids(table: &Table) -> Vec { + let query = table.query().nearest_to(&[1000.0_f32; 8]).unwrap().limit(1); + collect_ids(query.execute().await.unwrap()).await + } + + #[tokio::test] + async fn lsm_read_vector_search() { + let dir = tempdir().unwrap(); + let table = lsm_vector_table(dir.path(), None).await; assert_eq!( - ids, + nearest_ids(&table).await, vec![9999], "LSM vector search must rank the memtable row first" ); @@ -1893,4 +1896,417 @@ mod lsm_tests { assert_eq!(ids, vec![255]); } } + + /// The built-in vector plugin, except that its index answers no search. + #[derive(Debug)] + struct AnswersNothing; + + #[derive(Debug)] + struct AnswersNothingIndex(Arc); + + fn vector_plugin() -> Arc { + Arc::new(lance::dataset::mem_wal::index::HnswMemIndexPlugin) + } + + #[async_trait::async_trait] + impl lance::dataset::mem_wal::index::MemIndexPlugin for AnswersNothing { + fn name(&self) -> &str { + "AnswersNothing" + } + fn details_message(&self) -> &str { + "VectorIndexDetails" + } + fn flush_index_type(&self) -> lance_index::IndexType { + vector_plugin().flush_index_type() + } + fn training_criteria(&self) -> lance_index::scalar::registry::TrainingCriteria { + vector_plugin().training_criteria() + } + async fn resolve( + &self, + ctx: &lance::dataset::mem_wal::index::ResolveContext<'_>, + ) -> lance_core::Result { + vector_plugin().resolve(ctx).await + } + fn validate( + &self, + ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>, + ) -> lance_core::Result<()> { + vector_plugin().validate(ctx) + } + fn create( + &self, + ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>, + ) -> lance_core::Result> { + Ok(Arc::new(AnswersNothingIndex(vector_plugin().create(ctx)?))) + } + } + + #[async_trait::async_trait] + impl lance::dataset::mem_wal::index::MemIndex for AnswersNothingIndex { + fn columns(&self) -> &[String] { + self.0.columns() + } + fn can_answer(&self, _query: &dyn lance::dataset::mem_wal::index::MemQuery) -> bool { + false + } + fn insert(&self, batch: &RecordBatch, row_offset: u64) -> lance_core::Result<()> { + self.0.insert(batch, row_offset) + } + fn insert_batches( + &self, + batches: &[lance::dataset::mem_wal::write::StoredBatch], + ) -> lance_core::Result<()> { + self.0.insert_batches(batches) + } + fn resident_bytes(&self) -> usize { + self.0.resident_bytes() + } + fn search( + &self, + _query: &dyn lance::dataset::mem_wal::index::MemQuery, + _ctx: &lance::dataset::mem_wal::index::SearchContext, + ) -> lance_core::Result> { + Ok(None) + } + async fn flush( + &self, + ctx: &lance::dataset::mem_wal::index::FlushContext<'_>, + ) -> lance_core::Result { + self.0.flush(ctx).await + } + } + + /// A resident memtable whose vector index answers nothing still serves an + /// LSM vector search: Lance compares every row instead. + #[tokio::test] + async fn lsm_vector_search_compares_every_row_when_no_resident_index_answers() { + let dir = tempdir().unwrap(); + let mut registry = lance::dataset::mem_wal::MemIndexRegistry::default(); + registry.replace_plugin(Arc::new(AnswersNothing)).unwrap(); + let table = lsm_vector_table(dir.path(), Some(registry)).await; + assert_eq!(nearest_ids(&table).await, vec![9999]); + + // The second page starts after the memtable row. + let page = table + .query() + .nearest_to(&[1000.0_f32; 8]) + .unwrap() + .limit(2) + .offset(1); + assert_eq!( + collect_ids(page.execute().await.unwrap()).await, + vec![255, 254] + ); + } + + /// Claims the base table's bitmap index and keeps it as a B-tree, whose + /// flush rows are in the shape the bitmap trainer takes. + #[derive(Debug)] + struct BitmapAsBTree; + + fn btree_plugin() -> Arc { + lance::dataset::mem_wal::MemIndexRegistry::default() + .plugin_for_details_url("/lance.table.BTreeIndexDetails") + .unwrap() + .clone() + } + + #[async_trait::async_trait] + impl lance::dataset::mem_wal::index::MemIndexPlugin for BitmapAsBTree { + fn name(&self) -> &str { + "BitmapAsBTree" + } + fn details_message(&self) -> &str { + "BitmapIndexDetails" + } + fn flush_index_type(&self) -> lance_index::IndexType { + lance_index::IndexType::Bitmap + } + fn training_criteria(&self) -> lance_index::scalar::registry::TrainingCriteria { + btree_plugin().training_criteria() + } + fn validate( + &self, + ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>, + ) -> lance_core::Result<()> { + btree_plugin().validate(ctx) + } + fn create( + &self, + ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>, + ) -> lance_core::Result> { + btree_plugin().create(ctx) + } + } + + /// A kind Lance has no memtable index for is refused when named until the + /// table's registry maintains it; then it is maintained, flushed with each + /// generation, and answers searches on the flushed generations. + #[tokio::test] + async fn lsm_maintains_an_index_kind_from_the_tables_registry() { + let dir = tempdir().unwrap(); + let conn = connect(dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let table = conn + .create_table("t", id_region_reader(vec![(1, "us"), (2, "eu"), (3, "us")])) + .execute() + .await + .unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); + table + .create_index(&["region"], crate::index::Index::Bitmap(Default::default())) + .name("region_bitmap".to_string()) + .execute() + .await + .unwrap(); + let spec = + || LsmWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]); + + let refused = table.set_lsm_write_spec(spec()).await.unwrap_err(); + assert!( + refused.to_string().contains("region_bitmap") + && refused + .to_string() + .contains("no registered plugin maintains"), + "{refused:?}" + ); + + let registry = lance::dataset::mem_wal::MemIndexRegistry::default() + .with_plugin(Arc::new(BitmapAsBTree)) + .unwrap(); + table.as_native().unwrap().set_mem_index_registry(registry); + table.set_lsm_write_spec(spec()).await.unwrap(); + + // The registry is not stored with the table. + let other = conn.open_table("t").execute().await.unwrap(); + let mut builder = other.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + let refused = builder + .execute(id_region_reader(vec![(20, "us")])) + .await + .unwrap_err(); + assert!( + refused + .to_string() + .contains("no registered plugin maintains"), + "{refused:?}" + ); + + let mut builder = table.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + builder + .execute(id_region_reader(vec![ + (10, "fresh"), + (11, "fresh"), + (12, "eu"), + ])) + .await + .unwrap(); + + let fresh = || async { + let query = table.query().only_if("region = 'fresh'"); + let mut ids = collect_ids(query.execute().await.unwrap()).await; + ids.sort_unstable(); + ids + }; + assert_eq!(fresh().await, vec![10, 11]); + let (_, _, memtables) = table + .as_native() + .unwrap() + .dataset + .shard_writer() + .read_snapshot() + .await + .unwrap() + .expect("the merge opened a writer"); + let active = memtables.expect("the writer keeps memtables").active; + assert!( + active + .index_store + .index_names() + .contains(&"region_bitmap".to_string()), + "the memtable maintains the plugin's index: {:?}", + active.index_store.index_names() + ); + + table.close_lsm_writers().await.unwrap(); + assert_eq!(fresh().await, vec![10, 11]); + assert_generations_answer_through_region_bitmap(&table, vec![10, 11]).await; + } + + /// The generations each shard's manifest publishes. + async fn published_generations(table: &Table) -> Vec { + use lance::dataset::mem_wal::ShardManifestStore; + + let uri = table.uri().await.unwrap(); + let (store, base) = lance_io::object_store::ObjectStore::from_uri(&uri) + .await + .unwrap(); + let mem_wal = std::path::Path::new(&uri).join("_mem_wal"); + let mut generations = Vec::new(); + for shard in std::fs::read_dir(&mem_wal).unwrap() { + let Ok(shard_id) = shard.unwrap().file_name().to_string_lossy().parse() else { + continue; + }; + let manifest = ShardManifestStore::new(store.clone(), &base, shard_id, 2) + .latest() + .await + .unwrap() + .expect("a shard directory has a manifest"); + for sstable in manifest.sstables { + let path = mem_wal.join(shard_id.to_string()).join(&sstable.path); + generations.push(lance::Dataset::open(path.to_str().unwrap()).await.unwrap()); + } + } + generations + } + + /// Every published generation carries `region_bitmap` as a bitmap index on + /// `region`, and a search for `region = 'fresh'` on it uses that index. + async fn assert_generations_answer_through_region_bitmap(table: &Table, fresh: Vec) { + use lance::index::DatasetIndexExt; + + let generations = published_generations(table).await; + assert!( + !generations.is_empty(), + "closing the writer flushed a generation" + ); + let mut found = Vec::new(); + for generation in generations { + let region = generation.schema().field("region").unwrap().id; + let indices = generation.load_indices().await.unwrap(); + let index = indices + .iter() + .find(|index| index.name == "region_bitmap") + .expect("the generation carries the index the plugin trained"); + assert_eq!(index.fields, vec![region]); + assert!( + index + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("BitmapIndexDetails")), + "{:?}", + index.index_details + ); + + let mut scan = generation.scan(); + scan.filter("region = 'fresh'").unwrap(); + let plan = scan.explain_plan(false).await.unwrap(); + assert!(plan.contains("ScalarIndexQuery"), "{plan}"); + let batch = scan.try_into_batch().await.unwrap(); + found.extend( + batch["id"] + .as_primitive::() + .values() + .iter() + .copied(), + ); + } + found.sort_unstable(); + assert_eq!(found, fresh); + } + + /// Under the default set, a kind only the registry maintains is picked up + /// when its index is created after the writer opened, and the table's + /// clones share the registry. + #[tokio::test] + async fn lsm_default_set_maintains_a_registry_kind_created_while_writing() { + let dir = tempdir().unwrap(); + let conn = connect(dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let table = conn + .create_table("t", id_region_reader(vec![(1, "us"), (2, "eu")])) + .execute() + .await + .unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); + let registry = lance::dataset::mem_wal::MemIndexRegistry::default() + .with_plugin(Arc::new(BitmapAsBTree)) + .unwrap(); + table.as_native().unwrap().set_mem_index_registry(registry); + let clone = table.clone(); + assert!( + clone + .as_native() + .unwrap() + .mem_index_registry() + .plugin_for_details_url("/lance.table.BitmapIndexDetails") + .is_some(), + "a clone shares the handle's registry" + ); + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + + let upsert = |rows: Vec<(i64, &'static str)>| { + let table = table.clone(); + async move { + let mut builder = table.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + builder.execute(id_region_reader(rows)).await.unwrap(); + } + }; + // Opens the writer before the table has a bitmap index. + upsert(vec![(3, "us")]).await; + table + .create_index(&["region"], crate::index::Index::Bitmap(Default::default())) + .name("region_bitmap".to_string()) + .execute() + .await + .unwrap(); + upsert(vec![(10, "fresh"), (11, "fresh")]).await; + + let (_, _, memtables) = table + .as_native() + .unwrap() + .dataset + .shard_writer() + .read_snapshot() + .await + .unwrap() + .expect("the merge opened a writer"); + let active = memtables.expect("the writer keeps memtables").active; + assert!( + active + .index_store + .index_names() + .contains(&"region_bitmap".to_string()), + "the open writer picked up the new index: {:?}", + active.index_store.index_names() + ); + + table.close_lsm_writers().await.unwrap(); + // Only the generation written after the index exists carries it. + let generations = published_generations(&table).await; + let mut carrying = 0; + for generation in &generations { + use lance::index::DatasetIndexExt; + if generation + .load_indices() + .await + .unwrap() + .iter() + .any(|index| index.name == "region_bitmap") + { + carrying += 1; + } + } + assert!( + carrying >= 1, + "{} generations, none carry the index", + generations.len() + ); + } } diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index 0f86fdcac..33bf54535 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -29,7 +29,7 @@ use arrow_schema::{DataType, Schema as ArrowSchema, SchemaRef}; use lance::Dataset; use lance::dataset::mem_wal::{ DatasetMemWalExt, ShardWriter, ShardWriterConfig, evaluate_sharding_spec, - validate_maintained_indexes, + validate_maintained_indexes_with, }; use lance::index::DatasetIndexExt; use lance_core::datatypes::Schema as LanceSchema; @@ -103,7 +103,8 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) } } let dataset = table.dataset.get().await?; - validate_maintained_indexes(&dataset, requested).await?; + validate_maintained_indexes_with(&dataset, requested, &table.mem_index_registry()) + .await?; Some(requested.to_vec()) } None => None, @@ -126,7 +127,9 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) .into(), }); } - let mut builder = dataset.initialize_mem_wal(); + let mut builder = dataset + .initialize_mem_wal() + .mem_index_registry(table.mem_index_registry()); let writer_config_defaults = match spec { LsmWriteSpec::Bucket { column, @@ -725,7 +728,8 @@ pub(crate) async fn execute_lsm_merge_insert( return Ok(lsm_merge_result(0)); }; - let config = shard_writer_config_from_defaults(&plan.writer_config_defaults); + let mut config = shard_writer_config_from_defaults(&plan.writer_config_defaults); + config.mem_index_registry = table.mem_index_registry(); let writer = table .dataset .shard_writer() diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 20e36e95d..4798da431 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -33,7 +33,6 @@ use datafusion_physical_plan::expressions::Column; use datafusion_physical_plan::projection::ProjectionExec; use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr}; use lance::Dataset; -use lance::dataset::mem_wal::index::{IndexStore, VectorMemQuery}; use lance::dataset::mem_wal::scanner::{InMemoryMemTableRef, InMemoryMemTables}; use lance::dataset::mem_wal::{ DatasetMemWalExt, LsmScanner, ShardManifestStore, ShardSnapshot, ShardWriterConfig, @@ -711,17 +710,14 @@ async fn vector_plan( // The base arm relies on the column's vector index (`fast_search`). Unmaintained, // its catch-up is untracked and exclusion falls back to the compaction watermark, - // dropping compacted SSTables the lagging base index has not re-indexed; and a - // resident MemTable built before the index joined the set carries none either. + // dropping compacted SSTables the lagging base index has not re-indexed. // Reject rather than silently omit rows, mirroring the FTS arm. - if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? - || !resident_memtables_carry(&in_memory, |memtable| { - carries_vector_index(&memtable.index_store, &column) - }) - { + // A resident MemTable needs no index of its own: one that cannot answer + // compares every row. + if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only" + "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only" ), }); } @@ -764,22 +760,6 @@ async fn vector_plan( Ok(scanner.create_plan().await?) } -/// Probe for a resident vector index without imposing a distance metric, matching -/// the presence check used before Lance introduced query-based index lookup. -fn carries_vector_index(store: &IndexStore, column: &str) -> bool { - let probe = VectorMemQuery { - vector: arrow_array::FixedSizeListArray::new_null( - Arc::new(arrow_schema::Field::new("item", DataType::Float32, true)), - 1, - 1, - ), - k: 1, - ef: None, - distance_type: None, - }; - store.index_answering(column, &probe).is_some() -} - /// Whether `column` stores binary (uint8) vectors, which the LSM vector planner /// does not support. fn is_binary_vector_column(schema: &ArrowSchema, column: &str) -> bool { @@ -838,9 +818,9 @@ mod tests { } #[test] - fn resident_index_checks_distinguish_columns_and_query_types() { + fn resident_fts_check_distinguishes_columns() { use arrow_schema::Field; - use lance::dataset::mem_wal::index::MemIndexSpec; + use lance::dataset::mem_wal::index::{IndexStore, MemIndexSpec}; let schema = ArrowSchema::new(vec![ Field::new("id", DataType::Int32, false), @@ -863,16 +843,11 @@ mod tests { ), ]; let store = IndexStore::from_specs(&specs, &schema, 1_000, 16).unwrap(); - assert!(carries_vector_index(&store, "vector")); - for column in ["id", "text", "missing"] { - assert!(!carries_vector_index(&store, column)); - } assert!(!store.fts_granularities_on("text").is_empty()); for column in ["id", "vector", "missing"] { assert!(store.fts_granularities_on(column).is_empty()); } let empty = IndexStore::from_specs(&[], &schema, 1_000, 16).unwrap(); - assert!(!carries_vector_index(&empty, "vector")); assert!(empty.fts_granularities_on("text").is_empty()); } From 792ffd108251e0b4818485f2c171bc33e5dec39d Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:56:39 -0700 Subject: [PATCH 4/7] fix(python): materialize nested blob bytes in pandas (#4305) --- python/python/lancedb/_blob.py | 63 ++++++++++++++--- python/python/tests/test_blob.py | 116 +++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 11 deletions(-) diff --git a/python/python/lancedb/_blob.py b/python/python/lancedb/_blob.py index dc4ed37df..ee78a7312 100644 --- a/python/python/lancedb/_blob.py +++ b/python/python/lancedb/_blob.py @@ -110,7 +110,9 @@ def projection_includes_blob_column( if projection is None: return True for output, source in _iter_projection_pairs(projection): - if output in columns or source in columns: + if output in columns or any( + column == source or column.startswith(f"{source}.") for column in columns + ): return True return False @@ -122,14 +124,16 @@ def blob_v2_projection_sources( blob_columns = row_addressable_blob_v2_paths(schema) if not blob_columns: return {} - columns = set(blob_columns) if projection is None: return {column: column for column in blob_columns} - return { - output: source - for output, source in _iter_projection_pairs(projection) - if source in columns - } + sources = {} + for output, source in _iter_projection_pairs(projection): + for column in blob_columns: + if column == source: + sources[output] = column + elif column.startswith(f"{source}."): + sources[f"{output}{column[len(source) :]}"] = column + return sources def v2_projection_needs_row_id( @@ -180,7 +184,7 @@ async def replace_v2_blob_columns_with_bytes( fetch_blobs: FetchBlobsAsync, ) -> pa.Table: for output_name, source_name in blob_sources.items(): - if output_name not in tbl.column_names: + if not _has_column_path(tbl.schema, output_name): continue blobs = await fetch_blobs(source_name, tbl) tbl = _set_blob_column(tbl, output_name, blobs) @@ -193,7 +197,7 @@ def replace_v2_blob_columns_with_bytes_sync( fetch_blobs: FetchBlobsSync, ) -> pa.Table: for output_name, source_name in blob_sources.items(): - if output_name not in tbl.column_names: + if not _has_column_path(tbl.schema, output_name): continue blobs = fetch_blobs(source_name, tbl) tbl = _set_blob_column(tbl, output_name, blobs) @@ -287,9 +291,46 @@ def _iter_projection_pairs( yield name, source if source is not None else expr.to_sql() -def _set_blob_column(tbl: pa.Table, output_name: str, blobs: pa.Array) -> pa.Table: +def _has_column_path(schema: pa.Schema, path: str) -> bool: + if schema.get_field_index(path) >= 0: + return True + top_name, *rest = path.split(".") + index = schema.get_field_index(top_name) + if index < 0: + return False + field = schema.field(index) + for name in rest: + if not pa.types.is_struct(field.type): + return False + index = field.type.get_field_index(name) + if index < 0: + return False + field = field.type.field(index) + return True + + +def _set_blob_column( + tbl: pa.Table, output_name: str, blobs: pa.Array | pa.ChunkedArray +) -> pa.Table: index = tbl.schema.get_field_index(output_name) - return tbl.set_column(index, pa.field(output_name, blobs.type), [blobs]) + if index >= 0: + return tbl.set_column(index, pa.field(output_name, blobs.type), [blobs]) + + parent_path, leaf_name = output_name.rsplit(".", 1) + if isinstance(blobs, pa.ChunkedArray): + blobs = blobs.combine_chunks() + + def replace_blob(children: list, child_fields: list) -> None: + for i, field in enumerate(child_fields): + if field.name == leaf_name: + children[i] = blobs + child_fields[i] = pa.field( + leaf_name, blobs.type, nullable=field.nullable + ) + return + raise KeyError(f"Blob field '{leaf_name}' not found in struct '{parent_path}'") + + return _transform_struct_column(tbl, parent_path, replace_blob) def _embed_row_id_in_column(tbl: pa.Table, path: str, row_ids: pa.Array) -> pa.Table: diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 72107364c..2bb5b6bea 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -15,6 +15,7 @@ from lance.blob import Blob, BlobType as LanceBlobType import lancedb from lancedb._blob import ( + _set_blob_column, blob_v2_projection_sources, read_row_ids_from_hits, stash_auto_row_ids, @@ -297,6 +298,121 @@ def test_blob_v2_projection_sources_use_typed_column_name(): } +def test_blob_v2_projection_sources_include_selected_struct(): + schema = pa.schema( + [ + pa.field( + "info", pa.struct([pa.field("name", pa.string()), lancedb.blob("blob")]) + ) + ] + ) + + assert blob_v2_projection_sources(schema, ["info"]) == {"info.blob": "info.blob"} + assert blob_v2_projection_sources(schema, {"renamed": "info"}) == { + "renamed.blob": "info.blob" + } + + +@pytest.mark.parametrize("parent_path", ["info", "outer.info"]) +def test_set_blob_column_missing_nested_field_raises(parent_path): + row = {"name": "first"} + for name in reversed(parent_path.split(".")): + row = {name: row} + table = pa.Table.from_pylist([row]) + blobs = pa.array([b"hello"], type=pa.large_binary()) + + with pytest.raises(KeyError) as exc_info: + _set_blob_column(table, f"{parent_path}.blob", blobs) + + assert parent_path in str(exc_info.value) + assert "'blob'" in str(exc_info.value) + + +def _nested_blob_table(db, name): + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field( + "info", pa.struct([pa.field("name", pa.string()), lancedb.blob("blob")]) + ), + ] + ) + table = db.create_table(name, schema=schema) + info = pa.StructArray.from_arrays( + [ + pa.array(["first", "second", "third"]), + pa.array([b"hello", None, b"hidden"], type=pa.large_binary()), + ], + names=["name", "blob"], + mask=pa.array([False, False, True]), + ) + table.add(pa.table({"id": pa.array([0, 1, 2]), "info": info})) + return table + + +def test_nested_blob_to_pandas_bytes(tmp_path): + table = _nested_blob_table(lancedb.connect(tmp_path), "nested_pandas_bytes") + expected = [ + {"name": "first", "blob": b"hello"}, + {"name": "second", "blob": None}, + None, + ] + + assert table.to_pandas(blob_mode="bytes")["info"].tolist() == expected + assert table.search().to_pandas(blob_mode="bytes")["info"].tolist() == expected + assert ( + table.search().select(["info"]).to_pandas(blob_mode="bytes")["info"].tolist() + == expected + ) + assert ( + table.search() + .select({"renamed": "info"}) + .to_pandas(blob_mode="bytes")["renamed"] + .tolist() + == expected + ) + assert table.search().select({"b": "info.blob"}).to_pandas(blob_mode="bytes")[ + "b" + ].tolist() == [b"hello", None, None] + + +@pytest.mark.asyncio +async def test_nested_blob_to_pandas_bytes_async(tmp_path): + db = await lancedb.connect_async(tmp_path) + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field( + "info", pa.struct([pa.field("name", pa.string()), lancedb.blob("blob")]) + ), + ] + ) + table = await db.create_table("nested_pandas_bytes_async", schema=schema) + await table.add( + pa.table( + { + "id": pa.array([0, 1]), + "info": pa.StructArray.from_arrays( + [ + pa.array(["first", "second"]), + pa.array([b"hello", None], type=pa.large_binary()), + ], + names=["name", "blob"], + ), + } + ) + ) + expected = [ + {"name": "first", "blob": b"hello"}, + {"name": "second", "blob": None}, + ] + + assert (await table.to_pandas(blob_mode="bytes"))["info"].tolist() == expected + assert (await table.query().to_pandas(blob_mode="bytes"))[ + "info" + ].tolist() == expected + + def _legacy_v1_table(name): # Legacy v1 blob columns are only writable at file version <= 2.1. db = lancedb.connect( From b5565e19058a06e3c185d4e58aef510ab9b3b27d Mon Sep 17 00:00:00 2001 From: Daniel Peng <97350516+original4422@users.noreply.github.com> Date: Tue, 13 Oct 2026 05:48:13 +0800 Subject: [PATCH 5/7] docs: fix SDK docs working directory instructions (#4525) --- docs/README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/README.md b/docs/README.md index bce1ec668..c5a99943d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,18 +15,21 @@ unreleased features. ### Preview the docs +Run these commands from the LanceDB repository root. The subshell changes into +`docs` to load `mkdocs.yml`, while leaving your shell in the repository root. + ```shell -cd docs -mkdocs serve +(cd docs && PYTHONPATH=. mkdocs serve) ``` If you want to just generate the HTML files: ```shell -PYTHONPATH=. mkdocs build -f docs/mkdocs.yml +(cd docs && PYTHONPATH=. mkdocs build) ``` -If successful, you should see a `docs/site` directory that you can verify locally. +If successful, you should see a `docs/site` directory in the repository root that +you can verify locally. ## Adding examples From 3837b7a6221fb30edde55fc04797a94d0efeb04d Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Mon, 12 Oct 2026 19:19:32 -0500 Subject: [PATCH 6/7] feat(datafusion): let BaseTableAdapter set the MemWAL read routing (#4488) WAL queries have the ability to bypass WAL by using `use_lsm(bool)` to determine whether to explicitly use or not use it. This is exposed on many of the entry-points but not on the `BaseTableAdapter`. This PR adds that function to allow WAL use configuration. Co-authored-by: Claude Opus 5.5 (1M context) --- rust/lancedb/src/table/datafusion.rs | 10 ++++++++ rust/lancedb/src/table/merge.rs | 35 ++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/rust/lancedb/src/table/datafusion.rs b/rust/lancedb/src/table/datafusion.rs index 26c423497..0b820494f 100644 --- a/rust/lancedb/src/table/datafusion.rs +++ b/rust/lancedb/src/table/datafusion.rs @@ -166,6 +166,7 @@ pub struct BaseTableAdapter { table: Arc, schema: Arc, fts_query: Option, + use_lsm: Option, } impl BaseTableAdapter { @@ -181,9 +182,16 @@ impl BaseTableAdapter { table, schema: Arc::new(schema), fts_query: None, + use_lsm: None, }) } + /// Set the MemWAL read routing of every scan; see [`QueryRequest::use_lsm`]. + pub fn with_use_lsm(mut self, use_lsm: Option) -> Self { + self.use_lsm = use_lsm; + self + } + /// Create a new adapter with an FTS query applied. pub fn with_fts_query(&self, fts_query: FullTextSearchQuery) -> Self { // Add _score column to the schema @@ -196,6 +204,7 @@ impl BaseTableAdapter { table: self.table.clone(), schema, fts_query: Some(fts_query), + use_lsm: self.use_lsm, } } } @@ -223,6 +232,7 @@ impl TableProvider for BaseTableAdapter { let mut query = QueryRequest { full_text_search: self.fts_query.clone(), disable_scoring_autoprojection: disable_scoring, + use_lsm: self.use_lsm, ..Default::default() }; diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 28d4a0e13..fbe82a198 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -1349,6 +1349,41 @@ mod lsm_tests { ); } + #[tokio::test] + async fn base_table_adapter_use_lsm_false_scans_the_base_only() { + use crate::table::datafusion::BaseTableAdapter; + use datafusion::prelude::SessionContext; + + let dir = tempdir().unwrap(); + let table = id_value_table(&dir).await; // base: ids 1,2,3 + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + // Ids 4,5 land in the active memtable, not the base table. + lsm_upsert(&table, vec![4, 5]).await; + + let adapter = BaseTableAdapter::try_new(table.base_table().clone()) + .await + .unwrap() + .with_use_lsm(Some(false)); + let ctx = SessionContext::new(); + ctx.register_table("base_only", Arc::new(adapter)).unwrap(); + let batches = ctx + .sql("SELECT id FROM base_only") + .await + .unwrap() + .collect() + .await + .unwrap(); + let mut ids: Vec = batches + .iter() + .flat_map(|b| b.column(0).as_primitive::().values().to_vec()) + .collect(); + ids.sort(); + assert_eq!(ids, vec![1, 2, 3]); + } + #[tokio::test] async fn query_snapshot_preserves_lsm_read_semantics() { let dir = tempdir().unwrap(); From 1ea47a50559f02a0251d9d43237259b559606abd Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Tue, 13 Oct 2026 09:10:55 -0700 Subject: [PATCH 7/7] chore: update lance dependency to v14.0.0-beta.15 (#4527) Updates the Rust workspace Lance dependencies, Cargo lockfile, and Java lance-core dependency to [v14.0.0-beta.15](https://github.com/lance-format/lance/releases/tag/v14.0.0-beta.15). No compatibility fixes were required. Validation passed: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and `git diff --check`. --------- Co-authored-by: Daniel Rammer Co-authored-by: Claude Opus 5.5 (1M context) --- Cargo.lock | 92 +++++++++++++++---------------- Cargo.toml | 28 +++++----- java/pom.xml | 2 +- rust/lancedb/src/table/refresh.rs | 4 +- 4 files changed, 63 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f79185d9..ffed35bcc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arc-swap", "arrow", @@ -5133,8 +5133,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow-array", "arrow-buffer", @@ -5156,7 +5156,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow-array", "arrow-buffer", @@ -5170,7 +5170,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow-array", "arrow-schema", @@ -5179,8 +5179,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrayref", "crunchy", @@ -5190,8 +5190,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow-array", "arrow-buffer", @@ -5228,8 +5228,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow", "arrow-array", @@ -5260,8 +5260,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow", "arrow-array", @@ -5278,8 +5278,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "proc-macro2", "quote", @@ -5288,8 +5288,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow-arith", "arrow-array", @@ -5322,8 +5322,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow-arith", "arrow-array", @@ -5355,8 +5355,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "datafusion", "geo-traits", @@ -5370,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arc-swap", "arrow", @@ -5442,8 +5442,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow-array", "arrow-schema", @@ -5466,8 +5466,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow", "arrow-array", @@ -5509,8 +5509,8 @@ dependencies = [ [[package]] name = "lance-jsonb" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "byteorder", "ethnum", @@ -5527,8 +5527,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow-array", "arrow-schema", @@ -5542,8 +5542,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow", "async-trait", @@ -5557,8 +5557,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arc-swap", "arrow", @@ -5612,8 +5612,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow-array", "arrow-buffer", @@ -5627,8 +5627,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow", "arrow-array", @@ -5668,8 +5668,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "arrow-array", "arrow-schema", @@ -5682,8 +5682,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.13" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a" +version = "14.0.0-beta.15" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index d5821d993..78ed6bea8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 18bdfe413..33446040b 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 14.0.0-beta.13 + 14.0.0-beta.15 2.18.11 false 2.30.0 diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 884648f54..0119ba36d 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -1847,8 +1847,8 @@ mod tests { assert_eq!( layouts, vec![ - (1, Some(BlobKind::Managed)), - (2, Some(BlobKind::Managed)), + (1, Some(BlobKind::Dedicated)), + (2, Some(BlobKind::Packed)), (3, Some(BlobKind::Inline)), (4, None), ]