From c3176a47ce5206dcecc44605e20a1cd710a278d0 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:13:54 +0800 Subject: [PATCH 01/49] fix(python): report unsplittable IVF partition errors (#3846) ## Summary - add Python regression coverage for an IVF build that cannot form all requested non-empty partitions - verify hierarchical k-means returns an actionable RuntimeError instead of panicking or silently creating a degenerate index - exercise the current Lance v10.1.0-beta.1 dependency, which contains the upstream error-return fix ## Root cause Hierarchical k-means previously guarded a shortfall in generated clusters with only a debug assertion. Debug builds panicked, while release builds could silently publish an index with many empty partitions. The upstream Lance fix now returns a descriptive error and is already included in the dependency pinned on main; this test locks in propagation through the LanceDB Python API. ## Validation - uv run --extra tests pytest python/tests/test_index.py -q (24 passed) - uv run --extra tests pytest python/tests/test_index.py::test_create_ivf_index_reports_unsplittable_partitions -q (1 passed) - python/.venv/bin/ruff format python/python/tests/test_index.py - python/.venv/bin/ruff check . - git diff --check Fixes #3649 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_index.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/python/python/tests/test_index.py b/python/python/tests/test_index.py index 1cf2c733c..94268a53e 100644 --- a/python/python/tests/test_index.py +++ b/python/python/tests/test_index.py @@ -372,6 +372,31 @@ async def test_create_vector_index(some_table: AsyncTable): assert stats.num_indices == 1 +@pytest.mark.asyncio +async def test_create_ivf_index_reports_unsplittable_partitions(db_async): + dim = 8 + num_partitions = 300 # More than 256 selects hierarchical k-means. + base_vectors = [[float(row == column) for column in range(dim)] for row in range(5)] + vectors = pa.array(base_vectors * 200, pa.list_(pa.float32(), dim)) + table = await db_async.create_table( + "unsplittable_partitions", + pa.table({"vector": vectors}), + ) + + error_pattern = ( + rf"Cannot create {num_partitions} IVF partitions: k-means could only form" + ) + with pytest.raises(RuntimeError, match=error_pattern): + await table.create_index( + "vector", + config=IvfFlat( + distance_type="dot", + num_partitions=num_partitions, + max_iterations=10, + ), + ) + + @pytest.mark.asyncio async def test_create_4bit_ivfpq_index(some_table: AsyncTable): # Can create From 9e2e711c7a2311b35e6e5795c1bb71ebb5090b6f Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:36:05 +0800 Subject: [PATCH 02/49] test(python): cover OpenAI registry variable round-trip (#3863) ## Summary - replace the synthetic registry-variable metadata test with the OpenAI embedding function reported in #2387 - verify the resolved API key survives table metadata reconstruction - assert the OpenAI client receives the resolved key while serialized metadata retains the variable reference ## Root cause LanceDB 0.22.0 reconstructed embedding functions from table metadata with the model constructor, bypassing EmbeddingFunction.create and leaving the literal $var:api_key placeholder in OpenAI configuration. The production path was corrected for duplicate #2181 by #2640; this change gives that fix direct, network-free OpenAI regression coverage for #2387. ## Validation - uv run --extra tests pytest python/tests/test_embeddings.py -q (13 passed, 9 skipped) - uv run --project python --extra dev ruff check . - uv run --project python --extra dev ruff format --check python/python/tests/test_embeddings.py - git diff --check Fixes #2387 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_embeddings.py | 41 +++++++++----------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/python/python/tests/test_embeddings.py b/python/python/tests/test_embeddings.py index 5efb7d98a..f2d8971c3 100644 --- a/python/python/tests/test_embeddings.py +++ b/python/python/tests/test_embeddings.py @@ -115,34 +115,16 @@ def test_embedding_function_variables(): assert func.safe_model_dump()["secret_key"] == "$var:secret" -def test_parse_functions_with_variables(): - @register("variable-parsing-test") - class VariableParsingFunction(TextEmbeddingFunction): - api_key: str - base_url: Optional[str] = None - - @staticmethod - def sensitive_keys(): - return ["api_key"] - - def ndims(self): - return 10 - - def generate_embeddings(self, texts): - # Mock implementation that just returns random embeddings - # In real usage, this would use the api_key to call an API - return [np.random.rand(self.ndims()).tolist() for _ in texts] - +def test_openai_variables_survive_metadata_round_trip(): registry = EmbeddingFunctionRegistry.get_instance() registry.set_var("test_api_key", "sk-test-key-12345") - registry.set_var("test_base_url", "https://api.example.com") conf = EmbeddingFunctionConfig( source_column="text", vector_column="vector", - function=registry.get("variable-parsing-test").create( - api_key="$var:test_api_key", base_url="$var:test_base_url" + function=registry.get("openai").create( + api_key="$var:test_api_key", base_url="https://api.example.com" ), ) @@ -150,7 +132,10 @@ def test_parse_functions_with_variables(): # Create a mock arrow table with the metadata schema = pa.schema( - [pa.field("text", pa.string()), pa.field("vector", pa.list_(pa.float32(), 10))] + [ + pa.field("text", pa.string()), + pa.field("vector", pa.list_(pa.float32(), 1536)), + ] ) table = pa.table({"text": [], "vector": []}, schema=schema) table = table.replace_schema_metadata(metadata) @@ -164,13 +149,15 @@ def test_parse_functions_with_variables(): assert parsed_func.api_key == "sk-test-key-12345" assert parsed_func.base_url == "https://api.example.com" - - embeddings = parsed_func.generate_embeddings(["test text"]) - assert len(embeddings) == 1 - assert len(embeddings[0]) == 10 - assert parsed_func.safe_model_dump()["api_key"] == "$var:test_api_key" + with patch("lancedb.embeddings.openai.attempt_import_or_raise") as import_openai: + parsed_func._openai_client + + import_openai.return_value.OpenAI.assert_called_once_with( + api_key="sk-test-key-12345", base_url="https://api.example.com" + ) + def test_embedding_with_bad_results(tmp_path): @register("null-embedding") From 27dd92c67ebee5bbc37c0d085427d78fc2ffcd87 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:36:27 +0800 Subject: [PATCH 03/49] test(python): cover debugger-safe connection inspection (#3880) ## Summary - expand the synchronous debugger regression to enumerate every exposed connection attribute while the Python background loop is unavailable - retain direct representation checks for connections and tables ## Root cause VS Code debugpy suspends Python threads at a breakpoint and inspects local variables. Connection representation and property access previously dispatched asynchronous work to LanceDBBackgroundEventLoop and waited for the suspended loop thread, deadlocking the debugger. The production safeguards landed in #3620 and #3788; this regression exercises debugger-style whole-object expansion so a newly exposed property cannot reintroduce the original failure. ## Validation - uv run --no-sync pytest python/tests/test_db.py::test_sync_debugger_inspection_does_not_use_background_loop python/tests/test_db.py::test_read_consistency_interval_does_not_use_background_loop -q (2 passed) - uv run --no-sync pytest python/tests/test_db.py -q (48 passed) - python/.venv/bin/ruff format --check python/python/tests/test_db.py - python/.venv/bin/ruff check . - git diff --check Fixes #3611 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_db.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/python/python/tests/test_db.py b/python/python/tests/test_db.py index 93b791650..8f4a8850c 100644 --- a/python/python/tests/test_db.py +++ b/python/python/tests/test_db.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import inspect import re import sys from datetime import timedelta @@ -62,17 +63,23 @@ def test_basic(tmp_path): assert db.open_table("test").name == db["test"].name -def test_sync_repr_does_not_use_background_loop(tmp_path, monkeypatch): +def test_sync_debugger_inspection_does_not_use_background_loop(tmp_path, monkeypatch): from lancedb.background_loop import LOOP db = lancedb.connect(tmp_path) table = db.create_table("test", data=[{"id": 1}]) def fail_run(*args, **kwargs): - raise AssertionError("repr should not use the Python background loop") + raise AssertionError("debugger inspection should not use the background loop") monkeypatch.setattr(LOOP, "run", fail_run) + # Debuggers enumerate and evaluate every exposed attribute when expanding a + # variable. This must remain safe while their breakpoint suspends LOOP's thread. + members = dict(inspect.getmembers(db)) + + assert members["uri"] == str(tmp_path) + assert members["read_consistency_interval"] is None assert repr(db) == f"LanceDBConnection(uri={str(tmp_path)!r})" assert repr(table) == f"LanceTable(name='test', _conn={db!r})" From 16e1967efc0be9e1a9ddb44fb194374c93705607 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:36:58 +0800 Subject: [PATCH 04/49] fix(python): align wheel ABI with supported versions (#3884) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - align the PyO3 runtime and build ABI floor with the declared Python 3.10 minimum - add a regression test that keeps both ABI features synchronized with `requires-python` ## Root cause The Python 3.10 support-floor update originally changed PyO3 to `abi3-py310`, but a later dependency update reverted both PyO3 features to `abi3-py39`. Published Windows wheels were consequently tagged `cp39-abi3` while importing `PyCMethod_New`, a stable-ABI procedure absent from CPython 3.9.0 and 3.9.1. Windows reports that mismatch as “The specified procedure could not be found” while loading `_lancedb`. Restoring `abi3-py310` makes the wheel tag and native imports agree with the package metadata and prevents future wheels from advertising unsupported Python 3.9 compatibility. ## Validation - `uv run --extra tests pytest python/tests/test_package_metadata.py -q` - `uv run --extra tests --extra dev ruff format --check .` - `uv run --extra tests --extra dev ruff check .` - `cargo fmt --all` - `cargo check --quiet -p lancedb-python` - `uvx --from maturin==1.12.4 maturin build --profile ci` (built `lancedb-0.37.1b0-cp310-abi3-manylinux_2_34_x86_64.whl`) Fixes #2051 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/Cargo.toml | 4 +- python/python/tests/test_package_metadata.py | 42 ++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 python/python/tests/test_package_metadata.py diff --git a/python/Cargo.toml b/python/Cargo.toml index cc706e712..5a196e27c 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -26,7 +26,7 @@ lance-namespace-impls.workspace = true lance-io.workspace = true env_logger.workspace = true log.workspace = true -pyo3 = { version = "0.28", features = ["extension-module", "abi3-py39", "chrono"] } +pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "chrono"] } chrono = { version = "0.4", default-features = false, features = ["clock"] } pyo3-async-runtimes = { version = "0.28", features = [ "attributes", @@ -43,7 +43,7 @@ libc = "0.2" [build-dependencies] pyo3-build-config = { version = "0.28", features = [ "extension-module", - "abi3-py39", + "abi3-py310", ] } [features] diff --git a/python/python/tests/test_package_metadata.py b/python/python/tests/test_package_metadata.py new file mode 100644 index 000000000..5792f457b --- /dev/null +++ b/python/python/tests/test_package_metadata.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import importlib +import re +import sys +from pathlib import Path + +import pytest + + +def test_pyo3_abi_matches_minimum_supported_python(): + project_dir = Path(__file__).parents[2] + pyproject = (project_dir / "pyproject.toml").read_text() + cargo_manifest = (project_dir / "Cargo.toml").read_text() + + minimum_python = re.search( + r'^requires-python\s*=\s*">=(\d+)\.(\d+)"$', pyproject, re.MULTILINE + ) + assert minimum_python is not None + + major, minor = minimum_python.groups() + expected_abi = f"abi3-py{major}{minor}" + configured_abis = re.findall(r'"(abi3-py\d+)"', cargo_manifest) + + assert configured_abis == [expected_abi, expected_abi], ( + "the pyo3 runtime and build ABI features must both match requires-python" + ) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows wheel regression test") +def test_windows_wheel_tag_and_native_import(): + project_dir = Path(__file__).parents[2] + wheels = list((project_dir.parent / "target" / "wheels").glob("lancedb-*.whl")) + if not wheels: + pytest.skip("no wheel artifact is available in this development environment") + + assert len(wheels) == 1 + assert wheels[0].name.endswith("-cp310-abi3-win_amd64.whl") + + native_module = importlib.import_module("lancedb._lancedb") + assert Path(native_module.__file__).suffix == ".pyd" From 3956d9dbfa1e701ca461a078ee0f1ca2a9c175a6 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:38:27 +0800 Subject: [PATCH 05/49] fix(python): prevent OpenSSL linkage in Linux wheels (#3877) ## Summary - select rustls with native certificate roots explicitly for LanceDB's remote HTTP client - add a Linux regression test that rejects `libssl` or `libcrypto` dependencies in the built Python extension ## Root cause The Python remote client originally enabled reqwest's native TLS backend. During manylinux wheel repair, that caused OpenSSL 1.1 libraries to be bundled into the wheel. Loading those libraries on RHEL 9 with FIPS enabled aborts during the OpenSSL self-test before `import lancedb` can complete. LanceDB has since moved away from native TLS, but its own reqwest dependency relied on transitive rustls feature selection and the built extension had no regression guard. This change makes rustls selection explicit and tests the produced Linux native module's dynamic dependencies. ## Validation - `uv run --no-sync pytest python/tests/test_import.py -q` - `ruff format --check python` - `ruff check .` - `cargo fmt --all -- --check` - `cargo check --quiet --features remote --tests --examples` - `ldd python/lancedb/_lancedb.abi3.so` (no `libssl` or `libcrypto` dependency) - verified the resolved Python Rust dependency graph contains rustls and no `openssl-sys` or `native-tls` Fixes #1884 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_import.py | 33 ++++++++++++++++++++++++++++++ rust/lancedb/Cargo.toml | 2 ++ 2 files changed, 35 insertions(+) create mode 100644 python/python/tests/test_import.py diff --git a/python/python/tests/test_import.py b/python/python/tests/test_import.py new file mode 100644 index 000000000..4b87a0ce8 --- /dev/null +++ b/python/python/tests/test_import.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import re +import shutil +import subprocess +import sys + +import lancedb._lancedb as _lancedb +import pytest + + +@pytest.mark.skipif(sys.platform != "linux", reason="ldd is Linux-specific") +def test_native_extension_does_not_link_openssl(): + """OpenSSL-linked wheels abort when imported on RHEL hosts in FIPS mode.""" + ldd = shutil.which("ldd") + if ldd is None: + pytest.skip("ldd is not installed") + + result = subprocess.run( + [ldd, _lancedb.__file__], + check=True, + capture_output=True, + text=True, + ) + openssl_libraries = re.findall( + r"^\s*(lib(?:crypto|ssl)\S*)\s+=>", result.stdout, flags=re.MULTILINE + ) + + assert not openssl_libraries, ( + "the LanceDB native extension must use rustls instead of linking OpenSSL: " + f"{openssl_libraries}" + ) diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 66db3cf12..816f095de 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -75,6 +75,8 @@ reqwest = { version = "0.12.0", default-features = false, features = [ "http2", "json", "macos-system-configuration", + # Avoid linking OpenSSL into Python wheels, which breaks on FIPS hosts. + "rustls-tls-native-roots", "stream", ], optional = true } http = { version = "1", optional = true } # Matching what is in reqwest From 9e73d440a36986db57f16cf8a19cf850135c53f2 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:39:26 +0800 Subject: [PATCH 06/49] test(python): cover schema-only vector table creation (#3882) ## Summary - make the existing #1968 regression explicitly assert that schema-only table creation succeeds - verify the new table has zero rows and preserves the requested fixed-size vector schema before accepting subsequent data ## Root cause In v0.16.0, schema-only table creation sent an empty table through vector sanitization, which calculated a remainder using `len(data)` and raised `ZeroDivisionError`. Later refactors removed that runtime path, but the issue-specific regression only asserted the final row count after a subsequent add. This change makes the reported operation and its expected empty-table state explicit so the original defect remains directly covered. ## Validation - `uv run --extra tests pytest python/tests/test_table.py::test_create_table_without_data_with_vector_schema -q` - `uv --project python run --extra tests --extra dev ruff format --check python/python/tests/test_table.py` - `uv --project python run --extra tests --extra dev ruff check .` - `git diff --check` Fixes #1968 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 069527b21..1f2e14742 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -3489,8 +3489,8 @@ def test_create_table_empty_list_no_schema_error(mem_db: DBConnection): mem_db.create_table("test_empty_no_schema", data=[]) -def test_add_table_with_empty_embeddings(tmp_path): - """Test exact scenario from issue #1968 +def test_create_table_without_data_with_vector_schema(tmp_path): + """Test exact scenario from issue #1968. Regression test for issue #1968: https://github.com/lancedb/lancedb/issues/1968 @@ -3502,6 +3502,9 @@ def test_add_table_with_empty_embeddings(tmp_path): embedding: Vector(16) table = db.create_table("test", schema=MySchema) + assert table.count_rows() == 0 + assert table.schema == MySchema.to_arrow_schema() + table.add( [{"text": "bar", "embedding": [0.1] * 16}], on_bad_vectors="drop", From 99a68db78c149ceb3342be358c6b3178d18f0a2c Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:40:01 +0800 Subject: [PATCH 07/49] test(rust): cover concurrent appends during compaction (#3878) ## Summary - add a LanceDB core regression for compaction overlapping appends through separate table handles - verify concurrent commits preserve fragment ID order on an indexed table - run the follow-up compaction that exposed the original row-ID ordering failure and verify all rows remain ## Root cause Older Lance versions could reserve fragment IDs for compaction, allow concurrent appends to commit later IDs, and then commit the reserved compaction fragments at the end of the manifest. A later compaction could consequently receive row IDs out of order. Current Lance sorts fragments at the transaction boundary; this adds the missing LanceDB-level regression coverage for the Node-visible concurrency contract. ## Validation - `cargo fmt --all` - focused regression passed once with output and 20 repeated runs - `cargo test --quiet --features remote -p lancedb table::optimize::tests` (14 passed) - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` - `cargo test --quiet --features remote --tests` (867 passed, 1 ignored) Fixes #1498 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/table/optimize.rs | 90 ++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/rust/lancedb/src/table/optimize.rs b/rust/lancedb/src/table/optimize.rs index e29445b2d..3fdad0533 100644 --- a/rust/lancedb/src/table/optimize.rs +++ b/rust/lancedb/src/table/optimize.rs @@ -304,6 +304,96 @@ mod tests { assert_eq!(all_values, expected); } + #[tokio::test] + async fn test_compact_with_concurrent_add() { + const NUM_FRAGMENTS: usize = 5; + const ROWS_PER_FRAGMENT: i32 = 300; + + let tmpdir = tempfile::tempdir().unwrap(); + let conn = connect(tmpdir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(Int32Array::from_iter_values(0..ROWS_PER_FRAGMENT))], + ) + .unwrap(); + + let table = conn + .create_table("test_concurrent_compact", batch.clone()) + .execute() + .await + .unwrap(); + table + .create_index(&["id"], Index::BTree(BTreeIndexBuilder::default())) + .execute() + .await + .unwrap(); + for _ in 0..NUM_FRAGMENTS { + table.add(batch.clone()).execute().await.unwrap(); + } + + // Use separate handles so the two writes actually overlap, as they can + // when different Node connections operate on the same S3 table. + let compact_table = conn + .open_table("test_concurrent_compact") + .execute() + .await + .unwrap(); + let append_table = conn + .open_table("test_concurrent_compact") + .execute() + .await + .unwrap(); + let compact_task = tokio::spawn(async move { + compact_table + .optimize(OptimizeAction::Compact { + options: CompactionOptions { + target_rows_per_fragment: 1_000, + ..Default::default() + }, + remap_options: None, + }) + .await + }); + tokio::task::yield_now().await; + for _ in 0..NUM_FRAGMENTS { + append_table.add(batch.clone()).execute().await.unwrap(); + } + compact_task.await.unwrap().unwrap(); + + let table = conn + .open_table("test_concurrent_compact") + .execute() + .await + .unwrap(); + let dataset = table.dataset().unwrap().get().await.unwrap(); + let fragment_ids = dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id()) + .collect::>(); + assert!(fragment_ids.windows(2).all(|ids| ids[0] < ids[1])); + + // A second compaction exposed the original out-of-order row-id bug. + table + .optimize(OptimizeAction::Compact { + options: CompactionOptions { + target_rows_per_fragment: 1_000, + ..Default::default() + }, + remap_options: None, + }) + .await + .unwrap(); + assert_eq!( + table.count_rows(None).await.unwrap(), + ROWS_PER_FRAGMENT as usize * (NUM_FRAGMENTS * 2 + 1) + ); + } + #[tokio::test] async fn test_optimize_prune_versions() { let conn = connect("memory://").execute().await.unwrap(); From 123c921c4f6cf6ff350922d9236215d408efecb6 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:40:41 +0800 Subject: [PATCH 08/49] test(python): cover sliced nullable table search (#3875) ## Summary - add a Python regression for vector search over a sliced Arrow table with nullable scalar columns - verify the nearest row retains its non-null score values after the table is written ## Root cause Lance 0.19.2 deep-copied a validity bitmap without preserving its non-zero bit offset. For a sliced nullable table, scalar values and vectors began at the slice while the copied validity bitmap began at the parent table's first row. That made valid score values appear null even though the corresponding vector stayed intact. The upstream Lance repair is already present in the current dependency; this adds a LanceDB-level guard for the reported create/search path. ## Validation - reproduced on Python 3.12 with LanceDB 0.16.0, pylance 0.19.2, PyArrow 18.0.0, and Polars 1.14.0 - `uv run --project python --extra dev ruff format --check python/python/tests/test_table.py` - `uv run --project python --extra dev ruff check .` - `cd python && uv run --extra tests pytest python/tests/test_table.py::test_search_preserves_nulls_from_sliced_arrow_table -q` Fixes #1879 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 1f2e14742..ef59502d7 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -99,6 +99,30 @@ def test_basic(mem_db: DBConnection): assert table.to_arrow() == expected_data +def test_search_preserves_nulls_from_sliced_arrow_table(mem_db: DBConnection): + data = pa.table( + { + "id": [0, 1, 2, 3, 4], + "score_cn": [None, 22, None, 5, 8], + "score_mt": [None, 42, None, 5, 8], + "vector": [ + [20, 19, -1, -1], + [41, 38, 22, 42], + [10, 10, -1, -1], + [5, 5, 5, 5], + [8, 8, 8, 8], + ], + } + ).slice(1) + + table = mem_db.create_table("sliced_nullable", data=data) + result = table.search([41, 38, 22, 42]).limit(1).to_arrow() + + assert result["id"].to_pylist() == [1] + assert result["score_cn"].to_pylist() == [22] + assert result["score_mt"].to_pylist() == [42] + + def test_table_to_pandas_default_matches_arrow(tmp_db: DBConnection): pd = pytest.importorskip("pandas") data = pa.table({"id": [1, 2], "text": ["one", "two"]}) From f1f34dfdd33072319dffd821f69cb32c8e3a426f Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:41:11 +0800 Subject: [PATCH 09/49] fix(python): instruct dimension probe for instructor embeddings (#3874) ## Summary - pass an Instructor-compatible `[instruction, text]` pair when detecting embedding dimensions - add a regression test that verifies the dimension probe uses the configured source instruction ## Root cause `InstructorEmbeddingFunction.ndims()` encoded a bare string even though Instructor models require instruction/text pairs. With affected `sentence-transformers` versions, the bare input omitted `instruction_mask` and raised `KeyError` while defining the LanceDB schema. ## Validation - `uv run --extra tests pytest python/tests/test_embeddings.py -q` (`14 passed, 9 skipped`) - `uv run --project python --extra tests --extra dev ruff format --check python/python/lancedb/embeddings/instructor.py python/python/tests/test_embeddings.py` - `uv run --project python --extra tests --extra dev ruff check .` Fixes #2041 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/lancedb/embeddings/instructor.py | 3 +-- python/python/tests/test_embeddings.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/python/python/lancedb/embeddings/instructor.py b/python/python/lancedb/embeddings/instructor.py index 675a0139c..37ae1c296 100644 --- a/python/python/lancedb/embeddings/instructor.py +++ b/python/python/lancedb/embeddings/instructor.py @@ -101,8 +101,7 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction): @weak_lru(maxsize=1) def ndims(self): - model = self.get_model() - return model.encode("foo").shape[0] + return len(self.generate_embeddings([[self.source_instruction, "foo"]])[0]) def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]: return self.generate_embeddings([[self.query_instruction, query]]) diff --git a/python/python/tests/test_embeddings.py b/python/python/tests/test_embeddings.py index f2d8971c3..678270f19 100644 --- a/python/python/tests/test_embeddings.py +++ b/python/python/tests/test_embeddings.py @@ -64,6 +64,23 @@ def test_embedding_function(tmp_path): assert np.allclose(actual, expected) +def test_instructor_ndims_uses_instruction(): + instructor = get_registry().get("instructor").create() + model = MagicMock() + model.encode.return_value = np.zeros((1, 384)) + + with patch.object(type(instructor), "get_model", return_value=model): + assert instructor.ndims() == 384 + + model.encode.assert_called_once_with( + [[instructor.source_instruction, "foo"]], + batch_size=instructor.batch_size, + show_progress_bar=instructor.show_progress_bar, + normalize_embeddings=instructor.normalize_embeddings, + device=instructor.device, + ) + + def test_embedding_function_variables(): @register("variable-testing") class VariableTestingFunction(TextEmbeddingFunction): From 798e5364fb8ec24aca8b247bf7ed38f25745a688 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:42:20 +0800 Subject: [PATCH 10/49] test(python): cover VoyageAI text source routing (#3872) ## Summary - add fast regression coverage for VoyageAI `voyage-3` source embeddings - verify table text uses `client.embed` and never `client.multimodal_embed` ## Root cause The original VoyageAI source-embedding path treated table source values as images and always invoked the multimodal API. Production routing was corrected by later merged changes, but the table regression was covered only by API-gated slow tests. This test locks the corrected text routing into the regular unit suite. ## Validation - `cd python && uv run --extra tests pytest python/tests/test_voyageai_embeddings.py -q` - `uv run --project python --extra tests --extra dev ruff format --check python/python/tests/test_voyageai_embeddings.py` - `uv run --project python --extra tests --extra dev ruff check .` Fixes #2059 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_voyageai_embeddings.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/python/tests/test_voyageai_embeddings.py b/python/python/tests/test_voyageai_embeddings.py index ac1554cad..040cade1f 100644 --- a/python/python/tests/test_voyageai_embeddings.py +++ b/python/python/tests/test_voyageai_embeddings.py @@ -75,6 +75,22 @@ class TestVoyageAIModelRegistration: with pytest.raises(ValueError, match="not supported"): func.ndims() + def test_voyage3_source_embeddings_use_text_api(self, mock_voyageai_client): + """Regression test for text table data being sent to the multimodal API.""" + mock_voyageai_client.tokenize.return_value = [["hello", "world"]] + mock_voyageai_client.embed.return_value.embeddings = [[0.1] * 1024] + + registry = get_registry() + func = registry.get("voyageai").create(name="voyage-3") + + embeddings = func.compute_source_embeddings("hello world") + + assert embeddings == [[0.1] * 1024] + mock_voyageai_client.embed.assert_called_once_with( + texts=["hello world"], model="voyage-3", input_type="document" + ) + mock_voyageai_client.multimodal_embed.assert_not_called() + @pytest.mark.parametrize( "model_name", [ From 03b52e587788d4c57a8379b52408646c785b33e5 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:43:59 +0800 Subject: [PATCH 11/49] test(node): cover fixed-size list schemas with typed arrays (#3866) ## Summary - cover explicit FixedSizeList schemas populated from Float32Array values - verify the original vector.0 failure stays fixed across Arrow 15, 16, 17, and 18 ## Root cause and fix In v0.16, schema subset inference treated typed-array vectors as nested objects and looked up numeric paths such as vector.0, which do not exist in a FixedSizeList schema. Current typed-array handling correctly recognizes ArrayBuffer views as vector values instead of traversing their elements. This change adds the missing regression coverage for the reported explicit-schema path so that behavior cannot regress unnoticed. ## Validation - pnpm test __test__/arrow.test.ts --runInBand - pnpm lint - pnpm build - pnpm run docs - pnpm test --runInBand (681 passed, 5 skipped) Fixes #2134 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/arrow.test.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/nodejs/__test__/arrow.test.ts b/nodejs/__test__/arrow.test.ts index 9e20e3c04..c05849cb9 100644 --- a/nodejs/__test__/arrow.test.ts +++ b/nodejs/__test__/arrow.test.ts @@ -197,6 +197,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( expect(table.getChild("d")?.toJSON()).toEqual([9n, 10n, null]); }); + it("will use a provided FixedSizeList schema with typed array values", function () { + const schema = new Schema([ + new Field("text", new Utf8(), false), + new Field( + "vector", + new FixedSizeList(3, new Field("item", new Float32(), false)), + false, + ), + ]); + + const table = makeArrowTable( + [ + { + text: "foo", + vector: new Float32Array([1, 2, 3]), + }, + ], + { schema }, + ); + + expect(table.getChild("text")?.toJSON()).toEqual(["foo"]); + expect( + table + .getChild("vector") + ?.toJSON() + .map((value) => value.toJSON()), + ).toEqual([[1, 2, 3]]); + }); + it("will assume the column `vector` is FixedSizeList by default", async function () { const schema = new Schema([ new Field("a", new Float(Precision.DOUBLE), true), From 173f889d2afc35a84de5babdae1d38fb2e7be884 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:44:20 +0800 Subject: [PATCH 12/49] test(python): cover stale scalar prefilters in hybrid search (#3865) ## Summary - capture the stale-index state behind the reported fixed-size-binary panic: the vector and FTS indices cover newer fragments while the BTree prefilter does not - verify vector, FTS, and hybrid searches return matches from both scalar-indexed and unindexed fragments without panicking - preserve binding-level coverage for the Lance fix in https://github.com/lance-format/lance/pull/3768, which restricts incomplete scalar prefilters when search indices are further ahead The production root cause is in Lance and the current LanceDB dependency already contains that fix, so this change adds the missing LanceDB Python regression coverage. ## Validation - `cd python && uv run --no-sync pytest python/tests/test_hybrid_query.py::test_hybrid_query_with_stale_fixed_size_binary_prefilter -q` - `cd python && uv run --no-sync pytest python/tests/test_hybrid_query.py -q` - `python/.venv/bin/ruff check .` - `python/.venv/bin/ruff format --check python/python/tests/test_hybrid_query.py` Fixes #2370 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_hybrid_query.py | 82 +++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/python/python/tests/test_hybrid_query.py b/python/python/tests/test_hybrid_query.py index 65a7890bf..72dcaaa49 100644 --- a/python/python/tests/test_hybrid_query.py +++ b/python/python/tests/test_hybrid_query.py @@ -12,7 +12,7 @@ import pyarrow.compute as pc import pytest import pytest_asyncio -from lancedb.index import FTS +from lancedb.index import BTree, FTS, IvfPq from lancedb.table import AsyncTable, Table @@ -99,6 +99,86 @@ async def test_async_hybrid_query_filters(table: AsyncTable): assert result["text"].to_pylist() == ["cat", "b"] +@pytest.mark.asyncio +async def test_hybrid_query_with_stale_fixed_size_binary_prefilter( + tmpdir_factory, +): + tmp_path = str(tmpdir_factory.mktemp("stale_scalar_prefilter")) + db = await lancedb.connect_async(tmp_path) + + def fixed_size_binary(value: int) -> bytes: + return value.to_bytes(16, byteorder="big") + + num_rows = 1000 + data = pa.table( + { + "space_id": pa.array( + [fixed_size_binary(i) for i in range(num_rows)], + type=pa.binary(16), + ), + "text": ["book"] * num_rows, + "vector": pa.array( + [[float(i), float(i)] for i in range(num_rows)], + type=pa.list_(pa.float32(), 2), + ), + } + ) + table = await db.create_table("test", data) + await table.create_index( + "vector", config=IvfPq(num_partitions=4, num_sub_vectors=2) + ) + await table.create_index("space_id", config=BTree()) + await table.create_index("text", config=FTS(with_position=False)) + + # Advance the search indices without advancing the scalar index. This is the + # state that previously let hybrid search use an incomplete scalar prefilter. + await table.add(data) + lance_dataset = await table.to_lance() + lance_dataset.optimize.optimize_indices(index_names=["vector_idx", "text_idx"]) + await table.checkout_latest() + + scalar_stats = await table.index_stats("space_id_idx") + assert scalar_stats is not None + assert scalar_stats.num_indexed_rows == num_rows + assert scalar_stats.num_unindexed_rows == num_rows + + for index_name in ["vector_idx", "text_idx"]: + search_stats = await table.index_stats(index_name) + assert search_stats is not None + assert search_stats.num_indexed_rows == num_rows * 2 + assert search_stats.num_unindexed_rows == 0 + + matching_ids = [5, 10, 15, 20, 25, 30] + literals = [ + f"arrow_cast(0x{fixed_size_binary(i).hex()}, 'FixedSizeBinary(16)')" + for i in matching_ids + ] + predicate = f"space_id IN ({', '.join(literals)})" + expected_ids = sorted(fixed_size_binary(i) for i in matching_ids for _ in range(2)) + + vector_query = ( + table.query().where(predicate).nearest_to([5.0, 5.0]).limit(num_rows * 2) + ) + vector_results = await vector_query.to_arrow() + assert sorted(vector_results["space_id"].to_pylist()) == expected_ids + + fts_query = ( + table.query().where(predicate).nearest_to_text("book").limit(num_rows * 2) + ) + fts_results = await fts_query.to_arrow() + assert sorted(fts_results["space_id"].to_pylist()) == expected_ids + + hybrid_results = await ( + table.query() + .where(predicate) + .nearest_to([5.0, 5.0]) + .nearest_to_text("book") + .limit(num_rows * 2) + .to_arrow() + ) + assert sorted(hybrid_results["space_id"].to_pylist()) == expected_ids + + @pytest.mark.asyncio async def test_async_hybrid_query_default_limit(table: AsyncTable): # add 10 new rows From ac8b28c010360322557d31ca49bcd3826745b73c Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:44:38 +0800 Subject: [PATCH 13/49] fix(python): support nullable pandas merge input (#3864) ## Summary - add an end-to-end Python regression for pandas DataFrame inputs merged into a table created from a Pydantic model - verify reordered, nullable Arrow source fields can update and insert into a non-nullable target schema when the values contain no nulls ## Root cause Lance merge_insert previously compared source schema nullability with the target, unlike add. The upstream fix now pinned by LanceDB ignores declared nullability during schema compatibility and validates actual null values at write time. LanceDB lacked regression coverage for the full pandas-to-Pydantic path, so this test locks in the correct behavior without falsifying the input schema nullability. ## Validation - 5 focused merge-insert tests passed - Ruff lint passed for the repository - Ruff format check passed for the changed file - git diff --check passed Fixes #2366 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index ef59502d7..d7748095e 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -2387,6 +2387,55 @@ def test_merge_insert(mem_db: DBConnection): ) +def test_merge_insert_nullable_pandas_into_pydantic_schema(mem_db: DBConnection): + # Regression test for https://github.com/lancedb/lancedb/issues/2366 + pd = pytest.importorskip("pandas") + + class Document(LanceModel): + id: int + title: str + content: str + + table = mem_db.create_table("documents", schema=Document) + table.add( + pd.DataFrame( + { + "title": ["Old title", "Unchanged"], + "id": [2, 3], + "content": ["Old content", "Keep this"], + } + ) + ) + + # Pandas produces nullable Arrow fields, in an order that differs from the + # non-nullable Pydantic schema. This is valid as long as the data has no nulls. + new_data = pd.DataFrame( + { + "title": ["Inserted", "Updated"], + "id": [1, 2], + "content": ["New row", "New content"], + } + ) + result = ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(new_data) + ) + + assert result.num_inserted_rows == 1 + assert result.num_updated_rows == 1 + expected = pa.Table.from_pylist( + [ + {"id": 1, "title": "Inserted", "content": "New row"}, + {"id": 2, "title": "Updated", "content": "New content"}, + {"id": 3, "title": "Unchanged", "content": "Keep this"}, + ], + schema=Document.to_arrow_schema(), + ) + assert table.to_arrow().sort_by("id") == expected + + def test_merge_insert_by_source_delete_expr(mem_db: DBConnection): table = mem_db.create_table( "my_table", From 2c06a48bd848f400a690b6abc15c3fec662a1418 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:45:10 +0800 Subject: [PATCH 14/49] test(python): cover Arrow buffer release after add (#3860) ## Summary - add deterministic regression coverage that `Table.add()` releases backing Arrow buffers without cyclic garbage collection - track the foreign buffer owner rather than RSS, separating live input retention from allocator high-water behavior - preserve the bounded-lifetime behavior of the Scannable writer that superseded the historical preprocessing path ## Root cause The historical Python preprocessing/write path produced a high allocator RSS while ingesting very wide IPC batches. The current Scannable writer releases each input buffer when `Table.add()` completes; remaining RSS is allocator high-water rather than a live Arrow reference. The resolved behavior had no regression coverage, so a future native lifetime regression could silently reintroduce the original failure mode. ## Validation - `uv run --extra tests --extra dev maturin develop` - `uv run --project python --extra tests pytest python/python/tests/test_table.py::test_add python/python/tests/test_table.py::test_add_releases_arrow_buffers_without_gc -q` - `uv run --project python --extra dev ruff format --check python/python/tests/test_table.py` - `uv run --project python --extra dev ruff check .` Fixes #2512 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index d7748095e..70a52b100 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -2,10 +2,13 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import ctypes +import gc import os import sys import threading import warnings +import weakref from concurrent.futures import ThreadPoolExecutor from datetime import date, datetime, timedelta from time import sleep @@ -459,6 +462,38 @@ def test_add(mem_db: DBConnection): _add(table, schema) +def test_add_releases_arrow_buffers_without_gc(mem_db: DBConnection): + """Regression test for https://github.com/lancedb/lancedb/issues/2512.""" + schema = pa.schema([pa.field("x", pa.int64())]) + table = mem_db.create_table("test_add_releases_arrow_buffers", schema=schema) + + class BufferOwner: + def __init__(self, size: int): + self.memory = ctypes.create_string_buffer(size) + + owner_refs = [] + gc_was_enabled = gc.isenabled() + gc.disable() + try: + for _ in range(3): + size = 8 * 1024 + owner = BufferOwner(size) + arrow_buffer = pa.foreign_buffer( + ctypes.addressof(owner.memory), size, owner + ) + array = pa.Array.from_buffers(pa.int64(), 1024, [None, arrow_buffer]) + batch = pa.RecordBatch.from_arrays([array], schema=schema) + owner_refs.append(weakref.ref(owner)) + + table.add(batch) + del batch, array, arrow_buffer, owner + + assert all(owner_ref() is None for owner_ref in owner_refs) + finally: + if gc_was_enabled: + gc.enable() + + def test_add_write_parallelism(mem_db: DBConnection): schema = pa.schema([pa.field("id", pa.int64())]) table = mem_db.create_table("test", schema=schema) From 3af51541a02963563baac208a7bd768ac241a785 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:46:49 +0800 Subject: [PATCH 15/49] test(rust): cover fixed-size binary merge insert regression (#3854) ## Summary - add a LanceDB regression for `merge_insert` with a non-nullable `FixedSizeBinary` column - exercise matched updates, unmatched inserts, and source-missing deletes - assert the exact merge statistics and final row count ## Root cause The Arrow `take` kernel previously ignored nulls in the index array for `FixedSizeBinary`. DataFusion uses that kernel while constructing outer-join results, so the join behind `when_not_matched_by_source_delete` could place invalid values into non-nullable columns. The current Arrow dependency contains the upstream fix; this test locks the corrected behavior at the LanceDB API boundary. ## Validation - `cargo fmt --all -- --check` - `cargo test --quiet --features remote --tests` - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` Fixes #2869 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/table/merge.rs | 71 ++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 13a633c67..82a1d1473 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -315,7 +315,10 @@ pub(crate) async fn execute_merge_insert( #[cfg(test)] mod tests { - use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader}; + use arrow_array::builder::FixedSizeBinaryBuilder; + use arrow_array::{ + Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, UInt64Array, + }; use arrow_schema::{DataType, Field, Schema}; use std::sync::Arc; @@ -337,6 +340,42 @@ mod tests { Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema)) } + fn fixed_size_binary_merge_batch( + id_range: std::ops::Range, + price: u64, + ) -> Box { + let ids = id_range.collect::>(); + let mut id_builder = FixedSizeBinaryBuilder::new(16); + for id in &ids { + let mut bytes = [0; 16]; + bytes[..8].copy_from_slice(&id.to_le_bytes()); + id_builder.append_value(bytes).unwrap(); + } + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::FixedSizeBinary(16), false), + Field::new("id_as_int", DataType::UInt64, false), + Field::new("name", DataType::Utf8, false), + Field::new("market", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(id_builder.finish()), + Arc::new(UInt64Array::from_iter_values(ids.iter().copied())), + Arc::new(StringArray::from_iter_values( + ids.iter().map(|id| format!("name{id}")), + )), + Arc::new(StringArray::from_iter_values(std::iter::repeat_n( + format!("market_{price}"), + ids.len(), + ))), + ], + ) + .unwrap(); + Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema)) + } + #[tokio::test] async fn test_merge_insert() { let conn = connect("memory://").execute().await.unwrap(); @@ -388,6 +427,36 @@ mod tests { ); } + #[tokio::test] + async fn test_merge_insert_fixed_size_binary_non_nullable() { + // Regression test for #2869: an unrelated FixedSizeBinary column used to corrupt the + // outer join that implements when_not_matched_by_source_delete. + let conn = connect("memory://").execute().await.unwrap(); + let table = conn + .create_table( + "fixed_size_binary_merge", + fixed_size_binary_merge_batch(0..256, 100), + ) + .execute() + .await + .unwrap(); + + let mut merge_insert = table.merge_insert(&["id_as_int"]); + merge_insert + .when_matched_update_all(None) + .when_not_matched_insert_all() + .when_not_matched_by_source_delete(None); + let result = merge_insert + .execute(fixed_size_binary_merge_batch(100..356, 200)) + .await + .unwrap(); + + assert_eq!(result.num_updated_rows, 156); + assert_eq!(result.num_inserted_rows, 100); + assert_eq!(result.num_deleted_rows, 100); + assert_eq!(table.count_rows(None).await.unwrap(), 256); + } + #[tokio::test] async fn test_merge_insert_use_index() { let conn = connect("memory://").execute().await.unwrap(); From 0ba82873c5128f45ce6fbea75eb0135bcd08ead4 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:47:29 +0800 Subject: [PATCH 16/49] fix(python): cover nullable list v2.2 decoding (#3853) ## Summary - add a minimized regression for mostly-null `list` data at the v2.2 structural page boundary - verify scans preserve all 64,885 rows, including 64,668 null list values ## Root cause Lance 3.0.0 sliced repetition/definition state using top-level row offsets in the complex all-null decoder. At this page boundary, the list and validity children were materialized at different lengths. The current Lance dependency contains the upstream decoder repair; this test locks that behavior into the LanceDB Python suite without duplicating decoder logic. ## Validation - reproduced the attached 1,892,466-row case on `lancedb==0.30.0` with `expected 1024 got 285` - verified the full attachment reads on the current branch - `python/.venv/bin/ruff format --check python/python/tests/test_table.py` - `python/.venv/bin/ruff check .` - `cd python && uv run --extra tests pytest python/tests/test_table.py::test_read_mostly_null_list_v2_2_page_boundary -q` - `cd python && uv run --extra tests pytest python/tests/test_table.py -q` (137 passed) Fixes #3194 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 70a52b100..832b70569 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -1884,6 +1884,33 @@ def test_add_nullable_struct_with_none(mem_db: DBConnection): assert result.column("data").to_pylist() == [{"x": 1.0}, None] +def test_read_mostly_null_list_v2_2_page_boundary(tmp_path): + # Regression test for #3194. This row/value count crosses a v2.2 structural + # encoding page boundary where Lance 3.0.0 sliced repetition/definition + # levels by row offset and decoded child arrays at different lengths. + num_rows = 64_885 + num_values = 217 + list_type = pa.list_(pa.float32()) + source = pa.table( + { + "id": np.arange(num_rows, dtype=np.int64), + "coords": pa.array( + [[1.0, 2.0, 3.0, 4.0]] * num_values + [None] * (num_rows - num_values), + type=list_type, + ), + } + ) + db = lancedb.connect( + tmp_path, + storage_options={"new_table_data_storage_version": "2.2"}, + ) + table = db.create_table("test_sparse_nullable_list", data=source) + + result = table.search().select(["id", "coords"]).limit(num_rows).to_arrow() + + assert result.equals(source) + + def test_add_with_integer_embeddings_preserves_casting(mem_db: DBConnection): class Schema(LanceModel): text: str From c1a3fa7f51057160f930d4a86e5c96cc18993ef5 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:48:01 +0800 Subject: [PATCH 17/49] fix(python): preserve repeated indexed merge inserts (#3850) ## Summary - add a Python regression test for two partial-schema merge inserts against the same BTree-indexed rows - verify repeated updates retain one copy of every row and the final update values ## Root cause Lance 4.0, used by LanceDB 0.30.2, removed a rewritten fragment from the index bitmap while stale BTree entries for that fragment remained searchable. The next merge found each target through both the stale index and the unindexed-fragment scan, producing the ambiguous-match error. Lance fixed the root cause in lance-format/lance#6563 by applying the fragment-bitmap allow-list to index results, and the Lance release pinned by current LanceDB includes that fix. This test preserves the corrected behavior through the Python API. ## Validation - `cd python && uv run --extra tests pytest python/tests/test_table.py -k merge_insert -q` (9 passed) - `cd python && uv run --extra tests --extra dev ruff format --check python/tests/test_table.py` - `cd python && uv run --extra tests --extra dev ruff check python/tests/test_table.py` Repository-wide Ruff also reports 20 pre-existing violations in untouched CI and plugin scripts. Fixes #3280 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 832b70569..f20f42617 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -2598,6 +2598,36 @@ def test_merge_insert_subschema(mem_db: DBConnection, data_format): assert table.to_arrow().sort_by("id") == expected +def test_repeated_partial_merge_insert_with_scalar_index(mem_db: DBConnection): + def make_batch(start: int) -> pa.Table: + return pa.table( + { + "id": [f"id-{i:04}" for i in range(start, start + 100)], + "category": ["A"] * 100, + "value_a": [float(i) for i in range(start, start + 100)], + "value_b": [float(i) / 10 for i in range(100)], + } + ) + + table = mem_db.create_table("my_table", data=make_batch(0)) + table.add(make_batch(100)) + table.add(make_batch(200)) + table.create_index("id", config=BTree()) + + ids = [f"id-{i:04}" for i in range(100, 200)] + for value in (999.0, 888.0): + result = ( + table.merge_insert("id") + .when_matched_update_all() + .execute(pa.table({"id": ids, "value_a": [value] * 100})) + ) + assert result.num_updated_rows == 100 + + actual = table.to_arrow().sort_by("id") + assert actual.num_rows == 300 + assert actual["value_a"].to_pylist()[100:200] == [888.0] * 100 + + @pytest.mark.asyncio async def test_merge_insert_async(mem_db_async: AsyncConnection): data = pa.table({"a": [1, 2, 3], "b": ["a", "b", "c"]}) From 772bdeced8b4a2441c8c54d54d2a4a713a943855 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:48:24 +0800 Subject: [PATCH 18/49] fix(rust): prevent vector optimize regression after deletes (#3848) ## Summary - Adds a LanceDB regression for stable row IDs, scattered deletes, IVF_RQ, and default index optimization. - Verifies optimization completes and preserves the expected live-row count. ## Root cause Lance 3.0.1 built the stable-row-ID address list by dropping deleted IDs while retaining the original ID list. The subsequent positional zip misaligned IDs and addresses, so vector partition joins requested deleted rows and failed with batch.num_rows() != chunk.len(). Lance PR https://github.com/lance-format/lance/pull/7704 corrected the generic filter, and the LanceDB dependency currently pinned on main contains that correction. ## Fix Add regression coverage at the Rust Table optimize surface using the IVF_RQ configuration from the report. This locks the upstream correction into the LanceDB workflow that originally crashed. ## Validation - cargo fmt --all -- --check - cargo test --quiet --features remote -p lancedb table::optimize::tests (14 passed) - cargo check --quiet --features remote --tests --examples - cargo clippy --quiet --features remote --tests --examples -p lancedb Fixes #3330 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/table/optimize.rs | 59 +++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/rust/lancedb/src/table/optimize.rs b/rust/lancedb/src/table/optimize.rs index 3fdad0533..4ad58cffb 100644 --- a/rust/lancedb/src/table/optimize.rs +++ b/rust/lancedb/src/table/optimize.rs @@ -214,12 +214,17 @@ pub(crate) async fn execute_optimize( #[cfg(test)] mod tests { - use arrow_array::{Int32Array, RecordBatch, StringArray}; + use arrow_array::{ + Array, FixedSizeListArray, Float32Array, Int32Array, RecordBatch, StringArray, + }; use arrow_schema::{DataType, Field, Schema}; + use lance_arrow::FixedSizeListArrayExt; use rstest::rstest; use std::sync::Arc; use crate::connect; + use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS; + use crate::index::vector::IvfRqIndexBuilder; use crate::index::{Index, scalar::BTreeIndexBuilder}; use crate::query::ExecutableQuery; use crate::table::{CompactionOptions, OptimizeAction, OptimizeStats}; @@ -532,6 +537,58 @@ mod tests { assert_eq!(final_row_count, 200); } + #[tokio::test] + async fn test_optimize_vector_index_after_delete_with_stable_row_ids() { + const NUM_ROWS: i32 = 400; + const DIMENSION: i32 = 32; + + let conn = connect("memory://").execute().await.unwrap(); + let vectors = FixedSizeListArray::try_new_from_values( + Float32Array::from_iter_values((0..NUM_ROWS).flat_map(|id| { + (0..DIMENSION).map(move |offset| ((id as f32 * 0.1) + (offset as f32 * 0.3)).sin()) + })), + DIMENSION, + ) + .unwrap(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("vector", vectors.data_type().clone(), false), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from_iter_values(0..NUM_ROWS)), + Arc::new(vectors), + ], + ) + .unwrap(); + let table = conn + .create_table("test_vector_index_optimize_after_delete", batch) + .storage_option(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true") + .execute() + .await + .unwrap(); + + table + .create_index( + &["vector"], + Index::IvfRq(IvfRqIndexBuilder::default().num_partitions(4)), + ) + .execute() + .await + .unwrap(); + table.delete("id % 3 = 0").await.unwrap(); + + // Regression test for #3330: deleted stable row IDs used to become + // misaligned with row addresses while joining small IVF partitions. + table + .optimize(OptimizeAction::Index(Default::default())) + .await + .unwrap(); + + assert_eq!(table.count_rows(None).await.unwrap(), 266); + } + #[tokio::test] async fn test_optimize_all() { let conn = connect("memory://").execute().await.unwrap(); From b20696ef9ca2062165417d68c0a891260b3f8e0b Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:48:56 +0800 Subject: [PATCH 19/49] fix(remote): validate cloud DNS hostnames (#3845) ## Summary - validate the generated LanceDB Cloud hostname during connection setup - return a clear invalid-input error for empty, overlong, or oversized DNS names before network resolution - add Rust and Python regression coverage for malformed `db://` authorities ## Root cause The `db://` authority and region were interpolated into the Cloud API hostname without DNS length validation. Empty or overlong labels therefore reached the resolver and surfaced as an opaque IDNA `UnicodeError` instead of a useful connection error. ## Validation - `cargo test --quiet --features remote -p lancedb test_rejects_invalid_cloud_dns_hostname --lib` - `cargo check --quiet --features remote --tests --examples` - `uv run --no-sync --extra tests pytest python/tests/test_remote_db.py::test_async_remote_db python/tests/test_remote_db.py::test_connect_rejects_invalid_cloud_dns_hostname -q` - `cargo fmt --all -- --check` - `ruff check .` - `ruff format --check python/python/tests/test_remote_db.py` Fixes #799 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_remote_db.py | 6 +++ rust/lancedb/src/remote/client.rs | 60 ++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index d5d3569d3..ce8d5bd6e 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -35,6 +35,12 @@ def make_mock_http_handler(handler): return MockLanceDBHandler +@pytest.mark.parametrize("db_name", ["a" * 64, "invalid..database"]) +def test_connect_rejects_invalid_cloud_dns_hostname(db_name): + with pytest.raises(ValueError, match="DNS labels must contain 1 to 63 bytes"): + lancedb.connect(f"db://{db_name}", api_key="fake") + + @contextlib.contextmanager def mock_lancedb_connection(handler): with http.server.HTTPServer( diff --git a/rust/lancedb/src/remote/client.rs b/rust/lancedb/src/remote/client.rs index 9e34fca9f..57dd89890 100644 --- a/rust/lancedb/src/remote/client.rs +++ b/rust/lancedb/src/remote/client.rs @@ -373,6 +373,37 @@ pub fn parse_db_url(db_url: &str) -> Result { Ok(ParsedDbUrl { db_name, db_prefix }) } +fn validate_dns_hostname(hostname: &str) -> Result<()> { + let ascii_hostname = match url::Host::parse(hostname) { + Ok(url::Host::Domain(hostname)) => hostname, + Ok(_) => { + return Err(Error::InvalidInput { + message: "LanceDB Cloud database URI or region produced a non-DNS hostname" + .to_string(), + }); + } + Err(err) => { + return Err(Error::InvalidInput { + message: format!( + "LanceDB Cloud database URI or region produced an invalid hostname: {err}" + ), + }); + } + }; + + if ascii_hostname.len() > 253 + || ascii_hostname + .split('.') + .any(|label| label.is_empty() || label.len() > 63) + { + return Err(Error::InvalidInput { + message: "LanceDB Cloud database URI or region produced an invalid hostname: DNS labels must contain 1 to 63 bytes and the full hostname must not exceed 253 bytes".to_string(), + }); + } + + Ok(()) +} + impl RestfulLanceDbClient { fn get_timeout(passed: Option, env_var: &str) -> Result> { if let Some(passed) = passed { @@ -480,7 +511,11 @@ impl RestfulLanceDbClient { let host = match host_override { Some(host_override) => host_override, - None => format!("https://{}.{}.api.lancedb.com", parsed_url.db_name, region), + None => { + let hostname = format!("{}.{}.api.lancedb.com", parsed_url.db_name, region); + validate_dns_hostname(&hostname)?; + format!("https://{hostname}") + } }; debug!("Created client for host: {}", host); let retry_config = client_config.retry_config.clone().try_into()?; @@ -1157,6 +1192,29 @@ mod tests { assert_eq!(headers.get("x-api-key").unwrap(), "api-key"); } + #[test] + fn test_rejects_invalid_cloud_dns_hostname() { + let invalid_database_names = ["a".repeat(64), "invalid..database".to_string()]; + + for db_name in invalid_database_names { + let parsed_url = parse_db_url(&format!("db://{db_name}")).unwrap(); + let error = RestfulLanceDbClient::::try_new( + &parsed_url, + "us-east-1", + None, + HeaderMap::new(), + ClientConfig::default(), + None, + ) + .unwrap_err(); + + assert!( + matches!(error, Error::InvalidInput { ref message } if message.contains("DNS labels must contain 1 to 63 bytes")), + "unexpected error: {error}" + ); + } + } + // Test implementation of HeaderProvider #[derive(Debug, Clone)] struct TestHeaderProvider { From cc0139c136742e18d4a4038ca021d3725255110e Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:49:13 +0800 Subject: [PATCH 20/49] test(node): cover foreign Float64 vector schema workflow (#3844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - add an end-to-end regression for schemas created by a different Apache Arrow package instance - cover seeded table creation, filtered scanning, and Float64 vector search across Arrow 15–18 ## Root cause Apache Arrow's runtime identity checks historically rejected schemas created by another installed Arrow instance, producing the constructor failures reported in the issue. LanceDB's peer dependency and foreign-schema sanitization now handle that boundary, but the complete reported workflow was only covered by separate unit tests. This regression keeps the repaired behavior protected end to end. ## Validation - `pnpm exec jest --runInBand __test__/table.test.ts` (281 passed) - `pnpm lint-ci` - `pnpm build` - `pnpm run docs` Fixes #882 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/table.test.ts | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 4cad365af..15d6e0804 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -86,6 +86,44 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( await expect(table.countRows()).resolves.toBe(3); }); + it("should support a foreign Float64 vector schema end to end", async () => { + const conn = await connect(tmpDir.name); + const schema = new arrow.Schema([ + new arrow.Field("resource_id", new arrow.Int32(), false), + new arrow.Field( + "vector", + new arrow.FixedSizeList( + 3, + new arrow.Field("value", new arrow.Float64(), true), + ), + false, + ), + ]); + const data = [ + { + // biome-ignore lint/style/useNamingConvention: matches the reported schema + resource_id: 0, + vector: [0.1, 0.1, 0.1], + }, + ]; + + const resources = await conn.createTable("resources", data, { schema }); + + const existing = await resources + .query() + .where("resource_id = 0") + .limit(1) + .toArray(); + expect(existing).toHaveLength(1); + + const matched = await resources + .search(Float64Array.from(data[0].vector)) + .limit(1) + .toArray(); + expect(matched).toHaveLength(1); + expect(matched[0]["resource_id"]).toBe(0); + }); + it("should support branches", async () => { await table.add([{ id: 1 }]); expect(await table.countRows()).toBe(1); From e6444ecc058e909893bb0a2974e37ff45738da0b Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:49:49 +0800 Subject: [PATCH 21/49] fix(rust): handle missing mirrored copy sources (#3843) ## Summary - treat `NotFound` from the mirrored secondary copy as a cache miss while preserving every other secondary error - perform the durable primary copy after either a successful secondary copy or a secondary cache miss - cover both an initially missing secondary manifest and eviction immediately before the secondary copy ## Root cause Readers can use process-local secondary stores that do not contain a staging manifest written by another process, or that evict it before finalization. `MirroringObjectStore::copy_opts` propagated that secondary `NotFound`, so older object_store versions could loop indefinitely and the locked version aborted before performing the durable primary copy. ## Validation - `cargo fmt --all -- --check` - `cargo test --quiet --features remote -p lancedb io::object_store::test::test_copy_when -- --nocapture` - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` - `cargo test --quiet --features remote --tests` Fixes #1176 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/io/object_store.rs | 147 +++++++++++++++++++++++++++- 1 file changed, 143 insertions(+), 4 deletions(-) diff --git a/rust/lancedb/src/io/object_store.rs b/rust/lancedb/src/io/object_store.rs index d27357b82..d594bd857 100644 --- a/rust/lancedb/src/io/object_store.rs +++ b/rust/lancedb/src/io/object_store.rs @@ -132,9 +132,14 @@ impl ObjectStore for MirroringObjectStore { if to.primary_only() { self.primary.copy_opts(from, to, options).await } else { - self.secondary.copy_opts(from, to, options.clone()).await?; - self.primary.copy_opts(from, to, options).await?; - Ok(()) + // The secondary store can be process-local and less durable than the + // primary, so a source written by another process may not exist here + // or may be evicted before the copy begins. + match self.secondary.copy_opts(from, to, options.clone()).await { + Ok(()) | Err(Error::NotFound { .. }) => {} + Err(err) => return Err(err), + } + self.primary.copy_opts(from, to, options).await } } } @@ -192,7 +197,8 @@ mod test { use futures::TryStreamExt; use lance::{dataset::WriteParams, io::ObjectStoreParams}; use lance_testing::datagen::{BatchGenerator, IncrementingInt32, RandomVector}; - use object_store::local::LocalFileSystem; + use object_store::{local::LocalFileSystem, memory::InMemory}; + use std::time::Duration; use tempfile; use crate::{ @@ -201,6 +207,139 @@ mod test { table::WriteOptions, }; + #[derive(Debug)] + struct EvictBeforeCopyStore { + inner: Arc, + } + + impl std::fmt::Display for EvictBeforeCopyStore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "EvictBeforeCopyStore") + } + } + + #[async_trait] + impl ObjectStore for EvictBeforeCopyStore { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + options: PutOptions, + ) -> Result { + self.inner.put_opts(location, payload, options).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + options: PutMultipartOptions, + ) -> Result> { + self.inner.put_multipart_opts(location, options).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> Result { + self.inner.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, Result>, + ) -> BoxStream<'static, Result> { + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> { + self.inner.delete(from).await?; + self.inner.copy_opts(from, to, options).await + } + } + + #[tokio::test] + async fn test_copy_when_source_is_missing_from_secondary() { + let primary_dir = tempfile::tempdir().unwrap(); + let secondary_dir = tempfile::tempdir().unwrap(); + let primary: Arc = + Arc::new(LocalFileSystem::new_with_prefix(primary_dir.path()).unwrap()); + let secondary: Arc = + Arc::new(LocalFileSystem::new_with_prefix(secondary_dir.path()).unwrap()); + let store = MirroringObjectStore { + primary: primary.clone(), + secondary: secondary.clone(), + }; + let staging = Path::from("_versions/1.manifest-staging"); + let finalized = Path::from("_versions/1.manifest"); + + primary + .put(&staging, "manifest contents".into()) + .await + .unwrap(); + + tokio::time::timeout(Duration::from_secs(5), store.copy(&staging, &finalized)) + .await + .expect("copy should not hang when the secondary source is missing") + .unwrap(); + + let copied = primary + .get(&finalized) + .await + .unwrap() + .bytes() + .await + .unwrap(); + assert_eq!(copied, "manifest contents"); + assert!(matches!( + secondary.head(&finalized).await, + Err(Error::NotFound { .. }) + )); + } + + #[tokio::test] + async fn test_copy_when_secondary_source_disappears_after_head() { + let primary: Arc = Arc::new(InMemory::new()); + let secondary_inner: Arc = Arc::new(InMemory::new()); + let secondary: Arc = Arc::new(EvictBeforeCopyStore { + inner: secondary_inner.clone(), + }); + let store = MirroringObjectStore { + primary: primary.clone(), + secondary, + }; + let staging = Path::from("_versions/1.manifest-staging"); + let finalized = Path::from("_versions/1.manifest"); + + primary + .put(&staging, "manifest contents".into()) + .await + .unwrap(); + secondary_inner + .put(&staging, "manifest contents".into()) + .await + .unwrap(); + + store.copy(&staging, &finalized).await.unwrap(); + + let copied = primary + .get(&finalized) + .await + .unwrap() + .bytes() + .await + .unwrap(); + assert_eq!(copied, "manifest contents"); + assert!(matches!( + secondary_inner.head(&finalized).await, + Err(Error::NotFound { .. }) + )); + } + // This test is ignored because lance 3.0 introduced LocalWriter optimization // that bypasses the object store wrapper for local writes. The mirroring feature // still works for remote/cloud storage, but can't be tested with local storage. From 1493ece3de78fcae62b0620ae12383a7424af97b Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:50:05 +0800 Subject: [PATCH 22/49] test(node): cover remote table server errors (#3841) ## Summary - add a public Node API regression test for JSON server errors from remote table operations - verify countRows reports the server message instead of an ArrayBuffer decoding TypeError ## Root cause and fix The former TypeScript remote HTTP client passed an Axios-decoded JSON error object to TextDecoder, which masked the server response with an ArrayBuffer TypeError. The current Rust-backed remote client consumes non-success response bodies as text and propagates them through the Node error chain. This test exercises that corrected path through countRows and prevents the original failure from regressing. ## Validation - pnpm build - pnpm lint-ci - pnpm test --runInBand __test__/remote.test.ts - pnpm run docs Fixes #825 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/remote.test.ts | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 89a9e992c..e766b3d2a 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -170,6 +170,38 @@ describe("remote connection", () => { ); }); + it("surfaces JSON server errors from remote table operations", async () => { + await withMockDatabase( + (req, res) => { + const path = req.url ?? ""; + if (path.endsWith("/describe/")) { + res.writeHead(200, { "Content-Type": "application/json" }).end( + JSON.stringify({ + name: "broken_table", + version: 1, + schema: { fields: [] }, + }), + ); + return; + } + + if (path.endsWith("/count_rows/")) { + res + .writeHead(400, { "Content-Type": "application/json" }) + .end(JSON.stringify({ error: "count rows failed" })); + return; + } + + res.writeHead(404).end(); + }, + async (db) => { + const table = await db.openTable("broken_table"); + + await expect(table.countRows()).rejects.toThrow("count rows failed"); + }, + ); + }); + it("should pass on requested extra headers", async () => { await withMockDatabase( (req, res) => { From 62fe413a523b06775fe93d410c68994f3002dcc7 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Thu, 6 Aug 2026 01:53:09 -0700 Subject: [PATCH 23/49] fix: percent-encode index names in per-index remote REST paths (#3840) Nothing validates index names, so a `/` in one is reachable, and the remote client interpolates it straight into the URL, splitting the path so the router 404s. The index then reads back as missing and cannot be dropped, while `create_index` keeps succeeding because it sends the name in the body. Encode at the three affected sites, mirroring `fetch_blob_files`. The shared Rust client covers all bindings. --- rust/lancedb/src/remote/table.rs | 50 ++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 5fefabeb2..29b6b698d 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2791,9 +2791,10 @@ impl BaseTable for RemoteTable { } async fn index_stats(&self, index_name: &str) -> Result> { + let encoded_name = urlencoding::encode(index_name); let mut request = self.post_read(&format!( - "/v1/table/{}/index/{}/stats/", - self.identifier, index_name + "/v1/table/{}/index/{encoded_name}/stats/", + self.identifier )); let version = self.current_version().await; let mut body = serde_json::json!({ "version": version }); @@ -2820,9 +2821,10 @@ impl BaseTable for RemoteTable { } async fn drop_index(&self, index_name: &str) -> Result<()> { + let encoded_name = urlencoding::encode(index_name); let request = self.apply_branch_query(self.client.post(&format!( - "/v1/table/{}/index/{}/drop/", - self.identifier, index_name + "/v1/table/{}/index/{encoded_name}/drop/", + self.identifier ))); let (request_id, response) = self.send(request, true).await?; if response.status() == StatusCode::NOT_FOUND { @@ -2835,9 +2837,10 @@ impl BaseTable for RemoteTable { } async fn prewarm_index(&self, index_name: &str) -> Result<()> { + let encoded_name = urlencoding::encode(index_name); let request = self.client.post(&format!( - "/v1/table/{}/index/{}/prewarm/", - self.identifier, index_name + "/v1/table/{}/index/{encoded_name}/prewarm/", + self.identifier )); let (request_id, response) = self.send(request, true).await?; if response.status() == StatusCode::NOT_FOUND { @@ -6489,6 +6492,41 @@ mod tests { assert!(matches!(e, Error::IndexNotFound { .. })); } + /// Index names are unvalidated, so reserved characters must be + /// percent-encoded or they restructure the request path. + #[tokio::test] + async fn test_per_index_paths_encode_reserved_characters() { + const NAME: &str = "my/index?a#b c"; + const PREFIX: &str = "/v1/table/my_table/index/my%2Findex%3Fa%23b%20c"; + + let table = Table::new_with_handler("my_table", |request| { + assert_eq!(request.url().path(), format!("{PREFIX}/stats/")); + let body = serde_json::json!({ + "num_indexed_rows": 1, + "num_unindexed_rows": 0, + "index_type": "IVF_PQ", + "distance_type": "l2" + }); + http::Response::builder() + .status(200) + .body(serde_json::to_string(&body).unwrap()) + .unwrap() + }); + assert!(table.index_stats(NAME).await.unwrap().is_some()); + + let table = Table::new_with_handler("my_table", |request| { + assert_eq!(request.url().path(), format!("{PREFIX}/drop/")); + http::Response::builder().status(200).body("{}").unwrap() + }); + table.drop_index(NAME).await.unwrap(); + + let table = Table::new_with_handler("my_table", |request| { + assert_eq!(request.url().path(), format!("{PREFIX}/prewarm/")); + http::Response::builder().status(200).body("{}").unwrap() + }); + table.prewarm_index(NAME).await.unwrap(); + } + #[tokio::test] async fn test_set_lsm_write_spec_unsharded() { let table = Table::new_with_handler("my_table", |request| { From 9707966943ec2925f8f7c5572997ec4e00fccb34 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:53:32 +0800 Subject: [PATCH 24/49] test(rust): cover named memory databases on Windows (#3839) ## Summary\n\n- add a create-table regression for a named database\n- assert that the derived table URI uses URL separators\n- restore the four query tests that were moved to temporary files for #1051\n\n## Root cause\n\n historically joined table names with . On Windows this inserted a backslash into , so Lance interpreted the URI as an invalid local filename. The production URI builder now preserves forward slashes for URI schemes; this change restores the issue-specific tests and adds direct regression coverage for table creation and the derived URI.\n\n## Validation\n\n- \n- \n- (passes with four pre-existing warnings in unrelated remote-table code)\n- running 814 tests ....................................................................................... 87/814 .....................................i................................................. 174/814 ....................................................................................... 261/814 ....................................................................................... 348/814 ....................................................................................... 435/814 ....................................................................................... 522/814 ....................................................................................... 609/814 ....................................................................................... 696/814 ....................................................................................... 783/814 ............................... test result: ok. 813 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 7.76s running 39 tests ....................................... test result: ok. 39 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.23s running 6 tests ...... test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s running 5 tests ..... test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.10s running 0 tests test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s running 2 tests .. test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s running 2 tests .. test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s (867 passed, 1 ignored)\n- focused named-memory create and restored query tests\n\nFixes #1051\n\n Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/connection/create_table.rs | 11 +++++++ rust/lancedb/src/query.rs | 32 +++------------------ 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/rust/lancedb/src/connection/create_table.rs b/rust/lancedb/src/connection/create_table.rs index 66f6dfa8d..b10141beb 100644 --- a/rust/lancedb/src/connection/create_table.rs +++ b/rust/lancedb/src/connection/create_table.rs @@ -202,6 +202,17 @@ mod tests { assert_eq!(table.count_rows(None).await.unwrap(), 0); } + #[tokio::test] + async fn create_table_in_named_memory_database() { + let db = connect("memory://foo").execute().await.unwrap(); + let batch = record_batch!(("id", Int64, [1, 2, 3])).unwrap(); + + let table = db.create_table("my_table", batch).execute().await.unwrap(); + + assert_eq!(table.uri().await.unwrap(), "memory://foo/my_table.lance"); + assert_eq!(table.count_rows(None).await.unwrap(), 3); + } + async fn test_create_table_with_data(data: T) where T: Scannable + 'static, diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index b76865043..b2c5fefbe 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -1661,14 +1661,8 @@ mod tests { #[tokio::test] async fn test_setters_getters() { - // TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051 - // is fixed - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("test.lance"); - let uri = dataset_path.to_str().unwrap(); - let batches = make_test_batches(); - let conn = connect(uri).execute().await.unwrap(); + let conn = connect("memory://foo").execute().await.unwrap(); let table = conn .create_table("my_table", batches) .execute() @@ -1763,14 +1757,8 @@ mod tests { #[tokio::test] async fn test_execute() { - // TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051 - // is fixed - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("test.lance"); - let uri = dataset_path.to_str().unwrap(); - let batches = make_non_empty_batches(); - let conn = connect(uri).execute().await.unwrap(); + let conn = connect("memory://foo").execute().await.unwrap(); let table = conn .create_table("my_table", batches) .execute() @@ -1889,14 +1877,8 @@ mod tests { #[tokio::test] async fn test_select_with_transform() { - // TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051 - // is fixed - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("test.lance"); - let uri = dataset_path.to_str().unwrap(); - let batches = make_non_empty_batches(); - let conn = connect(uri).execute().await.unwrap(); + let conn = connect("memory://foo").execute().await.unwrap(); let table = conn .create_table("my_table", batches) .execute() @@ -1993,15 +1975,9 @@ mod tests { #[tokio::test] async fn test_execute_no_vector() { - // TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051 - // is fixed - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("test.lance"); - let uri = dataset_path.to_str().unwrap(); - // test that it's ok to not specify a query vector (just filter / limit) let batches = make_non_empty_batches(); - let conn = connect(uri).execute().await.unwrap(); + let conn = connect("memory://foo").execute().await.unwrap(); let table = conn .create_table("my_table", batches) .execute() From 1c3cd1d9184bb8c6d4088e077bd0825ef1e78235 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:54:04 +0800 Subject: [PATCH 25/49] fix(python): accept Arrow scalars in table updates (#3838) ## Summary - convert PyArrow scalar values through their Python representation before SQL literal rendering - add an end-to-end regression for updating a fixed-size-list vector from a queried FixedSizeListScalar ## Root cause Python update literal conversion used single dispatch for native Python and NumPy values but had no PyArrow Scalar registration. A FixedSizeListScalar returned by a query therefore reached the unsupported generic conversion instead of the existing recursive list converter. ## Validation - uv run --extra tests pytest python/tests/test_table.py::test_update python/tests/test_table.py::test_update_with_arrow_scalar python/tests/test_table.py::test_update_types -q - uv run --extra tests pytest python/tests/test_util.py -q - uv run --project python --extra tests --extra dev ruff format --check python/python/lancedb/util.py python/python/tests/test_table.py - uv run --project python --extra tests --extra dev ruff check . Fixes #1228 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/lancedb/util.py | 5 +++++ python/python/tests/test_table.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/python/python/lancedb/util.py b/python/python/lancedb/util.py index f582be7b4..dbc52bff6 100644 --- a/python/python/lancedb/util.py +++ b/python/python/lancedb/util.py @@ -395,6 +395,11 @@ def _(value: dict): ) +@value_to_sql.register(pa.Scalar) +def _(value: pa.Scalar): + return value_to_sql(value.as_py()) + + @value_to_sql.register(np.ndarray) def _(value: np.ndarray): return value_to_sql(value.tolist()) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index f20f42617..b2bfa2a68 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -2282,6 +2282,20 @@ def test_update(mem_db: DBConnection): assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]])) +def test_update_with_arrow_scalar(mem_db: DBConnection): + schema = pa.schema({"id": pa.int64(), "vector": pa.list_(pa.float32(), 4)}) + table = mem_db.create_table("my_table", schema=schema) + table.add([{"id": 1, "vector": [1.0, 2.0, 3.0, 4.0]}]) + + value = table.search().select(["vector"]).limit(1).to_arrow()["vector"][0] + assert isinstance(value, pa.FixedSizeListScalar) + + result = table.update(where="id == 1", values={"vector": value}) + + assert result.rows_updated == 1 + assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0, 3.0, 4.0]] + + def test_update_types(mem_db: DBConnection): table = mem_db.create_table( "my_table", From 369b10a377ed2e3bafd86d5999ac80d2e8aba49e Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:54:59 +0800 Subject: [PATCH 26/49] test(rust): cover object store reuse on table open (#3831) ## Summary - add regression coverage for repeated table opens through one database connection - assert that each open reuses the connection object-store client without another registry miss - exercise the table after every open so the test covers the complete dataset-loading path ## Root cause At the commit reported in #1600, opening a table constructed a separate object-store client rather than reusing the client that had already connected to the database. On S3 this repeated credential discovery, which could fail intermittently in AWS Lambda and surface as TableNotFound. The connection-owned Session reuse added later fixed the runtime path, but no focused test protected the open-table invariant. ## Fix Add a regression test backed by ObjectStoreRegistry statistics. Three successive opens must add cache hits while leaving the miss count unchanged, proving that open_table uses the connection Session and its authenticated object-store client. ## Validation - cargo fmt --all - cargo test --quiet --features remote -p lancedb database::listing::tests::test_open_table_reuses_connection_object_store - cargo check --quiet --features remote --tests --examples - cargo clippy --quiet --features remote --tests --examples - cargo test --quiet --features remote --tests Fixes #1600 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/database/listing.rs | 62 ++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index 454498d54..4fea6767c 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -1376,6 +1376,68 @@ mod tests { assert!(!tempdir.path().join("__manifest").exists()); } + /// Regression test for https://github.com/lancedb/lancedb/issues/1600. + /// + /// Opening a table used to create a separate object-store client instead of + /// reusing the one that successfully connected to the database. Repeating + /// credential discovery made S3 table opens intermittent, especially in AWS + /// Lambda, and the failed open was reported as `TableNotFound`. + #[tokio::test] + async fn test_open_table_reuses_connection_object_store() { + let tempdir = tempdir().unwrap(); + let uri = tempdir.path().to_str().unwrap(); + let registry = Arc::new(lance_io::object_store::ObjectStoreRegistry::default()); + let session = Arc::new(lance::session::Session::new(16, 16, registry.clone())); + + let request = ConnectRequest { + uri: uri.to_string(), + #[cfg(feature = "remote")] + client_config: Default::default(), + options: Default::default(), + namespace_client_properties: Default::default(), + manifest_enabled: false, + read_consistency_interval: None, + session: Some(session), + }; + let db = ListingDatabase::connect_with_options(&request) + .await + .unwrap(); + + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + db.create_table(CreateTableRequest { + name: "test".to_string(), + namespace_path: vec![], + data: Box::new(RecordBatch::new_empty(schema)) as Box, + mode: CreateTableMode::Create, + write_options: Default::default(), + location: None, + namespace_client: None, + }) + .await + .unwrap(); + + let before_open = registry.stats(); + for _ in 0..3 { + let table = db + .open_table(OpenTableRequest { + name: "test".to_string(), + namespace_path: vec![], + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await + .unwrap(); + assert_eq!(table.count_rows(None).await.unwrap(), 0); + } + + let after_open = registry.stats(); + assert_eq!(after_open.misses, before_open.misses); + assert!(after_open.hits >= before_open.hits + 3); + } + #[tokio::test] async fn test_clone_table_basic() { let (_tempdir, db) = setup_database().await; From 001237c7a42b62581dd3b43b39ede68eb3ce6cf3 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Thu, 6 Aug 2026 12:00:22 -0700 Subject: [PATCH 27/49] chore: update lance dependency to v11.0.0-beta.2 (#3886) Updates the Lance dependencies and Java lance-core to v11.0.0-beta.2. Includes required compatibility fixes for the LanceFileVersion module move and the updated GooseFS/OpenDAL dependency. Lance tag: https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.2 --------- Co-authored-by: Daniel Rammer Co-authored-by: Claude Opus 5 (1M context) --- .github/workflows/rust.yml | 14 +- Cargo.lock | 492 +++++++++++++----------- Cargo.toml | 28 +- java/pom.xml | 2 +- rust/lancedb/Cargo.toml | 4 +- rust/lancedb/src/blob.rs | 2 +- rust/lancedb/src/connection.rs | 2 +- rust/lancedb/src/database/listing.rs | 2 +- rust/lancedb/src/database/namespace.rs | 4 +- rust/lancedb/src/remote/table.rs | 8 +- rust/lancedb/src/remote/table/blobs.rs | 2 +- rust/lancedb/src/remote/table/insert.rs | 4 +- rust/lancedb/tests/blob_integration.rs | 2 +- 13 files changed, 297 insertions(+), 269 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1a0b65c4a..471da43e0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -296,16 +296,18 @@ jobs: cargo update -p aws-types --precise 1.3.9 cargo update -p aws-sigv4 --precise 1.3.5 cargo update -p aws-credential-types --precise 1.2.8 - cargo update -p aws-smithy-checksums --precise 0.63.9 + # aws-smithy-checksums must stay at or above 0.63.13: OpenDAL's S3 + # service needs crc-fast ~1.9, and older releases pin it to ~1.3. + cargo update -p aws-smithy-checksums --precise 0.63.13 cargo update -p aws-smithy-runtime --precise 1.9.3 - cargo update -p aws-smithy-http --precise 0.62.4 - cargo update -p aws-smithy-eventstream --precise 0.60.12 + cargo update -p aws-smithy-http --precise 0.62.6 + cargo update -p aws-smithy-eventstream --precise 0.60.14 cargo update -p aws-smithy-http-client --precise 1.1.3 cargo update -p aws-smithy-observability --precise 0.1.4 cargo update -p aws-smithy-query --precise 0.60.8 - cargo update -p aws-smithy-runtime-api --precise 1.9.1 - cargo update -p aws-smithy-async --precise 1.2.6 - cargo update -p aws-smithy-types --precise 1.3.5 + cargo update -p aws-smithy-runtime-api --precise 1.9.3 + cargo update -p aws-smithy-async --precise 1.2.7 + cargo update -p aws-smithy-types --precise 1.3.6 cargo update -p aws-smithy-xml --precise 0.60.11 cargo update -p home --precise 0.5.9 - name: cargo +${{ matrix.msrv }} check diff --git a/Cargo.lock b/Cargo.lock index 995424b20..93e16c06d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -775,7 +775,7 @@ dependencies = [ "http 0.2.12", "http 1.5.0", "http-body 1.1.0", - "lru", + "lru 0.16.4", "percent-encoding", "regex-lite", "sha2 0.11.0", @@ -1241,6 +1241,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1964,15 +1970,6 @@ dependencies = [ "spin 0.10.1", ] -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - [[package]] name = "crc32fast" version = "1.5.0" @@ -2174,9 +2171,9 @@ dependencies = [ [[package]] name = "ctor" -version = "1.0.5" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378f0974ae2468eaf63aa036dbe9c926b0dc7ea64c156f2ea618bc2f75b934f0" +checksum = "2d83cb7e7a873830708d6b02a78cd36a592c6fa14bf267b68725103b85c0d77f" dependencies = [ "link-section", "linktime-proc-macro", @@ -2902,6 +2899,37 @@ dependencies = [ "uuid", ] +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "der" version = "0.6.1" @@ -3413,6 +3441,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "frostem" +version = "1.20260804.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82eb03a32a1d50555353c85a7b9d3279a6f1e91af9890b789acdf544ed57c8d7" + [[package]] name = "fs_extra" version = "1.3.0" @@ -3421,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "rand 0.9.5", @@ -3779,14 +3813,23 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.1.5" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" +checksum = "e1ea4eee6dcbc31b25ab4fd577adc55b677d2bed3aa3016c44c58fbe1b2298a5" dependencies = [ + "arc-swap", "async-trait", "bytes", "dashmap", + "fastrand", + "futures", "hostname", + "io-uring", + "itoa", + "libc", + "lru 0.18.2", + "memmap2 0.9.10", + "moka", "prost", "prost-types", "rand 0.9.5", @@ -3799,6 +3842,7 @@ dependencies = [ "tonic-prost", "tracing", "uuid", + "xxhash-rust", ] [[package]] @@ -4599,10 +4643,12 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.24" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ + "defmt", + "jiff-core", "jiff-static", "jiff-tzdb-platform", "js-sys", @@ -4611,15 +4657,25 @@ dependencies = [ "portable-atomic-util", "serde_core", "wasm-bindgen", - "windows-sys 0.61.2", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", ] [[package]] name = "jiff-static" -version = "0.2.24" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" dependencies = [ + "jiff-core", "proc-macro2", "quote", "syn 2.0.117", @@ -4731,24 +4787,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "jsonwebtoken" -version = "10.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" -dependencies = [ - "aws-lc-rs", - "base64 0.22.1", - "getrandom 0.2.17", - "js-sys", - "pem", - "serde", - "serde_json", - "signature 2.2.0", - "simple_asn1", - "zeroize", -] - [[package]] name = "kanaria" version = "0.2.0" @@ -4777,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arc-swap", "arrow", @@ -4852,8 +4890,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-buffer", @@ -4875,7 +4913,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-buffer", @@ -4889,7 +4927,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-schema", @@ -4898,8 +4936,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrayref", "crunchy", @@ -4909,8 +4947,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-buffer", @@ -4950,8 +4988,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow", "arrow-array", @@ -4981,8 +5019,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow", "arrow-array", @@ -4999,8 +5037,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "proc-macro2", "quote", @@ -5009,8 +5047,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-arith", "arrow-array", @@ -5036,7 +5074,6 @@ dependencies = [ "prost", "prost-build", "rand 0.9.5", - "strum 0.26.3", "tokio", "tracing", "xxhash-rust", @@ -5045,8 +5082,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-arith", "arrow-array", @@ -5077,8 +5114,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arc-swap", "arrow", @@ -5145,8 +5182,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-schema", @@ -5168,8 +5205,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow", "arrow-array", @@ -5181,7 +5218,6 @@ dependencies = [ "bytes", "chrono", "futures", - "goosefs-sdk", "http 1.5.0", "io-uring", "lance-arrow", @@ -5206,8 +5242,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-buffer", @@ -5223,8 +5259,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow", "async-trait", @@ -5236,8 +5272,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow", "arrow-ipc", @@ -5291,8 +5327,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-buffer", @@ -5307,8 +5343,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow", "arrow-array", @@ -5347,8 +5383,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-schema", @@ -5361,13 +5397,13 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ + "frostem", "icu_segmenter", "jieba-rs", "lindera", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -5645,7 +5681,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml_ng", - "strum 0.28.0", + "strum", "strum_macros 0.28.0", "unicode-blocks", "unicode-normalization", @@ -5675,22 +5711,22 @@ dependencies = [ "rkyv", "serde", "serde_json", - "strum 0.28.0", + "strum", "strum_macros 0.28.0", "thiserror 2.0.18", ] [[package]] name = "link-section" -version = "0.16.1" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8600ca3dbe044f07955b443ff606c50f45295b863289bbe7d0844d50cf11e4" +checksum = "5ee1a0d6e252afe82e7bc2db42fba60e02ddf3b1accaf8cb21d96e34ba61f3d4" [[package]] name = "linktime-proc-macro" -version = "0.1.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44cd706ff0d503ee32b2071166510ca27e281228de10cd3aa8d35ff94560f81" +checksum = "348d0075b1fc163b26d72a7f75fc5141daf2fd1bdf128d873cbaf6785d495bdf" [[package]] name = "linux-raw-sys" @@ -5747,6 +5783,15 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "lru" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -6067,7 +6112,7 @@ checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941" dependencies = [ "bitflags 2.11.1", "chrono", - "ctor 1.0.5", + "ctor 1.0.12", "futures", "napi-build", "napi-sys", @@ -6091,7 +6136,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d5c9c02556ea6dc99dffd36c1ce60141411657438501a125b675776d011ce92" dependencies = [ "convert_case", - "ctor 1.0.5", + "ctor 1.0.12", "napi-derive-backend", "proc-macro2", "quote", @@ -6384,9 +6429,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" dependencies = [ "async-trait", "bytes", @@ -6447,12 +6492,13 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "opendal" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d" dependencies = [ - "ctor 1.0.5", + "ctor 1.0.12", "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -6469,24 +6515,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed" dependencies = [ "anyhow", - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures", "http 1.5.0", - "http-body 1.1.0", "jiff", "log", "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.3", "serde", "serde_json", "tokio", @@ -6496,10 +6540,24 @@ dependencies = [ ] [[package]] -name = "opendal-layer-concurrent-limit" -version = "0.57.0" +name = "opendal-http-transport-reqwest" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef" +dependencies = [ + "bytes", + "futures", + "http 1.5.0", + "http-body 1.1.0", + "opendal-core", + "reqwest 0.13.4", +] + +[[package]] +name = "opendal-layer-concurrent-limit" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081" dependencies = [ "futures", "http 1.5.0", @@ -6509,9 +6567,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c" dependencies = [ "log", "opendal-core", @@ -6519,9 +6577,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d" dependencies = [ "backon", "log", @@ -6530,9 +6588,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66" dependencies = [ "opendal-core", "tokio", @@ -6540,17 +6598,17 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "http 1.5.0", "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6561,17 +6619,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "http 1.5.0", "log", + "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6581,9 +6640,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df" dependencies = [ "http 1.5.0", "opendal-core", @@ -6591,15 +6650,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0" dependencies = [ "bytes", "http 1.5.0", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -6608,9 +6667,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6" dependencies = [ "async-trait", "bytes", @@ -6618,7 +6677,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -6629,9 +6688,9 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78" dependencies = [ "bytes", "goosefs-sdk", @@ -6643,9 +6702,9 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7" dependencies = [ "bytes", "hf-xet", @@ -6653,22 +6712,21 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "reqwest 0.13.3", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b" dependencies = [ "bytes", "http 1.5.0", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -6677,18 +6735,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", - "crc32c", + "crc-fast", "http 1.5.0", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -7583,7 +7641,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "itertools 0.14.0", "log", "multimap", @@ -7799,6 +7857,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick_cache" version = "0.6.24" @@ -8220,9 +8288,9 @@ dependencies = [ [[package]] name = "reqsign-aliyun-oss" -version = "3.0.0" +version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95" +checksum = "a5e6d659fcdbca6fe2d7ef109c2e28499b7be80501f1bb86c10caf5ec8ac1219" dependencies = [ "anyhow", "form_urlencoded", @@ -8236,38 +8304,52 @@ dependencies = [ ] [[package]] -name = "reqsign-aws-v4" -version = "3.0.0" +name = "reqsign-aws-core" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43" +checksum = "e4af084e1f3cbf3e67e0c972765399bce54ecec804cceba46b39a8331f3c1bff" dependencies = [ - "anyhow", "bytes", "form_urlencoded", + "hex", "http 1.5.0", "log", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "rust-ini", "serde", "serde_json", "serde_urlencoded", - "sha1 0.10.6", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac5b3b7cefa28933792b439186459f77f19f9b6edbeab41b8b187150361a206" +dependencies = [ + "bytes", + "http 1.5.0", + "log", + "quick-xml 0.41.0", + "reqsign-aws-core", + "reqsign-core", + "serde", ] [[package]] name = "reqsign-azure-storage" -version = "3.0.0" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6" +checksum = "2824e7da3c2cc42ac3406c674eb57c89127fdcd97f3a73c608cfc680505ea134" dependencies = [ "anyhow", - "base64 0.22.1", + "base64 0.23.1", "bytes", "form_urlencoded", "http 1.5.0", - "jsonwebtoken", "log", "pem", "percent-encoding", @@ -8275,36 +8357,38 @@ dependencies = [ "rsa", "serde", "serde_json", - "sha1 0.10.6", + "sha1 0.11.0", ] [[package]] name = "reqsign-core" -version = "3.0.0" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102" +checksum = "c07dd510b1e1b9b241883e483358147fb2ed2d497a7b39b065ba61eb93deceb0" dependencies = [ "anyhow", - "base64 0.22.1", + "base64 0.23.1", "bytes", - "form_urlencoded", "futures", "hex", - "hmac 0.12.1", + "hmac 0.13.0", "http 1.5.0", "jiff", "log", "percent-encoding", - "sha1 0.10.6", - "sha2 0.10.9", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.0", "windows-sys 0.61.2", ] [[package]] name = "reqsign-file-read-tokio" -version = "3.0.0" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5" +checksum = "663d9d55abd0df0830ef0ae43708297cc1371cf4e8ca91f3ac813c309cca8c98" dependencies = [ "anyhow", "reqsign-core", @@ -8313,13 +8397,12 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.0.0" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e" +checksum = "4080a227f82a09f68540ecd028622065d7ac4c0bcb8727a25bdcfc0526235792" dependencies = [ "form_urlencoded", "http 1.5.0", - "jsonwebtoken", "log", "percent-encoding", "reqsign-aws-v4", @@ -8327,15 +8410,14 @@ dependencies = [ "rsa", "serde", "serde_json", - "sha2 0.10.9", "tokio", ] [[package]] name = "reqsign-tencent-cos" -version = "3.0.0" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a" +checksum = "764629c90f7c3566a6d4e4641ebab9acd604ce02e16eda4d37c7d7e79e16ed90" dependencies = [ "anyhow", "http 1.5.0", @@ -8394,9 +8476,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64 0.22.1", "bytes", @@ -8457,7 +8539,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.5.0", - "reqwest 0.13.3", + "reqwest 0.13.4", "thiserror 2.0.18", "tower-service", ] @@ -8597,16 +8679,6 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-demangle" version = "0.1.27" @@ -9206,18 +9278,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" -[[package]] -name = "simple_asn1" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" -dependencies = [ - "num-bigint", - "num-traits", - "thiserror 2.0.18", - "time", -] - [[package]] name = "siphasher" version = "1.0.3" @@ -9277,7 +9337,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -9289,7 +9349,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -9492,15 +9552,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", -] - [[package]] name = "strum" version = "0.28.0" @@ -9523,19 +9574,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.117", -] - [[package]] name = "strum_macros" version = "0.28.0" @@ -9730,7 +9768,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", @@ -11161,7 +11199,7 @@ dependencies = [ "more-asserts", "rand 0.10.1", "redb", - "reqwest 0.13.3", + "reqwest 0.13.4", "reqwest-middleware", "serde", "serde_json", @@ -11274,7 +11312,7 @@ dependencies = [ "oneshot", "pin-project", "rand 0.10.1", - "reqwest 0.13.3", + "reqwest 0.13.4", "serde", "serde_json", "shellexpand", @@ -11370,20 +11408,6 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] [[package]] name = "zerotrie" diff --git a/Cargo.toml b/Cargo.toml index 35370e58f..a879d1f8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } ahash = "0.8" # Note that this one does not include pyarrow arrow = { version = "58.0.0", optional = false } diff --git a/java/pom.xml b/java/pom.xml index 1b96cbcab..e8f030b27 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 10.1.0-beta.1 + 11.0.0-beta.2 false 2.30.0 1.7 diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 816f095de..96ea9ec95 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -49,8 +49,8 @@ lance-namespace = { workspace = true } lance-namespace-impls = { workspace = true } metrics = { workspace = true, optional = true } metrics-util = { workspace = true, optional = true } -# Pin the transitive GooseFS SDK until the 0.1.6 compile break is fixed upstream. -goosefs-sdk = { version = "=0.1.5", optional = true } +# Pin the GooseFS SDK to the version required by Lance's OpenDAL dependency. +goosefs-sdk = { version = "=0.1.9", optional = true } moka = { workspace = true } pin-project = { workspace = true } tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] } diff --git a/rust/lancedb/src/blob.rs b/rust/lancedb/src/blob.rs index 3448257ab..e1c18dd84 100644 --- a/rust/lancedb/src/blob.rs +++ b/rust/lancedb/src/blob.rs @@ -17,7 +17,7 @@ use arrow_array::builder::LargeBinaryBuilder; use arrow_schema::{DataType, Field, Schema}; use lance::dataset::{BlobRangeRequest as LanceBlobRangeRequest, Dataset, WriteParams}; use lance_arrow::FieldExt; -use lance_encoding::version::LanceFileVersion; +use lance_file::version::LanceFileVersion; use lance_io::object_store::ObjectStore; use object_store::path::Path; diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 89e59e12e..dd53a2d2e 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -34,7 +34,7 @@ use crate::remote::{ db::{OPT_REMOTE_API_KEY, OPT_REMOTE_HOST_OVERRIDE, OPT_REMOTE_REGION}, }; use lance::io::ObjectStoreParams; -pub use lance_encoding::version::LanceFileVersion; +pub use lance_file::version::LanceFileVersion; #[cfg(feature = "remote")] use lance_io::object_store::StorageOptions; use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider}; diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index 4fea6767c..fea34bb48 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -12,7 +12,7 @@ use lance::dataset::refs::Ref; use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder}; use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore}; use lance_datafusion::utils::StreamingWriteSource; -use lance_encoding::version::LanceFileVersion; +use lance_file::version::LanceFileVersion; use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider}; use lance_table::io::commit::commit_handler_from_url; use object_store::local::LocalFileSystem; diff --git a/rust/lancedb/src/database/namespace.rs b/rust/lancedb/src/database/namespace.rs index d18c78682..740e11645 100644 --- a/rust/lancedb/src/database/namespace.rs +++ b/rust/lancedb/src/database/namespace.rs @@ -201,7 +201,7 @@ impl LanceNamespaceDatabase { &self, request: &DbCreateTableRequest, ) -> Result<( - Option, + Option, Option, Option, )> { @@ -214,7 +214,7 @@ impl LanceNamespaceDatabase { let storage_version_override = storage_options .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION)) - .map(|s| s.parse::()) + .map(|s| s.parse::()) .transpose()?; let v2_manifest_override = storage_options diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 29b6b698d..388bed0f7 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2942,7 +2942,7 @@ impl BaseTable for RemoteTable { } #[derive(Serialize, Clone, Debug)] -pub(crate) struct MergeInsertRequest { +pub struct MergeInsertRequest { on: String, when_matched_update_all: bool, when_matched_update_all_filt: Option, @@ -5907,16 +5907,18 @@ mod tests { .await .unwrap(); + // Positions are relative to the first retained token, so dropping the + // leading "hello" stop word does not shift the remaining tokens. assert_eq!( tokens, vec![ FtsToken { text: "こんにちは".to_string(), - position: 1, + position: 0, }, FtsToken { text: "世界".to_string(), - position: 2, + position: 1, }, ] ); diff --git a/rust/lancedb/src/remote/table/blobs.rs b/rust/lancedb/src/remote/table/blobs.rs index bb8a5d103..387d3c6dc 100644 --- a/rust/lancedb/src/remote/table/blobs.rs +++ b/rust/lancedb/src/remote/table/blobs.rs @@ -90,7 +90,7 @@ struct RemoteBlobState { /// Seekable Cloud blob handle over HTTP Range. #[derive(Debug)] -pub(crate) struct RemoteBlobFile { +pub struct RemoteBlobFile { requester: Arc, state: Mutex, closed: AtomicBool, diff --git a/rust/lancedb/src/remote/table/insert.rs b/rust/lancedb/src/remote/table/insert.rs index 67ea7765d..a4a28a9c6 100644 --- a/rust/lancedb/src/remote/table/insert.rs +++ b/rust/lancedb/src/remote/table/insert.rs @@ -33,7 +33,7 @@ use crate::table::{AddResult, MergeResult}; /// same Arrow-IPC streaming body and error side-channel; only the target /// endpoint, query parameters, and parsed result type differ. #[derive(Debug, Clone)] -pub(crate) enum WriteOp { +pub enum WriteOp { /// `add`: stream to `/v1/table/{id}/insert/`, optionally overwriting. Insert { overwrite: bool }, /// `merge_insert`: stream to `/v1/table/{id}/merge_insert/` with the merge @@ -49,7 +49,7 @@ pub(crate) enum WriteOp { /// The parsed server response for a completed write, discriminated by the /// operation that produced it. #[derive(Debug, Clone)] -pub(crate) enum WriteResult { +pub enum WriteResult { Add(AddResult), Merge(MergeResult), } diff --git a/rust/lancedb/tests/blob_integration.rs b/rust/lancedb/tests/blob_integration.rs index 3d327766e..77d49abd9 100644 --- a/rust/lancedb/tests/blob_integration.rs +++ b/rust/lancedb/tests/blob_integration.rs @@ -10,7 +10,7 @@ use arrow_array::{ use arrow_schema::{DataType, Field, Fields, Schema}; use futures::TryStreamExt; use lance::Dataset; -use lance_encoding::version::LanceFileVersion; +use lance_file::version::LanceFileVersion; use lancedb::{ Connection, Error, Result, Table, blob::{BlobRangeRequest, blob}, From b1cfe6edb15c3904001a99e98560b80970fd711e Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 7 Aug 2026 16:31:40 +0800 Subject: [PATCH 28/49] ci(docs): add scheduled doc link check (#3888) The docs have no link checking at all, so external links rot silently: a trial run already found `docs/src/python/python.md` pointing at `lancedb.github.io/lance-namespace`, which returns 404 since the repository moved to the lance-format org. Checking external links on the blocking path would be the wrong trade: third-party hosts rate-limit automated clients, reject non-browser user agents, and go down temporarily, so any of them having a bad minute would turn unrelated PRs red. Following lance-format/lance#8315, this adds a daily `lychee` run that reports broken links into a single tracking issue, rewritten in place on each run and closed automatically once every link resolves. The scan job runs the downloaded lychee binary with a read-only token; everything that writes lives in a separate report job, and a non-verdict lychee exit fails the run instead of publishing a bogus report. The check is restricted to http(s) links because much of `docs/src` is generated API reference (the `js/` tree comes from `npm run docs`) and the hand-written pages use mkdocstrings cross-references and nav-relative paths that only resolve in the site mkdocs builds, so relative links would be reported as broken on every run. The one broken link the trial run surfaced is fixed here; after the fix, a local run over all 154 files reports 0 errors across 216 unique links. --- .github/workflows/docs-link-check.yml | 222 ++++++++++++++++++++++++++ docs/src/python/python.md | 2 +- 2 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docs-link-check.yml diff --git a/.github/workflows/docs-link-check.yml b/.github/workflows/docs-link-check.yml new file mode 100644 index 000000000..0e22100eb --- /dev/null +++ b/.github/workflows/docs-link-check.yml @@ -0,0 +1,222 @@ +name: Check doc links + +# Checking external links is inherently noisy: third-party sites rate-limit +# automated clients, reject non-browser user agents, and go down temporarily. +# Blocking pull requests on that trades a lot of false failures for very little +# signal, so this runs on a schedule and reports findings in a single tracking +# issue instead of failing anyone's build. +on: + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + +# The report lives in one repository-global issue, so runs must not overlap: a +# lookup racing a create produces duplicate issues, and a healthy run closing +# the issue while a failing run only rewrites its body would leave a broken +# report closed. The group is deliberately ref-independent so that a manual +# dispatch serializes against the scheduled run. +concurrency: + group: docs-link-check + cancel-in-progress: false + +permissions: {} + +env: + REPORT_TITLE: "Docs link checker report" + +jobs: + scan: + name: Scan links + runs-on: ubuntu-24.04 + # lychee-action is pinned by SHA, but its wrapper downloads the lychee + # release tarball at run time without verifying a digest, and hands the + # resulting binary a GitHub token. Release assets remain replaceable, so + # that binary is confined to a job whose token can only read public + # content; everything that writes runs in the report job below. + permissions: + contents: read + outputs: + exit_code: ${{ steps.lychee.outputs.exit_code }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + # workflow_dispatch can run from any ref, but the report is + # repository-global. Always measure the default branch so a manual + # run from a topic branch cannot close a report that main warrants, + # or overwrite it with branch-only findings. + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Check links + id: lychee + uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0 + with: + # Restricted to http(s) on purpose. Much of docs/src is generated + # API reference (the js/ tree comes from `npm run docs` in nodejs) + # and the hand-written pages use mkdocstrings cross-references and + # nav-relative paths that only resolve in the site mkdocs builds, + # not in this checkout, so relative links would be reported as + # broken on every run. + args: >- + --scheme https + --scheme http + --no-progress + --max-retries 3 + --timeout 20 + 'docs/src/**/*.md' + format: json + output: ./lychee/out.json + jobSummary: false + # The report, not a red build, is the signal for broken links. The + # validation step below still fails the run if the check itself + # breaks. + fail: false + + - name: Validate report + # lychee does not reserve exit code 2 for broken links: its CLI + # parser also exits 2 on an invalid option, before any link was + # checked or any report written. Only a parseable report whose + # counts agree with the exit code counts as a link verdict; anything + # else fails here, and the report job below is skipped entirely, so + # the tracking issue is never touched. Exit 2 covers timeouts as + # well as errors, and a timed-out host is exactly the transient + # unavailability this report exists to surface, so both count as + # findings. Requiring total > 0 also catches a glob that silently + # stopped matching any file. + if: steps.lychee.outputs.exit_code == 0 || steps.lychee.outputs.exit_code == 2 + env: + EXIT_CODE: ${{ steps.lychee.outputs.exit_code }} + run: | + jq -e --argjson code "$EXIT_CODE" ' + (.total > 0) and + (if $code == 0 + then .errors == 0 and .timeouts == 0 + and (.error_map | length == 0) and (.timeout_map | length == 0) + else (.errors + .timeouts) > 0 + and ((.error_map | length) + (.timeout_map | length)) > 0 + end) + ' ./lychee/out.json + + - name: Upload report + if: steps.lychee.outputs.exit_code == 2 + uses: actions/upload-artifact@v7 + with: + name: link-report + path: ./lychee/out.json + retention-days: 7 + + report: + name: Update report issue + needs: scan + runs-on: ubuntu-24.04 + # Deliberately no checkout: this job needs the report artifact and the + # issues API, not the repository contents. + permissions: + issues: write + env: + EXIT_CODE: ${{ needs.scan.outputs.exit_code }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Classify checker result + # lychee exits 0 when every link resolves and 2 when links fail, + # both already cross-checked against the report by the scan job's + # validation step. Anything else (1 runtime, 3 bad config) means the + # check never produced a link verdict, which must surface as a failed + # run rather than be published as "broken documentation links". + run: | + case "$EXIT_CODE" in + 0|2) + echo "lychee exit code $EXIT_CODE" + ;; + *) + echo "::error::lychee exited with '$EXIT_CODE': the link check did not complete. Leaving the report issue untouched." + exit 1 + ;; + esac + + - name: Find existing report issue + id: report + # Matched on title alone, and through search rather than a listing: + # the issue action applies labels in a separate call after creating the + # issue, so a label filter misses a half-created report, and this + # repository has far more open issues than one listing page holds. + # Closed issues are included because a healthy run closes the report: + # an open-only lookup would forget that identity and the next failing + # run would open a duplicate. The oldest match stays the canonical + # report and is reopened below when links break again. + run: | + match=$(gh issue list --repo "$GITHUB_REPOSITORY" --state all \ + --search "in:title \"$REPORT_TITLE\" author:app/github-actions" \ + --limit 50 --json number,title,state \ + --jq "[.[] | select(.title == \"$REPORT_TITLE\")] | sort_by(.number) | first // empty") + echo "number=$(jq -r '.number // empty' <<<"$match")" >> "$GITHUB_OUTPUT" + echo "state=$(jq -r '.state // empty' <<<"$match")" >> "$GITHUB_OUTPUT" + + - name: Download report + if: env.EXIT_CODE == 2 + uses: actions/download-artifact@v8 + with: + name: link-report + path: ./lychee + + - name: Compose report + if: env.EXIT_CODE == 2 + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + { + echo "Broken documentation links found by [\`$GITHUB_WORKFLOW\`]($run_url)." + echo + echo "This issue is rewritten by every scheduled run and closed automatically once all links resolve." + echo + echo "Entries can be false positives: some sites rate-limit or block automated clients while working fine in a browser. Confirm before editing the docs, and add persistent offenders to \`--exclude\` in \`.github/workflows/docs-link-check.yml\`." + echo + # Timeouts are reported alongside errors: entries land in + # timeout_map with a status text instead of an HTTP code. + jq -r ' + "\(.errors) of \(.total) links failed, \(.timeouts) timed out.", + "", + ([(.error_map | to_entries[]), (.timeout_map | to_entries[])] + | group_by(.key)[] | + "### Errors in \(.[0].key)", + "", + (map(.value[])[] | "* [\(.status.code // .status.text // "ERR")] <\(.url)> — \(.status.details // .status.text // "unknown error")"), + "") + ' ./lychee/out.json + } > ./lychee/issue.md + + - name: Reopen report issue + # A healthy run closes the report, and the issue action below only + # rewrites the body of whatever number it is given. Without an + # explicit reopen, the 2 -> 0 -> 2 sequence would keep rewriting a + # closed issue while links are broken. A CLOSED state implies the + # lookup found a canonical issue, so no separate emptiness check. + if: env.EXIT_CODE == 2 && steps.report.outputs.state == 'CLOSED' + env: + ISSUE_NUMBER: ${{ steps.report.outputs.number }} + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + gh issue reopen "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --comment "Broken documentation links found again in [the latest run]($run_url)." + + - name: Report broken links + if: env.EXIT_CODE == 2 + uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6.0.0 + with: + # Empty on the first failing run, which creates the issue; afterwards + # the same issue is updated in place. + issue-number: ${{ steps.report.outputs.number }} + title: ${{ env.REPORT_TITLE }} + content-filepath: ./lychee/issue.md + labels: documentation + + - name: Close report issue once links are healthy + # An OPEN state implies the lookup found a canonical issue; a report + # that is already closed needs nothing. + if: env.EXIT_CODE == 0 && steps.report.outputs.state == 'OPEN' + env: + ISSUE_NUMBER: ${{ steps.report.outputs.number }} + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + gh issue close "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --comment "All documentation links resolved in [the latest run]($run_url)." diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 36044d35d..3dd6f59f4 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -31,7 +31,7 @@ is also an [asynchronous API client](#connections-asynchronous). ## Namespaces (Synchronous) A namespace-backed connection resolves tables through a -[Lance namespace](https://lancedb.github.io/lance-namespace/) service instead of +[Lance namespace](https://lance-format.github.io/lance-namespace/) service instead of listing a storage directory. ::: lancedb.connect_namespace From f4c668e2441a3c5ab31a024eb8e1ec6a68920f17 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Fri, 7 Aug 2026 02:24:38 -0700 Subject: [PATCH 29/49] chore(deps): declare more specific futures dependency (#3800) Lancedb does not work with any other version of `futures`. With futures 0.1 it fails like this: ```console error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryStreamExt` --> rust/lancedb/src/arrow.rs:21:23 | 21 | use futures::{Stream, StreamExt, TryStreamExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved import `futures::StreamExt` --> rust/lancedb/src/data/scannable.rs:24:5 | 24 | use futures::StreamExt; | ^^^^^^^^^^^^^^^^^^ no `StreamExt` in the root | error[E0432]: unresolved import `futures::TryStreamExt` --> rust/lancedb/src/dataloader/permutation/builder.rs:9:5 | 9 | use futures::TryStreamExt; | ^^^^^^^^^^^^^^^^^^^^^ no `TryStreamExt` in the root error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryStreamExt` --> rust/lancedb/src/dataloader/permutation/reader.rs:25:15 | 25 | use futures::{StreamExt, TryStreamExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryStreamExt` --> rust/lancedb/src/dataloader/permutation/shuffle.rs:8:15 | 8 | use futures::{StreamExt, TryStreamExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryStreamExt` --> rust/lancedb/src/dataloader/permutation/split.rs:12:15 | 12 | use futures::{StreamExt, TryStreamExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved import `futures::TryStreamExt` --> rust/lancedb/src/dataloader/permutation/util.rs:9:5 | 9 | use futures::TryStreamExt; | ^^^^^^^^^^^^^^^^^^^^^ no `TryStreamExt` in the root error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryFutureExt` --> rust/lancedb/src/io/object_store.rs:8:15 | 8 | use futures::{StreamExt, TryFutureExt, stream::BoxStream}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryFutureExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved imports `futures::FutureExt`, `futures::TryFutureExt`, `futures::TryStreamExt`, `futures::try_join` --> rust/lancedb/src/query.rs:12:15 | 12 | use futures::{FutureExt, TryFutureExt, TryStreamExt, stream, try_join}; | ^^^^^^^^^ ^^^^^^^^^^^^ ^^^^^^^^^^^^ ^^^^^^^^ no `try_join` in the root | | | | | | | no `TryStreamExt` in the root | | no `TryFutureExt` in the root | no `FutureExt` in the root | error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryStreamExt` --> rust/lancedb/src/remote/table/blobs.rs:13:15 | 13 | use futures::{StreamExt, TryStreamExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved imports `futures::SinkExt`, `futures::StreamExt` --> rust/lancedb/src/remote/table/insert.rs:20:15 | 20 | use futures::{SinkExt, StreamExt}; | ^^^^^^^ ^^^^^^^^^ no `StreamExt` in the root | | | no `SinkExt` in the root | error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryStreamExt` --> rust/lancedb/src/remote/table.rs:58:15 | 58 | use futures::{StreamExt, TryStreamExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved import `futures::StreamExt` --> rust/lancedb/src/remote/util.rs:5:23 | 5 | use futures::{Stream, StreamExt}; | ^^^^^^^^^ no `StreamExt` in the root | error[E0432]: unresolved import `futures::StreamExt` --> rust/lancedb/src/table.rs:14:5 | 14 | use futures::StreamExt; | ^^^^^^^^^^^^^^^^^^ no `StreamExt` in the root | error[E0432]: unresolved import `futures::TryStreamExt` --> rust/lancedb/src/table/datafusion/insert.rs:20:5 | 20 | use futures::TryStreamExt; | ^^^^^^^^^^^^^^^^^^^^^ no `TryStreamExt` in the root error[E0432]: unresolved import `futures::TryStreamExt` --> rust/lancedb/src/table/datafusion/scannable_exec.rs:14:5 | 14 | use futures::TryStreamExt; | ^^^^^^^^^^^^^^^^^^^^^ no `TryStreamExt` in the root error[E0432]: unresolved imports `futures::TryFutureExt`, `futures::TryStreamExt` --> rust/lancedb/src/table/datafusion.rs:25:15 | 25 | use futures::{TryFutureExt, TryStreamExt}; | ^^^^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `TryFutureExt` in the root error[E0432]: unresolved import `futures::FutureExt` --> rust/lancedb/src/table/delete.rs:3:5 | 3 | use futures::FutureExt; | ^^^^^^^^^^^^^^^^^^ no `FutureExt` in the root | error[E0432]: unresolved imports `futures::FutureExt`, `futures::TryFutureExt` --> rust/lancedb/src/table/merge.rs:9:15 | 9 | use futures::{FutureExt, TryFutureExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryFutureExt` in the root | | | no `FutureExt` in the root | error[E0432]: unresolved import `futures::future::try_join_all` --> rust/lancedb/src/table/query.rs:24:5 | 24 | use futures::future::try_join_all; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ no `try_join_all` in `future` | error[E0432]: unresolved import `futures::FutureExt` --> rust/lancedb/src/utils/background_cache.rs:12:5 | 12 | use futures::FutureExt; | ^^^^^^^^^^^^^^^^^^ no `FutureExt` in the root | error[E0432]: unresolved import `futures::FutureExt` --> rust/lancedb/src/utils/mod.rs:12:15 | 12 | use futures::{FutureExt, Stream}; | ^^^^^^^^^ no `FutureExt` in the root | error[E0433]: cannot find `join` in `futures` --> rust/lancedb/src/remote/table/insert.rs:504:55 | 504 | let (producer_result, send_result) = futures::join!(producer, send); | ^^^^ could not find `join` in `futures` error[E0407]: method `poll_next` is not a member of trait `Stream` --> rust/lancedb/src/arrow.rs:108:5 | 108 | / fn poll_next( 109 | | self: Pin<&mut Self>, 110 | | cx: &mut std::task::Context<'_>, 111 | | ) -> std::task::Poll> { 112 | | let this = self.project(); 113 | | this.stream.poll_next(cx) 114 | | } | |_____^ not a member of trait `Stream` error[E0407]: method `poll_next` is not a member of trait `Stream` --> rust/lancedb/src/utils/mod.rs:362:5 | 362 | / fn poll_next( 363 | | mut self: std::pin::Pin<&mut Self>, 364 | | cx: &mut std::task::Context<'_>, 365 | | ) -> std::task::Poll> { ... | 391 | | } | |_____^ not a member of trait `Stream` error[E0407]: method `poll_next` is not a member of trait `Stream` --> rust/lancedb/src/utils/mod.rs:433:5 | 433 | / fn poll_next( 434 | | mut self: Pin<&mut Self>, 435 | | cx: &mut std::task::Context<'_>, 436 | | ) -> std::task::Poll> { ... | 470 | | } | |_____^ not a member of trait `Stream` error[E0425]: cannot find function `try_unfold` in module `futures::stream` --> rust/lancedb/src/remote/table/insert.rs:230:39 | 230 | let stream = futures::stream::try_unfold( | ^^^^^^^^^^ not found in `futures::stream` error[E0433]: cannot find `channel` in `futures` --> rust/lancedb/src/remote/table/insert.rs:418:22 | 418 | futures::channel::mpsc::channel::, std::io::Error>>(2); | ^^^^^^^ could not find `channel` in `futures` | error[E0425]: cannot find function `try_join_all` in module `futures::future` --> rust/lancedb/src/remote/table.rs:1062:40 | 1062 | let streams = futures::future::try_join_all(futures); | ^^^^^^^^^^^^ | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:76:1 | 76 | / pub fn join_all(i: I) -> JoinAll 77 | | where I: IntoIterator, 78 | | I::Item: IntoFuture, | |______________________________- similarly named function `join_all` defined here | error[E0425]: cannot find function `try_join_all` in module `futures::future` --> rust/lancedb/src/remote/table.rs:1660:40 | 1660 | let results = futures::future::try_join_all(futures).await?; | ^^^^^^^^^^^^ | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:76:1 | 76 | / pub fn join_all(i: I) -> JoinAll 77 | | where I: IntoIterator, 78 | | I::Item: IntoFuture, | |______________________________- similarly named function `join_all` defined here | error[E0425]: cannot find function `try_join_all` in module `futures::future` --> rust/lancedb/src/remote/table.rs:2243:43 | 2243 | let plan_texts = futures::future::try_join_all(futures).await?; | ^^^^^^^^^^^^ | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:76:1 | 76 | / pub fn join_all(i: I) -> JoinAll 77 | | where I: IntoIterator, 78 | | I::Item: IntoFuture, | |______________________________- similarly named function `join_all` defined here | error[E0425]: cannot find function `try_join_all` in module `futures::future` --> rust/lancedb/src/remote/table.rs:2290:53 | 2290 | let analyze_result_texts = futures::future::try_join_all(futures).await?; | ^^^^^^^^^^^^ | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:76:1 | 76 | / pub fn join_all(i: I) -> JoinAll 77 | | where I: IntoIterator, 78 | | I::Item: IntoFuture, | |______________________________- similarly named function `join_all` defined here | error[E0425]: cannot find function `try_unfold` in module `futures::stream` --> rust/lancedb/src/remote/util.rs:21:35 | 21 | let stream = futures::stream::try_unfold( | ^^^^^^^^^^ not found in `futures::stream` error[E0191]: the value of the associated type `Error` in `futures::Stream` must be specified --> rust/lancedb/src/arrow.rs:70:50 | 70 | pub type SendableRecordBatchStream = Pin>; | ^^^^^^^^^^^^^^^^^ | help: specify the associated type | 70 | pub type SendableRecordBatchStream = Pin + Send>>; | ++++++++++++++++++++ error[E0107]: type alias takes 0 lifetime arguments but 1 lifetime argument was supplied --> rust/lancedb/src/utils/background_cache.rs:15:31 | 15 | type SharedFut = Shared>>>; | ^^^^^^^^^ ------- help: remove the lifetime argument | | | expected 0 lifetime arguments | note: type alias defined here, with 0 lifetime parameters --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/mod.rs:106:14 | 106 | pub type BoxFuture = ::std::boxed::Box + Send>; | ^^^^^^^^^ error[E0107]: type alias takes 2 generic arguments but 1 generic argument was supplied --> rust/lancedb/src/utils/background_cache.rs:15:31 | 15 | type SharedFut = Shared>>>; | ^^^^^^^^^ ----------------- supplied 1 generic argument | | | expected 2 generic arguments | note: type alias defined here, with 2 generic parameters: `T`, `E` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/mod.rs:106:14 | 106 | pub type BoxFuture = ::std::boxed::Box + Send>; | ^^^^^^^^^ - - help: add missing generic argument | 15 | type SharedFut = Shared>, E>>; | +++ error[E0046]: not all trait items implemented, missing: `Error`, `poll` --> rust/lancedb/src/arrow.rs:105:1 | 105 | impl>> Stream for SimpleRecordBatchStream { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ missing `Error`, `poll` in implementation | = help: implement the missing item: `type Error = /* Type */;` = help: implement the missing item: `fn poll(&mut self) -> std::result::Result::Item>>, ::Error> { todo!() }` error[E0107]: type alias takes 0 lifetime arguments but 1 lifetime argument was supplied --> rust/lancedb/src/io/object_store.rs:97:46 | 97 | fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { | ^^^^^^^^^ ------- help: remove the lifetime argument | | | expected 0 lifetime arguments | note: type alias defined here, with 0 lifetime parameters --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:132:14 | 132 | pub type BoxStream = ::std::boxed::Box + Send>; | ^^^^^^^^^ error[E0107]: type alias takes 2 generic arguments but 1 generic argument was supplied --> rust/lancedb/src/io/object_store.rs:97:46 | 97 | fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { | ^^^^^^^^^ ------------------ supplied 1 generic argument | | | expected 2 generic arguments | note: type alias defined here, with 2 generic parameters: `T`, `E` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:132:14 | 132 | pub type BoxStream = ::std::boxed::Box + Send>; | ^^^^^^^^^ - - help: add missing generic argument | 97 | fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result, E> { | +++ error[E0107]: type alias takes 0 lifetime arguments but 1 lifetime argument was supplied --> rust/lancedb/src/io/object_store.rs:107:20 | 107 | locations: BoxStream<'static, Result>, | ^^^^^^^^^ ------- help: remove the lifetime argument | | | expected 0 lifetime arguments | note: type alias defined here, with 0 lifetime parameters --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:132:14 | 132 | pub type BoxStream = ::std::boxed::Box + Send>; | ^^^^^^^^^ error[E0107]: type alias takes 2 generic arguments but 1 generic argument was supplied --> rust/lancedb/src/io/object_store.rs:107:20 | 107 | locations: BoxStream<'static, Result>, | ^^^^^^^^^ ------------ supplied 1 generic argument | | | expected 2 generic arguments | note: type alias defined here, with 2 generic parameters: `T`, `E` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:132:14 | 132 | pub type BoxStream = ::std::boxed::Box + Send>; | ^^^^^^^^^ - - help: add missing generic argument | 107 | locations: BoxStream<'static, Result, E>, | +++ error[E0107]: type alias takes 0 lifetime arguments but 1 lifetime argument was supplied --> rust/lancedb/src/io/object_store.rs:108:10 | 108 | ) -> BoxStream<'static, Result> { | ^^^^^^^^^ ------- help: remove the lifetime argument | | | expected 0 lifetime arguments | note: type alias defined here, with 0 lifetime parameters --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:132:14 | 132 | pub type BoxStream = ::std::boxed::Box + Send>; | ^^^^^^^^^ error[E0107]: type alias takes 2 generic arguments but 1 generic argument was supplied --> rust/lancedb/src/io/object_store.rs:108:10 | 108 | ) -> BoxStream<'static, Result> { | ^^^^^^^^^ ------------ supplied 1 generic argument | | | expected 2 generic arguments | note: type alias defined here, with 2 generic parameters: `T`, `E` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:132:14 | 132 | pub type BoxStream = ::std::boxed::Box + Send>; | ^^^^^^^^^ - - help: add missing generic argument | 108 | ) -> BoxStream<'static, Result, E> { | +++ error[E0599]: no method named `map_err` found for struct `Pin>` in the current scope --> rust/lancedb/src/dataloader/permutation/builder.rs:208:32 | 208 | let stream = df_stream.map_err(|e| Error::Other { | ----------^^^^^^^ method not found in `Pin>` | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-util-0.3.32/src/stream/try_stream/mod.rs:248:8 | 248 | fn map_err(self, f: F) -> MapErr | ------- the method is available for `Pin>` here | error[E0599]: no method named `try_collect` found for struct `DatasetRecordBatchStream` in the current scope --> rust/lancedb/src/dataloader/permutation/reader.rs:220:28 | 220 | let batches = data.try_collect::>().await?; | ^^^^^^^^^^^ | error[E0599]: no method named `map_err` found for struct `DatasetRecordBatchStream` in the current scope --> rust/lancedb/src/dataloader/permutation/reader.rs:287:14 | 286 | let mut stream = row_ids | __________________________- 287 | | .map_err(Error::from) | | -^^^^^^^ method not found in `DatasetRecordBatchStream` | |_____________| | error[E0599]: the method `chain` exists for struct `futures::stream::Once<_, _>`, but its trait bounds were not satisfied --> rust/lancedb/src/dataloader/permutation/reader.rs:307:81 | 307 | let stream = futures::stream::once(std::future::ready(Ok(first_batch))).chain(stream); | ^^^^^ method cannot be called on `futures::stream::Once<_, _>` due to unsatisfied trait bounds error[E0308]: mismatched types --> rust/lancedb/src/dataloader/permutation/shuffle.rs:120:35 | 120 | futures::stream::once(async move { Ok(shuffled) }), | --------------------- ^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<_, _>`, found `async` block | | | arguments to this function are incorrect | = note: expected enum `std::result::Result<_, _>` found `async` block `{async block@rust/lancedb/src/dataloader/permutation/shuffle.rs:120:35: 120:45}` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 120 | futures::stream::once(Ok(async move { Ok(shuffled) })), | +++ + 120 | futures::stream::once(Err(async move { Ok(shuffled) })), | ++++ + error[E0271]: type mismatch resolving ` as IntoIterator>::Item == Result<_, _>` --> rust/lancedb/src/dataloader/permutation/shuffle.rs:228:44 | 228 | let stream = futures::stream::iter(0..num_files) | --------------------- ^^^^^^^^^^^^ expected `Result<_, _>`, found `u64` | | | required by a bound introduced by this call | = note: expected enum `std::result::Result<_, _>` found type `u64` note: required by a bound in `futures::stream::iter` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/iter.rs:31:27 | 30 | pub fn iter(i: J) -> Iter | ---- required by a bound in this function 31 | where J: IntoIterator>, | ^^^^^^^^^^^^^^^^^ required by this bound in `iter` error[E0599]: no method named `then` found for struct `IterStream` in the current scope --> rust/lancedb/src/dataloader/permutation/shuffle.rs:229:14 | 228 | let stream = futures::stream::iter(0..num_files) | ______________________- 229 | | .then(move |file_index| { | | -^^^^ method not found in `IterStream>` | |_____________| | error[E0599]: no method named `try_collect` found for struct `Pin>` in the current scope --> rust/lancedb/src/dataloader/permutation/shuffle.rs:258:26 | 250 | let batches = reader | ___________________________________- 251 | | .read_stream( 252 | | ReadBatchParams::RangeFull, 253 | | reader.num_rows() as u32, ... | 257 | | .await? 258 | | .try_collect::>() | |_________________________-^^^^^^^^^^^ error[E0599]: no method named `and_then` found for associated type `impl Future, ...>> + Send` in the current scope --> rust/lancedb/src/query.rs:766:14 | 765 | / self.create_plan(QueryExecutionOptions::default()) 766 | | .and_then(|plan| std::future::ready(Ok(plan.schema()))) | |_____________-^^^^^^^^ | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-util-0.3.32/src/future/try_future/mod.rs:395:8 | 395 | fn and_then(self, f: F) -> AndThen | -------- the method is available for `impl std::future::Future, error::Error>> + std::marker::Send` here error[E0599]: no method named `boxed` found for `async` block `{async block@rust/lancedb/src/query.rs:1492:33: 1492:43}` in the current scope --> rust/lancedb/src/query.rs:1493:18 | 1492 | let hybrid_result = async move { self.execute_hybrid(options).await } | _________________________________- 1493 | | .boxed() | | -^^^^^ method not found in `{async block@rust/lancedb/src/query.rs:1492:33: 1492:43}` | |_________________| error[E0271]: expected `{closure@blobs.rs:181:58}` to return `Result<_, _>`, but it returns `impl Future>` --> rust/lancedb/src/remote/table/blobs.rs:181:66 | 181 | futures::stream::iter(ranges.iter().cloned().map(|range| self.read_range(range))) | --------------------- ------- ^^^^^^^^^^^^^^^^^^^^^^ expected `Result<_, _>`, found future | | | | | this closure | required by a bound introduced by this call error[E0599]: no method named `buffered` found for struct `IterStream` in the current scope --> rust/lancedb/src/remote/table/blobs.rs:182:14 | 181 | / futures::stream::iter(ranges.iter().cloned().map(|range| self.read_range(range))) 182 | | .buffered(BLOB_REQUEST_CONCURRENCY) | | -^^^^^^^^ method not found in `Iter>>, {closure@...}>>` | |_____________| error[E0599]: no method named `try_next` found for struct `Pin>` in the current scope --> rust/lancedb/src/remote/table/blobs.rs:379:40 | 379 | while let Some(batch) = stream.try_next().await? { | ^^^^^^^^ method not found in `Pin>` error[E0271]: type mismatch resolving ` as IntoIterator>::Item == Result<_, _>` --> rust/lancedb/src/remote/table/blobs.rs:481:27 | 481 | futures::stream::iter(probe_futures) | --------------------- ^^^^^^^^^^^^^ expected `Result<_, _>`, found future | | | required by a bound introduced by this call error[E0599]: no method named `buffered` found for struct `IterStream` in the current scope --> rust/lancedb/src/remote/table/blobs.rs:482:10 | 481 | / futures::stream::iter(probe_futures) 482 | | .buffered(BLOB_REQUEST_CONCURRENCY) | | -^^^^^^^^ method not found in `Iter>>>` | |_________| error[E0599]: no method named `next` found for struct `Pin>` in the current scope --> rust/lancedb/src/remote/table/insert.rs:324:37 | 324 | let mut first = match input.next().await { | ^^^^ method not found in `Pin>` error[E0599]: no method named `next` found for struct `Pin>` in the current scope --> rust/lancedb/src/remote/table/insert.rs:345:33 | 345 | first = match input.next().await { | ^^^^ method not found in `Pin>` error[E0599]: the method `next` exists for mutable reference `&mut Pin>`, but its trait bounds were not satisfied --> rust/lancedb/src/remote/table/insert.rs:446:41 | 446 | None => match input.next().await { | ^^^^ method cannot be called on `&mut Pin>` due to unsatisfied trait bounds | = note: the following trait bounds were not satisfied: `Pin>: Iterator` which is required by `&mut Pin>: Iterator` error[E0599]: no method named `map_err` found for struct `IterStream` in the current scope --> rust/lancedb/src/remote/table.rs:688:53 | 688 | let stream = futures::stream::iter(batches).map_err(DataFusionError::from); | ^^^^^^^ method not found in `Iter> + Send>>` error[E0599]: no method named `try_collect` found for struct `Pin>` in the current scope --> rust/lancedb/src/remote/table.rs:1378:49 | 1378 | let result: Result> = stream.try_collect().await.map_err(Error::from); | ^^^^^^^^^^^ error[E0599]: no method named `next` found for struct `Pin>` in the current scope --> rust/lancedb/src/remote/table.rs:1509:48 | 1509 | while let Some(batch) = stream.next().await { | ^^^^ method not found in `Pin>` error[E0599]: no method named `boxed` found for opaque type `impl Future>` in the current scope --> rust/lancedb/src/table/delete.rs:35:51 | 35 | let delete_result = dataset.delete(s).boxed().await?; | ^^^^^ method not found in `impl Future>` error[E0599]: no variant, associated function, or constant named `Left` found for enum `Either` in the current scope --> rust/lancedb/src/table/merge.rs:292:17 | 292 | Either::Left(tokio::time::timeout(timeout, future).map(|res| match res { | ^^^^ variant, associated function, or constant not found in `Either<_, _>` error[E0599]: `Timeout, ...), ...>>>` is not an iterator --> rust/lancedb/src/table/merge.rs:292:60 | 292 | Either::Left(tokio::time::timeout(timeout, future).map(|res| match res { | --------------------------------------^^^ `Timeout, ...), ...>>>` is not an iterator | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/pin-project-lite-0.2.17/src/lib.rs:745:9 | 745 | / $vis struct $ident $($def_generics)* 746 | | $(where 747 | | $($where_clause)*)? ... | 751 | | ),+ 752 | | } | |_________- doesn't satisfy `_: Iterator` | = note: the following trait bounds were not satisfied: `tokio::time::Timeout, MergeStats), lance::Error>>>: Iterator` which is required by `&mut tokio::time::Timeout, MergeStats), lance::Error>>>: Iterator` error[E0599]: no variant, associated function, or constant named `Right` found for enum `Either` in the current scope --> rust/lancedb/src/table/merge.rs:301:17 | 301 | Either::Right(job.execute_reader(new_data).map_err(|e| e.into())) | ^^^^^ variant, associated function, or constant not found in `Either<_, _>` error[E0599]: no method named `map_err` found for opaque type `impl Future, ...), ...>>` in the current scope --> rust/lancedb/src/table/merge.rs:301:52 | 301 | Either::Right(job.execute_reader(new_data).map_err(|e| e.into())) | ^^^^^^^ method not found in `impl Future, ...), ...>>` error[E0277]: the trait bound `Iter, ...>>: Stream` is not satisfied --> rust/lancedb/src/table/query.rs:681:38 | 681 | Ok(DatasetRecordBatchStream::new(record_batch_stream)) | ^^^^^^^^^^^^^^^^^^^ the trait `futures_core::stream::Stream` is not implemented for `Iter, ...>>` error[E0277]: the trait bound `TimeoutStream: futures_core::stream::Stream` is not satisfied --> rust/lancedb/src/utils/mod.rs:353:28 | 353 | impl RecordBatchStream for TimeoutStream { | ^^^^^^^^^^^^^ unsatisfied trait bound error[E0046]: not all trait items implemented, missing: `Error`, `poll` --> rust/lancedb/src/utils/mod.rs:359:1 | 359 | impl Stream for TimeoutStream { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ missing `Error`, `poll` in implementation | = help: implement the missing item: `type Error = /* Type */;` = help: implement the missing item: `fn poll(&mut self) -> std::result::Result::Item>>, ::Error> { todo!() }` error[E0277]: the trait bound `MaxBatchLengthStream: futures_core::stream::Stream` is not satisfied --> rust/lancedb/src/utils/mod.rs:424:28 | 424 | impl RecordBatchStream for MaxBatchLengthStream { | ^^^^^^^^^^^^^^^^^^^^ unsatisfied trait bound error[E0046]: not all trait items implemented, missing: `Error`, `poll` --> rust/lancedb/src/utils/mod.rs:430:1 | 430 | impl Stream for MaxBatchLengthStream { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ missing `Error`, `poll` in implementation | = help: implement the missing item: `type Error = /* Type */;` = help: implement the missing item: `fn poll(&mut self) -> std::result::Result::Item>>, ::Error> { todo!() }` error[E0599]: no method named `map` found for type parameter `I` in the current scope --> rust/lancedb/src/arrow.rs:75:45 | 72 | impl From for SendableRecordBatchStream { | - method `map` not found for this type parameter ... 75 | let mapped_stream = Box::pin(stream.map(|r| r.map_err(Into::into))); | ^^^ error[E0599]: no method named `poll_next` found for struct `Pin<&mut S>` in the current scope --> rust/lancedb/src/arrow.rs:113:21 | 113 | this.stream.poll_next(cx) | ^^^^^^^^^ | = help: items from traits can only be used if the trait is implemented and in scope = note: the following traits define an item `poll_next`, perhaps you need to implement one of them: candidate #1: `futures_core::stream::Stream` candidate #2: `sorts::stream::PartitionedStream` help: there is a method `collect` with a similar name, but with different arguments --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:563:5 | 563 | / fn collect(self) -> Collect 564 | | where Self: Sized | |_________________________^ error[E0599]: the method `map_err` exists for struct `Pin> + Send>>`, but its trait bounds were not satisfied --> rust/lancedb/src/arrow.rs:150:29 | 150 | let stream = stream.map_err(|err| Error::Arrow { source: err }); | ^^^^^^^ method cannot be called due to unsatisfied trait bounds error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:80:26 | 80 | stream: once(async move { Ok(batch) }), | ---- ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<_, _>`, found `async` block | | | arguments to this function are incorrect | = note: expected enum `std::result::Result<_, _>` found `async` block `{async block@rust/lancedb/src/data/scannable.rs:80:26: 80:36}` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 80 | stream: once(Ok(async move { Ok(batch) })), | +++ + 80 | stream: once(Err(async move { Ok(batch) })), | ++++ + error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:107:30 | 107 | stream: once(async { | _________________________----_^ | | | | | arguments to this function are incorrect 108 | | Err(Error::InvalidInput { 109 | | message: "Cannot scan an empty Vec".to_string(), 110 | | }) 111 | | }), | |_________________^ expected `Result<_, _>`, found `async` block | = note: expected enum `std::result::Result<_, _>` found `async` block `{async block@rust/lancedb/src/data/scannable.rs:107:30: 107:35}` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 107 ~ stream: once(Ok(async { 108 | Err(Error::InvalidInput { 109 | message: "Cannot scan an empty Vec".to_string(), 110 | }) 111 ~ })), | 107 ~ stream: once(Err(async { 108 | Err(Error::InvalidInput { 109 | message: "Cannot scan an empty Vec".to_string(), 110 | }) 111 ~ })), | error[E0271]: expected `Ok` to return `Result, _>`, but it returns `Result` --> rust/lancedb/src/data/scannable.rs:117:52 | 117 | Box::pin(SimpleRecordBatchStream { schema, stream }) | ^^^^^^ expected `Result, _>`, found `Result` error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:158:59 | 158 | let stream = futures::stream::unfold(rx, |mut rx| async move { | ___________________________________________________________^ 159 | | rx.recv().await.map(|batch| (batch, rx)) 160 | | }) | |_________^ expected `Option<_>`, found `async` block | = note: expected enum `std::option::Option<_>` found `async` block `{async block@rust/lancedb/src/data/scannable.rs:158:59: 158:69}` help: try wrapping the expression in `Some` | 158 ~ let stream = futures::stream::unfold(rx, |mut rx| Some(async move { 159 | rx.recv().await.map(|batch| (batch, rx)) 160 ~ })) | error[E0599]: the method `fuse` exists for struct `Unfold>, ..., _>`, but its trait bounds were not satisfied --> rust/lancedb/src/data/scannable.rs:161:10 | 158 | let stream = futures::stream::unfold(rx, |mut rx| async move { | ______________________- 159 | | rx.recv().await.map(|batch| (batch, rx)) 160 | | }) 161 | | .fuse(); | | -^^^^ method cannot be called due to unsatisfied trait bounds | |_________| error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:178:26 | 178 | stream: once(async { | _____________________----_^ | | | | | arguments to this function are incorrect 179 | | Err(Error::InvalidInput { 180 | | message: "Stream has already been consumed".to_string(), 181 | | }) 182 | | }), | |_____________^ expected `Result<_, _>`, found `async` block | = note: expected enum `std::result::Result<_, _>` found `async` block `{async block@rust/lancedb/src/data/scannable.rs:178:26: 178:31}` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 178 ~ stream: once(Ok(async { 179 | Err(Error::InvalidInput { 180 | message: "Stream has already been consumed".to_string(), 181 | }) 182 ~ })), | 178 ~ stream: once(Err(async { 179 | Err(Error::InvalidInput { 180 | message: "Stream has already been consumed".to_string(), 181 | }) 182 ~ })), | error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:474:53 | 474 | let prepend = futures::stream::once(std::future::ready(Ok(batch))); | --------------------- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<_, _>`, found `Ready>` | | | arguments to this function are incorrect | = note: expected enum `std::result::Result<_, _>` found struct `std::future::Ready>` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 474 | let prepend = futures::stream::once(Ok(std::future::ready(Ok(batch)))); | +++ + 474 | let prepend = futures::stream::once(Err(std::future::ready(Ok(batch)))); | ++++ + error[E0599]: the method `chain` exists for struct `futures::stream::Once<_, _>`, but its trait bounds were not satisfied --> rust/lancedb/src/data/scannable.rs:477:37 | 477 | stream: prepend.chain(rest), | ^^^^^ method cannot be called on `futures::stream::Once<_, _>` due to unsatisfied trait bounds error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:482:47 | 482 | stream: futures::stream::once(std::future::ready(Ok(batch))), | --------------------- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<_, _>`, found `Ready>` | | | arguments to this function are incorrect | = note: expected enum `std::result::Result<_, _>` found struct `std::future::Ready>` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 482 | stream: futures::stream::once(Ok(std::future::ready(Ok(batch)))), | +++ + 482 | stream: futures::stream::once(Err(std::future::ready(Ok(batch)))), | ++++ + error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:486:56 | 486 | let stream = futures::stream::once(std::future::ready(err)); | --------------------- ^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<_, _>`, found `Ready>` | | | arguments to this function are incorrect | = note: expected enum `std::result::Result<_, _>` found struct `std::future::Ready>` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 486 | let stream = futures::stream::once(Ok(std::future::ready(err))); | +++ + 486 | let stream = futures::stream::once(Err(std::future::ready(err))); | ++++ + error[E0599]: no method named `and_then` found for struct `Pin> + Send>>` in the current scope --> rust/lancedb/src/io/object_store.rs:153:32 | 153 | Box::pin(put_secondary.and_then(|_| put_primary)) | ^^^^^^^^ error[E0271]: expected `IntoIter, 1>` to be an iterator that yields `Result, _>`, but it yields `Result` --> rust/lancedb/src/query.rs:1465:25 | 1465 | return Box::pin(SimpleRecordBatchStream::new( | ^^^^^^^^^^^^^^^^^^^^^^^ expected `Result, _>`, found `Result` error[E0271]: expected `IntoIter>` to be an iterator that yields `Result, _>`, but it yields `Result` --> rust/lancedb/src/query.rs:1478:14 | 1478 | Box::pin(SimpleRecordBatchStream::new(stream::iter(batches), schema)) | ^^^^^^^^^^^^^^^^^^^^^^^ expected `Result, _>`, found `Result` error[E0308]: mismatched types --> rust/lancedb/src/remote/table/insert.rs:626:44 | 626 | let stream = futures::stream::once(async move { | ______________________---------------------_^ | | | | | arguments to this function are incorrect ... | 791 | | Ok::<_, DataFusionError>(batch) 792 | | }); | |_________^ expected `Result<_, _>`, found `async` block | = note: expected enum `std::result::Result<_, _>` found `async` block `{async block@rust/lancedb/src/remote/table/insert.rs:626:44: 626:54}` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 626 ~ let stream = futures::stream::once(Ok(async move { 627 | // Multipart writes with a byte budget split the partition into ... 791 | Ok::<_, DataFusionError>(batch) 792 ~ })); | 626 ~ let stream = futures::stream::once(Err(async move { 627 | // Multipart writes with a byte budget split the partition into ... 791 | Ok::<_, DataFusionError>(batch) 792 ~ })); | error[E0277]: the trait bound `futures::stream::Once<_, _>: futures_core::stream::Stream` is not satisfied --> rust/lancedb/src/remote/table/insert.rs:794:12 | 794 | Ok(Box::pin(RecordBatchStreamAdapter::new( | ____________^ 795 | | COUNT_SCHEMA.clone(), 796 | | stream, 797 | | ))) | |__________^ the trait `futures_core::stream::Stream` is not implemented for `futures::stream::Once<_, _>` error[E0599]: no method named `try_collect` found for struct `Pin>` in the current scope --> rust/lancedb/src/remote/table.rs:2442:49 | 2442 | let result: Result> = stream.try_collect().await.map_err(Error::from); | ^^^^^^^^^^^ error[E0277]: the trait bound `impl Stream>: TryStream` is not satisfied --> rust/lancedb/src/remote/util.rs:47:35 | 47 | Ok(reqwest::Body::wrap_stream(stream)) | -------------------------- ^^^^^^ unsatisfied trait bound | | | required by a bound introduced by this call error[E0599]: no method named `map_ok` found for struct `Pin>` in the current scope --> rust/lancedb/src/table/datafusion/insert.rs:200:30 | 200 | input_stream.map_ok(move |batch| { | -------------^^^^^^ method not found in `Pin>` error[E0308]: mismatched types --> rust/lancedb/src/table/datafusion/insert.rs:208:44 | 208 | let stream = futures::stream::once(async move { | ______________________---------------------_^ | | | | | arguments to this function are incorrect 209 | | if let Some(tracker) = tracker 210 | | && write_params.write_progress.is_none() ... | 255 | | )?) 256 | | }); | |_________^ expected `Result<_, _>`, found `async` block | = note: expected enum `std::result::Result<_, _>` found `async` block `{async block@rust/lancedb/src/table/datafusion/insert.rs:208:44: 208:54}` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 208 ~ let stream = futures::stream::once(Ok(async move { 209 | if let Some(tracker) = tracker ... 255 | )?) 256 ~ })); | 208 ~ let stream = futures::stream::once(Err(async move { 209 | if let Some(tracker) = tracker ... 255 | )?) 256 ~ })); | error[E0277]: the trait bound `futures::stream::Once<_, _>: futures_core::stream::Stream` is not satisfied --> rust/lancedb/src/table/datafusion/insert.rs:258:12 | 258 | Ok(Box::pin(RecordBatchStreamAdapter::new( | ____________^ 259 | | COUNT_SCHEMA.clone(), 260 | | stream, 261 | | ))) | |__________^ the trait `futures_core::stream::Stream` is not implemented for `futures::stream::Once<_, _>` error[E0599]: no method named `map_ok` found for struct `Pin>` in the current scope --> rust/lancedb/src/table/datafusion.rs:128:29 | 128 | let stream = stream.map_ok(move |batch| { | -------^^^^^^ method not found in `Pin>` error[E0599]: no method named `map_err` found for struct `Pin, ...>> + Send>>` in the current scope --> rust/lancedb/src/table/datafusion.rs:245:14 | 242 | let plan = self | ____________________- 243 | | .table 244 | | .create_plan(&AnyQuery::Query(query), options) 245 | | .map_err(|err| DataFusionError::External(err.into())) | | -^^^^^^^ method not found in `Pin, ...>> + Send>>` | |_____________| error[E0599]: no method named `next` found for struct `Pin>` in the current scope --> rust/lancedb/src/table.rs:3048:48 | 3048 | while let Some(batch) = stream.next().await { | ^^^^ method not found in `Pin>` error[E0277]: the trait bound `JoinHandle>: Future` is not satisfied --> rust/lancedb/src/table.rs:3038:23 | 3038 | let handles = FuturesUnordered::new(); | ^^^^^^^^^^^^^^^^^^^^^^^ the trait `futures::Future` is not implemented for `tokio::task::JoinHandle>` error[E0277]: `FuturesUnordered>>` is not an iterator --> rust/lancedb/src/table.rs:3054:23 | 3054 | for handle in handles { | ^^^^^^^ `FuturesUnordered>>` is not an iterator error[E0277]: the trait bound `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}: futures::IntoFuture` is not satisfied --> rust/lancedb/src/table.rs:3450:13 | 3449 | let mut sorted_sizes = join_all( | -------- required by a bound introduced by this call 3450 | / frags 3451 | | .iter() 3452 | | .map(|frag| async move { frag.physical_rows().await.unwrap_or(0) }), | |___________________________________________________________________________________^ the trait `futures::Future` is not implemented for `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` | = note: `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` implements similarly named trait `std::future::Future`, but not `futures::Future` = help: the following other types implement trait `futures::Future`: &'a mut F AssertUnwindSafe BiLockAcquire Box Concat2 Either Finished Fold and 43 others = note: required for `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` to implement `futures::IntoFuture` note: required by a bound in `join_all` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:78:20 | 76 | pub fn join_all(i: I) -> JoinAll | -------- required by a bound in this function 77 | where I: IntoIterator, 78 | I::Item: IntoFuture, | ^^^^^^^^^^ required by this bound in `join_all` error[E0277]: the trait bound `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}: futures::Future` is not satisfied --> rust/lancedb/src/table.rs:3449:32 | 3449 | let mut sorted_sizes = join_all( | ________________________________^ 3450 | | frags 3451 | | .iter() 3452 | | .map(|frag| async move { frag.physical_rows().await.unwrap_or(0) }), 3453 | | ) | |_________^ the trait `futures::Future` is not implemented for `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` | = note: `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` implements similarly named trait `std::future::Future`, but not `futures::Future` = help: the following other types implement trait `futures::Future`: &'a mut F AssertUnwindSafe BiLockAcquire Box Concat2 Either Finished Fold and 43 others = note: required for `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` to implement `futures::IntoFuture` note: required by a bound in `JoinAll` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:24:20 | 22 | pub struct JoinAll | ------- required by a bound in this struct 23 | where I: IntoIterator, 24 | I::Item: IntoFuture, | ^^^^^^^^^^ required by this bound in `JoinAll` error[E0277]: `JoinAll, {closure@...}>>` is not a future --> rust/lancedb/src/table.rs:3454:10 | 3449 | let mut sorted_sizes = join_all( | ________________________________- 3450 | | frags 3451 | | .iter() 3452 | | .map(|frag| async move { frag.physical_rows().await.unwrap_or(0) }), 3453 | | ) | |_________- this call returns `JoinAll, {closure@rust/lancedb/src/table.rs:3452:22: 3452:28}>>` 3454 | .await; | ^^^^^ `JoinAll, {closure@...}>>` is not a future error[E0277]: the trait bound `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}: futures::Future` is not satisfied --> rust/lancedb/src/table.rs:3454:10 | 3454 | .await; | ^^^^^ the trait `futures::Future` is not implemented for `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` | = note: `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` implements similarly named trait `std::future::Future`, but not `futures::Future` = help: the following other types implement trait `futures::Future`: &'a mut F AssertUnwindSafe BiLockAcquire Box Concat2 Either Finished Fold and 43 others = note: required for `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` to implement `futures::IntoFuture` note: required by a bound in `JoinAll` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:24:20 | 22 | pub struct JoinAll | ------- required by a bound in this struct 23 | where I: IntoIterator, 24 | I::Item: IntoFuture, | ^^^^^^^^^^ required by this bound in `JoinAll` error[E0282]: type annotations needed --> rust/lancedb/src/utils/background_cache.rs:119:40 | 119 | inner: Arc::new(Mutex::new(CacheInner { | ________________________________________^ 120 | | state: State::Empty, 121 | | generation: 0, 122 | | })), | |_____________^ cannot infer type of the type parameter `E` declared on the struct `CacheInner` | help: consider specifying the generic arguments | 119 | inner: Arc::new(Mutex::new(CacheInner:: { | ++++++++ error[E0282]: type annotations needed --> rust/lancedb/src/utils/background_cache.rs:134:9 | 134 | cache.state.fresh_value(self.ttl, self.refresh_window) | ^^^^^^^^^^^ cannot infer type for type parameter `E` error[E0282]: type annotations needed --> rust/lancedb/src/utils/background_cache.rs:173:23 | 173 | cache.state = State::Current(value, clock::now()); | ^^^^^^^^^^^^^^ cannot infer type of the type parameter `E` declared on the enum `State` | help: consider specifying the generic arguments | 173 | cache.state = State::::Current(value, clock::now()); | ++++++++ error[E0282]: type annotations needed --> rust/lancedb/src/utils/background_cache.rs:182:23 | 182 | cache.state = State::Empty; | ^^^^^^^^^^^^ cannot infer type of the type parameter `E` declared on the enum `State` | help: consider specifying the generic arguments | 182 | cache.state = State::::Empty; | ++++++++ error[E0599]: no method named `boxed` found for `async` block `{async block@rust/lancedb/src/utils/background_cache.rs:269:22: 269:32}` in the current scope --> rust/lancedb/src/utils/background_cache.rs:270:14 | 269 | let shared = async move { (fetch)().await.map_err(Arc::new) } | ______________________- 270 | | .boxed() | | -^^^^^ method not found in `{async block@rust/lancedb/src/utils/background_cache.rs:269:22: 269:32}` | |_____________| error[E0277]: the trait bound `TimeoutStream: futures_core::stream::Stream` is not satisfied --> rust/lancedb/src/utils/mod.rs:345:9 | 345 | Box::pin(Self::new(inner, timeout)) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ unsatisfied trait bound error[E0599]: no method named `poll_next` found for struct `Pin<&mut TimeoutStream>` in the current scope --> rust/lancedb/src/utils/mod.rs:376:22 | 376 | self.poll_next(cx) | ^^^^^^^^^ | = help: items from traits can only be used if the trait is implemented and in scope = note: the following traits define an item `poll_next`, perhaps you need to implement one of them: candidate #1: `futures_core::stream::Stream` candidate #2: `sorts::stream::PartitionedStream` help: there is a method `collect` with a similar name, but with different arguments --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:563:5 | 563 | / fn collect(self) -> Collect 564 | | where Self: Sized | |_________________________^ error[E0599]: no method named `poll_unpin` found for mutable reference `&mut Pin>` in the current scope --> rust/lancedb/src/utils/mod.rs:378:75 | 378 | TimeoutState::Started { deadline, timeout } => match deadline.poll_unpin(cx) { | ^^^^^^^^^^ method not found in `&mut Pin>` error[E0599]: no method named `poll_next` found for struct `Pin<&mut Pin>>` in the current scope --> rust/lancedb/src/utils/mod.rs:386:27 | 386 | inner.poll_next(cx) | ^^^^^^^^^ error[E0277]: the trait bound `MaxBatchLengthStream: futures_core::stream::Stream` is not satisfied --> rust/lancedb/src/utils/mod.rs:419:13 | 419 | Box::pin(Self::new(inner, max_batch_length)) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ unsatisfied trait bound error[E0599]: no method named `poll_next` found for struct `Pin<&mut Pin>>` in the current scope --> rust/lancedb/src/utils/mod.rs:439:50 | 439 | return Pin::new(&mut self.inner).poll_next(cx); | ^^^^^^^^^ error[E0599]: no method named `poll_next` found for struct `Pin<&mut Pin>>` in the current scope --> rust/lancedb/src/utils/mod.rs:459:45 | 459 | match Pin::new(&mut self.inner).poll_next(cx) { | ^^^^^^^^^ Some errors have detailed explanations: E0046, E0107, E0191, E0271, E0277, E0282, E0308, E0407, E0425... For more information about an error, try `rustc --explain E0046`. error: could not compile `lancedb` (lib) due to 118 previous errors ``` --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a879d1f8b..78474cd16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,7 @@ env_logger = "0.11" half = { "version" = "2.7.1", default-features = false, features = [ "num-traits", ] } -futures = "0" +futures = "0.3" log = "0.4" metrics = "0.24" metrics-util = "0.19" From c5f9efefe9396c058b3ce6aa2298c0ef635477fb Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:30:55 +0800 Subject: [PATCH 30/49] test(python): cover local sync multiple-vector search (#3830) ## Summary - add regression coverage for multiple query vectors in the local synchronous Python API - verify that each query vector receives its own limited nearest-neighbor result and `query_index` ## Root cause In LanceDB v0.16, the local synchronous scanner passed a nested vector array as one query, unlike the async and remote implementations. The subsequent sync-to-async table migration supplied the correct shared runtime path, but this local sync behavior was never regression-tested and issue #1857 remained open. ## Validation - `uv run --extra tests pytest python/tests/test_query.py::test_query_multiple_vectors -q` - `uv run --project python --extra tests --extra dev ruff format --check python/python/tests/test_query.py` - `uv run --project python --extra tests --extra dev ruff check .` Fixes #1857 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_query.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/python/tests/test_query.py b/python/python/tests/test_query.py index 6840be052..d2629d1a8 100644 --- a/python/python/tests/test_query.py +++ b/python/python/tests/test_query.py @@ -570,6 +570,15 @@ def test_query_builder(table): assert all(np.array(rs[0]["vector"]) == [1, 2]) +def test_query_multiple_vectors(table): + results = table.search([np.array([1, 2]), np.array([4, 5])]).limit(1).to_list() + + assert len(results) == 2 + results_by_query = {result["query_index"]: result for result in results} + assert results_by_query[0]["id"] == 1 + assert results_by_query[1]["id"] == 2 + + def test_with_row_id(table: lancedb.table.Table): rs = table.search().with_row_id(True).to_arrow() assert "_rowid" in rs.column_names From 2922c171f7feecc6ce6ab63621fb4e3791e36420 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:31:08 +0800 Subject: [PATCH 31/49] test(rust): cover Azure table URI separators (#3837) ## Root cause The former listing-database table URI builder used OS-native `Path::join` for object-store URIs. On Windows this inserted backslashes into `az://` table paths, so `table_names` found slash-delimited objects while `open_table` addressed a different key. The production path now builds URI paths with forward slashes after the equivalent S3 report was fixed in #2575, but #1072 remained open without Azure-specific regression coverage. ## Fix - Add Azure URI regression assertions at the Rust table URI construction boundary. - Cover connection bases both with and without a trailing slash, matching the behavior reported in #1072. - Verify the resulting table URI always uses forward slashes on every platform. ## Validation - `cargo fmt --all -- --check` - `cargo test --quiet -p lancedb --lib database::listing::tests::test_table_uri` - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` - `cargo test --quiet --features remote --tests` (866 passed, 1 ignored) Fixes #1072 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/database/listing.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index fea34bb48..a4624112e 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -2342,7 +2342,7 @@ mod tests { #[tokio::test] async fn test_table_uri() { - let (_tempdir, db) = setup_database().await; + let (_tempdir, mut db) = setup_database().await; let mut pb = PathBuf::new(); pb.push(db.uri.clone()); @@ -2351,6 +2351,18 @@ mod tests { let expected = pb.to_str().unwrap(); let uri = db.table_uri("test").ok().unwrap(); assert_eq!(uri, expected); + + // URI paths always use forward slashes, even on Windows. Using + // `Path::join` here used to produce `az://container/prefix\\test.lance`, + // which Azure treated as a different object from the table returned by + // `table_names` (https://github.com/lancedb/lancedb/issues/1072). + for base_uri in ["az://container/prefix", "az://container/prefix/"] { + db.uri = base_uri.to_string(); + assert_eq!( + db.table_uri("test").unwrap(), + "az://container/prefix/test.lance" + ); + } } /// Regression: connecting via a URL-style URI (which goes through From 4048150fdd3bc19d7c651f1e4e7e16c82b6b5888 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:31:19 +0800 Subject: [PATCH 32/49] test(python): cover nullable fixed-size-list ingestion (#3812) ## Summary - add regression coverage for adding dictionary rows with a nullable fixed-size-list column - verify ordinary list columns remain aligned alongside the null fixed-size-list value ## Root cause PyArrow infers an all-`None` dictionary column as the generic `null` type. The original schema-alignment path treated the target fixed-size-list type as proof that the inferred source was also list-like and unconditionally accessed `value_field`, which raised `AttributeError`. Current alignment logic correctly falls back to the target type when the source is not list-like; this test locks in that repair for the reported ingestion path. ## Validation - `uv run --extra tests pytest python/tests/test_table.py::test_add_with_empty_fixed_size_list_drops_bad_rows python/tests/test_table.py::test_add_nullable_fixed_size_list_with_none python/tests/test_table.py::test_add_nullable_struct_with_none -q` - `uv run --with pyarrow==19.0.1 --extra tests pytest python/tests/test_table.py::test_add_nullable_fixed_size_list_with_none -q` - `uv run --project python --extra dev ruff format --check python/python/tests/test_table.py` - `uv run --project python --extra dev ruff check .` Fixes #2340 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index b2bfa2a68..e5c4ad801 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -1845,6 +1845,27 @@ def test_add_with_empty_fixed_size_list_drops_bad_rows(mem_db: DBConnection): assert np.allclose(data["embedding"].to_pylist()[0], np.array([0.1] * 16)) +def test_add_nullable_fixed_size_list_with_none(mem_db: DBConnection): + """Regression test for issue #2340.""" + table = mem_db.create_table( + "test_nullable_fixed_size_list", + schema=pa.schema( + [ + pa.field("id", pa.string()), + pa.field("feature", pa.list_(pa.float32(), 256)), + pa.field("tags", pa.list_(pa.string())), + ] + ), + ) + + table.add([{"id": "1", "feature": None, "tags": ["tag1", "tag2"]}]) + + result = table.to_arrow() + assert result.to_pylist() == [ + {"id": "1", "feature": None, "tags": ["tag1", "tag2"]} + ] + + def test_add_nullable_struct_with_none(mem_db: DBConnection): """Regression test for issue #2654: a nullable struct column whose first batch contains only None values must not crash in From fc44535ceeac0fd73ae94d0a0151c35753cb3e40 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:31:30 +0800 Subject: [PATCH 33/49] fix(python): clarify bare Vector annotations (#3809) ## Summary - raise a clear `TypeError` when `Vector` is used without a dimension - preserve normal `Vector(dim)` behavior across Pydantic v1 and v2 - add a regression test that defines a model without importing PyArrow ## Root cause Pydantic interpreted the bare `Vector` factory as a callable field type and inspected its postponed annotations in the user model's namespace. Because that namespace did not define LanceDB's internal `pa` alias, model construction failed with the misleading `NameError: name 'pa' is not defined` instead of explaining that `Vector` must be parameterized. The factory now exposes Pydantic's v1 and v2 schema hooks and rejects bare use before signature introspection with guidance to use `Vector(dim)`. ## Validation - `uvx --from 'ruff==0.15.20' ruff check .` - `uvx --from 'ruff==0.15.20' ruff format --check python/python/lancedb/pydantic.py python/python/tests/test_pydantic.py` - `cd python && uv run --extra tests pytest python/tests/test_pydantic.py::test_bare_vector_raises_clear_error -q` - `cd python && uv run --extra tests pytest python/tests/test_pydantic.py -q` - compatibility checks with Pydantic 1.10.22, 2.11.4, and 2.13.4 Fixes #2384 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/lancedb/pydantic.py | 10 ++++++++++ python/python/tests/test_pydantic.py | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/python/python/lancedb/pydantic.py b/python/python/lancedb/pydantic.py index c4dedc0e6..1ab6e6fcc 100644 --- a/python/python/lancedb/pydantic.py +++ b/python/python/lancedb/pydantic.py @@ -153,6 +153,16 @@ def Vector( return FixedSizeList +def _raise_bare_vector_error(*_args): + raise TypeError("Vector must be parameterized with a dimension, e.g. Vector(128).") + + +# Pydantic v1 and v2 otherwise treat the bare Vector factory as a field validator +# and inspect its signature, which produces misleading errors about internal types. +setattr(Vector, "__get_validators__", _raise_bare_vector_error) +setattr(Vector, "__get_pydantic_core_schema__", _raise_bare_vector_error) + + def MultiVector( dim: int, value_type: pa.DataType = pa.float32(), nullable: bool = True ) -> Type: diff --git a/python/python/tests/test_pydantic.py b/python/python/tests/test_pydantic.py index e1d533784..db93d7c64 100644 --- a/python/python/tests/test_pydantic.py +++ b/python/python/tests/test_pydantic.py @@ -415,6 +415,17 @@ def test_nullable_vector(): assert schema == pa.schema([pa.field("vec", pa.list_(pa.float32(), 16), True)]) +def test_bare_vector_raises_clear_error(): + namespace = { + "__name__": "test_model_without_pyarrow", + "LanceModel": LanceModel, + "Vector": Vector, + } + + with pytest.raises(TypeError, match=r"Vector must be parameterized.*Vector\(128\)"): + exec("class TestModel(LanceModel):\n vector: Vector", namespace) + + def test_fixed_size_list_field(): class TestModel(pydantic.BaseModel): vec: Vector(16) From ec80acb668b361e395ecfa5ee381391bd3b3b859 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:31:42 +0800 Subject: [PATCH 34/49] fix(python): expose inline types to downstream checkers (#3817) ## Summary - publish the PEP 561 `py.typed` marker so downstream type checkers consume the inline public annotations - add a Pyright contract test that distinguishes synchronous `connect` from awaited `connect_async` - verify the marker is present in the installed package ## Root cause The public Python module already annotated `lancedb.connect` as synchronous and `lancedb.connect_async` as asynchronous. The private native `_lancedb.connect` stub is intentionally awaitable because it backs `connect_async`. However, the distribution did not include a PEP 561 marker, so downstream tools such as mypy could ignore the public inline annotations and expose misleading or incomplete type information. ## Validation - `python/.venv/bin/ruff format --check python/python/tests/test_db.py python/python/type_tests/connect.py` - `python/.venv/bin/ruff check .` - `cd python && .venv/bin/pytest python/tests/test_db.py::test_package_includes_pep_561_marker -q` - `cd python && .venv/bin/pyright --pythonpath .venv/bin/python` - downstream mypy contract check for both public connection functions Fixes #2159 Co-authored-by: lancedb-gatefixer[bot] <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/pyproject.toml | 1 + python/python/lancedb/py.typed | 1 + python/python/tests/test_db.py | 5 +++++ python/python/type_tests/connect.py | 15 +++++++++++++++ 4 files changed, 22 insertions(+) create mode 100644 python/python/lancedb/py.typed create mode 100644 python/python/type_tests/connect.py diff --git a/python/pyproject.toml b/python/pyproject.toml index cb175bd6d..348058957 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -140,6 +140,7 @@ include = [ "python/lancedb/remote/errors.py", "python/lancedb/embeddings/__init__.py", "python/lancedb/_lancedb.pyi", + "python/type_tests/connect.py", ] exclude = ["python/tests/"] pythonVersion = "3.13" diff --git a/python/python/lancedb/py.typed b/python/python/lancedb/py.typed new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/python/python/lancedb/py.typed @@ -0,0 +1 @@ + diff --git a/python/python/tests/test_db.py b/python/python/tests/test_db.py index 8f4a8850c..84e78fd8f 100644 --- a/python/python/tests/test_db.py +++ b/python/python/tests/test_db.py @@ -6,6 +6,7 @@ import inspect import re import sys from datetime import timedelta +from importlib import resources import os from types import SimpleNamespace @@ -18,6 +19,10 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError from lancedb.pydantic import LanceModel, Vector +def test_package_includes_pep_561_marker(): + assert resources.files(lancedb).joinpath("py.typed").is_file() + + def test_basic(tmp_path): db = lancedb.connect(tmp_path) diff --git a/python/python/type_tests/connect.py b/python/python/type_tests/connect.py new file mode 100644 index 000000000..eb2cba37c --- /dev/null +++ b/python/python/type_tests/connect.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +from typing import assert_type + +import lancedb +from lancedb import AsyncConnection, DBConnection + + +def check_connect_type() -> None: + assert_type(lancedb.connect("memory://"), DBConnection) + + +async def check_connect_async_type() -> None: + assert_type(await lancedb.connect_async("memory://"), AsyncConnection) From dbc3687c7b9cfc2e5923d2c8a28a94cb6b9e56da Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:31:53 +0800 Subject: [PATCH 35/49] fix(node): require compatible Node.js types (#3829) ## Summary - require Node.js 18-compatible type declarations when TypeScript consumers install them - keep the type peer optional for JavaScript-only consumers - add a regression test tying the Node type peer range to the supported runtime ## Root cause LanceDB requires Node.js 18 or newer, and its public types expose Apache Arrow declarations that import built-ins through the node: scheme. The package did not declare a matching @types/node peer requirement, so npm accepted projects pinned to Node 12 declarations and TypeScript then reported that node:stream and node:fs/promises did not exist. ## Validation - pnpm lint - pnpm build - pnpm run docs - pnpm test --runInBand (678 passed, 5 skipped) - packed-package consumer probe rejects @types/node 12.20.55 and installs with @types/node 18.19.130 Fixes #1713 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/package.test.ts | 14 ++++++++++++++ nodejs/package-lock.json | 6 ++++++ nodejs/package.json | 6 ++++++ 3 files changed, 26 insertions(+) create mode 100644 nodejs/__test__/package.test.ts diff --git a/nodejs/__test__/package.test.ts b/nodejs/__test__/package.test.ts new file mode 100644 index 000000000..7743d73d6 --- /dev/null +++ b/nodejs/__test__/package.test.ts @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import packageJson = require("../package.json"); + +describe("package metadata", () => { + it("requires Node.js type declarations compatible with the runtime", () => { + expect(packageJson.engines.node).toBe(">= 18"); + expect(packageJson.peerDependencies["@types/node"]).toBe(">=18"); + expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({ + optional: true, + }); + }); +}); diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 8e30b0fab..bdbd3cf79 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -55,7 +55,13 @@ "openai": "4.29.2" }, "peerDependencies": { + "@types/node": ">=18", "apache-arrow": ">=15.0.0 <=18.1.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@aws-crypto/crc32": { diff --git a/nodejs/package.json b/nodejs/package.json index f3f719af2..671f3f94d 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -101,6 +101,12 @@ "openai": "4.29.2" }, "peerDependencies": { + "@types/node": ">=18", "apache-arrow": ">=15.0.0 <=18.1.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } } From dd5cb4d805b6cd79c7ecf9fb25754285d5d631bd Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:32:05 +0800 Subject: [PATCH 36/49] test(python): cover float16 table creation from Arrow data (#3785) ## Summary - exercise float16 sanitization through the reported direct Arrow-data table creation path - assert that the inferred fixed-size vector schema remains float16 - retain end-to-end index creation and vector search coverage ## Root cause and fix PyArrow 16 does not provide an is_nan kernel for half-float arrays, so passing float16 vector values directly to that kernel raises ArrowNotImplementedError. LanceDB's sanitizer already carries the compatibility fix from #837: it casts float16 values to float32 only for NaN detection while preserving the stored vector type. The existing end-to-end regression created an empty schema-defined table and added data afterward. This change aligns that regression with the issue reproduction by creating a table directly from a FixedSizeList Arrow table and verifying the persisted schema. ## Validation - uv run --extra tests pytest python/tests/test_table.py::test_create_f16_table_from_arrow_data -q - direct 1,000-row by 128-dimension float16 Arrow-table reproduction - PyArrow 16.1 half-float is_nan kernel reproduction - uvx ruff@0.15.20 format --check python/python/tests/test_table.py - uvx ruff@0.15.20 check . Fixes #835 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index e5c4ad801..8fc06ea69 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -2759,15 +2759,40 @@ def test_create_with_embedding_function(mem_db: DBConnection): assert actual == expected +def test_create_f16_table_from_arrow_data(mem_db: DBConnection): + dimension = 32 + num_rows = 512 + values = pa.array( + np.random.default_rng(42) + .standard_normal(num_rows * dimension) + .astype(np.float16) + ) + df = pa.table( + { + "text": [f"s-{i}" for i in range(num_rows)], + "vector": pa.FixedSizeListArray.from_arrays(values, dimension), + } + ) + table = mem_db.create_table("f16_tbl", data=df) + assert table.schema.field("vector").type == pa.list_(pa.float16(), dimension) + table.create_index(num_partitions=2, num_sub_vectors=2) + + query = df["vector"][2].as_py() + expected = table.search(query).limit(2).to_arrow() + + assert "s-2" in expected["text"].to_pylist() + + def test_create_f16_table(mem_db: DBConnection): class MyTable(LanceModel): text: str vector: Vector(32, value_type=pa.float16()) + rng = np.random.default_rng(42) df = pa.table( { "text": [f"s-{i}" for i in range(512)], - "vector": [np.random.randn(32).astype(np.float16) for _ in range(512)], + "vector": [rng.standard_normal(32).astype(np.float16) for _ in range(512)], } ) table = mem_db.create_table( From 564e5d0d56802bfd2c025e9337b8dd810d42759a Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:32:17 +0800 Subject: [PATCH 37/49] fix(python): support Polars 1.32 table scans (#3801) ## Root cause `Table.to_polars()` disabled PyArrow predicate pushdown by selecting the non-PyArrow Polars scan callback. Polars 1.32.3 invokes that callback with `batch_size` both positionally and through its partial, so collecting the returned lazy frame raises `TypeError: _scan_pyarrow_dataset_impl() got multiple values for argument batch_size`. ## Fix - Keep the compatible PyArrow callback path. - Add an identity `map_batches` barrier so predicates stay in Polars instead of reaching the LanceDB adapter as unsupported PyArrow expressions. - Extend the tested Polars range through 1.32.3 and retain lazy-frame regression coverage. ## Validation - `python/tests/test_table.py::test_polars` with Polars 1.32.3 - `python/tests/test_table.py::test_polars` with the locked Polars 1.3.0 baseline - `ruff format --check` on the changed Python files - `ruff check .` - `uv lock --check` Fixes #2619 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/pyproject.toml | 2 +- python/python/lancedb/table.py | 28 +++++++++++++++++++++++----- python/python/tests/test_table.py | 1 + python/uv.lock | 2 +- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 348058957..ce71484de 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -60,7 +60,7 @@ tests = [ "pytest-asyncio>=0.21", "duckdb>=0.9.0", "pytz>=2023.3", - "polars>=0.19, <=1.3.0", + "polars>=0.19, <=1.32.3", "pyarrow<25", "pyarrow-stubs>=16.0", "pylance==9.0.0rc1", diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index ae36bac7a..59e2650eb 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -108,6 +108,11 @@ def _should_push_down_query_table( return namespace_client is not None and "QueryTable" in pushdown_operations +def _polars_predicate_pushdown_barrier(frame: Any) -> Any: + """Return a Polars frame unchanged while blocking predicate pushdown.""" + return frame + + _MODEL_BACKED_TOKENIZER_PREFIXES = ("jieba", "lindera") _MODEL_BACKED_TOKENIZER_ERRORS = ( "unknown base tokenizer", @@ -864,12 +869,18 @@ class Table(ABC): """ raise NotImplementedError - def to_polars(self, **kwargs) -> "pl.DataFrame": - """Return the table as a polars.DataFrame. + def to_polars(self, **kwargs) -> "pl.LazyFrame": + """Return the table as a Polars LazyFrame. + + Note + ---- + The Polars streaming engine is not supported because it does not currently + implement Python PyArrow dataset scans. Use the default engine when collecting + this LazyFrame. Returns ------- - polars.DataFrame + polars.LazyFrame """ raise NotImplementedError @@ -2569,6 +2580,9 @@ class LanceTable(Table): 2. Currently we've disabled push-down of the filters from polars because polars pushdown into pyarrow uses pyarrow compute expressions rather than SQl strings (which LanceDB supports) + 3. The Polars streaming engine is not supported because it does not + currently implement Python PyArrow dataset scans. Use the default + engine when collecting this LazyFrame. Returns ------- @@ -2577,8 +2591,12 @@ class LanceTable(Table): from lancedb.integrations.pyarrow import PyarrowDatasetAdapter dataset = PyarrowDatasetAdapter(self) - return pl.scan_pyarrow_dataset( - dataset, allow_pyarrow_filter=False, batch_size=batch_size + # Polars 1.32's non-PyArrow callback path passes batch_size twice. Keep + # the compatible PyArrow path, but block predicates because this adapter + # cannot translate PyArrow expressions into LanceDB filters. + return pl.scan_pyarrow_dataset(dataset, batch_size=batch_size).map_batches( + _polars_predicate_pushdown_barrier, + predicate_pushdown=False, ) # New unified API overload diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 8fc06ea69..4ad5d7c3d 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -929,6 +929,7 @@ def test_polars(mem_db: DBConnection): # enter table to polars dataframe result = table.to_polars() + assert isinstance(result, pl.LazyFrame) assert np.allclose(result.collect()["vector"].to_list(), data["vector"]) # make sure filtering isn't broken diff --git a/python/uv.lock b/python/uv.lock index 551dc3f68..2cdcb182e 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1998,7 +1998,7 @@ requires-dist = [ { name = "pillow", marker = "extra == 'clip'", specifier = ">=12.1.1" }, { name = "pillow", marker = "extra == 'embeddings'", specifier = ">=12.1.1" }, { name = "pillow", marker = "extra == 'siglip'", specifier = ">=12.1.1" }, - { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.3.0" }, + { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.32.3" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5.0" }, { name = "pyarrow", specifier = ">=16" }, { name = "pyarrow", marker = "extra == 'tests'", specifier = "<25" }, From 607e5569276e68fc9b7bd6803b2748e6028007e1 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:32:28 +0800 Subject: [PATCH 38/49] test(python): cover search after schema merge (#3784) ## Summary - add an end-to-end regression for indexed vector search after merging a pandas column - verify unmatched rows retain a null merged value instead of failing Arrow batch assembly ## Root cause Historical Lance readers could assemble schema-evolved columns in physical data-file order. Indexed row-ID reads after a merge could therefore omit or misorder the newly merged column for unmatched rows. The currently pinned Lance release contains the reader correction, but LanceDB did not cover the reported merge-then-search path. ## Validation - uv run --extra tests pytest python/tests/test_table.py::test_merge python/tests/test_table.py::test_search_after_merge -q - uv run --project python --extra dev ruff check . - uv run --project python --extra dev ruff format --check python/python/tests/test_table.py Fixes #599 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 4ad5d7c3d..eb6eaefaa 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -2218,6 +2218,45 @@ def test_merge(tmp_db: DBConnection, tmp_path): table.merge(other_dataset, left_on="id") +@pytest.mark.parametrize("storage_version", ["legacy", "stable"]) +def test_search_after_merge(tmp_path, storage_version): + pytest.importorskip("lance") + pd = pytest.importorskip("pandas") + + db = lancedb.connect( + tmp_path, + storage_options={"new_table_data_storage_version": storage_version}, + ) + rng = np.random.default_rng(42) + row_count = 512 + vectors = rng.standard_normal((row_count, 8)).astype(np.float32) + table = db.create_table( + "search_after_merge", + data=pd.DataFrame( + { + "id": [str(i) for i in range(row_count)], + "vector": list(vectors), + } + ), + ) + table.create_index("vector", config=IvfPq(num_partitions=1, num_sub_vectors=2)) + + links = pd.DataFrame( + { + "id": [str(i) for i in range(row_count // 2)], + "link": [f"https://example.com/{i}" for i in range(row_count // 2)], + } + ) + table.merge(links, left_on="id") + + query = table.search(vectors[-1]).refine_factor(50).limit(10) + assert "ANN" in query.explain_plan(verbose=True) + + result = query.to_arrow() + links_by_id = dict(zip(result["id"].to_pylist(), result["link"].to_pylist())) + assert links_by_id[str(row_count - 1)] is None + + def test_delete(mem_db: DBConnection): table = mem_db.create_table( "my_table", From 2ba7407dc36f4989dc720d96bd765601b94566ba Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:32:39 +0800 Subject: [PATCH 39/49] fix(node): cover non-nullable embedding schema append (#3835) ## Summary - Add an issue-specific regression for appending generated embeddings to an empty table with a non-nullable vector field. - Verify the custom embedding function produces the declared Float64 vectors and both appended rows are readable. ## Root cause In v0.4.19, records without a vector value were materialized against the explicit schema before embeddings were inserted. Apache Arrow inferred the generated batch vector field as nullable while the table retained the user-provided non-nullable field, then rejected the mismatched schemas. The current conversion path excludes the generated field from the initial record conversion and realigns the completed batch to the stored schema after embedding, but the reported empty-table append sequence lacked permanent regression coverage. ## Validation - `pnpm exec biome format --write __test__/embedding.test.ts` - `pnpm lint-ci` - `pnpm test -- --runInBand __test__/embedding.test.ts` (12 passed, 1 skipped integration test) - `pnpm build` - `pnpm run docs` Fixes #1281 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/embedding.test.ts | 60 +++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/nodejs/__test__/embedding.test.ts b/nodejs/__test__/embedding.test.ts index e56e80631..06184751e 100644 --- a/nodejs/__test__/embedding.test.ts +++ b/nodejs/__test__/embedding.test.ts @@ -11,8 +11,11 @@ import { Float16, Float32, Float64, + Int32, Schema, Utf8, + fromDataToBuffer, + tableFromIPC, } from "../lancedb/arrow"; import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding"; import { getRegistry, register } from "../lancedb/embedding/registry"; @@ -184,6 +187,63 @@ describe("embedding functions", () => { const vector0 = JSON.parse(JSON.stringify(arr[0].vector)); expect(vector0).toEqual([1, 2, 3]); }); + + it("should append generated vectors to a non-nullable schema", async () => { + @register("non_nullable_schema_test") + class MockEmbeddingFunction extends EmbeddingFunction { + ndims() { + return 3; + } + embeddingDataType(): Float { + return new Float64(); + } + async computeSourceEmbeddings(data: string[]) { + return data.map(() => [1, 2, 3]); + } + } + + const schema = new Schema([ + new Field("id", new Int32()), + new Field("text", new Utf8()), + new Field("type", new Utf8()), + new Field( + "vector", + new FixedSizeList(3, new Field("item", new Float64())), + ), + ]); + const func = new MockEmbeddingFunction(); + const db = await connect(tmpDir.name); + const table = await db.createEmptyTable("test_non_nullable", schema, { + embeddingFunction: { + function: func, + sourceColumn: "text", + }, + }); + + const data = [ + { id: 1, text: "Carrot", type: "vegetable" }, + { id: 2, text: "Apple", type: "fruit" }, + ]; + const buffer = await fromDataToBuffer( + data, + undefined, + await table.schema(), + ); + const generatedTable = tableFromIPC(buffer); + const vectorField = generatedTable.schema.fields.find( + (field) => field.name === "vector", + ); + expect(vectorField?.nullable).toBe(false); + + await table.add(data); + + const rows = await table.query().toArray(); + expect(rows).toHaveLength(2); + for (const row of rows) { + expect([...row.vector]).toEqual([1, 2, 3]); + } + }); + it("should error when appending to a table with an unregistered embedding function", async () => { @register("mock") class MockEmbeddingFunction extends EmbeddingFunction { From 11f24b1df408306d9a7801f83fe7fbed20b99e46 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:33:02 +0800 Subject: [PATCH 40/49] fix: explain unsupported object storage mounts (#3823) ## Summary - classify unsupported local-filesystem operations from Lance as a NotSupported error - explain that object-storage mounts cannot provide the safe commit operations Lance requires and direct users to native object-store URIs - preserve existing error behavior for other local I/O failures and non-local backends ## Root cause Mountpoint for Amazon S3 exposes an S3 bucket as a local path but does not implement atomic rename. Lance uses atomic rename for safe local commits, and the resulting unsupported I/O error was previously passed through as a generic Lance error, leaving Python users with an opaque low-level failure. Transparent support for such mounts is not safe; direct s3:// access remains the supported path. ## Validation - cargo test --quiet --features remote -p lancedb error::tests - cargo test --quiet --features remote -p lancedb --lib (807 passed, 1 ignored) - cargo check --quiet --features remote --tests --examples - cargo clippy --quiet --features remote --tests --examples - cargo fmt --all -- --check Fixes #2016 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/error.rs | 70 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/rust/lancedb/src/error.rs b/rust/lancedb/src/error.rs index f6f596f3d..4a6e6d8d9 100644 --- a/rust/lancedb/src/error.rs +++ b/rust/lancedb/src/error.rs @@ -169,6 +169,12 @@ impl From for Error { impl From for Error { fn from(source: lance::Error) -> Self { + if has_unsupported_local_filesystem_source(&source) { + return Self::NotSupported { + message: "the filesystem does not support an operation required for safe Lance commits (such as atomic rename). Object-storage mounts such as Mountpoint for Amazon S3 are not supported; use the native object-store URI (for example, s3://bucket/path) instead".to_string(), + }; + } + // Try to unwrap external errors that were wrapped by lance match source { lance::Error::Wrapped { error, .. } => Self::from_box_error(error), @@ -181,6 +187,27 @@ impl From for Error { } } +fn has_unsupported_local_filesystem_source(error: &(dyn std::error::Error + 'static)) -> bool { + let mut current = Some(error); + let mut is_local_filesystem = false; + let mut is_unsupported = false; + while let Some(error) = current { + is_local_filesystem |= error + .downcast_ref::() + .is_some_and(|error| { + matches!(error, object_store::Error::Generic { store, .. } if *store == "LocalFileSystem") + }); + is_unsupported |= error + .downcast_ref::() + .is_some_and(|error| error.kind() == std::io::ErrorKind::Unsupported); + if is_local_filesystem && is_unsupported { + return true; + } + current = error.source(); + } + false +} + impl Error { fn from_box_error(mut source: Box) -> Self { source = match source.downcast::() { @@ -270,3 +297,46 @@ impl From for Error { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_filesystem_operations_have_actionable_error() { + let object_store_error = object_store::Error::Generic { + store: "LocalFileSystem", + source: Box::new(std::io::Error::from(std::io::ErrorKind::Unsupported)), + }; + let lance_error = lance::Error::io_source(Box::new(object_store_error)); + + let error = Error::from(lance_error); + + assert!(matches!( + error, + Error::NotSupported { message } + if message.contains("Mountpoint for Amazon S3") + && message.contains("s3://bucket/path") + )); + } + + #[test] + fn other_io_errors_remain_lance_errors() { + let object_store_error = object_store::Error::Generic { + store: "LocalFileSystem", + source: Box::new(std::io::Error::from(std::io::ErrorKind::PermissionDenied)), + }; + let lance_error = lance::Error::io_source(Box::new(object_store_error)); + + assert!(matches!(Error::from(lance_error), Error::Lance { .. })); + } + + #[test] + fn unsupported_non_filesystem_errors_remain_lance_errors() { + let lance_error = lance::Error::io_source(Box::new(std::io::Error::from( + std::io::ErrorKind::Unsupported, + ))); + + assert!(matches!(Error::from(lance_error), Error::Lance { .. })); + } +} From 6ba80a960cd1a54f6d8b625124743301beaf4173 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:33:13 +0800 Subject: [PATCH 41/49] fix(node): cover offset pagination in search (#3814) ## Summary - add Node regression coverage for vector-search offset pagination - add equivalent coverage for full-text search - compare later pages with the corresponding complete-result slice and assert page sizes ## Root cause The historical query path requested only the user limit from nearest-neighbor or full-text search before applying the offset, so a page became empty when its offset reached that limit. The production query path on current main already incorporates the later fix from #2592; this change adds the missing Node binding coverage for the still-open report and protects both affected APIs from regression. ## Validation - corepack pnpm build - corepack pnpm test -- query.test.ts --runInBand --testNamePattern="Search pagination" - corepack pnpm lint-ci - corepack pnpm tsc - corepack pnpm run docs Fixes #2229 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/query.test.ts | 75 +++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/nodejs/__test__/query.test.ts b/nodejs/__test__/query.test.ts index da001b1eb..5f3e68b16 100644 --- a/nodejs/__test__/query.test.ts +++ b/nodejs/__test__/query.test.ts @@ -110,6 +110,81 @@ describe("Query outputSchema", () => { }); }); +describe("Search pagination", () => { + let tmpDir: tmp.DirResult; + let table: Table; + + beforeEach(async () => { + tmpDir = tmp.dirSync({ unsafeCleanup: true }); + const db = await connect(tmpDir.name); + const schema = new Schema([ + new Field("id", new Int64(), false), + new Field("text", new Utf8(), false), + new Field( + "vector", + new FixedSizeList(2, new Field("item", new Float32())), + false, + ), + ]); + const data = makeArrowTable( + [ + { id: 1n, text: "common", vector: [0, 0] }, + { id: 2n, text: "common common", vector: [1, 1] }, + { id: 3n, text: "common common common", vector: [2, 2] }, + { id: 4n, text: "common common common common", vector: [3, 3] }, + ], + { schema }, + ); + table = await db.createTable("test", data); + }); + + afterEach(() => { + tmpDir.removeCallback(); + }); + + it("applies offset after the vector search limit", async () => { + const allResults = await table + .vectorSearch([0, 0]) + .select(["id"]) + .limit(4) + .toArray(); + const secondPage = await table + .vectorSearch([0, 0]) + .select(["id"]) + .limit(2) + .offset(2) + .toArray(); + + expect(allResults).toHaveLength(4); + expect(secondPage).toHaveLength(2); + expect(secondPage.map((row) => row.id)).toEqual( + allResults.slice(2, 4).map((row) => row.id), + ); + }); + + it("applies offset after the full-text search limit", async () => { + await table.createIndex("text", { config: Index.fts() }); + + const allResults = await table + .search("common", "fts") + .select(["id"]) + .limit(4) + .toArray(); + const secondPage = await table + .search("common", "fts") + .select(["id"]) + .limit(2) + .offset(2) + .toArray(); + + expect(allResults).toHaveLength(4); + expect(secondPage).toHaveLength(2); + expect(secondPage.map((row) => row.id)).toEqual( + allResults.slice(2, 4).map((row) => row.id), + ); + }); +}); + describe("Query orderBy", () => { let tmpDir: tmp.DirResult; let table: Table; From ec21e370401a2d74b43ca3efa187238905bf5fa8 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:37:37 +0800 Subject: [PATCH 42/49] test(rust): cover Hugging Face table symlinks (#3887) ## Summary - cover Hugging Face cache layouts where both manifests and Lance data files are relative symlinks into a blob directory - reconnect with a fresh session before opening so the test exercises filesystem discovery instead of cached manifest metadata - scan the reopened table to verify both manifest recovery and data-file reads ## Root cause Lance 3.0.1 recorded Unix symlink metadata as the known manifest size, so the short link length caused a file size is too small error. The current Lance v11.0.0-beta.2 dependency repairs this by detecting an invalid footer from a stale known size and retrying with the target file metadata. This regression test locks that behavior into the LanceDB open-table path used by Node. ## Validation - cargo fmt --all - cargo test --quiet --features remote -p lancedb --lib test_open_table_follows_hugging_face_symlinks -- --nocapture - cargo test --quiet --features remote -p lancedb --lib database::listing::tests - cargo clippy --quiet --features remote -p lancedb --lib --tests -- -D warnings - cargo check --quiet --features remote --tests --examples Fixes #3197 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/database/listing.rs | 92 +++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index a4624112e..0ab3614e7 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -1294,9 +1294,11 @@ mod tests { use crate::connection::ConnectRequest; use crate::data::scannable::Scannable; use crate::database::{CreateTableMode, CreateTableRequest}; - use crate::table::WriteOptions; + use crate::query::QueryRequest; + use crate::table::{AnyQuery, WriteOptions}; use arrow_array::{Int32Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; + use futures::TryStreamExt; use std::path::PathBuf; use tempfile::tempdir; @@ -1438,6 +1440,94 @@ mod tests { assert!(after_open.hits >= before_open.hits + 3); } + /// Regression test for https://github.com/lancedb/lancedb/issues/3197. + #[cfg(unix)] + #[tokio::test] + async fn test_open_table_follows_hugging_face_symlinks() { + let (tempdir, db) = setup_database().await; + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + db.create_table(CreateTableRequest { + name: "test".to_string(), + namespace_path: vec![], + data: Box::new( + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]) + .unwrap(), + ) as Box, + mode: CreateTableMode::Create, + write_options: Default::default(), + location: None, + namespace_client: None, + }) + .await + .unwrap(); + + let table_dir = tempdir.path().join("test.lance"); + let versions_dir = table_dir.join("_versions"); + let manifest_path = std::fs::read_dir(&versions_dir) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| path.extension().is_some_and(|ext| ext == "manifest")) + .unwrap(); + let data_path = std::fs::read_dir(table_dir.join("data")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| path.extension().is_some_and(|ext| ext == "lance")) + .unwrap(); + + // Hugging Face snapshots keep dataset objects in a separate blob directory and + // expose them through relative symlinks. + let blobs_dir = tempdir.path().join("blobs"); + std::fs::create_dir(&blobs_dir).unwrap(); + let manifest_blob = "9b603c63d0e692e05d58be25605f2f2064cc781e5ff94fe983a405059547b816"; + let data_blob = "be64f20e5723bd0a27cfdbdb41cf7d6fad94cd572a71973b717fb8340f4310c5"; + std::fs::rename(&manifest_path, blobs_dir.join(manifest_blob)).unwrap(); + std::fs::rename(&data_path, blobs_dir.join(data_blob)).unwrap(); + std::os::unix::fs::symlink(Path::new("../../blobs").join(manifest_blob), &manifest_path) + .unwrap(); + std::os::unix::fs::symlink(Path::new("../../blobs").join(data_blob), &data_path).unwrap(); + let symlink_len = std::fs::symlink_metadata(&manifest_path).unwrap().len(); + let target_len = std::fs::metadata(&manifest_path).unwrap().len(); + assert_ne!(symlink_len, target_len); + + drop(db); + let db = ListingDatabase::connect_with_options(&ConnectRequest { + uri: tempdir.path().to_str().unwrap().to_string(), + #[cfg(feature = "remote")] + client_config: Default::default(), + options: Default::default(), + namespace_client_properties: Default::default(), + manifest_enabled: false, + read_consistency_interval: None, + session: None, + }) + .await + .unwrap(); + + let table = db + .open_table(OpenTableRequest { + name: "test".to_string(), + namespace_path: vec![], + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await + .unwrap(); + let batches = table + .query( + &AnyQuery::Query(QueryRequest::default()), + Default::default(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 3); + } + #[tokio::test] async fn test_clone_table_basic() { let (_tempdir, db) = setup_database().await; From 79ba076429d60dbdc245278ea3959d97bd5b5aa3 Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Fri, 7 Aug 2026 13:44:49 -0500 Subject: [PATCH 43/49] feat(table): checkpoint_lsm, flush_lsm, compact_lsm, get_lsm_stats (#3736) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Converge a table's LSM write path into its base table, and inspect it. `checkpoint_lsm` is `flush` then `compact`, repeated until the fresh tier is empty — and the loop runs **client-side**. Putting it on the server would mean a background task, which means a single-flight intent, an intent that leaks on panic, a bounded-iteration policy, an "is it done" observable, and a story for every way a client can vanish mid-operation. None of that exists in this shape: each request does a bounded unit of work and reports what is left, so completion is *carried in the responses* rather than inferred from a shared counter that cannot distinguish "converged" from "hasn't started yet". Best-effort by construction. Nothing is frozen, so `converged` means L0 was empty as of the last pass. It is idempotent, abandonable at any point with zero consequence, and safe to run on a cadence — an already-converged table costs one round trip and zero compaction passes, because `flush` reports `generations_remaining` and the loop is never entered. ## The failure taxonomy is the load-bearing part Five distinct conditions used to arrive at a client as one 503. `Error::LsmRoute` carries a classification read from the response body's namespace error code **at the point of receipt** — before any generic helper folds the body into a string and keeps only the status. | condition | wire | client action | |---|---|---| | contention (latch held / pool saturated) | 429, code 21 | retry with backoff | | owning node draining | 503, code 19 `InvalidTableState` | **stop** | | fenced / no slot / transport | 503, code 17 | retry with backoff | | registry entry vanished | 404 | re-issue from `flush` (capped) | | table being dropped / not WAL-backed | 409 / 400 | stop | Draining is terminal because the drain gate is a one-way latch — retrying spins until the deadline to report a failure that was knowable on the first response. Transport retry is disabled on these routes for the same reason: it treats every 503 alike and would burn its budget before the classifier ever saw the body. `get_lsm_stats` returns `Option`, matching `get_lsm_write_spec` — `None` only when the table has no LSM write path, since a struct of zeros would read as measurements. Python bindings mirror all four, preserving per-bucket detail rather than flattening to a table-level summary. ## Testing Six new unit tests against the mocked endpoint, plus the taxonomy round-trip: - flush into an empty L0 issues **zero** compact calls (asserts the call count — `generations_consumed: 0` is also true of a loop that ran a pointless pass) - the loop drives compact until the server reports zero remaining - **contention is not draining**: a 429 retries and converges; asserts the retry count - a draining node stops after **exactly one** request, no retries - stats round-trips fully populated; `include_generation_rows` off by default - every `(status, code)` pair classifies correctly, including unparseable 503 bodies falling back to *retryable* rather than terminal `cargo test -p lancedb --features remote --lib`: 723 passed. ## Notes for review - Depends on the sibling lance change returning `SealedGeneration` from `force_seal_active` only at the *server* level — no lance API is used here. - The branch is based on `codex/update-lance-10-0-0-beta-5`, so it carries one extra commit (`chore: update lance dependency to v10.0.0-beta.5`) that is not part of this change. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: lancedb automation Co-authored-by: Claude Opus 5 (1M context) --- python/python/lancedb/_lancedb.pyi | 4 + python/python/lancedb/table.py | 83 ++++ python/src/table.rs | 108 +++++- rust/lancedb/Cargo.toml | 2 +- rust/lancedb/src/remote/table.rs | 541 +++++++++++++++++++++++++++ rust/lancedb/src/table.rs | 107 ++++++ rust/lancedb/src/table/checkpoint.rs | 315 ++++++++++++++++ rust/lancedb/src/table/lsm_stats.rs | 162 ++++++++ 8 files changed, 1320 insertions(+), 2 deletions(-) create mode 100644 rust/lancedb/src/table/checkpoint.rs create mode 100644 rust/lancedb/src/table/lsm_stats.rs diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 47e727f99..fad2744d3 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -355,6 +355,10 @@ class Table: async def set_lsm_write_spec(self, spec: LsmWriteSpec) -> None: ... async def unset_lsm_write_spec(self) -> None: ... async def get_lsm_write_spec(self) -> Optional[LsmWriteSpec]: ... + async def checkpoint_lsm(self) -> None: ... + async def flush_lsm(self) -> None: ... + async def compact_lsm(self) -> None: ... + async def get_lsm_stats(self, include_generation_rows: bool) -> Optional[dict]: ... async def close_lsm_writers(self) -> None: ... @property def tags(self) -> Tags: ... diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 59e2650eb..0828f04dc 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -3976,6 +3976,28 @@ class LanceTable(Table): [`AsyncTable.get_lsm_write_spec`][lancedb.AsyncTable.get_lsm_write_spec].""" return LOOP.run(self._table.get_lsm_write_spec()) + def checkpoint_lsm(self) -> None: + """Synchronous version of + [`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm].""" + return LOOP.run(self._table.checkpoint_lsm()) + + def flush_lsm(self) -> None: + """Synchronous version of + [`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm].""" + return LOOP.run(self._table.flush_lsm()) + + def compact_lsm(self) -> None: + """Synchronous version of + [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm].""" + return LOOP.run(self._table.compact_lsm()) + + def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]: + """Synchronous version of + [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats].""" + return LOOP.run( + self._table.get_lsm_stats(include_generation_rows=include_generation_rows) + ) + def close_lsm_writers(self) -> None: """Close cached MemWAL shard writers. See [`AsyncTable.close_lsm_writers`][lancedb.AsyncTable.close_lsm_writers].""" @@ -4686,6 +4708,67 @@ class AsyncTable: """ return await self._inner.get_lsm_write_spec() + async def checkpoint_lsm(self) -> None: + """Converge this table's LSM write path into its base table. + + One flush, sealing every memtable into L0, then compaction triggers + until every generation that existed at that moment has reached base. + The loop runs client-side, reading progress from ``get_lsm_stats``. + + Best-effort: generations created *while* it runs are deliberately not + waited on, which is what lets it terminate on a table taking writes. + Idempotent and safe on a cadence. + + There is no deadline, and the caller owns that. It returns when the + target generations are gone, raises on a terminal server fault, and + otherwise waits however long the server takes. A slow table and a + stuck one are the same picture from the client: the compactor pool is + shared across every table on the node, so a checkpoint queued behind + unrelated work looks exactly like one that is merging. Wrap this in + ``asyncio.wait_for`` for a wall-clock bound; abandoning it partway + costs nothing. + """ + return await self._inner.checkpoint_lsm() + + async def flush_lsm(self) -> None: + """Seal every bucket's active memtable into L0. + + Does not touch the base table — moving L0 into base is + `compact_lsm`. On a node that has not claimed this table, this claims + it and replays its WAL log first. + """ + return await self._inner.flush_lsm() + + async def compact_lsm(self) -> None: + """Trigger a background L0 to base compaction pass per bucket. + + Returns once the passes are dispatched, not once they finish: watch + ``get_lsm_stats`` for progress, or use ``checkpoint_lsm`` to loop + until the current L0 has reached base. + """ + return await self._inner.compact_lsm() + + async def get_lsm_stats( + self, *, include_generation_rows: bool = False + ) -> Optional[dict]: + """Read live per-bucket LSM state. + + Answers "how far behind is my fresh tier", "which bucket is hot", and + "why is my fresh-tier vector search brute-force". Mutates no table + state, though on a node that has not claimed this table it claims it, + exactly as a read would. + + Returns ``None`` only when the LSM write path is not enabled. + + Parameters + ---------- + include_generation_rows + Report a row count per L0 generation. Off by default: each count + opens an uncached Lance dataset, and ``checkpoint_lsm`` polls this + needing only generation numbers. + """ + return await self._inner.get_lsm_stats(include_generation_rows) + async def close_lsm_writers(self) -> None: """Drain and close any cached MemWAL shard writers for this table. diff --git a/python/src/table.rs b/python/src/table.rs index 5b5d6596a..119388708 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -28,11 +28,72 @@ use pyo3::{ Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python, exceptions::{PyRuntimeError, PyValueError}, pyclass, pyfunction, pymethods, - types::{IntoPyDict, PyAnyMethods, PyBytes, PyDict, PyDictMethods}, + types::{IntoPyDict, PyAnyMethods, PyBytes, PyDict, PyDictMethods, PyList, PyListMethods}, }; mod scannable; +/// Convert `LsmStats` to a Python dict, preserving the per-bucket list. +/// +/// Deliberately not flattened to a table-level summary: a table is N +/// buckets on one node, and the per-bucket detail is the reason the +/// endpoint exists — flattening hides the single hot bucket someone opened +/// it to find. +fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult> { + let out = PyDict::new(py); + let buckets = PyList::empty(py); + for b in &stats.buckets { + let e = PyDict::new(py); + e.set_item("shard_id", &b.shard_id)?; + e.set_item("status", &b.status)?; + e.set_item("writer_epoch", b.writer_epoch)?; + e.set_item("manifest_version", b.manifest_version)?; + e.set_item("current_generation", b.current_generation)?; + e.set_item( + "replay_after_wal_entry_position", + b.replay_after_wal_entry_position, + )?; + e.set_item( + "wal_entry_position_last_seen", + b.wal_entry_position_last_seen, + )?; + + let generations = PyList::empty(py); + for g in &b.generations { + let ge = PyDict::new(py); + ge.set_item("generation", g.generation)?; + ge.set_item("bytes", g.bytes)?; + ge.set_item("rows", g.rows)?; + generations.append(ge)?; + } + e.set_item("generations", generations)?; + e.set_item("compacting", b.compacting)?; + + e.set_item( + "memtables", + b.memtables + .as_ref() + .map(|ms| { + let l = PyList::empty(py); + for m in ms { + let d = PyDict::new(py); + d.set_item("generation", m.generation)?; + d.set_item("rows", m.rows)?; + d.set_item("bytes", m.bytes)?; + d.set_item("batches", m.batches)?; + d.set_item("indexes", m.indexes.clone())?; + l.append(d)?; + } + PyResult::Ok(l.unbind()) + }) + .transpose()?, + )?; + buckets.append(e)?; + } + out.set_item("buckets", buckets)?; + Ok(out.unbind()) +} + #[derive(FromPyObject)] enum PredicateArg { Expr(PyExpr), @@ -1339,6 +1400,51 @@ impl Table { }) } + /// Converge the table's LSM write path into its base table. + /// + /// Best-effort: with writes flowing, new rows may land after the last + /// pass. Errors if the table stops making progress. + pub fn checkpoint_lsm(self_: PyRef<'_, Self>) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + future_into_py(self_.py(), async move { + inner.checkpoint_lsm().await.infer_error() + }) + } + + /// Seal every bucket's active memtable into L0. + pub fn flush_lsm(self_: PyRef<'_, Self>) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + future_into_py( + self_.py(), + async move { inner.flush_lsm().await.infer_error() }, + ) + } + + /// Trigger a background L0 → base pass per bucket. Returns once the + /// passes are dispatched, not once they finish — watch `get_lsm_stats`. + pub fn compact_lsm(self_: PyRef<'_, Self>) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + future_into_py(self_.py(), async move { + inner.compact_lsm().await.infer_error() + }) + } + + /// Live LSM state, or `None` when the LSM write path is not enabled. + #[pyo3(signature = (include_generation_rows=false))] + pub fn get_lsm_stats( + self_: PyRef<'_, Self>, + include_generation_rows: bool, + ) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + future_into_py(self_.py(), async move { + let stats = inner + .get_lsm_stats(include_generation_rows) + .await + .infer_error()?; + Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose()) + }) + } + pub fn close_lsm_writers(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 96ea9ec95..c2137d0b5 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -100,7 +100,7 @@ anyhow = "1" lance-testing = { workspace = true } tempfile = "3.5.0" random_word = { version = "0.4.3", features = ["en"] } -tokio = { version = "1.23", features = ["io-util", "macros", "net", "rt-multi-thread", "sync"] } +tokio = { version = "1.23", features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "test-util"] } uuid = { version = "1.7.0", features = ["v4"] } walkdir = "2" aws-sdk-dynamodb = { version = "1.55.0" } diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 388bed0f7..f3e872cbe 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -23,11 +23,13 @@ use crate::table::AddResult; use crate::table::BranchDiff; use crate::table::DeleteResult; use crate::table::DropColumnsResult; +use crate::table::LsmStats; use crate::table::LsmWriteSpec; use crate::table::MergeBranchResult; use crate::table::MergeResult; use crate::table::Tags; use crate::table::UpdateResult; +use crate::table::lsm_stats::GetLsmStatsResponse; use crate::table::merge::MergeFilter; use crate::table::query::create_multi_vector_plan; use crate::table::write_progress::FinishOnDrop; @@ -991,6 +993,18 @@ impl RemoteTable { } } + /// Send an LSM operator request with the transport retry layer **off**. + /// + /// Retry policy on these routes belongs to the checkpoint loop, which + /// reads the status and can tell contention from a lost claim. Leaving the + /// transport layer on would re-ask on its own schedule first, and surface + /// an `Error::Retry` whose status the loop would then have to unwrap. + async fn send_lsm_route(&self, request: RequestBuilder) -> Result<(String, reqwest::Response)> { + let (request_id, response) = self.send(request, false).await?; + let response = self.check_table_response(&request_id, response).await?; + Ok((request_id, response)) + } + /// Build a POST request and attach the read-freshness headers /// (`x-lancedb-min-version`, `x-lancedb-min-timestamp`). fn post_read(&self, uri: &str) -> RequestBuilder { @@ -2468,6 +2482,40 @@ impl BaseTable for RemoteTable { }) } + async fn flush_lsm(&self) -> Result<()> { + let request = self + .client + .post(&format!("/v1/table/{}/flush_lsm/", self.identifier)); + self.send_lsm_route(request).await?; + Ok(()) + } + + async fn compact_lsm(&self) -> Result<()> { + let request = self + .client + .post(&format!("/v1/table/{}/compact_lsm/", self.identifier)); + self.send_lsm_route(request).await?; + Ok(()) + } + + async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result> { + // Read-semantics POST, like `get_lsm_write_spec`. + let request = self + .post_read(&format!("/v1/table/{}/get_lsm_stats/", self.identifier)) + .json(&serde_json::json!({ + "include_generation_rows": include_generation_rows, + })); + let (request_id, response) = self.send_lsm_route(request).await?; + let body = response.text().await.err_to_http(request_id.clone())?; + let parsed: GetLsmStatsResponse = serde_json::from_str(&body).map_err(|e| Error::Http { + source: format!("Failed to parse get_lsm_stats response: {e}").into(), + request_id, + status_code: None, + })?; + // `null` — and only — when the table has no LSM write path. + Ok(parsed.lsm_stats) + } + async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> Result<()> { self.check_mutable().await?; @@ -6682,6 +6730,499 @@ mod tests { assert!(table.get_lsm_write_spec().await.unwrap().is_none()); } + /// Build a `get_lsm_stats` body for one bucket holding `generations`. + fn stats_body(generations: &[u64], compacting: bool) -> String { + serde_json::json!({ + "lsm_stats": { + "buckets": [{ + "shard_id": "b0", + "status": "Active", + "writer_epoch": 1, + "manifest_version": 1, + "current_generation": generations.iter().max().copied().unwrap_or(0) + 1, + "replay_after_wal_entry_position": 0, + "wal_entry_position_last_seen": 0, + "generations": generations.iter() + .map(|g| serde_json::json!({ "generation": g, "bytes": 1 })) + .collect::>(), + "compacting": compacting, + "memtables": [], + }], + } + }) + .to_string() + } + + /// `flush_lsm` / `compact_lsm` answer 202 with no body at all. + fn accepted() -> http::Response { + http::Response::builder() + .status(202) + .body(String::new()) + .unwrap() + } + + fn ok_json(body: String) -> http::Response { + http::Response::builder().status(200).body(body).unwrap() + } + + /// A flush landing in an empty L0 finishes on the opening stats read + /// alone. Asserting zero compacts is the point: "it returned Ok" is also + /// true of a loop that ran a pointless pass. + #[tokio::test(start_paused = true)] + async fn test_checkpoint_short_circuits_on_empty_l0() { + let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let seen = compacts.clone(); + let table = Table::new_with_handler("my_table", move |request| { + let path = request.url().path().to_string(); + if path.contains("compact_lsm") { + seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + panic!("an already-converged table must issue no compact calls"); + } + if path.contains("flush_lsm") { + return accepted(); + } + assert_eq!(path, "/v1/table/my_table/get_lsm_stats/"); + ok_json(stats_body(&[], false)) + }); + + table.checkpoint_lsm().await.unwrap(); + assert_eq!(compacts.load(std::sync::atomic::Ordering::SeqCst), 0); + } + + /// The loop triggers compaction until every generation that existed at + /// the start is gone, one bounded prefix per pass. + #[tokio::test(start_paused = true)] + async fn test_checkpoint_triggers_until_targets_are_drained() { + let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let seen = compacts.clone(); + let table = Table::new_with_handler("my_table", move |request| { + let path = request.url().path().to_string(); + if path.contains("flush_lsm") { + return accepted(); + } + if path.contains("compact_lsm") { + seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + return accepted(); + } + // Each pass drains the oldest generation. + let drained = seen.load(std::sync::atomic::Ordering::SeqCst); + let left: Vec = [1u64, 2, 3].into_iter().skip(drained).collect(); + ok_json(stats_body(&left, false)) + }); + + table.checkpoint_lsm().await.unwrap(); + assert_eq!( + compacts.load(std::sync::atomic::Ordering::SeqCst), + 3, + "one trigger per generation prefix, then stop" + ); + } + + /// Generations created *during* the checkpoint are not waited on, which + /// is what lets the loop terminate on a table taking writes where "L0 is + /// empty" never becomes true. + #[tokio::test(start_paused = true)] + async fn test_checkpoint_ignores_generations_created_while_it_runs() { + let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let seen = compacts.clone(); + let table = Table::new_with_handler("my_table", move |request| { + let path = request.url().path().to_string(); + if path.contains("flush_lsm") { + return accepted(); + } + if path.contains("compact_lsm") { + seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + return accepted(); + } + // Target is 5. One pass drains it; a writer keeps adding above. + let n = seen.load(std::sync::atomic::Ordering::SeqCst); + let body = if n == 0 { + stats_body(&[5], false) + } else { + stats_body(&[6, 7], false) + }; + ok_json(body) + }); + + table.checkpoint_lsm().await.unwrap(); + assert_eq!( + compacts.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the loop must not chase generations written after it started" + ); + } + + /// Contention is a 429 and must be retried. The server keeps it off 503 + /// precisely so the client can act on the status alone — reading it as + /// terminal stops the checkpoint early on a healthy node. + #[tokio::test(start_paused = true)] + async fn test_checkpoint_retries_contention() { + let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let seen = compacts.clone(); + let table = Table::new_with_handler("my_table", move |request| { + let path = request.url().path().to_string(); + if path.contains("flush_lsm") { + return accepted(); + } + if path.contains("compact_lsm") { + // First two triggers: every bucket already latched. + if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2 { + return http::Response::builder() + .status(429) + .body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string()) + .unwrap(); + } + return accepted(); + } + let accepted_triggers = seen + .load(std::sync::atomic::Ordering::SeqCst) + .saturating_sub(2); + let left: Vec = if accepted_triggers == 0 { + vec![1] + } else { + vec![] + }; + ok_json(stats_body(&left, false)) + }); + + table + .checkpoint_lsm() + .await + .expect("contention must not abort the checkpoint"); + assert_eq!( + compacts.load(std::sync::atomic::Ordering::SeqCst), + 3, + "assert the retry count, not just the outcome" + ); + } + + /// A transient fault on the poll must not abort the checkpoint. This route + /// meets the most contention — it runs every `POLL_INTERVAL` for the + /// checkpoint's whole life, with the transport retry layer disabled — yet + /// was the one call reached with a bare `?`. + #[tokio::test(start_paused = true)] + async fn test_checkpoint_retries_a_contended_stats_poll() { + let polls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let seen = polls.clone(); + let table = Table::new_with_handler("my_table", move |request| { + let path = request.url().path().to_string(); + if path.contains("flush_lsm") || path.contains("compact_lsm") { + return accepted(); + } + // The opening read lands; the next two polls are latched out. + let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if (1..3).contains(&n) { + return http::Response::builder() + .status(429) + .body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string()) + .unwrap(); + } + ok_json(stats_body(if n < 4 { &[1] } else { &[] }, false)) + }); + + table + .checkpoint_lsm() + .await + .expect("a contended poll must be retried, not surfaced"); + assert_eq!( + polls.load(std::sync::atomic::Ordering::SeqCst), + 5, + "the two rejected polls must be re-issued, not skipped" + ); + } + + /// Contention and a lost claim draw on separate budgets: five straight + /// 429s on `flush`, more than `MAX_REISSUES`, must still converge. On one + /// shared counter this spent the re-issue cap and then reported a lost + /// claim nothing had ever reported. + #[tokio::test(start_paused = true)] + async fn test_contention_does_not_exhaust_the_reissue_budget() { + let flushes = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let seen = flushes.clone(); + let table = Table::new_with_handler("my_table", move |request| { + let path = request.url().path().to_string(); + if path.contains("flush_lsm") { + if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 5 { + return http::Response::builder() + .status(429) + .body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string()) + .unwrap(); + } + return accepted(); + } + if path.contains("compact_lsm") { + return accepted(); + } + ok_json(stats_body(&[], false)) + }); + + table + .checkpoint_lsm() + .await + .expect("contention must not be reported as a lost claim"); + assert_eq!( + flushes.load(std::sync::atomic::Ordering::SeqCst), + 6, + "five retries against one seal, then it lands" + ); + } + + /// An exhausted retry budget surfaces the fault that consumed it, not a + /// message the loop invented: "429, nine times" points an operator at a + /// saturated pool, a generic runtime error points them nowhere. + #[tokio::test(start_paused = true)] + async fn test_exhausted_retries_surface_the_underlying_fault() { + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let seen = calls.clone(); + let table = Table::new_with_handler("my_table", move |_request| { + seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + http::Response::builder() + .status(429) + .body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string()) + .unwrap() + }); + + let err = table.checkpoint_lsm().await.unwrap_err(); + assert!( + matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 429), + "the fault that spent the budget must be the one reported: {err:?}" + ); + assert_eq!( + calls.load(std::sync::atomic::Ordering::SeqCst), + 9, + "one call plus MAX_RETRIES — the re-issue budget is not spent on top" + ); + } + + /// A draining node is terminal, but the client does not know that from the + /// status: draining and a proxy blip are both 503, and telling them apart + /// takes parsing the body for a namespace code. So it spends the retry + /// budget and then reports what the server said — the drain gate never + /// releases, so the answer does not change, and the operator still reads + /// "WAL node draining" in the error. + #[tokio::test(start_paused = true)] + async fn test_draining_surfaces_after_the_retry_budget() { + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let seen = calls.clone(); + let table = Table::new_with_handler("my_table", move |_request| { + seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + http::Response::builder() + .status(503) + .body(r#"{"code":19,"error":"WAL node draining"}"#.to_string()) + .unwrap() + }); + + let err = table.checkpoint_lsm().await.unwrap_err(); + let message = err.to_string(); + assert!( + matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 503), + "the 503 must surface as itself: {err:?}" + ); + assert!( + message.contains("WAL node draining"), + "the server's own diagnosis must survive to the caller: {message}" + ); + assert_eq!( + calls.load(std::sync::atomic::Ordering::SeqCst), + 9, + "one call plus MAX_RETRIES, then it reports rather than spinning" + ); + } + + /// A long stall with nothing compacting must keep waiting, not fail. The + /// client cannot judge this: a checkpoint queued behind unrelated tables + /// on the pod-wide compactor pool reports exactly these numbers — flat + /// generations, an idle latch — as one whose merges are failing. The + /// deadline is the caller's. + #[tokio::test(start_paused = true)] + async fn test_checkpoint_waits_out_a_long_stall_rather_than_failing() { + let polls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let seen = polls.clone(); + let table = Table::new_with_handler("my_table", move |request| { + let path = request.url().path().to_string(); + if path.contains("flush_lsm") || path.contains("compact_lsm") { + return accepted(); + } + // Flat for far longer than any bound this loop ever had, with + // `compacting: false` throughout — then it drains. + let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + ok_json(stats_body(if n < 40 { &[1, 2] } else { &[] }, false)) + }); + + table + .checkpoint_lsm() + .await + .expect("a stall is the server being slow, not the client's call to make"); + assert!( + polls.load(std::sync::atomic::Ordering::SeqCst) > 40, + "the loop must have kept polling well past the old ten-poll bound" + ); + } + + /// A pass already owns the latch on every outstanding bucket, so the loop + /// waits rather than piling on triggers it would only refuse. This is the + /// sole thing `compacting` is read for. + #[tokio::test(start_paused = true)] + async fn test_checkpoint_waits_while_a_pass_is_running() { + let polls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let seen_polls = polls.clone(); + let seen_compacts = compacts.clone(); + let table = Table::new_with_handler("my_table", move |request| { + let path = request.url().path().to_string(); + if path.contains("flush_lsm") { + return accepted(); + } + if path.contains("compact_lsm") { + seen_compacts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + return accepted(); + } + // Latched for many polls, then done. + let n = seen_polls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + ok_json(if n > 15 { + stats_body(&[], false) + } else { + stats_body(&[1], true) + }) + }); + + table + .checkpoint_lsm() + .await + .expect("a running pass is progress, not a stall"); + assert_eq!( + compacts.load(std::sync::atomic::Ordering::SeqCst), + 0, + "never trigger against a bucket already compacting" + ); + } + + /// WAL off ⇒ `None`; WAL on ⇒ a fully populated `Some` with no field + /// defaulting to a zero it did not measure. `include_generation_rows` + /// rides in the body and is off unless asked for. + #[tokio::test] + async fn test_get_lsm_stats_round_trip() { + let table = Table::new_with_handler("my_table", |request| { + assert_eq!(request.url().path(), "/v1/table/my_table/get_lsm_stats/"); + let body = request.body().unwrap().as_bytes().unwrap(); + let body: serde_json::Value = serde_json::from_slice(body).unwrap(); + assert_eq!( + body["include_generation_rows"], true, + "the flag must reach the server, not be silently dropped" + ); + let response = serde_json::json!({ + "lsm_stats": { + "buckets": [{ + "shard_id": "b0", + "status": "Active", + "writer_epoch": 3, + "manifest_version": 11, + "current_generation": 9, + "replay_after_wal_entry_position": 100, + "wal_entry_position_last_seen": 140, + "generations": [{ "generation": 8, "bytes": 4096, "rows": 30 }], + "compacting": false, + "memtables": [ + { "generation": 9, "rows": 12, "bytes": 900, "batches": 2, + "indexes": ["vec_idx"] } + ], + }], + } + }); + http::Response::builder() + .status(200) + .body(response.to_string()) + .unwrap() + }); + + let stats = table + .get_lsm_stats(true) + .await + .unwrap() + .expect("a WAL-backed table reports Some"); + let bucket = &stats.buckets[0]; + assert_eq!(bucket.replay_after_wal_entry_position, 100); + assert_eq!(bucket.wal_entry_position_last_seen, 140); + assert!(!bucket.compacting); + assert_eq!(bucket.generations[0].generation, 8); + assert_eq!(bucket.generations[0].rows, Some(30)); + // The line that answers "why is my fresh-tier vector search + // brute-force" — an absent index name is the whole explanation. + let memtables = bucket.memtables.as_ref().unwrap(); + assert_eq!(memtables[0].indexes, vec!["vec_idx".to_string()]); + } + + /// A 404 arrives as `TableNotFound`, not as a lost claim the loop + /// re-issues from flush until its cap. The two are distinguished by + /// status: 404 is "no such table", 421 is "this node holds no claim". + /// They shared 404 once, and the loop chased a name that never existed. + #[tokio::test(start_paused = true)] + async fn test_missing_table_is_not_read_as_a_lost_claim() { + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let seen = calls.clone(); + let table = Table::new_with_handler("my_table", move |_request| { + seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + http::Response::builder() + .status(404) + .body(r#"{"code":4,"error":"Not found: Table not found: my_table"}"#.to_string()) + .unwrap() + }); + + let err = table.checkpoint_lsm().await.unwrap_err(); + assert!( + matches!(err, Error::TableNotFound { .. }), + "a missing table must say so: {err:?}" + ); + assert_eq!( + calls.load(std::sync::atomic::Ordering::SeqCst), + 1, + "no point re-claiming a table that does not exist" + ); + } + + /// A lost claim — 421, not 404 — does re-issue from flush, the call that + /// re-claims and replays. + #[tokio::test(start_paused = true)] + async fn test_registry_miss_reissues_from_flush() { + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let seen = calls.clone(); + let table = Table::new_with_handler("my_table", move |request| { + let path = request.url().path().to_string(); + let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if path.contains("flush_lsm") { + // First flush lands; the claim is then lost, and the + // re-issued flush succeeds. + return accepted(); + } + if path.contains("compact_lsm") { + if n < 4 { + return http::Response::builder() + .status(421) + .body(r#"{"code":19,"error":"table not claimed"}"#.to_string()) + .unwrap(); + } + return accepted(); + } + ok_json(stats_body(if n < 6 { &[1] } else { &[] }, false)) + }); + + table + .checkpoint_lsm() + .await + .expect("a lost claim must be recovered by re-flushing, not surfaced"); + } + + #[tokio::test] + async fn test_get_lsm_stats_absent_when_wal_off() { + let table = Table::new_with_handler("my_table", |_request| { + http::Response::builder() + .status(200) + .body(serde_json::json!({ "lsm_stats": null }).to_string()) + .unwrap() + }); + assert!(table.get_lsm_stats(false).await.unwrap().is_none()); + } + #[tokio::test] async fn test_wait_for_index() { let table = _make_table_with_indices(0); diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 0d8a8e8b9..74ab17921 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -68,10 +68,12 @@ use self::merge::MergeInsertBuilder; pub mod add_columns; mod add_data; pub mod branch_merge; +pub mod checkpoint; mod create_index; pub mod datafusion; pub(crate) mod dataset; pub mod delete; +pub mod lsm_stats; pub mod merge; pub mod optimize; mod primary_key; @@ -95,6 +97,7 @@ pub use lance::dataset::refs::{BranchContents, Ref, TagContents, Tags as LanceTa pub use lance::dataset::scanner::DatasetRecordBatchStream; use lance::dataset::statistics::DatasetStatisticsExt; pub use lance_index::optimize::OptimizeOptions; +pub use lsm_stats::{BucketStats, GenerationStats, LsmStats, MemtableStats}; pub use optimize::{CompactionOptions, OptimizeAction, OptimizeStats}; pub use schema_evolution::{ AddColumnsResult, AlterColumnsResult, DropColumnsResult, FieldMetadataUpdate, @@ -685,6 +688,31 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { message: "get_lsm_write_spec is not supported on this table type".into(), }) } + /// Seal every bucket's active memtable into L0. + /// + /// The default implementation returns `NotSupported`. + async fn flush_lsm(&self) -> Result<()> { + Err(Error::NotSupported { + message: "flush_lsm is not supported on this table type".into(), + }) + } + /// Trigger a background L0 → base compaction pass per bucket. + /// + /// The default implementation returns `NotSupported`. + async fn compact_lsm(&self) -> Result<()> { + Err(Error::NotSupported { + message: "compact_lsm is not supported on this table type".into(), + }) + } + /// Read live LSM state, or `None` when the LSM write path is not + /// enabled for this table. + /// + /// The default implementation returns `NotSupported`. + async fn get_lsm_stats(&self, _include_generation_rows: bool) -> Result> { + Err(Error::NotSupported { + message: "get_lsm_stats is not supported on this table type".into(), + }) + } /// Drain and close any cached MemWAL shard writers for this table. /// /// The default implementation is a no-op; table types that maintain @@ -1726,6 +1754,85 @@ impl Table { self.inner.get_lsm_write_spec().await } + /// Converge this table's LSM write path into its base table. + /// + /// One `flush` to seal every memtable into L0, then compaction triggers + /// until every generation that existed at that moment has reached base. + /// The loop runs client-side, reading progress from `get_lsm_stats`, so + /// there is no held socket and nothing to reconcile if you drop this + /// future partway through. + /// + /// **Best-effort.** Generations created *after* the opening flush are + /// deliberately not waited on — that is what lets this terminate on a + /// table taking writes. Idempotent and safe on a cadence: an + /// already-converged table costs two round trips and triggers nothing. + /// + /// **No deadline, and the caller owns that.** It returns when the target + /// generations are gone, propagates a terminal server fault, and + /// otherwise waits however long the server takes. A slow table and a + /// stuck one are the same picture from here: the compactor pool is shared + /// across every table on the node, so a checkpoint queued behind + /// unrelated work is indistinguishable from one that is merging. Wrap + /// this in `tokio::time::timeout` for a wall-clock bound; abandoning it + /// partway costs nothing. + /// + /// # Example + /// + /// ```no_run + /// # use lancedb::Table; + /// # async fn example(table: &Table) -> Result<(), Box> { + /// let before = table.get_lsm_stats(false).await?; + /// table.checkpoint_lsm().await?; + /// let after = table.get_lsm_stats(false).await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn checkpoint_lsm(&self) -> Result<()> { + checkpoint::checkpoint_lsm(self).await + } + + /// Seal every bucket's active memtable into L0 without touching the + /// base table. + /// + /// Independently useful: flushing makes memtable rows readable from L0 at + /// a lower per-query cost. On a node that has not claimed this table it + /// claims it and replays the WAL log first — reporting "nothing to flush" + /// without replaying would lie about durable data. + pub async fn flush_lsm(&self) -> Result<()> { + self.inner.flush_lsm().await + } + + /// Run one bounded L0 → base compaction pass per bucket, reporting what + /// it merged and what is left. + /// + /// One pass, not convergence: that bounds each request's cost and gives a + /// caller driving its own cadence a progress signal per round trip. + pub async fn compact_lsm(&self) -> Result<()> { + self.inner.compact_lsm().await + } + + /// Read live per-bucket LSM state. + /// + /// Answers "how far behind is my fresh tier", "which bucket is hot", and + /// "why is my fresh-tier vector search brute-force". Mutates no table + /// state, though on a node that has not claimed this table it claims it, + /// exactly as a read would. + /// + /// `include_generation_rows` reports a row count per L0 generation. Off by + /// default: each count opens an uncached Lance dataset, and + /// `checkpoint_lsm` polls this needing only generation numbers. + /// + /// `Ok(None)` only when the LSM write path is not enabled, matching + /// [`Table::get_lsm_write_spec`]. Stats is fresh-tier only, so with the + /// WAL off there is no manifest to report and a struct of zeros would + /// read as measurements. + /// + /// Do not build a checkpoint's termination on this: the completion + /// predicate lives in the `flush` and `compact` responses. + pub async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result> { + self.inner.get_lsm_stats(include_generation_rows).await + } + /// Drain and close any cached MemWAL shard writers held for this table. /// /// When an [`LsmWriteSpec`] is installed, `merge_insert` opens MemWAL shard diff --git a/rust/lancedb/src/table/checkpoint.rs b/rust/lancedb/src/table/checkpoint.rs new file mode 100644 index 000000000..bb76604ed --- /dev/null +++ b/rust/lancedb/src/table/checkpoint.rs @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Converging a table's LSM write path into its base table. +//! +//! `checkpoint_lsm` seals once, then triggers compaction and watches +//! generation numbers until the L0 that existed at the start is gone. +//! +//! The loop runs in the client, not the server: `compact_lsm` dispatches a +//! pass and returns, so nothing holds a socket and a client can vanish +//! mid-operation with nothing to reconcile. Completion is read from +//! generation numbers in the shard manifest — durable state, unlike a count +//! in a compact response, which a concurrent write invalidates. +//! +//! The target set is fixed at the start, so generations created *during* the +//! checkpoint are ignored. That is what lets it terminate under write load, +//! and what makes it best-effort: it converges the fresh tier as of some +//! instant. Idempotent, abandonable at any point, safe on a cadence. +//! +//! No liveness bound — the caller owns the deadline. The compactor pool is +//! shared pod-wide, so a checkpoint queued behind unrelated tables looks +//! exactly like one that is merging. + +use std::collections::HashMap; +use std::future::Future; +use std::time::Duration; + +use crate::{Error, Result, Table}; + +/// The HTTP status a failed request carried, if it carried one. +/// +/// `None` for anything with no retry story: a `TableNotFound` that +/// `check_table_response` already translated, or a connection failure that +/// never reached the server. Both are terminal. +fn status_of(e: &Error) -> Option { + #[cfg(feature = "remote")] + { + match e { + Error::Http { + status_code: Some(status), + .. + } => Some(status.as_u16()), + _ => None, + } + } + #[cfg(not(feature = "remote"))] + { + let _ = e; + None + } +} + +/// 429 (latch held, pool saturated, or the pod replaying its WAL) and 503 (a +/// draining node, or a proxy between here and it). +/// +/// The status is the whole signal: the server deliberately keeps contention +/// off 503, so a latch collision is a 429. A draining node *is* terminal, but +/// it is also a 503 that stays a 503, so retrying spends one budget and then +/// reports the server's own message — cheaper than parsing the body for the +/// namespace code it would take to tell the two apart. +fn is_retryable(e: &Error) -> bool { + matches!(status_of(e), Some(429 | 503)) +} + +/// 421: the owning node holds no claim. Only `flush` re-claims and replays, +/// so this cannot be retried in place — the caller has to start over. +fn is_lost_claim(e: &Error) -> bool { + status_of(e) == Some(421) +} + +/// Interval between `get_lsm_stats` polls. One interval is roughly one +/// compaction pass, the granularity at which the answer can change. +/// +/// Fixed rather than configurable, matching `wait_for_index`. It costs +/// nothing on an already-converged table and at most one interval of tail +/// latency after the final pass lands. +const POLL_INTERVAL: Duration = Duration::from_secs(5); + +/// Cap on re-issues from `flush` after a 421, so a crash-looping node cannot +/// turn flush → compact → 421 → flush into a spin. +/// +/// Deliberately not shared with [`MAX_RETRIES`]: a claim that keeps +/// evaporating is a broken node, while contention is routine and wants a real +/// budget. One shared counter let a merely contended table exhaust this cap +/// and then blame a claim it never lost. +const MAX_REISSUES: usize = 3; + +/// Retryable faults tolerated on a *single* request, reset on every success — +/// scattered contention across a long checkpoint must not accumulate toward a +/// cap. Roughly 16s of retrying against the backoff below. +const MAX_RETRIES: usize = 8; + +/// Backoff between retries, doubling up to [`RETRY_BACKOFF_MAX`]. Latch +/// contention clears in about the time one pass takes, so start small; a +/// saturated pool wants the ceiling. +const RETRY_BACKOFF_BASE: Duration = Duration::from_millis(100); +const RETRY_BACKOFF_MAX: Duration = Duration::from_secs(5); + +/// Sleep before re-issuing a retryable request. +async fn backoff(attempt: usize) { + let delay = RETRY_BACKOFF_BASE + .saturating_mul(1u32 << attempt.min(8) as u32) + .min(RETRY_BACKOFF_MAX); + tokio::time::sleep(delay).await; +} + +/// Whether the drain loop finished or needs the table re-claimed first. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CheckpointOutcome { + Done, + ReissueFromFlush, +} + +/// What one LSM request produced: its value, or word that the owning node +/// holds no claim and only `flush` can get it back. +enum Attempt { + Ok(T), + ReissueFromFlush, +} + +/// Issue one LSM request, retrying in place while the fault is retryable. +/// +/// The two recoverable faults have separate budgets: contention clears on its +/// own and retries here against [`MAX_RETRIES`], while a 421 needs `flush` to +/// re-claim, which only the caller can drive. +/// +/// An exhausted budget propagates the last error *as itself* rather than a +/// synthesized one — "429 after nine tries" beats "checkpoint failed", and a +/// draining node arrives carrying the server's own message. +async fn issue(mut call: F) -> Result> +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut retries = 0; + loop { + let e = match call().await { + Ok(value) => return Ok(Attempt::Ok(value)), + Err(e) => e, + }; + if is_lost_claim(&e) { + return Ok(Attempt::ReissueFromFlush); + } + if !is_retryable(&e) || retries >= MAX_RETRIES { + return Err(e); + } + backoff(retries).await; + retries += 1; + } +} + +/// Drive [`Table::checkpoint_lsm`]: seal once, fix the target watermark +/// from the resulting L0, then trigger and poll until it drains. +pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> { + for reissue in 0..=MAX_REISSUES { + // The seal turns everything written before this call into a + // generation, so the watermark has to be read after it. Idempotent: + // sealing an empty memtable is a no-op, so a re-issue does not churn + // empty generations. + match issue(|| table.flush_lsm()).await? { + Attempt::Ok(()) => {} + Attempt::ReissueFromFlush => { + backoff(reissue).await; + continue; + } + } + + let stats = match issue(|| table.get_lsm_stats(false)).await? { + Attempt::Ok(stats) => stats, + Attempt::ReissueFromFlush => { + backoff(reissue).await; + continue; + } + }; + let Some(stats) = stats else { + // Not WAL-backed; `flush_lsm` would have errored first but for a race. + return Ok(()); + }; + let targets: HashMap = stats + .buckets + .iter() + .filter_map(|b| Some((b.shard_id.clone(), b.newest_generation()?))) + .collect(); + if targets.is_empty() { + return Ok(()); + } + + match drain_to_targets(table, &targets).await? { + CheckpointOutcome::Done => return Ok(()), + CheckpointOutcome::ReissueFromFlush => { + backoff(reissue).await; + continue; + } + } + } + Err(Error::Runtime { + message: "checkpoint_lsm: the owning node kept losing its claim; \ + re-issued from flush the maximum number of times" + .into(), + }) +} + +/// Trigger and poll until no bucket holds a generation at or below its +/// target. +/// +/// No liveness bound, deliberately. The pod-wide compactor pool (a semaphore +/// of 2 by default, shared across every table on the node) is taken *inside* +/// the pass, after the bucket latch, so a checkpoint queued behind unrelated +/// tables is indistinguishable from one that is merging. An idle-poll counter +/// here could only ever have fired on a table that would have finished. +async fn drain_to_targets( + table: &Table, + targets: &HashMap, +) -> Result { + loop { + let stats = match issue(|| table.get_lsm_stats(false)).await? { + Attempt::Ok(stats) => stats, + Attempt::ReissueFromFlush => return Ok(CheckpointOutcome::ReissueFromFlush), + }; + let Some(stats) = stats else { + return Ok(CheckpointOutcome::Done); + }; + // `compacting` is the bucket's compaction latch, held from dispatch + // until the pass ends — including while it waits on the pod-wide + // permit. So it answers one question only: do not pile on. Buckets + // with nothing outstanding are skipped, not counted as idle. + let mut outstanding = 0; + let mut all_compacting = true; + for b in &stats.buckets { + let Some(target) = targets.get(&b.shard_id) else { + continue; + }; + let n = b.outstanding_generations(*target); + if n > 0 { + outstanding += n; + all_compacting &= b.compacting; + } + } + if outstanding == 0 { + return Ok(CheckpointOutcome::Done); + } + + if !all_compacting { + match table.compact_lsm().await { + Ok(()) => {} + Err(e) if is_lost_claim(&e) => return Ok(CheckpointOutcome::ReissueFromFlush), + Err(e) if !is_retryable(&e) => return Err(e), + // A 429 here means the server could latch no bucket at all, + // which the poll above already handles. Not retried in place: + // the latch it would contend for is the one doing the work, so + // fall through and re-read — `POLL_INTERVAL` is the backoff. + Err(_) => {} + } + } + tokio::time::sleep(POLL_INTERVAL).await; + } +} + +#[cfg(all(test, feature = "remote"))] +mod tests { + use super::*; + + fn http(status: u16) -> Error { + Error::Http { + source: "server said no".into(), + request_id: "rid".into(), + status_code: reqwest::StatusCode::from_u16(status).ok(), + } + } + + /// Every status the loop acts on. The two predicates are checked together + /// because their overlap is what would be wrong: a status must never be + /// both, and 421 in particular must not read as retryable — retrying it in + /// place re-issues the call that just said the node holds no claim. + #[test] + fn taxonomy_round_trips() { + for status in [429, 503] { + assert!(is_retryable(&http(status)), "{status} must retry"); + assert!( + !is_lost_claim(&http(status)), + "{status} is not a lost claim" + ); + } + assert!(is_lost_claim(&http(421)), "a lost claim must re-claim"); + assert!( + !is_retryable(&http(421)), + "retrying a lost claim in place only asks the same node again" + ); + for status in [400, 404, 409, 500] { + assert!(!is_retryable(&http(status)), "{status} is terminal"); + assert!(!is_lost_claim(&http(status)), "{status} is terminal"); + } + } + + /// An error carrying no status has no retry story and must be terminal — + /// a connection that never reached the server, or a `TableNotFound` that + /// `check_table_response` translated before the loop saw it. + #[test] + fn errors_without_a_status_are_terminal() { + let no_status = Error::Http { + source: "connection reset".into(), + request_id: "rid".into(), + status_code: None, + }; + assert!(!is_retryable(&no_status)); + assert!(!is_lost_claim(&no_status)); + + let translated = Error::TableNotFound { + name: "t".into(), + source: "gone".into(), + }; + assert!(!is_retryable(&translated)); + assert!(!is_lost_claim(&translated)); + } +} diff --git a/rust/lancedb/src/table/lsm_stats.rs b/rust/lancedb/src/table/lsm_stats.rs new file mode 100644 index 000000000..953aea90f --- /dev/null +++ b/rust/lancedb/src/table/lsm_stats.rs @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Live per-bucket LSM state — the shape [`crate::Table::get_lsm_stats`] +//! returns and [`super::checkpoint`] polls. +//! +//! Nothing here is derived: sums and differences (total L0 bytes, WAL lag) +//! are the caller's to compute. There is no "WAL is off" shape — that case is +//! `None`, because a struct of zeros would read as measurements. + +use serde::Deserialize; + +/// One flushed L0 generation. +#[derive(Debug, Clone, Deserialize)] +pub struct GenerationStats { + pub generation: u64, + pub bytes: u64, + /// Present only when `include_generation_rows` was requested. Off by + /// default because each count opens an uncached Lance dataset, and the + /// checkpoint loop polls this route needing only generation numbers. + #[serde(default)] + pub rows: Option, +} + +/// One in-memory memtable. +#[derive(Debug, Clone, Deserialize)] +pub struct MemtableStats { + pub generation: u64, + pub rows: u64, + pub bytes: u64, + pub batches: u64, + /// Names of the indexes this memtable carries. An absent name is the whole + /// answer to "why is my fresh-tier search on that column brute-force". + pub indexes: Vec, +} + +/// Live state of one bucket. A table is N buckets on one node; flattening to +/// a single number hides the one hot bucket that is usually why someone +/// opened this endpoint. +#[derive(Debug, Clone, Deserialize)] +pub struct BucketStats { + pub shard_id: String, + /// `Active` | `Sealed` (drop-table 2PC in flight). + pub status: String, + pub writer_epoch: u64, + pub manifest_version: u64, + pub current_generation: u64, + pub replay_after_wal_entry_position: u64, + pub wal_entry_position_last_seen: u64, + pub generations: Vec, + /// Whether a pass owns this bucket's compaction latch right now. Says *a* + /// driver is running, not *whose*, and the latch is held from dispatch — + /// including while the pass queues for a pod-wide compactor permit. Read + /// it as "do not pile on", never as "mine is progressing". + pub compacting: bool, + /// Oldest first, active last. Absent for a `Sealed` bucket, whose + /// in-memory state is torn down. + #[serde(default)] + pub memtables: Option>, +} + +impl BucketStats { + /// The newest flushed generation, or `None` when L0 is empty. + pub(crate) fn newest_generation(&self) -> Option { + self.generations.iter().map(|g| g.generation).max() + } + + /// How many generations at or below `target` are still in L0. + /// + /// A count, not a boolean: one pass drains a bounded prefix rather than + /// the whole target set, so a boolean would read as "no progress" for + /// every pass but the last. Compaction drains oldest-first, so this + /// decreases monotonically. + pub(crate) fn outstanding_generations(&self, target: u64) -> usize { + self.generations + .iter() + .filter(|g| g.generation <= target) + .count() + } +} + +/// Live LSM state, one entry per bucket. +#[derive(Debug, Clone, Deserialize)] +pub struct LsmStats { + pub buckets: Vec, +} + +/// Server-side JSON envelope for `get_lsm_stats`. `lsm_stats` is null when +/// the table has no LSM write path. +#[derive(Debug, Deserialize)] +pub(crate) struct GetLsmStatsResponse { + #[serde(default)] + pub lsm_stats: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn bucket(shard: &str, generations: &[u64], compacting: bool) -> BucketStats { + BucketStats { + shard_id: shard.into(), + status: "Active".into(), + writer_epoch: 1, + manifest_version: 1, + current_generation: generations.iter().max().copied().unwrap_or(0) + 1, + replay_after_wal_entry_position: 0, + wal_entry_position_last_seen: 0, + generations: generations + .iter() + .map(|g| GenerationStats { + generation: *g, + bytes: 1, + rows: None, + }) + .collect(), + compacting, + memtables: None, + } + } + + /// The target watermark is the newest generation at the start, and a + /// generation created after it must not hold the loop open — that is why + /// the predicate terminates under write load. + #[test] + fn newer_generations_do_not_extend_the_target() { + let start = bucket("b0", &[7, 8], false); + let target = start.newest_generation().expect("L0 is non-empty"); + assert_eq!(target, 8); + + // Compaction drained 7 and 8; 9 and 10 arrived while it ran. + let later = bucket("b0", &[9, 10], false); + assert_eq!( + later.outstanding_generations(target), + 0, + "generations above the target are somebody else's problem" + ); + + // Still holding 8 means still outstanding. + assert_eq!( + bucket("b0", &[8, 9], false).outstanding_generations(target), + 1 + ); + } + + /// The metric counts generations, not buckets: a pass drains a bounded + /// prefix, so one bucket going 3 → 2 → 1 → 0 is three steps. + #[test] + fn progress_is_measured_in_generations() { + let target = 3; + let counts: Vec = [&[1u64, 2, 3][..], &[2, 3][..], &[3][..], &[][..]] + .iter() + .map(|gens| bucket("b0", gens, false).outstanding_generations(target)) + .collect(); + assert_eq!(counts, vec![3, 2, 1, 0]); + } + + #[test] + fn empty_l0_has_no_target() { + assert!(bucket("b0", &[], false).newest_generation().is_none()); + } +} From be290447d9fa156f17d3c1eea028bb532645f2f0 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Fri, 7 Aug 2026 11:54:32 -0700 Subject: [PATCH 44/49] chore: update lance dependency to v11.0.0-beta.3 (#3896) Updates the Rust workspace Lance dependencies and Java lance-core dependency to v11.0.0-beta.3. No compatibility fixes were required; all-features clippy and Rust formatting pass. Triggering tag: https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.3 --- Cargo.lock | 86 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 28 ++++++++--------- java/pom.xml | 2 +- 3 files changed, 58 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 93e16c06d..f6186672c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arc-swap", "arrow", @@ -4890,8 +4890,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow-array", "arrow-buffer", @@ -4913,7 +4913,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow-array", "arrow-buffer", @@ -4927,7 +4927,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow-array", "arrow-schema", @@ -4936,8 +4936,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrayref", "crunchy", @@ -4947,8 +4947,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow-array", "arrow-buffer", @@ -4988,8 +4988,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow", "arrow-array", @@ -5019,8 +5019,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow", "arrow-array", @@ -5037,8 +5037,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "proc-macro2", "quote", @@ -5047,8 +5047,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow-arith", "arrow-array", @@ -5082,8 +5082,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow-arith", "arrow-array", @@ -5114,8 +5114,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arc-swap", "arrow", @@ -5182,8 +5182,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow-array", "arrow-schema", @@ -5205,8 +5205,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow", "arrow-array", @@ -5242,8 +5242,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow-array", "arrow-buffer", @@ -5259,8 +5259,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow", "async-trait", @@ -5272,8 +5272,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow", "arrow-ipc", @@ -5303,7 +5303,6 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", - "time", "tokio", "tower", "tower-http 0.5.2", @@ -5327,8 +5326,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow-array", "arrow-buffer", @@ -5343,8 +5342,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow", "arrow-array", @@ -5354,6 +5353,7 @@ dependencies = [ "async-trait", "aws-credential-types", "aws-sdk-dynamodb", + "blake3", "byteorder", "bytes", "chrono", @@ -5383,8 +5383,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "arrow-array", "arrow-schema", @@ -5397,8 +5397,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "11.0.0-beta.2" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" +version = "11.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 78474cd16..936660d78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=11.0.0-beta.3", default-features = false, "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=11.0.0-beta.3", default-features = false, "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=11.0.0-beta.3", default-features = false, "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } ahash = "0.8" # Note that this one does not include pyarrow arrow = { version = "58.0.0", optional = false } diff --git a/java/pom.xml b/java/pom.xml index e8f030b27..e1c39c3e2 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 11.0.0-beta.2 + 11.0.0-beta.3 false 2.30.0 1.7 From 706a9c327fb324e7961620cf1e159fe8c15f7fab Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Fri, 7 Aug 2026 14:50:22 -0500 Subject: [PATCH 45/49] feat: infer maintained indexes when an LsmWriteSpec omits them (#3748) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What `LsmWriteSpec::maintained_indexes` becomes `Option>`: | value | meaning | |---|---| | `None` (new default) | every index the MemWAL supports, resolved when the spec is installed | | `Some([])` | maintain nothing — a scan/filter-only WAL table | | `Some([..])` | exactly these, taken verbatim | `with_maintained_indexes` keeps its signature; `with_no_maintained_indexes()` is new. Surfaced through the remote path (null on the wire), Python, and Node. ## Why Callers had to state the maintained set by hand every time, which is both tedious and easy to get wrong — the common case is "maintain what I already built." Resolution filters on `IndexConfig::is_memwal_maintainable`, delegating to lance's `is_maintainable_index_type`. This is load-bearing rather than cosmetic: lance does **not** skip an index type its memtable cannot build, it errors when the shard writer opens, so sweeping up a bitmap index would fail every memtable claim and leave the table unwritable. The inferred set excludes those, and an explicit list naming one is now rejected at spec time instead of at claim time. ## Behavior change A freshly constructed spec used to maintain **nothing**; it now maintains **everything supported**. This flipped because napi collapses `undefined` and `null` to `None`, so TypeScript cannot express "absent means nothing, null means all" — any other choice makes the bindings disagree with the wire. The error direction also favors it: an unwanted maintained index costs memory, while a silently unmaintained one degrades FTS to an unscored scan. Three existing tests encoded the old default and are updated rather than worked around. ## Caveat The resolved set is a snapshot, not a subscription. An index created after the spec is installed is not maintained until the spec is unset and set again. `get_lsm_write_spec` therefore always reports a concrete list — `None` never round-trips. ## Dependency Needs a lance release carrying `is_maintainable_index_type` (lance-format/lance#8095) before this builds against the pinned tag. Draft until then. ## Testing 38 Rust LSM tests and 10 Python tests pass against a local lance build, including new coverage that a bitmap index is excluded from inference and rejected when named, and that `[]` stays distinguishable from null on the wire. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 (1M context) --- docs/src/js/classes/Table.md | 12 +- docs/src/js/interfaces/LsmWriteSpec.md | 4 +- nodejs/lancedb/table.ts | 18 +- nodejs/src/table.rs | 12 +- python/python/lancedb/_lancedb.pyi | 11 +- python/python/lancedb/table.py | 13 +- python/python/tests/test_lsm_write_spec.py | 15 +- python/python/tests/test_merge_insert_lsm.py | 4 +- python/src/table.rs | 46 +++-- rust/lancedb/src/remote/table.rs | 30 ++- rust/lancedb/src/table.rs | 202 +++++++++++++++---- rust/lancedb/src/table/merge.rs | 4 +- rust/lancedb/src/table/merge/lsm.rs | 82 +++++++- 13 files changed, 360 insertions(+), 93 deletions(-) diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 11fca32d0..3fa3b08db 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -431,9 +431,10 @@ Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on th Resolves to `undefined` when the MemWAL LSM write path is not enabled (no spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)). -The returned spec — including its `maintainedIndexes` and -`writerConfigDefaults` — mirrors what was passed to -[Table#setLsmWriteSpec](Table.md#setlsmwritespec). +The returned spec mirrors what was passed to +[Table#setLsmWriteSpec](Table.md#setlsmwritespec), except that `maintainedIndexes` always +reports the concrete list resolved when the spec was set — `undefined` +never round-trips. #### Returns @@ -806,6 +807,11 @@ All variants require the table to have an unenforced primary key ([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally requires it to be the single column being bucketed. +Omitting `maintainedIndexes` maintains every index on the table, resolved +here, failing if one cannot be maintained — name them to install anyway. +Naming them pins an exact set, and a still-building index is rejected +rather than quietly omitted. + #### Parameters * **spec**: [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md) diff --git a/docs/src/js/interfaces/LsmWriteSpec.md b/docs/src/js/interfaces/LsmWriteSpec.md index 8a588df6a..f2ae91186 100644 --- a/docs/src/js/interfaces/LsmWriteSpec.md +++ b/docs/src/js/interfaces/LsmWriteSpec.md @@ -34,7 +34,9 @@ Bucket and identity variants: the sharding column. optional maintainedIndexes: string[]; ``` -Names of indexes the MemWAL should keep up to date during writes. +Indexes the MemWAL keeps up to date. Omit to maintain every supported +index, resolved on install — a snapshot, so indexes created later are not +maintained. Pass `[]` for none. *** diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 3359a2643..04705475b 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -197,7 +197,11 @@ export interface LsmWriteSpec { column?: string; /** Bucket variant: the number of buckets, in `[1, 1024]`. */ numBuckets?: number; - /** Names of indexes the MemWAL should keep up to date during writes. */ + /** + * Indexes the MemWAL keeps up to date. Omit to maintain every supported + * index, resolved on install — a snapshot, so indexes created later are not + * maintained. Pass `[]` for none. + */ maintainedIndexes?: string[]; /** Default `ShardWriter` configuration recorded in the MemWAL index. */ writerConfigDefaults?: Record; @@ -595,6 +599,11 @@ export abstract class Table { * All variants require the table to have an unenforced primary key * ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally * requires it to be the single column being bucketed. + * + * Omitting `maintainedIndexes` maintains every index on the table, resolved + * here, failing if one cannot be maintained — name them to install anyway. + * Naming them pins an exact set, and a still-building index is rejected + * rather than quietly omitted. * @param {LsmWriteSpec} spec The sharding spec to install. * @returns {Promise} * @example @@ -622,9 +631,10 @@ export abstract class Table { * * Resolves to `undefined` when the MemWAL LSM write path is not enabled (no * spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}). - * The returned spec — including its `maintainedIndexes` and - * `writerConfigDefaults` — mirrors what was passed to - * {@link Table#setLsmWriteSpec}. + * The returned spec mirrors what was passed to + * {@link Table#setLsmWriteSpec}, except that `maintainedIndexes` always + * reports the concrete list resolved when the spec was set — `undefined` + * never round-trips. * @returns {Promise} */ abstract getLsmWriteSpec(): Promise; diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 2ac2fecb2..d26a44845 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -772,7 +772,8 @@ pub struct LsmWriteSpec { pub column: Option, /// Bucket variant: the number of buckets, in `[1, 1024]`. pub num_buckets: Option, - /// Names of indexes the MemWAL should keep up to date during writes. + /// Indexes the MemWAL keeps up to date. Omitted resolves every + /// maintainable index on install; an empty array means none. pub maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. pub writer_config_defaults: Option>, @@ -782,7 +783,6 @@ impl TryFrom for lancedb::table::LsmWriteSpec { type Error = napi::Error; fn try_from(value: LsmWriteSpec) -> napi::Result { - let maintained = value.maintained_indexes.unwrap_or_default(); let writer_config_defaults = value.writer_config_defaults.unwrap_or_default(); let spec = match value.spec_type.as_str() { "bucket" => { @@ -809,7 +809,7 @@ impl TryFrom for lancedb::table::LsmWriteSpec { } }; Ok(spec - .with_maintained_indexes(maintained) + .with_maintained_indexes(value.maintained_indexes) .with_writer_config_defaults(writer_config_defaults)) } } @@ -827,7 +827,7 @@ impl From for LsmWriteSpec { spec_type: "bucket".to_string(), column: Some(column), num_buckets: Some(num_buckets), - maintained_indexes: Some(maintained_indexes), + maintained_indexes, writer_config_defaults: Some(writer_config_defaults), }, Native::Identity { @@ -838,7 +838,7 @@ impl From for LsmWriteSpec { spec_type: "identity".to_string(), column: Some(column), num_buckets: None, - maintained_indexes: Some(maintained_indexes), + maintained_indexes, writer_config_defaults: Some(writer_config_defaults), }, Native::Unsharded { @@ -848,7 +848,7 @@ impl From for LsmWriteSpec { spec_type: "unsharded".to_string(), column: None, num_buckets: None, - maintained_indexes: Some(maintained_indexes), + maintained_indexes, writer_config_defaults: Some(writer_config_defaults), }, } diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index fad2744d3..f87fd3d13 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -653,9 +653,10 @@ class LsmWriteSpec: def identity(column: str) -> "LsmWriteSpec": ... @staticmethod def unsharded() -> "LsmWriteSpec": ... - def with_maintained_indexes(self, indexes: List[str]) -> "LsmWriteSpec": - """Return a copy of this spec asking the MemWAL to keep the named - indexes up to date as rows are appended.""" + def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "LsmWriteSpec": + """Set which indexes the MemWAL keeps up to date. None resolves every + index on the table at install, failing if one cannot be maintained; + a list is verbatim, empty means none.""" ... def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "LsmWriteSpec": """Return a copy of this spec recording the given default @@ -670,7 +671,9 @@ class LsmWriteSpec: @property def num_buckets(self) -> Optional[int]: ... @property - def maintained_indexes(self) -> List[str]: ... + def maintained_indexes(self) -> Optional[List[str]]: + """Indexes the MemWAL keeps up to date, or None for every supported one.""" + ... @property def writer_config_defaults(self) -> Dict[str, str]: ... diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 0828f04dc..f0d7dc8c8 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -4676,6 +4676,13 @@ class AsyncTable: via [`set_unenforced_primary_key`]; bucket sharding additionally requires it to be the single column being bucketed. + By default the MemWAL maintains every index on the table, resolved + here — a snapshot, so an index created afterwards needs the spec unset + and set again. This fails if one cannot be maintained; name the set + with ``with_maintained_indexes`` to install anyway. That pins an exact + set (a still-building index is rejected, not omitted); ``[]`` maintains + none. + Parameters ---------- spec : LsmWriteSpec @@ -4702,9 +4709,9 @@ class AsyncTable: Returns ``None`` when the MemWAL LSM write path is not enabled (no spec has been set, or it was removed with `unset_lsm_write_spec`). - The returned spec — including its ``maintained_indexes`` and - ``writer_config_defaults`` — mirrors what was passed to - `set_lsm_write_spec`. + The returned spec mirrors what was passed to `set_lsm_write_spec`, + except that ``maintained_indexes`` always reports the concrete list + resolved when the spec was set — ``None`` never round-trips. """ return await self._inner.get_lsm_write_spec() diff --git a/python/python/tests/test_lsm_write_spec.py b/python/python/tests/test_lsm_write_spec.py index d38918f09..218793b89 100644 --- a/python/python/tests/test_lsm_write_spec.py +++ b/python/python/tests/test_lsm_write_spec.py @@ -83,7 +83,9 @@ def test_lsm_write_spec_repr(): assert s.spec_type == "bucket" assert s.column == "id" assert s.num_buckets == 4 - assert s.maintained_indexes == [] + # A fresh spec defers its maintained set to install time. + assert s.maintained_indexes is None + assert s.with_maintained_indexes([]).maintained_indexes == [] assert "bucket" in repr(s) assert "id" in repr(s) assert "4" in repr(s) @@ -169,18 +171,23 @@ def test_get_lsm_write_spec(tmp_path): table.unset_lsm_write_spec() assert table.get_lsm_write_spec() is None - # Identity round-trips (column recovered from the schema). + # Identity round-trips (column recovered from the schema). Leaving the + # maintained set to be inferred picks up the index on the table, so the + # spec reads back naming it rather than as "infer". table.set_lsm_write_spec(LsmWriteSpec.identity("id")) spec = table.get_lsm_write_spec() assert spec.spec_type == "identity" assert spec.column == "id" + assert spec.maintained_indexes == [idx_name] table.unset_lsm_write_spec() - # Unsharded round-trips (no routing column). - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + # Unsharded round-trips (no routing column). Opting out is distinct from + # the inferred default. + table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) spec = table.get_lsm_write_spec() assert spec.spec_type == "unsharded" assert spec.column is None + assert spec.maintained_indexes == [] @pytest.mark.asyncio diff --git a/python/python/tests/test_merge_insert_lsm.py b/python/python/tests/test_merge_insert_lsm.py index 5674a05ab..e74c21589 100644 --- a/python/python/tests/test_merge_insert_lsm.py +++ b/python/python/tests/test_merge_insert_lsm.py @@ -544,7 +544,7 @@ def test_lsm_read_fts_unmaintained_index_errors(tmp_path): table.create_index("text", config=FTS()) # No maintained indexes: the active memtable FTS arm cannot serve un-compacted # docs, so the search would silently omit them — reject instead. - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) with pytest.raises(Exception, match="maintained"): table.search("fox", query_type="fts", fts_columns="text").to_arrow() @@ -631,7 +631,7 @@ def test_lsm_read_vector_unmaintained_index_errors(tmp_path): ) # Spec with NO maintained indexes: the base vector index's catch-up is untracked, # so the scanner rejects rather than risk dropping compacted-but-unindexed rows. - table.set_lsm_write_spec(LsmWriteSpec.unsharded()) + table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) with pytest.raises(Exception, match="maintained"): table.search([1.0] * VECTOR_DIM).to_arrow() diff --git a/python/src/table.rs b/python/src/table.rs index 119388708..20a93556f 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -246,12 +246,22 @@ impl From for MergeResult { } } +/// Render for `__repr__`, so the default reads as Python's `None` rather than +/// Rust's `Some([..])`. +fn fmt_maintained(maintained: &Option>) -> String { + match maintained { + Some(names) => format!("{:?}", names), + None => "None".to_string(), + } +} + /// Specification selecting Lance's MemWAL LSM-style write path for /// `merge_insert`. /// /// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()` /// classmethods, then optionally chain `with_maintained_indexes(...)` and -/// `with_writer_config_defaults(...)`. +/// `with_writer_config_defaults(...)`. A fresh spec maintains every index the +/// MemWAL supports, resolved on install. #[pyclass(from_py_object)] #[derive(Clone, Debug)] pub struct LsmWriteSpec { @@ -291,11 +301,11 @@ impl LsmWriteSpec { } } - /// Replace the list of indexes the MemWAL should keep up to date as - /// rows are appended. Each name must reference an index that - /// already exists on the table at the time `set_lsm_write_spec` - /// is called. - pub fn with_maintained_indexes(&self, indexes: Vec) -> Self { + /// Set which indexes the MemWAL maintains. `None` (the default) + /// resolves every supported index on install; a list is verbatim, + /// and an empty list maintains nothing. + #[pyo3(signature = (indexes))] + pub fn with_maintained_indexes(&self, indexes: Option>) -> Self { Self { inner: self.inner.clone().with_maintained_indexes(indexes), } @@ -317,23 +327,29 @@ impl LsmWriteSpec { maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={:?}, writer_config_defaults={:?})", - column, num_buckets, maintained_indexes, writer_config_defaults, + "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})", + column, + num_buckets, + fmt_maintained(maintained_indexes), + writer_config_defaults, ), lancedb::table::LsmWriteSpec::Identity { column, maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.identity(column={:?}, maintained_indexes={:?}, writer_config_defaults={:?})", - column, maintained_indexes, writer_config_defaults, + "LsmWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})", + column, + fmt_maintained(maintained_indexes), + writer_config_defaults, ), lancedb::table::LsmWriteSpec::Unsharded { maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.unsharded(maintained_indexes={:?}, writer_config_defaults={:?})", - maintained_indexes, writer_config_defaults, + "LsmWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})", + fmt_maintained(maintained_indexes), + writer_config_defaults, ), } } @@ -368,10 +384,10 @@ impl LsmWriteSpec { } } - /// Names of indexes the MemWAL should keep up to date during writes. + /// Indexes the MemWAL keeps up to date, or `None` for every supported one. #[getter] - pub fn maintained_indexes(&self) -> Vec { - self.inner.maintained_indexes().to_vec() + pub fn maintained_indexes(&self) -> Option> { + self.inner.maintained_indexes().map(<[String]>::to_vec) } /// Default `ShardWriter` configuration recorded by this spec. diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index f3e872cbe..0d843dd54 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2520,9 +2520,9 @@ impl BaseTable for RemoteTable { self.check_mutable().await?; // Map the spec onto the server's request DTO. `sharding` is internally - // tagged on `mode` to mirror sophon's `Sharding` enum; `maintained_indexes` - // and `writer_config_defaults` are sent verbatim (an empty list means "no - // maintained indexes", not "default to all"). + // tagged on `mode` to mirror sophon's `Sharding` enum. A null + // `maintained_indexes` asks the server to resolve every maintainable + // index at HEAD; a list is verbatim, an empty one meaning none. let sharding = match &spec { LsmWriteSpec::Bucket { column, @@ -6599,7 +6599,7 @@ mod tests { .unwrap() }); let spec = crate::table::LsmWriteSpec::unsharded() - .with_maintained_indexes(["id_idx"]) + .with_maintained_indexes(vec!["id_idx".to_string()]) .with_writer_config_defaults([("max_memtable_rows", "1000")]); table.set_lsm_write_spec(spec).await.unwrap(); } @@ -6618,7 +6618,8 @@ mod tests { body["sharding"], serde_json::json!({ "mode": "bucket", "column": "id", "num_buckets": 16 }) ); - assert_eq!(body["maintained_indexes"], serde_json::json!([])); + // An unpinned maintained set sends null: resolve server-side. + assert_eq!(body["maintained_indexes"], serde_json::Value::Null); http::Response::builder().status(200).body("{}").unwrap() }); table @@ -6627,6 +6628,23 @@ mod tests { .unwrap(); } + /// `[]` (none) must stay distinguishable on the wire from null (all). + #[tokio::test] + async fn test_set_lsm_write_spec_no_maintained_indexes() { + let table = Table::new_with_handler("my_table", |request| { + let body = request.body().unwrap().as_bytes().unwrap(); + let body: serde_json::Value = serde_json::from_slice(body).unwrap(); + assert_eq!(body["maintained_indexes"], serde_json::json!([])); + http::Response::builder().status(200).body("{}").unwrap() + }); + table + .set_lsm_write_spec( + crate::table::LsmWriteSpec::bucket("id", 16).with_maintained_indexes(Vec::new()), + ) + .await + .unwrap(); + } + #[tokio::test] async fn test_set_lsm_write_spec_identity() { let table = Table::new_with_handler("my_table", |request| { @@ -6701,7 +6719,7 @@ mod tests { } => { assert_eq!(column, "id"); assert_eq!(num_buckets, 4); - assert_eq!(maintained_indexes, vec!["id_idx".to_string()]); + assert_eq!(maintained_indexes, Some(vec!["id_idx".to_string()])); assert_eq!( writer_config_defaults .get("durable_write") diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 74ab17921..e23bb7c47 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -371,6 +371,8 @@ pub use self::merge::MergeResult; /// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default /// `ShardWriter` configuration recorded in the MemWAL index). /// +/// A fresh spec maintains every index on the table, resolved on install. +/// /// Install a spec with [`Table::set_lsm_write_spec`] and remove it with /// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch /// onto the MemWAL writer is a follow-up. @@ -385,9 +387,12 @@ pub enum LsmWriteSpec { Bucket { column: String, num_buckets: u32, - /// Names of indexes (already created on the table) that the - /// MemWAL should maintain in-memory as rows are appended. - maintained_indexes: Vec, + /// Indexes the MemWAL maintains in-memory as rows are appended. + /// + /// `None` means every index it can maintain, resolved on install — a + /// snapshot, so indexes created later need the spec unset and re-set. + /// `Some([])` maintains nothing. + maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, }, @@ -397,35 +402,41 @@ pub enum LsmWriteSpec { /// distinct value of `column` becomes its own shard. Identity { column: String, - /// Names of indexes (already created on the table) that the - /// MemWAL should maintain in-memory as rows are appended. - maintained_indexes: Vec, + /// Indexes the MemWAL maintains in-memory as rows are appended. + /// + /// `None` means every index it can maintain, resolved on install — a + /// snapshot, so indexes created later need the spec unset and re-set. + /// `Some([])` maintains nothing. + maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, }, /// No sharding — every `merge_insert` call writes to a single MemWAL shard. Unsharded { - /// Names of indexes (already created on the table) that the - /// MemWAL should maintain in-memory as rows are appended. - maintained_indexes: Vec, + /// Indexes the MemWAL maintains in-memory as rows are appended. + /// + /// `None` means every index it can maintain, resolved on install — a + /// snapshot, so indexes created later need the spec unset and re-set. + /// `Some([])` maintains nothing. + maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, }, } impl LsmWriteSpec { - /// Construct a hash-bucket sharding spec with no maintained indexes. + /// Construct a hash-bucket sharding spec maintaining every index on the table. pub fn bucket(column: impl Into, num_buckets: u32) -> Self { Self::Bucket { column: column.into(), num_buckets, - maintained_indexes: Vec::new(), + maintained_indexes: None, writer_config_defaults: HashMap::new(), } } /// Construct an identity-sharding spec (shard by the raw value of - /// `column`) with no maintained indexes. + /// `column`) maintaining every index on the table. /// /// `column` must be a deterministic function of the unenforced primary /// key: every row with a given primary key must always produce the same @@ -437,28 +448,37 @@ impl LsmWriteSpec { pub fn identity(column: impl Into) -> Self { Self::Identity { column: column.into(), - maintained_indexes: Vec::new(), + maintained_indexes: None, writer_config_defaults: HashMap::new(), } } - /// Construct an unsharded spec with no maintained indexes. + /// Construct an unsharded spec maintaining every index on the table. pub fn unsharded() -> Self { Self::Unsharded { - maintained_indexes: Vec::new(), + maintained_indexes: None, writer_config_defaults: HashMap::new(), } } - /// Replace the list of indexes the MemWAL should keep up to date as - /// rows are appended. Each name must reference an index that already - /// exists on the table at the time `set_lsm_write_spec` is called. - pub fn with_maintained_indexes(mut self, indexes: I) -> Self - where - I: IntoIterator, - S: Into, - { - let v: Vec = indexes.into_iter().map(Into::into).collect(); + /// Set which indexes the MemWAL maintains. + /// + /// `None` (the default) resolves to every index on the table at install, + /// failing if one cannot be maintained — name the set to install anyway. A + /// list is verbatim: each name must already exist and be maintainable, and + /// an empty list maintains nothing. + /// + /// ``` + /// # use lancedb::table::LsmWriteSpec; + /// // Every index the table has when the spec is installed: + /// LsmWriteSpec::unsharded().with_maintained_indexes(None); + /// // Exactly these: + /// LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]); + /// // None at all: + /// LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new()); + /// ``` + pub fn with_maintained_indexes(mut self, indexes: impl Into>>) -> Self { + let indexes = indexes.into(); match &mut self { Self::Bucket { maintained_indexes, .. @@ -468,7 +488,7 @@ impl LsmWriteSpec { } | Self::Unsharded { maintained_indexes, .. - } => *maintained_indexes = v, + } => *maintained_indexes = indexes, } self } @@ -504,8 +524,9 @@ impl LsmWriteSpec { self } - /// Borrow the list of index names this spec asks MemWAL to maintain. - pub fn maintained_indexes(&self) -> &[String] { + /// Borrow the list of index names this spec asks MemWAL to maintain, or + /// `None` when it asks for every index on the table. + pub fn maintained_indexes(&self) -> Option<&[String]> { match self { Self::Bucket { maintained_indexes, .. @@ -515,7 +536,7 @@ impl LsmWriteSpec { } | Self::Unsharded { maintained_indexes, .. - } => maintained_indexes, + } => maintained_indexes.as_deref(), } } @@ -1713,7 +1734,7 @@ impl Table { /// # async fn example(table: &Table) -> Result<(), Box> { /// table /// .set_lsm_write_spec( - /// LsmWriteSpec::bucket("id", 16).with_maintained_indexes(["id_idx"]), + /// LsmWriteSpec::bucket("id", 16).with_maintained_indexes(vec!["id_idx".to_string()]), /// ) /// .await?; /// # Ok(()) @@ -1735,9 +1756,10 @@ impl Table { /// /// Returns `Ok(None)` when the MemWAL LSM write path is not enabled (no /// spec has been set, or it was removed with [`Table::unset_lsm_write_spec`]). - /// The returned spec — including its [`LsmWriteSpec::maintained_indexes`] and - /// [`LsmWriteSpec::writer_config_defaults`] — mirrors what was passed to - /// [`Table::set_lsm_write_spec`]. + /// The returned spec mirrors what was passed to + /// [`Table::set_lsm_write_spec`], except that + /// [`LsmWriteSpec::maintained_indexes`] always reports the concrete list + /// resolved when the spec was set — `None` never round-trips. /// /// # Example /// @@ -5065,7 +5087,7 @@ mod tests { // Bucket spec round-trips exactly, including the routing column (recovered // from its field id), maintained indexes, and writer config defaults. let spec = LsmWriteSpec::bucket("id", 4) - .with_maintained_indexes([idx_name]) + .with_maintained_indexes(vec![idx_name.clone()]) .with_writer_config_defaults([("durable_write", "false")]); table.set_lsm_write_spec(spec.clone()).await.unwrap(); assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); @@ -5075,15 +5097,125 @@ mod tests { assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); // Identity sharding round-trips (column recovered from the schema). + // A spec left at its default maintains every index on the table, so it + // reads back naming the one on the table rather than as "infer". let spec = LsmWriteSpec::identity("region"); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); + assert_eq!( + table.get_lsm_write_spec().await.unwrap(), + Some(spec.with_maintained_indexes(vec![idx_name.clone()])) + ); table.unset_lsm_write_spec().await.unwrap(); // Unsharded round-trips (no routing column). let spec = LsmWriteSpec::unsharded(); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); + assert_eq!( + table.get_lsm_write_spec().await.unwrap(), + Some(spec.with_maintained_indexes(vec![idx_name])) + ); + } + + /// The maintained set defaults to every index on the table, resolved at + /// install. An index the memtable cannot build fails the install rather + /// than being dropped: maintaining it would take the table offline for + /// writes, dropping it would hide that from the caller. + #[tokio::test] + async fn test_set_lsm_write_spec_infers_maintained_indexes() { + let tmp_dir = tempdir().unwrap(); + let uri = tmp_dir.path().to_str().unwrap(); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("tag", DataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(arrow_array::Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + ], + ) + .unwrap(); + let reader: Box = + Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema.clone())); + let conn = ConnectBuilder::new(uri) + .read_consistency_interval(Duration::from_secs(0)) + .execute() + .await + .unwrap(); + let table = conn.create_table("t", reader).execute().await.unwrap(); + + table + .create_index(&["id"], Index::BTree(Default::default())) + .name("id_btree".to_string()) + .execute() + .await + .unwrap(); + table + .create_index(&["tag"], Index::Bitmap(Default::default())) + .name("tag_bitmap".to_string()) + .execute() + .await + .unwrap(); + + // Explicitly naming the bitmap index fails before anything commits. + let err = table + .set_lsm_write_spec( + LsmWriteSpec::unsharded().with_maintained_indexes(vec!["tag_bitmap".to_string()]), + ) + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { ref message } if message.contains("tag_bitmap")), + "expected the bitmap index to be rejected, got {err:?}" + ); + assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); + + // The default covers every index, so the bitmap fails it too. + let err = table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { ref message } + if message.contains("tag_bitmap") && message.contains("maintained_indexes")), + "expected the inferred set to be rejected, got {err:?}" + ); + assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); + + // Naming the maintainable subset installs. + table + .set_lsm_write_spec( + LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]), + ) + .await + .unwrap(); + assert_eq!( + table + .get_lsm_write_spec() + .await + .unwrap() + .unwrap() + .maintained_indexes(), + Some(["id_btree".to_string()].as_slice()) + ); + + // Opting out entirely is distinct from the default. + table.unset_lsm_write_spec().await.unwrap(); + table + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new())) + .await + .unwrap(); + assert_eq!( + table + .get_lsm_write_spec() + .await + .unwrap() + .unwrap() + .maintained_indexes(), + Some([].as_slice()) + ); } #[tokio::test] diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 82a1d1473..3a5b6882d 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -1161,7 +1161,7 @@ mod lsm_tests { .unwrap(); let fts_index = table.list_indices().await.unwrap()[0].name.clone(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([fts_index])) + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) .await .unwrap(); @@ -1254,7 +1254,7 @@ mod lsm_tests { .unwrap(); let vec_index = table.list_indices().await.unwrap()[0].name.clone(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([vec_index])) + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![vec_index])) .await .unwrap(); diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index 0eb7c0231..87c427b3c 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -29,6 +29,7 @@ use arrow_schema::{DataType, Schema as ArrowSchema, SchemaRef}; use lance::Dataset; use lance::dataset::mem_wal::{ DatasetMemWalExt, ShardWriter, ShardWriterConfig, evaluate_sharding_spec, + validate_maintained_indexes, }; use lance::index::DatasetIndexExt; use lance_core::datatypes::Schema as LanceSchema; @@ -37,8 +38,9 @@ use tokio::sync::RwLock; use uuid::Uuid; use crate::error::{Error, Result}; +use crate::index::IndexConfig; use crate::table::merge::{MergeInsertBuilder, MergeResult}; -use crate::table::{LsmWriteSpec, NativeTable}; +use crate::table::{BaseTable, LsmWriteSpec, NativeTable}; /// Spec id of the sole sharding spec installed by [`set_lsm_write_spec`]. /// Must match Lance's `InitializeMemWalBuilder` (`SHARDING_SPEC_ID`). @@ -80,32 +82,44 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) } } + // Before the builder borrows the dataset clone. `list_indices` merges an + // index's segments into one entry, so the result needs no dedup. + let maintained_indexes = { + let dataset = table.dataset.get().await?; + resolve_maintained_indexes( + &dataset, + &table.list_indices().await?, + spec.maintained_indexes(), + ) + .await? + }; + let mut dataset = (*table.dataset.get().await?).clone(); let mut builder = dataset.initialize_mem_wal(); - let (maintained_indexes, writer_config_defaults) = match spec { + let writer_config_defaults = match spec { LsmWriteSpec::Bucket { column, num_buckets, - maintained_indexes, writer_config_defaults, + .. } => { builder = builder.bucket_sharding(column, num_buckets); - (maintained_indexes, writer_config_defaults) + writer_config_defaults } LsmWriteSpec::Identity { column, - maintained_indexes, writer_config_defaults, + .. } => { builder = builder.identity_sharding(column); - (maintained_indexes, writer_config_defaults) + writer_config_defaults } LsmWriteSpec::Unsharded { - maintained_indexes, writer_config_defaults, + .. } => { builder = builder.unsharded(); - (maintained_indexes, writer_config_defaults) + writer_config_defaults } }; builder = builder.maintained_indexes(maintained_indexes); @@ -117,6 +131,58 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) Ok(()) } +/// Resolve a spec's maintained-index selection against `indices`, as reported +/// by [`Table::list_indices`](crate::Table::list_indices). +/// +/// `None` means every index on the table, snapshotted now. Lance validates +/// either selection against its shard-writer rules, so a spec that installs is +/// one the MemWAL can open. +/// +/// An unmaintainable index fails an inferred set rather than being dropped from +/// it — dropping would leave the caller believing it is maintained. +async fn resolve_maintained_indexes( + dataset: &Dataset, + indices: &[IndexConfig], + requested: Option<&[String]>, +) -> Result> { + let Some(requested) = requested else { + let all: Vec = indices.iter().map(|index| index.name.clone()).collect(); + validate_maintained_indexes(dataset, &all) + .await + .map_err(|source| Error::InvalidInput { + message: format!( + "cannot maintain every index on this table: {source}. Set \ + maintained_indexes explicitly to choose from {}", + index_name_list(indices), + ), + })?; + return Ok(all); + }; + for name in requested { + if !indices.iter().any(|index| &index.name == name) { + return Err(Error::InvalidInput { + message: format!( + "maintained index '{}' does not exist on this table; it has {}", + name, + index_name_list(indices), + ), + }); + } + } + validate_maintained_indexes(dataset, requested).await?; + Ok(requested.to_vec()) +} + +/// Index names for an error message. +fn index_name_list(indices: &[IndexConfig]) -> String { + if indices.is_empty() { + return "no indexes".to_string(); + } + let mut names: Vec<&str> = indices.iter().map(|index| index.name.as_str()).collect(); + names.sort_unstable(); + format!("[{}]", names.join(", ")) +} + // ============================================================================= // unset_lsm_write_spec // ============================================================================= From 5b347afd9925dfe9017feae5725a93230b123b30 Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Sat, 8 Aug 2026 05:05:45 +0800 Subject: [PATCH 46/49] fix: avoid AttributeError in JinaEmbeddings image input for str/Path (#3670) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What `JinaEmbeddings._generate_image_input_dict()` crashes with `AttributeError: 'function' object has no attribute 'urlparse'` on any image given as a URL string, local path string, or `pathlib.Path` — i.e. every documented `jina-clip-v1` image-embedding use case except raw `bytes`. ## Why ```python from urllib.parse import urlparse ... parsed = urlparse.urlparse(image) ``` `urlparse` is imported as a function, then called as if it were the `urllib.parse` module (`urlparse.urlparse(...)`). The module-level `is_valid_url()` a few lines above does it correctly (`urlparse(text)`), which is why this reads as a typo rather than intentional. Fixed to `urlparse(str(image))` — `str()` is needed because `urlparse()` only accepts `str`/`bytes` and raises a different `AttributeError` on a raw `Path`. ## Testing Added `test_jina_generate_image_input_dict_local_path`, which fails with the original `AttributeError` before the fix and passes after, covering both a `str` path and a `pathlib.Path`. Verified locally (built the Rust extension, ran red→green, then the full `test_embeddings.py` file: 15 passed / 8 skipped, no regressions) and with `ruff check`/`ruff format`. --- Disclosure: this PR was drafted with AI assistance (Claude); I reviewed, tested, and take responsibility for the change. --------- Co-authored-by: Claude Opus 4.8 --- python/python/lancedb/embeddings/jinaai.py | 7 ++++--- python/python/tests/test_embeddings.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/python/python/lancedb/embeddings/jinaai.py b/python/python/lancedb/embeddings/jinaai.py index 9656f041f..f6ab601b3 100644 --- a/python/python/lancedb/embeddings/jinaai.py +++ b/python/python/lancedb/embeddings/jinaai.py @@ -87,12 +87,13 @@ class JinaEmbeddings(EmbeddingFunction): if isinstance(image, bytes): image_dict = {"image": base64.b64encode(image).decode("utf-8")} elif isinstance(image, (str, Path)): - parsed = urlparse.urlparse(image) - # TODO handle drive letter on windows. + parsed = urlparse(str(image)) PIL_Image = attempt_import_or_raise("PIL.Image", "pillow") if parsed.scheme == "file": pil_image = PIL_Image.open(parsed.path) - elif parsed.scheme == "": + elif parsed.scheme == "" or (os.name == "nt" and len(parsed.scheme) == 1): + # A Windows drive letter parses as a one-character scheme + # ("C:\\img.png" -> scheme="c"), so treat it as a local path. pil_image = PIL_Image.open(image if os.name == "nt" else parsed.path) elif parsed.scheme.startswith("http"): pil_image = PIL_Image.open(io.BytesIO(url_retrieve(image))) diff --git a/python/python/tests/test_embeddings.py b/python/python/tests/test_embeddings.py index 678270f19..9850669eb 100644 --- a/python/python/tests/test_embeddings.py +++ b/python/python/tests/test_embeddings.py @@ -631,3 +631,23 @@ def test_url_retrieve_downloads_image(): image_bytes = url_retrieve(image_url) img = Image.open(io.BytesIO(image_bytes)) assert img.size[0] > 0 and img.size[1] > 0 + + +def test_jina_generate_image_input_dict_local_path(tmp_path): + """ + JinaEmbeddings._generate_image_input_dict must accept a local image path + (str or Path), not just bytes. Previously it crashed with + `AttributeError: 'function' object has no attribute 'urlparse'` on any + str/Path input because it called `urlparse.urlparse(image)` instead of + `urlparse(image)` (urlparse was imported as a function, not a module). + """ + Image = pytest.importorskip("PIL.Image") + from lancedb.embeddings.jinaai import JinaEmbeddings + + image_path = tmp_path / "test.png" + Image.new("RGB", (4, 4), color="red").save(image_path, format="PNG") + + for image in (str(image_path), image_path): + image_dict = JinaEmbeddings._generate_image_input_dict(image) + assert "image" in image_dict + assert isinstance(image_dict["image"], str) and len(image_dict["image"]) > 0 From 7bb501839a18f5160bf858d6a10438630a7099ea Mon Sep 17 00:00:00 2001 From: Lance Release Date: Fri, 7 Aug 2026 21:15:20 +0000 Subject: [PATCH 47/49] =?UTF-8?q?Bump=20version:=200.37.1-beta.0=20?= =?UTF-8?q?=E2=86=92=200.37.1-beta.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 601b14d3f..a015353cb 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.37.1-beta.0" +current_version = "0.37.1-beta.1" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index f6186672c..04ab7b099 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5411,7 +5411,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.37.1-beta.0" +version = "0.37.1-beta.1" dependencies = [ "ahash", "anyhow", @@ -5499,7 +5499,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.37.1-beta.0" +version = "0.37.1-beta.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -5524,7 +5524,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.37.1-beta.0" +version = "0.37.1-beta.1" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 11e901ad0..091588922 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.37.1-beta.0 + 0.37.1-beta.1 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 3df2ac178..20f69e134 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.37.1-beta.0 + 0.37.1-beta.1 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index e1c39c3e2..f85b5c906 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.37.1-beta.0 + 0.37.1-beta.1 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 10e12edc8..48e5f5295 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.37.1-beta.0" +version = "0.37.1-beta.1" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 3c93ed470..d3792f9c2 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.37.1-beta.0", + "version": "0.37.1-beta.1", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 5ade5aaa3..44bc309ca 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.37.1-beta.0", + "version": "0.37.1-beta.1", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 16bb0edd0..e78f0fe6a 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.37.1-beta.0", + "version": "0.37.1-beta.1", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 6ee11e4bc..0e27c5f51 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.37.1-beta.0", + "version": "0.37.1-beta.1", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index c2e15bb9f..7bd27ba18 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.37.1-beta.0", + "version": "0.37.1-beta.1", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index d2820b1a1..5c76024b2 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.37.1-beta.0", + "version": "0.37.1-beta.1", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 601b51380..f8cc7d8e0 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.37.1-beta.0", + "version": "0.37.1-beta.1", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index bdbd3cf79..f7b6670e4 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.37.1-beta.0", + "version": "0.37.1-beta.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.37.1-beta.0", + "version": "0.37.1-beta.1", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index 671f3f94d..0416ce81b 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.37.1-beta.0", + "version": "0.37.1-beta.1", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 5a196e27c..9d36edd5c 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.37.1-beta.0" +version = "0.37.1-beta.1" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index c2137d0b5..05251f59a 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.37.1-beta.0" +version = "0.37.1-beta.1" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 77a93fee76f091445450513dfa32252e3bda00fa Mon Sep 17 00:00:00 2001 From: Dan Tasse <105866+dantasse@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:41:41 -0400 Subject: [PATCH 48/49] fix: get table size from metadata, not files (#3790) Some issues: - file_size_bytes is optional in the manifest, so if it's not there (old writer I guess) it'll under-report the table size. - it changes results a little bit from the old way by including per-file footers and metadata (probably not a big difference at real scale) --------- Co-authored-by: Will Jones --- Cargo.lock | 1 + docs/src/js/interfaces/TableStatistics.md | 5 +- nodejs/__test__/table.test.ts | 10 +- nodejs/src/table.rs | 5 +- python/python/lancedb/table.py | 4 +- python/python/tests/test_table.py | 10 +- rust/lancedb/Cargo.toml | 1 + rust/lancedb/src/table.rs | 228 +++++++++++++++++++++- 8 files changed, 253 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 04ab7b099..20f20a0ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5480,6 +5480,7 @@ dependencies = [ "random_word", "regex", "reqwest 0.12.28", + "roaring", "rstest", "semver", "serde", diff --git a/docs/src/js/interfaces/TableStatistics.md b/docs/src/js/interfaces/TableStatistics.md index e19cba119..e2e8ef34d 100644 --- a/docs/src/js/interfaces/TableStatistics.md +++ b/docs/src/js/interfaces/TableStatistics.md @@ -44,4 +44,7 @@ The number of rows in the table totalBytes: number; ``` -The total number of bytes in the table +The total size, in bytes, of the table's data files, index files, and +overlay files + +Read from the manifest, so this excludes deletion files and manifests. diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 15d6e0804..d263d9cab 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -277,8 +277,16 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( }, numIndices: 0, numRows: 3, - totalBytes: 44, + // Full on-disk size of the two data files, footers and metadata included. + totalBytes: 684, }); + + // Index files count toward totalBytes too (only deletion files and + // manifests are excluded). + await table.createIndex("id", { config: Index.btree() }); + const statsWithIndex = await table.stats(); + expect(statsWithIndex.numIndices).toBe(1); + expect(statsWithIndex.totalBytes).toBeGreaterThan(684); }); it("should overwrite data if asked", async () => { diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index d26a44845..c4ece20e2 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -1043,7 +1043,10 @@ impl From for IndexStatistics { #[napi(object)] pub struct TableStatistics { - /// The total number of bytes in the table + /// The total size, in bytes, of the table's data files, index files, and + /// overlay files + /// + /// Read from the manifest, so this excludes deletion files and manifests. pub total_bytes: i64, /// The number of rows in the table diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index f0d7dc8c8..c566fc532 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -6341,7 +6341,9 @@ class TableStatistics: Attributes ---------- total_bytes: int - The total number of bytes in the table. + The total size, in bytes, of the table's data files, index files, and + overlay files. Read from the manifest, so this excludes deletion files + and manifests. num_rows: int The total number of rows in the table. num_indices: int diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index eb6eaefaa..2a069c712 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -3713,7 +3713,8 @@ def test_stats(mem_db: DBConnection): stats = table.stats() print(f"{stats=}") assert stats == { - "total_bytes": 60, + # Full on-disk size of the data file, footer and metadata included. + "total_bytes": 633, "num_rows": 2, "num_indices": 0, "fragment_stats": { @@ -3731,6 +3732,13 @@ def test_stats(mem_db: DBConnection): }, } + # Index files count toward total_bytes too (only deletion files and + # manifests are excluded). + table.create_index("id", config=BTree()) + stats_with_index = table.stats() + assert stats_with_index["num_indices"] == 1 + assert stats_with_index["total_bytes"] > stats["total_bytes"] + def test_create_table_empty_list_with_schema(mem_db: DBConnection): """Test creating table with empty list data and schema diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 05251f59a..2dbd9d895 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -100,6 +100,7 @@ anyhow = "1" lance-testing = { workspace = true } tempfile = "3.5.0" random_word = { version = "0.4.3", features = ["en"] } +roaring = "0.11.4" tokio = { version = "1.23", features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "test-util"] } uuid = { version = "1.7.0", features = ["v4"] } walkdir = "2" diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index e23bb7c47..5120c48b7 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -95,7 +95,6 @@ pub use delete::DeleteResult; use futures::future::join_all; pub use lance::dataset::refs::{BranchContents, Ref, TagContents, Tags as LanceTags}; pub use lance::dataset::scanner::DatasetRecordBatchStream; -use lance::dataset::statistics::DatasetStatisticsExt; pub use lance_index::optimize::OptimizeOptions; pub use lsm_stats::{BucketStats, GenerationStats, LsmStats, MemtableStats}; pub use optimize::{CompactionOptions, OptimizeAction, OptimizeStats}; @@ -3570,9 +3569,24 @@ impl BaseTable for NativeTable { let num_rows = self.count_rows(None).await?; let num_indices = self.list_indices().await?.len(); let ds = self.dataset.get().await?; - let ds_clone = (*ds).clone(); - let ds_stats = Arc::new(ds_clone).calculate_data_stats().await?; - let total_bytes = ds_stats.fields.iter().map(|f| f.bytes_on_disk).sum::() as usize; + // Sizes come from the manifest. Summing per-field `bytes_on_disk` instead + // would open every data file to read its column metadata, which costs one + // IO per fragment and reports 0 for legacy v1 storage. + // + // The manifest summary covers only the fragments' base data files, so + // overlay files (recorded on each fragment) and index files (recorded in + // the manifest's index section) are added separately. + let mut total_bytes = ds.manifest().summary().total_files_size as usize; + for frag in ds.manifest().fragments.iter() { + for overlay in &frag.overlays { + if let Some(size) = overlay.data_file.file_size_bytes.get() { + total_bytes += size.get() as usize; + } + } + } + for index in ds.load_indices().await?.iter() { + total_bytes += index.total_size_bytes().unwrap_or(0) as usize; + } let frags = ds.get_fragments(); let mut sorted_sizes = join_all( @@ -3644,7 +3658,12 @@ impl BaseTable for NativeTable { #[skip_serializing_none] #[derive(Debug, Deserialize, PartialEq)] pub struct TableStatistics { - /// The total number of bytes in the table + /// The total size, in bytes, of the table's data files, index files, and + /// overlay files + /// + /// Read from the manifest, so this excludes deletion files and manifests, + /// and it excludes any file whose size the manifest does not record + /// (tables and indices written before writers persisted file sizes). pub total_bytes: usize, /// The number of rows in the table @@ -3705,6 +3724,7 @@ mod tests { use super::*; use crate::connect; use crate::connection::ConnectBuilder; + use crate::io::object_store::io_tracking::IoTrackingStore; use crate::query::Select; use crate::query::{ExecutableQuery, QueryBase}; use crate::test_utils::connection::new_test_connection; @@ -5263,12 +5283,16 @@ mod tests { let res = table.stats().await.unwrap(); println!("{:#?}", res); + // `total_bytes` is the full on-disk size of the 11 data files (this table + // has no index or overlay files), so it is well above the 2000 bytes of + // column data these 250 int32 pairs hold: each file carries its own footer + // and metadata. assert_eq!( res, TableStatistics { num_rows: 250, num_indices: 0, - total_bytes: 2300, + total_bytes: 8925, fragment_stats: FragmentStatistics { num_fragments: 11, num_small_fragments: 11, @@ -5308,4 +5332,196 @@ mod tests { } ) } + + /// `total_bytes` counts more than the base data files: index files and + /// overlay files recorded in the manifest are included too. + #[tokio::test] + pub async fn test_stats_includes_index_and_overlay_files() { + use lance::dataset::WriteDestination; + use lance::dataset::transaction::{DataOverlayGroup, Operation}; + use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; + use lance_file::writer::FileWriterOptions; + use lance_io::utils::CachedFileSize; + use lance_table::format::DataFile; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let tmp_dir = tempdir().unwrap(); + let uri = tmp_dir.path().to_str().unwrap(); + let conn = ConnectBuilder::new(uri) + .read_consistency_interval(Duration::from_secs(0)) + .execute() + .await + .unwrap(); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("foo", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..100)), + Arc::new(Int32Array::from_iter_values(0..100)), + ], + ) + .unwrap(); + let table = conn + .create_table("test_stats_extra_files", batch) + .execute() + .await + .unwrap(); + + let data_only = table.stats().await.unwrap().total_bytes; + assert!(data_only > 0); + + // A scalar index adds index files whose sizes are recorded in the + // manifest's index section. + table + .create_index(&["id"], Index::Auto) + .execute() + .await + .unwrap(); + let with_index = table.stats().await.unwrap().total_bytes; + let dataset = { + let native = table.as_native().unwrap(); + (*native.dataset.get().await.unwrap()).clone() + }; + let index_bytes: usize = dataset + .load_indices() + .await + .unwrap() + .iter() + .map(|idx| idx.total_size_bytes().unwrap_or(0) as usize) + .sum(); + assert!(index_bytes > 0); + assert_eq!(with_index, data_only + index_bytes); + + // Commit an overlay file supplying new `foo` values for the first three + // rows of fragment 0. There is no high-level API that writes overlays + // yet, so write the overlay's data file and commit the `DataOverlay` + // operation by hand. + let read_version = dataset.version().version; + let fragment_id = dataset.get_fragments()[0].id() as u64; + let foo_field_id = dataset.schema().field("foo").unwrap().id; + let overlay_schema = dataset.schema().project_by_ids(&[foo_field_id], true); + let file_version = ConcreteFileVersion::from(LanceFileVersion::Stable); + + let filename = "overlay.lance".to_string(); + let store = dataset.object_store(None).await.unwrap(); + let path = dataset.data_dir().child(filename.clone()); + let obj_writer = store.create(&path).await.unwrap(); + let mut writer = lance_file::versions::create_writer( + file_version, + obj_writer, + overlay_schema, + FileWriterOptions::default(), + ) + .unwrap(); + writer + .write_column(0, Arc::new(Int32Array::from(vec![1000, 1001, 1002])) as _) + .await + .unwrap(); + let summary = writer.finish().await.unwrap(); + let overlay_bytes = summary.size_bytes as usize; + assert!(overlay_bytes > 0); + + let mut data_file = DataFile::new_unstarted(filename, file_version); + data_file.fields = writer + .field_id_to_column_indices() + .iter() + .map(|(field_id, _)| *field_id as i32) + .collect::>() + .into(); + data_file.column_indices = writer + .field_id_to_column_indices() + .iter() + .map(|(_, column_index)| *column_index as i32) + .collect::>() + .into(); + data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes); + + let overlay = DataOverlayFile { + data_file, + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter(0..3)), + committed_version: 0, + }; + Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset)), + Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![overlay], + }], + }, + Some(read_version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .unwrap(); + + table.checkout_latest().await.unwrap(); + let with_overlay = table.stats().await.unwrap().total_bytes; + assert_eq!(with_overlay, with_index + overlay_bytes); + } + + /// `stats()` must stay manifest-only. Summing per-field `bytes_on_disk` + /// instead opens every data file, so cost would grow with fragment count. + #[tokio::test] + pub async fn test_stats_does_not_read_data_files() { + let tmp_dir = tempdir().unwrap(); + let uri = tmp_dir.path().to_str().unwrap(); + + let conn = ConnectBuilder::new(uri).execute().await.unwrap(); + + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..10))], + ) + .unwrap(); + + conn.create_table("test_stats_io", batch.clone()) + .execute() + .await + .unwrap(); + let table = conn.open_table("test_stats_io").execute().await.unwrap(); + const NUM_APPENDS: usize = 20; + for _ in 0..NUM_APPENDS { + table.add(batch.clone()).execute().await.unwrap(); + } + + // Reopen through a tracking store so the counters cover `stats()` alone and + // not the writes above. + let (wrapper, io_stats) = IoTrackingStore::new_wrapper(); + let table = conn + .open_table("test_stats_io") + .lance_read_params(ReadParams { + store_options: Some(ObjectStoreParams { + object_store_wrapper: Some(wrapper), + ..Default::default() + }), + ..Default::default() + }) + .execute() + .await + .unwrap(); + io_stats.lock().unwrap().read_iops = 0; + + let stats = table.stats().await.unwrap(); + let read_iops = io_stats.lock().unwrap().read_iops; + + assert_eq!(stats.fragment_stats.num_fragments, NUM_APPENDS + 1); + assert!(stats.total_bytes > 0); + // Reading the fragments' data files would take at least one IOP each. + assert!( + read_iops < stats.fragment_stats.num_fragments as u64, + "stats() issued {} read IOPs across {} fragments", + read_iops, + stats.fragment_stats.num_fragments + ); + } } From 36054be5760f54042549279333431fd8b4aaea76 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:34:39 +0800 Subject: [PATCH 49/49] fix(node): preserve nested Arrow data across versions (#3900) ## Root cause When LanceDB accepted an Arrow table created by a different installed Arrow package, its compatibility sanitizer rebuilt each Data node without converting the foreign type or preserving nested children. It also dropped the separate dictionary vector payload and did not preserve identity shared by dictionary schema types, vector wrappers, or growing dictionary chunks. ## Fix Recursively sanitize nested Arrow data types and child data. Use one table-scoped sanitization context to rebuild and memoize source type objects, dictionary vectors, and Data nodes in the local Arrow realm, preserving all identities required by Arrow IPC. Add Arrow 15 through 18 regressions for list serialization, ordinary dictionaries, dictionaries shared across fields and batches, growing dictionaries, and IPC round trips. ## Validation - pnpm test __test__/arrow.test.ts --runInBand (188 passed) - pnpm lint - pnpm build - pnpm test --runInBand (706 passed, 5 skipped) - pnpm run docs Fixes #2256 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/arrow.test.ts | 115 +++++++++++++++++++ nodejs/lancedb/sanitize.ts | 203 +++++++++++++++++++++++++++++----- 2 files changed, 289 insertions(+), 29 deletions(-) diff --git a/nodejs/__test__/arrow.test.ts b/nodejs/__test__/arrow.test.ts index c05849cb9..29030d4f8 100644 --- a/nodejs/__test__/arrow.test.ts +++ b/nodejs/__test__/arrow.test.ts @@ -6,7 +6,9 @@ import * as arrow17 from "apache-arrow-17"; import * as arrow18 from "apache-arrow-18"; import { + Vector as CurrentVector, convertToTable, + tableFromIPC as currentTableFromIPC, fromBufferToRecordBatch, fromDataToBuffer, fromRecordBatchToBuffer, @@ -19,6 +21,7 @@ import { FunctionOptions, } from "../lancedb/embedding/embedding_function"; import { EmbeddingFunctionConfig } from "../lancedb/embedding/registry"; +import { sanitizeTable } from "../lancedb/sanitize"; // biome-ignore lint/suspicious/noExplicitAny: skip function sampleRecords(): Array> { @@ -64,7 +67,11 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( tableFromIPC, DataType, Dictionary, + RecordBatch: ArrowRecordBatch, + Table: ArrowTable, Uint8: ArrowUint8, + makeData: arrowMakeData, + vectorFromArray, // biome-ignore lint/suspicious/noExplicitAny: } = arrow; type Schema = ApacheArrow["Schema"]; @@ -1054,6 +1061,114 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( }); describe("when using two versions of arrow", function () { + it("preserves a dictionary shared by multiple fields", async function () { + const values = ["alpha", "beta", "alpha"]; + const dictionaryVector = vectorFromArray(values); + const batch = new ArrowRecordBatch({ + first: dictionaryVector.data[0], + second: dictionaryVector.data[0], + }); + const table = new ArrowTable([batch]); + + const sanitized = sanitizeTable(table); + expect([...sanitized.getChild("first")!]).toEqual(values); + expect([...sanitized.getChild("second")!]).toEqual(values); + const firstType = sanitized.schema.fields[0].type as { + dictionary: unknown; + }; + const secondType = sanitized.schema.fields[1].type as { + dictionary: unknown; + }; + expect(secondType.dictionary).toBe(firstType.dictionary); + expect(sanitized.batches[0].data.children[1].dictionary).toBe( + sanitized.batches[0].data.children[0].dictionary, + ); + + const buf = await fromDataToBuffer(table); + const actual = currentTableFromIPC(buf); + expect([...actual.getChild("first")!]).toEqual(values); + expect([...actual.getChild("second")!]).toEqual(values); + }); + + it("preserves shared dictionary data from another Arrow version", async function () { + const values = ["alpha", "beta", "alpha"]; + const dictionaryVector = vectorFromArray(values); + const firstBatch = new ArrowRecordBatch({ + label: dictionaryVector.slice(0, 2).data[0], + }); + const secondBatch = new ArrowRecordBatch({ + label: dictionaryVector.slice(2).data[0], + }); + const table = new ArrowTable([firstBatch, secondBatch]); + + const sanitized = sanitizeTable(table); + expect([...sanitized.getChild("label")!]).toEqual(values); + + const dictionaries = sanitized.batches.map( + (batch) => batch.data.children[0].dictionary, + ); + expect(dictionaries[0]).toBeInstanceOf(CurrentVector); + expect(dictionaries[1]).toBe(dictionaries[0]); + + const buf = await fromDataToBuffer(table); + const actual = currentTableFromIPC(buf); + expect([...actual.getChild("label")!]).toEqual(values); + }); + + it("preserves shared chunks in growing dictionaries", async function () { + const type = new Dictionary(new Utf8(), new Int32(), 42, false); + const firstDictionary = vectorFromArray(["alpha", "beta"], new Utf8()); + const secondDictionary = firstDictionary.concat( + vectorFromArray(["gamma"], new Utf8()), + ); + const firstData = arrowMakeData({ + type, + data: Int32Array.from([0, 1]), + dictionary: firstDictionary, + }); + const secondData = arrowMakeData({ + type, + data: Int32Array.from([2]), + dictionary: secondDictionary, + }); + const table = new ArrowTable([ + new ArrowRecordBatch({ label: firstData }), + new ArrowRecordBatch({ label: secondData }), + ]); + + const sanitized = sanitizeTable(table); + const expected = ["alpha", "beta", "gamma"]; + expect([...sanitized.getChild("label")!]).toEqual(expected); + const firstLocalDictionary = + sanitized.batches[0].data.children[0].dictionary!; + const secondLocalDictionary = + sanitized.batches[1].data.children[0].dictionary!; + expect(secondLocalDictionary.data[0]).toBe( + firstLocalDictionary.data[0], + ); + + const buf = await fromTableToBuffer(sanitized); + const actual = currentTableFromIPC(buf); + expect([...actual.getChild("label")!]).toEqual(expected); + }); + + it("can serialize list data from another Arrow version", async function () { + const values = [["anime", "action"], [], null]; + const vector = vectorFromArray( + values, + new List(new Field("item", new Utf8(), true)), + ); + const table = new ArrowTable({ tags: vector }); + + const buf = await fromDataToBuffer(table); + const actual = currentTableFromIPC(buf); + const actualTags = actual.getChild("tags"); + + expect(actualTags?.get(0)?.toJSON()).toEqual(values[0]); + expect(actualTags?.get(1)?.toJSON()).toEqual(values[1]); + expect(actualTags?.get(2)).toBeNull(); + }); + it("can still import data", async function () { const schema = new arrow15.Schema([ new arrow15.Field("id", new arrow15.Int32()), diff --git a/nodejs/lancedb/sanitize.ts b/nodejs/lancedb/sanitize.ts index ae0bc0179..8fb2f1a0a 100644 --- a/nodejs/lancedb/sanitize.ts +++ b/nodejs/lancedb/sanitize.ts @@ -9,7 +9,7 @@ // comes from the exact same library instance. This is not always the case // and so we must sanitize the input to ensure that it is compatible. -import { BufferType, Data } from "apache-arrow"; +import { BufferType, Data, Vector } from "apache-arrow"; import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type"; import { Binary, @@ -74,6 +74,20 @@ import { Utf8, } from "./arrow"; +type SanitizationContext = { + types: WeakMap; + vectors: WeakMap; + data: WeakMap>; +}; + +function createSanitizationContext(): SanitizationContext { + return { + types: new WeakMap(), + vectors: new WeakMap(), + data: new WeakMap(), + }; +} + export function sanitizeMetadata( metadataLike?: unknown, ): Map | undefined { @@ -186,6 +200,13 @@ export function sanitizeInterval(typeLike: object) { } export function sanitizeList(typeLike: object) { + return sanitizeListWithContext(typeLike, createSanitizationContext()); +} + +function sanitizeListWithContext( + typeLike: object, + context: SanitizationContext, +) { if (!("children" in typeLike) || !Array.isArray(typeLike.children)) { throw Error( "Expected a List type to have an array-like `children` property", @@ -194,19 +215,35 @@ export function sanitizeList(typeLike: object) { if (typeLike.children.length !== 1) { throw Error("Expected a List type to have exactly one child"); } - return new List(sanitizeField(typeLike.children[0])); + return new List(sanitizeFieldWithContext(typeLike.children[0], context)); } export function sanitizeStruct(typeLike: object) { + return sanitizeStructWithContext(typeLike, createSanitizationContext()); +} + +function sanitizeStructWithContext( + typeLike: object, + context: SanitizationContext, +) { if (!("children" in typeLike) || !Array.isArray(typeLike.children)) { throw Error( "Expected a Struct type to have an array-like `children` property", ); } - return new Struct(typeLike.children.map((child) => sanitizeField(child))); + return new Struct( + typeLike.children.map((child) => sanitizeFieldWithContext(child, context)), + ); } export function sanitizeUnion(typeLike: object) { + return sanitizeUnionWithContext(typeLike, createSanitizationContext()); +} + +function sanitizeUnionWithContext( + typeLike: object, + context: SanitizationContext, +) { if ( !("typeIds" in typeLike) || !("mode" in typeLike) || @@ -226,7 +263,7 @@ export function sanitizeUnion(typeLike: object) { typeLike.mode, // biome-ignore lint/suspicious/noExplicitAny: skip typeLike.typeIds as any, - typeLike.children.map((child) => sanitizeField(child)), + typeLike.children.map((child) => sanitizeFieldWithContext(child, context)), ); } @@ -234,6 +271,19 @@ export function sanitizeTypedUnion( typeLike: object, // eslint-disable-next-line @typescript-eslint/naming-convention UnionType: typeof DenseUnion | typeof SparseUnion, +) { + return sanitizeTypedUnionWithContext( + typeLike, + UnionType, + createSanitizationContext(), + ); +} + +function sanitizeTypedUnionWithContext( + typeLike: object, + // eslint-disable-next-line @typescript-eslint/naming-convention + UnionType: typeof DenseUnion | typeof SparseUnion, + context: SanitizationContext, ) { if (!("typeIds" in typeLike)) { throw Error( @@ -248,7 +298,7 @@ export function sanitizeTypedUnion( return new UnionType( typeLike.typeIds as Int32Array | number[], - typeLike.children.map((child) => sanitizeField(child)), + typeLike.children.map((child) => sanitizeFieldWithContext(child, context)), ); } @@ -262,6 +312,16 @@ export function sanitizeFixedSizeBinary(typeLike: object) { } export function sanitizeFixedSizeList(typeLike: object) { + return sanitizeFixedSizeListWithContext( + typeLike, + createSanitizationContext(), + ); +} + +function sanitizeFixedSizeListWithContext( + typeLike: object, + context: SanitizationContext, +) { if (!("listSize" in typeLike) || typeof typeLike.listSize !== "number") { throw Error("Expected a FixedSizeList type to have a `listSize` property"); } @@ -275,11 +335,18 @@ export function sanitizeFixedSizeList(typeLike: object) { } return new FixedSizeList( typeLike.listSize, - sanitizeField(typeLike.children[0]), + sanitizeFieldWithContext(typeLike.children[0], context), ); } export function sanitizeMap(typeLike: object) { + return sanitizeMapWithContext(typeLike, createSanitizationContext()); +} + +function sanitizeMapWithContext( + typeLike: object, + context: SanitizationContext, +) { if (!("children" in typeLike) || !Array.isArray(typeLike.children)) { throw Error( "Expected a Map type to have an array-like `children` property", @@ -292,7 +359,10 @@ export function sanitizeMap(typeLike: object) { throw Error("Expected a Map type to have exactly one child"); } - return new Map_(sanitizeField(typeLike.children[0]), typeLike.keysSorted); + return new Map_( + sanitizeFieldWithContext(typeLike.children[0], context), + typeLike.keysSorted, + ); } export function sanitizeDuration(typeLike: object) { @@ -303,6 +373,13 @@ export function sanitizeDuration(typeLike: object) { } export function sanitizeDictionary(typeLike: object) { + return sanitizeDictionaryWithContext(typeLike, createSanitizationContext()); +} + +function sanitizeDictionaryWithContext( + typeLike: object, + context: SanitizationContext, +) { if (!("id" in typeLike) || typeof typeLike.id !== "number") { throw Error("Expected a Dictionary type to have an `id` property"); } @@ -316,8 +393,8 @@ export function sanitizeDictionary(typeLike: object) { throw Error("Expected a Dictionary type to have an `isOrdered` property"); } return new Dictionary( - sanitizeType(typeLike.dictionary), - sanitizeType(typeLike.indices) as TKeys, + sanitizeTypeWithContext(typeLike.dictionary, context), + sanitizeTypeWithContext(typeLike.indices, context) as TKeys, typeLike.id, typeLike.isOrdered, ); @@ -325,12 +402,23 @@ export function sanitizeDictionary(typeLike: object) { // biome-ignore lint/suspicious/noExplicitAny: skip export function sanitizeType(typeLike: unknown): DataType { + return sanitizeTypeWithContext(typeLike, createSanitizationContext()); +} + +function sanitizeTypeWithContext( + typeLike: unknown, + context: SanitizationContext, +): DataType { if (typeof typeLike === "string") { return dataTypeFromName(typeLike); } if (typeof typeLike !== "object" || typeLike === null) { throw Error("Expected a Type but object was null/undefined"); } + const cached = context.types.get(typeLike); + if (cached !== undefined) { + return cached; + } if ( !("typeId" in typeLike) || !( @@ -349,6 +437,16 @@ export function sanitizeType(typeLike: unknown): DataType { throw Error("Type's typeId property was not a function or number"); } + const type = sanitizeTypeById(typeLike, typeId, context); + context.types.set(typeLike, type); + return type; +} + +function sanitizeTypeById( + typeLike: object, + typeId: Type, + context: SanitizationContext, +): DataType { switch (typeId) { case Type.NONE: throw Error("Received a Type with a typeId of NONE"); @@ -375,21 +473,21 @@ export function sanitizeType(typeLike: unknown): DataType { case Type.Interval: return sanitizeInterval(typeLike); case Type.List: - return sanitizeList(typeLike); + return sanitizeListWithContext(typeLike, context); case Type.Struct: - return sanitizeStruct(typeLike); + return sanitizeStructWithContext(typeLike, context); case Type.Union: - return sanitizeUnion(typeLike); + return sanitizeUnionWithContext(typeLike, context); case Type.FixedSizeBinary: return sanitizeFixedSizeBinary(typeLike); case Type.FixedSizeList: - return sanitizeFixedSizeList(typeLike); + return sanitizeFixedSizeListWithContext(typeLike, context); case Type.Map: - return sanitizeMap(typeLike); + return sanitizeMapWithContext(typeLike, context); case Type.Duration: return sanitizeDuration(typeLike); case Type.Dictionary: - return sanitizeDictionary(typeLike); + return sanitizeDictionaryWithContext(typeLike, context); case Type.Int8: return new Int8(); case Type.Int16: @@ -433,9 +531,9 @@ export function sanitizeType(typeLike: unknown): DataType { case Type.TimestampSecond: return sanitizeTypedTimestamp(typeLike, TimestampSecond); case Type.DenseUnion: - return sanitizeTypedUnion(typeLike, DenseUnion); + return sanitizeTypedUnionWithContext(typeLike, DenseUnion, context); case Type.SparseUnion: - return sanitizeTypedUnion(typeLike, SparseUnion); + return sanitizeTypedUnionWithContext(typeLike, SparseUnion, context); case Type.IntervalDayTime: return new IntervalDayTime(); case Type.IntervalYearMonth: @@ -454,6 +552,13 @@ export function sanitizeType(typeLike: unknown): DataType { } export function sanitizeField(fieldLike: unknown): Field { + return sanitizeFieldWithContext(fieldLike, createSanitizationContext()); +} + +function sanitizeFieldWithContext( + fieldLike: unknown, + context: SanitizationContext, +): Field { if (fieldLike instanceof Field) { return fieldLike; } @@ -471,7 +576,7 @@ export function sanitizeField(fieldLike: unknown): Field { } let type: DataType; try { - type = sanitizeType(fieldLike.type); + type = sanitizeTypeWithContext(fieldLike.type, context); } catch (error: unknown) { throw Error( `Unable to sanitize type for field: ${fieldLike.name} due to error: ${error}`, @@ -501,6 +606,13 @@ export function sanitizeField(fieldLike: unknown): Field { * than lancedb is using. */ export function sanitizeSchema(schemaLike: SchemaLike): Schema { + return sanitizeSchemaWithContext(schemaLike, createSanitizationContext()); +} + +function sanitizeSchemaWithContext( + schemaLike: SchemaLike, + context: SanitizationContext, +): Schema { if (schemaLike instanceof Schema) { return schemaLike; } @@ -522,7 +634,7 @@ export function sanitizeSchema(schemaLike: SchemaLike): Schema { ); } const sanitizedFields = schemaLike.fields.map((field) => - sanitizeField(field), + sanitizeFieldWithContext(field, context), ); return new Schema(sanitizedFields, metadata); } @@ -544,13 +656,18 @@ export function sanitizeTable(tableLike: TableLike): Table { "The table passed in does not appear to be a table (no 'columns' property)", ); } - const schema = sanitizeSchema(tableLike.schema); - - const batches = tableLike.batches.map(sanitizeRecordBatch); + const context = createSanitizationContext(); + const schema = sanitizeSchemaWithContext(tableLike.schema, context); + const batches = tableLike.batches.map((batch) => + sanitizeRecordBatch(batch, context), + ); return new Table(schema, batches); } -function sanitizeRecordBatch(batchLike: RecordBatchLike): RecordBatch { +function sanitizeRecordBatch( + batchLike: RecordBatchLike, + context: SanitizationContext, +): RecordBatch { if (batchLike instanceof RecordBatch) { return batchLike; } @@ -567,19 +684,43 @@ function sanitizeRecordBatch(batchLike: RecordBatchLike): RecordBatch { "The record batch passed in does not appear to be a record batch (no 'data' property)", ); } - const schema = sanitizeSchema(batchLike.schema); - const data = sanitizeData(batchLike.data); + const schema = sanitizeSchemaWithContext(batchLike.schema, context); + const data = sanitizeData(batchLike.data, context) as Data; return new RecordBatch(schema, data); } + +type DictionaryVectorLike = { + data: readonly DataLike[]; +}; + +type DictionaryDataLike = DataLike & { + dictionary?: DictionaryVectorLike; +}; + function sanitizeData( dataLike: DataLike, - // biome-ignore lint/suspicious/noExplicitAny: -): import("apache-arrow").Data> { + context: SanitizationContext, +): Data { if (dataLike instanceof Data) { return dataLike; } - return new Data( - dataLike.type, + const cachedData = context.data.get(dataLike); + if (cachedData !== undefined) { + return cachedData; + } + const dictionaryLike = (dataLike as DictionaryDataLike).dictionary; + let dictionary: Vector | undefined; + if (dictionaryLike !== undefined) { + dictionary = context.vectors.get(dictionaryLike); + if (dictionary === undefined) { + dictionary = new Vector( + dictionaryLike.data.map((data) => sanitizeData(data, context)), + ); + context.vectors.set(dictionaryLike, dictionary); + } + } + const data = new Data( + sanitizeTypeWithContext(dataLike.type, context), dataLike.offset, dataLike.length, dataLike.nullCount, @@ -589,7 +730,11 @@ function sanitizeData( [BufferType.VALIDITY]: dataLike.nullBitmap, [BufferType.TYPE]: dataLike.typeIds, }, + dataLike.children.map((child) => sanitizeData(child, context)), + dictionary, ); + context.data.set(dataLike, data); + return data; } const constructorsByTypeName = {