From c3176a47ce5206dcecc44605e20a1cd710a278d0 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:13:54 +0800 Subject: [PATCH 01/41] fix(python): report unsplittable IVF partition errors (#3846) ## Summary - add Python regression coverage for an IVF build that cannot form all requested non-empty partitions - verify hierarchical k-means returns an actionable RuntimeError instead of panicking or silently creating a degenerate index - exercise the current Lance v10.1.0-beta.1 dependency, which contains the upstream error-return fix ## Root cause Hierarchical k-means previously guarded a shortfall in generated clusters with only a debug assertion. Debug builds panicked, while release builds could silently publish an index with many empty partitions. The upstream Lance fix now returns a descriptive error and is already included in the dependency pinned on main; this test locks in propagation through the LanceDB Python API. ## Validation - uv run --extra tests pytest python/tests/test_index.py -q (24 passed) - uv run --extra tests pytest python/tests/test_index.py::test_create_ivf_index_reports_unsplittable_partitions -q (1 passed) - python/.venv/bin/ruff format python/python/tests/test_index.py - python/.venv/bin/ruff check . - git diff --check Fixes #3649 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_index.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/python/python/tests/test_index.py b/python/python/tests/test_index.py index 1cf2c733c..94268a53e 100644 --- a/python/python/tests/test_index.py +++ b/python/python/tests/test_index.py @@ -372,6 +372,31 @@ async def test_create_vector_index(some_table: AsyncTable): assert stats.num_indices == 1 +@pytest.mark.asyncio +async def test_create_ivf_index_reports_unsplittable_partitions(db_async): + dim = 8 + num_partitions = 300 # More than 256 selects hierarchical k-means. + base_vectors = [[float(row == column) for column in range(dim)] for row in range(5)] + vectors = pa.array(base_vectors * 200, pa.list_(pa.float32(), dim)) + table = await db_async.create_table( + "unsplittable_partitions", + pa.table({"vector": vectors}), + ) + + error_pattern = ( + rf"Cannot create {num_partitions} IVF partitions: k-means could only form" + ) + with pytest.raises(RuntimeError, match=error_pattern): + await table.create_index( + "vector", + config=IvfFlat( + distance_type="dot", + num_partitions=num_partitions, + max_iterations=10, + ), + ) + + @pytest.mark.asyncio async def test_create_4bit_ivfpq_index(some_table: AsyncTable): # Can create From 9e2e711c7a2311b35e6e5795c1bb71ebb5090b6f Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:36:05 +0800 Subject: [PATCH 02/41] test(python): cover OpenAI registry variable round-trip (#3863) ## Summary - replace the synthetic registry-variable metadata test with the OpenAI embedding function reported in #2387 - verify the resolved API key survives table metadata reconstruction - assert the OpenAI client receives the resolved key while serialized metadata retains the variable reference ## Root cause LanceDB 0.22.0 reconstructed embedding functions from table metadata with the model constructor, bypassing EmbeddingFunction.create and leaving the literal $var:api_key placeholder in OpenAI configuration. The production path was corrected for duplicate #2181 by #2640; this change gives that fix direct, network-free OpenAI regression coverage for #2387. ## Validation - uv run --extra tests pytest python/tests/test_embeddings.py -q (13 passed, 9 skipped) - uv run --project python --extra dev ruff check . - uv run --project python --extra dev ruff format --check python/python/tests/test_embeddings.py - git diff --check Fixes #2387 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_embeddings.py | 41 +++++++++----------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/python/python/tests/test_embeddings.py b/python/python/tests/test_embeddings.py index 5efb7d98a..f2d8971c3 100644 --- a/python/python/tests/test_embeddings.py +++ b/python/python/tests/test_embeddings.py @@ -115,34 +115,16 @@ def test_embedding_function_variables(): assert func.safe_model_dump()["secret_key"] == "$var:secret" -def test_parse_functions_with_variables(): - @register("variable-parsing-test") - class VariableParsingFunction(TextEmbeddingFunction): - api_key: str - base_url: Optional[str] = None - - @staticmethod - def sensitive_keys(): - return ["api_key"] - - def ndims(self): - return 10 - - def generate_embeddings(self, texts): - # Mock implementation that just returns random embeddings - # In real usage, this would use the api_key to call an API - return [np.random.rand(self.ndims()).tolist() for _ in texts] - +def test_openai_variables_survive_metadata_round_trip(): registry = EmbeddingFunctionRegistry.get_instance() registry.set_var("test_api_key", "sk-test-key-12345") - registry.set_var("test_base_url", "https://api.example.com") conf = EmbeddingFunctionConfig( source_column="text", vector_column="vector", - function=registry.get("variable-parsing-test").create( - api_key="$var:test_api_key", base_url="$var:test_base_url" + function=registry.get("openai").create( + api_key="$var:test_api_key", base_url="https://api.example.com" ), ) @@ -150,7 +132,10 @@ def test_parse_functions_with_variables(): # Create a mock arrow table with the metadata schema = pa.schema( - [pa.field("text", pa.string()), pa.field("vector", pa.list_(pa.float32(), 10))] + [ + pa.field("text", pa.string()), + pa.field("vector", pa.list_(pa.float32(), 1536)), + ] ) table = pa.table({"text": [], "vector": []}, schema=schema) table = table.replace_schema_metadata(metadata) @@ -164,13 +149,15 @@ def test_parse_functions_with_variables(): assert parsed_func.api_key == "sk-test-key-12345" assert parsed_func.base_url == "https://api.example.com" - - embeddings = parsed_func.generate_embeddings(["test text"]) - assert len(embeddings) == 1 - assert len(embeddings[0]) == 10 - assert parsed_func.safe_model_dump()["api_key"] == "$var:test_api_key" + with patch("lancedb.embeddings.openai.attempt_import_or_raise") as import_openai: + parsed_func._openai_client + + import_openai.return_value.OpenAI.assert_called_once_with( + api_key="sk-test-key-12345", base_url="https://api.example.com" + ) + def test_embedding_with_bad_results(tmp_path): @register("null-embedding") From 27dd92c67ebee5bbc37c0d085427d78fc2ffcd87 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:36:27 +0800 Subject: [PATCH 03/41] test(python): cover debugger-safe connection inspection (#3880) ## Summary - expand the synchronous debugger regression to enumerate every exposed connection attribute while the Python background loop is unavailable - retain direct representation checks for connections and tables ## Root cause VS Code debugpy suspends Python threads at a breakpoint and inspects local variables. Connection representation and property access previously dispatched asynchronous work to LanceDBBackgroundEventLoop and waited for the suspended loop thread, deadlocking the debugger. The production safeguards landed in #3620 and #3788; this regression exercises debugger-style whole-object expansion so a newly exposed property cannot reintroduce the original failure. ## Validation - uv run --no-sync pytest python/tests/test_db.py::test_sync_debugger_inspection_does_not_use_background_loop python/tests/test_db.py::test_read_consistency_interval_does_not_use_background_loop -q (2 passed) - uv run --no-sync pytest python/tests/test_db.py -q (48 passed) - python/.venv/bin/ruff format --check python/python/tests/test_db.py - python/.venv/bin/ruff check . - git diff --check Fixes #3611 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_db.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/python/python/tests/test_db.py b/python/python/tests/test_db.py index 93b791650..8f4a8850c 100644 --- a/python/python/tests/test_db.py +++ b/python/python/tests/test_db.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import inspect import re import sys from datetime import timedelta @@ -62,17 +63,23 @@ def test_basic(tmp_path): assert db.open_table("test").name == db["test"].name -def test_sync_repr_does_not_use_background_loop(tmp_path, monkeypatch): +def test_sync_debugger_inspection_does_not_use_background_loop(tmp_path, monkeypatch): from lancedb.background_loop import LOOP db = lancedb.connect(tmp_path) table = db.create_table("test", data=[{"id": 1}]) def fail_run(*args, **kwargs): - raise AssertionError("repr should not use the Python background loop") + raise AssertionError("debugger inspection should not use the background loop") monkeypatch.setattr(LOOP, "run", fail_run) + # Debuggers enumerate and evaluate every exposed attribute when expanding a + # variable. This must remain safe while their breakpoint suspends LOOP's thread. + members = dict(inspect.getmembers(db)) + + assert members["uri"] == str(tmp_path) + assert members["read_consistency_interval"] is None assert repr(db) == f"LanceDBConnection(uri={str(tmp_path)!r})" assert repr(table) == f"LanceTable(name='test', _conn={db!r})" From 16e1967efc0be9e1a9ddb44fb194374c93705607 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:36:58 +0800 Subject: [PATCH 04/41] fix(python): align wheel ABI with supported versions (#3884) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - align the PyO3 runtime and build ABI floor with the declared Python 3.10 minimum - add a regression test that keeps both ABI features synchronized with `requires-python` ## Root cause The Python 3.10 support-floor update originally changed PyO3 to `abi3-py310`, but a later dependency update reverted both PyO3 features to `abi3-py39`. Published Windows wheels were consequently tagged `cp39-abi3` while importing `PyCMethod_New`, a stable-ABI procedure absent from CPython 3.9.0 and 3.9.1. Windows reports that mismatch as “The specified procedure could not be found” while loading `_lancedb`. Restoring `abi3-py310` makes the wheel tag and native imports agree with the package metadata and prevents future wheels from advertising unsupported Python 3.9 compatibility. ## Validation - `uv run --extra tests pytest python/tests/test_package_metadata.py -q` - `uv run --extra tests --extra dev ruff format --check .` - `uv run --extra tests --extra dev ruff check .` - `cargo fmt --all` - `cargo check --quiet -p lancedb-python` - `uvx --from maturin==1.12.4 maturin build --profile ci` (built `lancedb-0.37.1b0-cp310-abi3-manylinux_2_34_x86_64.whl`) Fixes #2051 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/Cargo.toml | 4 +- python/python/tests/test_package_metadata.py | 42 ++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 python/python/tests/test_package_metadata.py diff --git a/python/Cargo.toml b/python/Cargo.toml index cc706e712..5a196e27c 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -26,7 +26,7 @@ lance-namespace-impls.workspace = true lance-io.workspace = true env_logger.workspace = true log.workspace = true -pyo3 = { version = "0.28", features = ["extension-module", "abi3-py39", "chrono"] } +pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "chrono"] } chrono = { version = "0.4", default-features = false, features = ["clock"] } pyo3-async-runtimes = { version = "0.28", features = [ "attributes", @@ -43,7 +43,7 @@ libc = "0.2" [build-dependencies] pyo3-build-config = { version = "0.28", features = [ "extension-module", - "abi3-py39", + "abi3-py310", ] } [features] diff --git a/python/python/tests/test_package_metadata.py b/python/python/tests/test_package_metadata.py new file mode 100644 index 000000000..5792f457b --- /dev/null +++ b/python/python/tests/test_package_metadata.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import importlib +import re +import sys +from pathlib import Path + +import pytest + + +def test_pyo3_abi_matches_minimum_supported_python(): + project_dir = Path(__file__).parents[2] + pyproject = (project_dir / "pyproject.toml").read_text() + cargo_manifest = (project_dir / "Cargo.toml").read_text() + + minimum_python = re.search( + r'^requires-python\s*=\s*">=(\d+)\.(\d+)"$', pyproject, re.MULTILINE + ) + assert minimum_python is not None + + major, minor = minimum_python.groups() + expected_abi = f"abi3-py{major}{minor}" + configured_abis = re.findall(r'"(abi3-py\d+)"', cargo_manifest) + + assert configured_abis == [expected_abi, expected_abi], ( + "the pyo3 runtime and build ABI features must both match requires-python" + ) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows wheel regression test") +def test_windows_wheel_tag_and_native_import(): + project_dir = Path(__file__).parents[2] + wheels = list((project_dir.parent / "target" / "wheels").glob("lancedb-*.whl")) + if not wheels: + pytest.skip("no wheel artifact is available in this development environment") + + assert len(wheels) == 1 + assert wheels[0].name.endswith("-cp310-abi3-win_amd64.whl") + + native_module = importlib.import_module("lancedb._lancedb") + assert Path(native_module.__file__).suffix == ".pyd" From 3956d9dbfa1e701ca461a078ee0f1ca2a9c175a6 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:38:27 +0800 Subject: [PATCH 05/41] fix(python): prevent OpenSSL linkage in Linux wheels (#3877) ## Summary - select rustls with native certificate roots explicitly for LanceDB's remote HTTP client - add a Linux regression test that rejects `libssl` or `libcrypto` dependencies in the built Python extension ## Root cause The Python remote client originally enabled reqwest's native TLS backend. During manylinux wheel repair, that caused OpenSSL 1.1 libraries to be bundled into the wheel. Loading those libraries on RHEL 9 with FIPS enabled aborts during the OpenSSL self-test before `import lancedb` can complete. LanceDB has since moved away from native TLS, but its own reqwest dependency relied on transitive rustls feature selection and the built extension had no regression guard. This change makes rustls selection explicit and tests the produced Linux native module's dynamic dependencies. ## Validation - `uv run --no-sync pytest python/tests/test_import.py -q` - `ruff format --check python` - `ruff check .` - `cargo fmt --all -- --check` - `cargo check --quiet --features remote --tests --examples` - `ldd python/lancedb/_lancedb.abi3.so` (no `libssl` or `libcrypto` dependency) - verified the resolved Python Rust dependency graph contains rustls and no `openssl-sys` or `native-tls` Fixes #1884 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_import.py | 33 ++++++++++++++++++++++++++++++ rust/lancedb/Cargo.toml | 2 ++ 2 files changed, 35 insertions(+) create mode 100644 python/python/tests/test_import.py diff --git a/python/python/tests/test_import.py b/python/python/tests/test_import.py new file mode 100644 index 000000000..4b87a0ce8 --- /dev/null +++ b/python/python/tests/test_import.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import re +import shutil +import subprocess +import sys + +import lancedb._lancedb as _lancedb +import pytest + + +@pytest.mark.skipif(sys.platform != "linux", reason="ldd is Linux-specific") +def test_native_extension_does_not_link_openssl(): + """OpenSSL-linked wheels abort when imported on RHEL hosts in FIPS mode.""" + ldd = shutil.which("ldd") + if ldd is None: + pytest.skip("ldd is not installed") + + result = subprocess.run( + [ldd, _lancedb.__file__], + check=True, + capture_output=True, + text=True, + ) + openssl_libraries = re.findall( + r"^\s*(lib(?:crypto|ssl)\S*)\s+=>", result.stdout, flags=re.MULTILINE + ) + + assert not openssl_libraries, ( + "the LanceDB native extension must use rustls instead of linking OpenSSL: " + f"{openssl_libraries}" + ) diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 66db3cf12..816f095de 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -75,6 +75,8 @@ reqwest = { version = "0.12.0", default-features = false, features = [ "http2", "json", "macos-system-configuration", + # Avoid linking OpenSSL into Python wheels, which breaks on FIPS hosts. + "rustls-tls-native-roots", "stream", ], optional = true } http = { version = "1", optional = true } # Matching what is in reqwest From 9e73d440a36986db57f16cf8a19cf850135c53f2 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:39:26 +0800 Subject: [PATCH 06/41] test(python): cover schema-only vector table creation (#3882) ## Summary - make the existing #1968 regression explicitly assert that schema-only table creation succeeds - verify the new table has zero rows and preserves the requested fixed-size vector schema before accepting subsequent data ## Root cause In v0.16.0, schema-only table creation sent an empty table through vector sanitization, which calculated a remainder using `len(data)` and raised `ZeroDivisionError`. Later refactors removed that runtime path, but the issue-specific regression only asserted the final row count after a subsequent add. This change makes the reported operation and its expected empty-table state explicit so the original defect remains directly covered. ## Validation - `uv run --extra tests pytest python/tests/test_table.py::test_create_table_without_data_with_vector_schema -q` - `uv --project python run --extra tests --extra dev ruff format --check python/python/tests/test_table.py` - `uv --project python run --extra tests --extra dev ruff check .` - `git diff --check` Fixes #1968 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 069527b21..1f2e14742 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -3489,8 +3489,8 @@ def test_create_table_empty_list_no_schema_error(mem_db: DBConnection): mem_db.create_table("test_empty_no_schema", data=[]) -def test_add_table_with_empty_embeddings(tmp_path): - """Test exact scenario from issue #1968 +def test_create_table_without_data_with_vector_schema(tmp_path): + """Test exact scenario from issue #1968. Regression test for issue #1968: https://github.com/lancedb/lancedb/issues/1968 @@ -3502,6 +3502,9 @@ def test_add_table_with_empty_embeddings(tmp_path): embedding: Vector(16) table = db.create_table("test", schema=MySchema) + assert table.count_rows() == 0 + assert table.schema == MySchema.to_arrow_schema() + table.add( [{"text": "bar", "embedding": [0.1] * 16}], on_bad_vectors="drop", From 99a68db78c149ceb3342be358c6b3178d18f0a2c Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:40:01 +0800 Subject: [PATCH 07/41] test(rust): cover concurrent appends during compaction (#3878) ## Summary - add a LanceDB core regression for compaction overlapping appends through separate table handles - verify concurrent commits preserve fragment ID order on an indexed table - run the follow-up compaction that exposed the original row-ID ordering failure and verify all rows remain ## Root cause Older Lance versions could reserve fragment IDs for compaction, allow concurrent appends to commit later IDs, and then commit the reserved compaction fragments at the end of the manifest. A later compaction could consequently receive row IDs out of order. Current Lance sorts fragments at the transaction boundary; this adds the missing LanceDB-level regression coverage for the Node-visible concurrency contract. ## Validation - `cargo fmt --all` - focused regression passed once with output and 20 repeated runs - `cargo test --quiet --features remote -p lancedb table::optimize::tests` (14 passed) - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` - `cargo test --quiet --features remote --tests` (867 passed, 1 ignored) Fixes #1498 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/table/optimize.rs | 90 ++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/rust/lancedb/src/table/optimize.rs b/rust/lancedb/src/table/optimize.rs index e29445b2d..3fdad0533 100644 --- a/rust/lancedb/src/table/optimize.rs +++ b/rust/lancedb/src/table/optimize.rs @@ -304,6 +304,96 @@ mod tests { assert_eq!(all_values, expected); } + #[tokio::test] + async fn test_compact_with_concurrent_add() { + const NUM_FRAGMENTS: usize = 5; + const ROWS_PER_FRAGMENT: i32 = 300; + + let tmpdir = tempfile::tempdir().unwrap(); + let conn = connect(tmpdir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(Int32Array::from_iter_values(0..ROWS_PER_FRAGMENT))], + ) + .unwrap(); + + let table = conn + .create_table("test_concurrent_compact", batch.clone()) + .execute() + .await + .unwrap(); + table + .create_index(&["id"], Index::BTree(BTreeIndexBuilder::default())) + .execute() + .await + .unwrap(); + for _ in 0..NUM_FRAGMENTS { + table.add(batch.clone()).execute().await.unwrap(); + } + + // Use separate handles so the two writes actually overlap, as they can + // when different Node connections operate on the same S3 table. + let compact_table = conn + .open_table("test_concurrent_compact") + .execute() + .await + .unwrap(); + let append_table = conn + .open_table("test_concurrent_compact") + .execute() + .await + .unwrap(); + let compact_task = tokio::spawn(async move { + compact_table + .optimize(OptimizeAction::Compact { + options: CompactionOptions { + target_rows_per_fragment: 1_000, + ..Default::default() + }, + remap_options: None, + }) + .await + }); + tokio::task::yield_now().await; + for _ in 0..NUM_FRAGMENTS { + append_table.add(batch.clone()).execute().await.unwrap(); + } + compact_task.await.unwrap().unwrap(); + + let table = conn + .open_table("test_concurrent_compact") + .execute() + .await + .unwrap(); + let dataset = table.dataset().unwrap().get().await.unwrap(); + let fragment_ids = dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id()) + .collect::>(); + assert!(fragment_ids.windows(2).all(|ids| ids[0] < ids[1])); + + // A second compaction exposed the original out-of-order row-id bug. + table + .optimize(OptimizeAction::Compact { + options: CompactionOptions { + target_rows_per_fragment: 1_000, + ..Default::default() + }, + remap_options: None, + }) + .await + .unwrap(); + assert_eq!( + table.count_rows(None).await.unwrap(), + ROWS_PER_FRAGMENT as usize * (NUM_FRAGMENTS * 2 + 1) + ); + } + #[tokio::test] async fn test_optimize_prune_versions() { let conn = connect("memory://").execute().await.unwrap(); From 123c921c4f6cf6ff350922d9236215d408efecb6 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:40:41 +0800 Subject: [PATCH 08/41] test(python): cover sliced nullable table search (#3875) ## Summary - add a Python regression for vector search over a sliced Arrow table with nullable scalar columns - verify the nearest row retains its non-null score values after the table is written ## Root cause Lance 0.19.2 deep-copied a validity bitmap without preserving its non-zero bit offset. For a sliced nullable table, scalar values and vectors began at the slice while the copied validity bitmap began at the parent table's first row. That made valid score values appear null even though the corresponding vector stayed intact. The upstream Lance repair is already present in the current dependency; this adds a LanceDB-level guard for the reported create/search path. ## Validation - reproduced on Python 3.12 with LanceDB 0.16.0, pylance 0.19.2, PyArrow 18.0.0, and Polars 1.14.0 - `uv run --project python --extra dev ruff format --check python/python/tests/test_table.py` - `uv run --project python --extra dev ruff check .` - `cd python && uv run --extra tests pytest python/tests/test_table.py::test_search_preserves_nulls_from_sliced_arrow_table -q` Fixes #1879 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 1f2e14742..ef59502d7 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -99,6 +99,30 @@ def test_basic(mem_db: DBConnection): assert table.to_arrow() == expected_data +def test_search_preserves_nulls_from_sliced_arrow_table(mem_db: DBConnection): + data = pa.table( + { + "id": [0, 1, 2, 3, 4], + "score_cn": [None, 22, None, 5, 8], + "score_mt": [None, 42, None, 5, 8], + "vector": [ + [20, 19, -1, -1], + [41, 38, 22, 42], + [10, 10, -1, -1], + [5, 5, 5, 5], + [8, 8, 8, 8], + ], + } + ).slice(1) + + table = mem_db.create_table("sliced_nullable", data=data) + result = table.search([41, 38, 22, 42]).limit(1).to_arrow() + + assert result["id"].to_pylist() == [1] + assert result["score_cn"].to_pylist() == [22] + assert result["score_mt"].to_pylist() == [42] + + def test_table_to_pandas_default_matches_arrow(tmp_db: DBConnection): pd = pytest.importorskip("pandas") data = pa.table({"id": [1, 2], "text": ["one", "two"]}) From f1f34dfdd33072319dffd821f69cb32c8e3a426f Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:41:11 +0800 Subject: [PATCH 09/41] fix(python): instruct dimension probe for instructor embeddings (#3874) ## Summary - pass an Instructor-compatible `[instruction, text]` pair when detecting embedding dimensions - add a regression test that verifies the dimension probe uses the configured source instruction ## Root cause `InstructorEmbeddingFunction.ndims()` encoded a bare string even though Instructor models require instruction/text pairs. With affected `sentence-transformers` versions, the bare input omitted `instruction_mask` and raised `KeyError` while defining the LanceDB schema. ## Validation - `uv run --extra tests pytest python/tests/test_embeddings.py -q` (`14 passed, 9 skipped`) - `uv run --project python --extra tests --extra dev ruff format --check python/python/lancedb/embeddings/instructor.py python/python/tests/test_embeddings.py` - `uv run --project python --extra tests --extra dev ruff check .` Fixes #2041 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/lancedb/embeddings/instructor.py | 3 +-- python/python/tests/test_embeddings.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/python/python/lancedb/embeddings/instructor.py b/python/python/lancedb/embeddings/instructor.py index 675a0139c..37ae1c296 100644 --- a/python/python/lancedb/embeddings/instructor.py +++ b/python/python/lancedb/embeddings/instructor.py @@ -101,8 +101,7 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction): @weak_lru(maxsize=1) def ndims(self): - model = self.get_model() - return model.encode("foo").shape[0] + return len(self.generate_embeddings([[self.source_instruction, "foo"]])[0]) def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]: return self.generate_embeddings([[self.query_instruction, query]]) diff --git a/python/python/tests/test_embeddings.py b/python/python/tests/test_embeddings.py index f2d8971c3..678270f19 100644 --- a/python/python/tests/test_embeddings.py +++ b/python/python/tests/test_embeddings.py @@ -64,6 +64,23 @@ def test_embedding_function(tmp_path): assert np.allclose(actual, expected) +def test_instructor_ndims_uses_instruction(): + instructor = get_registry().get("instructor").create() + model = MagicMock() + model.encode.return_value = np.zeros((1, 384)) + + with patch.object(type(instructor), "get_model", return_value=model): + assert instructor.ndims() == 384 + + model.encode.assert_called_once_with( + [[instructor.source_instruction, "foo"]], + batch_size=instructor.batch_size, + show_progress_bar=instructor.show_progress_bar, + normalize_embeddings=instructor.normalize_embeddings, + device=instructor.device, + ) + + def test_embedding_function_variables(): @register("variable-testing") class VariableTestingFunction(TextEmbeddingFunction): From 798e5364fb8ec24aca8b247bf7ed38f25745a688 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:42:20 +0800 Subject: [PATCH 10/41] test(python): cover VoyageAI text source routing (#3872) ## Summary - add fast regression coverage for VoyageAI `voyage-3` source embeddings - verify table text uses `client.embed` and never `client.multimodal_embed` ## Root cause The original VoyageAI source-embedding path treated table source values as images and always invoked the multimodal API. Production routing was corrected by later merged changes, but the table regression was covered only by API-gated slow tests. This test locks the corrected text routing into the regular unit suite. ## Validation - `cd python && uv run --extra tests pytest python/tests/test_voyageai_embeddings.py -q` - `uv run --project python --extra tests --extra dev ruff format --check python/python/tests/test_voyageai_embeddings.py` - `uv run --project python --extra tests --extra dev ruff check .` Fixes #2059 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_voyageai_embeddings.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/python/tests/test_voyageai_embeddings.py b/python/python/tests/test_voyageai_embeddings.py index ac1554cad..040cade1f 100644 --- a/python/python/tests/test_voyageai_embeddings.py +++ b/python/python/tests/test_voyageai_embeddings.py @@ -75,6 +75,22 @@ class TestVoyageAIModelRegistration: with pytest.raises(ValueError, match="not supported"): func.ndims() + def test_voyage3_source_embeddings_use_text_api(self, mock_voyageai_client): + """Regression test for text table data being sent to the multimodal API.""" + mock_voyageai_client.tokenize.return_value = [["hello", "world"]] + mock_voyageai_client.embed.return_value.embeddings = [[0.1] * 1024] + + registry = get_registry() + func = registry.get("voyageai").create(name="voyage-3") + + embeddings = func.compute_source_embeddings("hello world") + + assert embeddings == [[0.1] * 1024] + mock_voyageai_client.embed.assert_called_once_with( + texts=["hello world"], model="voyage-3", input_type="document" + ) + mock_voyageai_client.multimodal_embed.assert_not_called() + @pytest.mark.parametrize( "model_name", [ From 03b52e587788d4c57a8379b52408646c785b33e5 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:43:59 +0800 Subject: [PATCH 11/41] test(node): cover fixed-size list schemas with typed arrays (#3866) ## Summary - cover explicit FixedSizeList schemas populated from Float32Array values - verify the original vector.0 failure stays fixed across Arrow 15, 16, 17, and 18 ## Root cause and fix In v0.16, schema subset inference treated typed-array vectors as nested objects and looked up numeric paths such as vector.0, which do not exist in a FixedSizeList schema. Current typed-array handling correctly recognizes ArrayBuffer views as vector values instead of traversing their elements. This change adds the missing regression coverage for the reported explicit-schema path so that behavior cannot regress unnoticed. ## Validation - pnpm test __test__/arrow.test.ts --runInBand - pnpm lint - pnpm build - pnpm run docs - pnpm test --runInBand (681 passed, 5 skipped) Fixes #2134 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/arrow.test.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/nodejs/__test__/arrow.test.ts b/nodejs/__test__/arrow.test.ts index 9e20e3c04..c05849cb9 100644 --- a/nodejs/__test__/arrow.test.ts +++ b/nodejs/__test__/arrow.test.ts @@ -197,6 +197,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( expect(table.getChild("d")?.toJSON()).toEqual([9n, 10n, null]); }); + it("will use a provided FixedSizeList schema with typed array values", function () { + const schema = new Schema([ + new Field("text", new Utf8(), false), + new Field( + "vector", + new FixedSizeList(3, new Field("item", new Float32(), false)), + false, + ), + ]); + + const table = makeArrowTable( + [ + { + text: "foo", + vector: new Float32Array([1, 2, 3]), + }, + ], + { schema }, + ); + + expect(table.getChild("text")?.toJSON()).toEqual(["foo"]); + expect( + table + .getChild("vector") + ?.toJSON() + .map((value) => value.toJSON()), + ).toEqual([[1, 2, 3]]); + }); + it("will assume the column `vector` is FixedSizeList by default", async function () { const schema = new Schema([ new Field("a", new Float(Precision.DOUBLE), true), From 173f889d2afc35a84de5babdae1d38fb2e7be884 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:44:20 +0800 Subject: [PATCH 12/41] test(python): cover stale scalar prefilters in hybrid search (#3865) ## Summary - capture the stale-index state behind the reported fixed-size-binary panic: the vector and FTS indices cover newer fragments while the BTree prefilter does not - verify vector, FTS, and hybrid searches return matches from both scalar-indexed and unindexed fragments without panicking - preserve binding-level coverage for the Lance fix in https://github.com/lance-format/lance/pull/3768, which restricts incomplete scalar prefilters when search indices are further ahead The production root cause is in Lance and the current LanceDB dependency already contains that fix, so this change adds the missing LanceDB Python regression coverage. ## Validation - `cd python && uv run --no-sync pytest python/tests/test_hybrid_query.py::test_hybrid_query_with_stale_fixed_size_binary_prefilter -q` - `cd python && uv run --no-sync pytest python/tests/test_hybrid_query.py -q` - `python/.venv/bin/ruff check .` - `python/.venv/bin/ruff format --check python/python/tests/test_hybrid_query.py` Fixes #2370 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_hybrid_query.py | 82 +++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/python/python/tests/test_hybrid_query.py b/python/python/tests/test_hybrid_query.py index 65a7890bf..72dcaaa49 100644 --- a/python/python/tests/test_hybrid_query.py +++ b/python/python/tests/test_hybrid_query.py @@ -12,7 +12,7 @@ import pyarrow.compute as pc import pytest import pytest_asyncio -from lancedb.index import FTS +from lancedb.index import BTree, FTS, IvfPq from lancedb.table import AsyncTable, Table @@ -99,6 +99,86 @@ async def test_async_hybrid_query_filters(table: AsyncTable): assert result["text"].to_pylist() == ["cat", "b"] +@pytest.mark.asyncio +async def test_hybrid_query_with_stale_fixed_size_binary_prefilter( + tmpdir_factory, +): + tmp_path = str(tmpdir_factory.mktemp("stale_scalar_prefilter")) + db = await lancedb.connect_async(tmp_path) + + def fixed_size_binary(value: int) -> bytes: + return value.to_bytes(16, byteorder="big") + + num_rows = 1000 + data = pa.table( + { + "space_id": pa.array( + [fixed_size_binary(i) for i in range(num_rows)], + type=pa.binary(16), + ), + "text": ["book"] * num_rows, + "vector": pa.array( + [[float(i), float(i)] for i in range(num_rows)], + type=pa.list_(pa.float32(), 2), + ), + } + ) + table = await db.create_table("test", data) + await table.create_index( + "vector", config=IvfPq(num_partitions=4, num_sub_vectors=2) + ) + await table.create_index("space_id", config=BTree()) + await table.create_index("text", config=FTS(with_position=False)) + + # Advance the search indices without advancing the scalar index. This is the + # state that previously let hybrid search use an incomplete scalar prefilter. + await table.add(data) + lance_dataset = await table.to_lance() + lance_dataset.optimize.optimize_indices(index_names=["vector_idx", "text_idx"]) + await table.checkout_latest() + + scalar_stats = await table.index_stats("space_id_idx") + assert scalar_stats is not None + assert scalar_stats.num_indexed_rows == num_rows + assert scalar_stats.num_unindexed_rows == num_rows + + for index_name in ["vector_idx", "text_idx"]: + search_stats = await table.index_stats(index_name) + assert search_stats is not None + assert search_stats.num_indexed_rows == num_rows * 2 + assert search_stats.num_unindexed_rows == 0 + + matching_ids = [5, 10, 15, 20, 25, 30] + literals = [ + f"arrow_cast(0x{fixed_size_binary(i).hex()}, 'FixedSizeBinary(16)')" + for i in matching_ids + ] + predicate = f"space_id IN ({', '.join(literals)})" + expected_ids = sorted(fixed_size_binary(i) for i in matching_ids for _ in range(2)) + + vector_query = ( + table.query().where(predicate).nearest_to([5.0, 5.0]).limit(num_rows * 2) + ) + vector_results = await vector_query.to_arrow() + assert sorted(vector_results["space_id"].to_pylist()) == expected_ids + + fts_query = ( + table.query().where(predicate).nearest_to_text("book").limit(num_rows * 2) + ) + fts_results = await fts_query.to_arrow() + assert sorted(fts_results["space_id"].to_pylist()) == expected_ids + + hybrid_results = await ( + table.query() + .where(predicate) + .nearest_to([5.0, 5.0]) + .nearest_to_text("book") + .limit(num_rows * 2) + .to_arrow() + ) + assert sorted(hybrid_results["space_id"].to_pylist()) == expected_ids + + @pytest.mark.asyncio async def test_async_hybrid_query_default_limit(table: AsyncTable): # add 10 new rows From ac8b28c010360322557d31ca49bcd3826745b73c Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:44:38 +0800 Subject: [PATCH 13/41] fix(python): support nullable pandas merge input (#3864) ## Summary - add an end-to-end Python regression for pandas DataFrame inputs merged into a table created from a Pydantic model - verify reordered, nullable Arrow source fields can update and insert into a non-nullable target schema when the values contain no nulls ## Root cause Lance merge_insert previously compared source schema nullability with the target, unlike add. The upstream fix now pinned by LanceDB ignores declared nullability during schema compatibility and validates actual null values at write time. LanceDB lacked regression coverage for the full pandas-to-Pydantic path, so this test locks in the correct behavior without falsifying the input schema nullability. ## Validation - 5 focused merge-insert tests passed - Ruff lint passed for the repository - Ruff format check passed for the changed file - git diff --check passed Fixes #2366 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index ef59502d7..d7748095e 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -2387,6 +2387,55 @@ def test_merge_insert(mem_db: DBConnection): ) +def test_merge_insert_nullable_pandas_into_pydantic_schema(mem_db: DBConnection): + # Regression test for https://github.com/lancedb/lancedb/issues/2366 + pd = pytest.importorskip("pandas") + + class Document(LanceModel): + id: int + title: str + content: str + + table = mem_db.create_table("documents", schema=Document) + table.add( + pd.DataFrame( + { + "title": ["Old title", "Unchanged"], + "id": [2, 3], + "content": ["Old content", "Keep this"], + } + ) + ) + + # Pandas produces nullable Arrow fields, in an order that differs from the + # non-nullable Pydantic schema. This is valid as long as the data has no nulls. + new_data = pd.DataFrame( + { + "title": ["Inserted", "Updated"], + "id": [1, 2], + "content": ["New row", "New content"], + } + ) + result = ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(new_data) + ) + + assert result.num_inserted_rows == 1 + assert result.num_updated_rows == 1 + expected = pa.Table.from_pylist( + [ + {"id": 1, "title": "Inserted", "content": "New row"}, + {"id": 2, "title": "Updated", "content": "New content"}, + {"id": 3, "title": "Unchanged", "content": "Keep this"}, + ], + schema=Document.to_arrow_schema(), + ) + assert table.to_arrow().sort_by("id") == expected + + def test_merge_insert_by_source_delete_expr(mem_db: DBConnection): table = mem_db.create_table( "my_table", From 2c06a48bd848f400a690b6abc15c3fec662a1418 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:45:10 +0800 Subject: [PATCH 14/41] test(python): cover Arrow buffer release after add (#3860) ## Summary - add deterministic regression coverage that `Table.add()` releases backing Arrow buffers without cyclic garbage collection - track the foreign buffer owner rather than RSS, separating live input retention from allocator high-water behavior - preserve the bounded-lifetime behavior of the Scannable writer that superseded the historical preprocessing path ## Root cause The historical Python preprocessing/write path produced a high allocator RSS while ingesting very wide IPC batches. The current Scannable writer releases each input buffer when `Table.add()` completes; remaining RSS is allocator high-water rather than a live Arrow reference. The resolved behavior had no regression coverage, so a future native lifetime regression could silently reintroduce the original failure mode. ## Validation - `uv run --extra tests --extra dev maturin develop` - `uv run --project python --extra tests pytest python/python/tests/test_table.py::test_add python/python/tests/test_table.py::test_add_releases_arrow_buffers_without_gc -q` - `uv run --project python --extra dev ruff format --check python/python/tests/test_table.py` - `uv run --project python --extra dev ruff check .` Fixes #2512 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index d7748095e..70a52b100 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -2,10 +2,13 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import ctypes +import gc import os import sys import threading import warnings +import weakref from concurrent.futures import ThreadPoolExecutor from datetime import date, datetime, timedelta from time import sleep @@ -459,6 +462,38 @@ def test_add(mem_db: DBConnection): _add(table, schema) +def test_add_releases_arrow_buffers_without_gc(mem_db: DBConnection): + """Regression test for https://github.com/lancedb/lancedb/issues/2512.""" + schema = pa.schema([pa.field("x", pa.int64())]) + table = mem_db.create_table("test_add_releases_arrow_buffers", schema=schema) + + class BufferOwner: + def __init__(self, size: int): + self.memory = ctypes.create_string_buffer(size) + + owner_refs = [] + gc_was_enabled = gc.isenabled() + gc.disable() + try: + for _ in range(3): + size = 8 * 1024 + owner = BufferOwner(size) + arrow_buffer = pa.foreign_buffer( + ctypes.addressof(owner.memory), size, owner + ) + array = pa.Array.from_buffers(pa.int64(), 1024, [None, arrow_buffer]) + batch = pa.RecordBatch.from_arrays([array], schema=schema) + owner_refs.append(weakref.ref(owner)) + + table.add(batch) + del batch, array, arrow_buffer, owner + + assert all(owner_ref() is None for owner_ref in owner_refs) + finally: + if gc_was_enabled: + gc.enable() + + def test_add_write_parallelism(mem_db: DBConnection): schema = pa.schema([pa.field("id", pa.int64())]) table = mem_db.create_table("test", schema=schema) From 3af51541a02963563baac208a7bd768ac241a785 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:46:49 +0800 Subject: [PATCH 15/41] test(rust): cover fixed-size binary merge insert regression (#3854) ## Summary - add a LanceDB regression for `merge_insert` with a non-nullable `FixedSizeBinary` column - exercise matched updates, unmatched inserts, and source-missing deletes - assert the exact merge statistics and final row count ## Root cause The Arrow `take` kernel previously ignored nulls in the index array for `FixedSizeBinary`. DataFusion uses that kernel while constructing outer-join results, so the join behind `when_not_matched_by_source_delete` could place invalid values into non-nullable columns. The current Arrow dependency contains the upstream fix; this test locks the corrected behavior at the LanceDB API boundary. ## Validation - `cargo fmt --all -- --check` - `cargo test --quiet --features remote --tests` - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` Fixes #2869 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/table/merge.rs | 71 ++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 13a633c67..82a1d1473 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -315,7 +315,10 @@ pub(crate) async fn execute_merge_insert( #[cfg(test)] mod tests { - use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader}; + use arrow_array::builder::FixedSizeBinaryBuilder; + use arrow_array::{ + Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, UInt64Array, + }; use arrow_schema::{DataType, Field, Schema}; use std::sync::Arc; @@ -337,6 +340,42 @@ mod tests { Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema)) } + fn fixed_size_binary_merge_batch( + id_range: std::ops::Range, + price: u64, + ) -> Box { + let ids = id_range.collect::>(); + let mut id_builder = FixedSizeBinaryBuilder::new(16); + for id in &ids { + let mut bytes = [0; 16]; + bytes[..8].copy_from_slice(&id.to_le_bytes()); + id_builder.append_value(bytes).unwrap(); + } + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::FixedSizeBinary(16), false), + Field::new("id_as_int", DataType::UInt64, false), + Field::new("name", DataType::Utf8, false), + Field::new("market", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(id_builder.finish()), + Arc::new(UInt64Array::from_iter_values(ids.iter().copied())), + Arc::new(StringArray::from_iter_values( + ids.iter().map(|id| format!("name{id}")), + )), + Arc::new(StringArray::from_iter_values(std::iter::repeat_n( + format!("market_{price}"), + ids.len(), + ))), + ], + ) + .unwrap(); + Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema)) + } + #[tokio::test] async fn test_merge_insert() { let conn = connect("memory://").execute().await.unwrap(); @@ -388,6 +427,36 @@ mod tests { ); } + #[tokio::test] + async fn test_merge_insert_fixed_size_binary_non_nullable() { + // Regression test for #2869: an unrelated FixedSizeBinary column used to corrupt the + // outer join that implements when_not_matched_by_source_delete. + let conn = connect("memory://").execute().await.unwrap(); + let table = conn + .create_table( + "fixed_size_binary_merge", + fixed_size_binary_merge_batch(0..256, 100), + ) + .execute() + .await + .unwrap(); + + let mut merge_insert = table.merge_insert(&["id_as_int"]); + merge_insert + .when_matched_update_all(None) + .when_not_matched_insert_all() + .when_not_matched_by_source_delete(None); + let result = merge_insert + .execute(fixed_size_binary_merge_batch(100..356, 200)) + .await + .unwrap(); + + assert_eq!(result.num_updated_rows, 156); + assert_eq!(result.num_inserted_rows, 100); + assert_eq!(result.num_deleted_rows, 100); + assert_eq!(table.count_rows(None).await.unwrap(), 256); + } + #[tokio::test] async fn test_merge_insert_use_index() { let conn = connect("memory://").execute().await.unwrap(); From 0ba82873c5128f45ce6fbea75eb0135bcd08ead4 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:47:29 +0800 Subject: [PATCH 16/41] fix(python): cover nullable list v2.2 decoding (#3853) ## Summary - add a minimized regression for mostly-null `list` data at the v2.2 structural page boundary - verify scans preserve all 64,885 rows, including 64,668 null list values ## Root cause Lance 3.0.0 sliced repetition/definition state using top-level row offsets in the complex all-null decoder. At this page boundary, the list and validity children were materialized at different lengths. The current Lance dependency contains the upstream decoder repair; this test locks that behavior into the LanceDB Python suite without duplicating decoder logic. ## Validation - reproduced the attached 1,892,466-row case on `lancedb==0.30.0` with `expected 1024 got 285` - verified the full attachment reads on the current branch - `python/.venv/bin/ruff format --check python/python/tests/test_table.py` - `python/.venv/bin/ruff check .` - `cd python && uv run --extra tests pytest python/tests/test_table.py::test_read_mostly_null_list_v2_2_page_boundary -q` - `cd python && uv run --extra tests pytest python/tests/test_table.py -q` (137 passed) Fixes #3194 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 70a52b100..832b70569 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -1884,6 +1884,33 @@ def test_add_nullable_struct_with_none(mem_db: DBConnection): assert result.column("data").to_pylist() == [{"x": 1.0}, None] +def test_read_mostly_null_list_v2_2_page_boundary(tmp_path): + # Regression test for #3194. This row/value count crosses a v2.2 structural + # encoding page boundary where Lance 3.0.0 sliced repetition/definition + # levels by row offset and decoded child arrays at different lengths. + num_rows = 64_885 + num_values = 217 + list_type = pa.list_(pa.float32()) + source = pa.table( + { + "id": np.arange(num_rows, dtype=np.int64), + "coords": pa.array( + [[1.0, 2.0, 3.0, 4.0]] * num_values + [None] * (num_rows - num_values), + type=list_type, + ), + } + ) + db = lancedb.connect( + tmp_path, + storage_options={"new_table_data_storage_version": "2.2"}, + ) + table = db.create_table("test_sparse_nullable_list", data=source) + + result = table.search().select(["id", "coords"]).limit(num_rows).to_arrow() + + assert result.equals(source) + + def test_add_with_integer_embeddings_preserves_casting(mem_db: DBConnection): class Schema(LanceModel): text: str From c1a3fa7f51057160f930d4a86e5c96cc18993ef5 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:48:01 +0800 Subject: [PATCH 17/41] fix(python): preserve repeated indexed merge inserts (#3850) ## Summary - add a Python regression test for two partial-schema merge inserts against the same BTree-indexed rows - verify repeated updates retain one copy of every row and the final update values ## Root cause Lance 4.0, used by LanceDB 0.30.2, removed a rewritten fragment from the index bitmap while stale BTree entries for that fragment remained searchable. The next merge found each target through both the stale index and the unindexed-fragment scan, producing the ambiguous-match error. Lance fixed the root cause in lance-format/lance#6563 by applying the fragment-bitmap allow-list to index results, and the Lance release pinned by current LanceDB includes that fix. This test preserves the corrected behavior through the Python API. ## Validation - `cd python && uv run --extra tests pytest python/tests/test_table.py -k merge_insert -q` (9 passed) - `cd python && uv run --extra tests --extra dev ruff format --check python/tests/test_table.py` - `cd python && uv run --extra tests --extra dev ruff check python/tests/test_table.py` Repository-wide Ruff also reports 20 pre-existing violations in untouched CI and plugin scripts. Fixes #3280 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 832b70569..f20f42617 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -2598,6 +2598,36 @@ def test_merge_insert_subschema(mem_db: DBConnection, data_format): assert table.to_arrow().sort_by("id") == expected +def test_repeated_partial_merge_insert_with_scalar_index(mem_db: DBConnection): + def make_batch(start: int) -> pa.Table: + return pa.table( + { + "id": [f"id-{i:04}" for i in range(start, start + 100)], + "category": ["A"] * 100, + "value_a": [float(i) for i in range(start, start + 100)], + "value_b": [float(i) / 10 for i in range(100)], + } + ) + + table = mem_db.create_table("my_table", data=make_batch(0)) + table.add(make_batch(100)) + table.add(make_batch(200)) + table.create_index("id", config=BTree()) + + ids = [f"id-{i:04}" for i in range(100, 200)] + for value in (999.0, 888.0): + result = ( + table.merge_insert("id") + .when_matched_update_all() + .execute(pa.table({"id": ids, "value_a": [value] * 100})) + ) + assert result.num_updated_rows == 100 + + actual = table.to_arrow().sort_by("id") + assert actual.num_rows == 300 + assert actual["value_a"].to_pylist()[100:200] == [888.0] * 100 + + @pytest.mark.asyncio async def test_merge_insert_async(mem_db_async: AsyncConnection): data = pa.table({"a": [1, 2, 3], "b": ["a", "b", "c"]}) From 772bdeced8b4a2441c8c54d54d2a4a713a943855 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:48:24 +0800 Subject: [PATCH 18/41] fix(rust): prevent vector optimize regression after deletes (#3848) ## Summary - Adds a LanceDB regression for stable row IDs, scattered deletes, IVF_RQ, and default index optimization. - Verifies optimization completes and preserves the expected live-row count. ## Root cause Lance 3.0.1 built the stable-row-ID address list by dropping deleted IDs while retaining the original ID list. The subsequent positional zip misaligned IDs and addresses, so vector partition joins requested deleted rows and failed with batch.num_rows() != chunk.len(). Lance PR https://github.com/lance-format/lance/pull/7704 corrected the generic filter, and the LanceDB dependency currently pinned on main contains that correction. ## Fix Add regression coverage at the Rust Table optimize surface using the IVF_RQ configuration from the report. This locks the upstream correction into the LanceDB workflow that originally crashed. ## Validation - cargo fmt --all -- --check - cargo test --quiet --features remote -p lancedb table::optimize::tests (14 passed) - cargo check --quiet --features remote --tests --examples - cargo clippy --quiet --features remote --tests --examples -p lancedb Fixes #3330 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/table/optimize.rs | 59 +++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/rust/lancedb/src/table/optimize.rs b/rust/lancedb/src/table/optimize.rs index 3fdad0533..4ad58cffb 100644 --- a/rust/lancedb/src/table/optimize.rs +++ b/rust/lancedb/src/table/optimize.rs @@ -214,12 +214,17 @@ pub(crate) async fn execute_optimize( #[cfg(test)] mod tests { - use arrow_array::{Int32Array, RecordBatch, StringArray}; + use arrow_array::{ + Array, FixedSizeListArray, Float32Array, Int32Array, RecordBatch, StringArray, + }; use arrow_schema::{DataType, Field, Schema}; + use lance_arrow::FixedSizeListArrayExt; use rstest::rstest; use std::sync::Arc; use crate::connect; + use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS; + use crate::index::vector::IvfRqIndexBuilder; use crate::index::{Index, scalar::BTreeIndexBuilder}; use crate::query::ExecutableQuery; use crate::table::{CompactionOptions, OptimizeAction, OptimizeStats}; @@ -532,6 +537,58 @@ mod tests { assert_eq!(final_row_count, 200); } + #[tokio::test] + async fn test_optimize_vector_index_after_delete_with_stable_row_ids() { + const NUM_ROWS: i32 = 400; + const DIMENSION: i32 = 32; + + let conn = connect("memory://").execute().await.unwrap(); + let vectors = FixedSizeListArray::try_new_from_values( + Float32Array::from_iter_values((0..NUM_ROWS).flat_map(|id| { + (0..DIMENSION).map(move |offset| ((id as f32 * 0.1) + (offset as f32 * 0.3)).sin()) + })), + DIMENSION, + ) + .unwrap(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("vector", vectors.data_type().clone(), false), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from_iter_values(0..NUM_ROWS)), + Arc::new(vectors), + ], + ) + .unwrap(); + let table = conn + .create_table("test_vector_index_optimize_after_delete", batch) + .storage_option(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true") + .execute() + .await + .unwrap(); + + table + .create_index( + &["vector"], + Index::IvfRq(IvfRqIndexBuilder::default().num_partitions(4)), + ) + .execute() + .await + .unwrap(); + table.delete("id % 3 = 0").await.unwrap(); + + // Regression test for #3330: deleted stable row IDs used to become + // misaligned with row addresses while joining small IVF partitions. + table + .optimize(OptimizeAction::Index(Default::default())) + .await + .unwrap(); + + assert_eq!(table.count_rows(None).await.unwrap(), 266); + } + #[tokio::test] async fn test_optimize_all() { let conn = connect("memory://").execute().await.unwrap(); From b20696ef9ca2062165417d68c0a891260b3f8e0b Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:48:56 +0800 Subject: [PATCH 19/41] fix(remote): validate cloud DNS hostnames (#3845) ## Summary - validate the generated LanceDB Cloud hostname during connection setup - return a clear invalid-input error for empty, overlong, or oversized DNS names before network resolution - add Rust and Python regression coverage for malformed `db://` authorities ## Root cause The `db://` authority and region were interpolated into the Cloud API hostname without DNS length validation. Empty or overlong labels therefore reached the resolver and surfaced as an opaque IDNA `UnicodeError` instead of a useful connection error. ## Validation - `cargo test --quiet --features remote -p lancedb test_rejects_invalid_cloud_dns_hostname --lib` - `cargo check --quiet --features remote --tests --examples` - `uv run --no-sync --extra tests pytest python/tests/test_remote_db.py::test_async_remote_db python/tests/test_remote_db.py::test_connect_rejects_invalid_cloud_dns_hostname -q` - `cargo fmt --all -- --check` - `ruff check .` - `ruff format --check python/python/tests/test_remote_db.py` Fixes #799 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_remote_db.py | 6 +++ rust/lancedb/src/remote/client.rs | 60 ++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index d5d3569d3..ce8d5bd6e 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -35,6 +35,12 @@ def make_mock_http_handler(handler): return MockLanceDBHandler +@pytest.mark.parametrize("db_name", ["a" * 64, "invalid..database"]) +def test_connect_rejects_invalid_cloud_dns_hostname(db_name): + with pytest.raises(ValueError, match="DNS labels must contain 1 to 63 bytes"): + lancedb.connect(f"db://{db_name}", api_key="fake") + + @contextlib.contextmanager def mock_lancedb_connection(handler): with http.server.HTTPServer( diff --git a/rust/lancedb/src/remote/client.rs b/rust/lancedb/src/remote/client.rs index 9e34fca9f..57dd89890 100644 --- a/rust/lancedb/src/remote/client.rs +++ b/rust/lancedb/src/remote/client.rs @@ -373,6 +373,37 @@ pub fn parse_db_url(db_url: &str) -> Result { Ok(ParsedDbUrl { db_name, db_prefix }) } +fn validate_dns_hostname(hostname: &str) -> Result<()> { + let ascii_hostname = match url::Host::parse(hostname) { + Ok(url::Host::Domain(hostname)) => hostname, + Ok(_) => { + return Err(Error::InvalidInput { + message: "LanceDB Cloud database URI or region produced a non-DNS hostname" + .to_string(), + }); + } + Err(err) => { + return Err(Error::InvalidInput { + message: format!( + "LanceDB Cloud database URI or region produced an invalid hostname: {err}" + ), + }); + } + }; + + if ascii_hostname.len() > 253 + || ascii_hostname + .split('.') + .any(|label| label.is_empty() || label.len() > 63) + { + return Err(Error::InvalidInput { + message: "LanceDB Cloud database URI or region produced an invalid hostname: DNS labels must contain 1 to 63 bytes and the full hostname must not exceed 253 bytes".to_string(), + }); + } + + Ok(()) +} + impl RestfulLanceDbClient { fn get_timeout(passed: Option, env_var: &str) -> Result> { if let Some(passed) = passed { @@ -480,7 +511,11 @@ impl RestfulLanceDbClient { let host = match host_override { Some(host_override) => host_override, - None => format!("https://{}.{}.api.lancedb.com", parsed_url.db_name, region), + None => { + let hostname = format!("{}.{}.api.lancedb.com", parsed_url.db_name, region); + validate_dns_hostname(&hostname)?; + format!("https://{hostname}") + } }; debug!("Created client for host: {}", host); let retry_config = client_config.retry_config.clone().try_into()?; @@ -1157,6 +1192,29 @@ mod tests { assert_eq!(headers.get("x-api-key").unwrap(), "api-key"); } + #[test] + fn test_rejects_invalid_cloud_dns_hostname() { + let invalid_database_names = ["a".repeat(64), "invalid..database".to_string()]; + + for db_name in invalid_database_names { + let parsed_url = parse_db_url(&format!("db://{db_name}")).unwrap(); + let error = RestfulLanceDbClient::::try_new( + &parsed_url, + "us-east-1", + None, + HeaderMap::new(), + ClientConfig::default(), + None, + ) + .unwrap_err(); + + assert!( + matches!(error, Error::InvalidInput { ref message } if message.contains("DNS labels must contain 1 to 63 bytes")), + "unexpected error: {error}" + ); + } + } + // Test implementation of HeaderProvider #[derive(Debug, Clone)] struct TestHeaderProvider { From cc0139c136742e18d4a4038ca021d3725255110e Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:49:13 +0800 Subject: [PATCH 20/41] test(node): cover foreign Float64 vector schema workflow (#3844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - add an end-to-end regression for schemas created by a different Apache Arrow package instance - cover seeded table creation, filtered scanning, and Float64 vector search across Arrow 15–18 ## Root cause Apache Arrow's runtime identity checks historically rejected schemas created by another installed Arrow instance, producing the constructor failures reported in the issue. LanceDB's peer dependency and foreign-schema sanitization now handle that boundary, but the complete reported workflow was only covered by separate unit tests. This regression keeps the repaired behavior protected end to end. ## Validation - `pnpm exec jest --runInBand __test__/table.test.ts` (281 passed) - `pnpm lint-ci` - `pnpm build` - `pnpm run docs` Fixes #882 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/table.test.ts | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 4cad365af..15d6e0804 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -86,6 +86,44 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( await expect(table.countRows()).resolves.toBe(3); }); + it("should support a foreign Float64 vector schema end to end", async () => { + const conn = await connect(tmpDir.name); + const schema = new arrow.Schema([ + new arrow.Field("resource_id", new arrow.Int32(), false), + new arrow.Field( + "vector", + new arrow.FixedSizeList( + 3, + new arrow.Field("value", new arrow.Float64(), true), + ), + false, + ), + ]); + const data = [ + { + // biome-ignore lint/style/useNamingConvention: matches the reported schema + resource_id: 0, + vector: [0.1, 0.1, 0.1], + }, + ]; + + const resources = await conn.createTable("resources", data, { schema }); + + const existing = await resources + .query() + .where("resource_id = 0") + .limit(1) + .toArray(); + expect(existing).toHaveLength(1); + + const matched = await resources + .search(Float64Array.from(data[0].vector)) + .limit(1) + .toArray(); + expect(matched).toHaveLength(1); + expect(matched[0]["resource_id"]).toBe(0); + }); + it("should support branches", async () => { await table.add([{ id: 1 }]); expect(await table.countRows()).toBe(1); From e6444ecc058e909893bb0a2974e37ff45738da0b Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:49:49 +0800 Subject: [PATCH 21/41] fix(rust): handle missing mirrored copy sources (#3843) ## Summary - treat `NotFound` from the mirrored secondary copy as a cache miss while preserving every other secondary error - perform the durable primary copy after either a successful secondary copy or a secondary cache miss - cover both an initially missing secondary manifest and eviction immediately before the secondary copy ## Root cause Readers can use process-local secondary stores that do not contain a staging manifest written by another process, or that evict it before finalization. `MirroringObjectStore::copy_opts` propagated that secondary `NotFound`, so older object_store versions could loop indefinitely and the locked version aborted before performing the durable primary copy. ## Validation - `cargo fmt --all -- --check` - `cargo test --quiet --features remote -p lancedb io::object_store::test::test_copy_when -- --nocapture` - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` - `cargo test --quiet --features remote --tests` Fixes #1176 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/io/object_store.rs | 147 +++++++++++++++++++++++++++- 1 file changed, 143 insertions(+), 4 deletions(-) diff --git a/rust/lancedb/src/io/object_store.rs b/rust/lancedb/src/io/object_store.rs index d27357b82..d594bd857 100644 --- a/rust/lancedb/src/io/object_store.rs +++ b/rust/lancedb/src/io/object_store.rs @@ -132,9 +132,14 @@ impl ObjectStore for MirroringObjectStore { if to.primary_only() { self.primary.copy_opts(from, to, options).await } else { - self.secondary.copy_opts(from, to, options.clone()).await?; - self.primary.copy_opts(from, to, options).await?; - Ok(()) + // The secondary store can be process-local and less durable than the + // primary, so a source written by another process may not exist here + // or may be evicted before the copy begins. + match self.secondary.copy_opts(from, to, options.clone()).await { + Ok(()) | Err(Error::NotFound { .. }) => {} + Err(err) => return Err(err), + } + self.primary.copy_opts(from, to, options).await } } } @@ -192,7 +197,8 @@ mod test { use futures::TryStreamExt; use lance::{dataset::WriteParams, io::ObjectStoreParams}; use lance_testing::datagen::{BatchGenerator, IncrementingInt32, RandomVector}; - use object_store::local::LocalFileSystem; + use object_store::{local::LocalFileSystem, memory::InMemory}; + use std::time::Duration; use tempfile; use crate::{ @@ -201,6 +207,139 @@ mod test { table::WriteOptions, }; + #[derive(Debug)] + struct EvictBeforeCopyStore { + inner: Arc, + } + + impl std::fmt::Display for EvictBeforeCopyStore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "EvictBeforeCopyStore") + } + } + + #[async_trait] + impl ObjectStore for EvictBeforeCopyStore { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + options: PutOptions, + ) -> Result { + self.inner.put_opts(location, payload, options).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + options: PutMultipartOptions, + ) -> Result> { + self.inner.put_multipart_opts(location, options).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> Result { + self.inner.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, Result>, + ) -> BoxStream<'static, Result> { + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> { + self.inner.delete(from).await?; + self.inner.copy_opts(from, to, options).await + } + } + + #[tokio::test] + async fn test_copy_when_source_is_missing_from_secondary() { + let primary_dir = tempfile::tempdir().unwrap(); + let secondary_dir = tempfile::tempdir().unwrap(); + let primary: Arc = + Arc::new(LocalFileSystem::new_with_prefix(primary_dir.path()).unwrap()); + let secondary: Arc = + Arc::new(LocalFileSystem::new_with_prefix(secondary_dir.path()).unwrap()); + let store = MirroringObjectStore { + primary: primary.clone(), + secondary: secondary.clone(), + }; + let staging = Path::from("_versions/1.manifest-staging"); + let finalized = Path::from("_versions/1.manifest"); + + primary + .put(&staging, "manifest contents".into()) + .await + .unwrap(); + + tokio::time::timeout(Duration::from_secs(5), store.copy(&staging, &finalized)) + .await + .expect("copy should not hang when the secondary source is missing") + .unwrap(); + + let copied = primary + .get(&finalized) + .await + .unwrap() + .bytes() + .await + .unwrap(); + assert_eq!(copied, "manifest contents"); + assert!(matches!( + secondary.head(&finalized).await, + Err(Error::NotFound { .. }) + )); + } + + #[tokio::test] + async fn test_copy_when_secondary_source_disappears_after_head() { + let primary: Arc = Arc::new(InMemory::new()); + let secondary_inner: Arc = Arc::new(InMemory::new()); + let secondary: Arc = Arc::new(EvictBeforeCopyStore { + inner: secondary_inner.clone(), + }); + let store = MirroringObjectStore { + primary: primary.clone(), + secondary, + }; + let staging = Path::from("_versions/1.manifest-staging"); + let finalized = Path::from("_versions/1.manifest"); + + primary + .put(&staging, "manifest contents".into()) + .await + .unwrap(); + secondary_inner + .put(&staging, "manifest contents".into()) + .await + .unwrap(); + + store.copy(&staging, &finalized).await.unwrap(); + + let copied = primary + .get(&finalized) + .await + .unwrap() + .bytes() + .await + .unwrap(); + assert_eq!(copied, "manifest contents"); + assert!(matches!( + secondary_inner.head(&finalized).await, + Err(Error::NotFound { .. }) + )); + } + // This test is ignored because lance 3.0 introduced LocalWriter optimization // that bypasses the object store wrapper for local writes. The mirroring feature // still works for remote/cloud storage, but can't be tested with local storage. From 1493ece3de78fcae62b0620ae12383a7424af97b Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:50:05 +0800 Subject: [PATCH 22/41] test(node): cover remote table server errors (#3841) ## Summary - add a public Node API regression test for JSON server errors from remote table operations - verify countRows reports the server message instead of an ArrayBuffer decoding TypeError ## Root cause and fix The former TypeScript remote HTTP client passed an Axios-decoded JSON error object to TextDecoder, which masked the server response with an ArrayBuffer TypeError. The current Rust-backed remote client consumes non-success response bodies as text and propagates them through the Node error chain. This test exercises that corrected path through countRows and prevents the original failure from regressing. ## Validation - pnpm build - pnpm lint-ci - pnpm test --runInBand __test__/remote.test.ts - pnpm run docs Fixes #825 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/remote.test.ts | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 89a9e992c..e766b3d2a 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -170,6 +170,38 @@ describe("remote connection", () => { ); }); + it("surfaces JSON server errors from remote table operations", async () => { + await withMockDatabase( + (req, res) => { + const path = req.url ?? ""; + if (path.endsWith("/describe/")) { + res.writeHead(200, { "Content-Type": "application/json" }).end( + JSON.stringify({ + name: "broken_table", + version: 1, + schema: { fields: [] }, + }), + ); + return; + } + + if (path.endsWith("/count_rows/")) { + res + .writeHead(400, { "Content-Type": "application/json" }) + .end(JSON.stringify({ error: "count rows failed" })); + return; + } + + res.writeHead(404).end(); + }, + async (db) => { + const table = await db.openTable("broken_table"); + + await expect(table.countRows()).rejects.toThrow("count rows failed"); + }, + ); + }); + it("should pass on requested extra headers", async () => { await withMockDatabase( (req, res) => { From 62fe413a523b06775fe93d410c68994f3002dcc7 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Thu, 6 Aug 2026 01:53:09 -0700 Subject: [PATCH 23/41] fix: percent-encode index names in per-index remote REST paths (#3840) Nothing validates index names, so a `/` in one is reachable, and the remote client interpolates it straight into the URL, splitting the path so the router 404s. The index then reads back as missing and cannot be dropped, while `create_index` keeps succeeding because it sends the name in the body. Encode at the three affected sites, mirroring `fetch_blob_files`. The shared Rust client covers all bindings. --- rust/lancedb/src/remote/table.rs | 50 ++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 5fefabeb2..29b6b698d 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2791,9 +2791,10 @@ impl BaseTable for RemoteTable { } async fn index_stats(&self, index_name: &str) -> Result> { + let encoded_name = urlencoding::encode(index_name); let mut request = self.post_read(&format!( - "/v1/table/{}/index/{}/stats/", - self.identifier, index_name + "/v1/table/{}/index/{encoded_name}/stats/", + self.identifier )); let version = self.current_version().await; let mut body = serde_json::json!({ "version": version }); @@ -2820,9 +2821,10 @@ impl BaseTable for RemoteTable { } async fn drop_index(&self, index_name: &str) -> Result<()> { + let encoded_name = urlencoding::encode(index_name); let request = self.apply_branch_query(self.client.post(&format!( - "/v1/table/{}/index/{}/drop/", - self.identifier, index_name + "/v1/table/{}/index/{encoded_name}/drop/", + self.identifier ))); let (request_id, response) = self.send(request, true).await?; if response.status() == StatusCode::NOT_FOUND { @@ -2835,9 +2837,10 @@ impl BaseTable for RemoteTable { } async fn prewarm_index(&self, index_name: &str) -> Result<()> { + let encoded_name = urlencoding::encode(index_name); let request = self.client.post(&format!( - "/v1/table/{}/index/{}/prewarm/", - self.identifier, index_name + "/v1/table/{}/index/{encoded_name}/prewarm/", + self.identifier )); let (request_id, response) = self.send(request, true).await?; if response.status() == StatusCode::NOT_FOUND { @@ -6489,6 +6492,41 @@ mod tests { assert!(matches!(e, Error::IndexNotFound { .. })); } + /// Index names are unvalidated, so reserved characters must be + /// percent-encoded or they restructure the request path. + #[tokio::test] + async fn test_per_index_paths_encode_reserved_characters() { + const NAME: &str = "my/index?a#b c"; + const PREFIX: &str = "/v1/table/my_table/index/my%2Findex%3Fa%23b%20c"; + + let table = Table::new_with_handler("my_table", |request| { + assert_eq!(request.url().path(), format!("{PREFIX}/stats/")); + let body = serde_json::json!({ + "num_indexed_rows": 1, + "num_unindexed_rows": 0, + "index_type": "IVF_PQ", + "distance_type": "l2" + }); + http::Response::builder() + .status(200) + .body(serde_json::to_string(&body).unwrap()) + .unwrap() + }); + assert!(table.index_stats(NAME).await.unwrap().is_some()); + + let table = Table::new_with_handler("my_table", |request| { + assert_eq!(request.url().path(), format!("{PREFIX}/drop/")); + http::Response::builder().status(200).body("{}").unwrap() + }); + table.drop_index(NAME).await.unwrap(); + + let table = Table::new_with_handler("my_table", |request| { + assert_eq!(request.url().path(), format!("{PREFIX}/prewarm/")); + http::Response::builder().status(200).body("{}").unwrap() + }); + table.prewarm_index(NAME).await.unwrap(); + } + #[tokio::test] async fn test_set_lsm_write_spec_unsharded() { let table = Table::new_with_handler("my_table", |request| { From 9707966943ec2925f8f7c5572997ec4e00fccb34 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:53:32 +0800 Subject: [PATCH 24/41] test(rust): cover named memory databases on Windows (#3839) ## Summary\n\n- add a create-table regression for a named database\n- assert that the derived table URI uses URL separators\n- restore the four query tests that were moved to temporary files for #1051\n\n## Root cause\n\n historically joined table names with . On Windows this inserted a backslash into , so Lance interpreted the URI as an invalid local filename. The production URI builder now preserves forward slashes for URI schemes; this change restores the issue-specific tests and adds direct regression coverage for table creation and the derived URI.\n\n## Validation\n\n- \n- \n- (passes with four pre-existing warnings in unrelated remote-table code)\n- running 814 tests ....................................................................................... 87/814 .....................................i................................................. 174/814 ....................................................................................... 261/814 ....................................................................................... 348/814 ....................................................................................... 435/814 ....................................................................................... 522/814 ....................................................................................... 609/814 ....................................................................................... 696/814 ....................................................................................... 783/814 ............................... test result: ok. 813 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 7.76s running 39 tests ....................................... test result: ok. 39 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.23s running 6 tests ...... test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s running 5 tests ..... test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.10s running 0 tests test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s running 2 tests .. test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s running 2 tests .. test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s (867 passed, 1 ignored)\n- focused named-memory create and restored query tests\n\nFixes #1051\n\n Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/connection/create_table.rs | 11 +++++++ rust/lancedb/src/query.rs | 32 +++------------------ 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/rust/lancedb/src/connection/create_table.rs b/rust/lancedb/src/connection/create_table.rs index 66f6dfa8d..b10141beb 100644 --- a/rust/lancedb/src/connection/create_table.rs +++ b/rust/lancedb/src/connection/create_table.rs @@ -202,6 +202,17 @@ mod tests { assert_eq!(table.count_rows(None).await.unwrap(), 0); } + #[tokio::test] + async fn create_table_in_named_memory_database() { + let db = connect("memory://foo").execute().await.unwrap(); + let batch = record_batch!(("id", Int64, [1, 2, 3])).unwrap(); + + let table = db.create_table("my_table", batch).execute().await.unwrap(); + + assert_eq!(table.uri().await.unwrap(), "memory://foo/my_table.lance"); + assert_eq!(table.count_rows(None).await.unwrap(), 3); + } + async fn test_create_table_with_data(data: T) where T: Scannable + 'static, diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index b76865043..b2c5fefbe 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -1661,14 +1661,8 @@ mod tests { #[tokio::test] async fn test_setters_getters() { - // TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051 - // is fixed - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("test.lance"); - let uri = dataset_path.to_str().unwrap(); - let batches = make_test_batches(); - let conn = connect(uri).execute().await.unwrap(); + let conn = connect("memory://foo").execute().await.unwrap(); let table = conn .create_table("my_table", batches) .execute() @@ -1763,14 +1757,8 @@ mod tests { #[tokio::test] async fn test_execute() { - // TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051 - // is fixed - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("test.lance"); - let uri = dataset_path.to_str().unwrap(); - let batches = make_non_empty_batches(); - let conn = connect(uri).execute().await.unwrap(); + let conn = connect("memory://foo").execute().await.unwrap(); let table = conn .create_table("my_table", batches) .execute() @@ -1889,14 +1877,8 @@ mod tests { #[tokio::test] async fn test_select_with_transform() { - // TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051 - // is fixed - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("test.lance"); - let uri = dataset_path.to_str().unwrap(); - let batches = make_non_empty_batches(); - let conn = connect(uri).execute().await.unwrap(); + let conn = connect("memory://foo").execute().await.unwrap(); let table = conn .create_table("my_table", batches) .execute() @@ -1993,15 +1975,9 @@ mod tests { #[tokio::test] async fn test_execute_no_vector() { - // TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051 - // is fixed - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("test.lance"); - let uri = dataset_path.to_str().unwrap(); - // test that it's ok to not specify a query vector (just filter / limit) let batches = make_non_empty_batches(); - let conn = connect(uri).execute().await.unwrap(); + let conn = connect("memory://foo").execute().await.unwrap(); let table = conn .create_table("my_table", batches) .execute() From 1c3cd1d9184bb8c6d4088e077bd0825ef1e78235 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:54:04 +0800 Subject: [PATCH 25/41] fix(python): accept Arrow scalars in table updates (#3838) ## Summary - convert PyArrow scalar values through their Python representation before SQL literal rendering - add an end-to-end regression for updating a fixed-size-list vector from a queried FixedSizeListScalar ## Root cause Python update literal conversion used single dispatch for native Python and NumPy values but had no PyArrow Scalar registration. A FixedSizeListScalar returned by a query therefore reached the unsupported generic conversion instead of the existing recursive list converter. ## Validation - uv run --extra tests pytest python/tests/test_table.py::test_update python/tests/test_table.py::test_update_with_arrow_scalar python/tests/test_table.py::test_update_types -q - uv run --extra tests pytest python/tests/test_util.py -q - uv run --project python --extra tests --extra dev ruff format --check python/python/lancedb/util.py python/python/tests/test_table.py - uv run --project python --extra tests --extra dev ruff check . Fixes #1228 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/lancedb/util.py | 5 +++++ python/python/tests/test_table.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/python/python/lancedb/util.py b/python/python/lancedb/util.py index f582be7b4..dbc52bff6 100644 --- a/python/python/lancedb/util.py +++ b/python/python/lancedb/util.py @@ -395,6 +395,11 @@ def _(value: dict): ) +@value_to_sql.register(pa.Scalar) +def _(value: pa.Scalar): + return value_to_sql(value.as_py()) + + @value_to_sql.register(np.ndarray) def _(value: np.ndarray): return value_to_sql(value.tolist()) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index f20f42617..b2bfa2a68 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -2282,6 +2282,20 @@ def test_update(mem_db: DBConnection): assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]])) +def test_update_with_arrow_scalar(mem_db: DBConnection): + schema = pa.schema({"id": pa.int64(), "vector": pa.list_(pa.float32(), 4)}) + table = mem_db.create_table("my_table", schema=schema) + table.add([{"id": 1, "vector": [1.0, 2.0, 3.0, 4.0]}]) + + value = table.search().select(["vector"]).limit(1).to_arrow()["vector"][0] + assert isinstance(value, pa.FixedSizeListScalar) + + result = table.update(where="id == 1", values={"vector": value}) + + assert result.rows_updated == 1 + assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0, 3.0, 4.0]] + + def test_update_types(mem_db: DBConnection): table = mem_db.create_table( "my_table", From 369b10a377ed2e3bafd86d5999ac80d2e8aba49e Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:54:59 +0800 Subject: [PATCH 26/41] test(rust): cover object store reuse on table open (#3831) ## Summary - add regression coverage for repeated table opens through one database connection - assert that each open reuses the connection object-store client without another registry miss - exercise the table after every open so the test covers the complete dataset-loading path ## Root cause At the commit reported in #1600, opening a table constructed a separate object-store client rather than reusing the client that had already connected to the database. On S3 this repeated credential discovery, which could fail intermittently in AWS Lambda and surface as TableNotFound. The connection-owned Session reuse added later fixed the runtime path, but no focused test protected the open-table invariant. ## Fix Add a regression test backed by ObjectStoreRegistry statistics. Three successive opens must add cache hits while leaving the miss count unchanged, proving that open_table uses the connection Session and its authenticated object-store client. ## Validation - cargo fmt --all - cargo test --quiet --features remote -p lancedb database::listing::tests::test_open_table_reuses_connection_object_store - cargo check --quiet --features remote --tests --examples - cargo clippy --quiet --features remote --tests --examples - cargo test --quiet --features remote --tests Fixes #1600 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/database/listing.rs | 62 ++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index 454498d54..4fea6767c 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -1376,6 +1376,68 @@ mod tests { assert!(!tempdir.path().join("__manifest").exists()); } + /// Regression test for https://github.com/lancedb/lancedb/issues/1600. + /// + /// Opening a table used to create a separate object-store client instead of + /// reusing the one that successfully connected to the database. Repeating + /// credential discovery made S3 table opens intermittent, especially in AWS + /// Lambda, and the failed open was reported as `TableNotFound`. + #[tokio::test] + async fn test_open_table_reuses_connection_object_store() { + let tempdir = tempdir().unwrap(); + let uri = tempdir.path().to_str().unwrap(); + let registry = Arc::new(lance_io::object_store::ObjectStoreRegistry::default()); + let session = Arc::new(lance::session::Session::new(16, 16, registry.clone())); + + let request = ConnectRequest { + uri: uri.to_string(), + #[cfg(feature = "remote")] + client_config: Default::default(), + options: Default::default(), + namespace_client_properties: Default::default(), + manifest_enabled: false, + read_consistency_interval: None, + session: Some(session), + }; + let db = ListingDatabase::connect_with_options(&request) + .await + .unwrap(); + + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + db.create_table(CreateTableRequest { + name: "test".to_string(), + namespace_path: vec![], + data: Box::new(RecordBatch::new_empty(schema)) as Box, + mode: CreateTableMode::Create, + write_options: Default::default(), + location: None, + namespace_client: None, + }) + .await + .unwrap(); + + let before_open = registry.stats(); + for _ in 0..3 { + let table = db + .open_table(OpenTableRequest { + name: "test".to_string(), + namespace_path: vec![], + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await + .unwrap(); + assert_eq!(table.count_rows(None).await.unwrap(), 0); + } + + let after_open = registry.stats(); + assert_eq!(after_open.misses, before_open.misses); + assert!(after_open.hits >= before_open.hits + 3); + } + #[tokio::test] async fn test_clone_table_basic() { let (_tempdir, db) = setup_database().await; From 001237c7a42b62581dd3b43b39ede68eb3ce6cf3 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Thu, 6 Aug 2026 12:00:22 -0700 Subject: [PATCH 27/41] chore: update lance dependency to v11.0.0-beta.2 (#3886) Updates the Lance dependencies and Java lance-core to v11.0.0-beta.2. Includes required compatibility fixes for the LanceFileVersion module move and the updated GooseFS/OpenDAL dependency. Lance tag: https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.2 --------- Co-authored-by: Daniel Rammer Co-authored-by: Claude Opus 5 (1M context) --- .github/workflows/rust.yml | 14 +- Cargo.lock | 492 +++++++++++++----------- Cargo.toml | 28 +- java/pom.xml | 2 +- rust/lancedb/Cargo.toml | 4 +- rust/lancedb/src/blob.rs | 2 +- rust/lancedb/src/connection.rs | 2 +- rust/lancedb/src/database/listing.rs | 2 +- rust/lancedb/src/database/namespace.rs | 4 +- rust/lancedb/src/remote/table.rs | 8 +- rust/lancedb/src/remote/table/blobs.rs | 2 +- rust/lancedb/src/remote/table/insert.rs | 4 +- rust/lancedb/tests/blob_integration.rs | 2 +- 13 files changed, 297 insertions(+), 269 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1a0b65c4a..471da43e0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -296,16 +296,18 @@ jobs: cargo update -p aws-types --precise 1.3.9 cargo update -p aws-sigv4 --precise 1.3.5 cargo update -p aws-credential-types --precise 1.2.8 - cargo update -p aws-smithy-checksums --precise 0.63.9 + # aws-smithy-checksums must stay at or above 0.63.13: OpenDAL's S3 + # service needs crc-fast ~1.9, and older releases pin it to ~1.3. + cargo update -p aws-smithy-checksums --precise 0.63.13 cargo update -p aws-smithy-runtime --precise 1.9.3 - cargo update -p aws-smithy-http --precise 0.62.4 - cargo update -p aws-smithy-eventstream --precise 0.60.12 + cargo update -p aws-smithy-http --precise 0.62.6 + cargo update -p aws-smithy-eventstream --precise 0.60.14 cargo update -p aws-smithy-http-client --precise 1.1.3 cargo update -p aws-smithy-observability --precise 0.1.4 cargo update -p aws-smithy-query --precise 0.60.8 - cargo update -p aws-smithy-runtime-api --precise 1.9.1 - cargo update -p aws-smithy-async --precise 1.2.6 - cargo update -p aws-smithy-types --precise 1.3.5 + cargo update -p aws-smithy-runtime-api --precise 1.9.3 + cargo update -p aws-smithy-async --precise 1.2.7 + cargo update -p aws-smithy-types --precise 1.3.6 cargo update -p aws-smithy-xml --precise 0.60.11 cargo update -p home --precise 0.5.9 - name: cargo +${{ matrix.msrv }} check diff --git a/Cargo.lock b/Cargo.lock index 995424b20..93e16c06d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -775,7 +775,7 @@ dependencies = [ "http 0.2.12", "http 1.5.0", "http-body 1.1.0", - "lru", + "lru 0.16.4", "percent-encoding", "regex-lite", "sha2 0.11.0", @@ -1241,6 +1241,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1964,15 +1970,6 @@ dependencies = [ "spin 0.10.1", ] -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - [[package]] name = "crc32fast" version = "1.5.0" @@ -2174,9 +2171,9 @@ dependencies = [ [[package]] name = "ctor" -version = "1.0.5" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378f0974ae2468eaf63aa036dbe9c926b0dc7ea64c156f2ea618bc2f75b934f0" +checksum = "2d83cb7e7a873830708d6b02a78cd36a592c6fa14bf267b68725103b85c0d77f" dependencies = [ "link-section", "linktime-proc-macro", @@ -2902,6 +2899,37 @@ dependencies = [ "uuid", ] +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "der" version = "0.6.1" @@ -3413,6 +3441,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "frostem" +version = "1.20260804.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82eb03a32a1d50555353c85a7b9d3279a6f1e91af9890b789acdf544ed57c8d7" + [[package]] name = "fs_extra" version = "1.3.0" @@ -3421,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "rand 0.9.5", @@ -3779,14 +3813,23 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.1.5" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" +checksum = "e1ea4eee6dcbc31b25ab4fd577adc55b677d2bed3aa3016c44c58fbe1b2298a5" dependencies = [ + "arc-swap", "async-trait", "bytes", "dashmap", + "fastrand", + "futures", "hostname", + "io-uring", + "itoa", + "libc", + "lru 0.18.2", + "memmap2 0.9.10", + "moka", "prost", "prost-types", "rand 0.9.5", @@ -3799,6 +3842,7 @@ dependencies = [ "tonic-prost", "tracing", "uuid", + "xxhash-rust", ] [[package]] @@ -4599,10 +4643,12 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.24" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ + "defmt", + "jiff-core", "jiff-static", "jiff-tzdb-platform", "js-sys", @@ -4611,15 +4657,25 @@ dependencies = [ "portable-atomic-util", "serde_core", "wasm-bindgen", - "windows-sys 0.61.2", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", ] [[package]] name = "jiff-static" -version = "0.2.24" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" dependencies = [ + "jiff-core", "proc-macro2", "quote", "syn 2.0.117", @@ -4731,24 +4787,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "jsonwebtoken" -version = "10.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" -dependencies = [ - "aws-lc-rs", - "base64 0.22.1", - "getrandom 0.2.17", - "js-sys", - "pem", - "serde", - "serde_json", - "signature 2.2.0", - "simple_asn1", - "zeroize", -] - [[package]] name = "kanaria" version = "0.2.0" @@ -4777,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arc-swap", "arrow", @@ -4852,8 +4890,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-buffer", @@ -4875,7 +4913,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-buffer", @@ -4889,7 +4927,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-schema", @@ -4898,8 +4936,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrayref", "crunchy", @@ -4909,8 +4947,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-buffer", @@ -4950,8 +4988,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow", "arrow-array", @@ -4981,8 +5019,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow", "arrow-array", @@ -4999,8 +5037,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "proc-macro2", "quote", @@ -5009,8 +5047,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-arith", "arrow-array", @@ -5036,7 +5074,6 @@ dependencies = [ "prost", "prost-build", "rand 0.9.5", - "strum 0.26.3", "tokio", "tracing", "xxhash-rust", @@ -5045,8 +5082,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-arith", "arrow-array", @@ -5077,8 +5114,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arc-swap", "arrow", @@ -5145,8 +5182,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-schema", @@ -5168,8 +5205,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow", "arrow-array", @@ -5181,7 +5218,6 @@ dependencies = [ "bytes", "chrono", "futures", - "goosefs-sdk", "http 1.5.0", "io-uring", "lance-arrow", @@ -5206,8 +5242,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-buffer", @@ -5223,8 +5259,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow", "async-trait", @@ -5236,8 +5272,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow", "arrow-ipc", @@ -5291,8 +5327,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-buffer", @@ -5307,8 +5343,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow", "arrow-array", @@ -5347,8 +5383,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ "arrow-array", "arrow-schema", @@ -5361,13 +5397,13 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "10.1.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" +version = "11.0.0-beta.2" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160" dependencies = [ + "frostem", "icu_segmenter", "jieba-rs", "lindera", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -5645,7 +5681,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml_ng", - "strum 0.28.0", + "strum", "strum_macros 0.28.0", "unicode-blocks", "unicode-normalization", @@ -5675,22 +5711,22 @@ dependencies = [ "rkyv", "serde", "serde_json", - "strum 0.28.0", + "strum", "strum_macros 0.28.0", "thiserror 2.0.18", ] [[package]] name = "link-section" -version = "0.16.1" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8600ca3dbe044f07955b443ff606c50f45295b863289bbe7d0844d50cf11e4" +checksum = "5ee1a0d6e252afe82e7bc2db42fba60e02ddf3b1accaf8cb21d96e34ba61f3d4" [[package]] name = "linktime-proc-macro" -version = "0.1.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44cd706ff0d503ee32b2071166510ca27e281228de10cd3aa8d35ff94560f81" +checksum = "348d0075b1fc163b26d72a7f75fc5141daf2fd1bdf128d873cbaf6785d495bdf" [[package]] name = "linux-raw-sys" @@ -5747,6 +5783,15 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "lru" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -6067,7 +6112,7 @@ checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941" dependencies = [ "bitflags 2.11.1", "chrono", - "ctor 1.0.5", + "ctor 1.0.12", "futures", "napi-build", "napi-sys", @@ -6091,7 +6136,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d5c9c02556ea6dc99dffd36c1ce60141411657438501a125b675776d011ce92" dependencies = [ "convert_case", - "ctor 1.0.5", + "ctor 1.0.12", "napi-derive-backend", "proc-macro2", "quote", @@ -6384,9 +6429,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" dependencies = [ "async-trait", "bytes", @@ -6447,12 +6492,13 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "opendal" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d" dependencies = [ - "ctor 1.0.5", + "ctor 1.0.12", "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -6469,24 +6515,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed" dependencies = [ "anyhow", - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures", "http 1.5.0", - "http-body 1.1.0", "jiff", "log", "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.3", "serde", "serde_json", "tokio", @@ -6496,10 +6540,24 @@ dependencies = [ ] [[package]] -name = "opendal-layer-concurrent-limit" -version = "0.57.0" +name = "opendal-http-transport-reqwest" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef" +dependencies = [ + "bytes", + "futures", + "http 1.5.0", + "http-body 1.1.0", + "opendal-core", + "reqwest 0.13.4", +] + +[[package]] +name = "opendal-layer-concurrent-limit" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081" dependencies = [ "futures", "http 1.5.0", @@ -6509,9 +6567,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c" dependencies = [ "log", "opendal-core", @@ -6519,9 +6577,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d" dependencies = [ "backon", "log", @@ -6530,9 +6588,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66" dependencies = [ "opendal-core", "tokio", @@ -6540,17 +6598,17 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "http 1.5.0", "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6561,17 +6619,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "http 1.5.0", "log", + "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6581,9 +6640,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df" dependencies = [ "http 1.5.0", "opendal-core", @@ -6591,15 +6650,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0" dependencies = [ "bytes", "http 1.5.0", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -6608,9 +6667,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6" dependencies = [ "async-trait", "bytes", @@ -6618,7 +6677,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -6629,9 +6688,9 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78" dependencies = [ "bytes", "goosefs-sdk", @@ -6643,9 +6702,9 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7" dependencies = [ "bytes", "hf-xet", @@ -6653,22 +6712,21 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "reqwest 0.13.3", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b" dependencies = [ "bytes", "http 1.5.0", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -6677,18 +6735,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", - "crc32c", + "crc-fast", "http 1.5.0", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -7583,7 +7641,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "itertools 0.14.0", "log", "multimap", @@ -7799,6 +7857,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick_cache" version = "0.6.24" @@ -8220,9 +8288,9 @@ dependencies = [ [[package]] name = "reqsign-aliyun-oss" -version = "3.0.0" +version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95" +checksum = "a5e6d659fcdbca6fe2d7ef109c2e28499b7be80501f1bb86c10caf5ec8ac1219" dependencies = [ "anyhow", "form_urlencoded", @@ -8236,38 +8304,52 @@ dependencies = [ ] [[package]] -name = "reqsign-aws-v4" -version = "3.0.0" +name = "reqsign-aws-core" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43" +checksum = "e4af084e1f3cbf3e67e0c972765399bce54ecec804cceba46b39a8331f3c1bff" dependencies = [ - "anyhow", "bytes", "form_urlencoded", + "hex", "http 1.5.0", "log", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "rust-ini", "serde", "serde_json", "serde_urlencoded", - "sha1 0.10.6", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac5b3b7cefa28933792b439186459f77f19f9b6edbeab41b8b187150361a206" +dependencies = [ + "bytes", + "http 1.5.0", + "log", + "quick-xml 0.41.0", + "reqsign-aws-core", + "reqsign-core", + "serde", ] [[package]] name = "reqsign-azure-storage" -version = "3.0.0" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6" +checksum = "2824e7da3c2cc42ac3406c674eb57c89127fdcd97f3a73c608cfc680505ea134" dependencies = [ "anyhow", - "base64 0.22.1", + "base64 0.23.1", "bytes", "form_urlencoded", "http 1.5.0", - "jsonwebtoken", "log", "pem", "percent-encoding", @@ -8275,36 +8357,38 @@ dependencies = [ "rsa", "serde", "serde_json", - "sha1 0.10.6", + "sha1 0.11.0", ] [[package]] name = "reqsign-core" -version = "3.0.0" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102" +checksum = "c07dd510b1e1b9b241883e483358147fb2ed2d497a7b39b065ba61eb93deceb0" dependencies = [ "anyhow", - "base64 0.22.1", + "base64 0.23.1", "bytes", - "form_urlencoded", "futures", "hex", - "hmac 0.12.1", + "hmac 0.13.0", "http 1.5.0", "jiff", "log", "percent-encoding", - "sha1 0.10.6", - "sha2 0.10.9", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.0", "windows-sys 0.61.2", ] [[package]] name = "reqsign-file-read-tokio" -version = "3.0.0" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5" +checksum = "663d9d55abd0df0830ef0ae43708297cc1371cf4e8ca91f3ac813c309cca8c98" dependencies = [ "anyhow", "reqsign-core", @@ -8313,13 +8397,12 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.0.0" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e" +checksum = "4080a227f82a09f68540ecd028622065d7ac4c0bcb8727a25bdcfc0526235792" dependencies = [ "form_urlencoded", "http 1.5.0", - "jsonwebtoken", "log", "percent-encoding", "reqsign-aws-v4", @@ -8327,15 +8410,14 @@ dependencies = [ "rsa", "serde", "serde_json", - "sha2 0.10.9", "tokio", ] [[package]] name = "reqsign-tencent-cos" -version = "3.0.0" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a" +checksum = "764629c90f7c3566a6d4e4641ebab9acd604ce02e16eda4d37c7d7e79e16ed90" dependencies = [ "anyhow", "http 1.5.0", @@ -8394,9 +8476,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64 0.22.1", "bytes", @@ -8457,7 +8539,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.5.0", - "reqwest 0.13.3", + "reqwest 0.13.4", "thiserror 2.0.18", "tower-service", ] @@ -8597,16 +8679,6 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-demangle" version = "0.1.27" @@ -9206,18 +9278,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" -[[package]] -name = "simple_asn1" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" -dependencies = [ - "num-bigint", - "num-traits", - "thiserror 2.0.18", - "time", -] - [[package]] name = "siphasher" version = "1.0.3" @@ -9277,7 +9337,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -9289,7 +9349,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -9492,15 +9552,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", -] - [[package]] name = "strum" version = "0.28.0" @@ -9523,19 +9574,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.117", -] - [[package]] name = "strum_macros" version = "0.28.0" @@ -9730,7 +9768,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", @@ -11161,7 +11199,7 @@ dependencies = [ "more-asserts", "rand 0.10.1", "redb", - "reqwest 0.13.3", + "reqwest 0.13.4", "reqwest-middleware", "serde", "serde_json", @@ -11274,7 +11312,7 @@ dependencies = [ "oneshot", "pin-project", "rand 0.10.1", - "reqwest 0.13.3", + "reqwest 0.13.4", "serde", "serde_json", "shellexpand", @@ -11370,20 +11408,6 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] [[package]] name = "zerotrie" diff --git a/Cargo.toml b/Cargo.toml index 35370e58f..a879d1f8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } ahash = "0.8" # Note that this one does not include pyarrow arrow = { version = "58.0.0", optional = false } diff --git a/java/pom.xml b/java/pom.xml index 1b96cbcab..e8f030b27 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 10.1.0-beta.1 + 11.0.0-beta.2 false 2.30.0 1.7 diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 816f095de..96ea9ec95 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -49,8 +49,8 @@ lance-namespace = { workspace = true } lance-namespace-impls = { workspace = true } metrics = { workspace = true, optional = true } metrics-util = { workspace = true, optional = true } -# Pin the transitive GooseFS SDK until the 0.1.6 compile break is fixed upstream. -goosefs-sdk = { version = "=0.1.5", optional = true } +# Pin the GooseFS SDK to the version required by Lance's OpenDAL dependency. +goosefs-sdk = { version = "=0.1.9", optional = true } moka = { workspace = true } pin-project = { workspace = true } tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] } diff --git a/rust/lancedb/src/blob.rs b/rust/lancedb/src/blob.rs index 3448257ab..e1c18dd84 100644 --- a/rust/lancedb/src/blob.rs +++ b/rust/lancedb/src/blob.rs @@ -17,7 +17,7 @@ use arrow_array::builder::LargeBinaryBuilder; use arrow_schema::{DataType, Field, Schema}; use lance::dataset::{BlobRangeRequest as LanceBlobRangeRequest, Dataset, WriteParams}; use lance_arrow::FieldExt; -use lance_encoding::version::LanceFileVersion; +use lance_file::version::LanceFileVersion; use lance_io::object_store::ObjectStore; use object_store::path::Path; diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 89e59e12e..dd53a2d2e 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -34,7 +34,7 @@ use crate::remote::{ db::{OPT_REMOTE_API_KEY, OPT_REMOTE_HOST_OVERRIDE, OPT_REMOTE_REGION}, }; use lance::io::ObjectStoreParams; -pub use lance_encoding::version::LanceFileVersion; +pub use lance_file::version::LanceFileVersion; #[cfg(feature = "remote")] use lance_io::object_store::StorageOptions; use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider}; diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index 4fea6767c..fea34bb48 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -12,7 +12,7 @@ use lance::dataset::refs::Ref; use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder}; use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore}; use lance_datafusion::utils::StreamingWriteSource; -use lance_encoding::version::LanceFileVersion; +use lance_file::version::LanceFileVersion; use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider}; use lance_table::io::commit::commit_handler_from_url; use object_store::local::LocalFileSystem; diff --git a/rust/lancedb/src/database/namespace.rs b/rust/lancedb/src/database/namespace.rs index d18c78682..740e11645 100644 --- a/rust/lancedb/src/database/namespace.rs +++ b/rust/lancedb/src/database/namespace.rs @@ -201,7 +201,7 @@ impl LanceNamespaceDatabase { &self, request: &DbCreateTableRequest, ) -> Result<( - Option, + Option, Option, Option, )> { @@ -214,7 +214,7 @@ impl LanceNamespaceDatabase { let storage_version_override = storage_options .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION)) - .map(|s| s.parse::()) + .map(|s| s.parse::()) .transpose()?; let v2_manifest_override = storage_options diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 29b6b698d..388bed0f7 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2942,7 +2942,7 @@ impl BaseTable for RemoteTable { } #[derive(Serialize, Clone, Debug)] -pub(crate) struct MergeInsertRequest { +pub struct MergeInsertRequest { on: String, when_matched_update_all: bool, when_matched_update_all_filt: Option, @@ -5907,16 +5907,18 @@ mod tests { .await .unwrap(); + // Positions are relative to the first retained token, so dropping the + // leading "hello" stop word does not shift the remaining tokens. assert_eq!( tokens, vec![ FtsToken { text: "こんにちは".to_string(), - position: 1, + position: 0, }, FtsToken { text: "世界".to_string(), - position: 2, + position: 1, }, ] ); diff --git a/rust/lancedb/src/remote/table/blobs.rs b/rust/lancedb/src/remote/table/blobs.rs index bb8a5d103..387d3c6dc 100644 --- a/rust/lancedb/src/remote/table/blobs.rs +++ b/rust/lancedb/src/remote/table/blobs.rs @@ -90,7 +90,7 @@ struct RemoteBlobState { /// Seekable Cloud blob handle over HTTP Range. #[derive(Debug)] -pub(crate) struct RemoteBlobFile { +pub struct RemoteBlobFile { requester: Arc, state: Mutex, closed: AtomicBool, diff --git a/rust/lancedb/src/remote/table/insert.rs b/rust/lancedb/src/remote/table/insert.rs index 67ea7765d..a4a28a9c6 100644 --- a/rust/lancedb/src/remote/table/insert.rs +++ b/rust/lancedb/src/remote/table/insert.rs @@ -33,7 +33,7 @@ use crate::table::{AddResult, MergeResult}; /// same Arrow-IPC streaming body and error side-channel; only the target /// endpoint, query parameters, and parsed result type differ. #[derive(Debug, Clone)] -pub(crate) enum WriteOp { +pub enum WriteOp { /// `add`: stream to `/v1/table/{id}/insert/`, optionally overwriting. Insert { overwrite: bool }, /// `merge_insert`: stream to `/v1/table/{id}/merge_insert/` with the merge @@ -49,7 +49,7 @@ pub(crate) enum WriteOp { /// The parsed server response for a completed write, discriminated by the /// operation that produced it. #[derive(Debug, Clone)] -pub(crate) enum WriteResult { +pub enum WriteResult { Add(AddResult), Merge(MergeResult), } diff --git a/rust/lancedb/tests/blob_integration.rs b/rust/lancedb/tests/blob_integration.rs index 3d327766e..77d49abd9 100644 --- a/rust/lancedb/tests/blob_integration.rs +++ b/rust/lancedb/tests/blob_integration.rs @@ -10,7 +10,7 @@ use arrow_array::{ use arrow_schema::{DataType, Field, Fields, Schema}; use futures::TryStreamExt; use lance::Dataset; -use lance_encoding::version::LanceFileVersion; +use lance_file::version::LanceFileVersion; use lancedb::{ Connection, Error, Result, Table, blob::{BlobRangeRequest, blob}, From b1cfe6edb15c3904001a99e98560b80970fd711e Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 7 Aug 2026 16:31:40 +0800 Subject: [PATCH 28/41] ci(docs): add scheduled doc link check (#3888) The docs have no link checking at all, so external links rot silently: a trial run already found `docs/src/python/python.md` pointing at `lancedb.github.io/lance-namespace`, which returns 404 since the repository moved to the lance-format org. Checking external links on the blocking path would be the wrong trade: third-party hosts rate-limit automated clients, reject non-browser user agents, and go down temporarily, so any of them having a bad minute would turn unrelated PRs red. Following lance-format/lance#8315, this adds a daily `lychee` run that reports broken links into a single tracking issue, rewritten in place on each run and closed automatically once every link resolves. The scan job runs the downloaded lychee binary with a read-only token; everything that writes lives in a separate report job, and a non-verdict lychee exit fails the run instead of publishing a bogus report. The check is restricted to http(s) links because much of `docs/src` is generated API reference (the `js/` tree comes from `npm run docs`) and the hand-written pages use mkdocstrings cross-references and nav-relative paths that only resolve in the site mkdocs builds, so relative links would be reported as broken on every run. The one broken link the trial run surfaced is fixed here; after the fix, a local run over all 154 files reports 0 errors across 216 unique links. --- .github/workflows/docs-link-check.yml | 222 ++++++++++++++++++++++++++ docs/src/python/python.md | 2 +- 2 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docs-link-check.yml diff --git a/.github/workflows/docs-link-check.yml b/.github/workflows/docs-link-check.yml new file mode 100644 index 000000000..0e22100eb --- /dev/null +++ b/.github/workflows/docs-link-check.yml @@ -0,0 +1,222 @@ +name: Check doc links + +# Checking external links is inherently noisy: third-party sites rate-limit +# automated clients, reject non-browser user agents, and go down temporarily. +# Blocking pull requests on that trades a lot of false failures for very little +# signal, so this runs on a schedule and reports findings in a single tracking +# issue instead of failing anyone's build. +on: + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + +# The report lives in one repository-global issue, so runs must not overlap: a +# lookup racing a create produces duplicate issues, and a healthy run closing +# the issue while a failing run only rewrites its body would leave a broken +# report closed. The group is deliberately ref-independent so that a manual +# dispatch serializes against the scheduled run. +concurrency: + group: docs-link-check + cancel-in-progress: false + +permissions: {} + +env: + REPORT_TITLE: "Docs link checker report" + +jobs: + scan: + name: Scan links + runs-on: ubuntu-24.04 + # lychee-action is pinned by SHA, but its wrapper downloads the lychee + # release tarball at run time without verifying a digest, and hands the + # resulting binary a GitHub token. Release assets remain replaceable, so + # that binary is confined to a job whose token can only read public + # content; everything that writes runs in the report job below. + permissions: + contents: read + outputs: + exit_code: ${{ steps.lychee.outputs.exit_code }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + # workflow_dispatch can run from any ref, but the report is + # repository-global. Always measure the default branch so a manual + # run from a topic branch cannot close a report that main warrants, + # or overwrite it with branch-only findings. + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Check links + id: lychee + uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0 + with: + # Restricted to http(s) on purpose. Much of docs/src is generated + # API reference (the js/ tree comes from `npm run docs` in nodejs) + # and the hand-written pages use mkdocstrings cross-references and + # nav-relative paths that only resolve in the site mkdocs builds, + # not in this checkout, so relative links would be reported as + # broken on every run. + args: >- + --scheme https + --scheme http + --no-progress + --max-retries 3 + --timeout 20 + 'docs/src/**/*.md' + format: json + output: ./lychee/out.json + jobSummary: false + # The report, not a red build, is the signal for broken links. The + # validation step below still fails the run if the check itself + # breaks. + fail: false + + - name: Validate report + # lychee does not reserve exit code 2 for broken links: its CLI + # parser also exits 2 on an invalid option, before any link was + # checked or any report written. Only a parseable report whose + # counts agree with the exit code counts as a link verdict; anything + # else fails here, and the report job below is skipped entirely, so + # the tracking issue is never touched. Exit 2 covers timeouts as + # well as errors, and a timed-out host is exactly the transient + # unavailability this report exists to surface, so both count as + # findings. Requiring total > 0 also catches a glob that silently + # stopped matching any file. + if: steps.lychee.outputs.exit_code == 0 || steps.lychee.outputs.exit_code == 2 + env: + EXIT_CODE: ${{ steps.lychee.outputs.exit_code }} + run: | + jq -e --argjson code "$EXIT_CODE" ' + (.total > 0) and + (if $code == 0 + then .errors == 0 and .timeouts == 0 + and (.error_map | length == 0) and (.timeout_map | length == 0) + else (.errors + .timeouts) > 0 + and ((.error_map | length) + (.timeout_map | length)) > 0 + end) + ' ./lychee/out.json + + - name: Upload report + if: steps.lychee.outputs.exit_code == 2 + uses: actions/upload-artifact@v7 + with: + name: link-report + path: ./lychee/out.json + retention-days: 7 + + report: + name: Update report issue + needs: scan + runs-on: ubuntu-24.04 + # Deliberately no checkout: this job needs the report artifact and the + # issues API, not the repository contents. + permissions: + issues: write + env: + EXIT_CODE: ${{ needs.scan.outputs.exit_code }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Classify checker result + # lychee exits 0 when every link resolves and 2 when links fail, + # both already cross-checked against the report by the scan job's + # validation step. Anything else (1 runtime, 3 bad config) means the + # check never produced a link verdict, which must surface as a failed + # run rather than be published as "broken documentation links". + run: | + case "$EXIT_CODE" in + 0|2) + echo "lychee exit code $EXIT_CODE" + ;; + *) + echo "::error::lychee exited with '$EXIT_CODE': the link check did not complete. Leaving the report issue untouched." + exit 1 + ;; + esac + + - name: Find existing report issue + id: report + # Matched on title alone, and through search rather than a listing: + # the issue action applies labels in a separate call after creating the + # issue, so a label filter misses a half-created report, and this + # repository has far more open issues than one listing page holds. + # Closed issues are included because a healthy run closes the report: + # an open-only lookup would forget that identity and the next failing + # run would open a duplicate. The oldest match stays the canonical + # report and is reopened below when links break again. + run: | + match=$(gh issue list --repo "$GITHUB_REPOSITORY" --state all \ + --search "in:title \"$REPORT_TITLE\" author:app/github-actions" \ + --limit 50 --json number,title,state \ + --jq "[.[] | select(.title == \"$REPORT_TITLE\")] | sort_by(.number) | first // empty") + echo "number=$(jq -r '.number // empty' <<<"$match")" >> "$GITHUB_OUTPUT" + echo "state=$(jq -r '.state // empty' <<<"$match")" >> "$GITHUB_OUTPUT" + + - name: Download report + if: env.EXIT_CODE == 2 + uses: actions/download-artifact@v8 + with: + name: link-report + path: ./lychee + + - name: Compose report + if: env.EXIT_CODE == 2 + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + { + echo "Broken documentation links found by [\`$GITHUB_WORKFLOW\`]($run_url)." + echo + echo "This issue is rewritten by every scheduled run and closed automatically once all links resolve." + echo + echo "Entries can be false positives: some sites rate-limit or block automated clients while working fine in a browser. Confirm before editing the docs, and add persistent offenders to \`--exclude\` in \`.github/workflows/docs-link-check.yml\`." + echo + # Timeouts are reported alongside errors: entries land in + # timeout_map with a status text instead of an HTTP code. + jq -r ' + "\(.errors) of \(.total) links failed, \(.timeouts) timed out.", + "", + ([(.error_map | to_entries[]), (.timeout_map | to_entries[])] + | group_by(.key)[] | + "### Errors in \(.[0].key)", + "", + (map(.value[])[] | "* [\(.status.code // .status.text // "ERR")] <\(.url)> — \(.status.details // .status.text // "unknown error")"), + "") + ' ./lychee/out.json + } > ./lychee/issue.md + + - name: Reopen report issue + # A healthy run closes the report, and the issue action below only + # rewrites the body of whatever number it is given. Without an + # explicit reopen, the 2 -> 0 -> 2 sequence would keep rewriting a + # closed issue while links are broken. A CLOSED state implies the + # lookup found a canonical issue, so no separate emptiness check. + if: env.EXIT_CODE == 2 && steps.report.outputs.state == 'CLOSED' + env: + ISSUE_NUMBER: ${{ steps.report.outputs.number }} + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + gh issue reopen "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --comment "Broken documentation links found again in [the latest run]($run_url)." + + - name: Report broken links + if: env.EXIT_CODE == 2 + uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6.0.0 + with: + # Empty on the first failing run, which creates the issue; afterwards + # the same issue is updated in place. + issue-number: ${{ steps.report.outputs.number }} + title: ${{ env.REPORT_TITLE }} + content-filepath: ./lychee/issue.md + labels: documentation + + - name: Close report issue once links are healthy + # An OPEN state implies the lookup found a canonical issue; a report + # that is already closed needs nothing. + if: env.EXIT_CODE == 0 && steps.report.outputs.state == 'OPEN' + env: + ISSUE_NUMBER: ${{ steps.report.outputs.number }} + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + gh issue close "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --comment "All documentation links resolved in [the latest run]($run_url)." diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 36044d35d..3dd6f59f4 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -31,7 +31,7 @@ is also an [asynchronous API client](#connections-asynchronous). ## Namespaces (Synchronous) A namespace-backed connection resolves tables through a -[Lance namespace](https://lancedb.github.io/lance-namespace/) service instead of +[Lance namespace](https://lance-format.github.io/lance-namespace/) service instead of listing a storage directory. ::: lancedb.connect_namespace From f4c668e2441a3c5ab31a024eb8e1ec6a68920f17 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Fri, 7 Aug 2026 02:24:38 -0700 Subject: [PATCH 29/41] chore(deps): declare more specific futures dependency (#3800) Lancedb does not work with any other version of `futures`. With futures 0.1 it fails like this: ```console error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryStreamExt` --> rust/lancedb/src/arrow.rs:21:23 | 21 | use futures::{Stream, StreamExt, TryStreamExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved import `futures::StreamExt` --> rust/lancedb/src/data/scannable.rs:24:5 | 24 | use futures::StreamExt; | ^^^^^^^^^^^^^^^^^^ no `StreamExt` in the root | error[E0432]: unresolved import `futures::TryStreamExt` --> rust/lancedb/src/dataloader/permutation/builder.rs:9:5 | 9 | use futures::TryStreamExt; | ^^^^^^^^^^^^^^^^^^^^^ no `TryStreamExt` in the root error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryStreamExt` --> rust/lancedb/src/dataloader/permutation/reader.rs:25:15 | 25 | use futures::{StreamExt, TryStreamExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryStreamExt` --> rust/lancedb/src/dataloader/permutation/shuffle.rs:8:15 | 8 | use futures::{StreamExt, TryStreamExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryStreamExt` --> rust/lancedb/src/dataloader/permutation/split.rs:12:15 | 12 | use futures::{StreamExt, TryStreamExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved import `futures::TryStreamExt` --> rust/lancedb/src/dataloader/permutation/util.rs:9:5 | 9 | use futures::TryStreamExt; | ^^^^^^^^^^^^^^^^^^^^^ no `TryStreamExt` in the root error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryFutureExt` --> rust/lancedb/src/io/object_store.rs:8:15 | 8 | use futures::{StreamExt, TryFutureExt, stream::BoxStream}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryFutureExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved imports `futures::FutureExt`, `futures::TryFutureExt`, `futures::TryStreamExt`, `futures::try_join` --> rust/lancedb/src/query.rs:12:15 | 12 | use futures::{FutureExt, TryFutureExt, TryStreamExt, stream, try_join}; | ^^^^^^^^^ ^^^^^^^^^^^^ ^^^^^^^^^^^^ ^^^^^^^^ no `try_join` in the root | | | | | | | no `TryStreamExt` in the root | | no `TryFutureExt` in the root | no `FutureExt` in the root | error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryStreamExt` --> rust/lancedb/src/remote/table/blobs.rs:13:15 | 13 | use futures::{StreamExt, TryStreamExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved imports `futures::SinkExt`, `futures::StreamExt` --> rust/lancedb/src/remote/table/insert.rs:20:15 | 20 | use futures::{SinkExt, StreamExt}; | ^^^^^^^ ^^^^^^^^^ no `StreamExt` in the root | | | no `SinkExt` in the root | error[E0432]: unresolved imports `futures::StreamExt`, `futures::TryStreamExt` --> rust/lancedb/src/remote/table.rs:58:15 | 58 | use futures::{StreamExt, TryStreamExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `StreamExt` in the root | error[E0432]: unresolved import `futures::StreamExt` --> rust/lancedb/src/remote/util.rs:5:23 | 5 | use futures::{Stream, StreamExt}; | ^^^^^^^^^ no `StreamExt` in the root | error[E0432]: unresolved import `futures::StreamExt` --> rust/lancedb/src/table.rs:14:5 | 14 | use futures::StreamExt; | ^^^^^^^^^^^^^^^^^^ no `StreamExt` in the root | error[E0432]: unresolved import `futures::TryStreamExt` --> rust/lancedb/src/table/datafusion/insert.rs:20:5 | 20 | use futures::TryStreamExt; | ^^^^^^^^^^^^^^^^^^^^^ no `TryStreamExt` in the root error[E0432]: unresolved import `futures::TryStreamExt` --> rust/lancedb/src/table/datafusion/scannable_exec.rs:14:5 | 14 | use futures::TryStreamExt; | ^^^^^^^^^^^^^^^^^^^^^ no `TryStreamExt` in the root error[E0432]: unresolved imports `futures::TryFutureExt`, `futures::TryStreamExt` --> rust/lancedb/src/table/datafusion.rs:25:15 | 25 | use futures::{TryFutureExt, TryStreamExt}; | ^^^^^^^^^^^^ ^^^^^^^^^^^^ no `TryStreamExt` in the root | | | no `TryFutureExt` in the root error[E0432]: unresolved import `futures::FutureExt` --> rust/lancedb/src/table/delete.rs:3:5 | 3 | use futures::FutureExt; | ^^^^^^^^^^^^^^^^^^ no `FutureExt` in the root | error[E0432]: unresolved imports `futures::FutureExt`, `futures::TryFutureExt` --> rust/lancedb/src/table/merge.rs:9:15 | 9 | use futures::{FutureExt, TryFutureExt}; | ^^^^^^^^^ ^^^^^^^^^^^^ no `TryFutureExt` in the root | | | no `FutureExt` in the root | error[E0432]: unresolved import `futures::future::try_join_all` --> rust/lancedb/src/table/query.rs:24:5 | 24 | use futures::future::try_join_all; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ no `try_join_all` in `future` | error[E0432]: unresolved import `futures::FutureExt` --> rust/lancedb/src/utils/background_cache.rs:12:5 | 12 | use futures::FutureExt; | ^^^^^^^^^^^^^^^^^^ no `FutureExt` in the root | error[E0432]: unresolved import `futures::FutureExt` --> rust/lancedb/src/utils/mod.rs:12:15 | 12 | use futures::{FutureExt, Stream}; | ^^^^^^^^^ no `FutureExt` in the root | error[E0433]: cannot find `join` in `futures` --> rust/lancedb/src/remote/table/insert.rs:504:55 | 504 | let (producer_result, send_result) = futures::join!(producer, send); | ^^^^ could not find `join` in `futures` error[E0407]: method `poll_next` is not a member of trait `Stream` --> rust/lancedb/src/arrow.rs:108:5 | 108 | / fn poll_next( 109 | | self: Pin<&mut Self>, 110 | | cx: &mut std::task::Context<'_>, 111 | | ) -> std::task::Poll> { 112 | | let this = self.project(); 113 | | this.stream.poll_next(cx) 114 | | } | |_____^ not a member of trait `Stream` error[E0407]: method `poll_next` is not a member of trait `Stream` --> rust/lancedb/src/utils/mod.rs:362:5 | 362 | / fn poll_next( 363 | | mut self: std::pin::Pin<&mut Self>, 364 | | cx: &mut std::task::Context<'_>, 365 | | ) -> std::task::Poll> { ... | 391 | | } | |_____^ not a member of trait `Stream` error[E0407]: method `poll_next` is not a member of trait `Stream` --> rust/lancedb/src/utils/mod.rs:433:5 | 433 | / fn poll_next( 434 | | mut self: Pin<&mut Self>, 435 | | cx: &mut std::task::Context<'_>, 436 | | ) -> std::task::Poll> { ... | 470 | | } | |_____^ not a member of trait `Stream` error[E0425]: cannot find function `try_unfold` in module `futures::stream` --> rust/lancedb/src/remote/table/insert.rs:230:39 | 230 | let stream = futures::stream::try_unfold( | ^^^^^^^^^^ not found in `futures::stream` error[E0433]: cannot find `channel` in `futures` --> rust/lancedb/src/remote/table/insert.rs:418:22 | 418 | futures::channel::mpsc::channel::, std::io::Error>>(2); | ^^^^^^^ could not find `channel` in `futures` | error[E0425]: cannot find function `try_join_all` in module `futures::future` --> rust/lancedb/src/remote/table.rs:1062:40 | 1062 | let streams = futures::future::try_join_all(futures); | ^^^^^^^^^^^^ | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:76:1 | 76 | / pub fn join_all(i: I) -> JoinAll 77 | | where I: IntoIterator, 78 | | I::Item: IntoFuture, | |______________________________- similarly named function `join_all` defined here | error[E0425]: cannot find function `try_join_all` in module `futures::future` --> rust/lancedb/src/remote/table.rs:1660:40 | 1660 | let results = futures::future::try_join_all(futures).await?; | ^^^^^^^^^^^^ | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:76:1 | 76 | / pub fn join_all(i: I) -> JoinAll 77 | | where I: IntoIterator, 78 | | I::Item: IntoFuture, | |______________________________- similarly named function `join_all` defined here | error[E0425]: cannot find function `try_join_all` in module `futures::future` --> rust/lancedb/src/remote/table.rs:2243:43 | 2243 | let plan_texts = futures::future::try_join_all(futures).await?; | ^^^^^^^^^^^^ | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:76:1 | 76 | / pub fn join_all(i: I) -> JoinAll 77 | | where I: IntoIterator, 78 | | I::Item: IntoFuture, | |______________________________- similarly named function `join_all` defined here | error[E0425]: cannot find function `try_join_all` in module `futures::future` --> rust/lancedb/src/remote/table.rs:2290:53 | 2290 | let analyze_result_texts = futures::future::try_join_all(futures).await?; | ^^^^^^^^^^^^ | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:76:1 | 76 | / pub fn join_all(i: I) -> JoinAll 77 | | where I: IntoIterator, 78 | | I::Item: IntoFuture, | |______________________________- similarly named function `join_all` defined here | error[E0425]: cannot find function `try_unfold` in module `futures::stream` --> rust/lancedb/src/remote/util.rs:21:35 | 21 | let stream = futures::stream::try_unfold( | ^^^^^^^^^^ not found in `futures::stream` error[E0191]: the value of the associated type `Error` in `futures::Stream` must be specified --> rust/lancedb/src/arrow.rs:70:50 | 70 | pub type SendableRecordBatchStream = Pin>; | ^^^^^^^^^^^^^^^^^ | help: specify the associated type | 70 | pub type SendableRecordBatchStream = Pin + Send>>; | ++++++++++++++++++++ error[E0107]: type alias takes 0 lifetime arguments but 1 lifetime argument was supplied --> rust/lancedb/src/utils/background_cache.rs:15:31 | 15 | type SharedFut = Shared>>>; | ^^^^^^^^^ ------- help: remove the lifetime argument | | | expected 0 lifetime arguments | note: type alias defined here, with 0 lifetime parameters --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/mod.rs:106:14 | 106 | pub type BoxFuture = ::std::boxed::Box + Send>; | ^^^^^^^^^ error[E0107]: type alias takes 2 generic arguments but 1 generic argument was supplied --> rust/lancedb/src/utils/background_cache.rs:15:31 | 15 | type SharedFut = Shared>>>; | ^^^^^^^^^ ----------------- supplied 1 generic argument | | | expected 2 generic arguments | note: type alias defined here, with 2 generic parameters: `T`, `E` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/mod.rs:106:14 | 106 | pub type BoxFuture = ::std::boxed::Box + Send>; | ^^^^^^^^^ - - help: add missing generic argument | 15 | type SharedFut = Shared>, E>>; | +++ error[E0046]: not all trait items implemented, missing: `Error`, `poll` --> rust/lancedb/src/arrow.rs:105:1 | 105 | impl>> Stream for SimpleRecordBatchStream { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ missing `Error`, `poll` in implementation | = help: implement the missing item: `type Error = /* Type */;` = help: implement the missing item: `fn poll(&mut self) -> std::result::Result::Item>>, ::Error> { todo!() }` error[E0107]: type alias takes 0 lifetime arguments but 1 lifetime argument was supplied --> rust/lancedb/src/io/object_store.rs:97:46 | 97 | fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { | ^^^^^^^^^ ------- help: remove the lifetime argument | | | expected 0 lifetime arguments | note: type alias defined here, with 0 lifetime parameters --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:132:14 | 132 | pub type BoxStream = ::std::boxed::Box + Send>; | ^^^^^^^^^ error[E0107]: type alias takes 2 generic arguments but 1 generic argument was supplied --> rust/lancedb/src/io/object_store.rs:97:46 | 97 | fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { | ^^^^^^^^^ ------------------ supplied 1 generic argument | | | expected 2 generic arguments | note: type alias defined here, with 2 generic parameters: `T`, `E` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:132:14 | 132 | pub type BoxStream = ::std::boxed::Box + Send>; | ^^^^^^^^^ - - help: add missing generic argument | 97 | fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result, E> { | +++ error[E0107]: type alias takes 0 lifetime arguments but 1 lifetime argument was supplied --> rust/lancedb/src/io/object_store.rs:107:20 | 107 | locations: BoxStream<'static, Result>, | ^^^^^^^^^ ------- help: remove the lifetime argument | | | expected 0 lifetime arguments | note: type alias defined here, with 0 lifetime parameters --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:132:14 | 132 | pub type BoxStream = ::std::boxed::Box + Send>; | ^^^^^^^^^ error[E0107]: type alias takes 2 generic arguments but 1 generic argument was supplied --> rust/lancedb/src/io/object_store.rs:107:20 | 107 | locations: BoxStream<'static, Result>, | ^^^^^^^^^ ------------ supplied 1 generic argument | | | expected 2 generic arguments | note: type alias defined here, with 2 generic parameters: `T`, `E` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:132:14 | 132 | pub type BoxStream = ::std::boxed::Box + Send>; | ^^^^^^^^^ - - help: add missing generic argument | 107 | locations: BoxStream<'static, Result, E>, | +++ error[E0107]: type alias takes 0 lifetime arguments but 1 lifetime argument was supplied --> rust/lancedb/src/io/object_store.rs:108:10 | 108 | ) -> BoxStream<'static, Result> { | ^^^^^^^^^ ------- help: remove the lifetime argument | | | expected 0 lifetime arguments | note: type alias defined here, with 0 lifetime parameters --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:132:14 | 132 | pub type BoxStream = ::std::boxed::Box + Send>; | ^^^^^^^^^ error[E0107]: type alias takes 2 generic arguments but 1 generic argument was supplied --> rust/lancedb/src/io/object_store.rs:108:10 | 108 | ) -> BoxStream<'static, Result> { | ^^^^^^^^^ ------------ supplied 1 generic argument | | | expected 2 generic arguments | note: type alias defined here, with 2 generic parameters: `T`, `E` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:132:14 | 132 | pub type BoxStream = ::std::boxed::Box + Send>; | ^^^^^^^^^ - - help: add missing generic argument | 108 | ) -> BoxStream<'static, Result, E> { | +++ error[E0599]: no method named `map_err` found for struct `Pin>` in the current scope --> rust/lancedb/src/dataloader/permutation/builder.rs:208:32 | 208 | let stream = df_stream.map_err(|e| Error::Other { | ----------^^^^^^^ method not found in `Pin>` | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-util-0.3.32/src/stream/try_stream/mod.rs:248:8 | 248 | fn map_err(self, f: F) -> MapErr | ------- the method is available for `Pin>` here | error[E0599]: no method named `try_collect` found for struct `DatasetRecordBatchStream` in the current scope --> rust/lancedb/src/dataloader/permutation/reader.rs:220:28 | 220 | let batches = data.try_collect::>().await?; | ^^^^^^^^^^^ | error[E0599]: no method named `map_err` found for struct `DatasetRecordBatchStream` in the current scope --> rust/lancedb/src/dataloader/permutation/reader.rs:287:14 | 286 | let mut stream = row_ids | __________________________- 287 | | .map_err(Error::from) | | -^^^^^^^ method not found in `DatasetRecordBatchStream` | |_____________| | error[E0599]: the method `chain` exists for struct `futures::stream::Once<_, _>`, but its trait bounds were not satisfied --> rust/lancedb/src/dataloader/permutation/reader.rs:307:81 | 307 | let stream = futures::stream::once(std::future::ready(Ok(first_batch))).chain(stream); | ^^^^^ method cannot be called on `futures::stream::Once<_, _>` due to unsatisfied trait bounds error[E0308]: mismatched types --> rust/lancedb/src/dataloader/permutation/shuffle.rs:120:35 | 120 | futures::stream::once(async move { Ok(shuffled) }), | --------------------- ^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<_, _>`, found `async` block | | | arguments to this function are incorrect | = note: expected enum `std::result::Result<_, _>` found `async` block `{async block@rust/lancedb/src/dataloader/permutation/shuffle.rs:120:35: 120:45}` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 120 | futures::stream::once(Ok(async move { Ok(shuffled) })), | +++ + 120 | futures::stream::once(Err(async move { Ok(shuffled) })), | ++++ + error[E0271]: type mismatch resolving ` as IntoIterator>::Item == Result<_, _>` --> rust/lancedb/src/dataloader/permutation/shuffle.rs:228:44 | 228 | let stream = futures::stream::iter(0..num_files) | --------------------- ^^^^^^^^^^^^ expected `Result<_, _>`, found `u64` | | | required by a bound introduced by this call | = note: expected enum `std::result::Result<_, _>` found type `u64` note: required by a bound in `futures::stream::iter` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/iter.rs:31:27 | 30 | pub fn iter(i: J) -> Iter | ---- required by a bound in this function 31 | where J: IntoIterator>, | ^^^^^^^^^^^^^^^^^ required by this bound in `iter` error[E0599]: no method named `then` found for struct `IterStream` in the current scope --> rust/lancedb/src/dataloader/permutation/shuffle.rs:229:14 | 228 | let stream = futures::stream::iter(0..num_files) | ______________________- 229 | | .then(move |file_index| { | | -^^^^ method not found in `IterStream>` | |_____________| | error[E0599]: no method named `try_collect` found for struct `Pin>` in the current scope --> rust/lancedb/src/dataloader/permutation/shuffle.rs:258:26 | 250 | let batches = reader | ___________________________________- 251 | | .read_stream( 252 | | ReadBatchParams::RangeFull, 253 | | reader.num_rows() as u32, ... | 257 | | .await? 258 | | .try_collect::>() | |_________________________-^^^^^^^^^^^ error[E0599]: no method named `and_then` found for associated type `impl Future, ...>> + Send` in the current scope --> rust/lancedb/src/query.rs:766:14 | 765 | / self.create_plan(QueryExecutionOptions::default()) 766 | | .and_then(|plan| std::future::ready(Ok(plan.schema()))) | |_____________-^^^^^^^^ | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-util-0.3.32/src/future/try_future/mod.rs:395:8 | 395 | fn and_then(self, f: F) -> AndThen | -------- the method is available for `impl std::future::Future, error::Error>> + std::marker::Send` here error[E0599]: no method named `boxed` found for `async` block `{async block@rust/lancedb/src/query.rs:1492:33: 1492:43}` in the current scope --> rust/lancedb/src/query.rs:1493:18 | 1492 | let hybrid_result = async move { self.execute_hybrid(options).await } | _________________________________- 1493 | | .boxed() | | -^^^^^ method not found in `{async block@rust/lancedb/src/query.rs:1492:33: 1492:43}` | |_________________| error[E0271]: expected `{closure@blobs.rs:181:58}` to return `Result<_, _>`, but it returns `impl Future>` --> rust/lancedb/src/remote/table/blobs.rs:181:66 | 181 | futures::stream::iter(ranges.iter().cloned().map(|range| self.read_range(range))) | --------------------- ------- ^^^^^^^^^^^^^^^^^^^^^^ expected `Result<_, _>`, found future | | | | | this closure | required by a bound introduced by this call error[E0599]: no method named `buffered` found for struct `IterStream` in the current scope --> rust/lancedb/src/remote/table/blobs.rs:182:14 | 181 | / futures::stream::iter(ranges.iter().cloned().map(|range| self.read_range(range))) 182 | | .buffered(BLOB_REQUEST_CONCURRENCY) | | -^^^^^^^^ method not found in `Iter>>, {closure@...}>>` | |_____________| error[E0599]: no method named `try_next` found for struct `Pin>` in the current scope --> rust/lancedb/src/remote/table/blobs.rs:379:40 | 379 | while let Some(batch) = stream.try_next().await? { | ^^^^^^^^ method not found in `Pin>` error[E0271]: type mismatch resolving ` as IntoIterator>::Item == Result<_, _>` --> rust/lancedb/src/remote/table/blobs.rs:481:27 | 481 | futures::stream::iter(probe_futures) | --------------------- ^^^^^^^^^^^^^ expected `Result<_, _>`, found future | | | required by a bound introduced by this call error[E0599]: no method named `buffered` found for struct `IterStream` in the current scope --> rust/lancedb/src/remote/table/blobs.rs:482:10 | 481 | / futures::stream::iter(probe_futures) 482 | | .buffered(BLOB_REQUEST_CONCURRENCY) | | -^^^^^^^^ method not found in `Iter>>>` | |_________| error[E0599]: no method named `next` found for struct `Pin>` in the current scope --> rust/lancedb/src/remote/table/insert.rs:324:37 | 324 | let mut first = match input.next().await { | ^^^^ method not found in `Pin>` error[E0599]: no method named `next` found for struct `Pin>` in the current scope --> rust/lancedb/src/remote/table/insert.rs:345:33 | 345 | first = match input.next().await { | ^^^^ method not found in `Pin>` error[E0599]: the method `next` exists for mutable reference `&mut Pin>`, but its trait bounds were not satisfied --> rust/lancedb/src/remote/table/insert.rs:446:41 | 446 | None => match input.next().await { | ^^^^ method cannot be called on `&mut Pin>` due to unsatisfied trait bounds | = note: the following trait bounds were not satisfied: `Pin>: Iterator` which is required by `&mut Pin>: Iterator` error[E0599]: no method named `map_err` found for struct `IterStream` in the current scope --> rust/lancedb/src/remote/table.rs:688:53 | 688 | let stream = futures::stream::iter(batches).map_err(DataFusionError::from); | ^^^^^^^ method not found in `Iter> + Send>>` error[E0599]: no method named `try_collect` found for struct `Pin>` in the current scope --> rust/lancedb/src/remote/table.rs:1378:49 | 1378 | let result: Result> = stream.try_collect().await.map_err(Error::from); | ^^^^^^^^^^^ error[E0599]: no method named `next` found for struct `Pin>` in the current scope --> rust/lancedb/src/remote/table.rs:1509:48 | 1509 | while let Some(batch) = stream.next().await { | ^^^^ method not found in `Pin>` error[E0599]: no method named `boxed` found for opaque type `impl Future>` in the current scope --> rust/lancedb/src/table/delete.rs:35:51 | 35 | let delete_result = dataset.delete(s).boxed().await?; | ^^^^^ method not found in `impl Future>` error[E0599]: no variant, associated function, or constant named `Left` found for enum `Either` in the current scope --> rust/lancedb/src/table/merge.rs:292:17 | 292 | Either::Left(tokio::time::timeout(timeout, future).map(|res| match res { | ^^^^ variant, associated function, or constant not found in `Either<_, _>` error[E0599]: `Timeout, ...), ...>>>` is not an iterator --> rust/lancedb/src/table/merge.rs:292:60 | 292 | Either::Left(tokio::time::timeout(timeout, future).map(|res| match res { | --------------------------------------^^^ `Timeout, ...), ...>>>` is not an iterator | ::: $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/pin-project-lite-0.2.17/src/lib.rs:745:9 | 745 | / $vis struct $ident $($def_generics)* 746 | | $(where 747 | | $($where_clause)*)? ... | 751 | | ),+ 752 | | } | |_________- doesn't satisfy `_: Iterator` | = note: the following trait bounds were not satisfied: `tokio::time::Timeout, MergeStats), lance::Error>>>: Iterator` which is required by `&mut tokio::time::Timeout, MergeStats), lance::Error>>>: Iterator` error[E0599]: no variant, associated function, or constant named `Right` found for enum `Either` in the current scope --> rust/lancedb/src/table/merge.rs:301:17 | 301 | Either::Right(job.execute_reader(new_data).map_err(|e| e.into())) | ^^^^^ variant, associated function, or constant not found in `Either<_, _>` error[E0599]: no method named `map_err` found for opaque type `impl Future, ...), ...>>` in the current scope --> rust/lancedb/src/table/merge.rs:301:52 | 301 | Either::Right(job.execute_reader(new_data).map_err(|e| e.into())) | ^^^^^^^ method not found in `impl Future, ...), ...>>` error[E0277]: the trait bound `Iter, ...>>: Stream` is not satisfied --> rust/lancedb/src/table/query.rs:681:38 | 681 | Ok(DatasetRecordBatchStream::new(record_batch_stream)) | ^^^^^^^^^^^^^^^^^^^ the trait `futures_core::stream::Stream` is not implemented for `Iter, ...>>` error[E0277]: the trait bound `TimeoutStream: futures_core::stream::Stream` is not satisfied --> rust/lancedb/src/utils/mod.rs:353:28 | 353 | impl RecordBatchStream for TimeoutStream { | ^^^^^^^^^^^^^ unsatisfied trait bound error[E0046]: not all trait items implemented, missing: `Error`, `poll` --> rust/lancedb/src/utils/mod.rs:359:1 | 359 | impl Stream for TimeoutStream { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ missing `Error`, `poll` in implementation | = help: implement the missing item: `type Error = /* Type */;` = help: implement the missing item: `fn poll(&mut self) -> std::result::Result::Item>>, ::Error> { todo!() }` error[E0277]: the trait bound `MaxBatchLengthStream: futures_core::stream::Stream` is not satisfied --> rust/lancedb/src/utils/mod.rs:424:28 | 424 | impl RecordBatchStream for MaxBatchLengthStream { | ^^^^^^^^^^^^^^^^^^^^ unsatisfied trait bound error[E0046]: not all trait items implemented, missing: `Error`, `poll` --> rust/lancedb/src/utils/mod.rs:430:1 | 430 | impl Stream for MaxBatchLengthStream { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ missing `Error`, `poll` in implementation | = help: implement the missing item: `type Error = /* Type */;` = help: implement the missing item: `fn poll(&mut self) -> std::result::Result::Item>>, ::Error> { todo!() }` error[E0599]: no method named `map` found for type parameter `I` in the current scope --> rust/lancedb/src/arrow.rs:75:45 | 72 | impl From for SendableRecordBatchStream { | - method `map` not found for this type parameter ... 75 | let mapped_stream = Box::pin(stream.map(|r| r.map_err(Into::into))); | ^^^ error[E0599]: no method named `poll_next` found for struct `Pin<&mut S>` in the current scope --> rust/lancedb/src/arrow.rs:113:21 | 113 | this.stream.poll_next(cx) | ^^^^^^^^^ | = help: items from traits can only be used if the trait is implemented and in scope = note: the following traits define an item `poll_next`, perhaps you need to implement one of them: candidate #1: `futures_core::stream::Stream` candidate #2: `sorts::stream::PartitionedStream` help: there is a method `collect` with a similar name, but with different arguments --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:563:5 | 563 | / fn collect(self) -> Collect 564 | | where Self: Sized | |_________________________^ error[E0599]: the method `map_err` exists for struct `Pin> + Send>>`, but its trait bounds were not satisfied --> rust/lancedb/src/arrow.rs:150:29 | 150 | let stream = stream.map_err(|err| Error::Arrow { source: err }); | ^^^^^^^ method cannot be called due to unsatisfied trait bounds error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:80:26 | 80 | stream: once(async move { Ok(batch) }), | ---- ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<_, _>`, found `async` block | | | arguments to this function are incorrect | = note: expected enum `std::result::Result<_, _>` found `async` block `{async block@rust/lancedb/src/data/scannable.rs:80:26: 80:36}` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 80 | stream: once(Ok(async move { Ok(batch) })), | +++ + 80 | stream: once(Err(async move { Ok(batch) })), | ++++ + error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:107:30 | 107 | stream: once(async { | _________________________----_^ | | | | | arguments to this function are incorrect 108 | | Err(Error::InvalidInput { 109 | | message: "Cannot scan an empty Vec".to_string(), 110 | | }) 111 | | }), | |_________________^ expected `Result<_, _>`, found `async` block | = note: expected enum `std::result::Result<_, _>` found `async` block `{async block@rust/lancedb/src/data/scannable.rs:107:30: 107:35}` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 107 ~ stream: once(Ok(async { 108 | Err(Error::InvalidInput { 109 | message: "Cannot scan an empty Vec".to_string(), 110 | }) 111 ~ })), | 107 ~ stream: once(Err(async { 108 | Err(Error::InvalidInput { 109 | message: "Cannot scan an empty Vec".to_string(), 110 | }) 111 ~ })), | error[E0271]: expected `Ok` to return `Result, _>`, but it returns `Result` --> rust/lancedb/src/data/scannable.rs:117:52 | 117 | Box::pin(SimpleRecordBatchStream { schema, stream }) | ^^^^^^ expected `Result, _>`, found `Result` error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:158:59 | 158 | let stream = futures::stream::unfold(rx, |mut rx| async move { | ___________________________________________________________^ 159 | | rx.recv().await.map(|batch| (batch, rx)) 160 | | }) | |_________^ expected `Option<_>`, found `async` block | = note: expected enum `std::option::Option<_>` found `async` block `{async block@rust/lancedb/src/data/scannable.rs:158:59: 158:69}` help: try wrapping the expression in `Some` | 158 ~ let stream = futures::stream::unfold(rx, |mut rx| Some(async move { 159 | rx.recv().await.map(|batch| (batch, rx)) 160 ~ })) | error[E0599]: the method `fuse` exists for struct `Unfold>, ..., _>`, but its trait bounds were not satisfied --> rust/lancedb/src/data/scannable.rs:161:10 | 158 | let stream = futures::stream::unfold(rx, |mut rx| async move { | ______________________- 159 | | rx.recv().await.map(|batch| (batch, rx)) 160 | | }) 161 | | .fuse(); | | -^^^^ method cannot be called due to unsatisfied trait bounds | |_________| error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:178:26 | 178 | stream: once(async { | _____________________----_^ | | | | | arguments to this function are incorrect 179 | | Err(Error::InvalidInput { 180 | | message: "Stream has already been consumed".to_string(), 181 | | }) 182 | | }), | |_____________^ expected `Result<_, _>`, found `async` block | = note: expected enum `std::result::Result<_, _>` found `async` block `{async block@rust/lancedb/src/data/scannable.rs:178:26: 178:31}` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 178 ~ stream: once(Ok(async { 179 | Err(Error::InvalidInput { 180 | message: "Stream has already been consumed".to_string(), 181 | }) 182 ~ })), | 178 ~ stream: once(Err(async { 179 | Err(Error::InvalidInput { 180 | message: "Stream has already been consumed".to_string(), 181 | }) 182 ~ })), | error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:474:53 | 474 | let prepend = futures::stream::once(std::future::ready(Ok(batch))); | --------------------- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<_, _>`, found `Ready>` | | | arguments to this function are incorrect | = note: expected enum `std::result::Result<_, _>` found struct `std::future::Ready>` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 474 | let prepend = futures::stream::once(Ok(std::future::ready(Ok(batch)))); | +++ + 474 | let prepend = futures::stream::once(Err(std::future::ready(Ok(batch)))); | ++++ + error[E0599]: the method `chain` exists for struct `futures::stream::Once<_, _>`, but its trait bounds were not satisfied --> rust/lancedb/src/data/scannable.rs:477:37 | 477 | stream: prepend.chain(rest), | ^^^^^ method cannot be called on `futures::stream::Once<_, _>` due to unsatisfied trait bounds error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:482:47 | 482 | stream: futures::stream::once(std::future::ready(Ok(batch))), | --------------------- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<_, _>`, found `Ready>` | | | arguments to this function are incorrect | = note: expected enum `std::result::Result<_, _>` found struct `std::future::Ready>` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 482 | stream: futures::stream::once(Ok(std::future::ready(Ok(batch)))), | +++ + 482 | stream: futures::stream::once(Err(std::future::ready(Ok(batch)))), | ++++ + error[E0308]: mismatched types --> rust/lancedb/src/data/scannable.rs:486:56 | 486 | let stream = futures::stream::once(std::future::ready(err)); | --------------------- ^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<_, _>`, found `Ready>` | | | arguments to this function are incorrect | = note: expected enum `std::result::Result<_, _>` found struct `std::future::Ready>` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 486 | let stream = futures::stream::once(Ok(std::future::ready(err))); | +++ + 486 | let stream = futures::stream::once(Err(std::future::ready(err))); | ++++ + error[E0599]: no method named `and_then` found for struct `Pin> + Send>>` in the current scope --> rust/lancedb/src/io/object_store.rs:153:32 | 153 | Box::pin(put_secondary.and_then(|_| put_primary)) | ^^^^^^^^ error[E0271]: expected `IntoIter, 1>` to be an iterator that yields `Result, _>`, but it yields `Result` --> rust/lancedb/src/query.rs:1465:25 | 1465 | return Box::pin(SimpleRecordBatchStream::new( | ^^^^^^^^^^^^^^^^^^^^^^^ expected `Result, _>`, found `Result` error[E0271]: expected `IntoIter>` to be an iterator that yields `Result, _>`, but it yields `Result` --> rust/lancedb/src/query.rs:1478:14 | 1478 | Box::pin(SimpleRecordBatchStream::new(stream::iter(batches), schema)) | ^^^^^^^^^^^^^^^^^^^^^^^ expected `Result, _>`, found `Result` error[E0308]: mismatched types --> rust/lancedb/src/remote/table/insert.rs:626:44 | 626 | let stream = futures::stream::once(async move { | ______________________---------------------_^ | | | | | arguments to this function are incorrect ... | 791 | | Ok::<_, DataFusionError>(batch) 792 | | }); | |_________^ expected `Result<_, _>`, found `async` block | = note: expected enum `std::result::Result<_, _>` found `async` block `{async block@rust/lancedb/src/remote/table/insert.rs:626:44: 626:54}` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 626 ~ let stream = futures::stream::once(Ok(async move { 627 | // Multipart writes with a byte budget split the partition into ... 791 | Ok::<_, DataFusionError>(batch) 792 ~ })); | 626 ~ let stream = futures::stream::once(Err(async move { 627 | // Multipart writes with a byte budget split the partition into ... 791 | Ok::<_, DataFusionError>(batch) 792 ~ })); | error[E0277]: the trait bound `futures::stream::Once<_, _>: futures_core::stream::Stream` is not satisfied --> rust/lancedb/src/remote/table/insert.rs:794:12 | 794 | Ok(Box::pin(RecordBatchStreamAdapter::new( | ____________^ 795 | | COUNT_SCHEMA.clone(), 796 | | stream, 797 | | ))) | |__________^ the trait `futures_core::stream::Stream` is not implemented for `futures::stream::Once<_, _>` error[E0599]: no method named `try_collect` found for struct `Pin>` in the current scope --> rust/lancedb/src/remote/table.rs:2442:49 | 2442 | let result: Result> = stream.try_collect().await.map_err(Error::from); | ^^^^^^^^^^^ error[E0277]: the trait bound `impl Stream>: TryStream` is not satisfied --> rust/lancedb/src/remote/util.rs:47:35 | 47 | Ok(reqwest::Body::wrap_stream(stream)) | -------------------------- ^^^^^^ unsatisfied trait bound | | | required by a bound introduced by this call error[E0599]: no method named `map_ok` found for struct `Pin>` in the current scope --> rust/lancedb/src/table/datafusion/insert.rs:200:30 | 200 | input_stream.map_ok(move |batch| { | -------------^^^^^^ method not found in `Pin>` error[E0308]: mismatched types --> rust/lancedb/src/table/datafusion/insert.rs:208:44 | 208 | let stream = futures::stream::once(async move { | ______________________---------------------_^ | | | | | arguments to this function are incorrect 209 | | if let Some(tracker) = tracker 210 | | && write_params.write_progress.is_none() ... | 255 | | )?) 256 | | }); | |_________^ expected `Result<_, _>`, found `async` block | = note: expected enum `std::result::Result<_, _>` found `async` block `{async block@rust/lancedb/src/table/datafusion/insert.rs:208:44: 208:54}` note: function defined here --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/once.rs:20:8 | 20 | pub fn once(item: Result) -> Once { | ^^^^ help: try wrapping the expression in a variant of `std::result::Result` | 208 ~ let stream = futures::stream::once(Ok(async move { 209 | if let Some(tracker) = tracker ... 255 | )?) 256 ~ })); | 208 ~ let stream = futures::stream::once(Err(async move { 209 | if let Some(tracker) = tracker ... 255 | )?) 256 ~ })); | error[E0277]: the trait bound `futures::stream::Once<_, _>: futures_core::stream::Stream` is not satisfied --> rust/lancedb/src/table/datafusion/insert.rs:258:12 | 258 | Ok(Box::pin(RecordBatchStreamAdapter::new( | ____________^ 259 | | COUNT_SCHEMA.clone(), 260 | | stream, 261 | | ))) | |__________^ the trait `futures_core::stream::Stream` is not implemented for `futures::stream::Once<_, _>` error[E0599]: no method named `map_ok` found for struct `Pin>` in the current scope --> rust/lancedb/src/table/datafusion.rs:128:29 | 128 | let stream = stream.map_ok(move |batch| { | -------^^^^^^ method not found in `Pin>` error[E0599]: no method named `map_err` found for struct `Pin, ...>> + Send>>` in the current scope --> rust/lancedb/src/table/datafusion.rs:245:14 | 242 | let plan = self | ____________________- 243 | | .table 244 | | .create_plan(&AnyQuery::Query(query), options) 245 | | .map_err(|err| DataFusionError::External(err.into())) | | -^^^^^^^ method not found in `Pin, ...>> + Send>>` | |_____________| error[E0599]: no method named `next` found for struct `Pin>` in the current scope --> rust/lancedb/src/table.rs:3048:48 | 3048 | while let Some(batch) = stream.next().await { | ^^^^ method not found in `Pin>` error[E0277]: the trait bound `JoinHandle>: Future` is not satisfied --> rust/lancedb/src/table.rs:3038:23 | 3038 | let handles = FuturesUnordered::new(); | ^^^^^^^^^^^^^^^^^^^^^^^ the trait `futures::Future` is not implemented for `tokio::task::JoinHandle>` error[E0277]: `FuturesUnordered>>` is not an iterator --> rust/lancedb/src/table.rs:3054:23 | 3054 | for handle in handles { | ^^^^^^^ `FuturesUnordered>>` is not an iterator error[E0277]: the trait bound `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}: futures::IntoFuture` is not satisfied --> rust/lancedb/src/table.rs:3450:13 | 3449 | let mut sorted_sizes = join_all( | -------- required by a bound introduced by this call 3450 | / frags 3451 | | .iter() 3452 | | .map(|frag| async move { frag.physical_rows().await.unwrap_or(0) }), | |___________________________________________________________________________________^ the trait `futures::Future` is not implemented for `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` | = note: `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` implements similarly named trait `std::future::Future`, but not `futures::Future` = help: the following other types implement trait `futures::Future`: &'a mut F AssertUnwindSafe BiLockAcquire Box Concat2 Either Finished Fold and 43 others = note: required for `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` to implement `futures::IntoFuture` note: required by a bound in `join_all` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:78:20 | 76 | pub fn join_all(i: I) -> JoinAll | -------- required by a bound in this function 77 | where I: IntoIterator, 78 | I::Item: IntoFuture, | ^^^^^^^^^^ required by this bound in `join_all` error[E0277]: the trait bound `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}: futures::Future` is not satisfied --> rust/lancedb/src/table.rs:3449:32 | 3449 | let mut sorted_sizes = join_all( | ________________________________^ 3450 | | frags 3451 | | .iter() 3452 | | .map(|frag| async move { frag.physical_rows().await.unwrap_or(0) }), 3453 | | ) | |_________^ the trait `futures::Future` is not implemented for `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` | = note: `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` implements similarly named trait `std::future::Future`, but not `futures::Future` = help: the following other types implement trait `futures::Future`: &'a mut F AssertUnwindSafe BiLockAcquire Box Concat2 Either Finished Fold and 43 others = note: required for `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` to implement `futures::IntoFuture` note: required by a bound in `JoinAll` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:24:20 | 22 | pub struct JoinAll | ------- required by a bound in this struct 23 | where I: IntoIterator, 24 | I::Item: IntoFuture, | ^^^^^^^^^^ required by this bound in `JoinAll` error[E0277]: `JoinAll, {closure@...}>>` is not a future --> rust/lancedb/src/table.rs:3454:10 | 3449 | let mut sorted_sizes = join_all( | ________________________________- 3450 | | frags 3451 | | .iter() 3452 | | .map(|frag| async move { frag.physical_rows().await.unwrap_or(0) }), 3453 | | ) | |_________- this call returns `JoinAll, {closure@rust/lancedb/src/table.rs:3452:22: 3452:28}>>` 3454 | .await; | ^^^^^ `JoinAll, {closure@...}>>` is not a future error[E0277]: the trait bound `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}: futures::Future` is not satisfied --> rust/lancedb/src/table.rs:3454:10 | 3454 | .await; | ^^^^^ the trait `futures::Future` is not implemented for `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` | = note: `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` implements similarly named trait `std::future::Future`, but not `futures::Future` = help: the following other types implement trait `futures::Future`: &'a mut F AssertUnwindSafe BiLockAcquire Box Concat2 Either Finished Fold and 43 others = note: required for `{async block@rust/lancedb/src/table.rs:3452:29: 3452:39}` to implement `futures::IntoFuture` note: required by a bound in `JoinAll` --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/future/join_all.rs:24:20 | 22 | pub struct JoinAll | ------- required by a bound in this struct 23 | where I: IntoIterator, 24 | I::Item: IntoFuture, | ^^^^^^^^^^ required by this bound in `JoinAll` error[E0282]: type annotations needed --> rust/lancedb/src/utils/background_cache.rs:119:40 | 119 | inner: Arc::new(Mutex::new(CacheInner { | ________________________________________^ 120 | | state: State::Empty, 121 | | generation: 0, 122 | | })), | |_____________^ cannot infer type of the type parameter `E` declared on the struct `CacheInner` | help: consider specifying the generic arguments | 119 | inner: Arc::new(Mutex::new(CacheInner:: { | ++++++++ error[E0282]: type annotations needed --> rust/lancedb/src/utils/background_cache.rs:134:9 | 134 | cache.state.fresh_value(self.ttl, self.refresh_window) | ^^^^^^^^^^^ cannot infer type for type parameter `E` error[E0282]: type annotations needed --> rust/lancedb/src/utils/background_cache.rs:173:23 | 173 | cache.state = State::Current(value, clock::now()); | ^^^^^^^^^^^^^^ cannot infer type of the type parameter `E` declared on the enum `State` | help: consider specifying the generic arguments | 173 | cache.state = State::::Current(value, clock::now()); | ++++++++ error[E0282]: type annotations needed --> rust/lancedb/src/utils/background_cache.rs:182:23 | 182 | cache.state = State::Empty; | ^^^^^^^^^^^^ cannot infer type of the type parameter `E` declared on the enum `State` | help: consider specifying the generic arguments | 182 | cache.state = State::::Empty; | ++++++++ error[E0599]: no method named `boxed` found for `async` block `{async block@rust/lancedb/src/utils/background_cache.rs:269:22: 269:32}` in the current scope --> rust/lancedb/src/utils/background_cache.rs:270:14 | 269 | let shared = async move { (fetch)().await.map_err(Arc::new) } | ______________________- 270 | | .boxed() | | -^^^^^ method not found in `{async block@rust/lancedb/src/utils/background_cache.rs:269:22: 269:32}` | |_____________| error[E0277]: the trait bound `TimeoutStream: futures_core::stream::Stream` is not satisfied --> rust/lancedb/src/utils/mod.rs:345:9 | 345 | Box::pin(Self::new(inner, timeout)) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ unsatisfied trait bound error[E0599]: no method named `poll_next` found for struct `Pin<&mut TimeoutStream>` in the current scope --> rust/lancedb/src/utils/mod.rs:376:22 | 376 | self.poll_next(cx) | ^^^^^^^^^ | = help: items from traits can only be used if the trait is implemented and in scope = note: the following traits define an item `poll_next`, perhaps you need to implement one of them: candidate #1: `futures_core::stream::Stream` candidate #2: `sorts::stream::PartitionedStream` help: there is a method `collect` with a similar name, but with different arguments --> $CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/futures-0.1.31/src/stream/mod.rs:563:5 | 563 | / fn collect(self) -> Collect 564 | | where Self: Sized | |_________________________^ error[E0599]: no method named `poll_unpin` found for mutable reference `&mut Pin>` in the current scope --> rust/lancedb/src/utils/mod.rs:378:75 | 378 | TimeoutState::Started { deadline, timeout } => match deadline.poll_unpin(cx) { | ^^^^^^^^^^ method not found in `&mut Pin>` error[E0599]: no method named `poll_next` found for struct `Pin<&mut Pin>>` in the current scope --> rust/lancedb/src/utils/mod.rs:386:27 | 386 | inner.poll_next(cx) | ^^^^^^^^^ error[E0277]: the trait bound `MaxBatchLengthStream: futures_core::stream::Stream` is not satisfied --> rust/lancedb/src/utils/mod.rs:419:13 | 419 | Box::pin(Self::new(inner, max_batch_length)) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ unsatisfied trait bound error[E0599]: no method named `poll_next` found for struct `Pin<&mut Pin>>` in the current scope --> rust/lancedb/src/utils/mod.rs:439:50 | 439 | return Pin::new(&mut self.inner).poll_next(cx); | ^^^^^^^^^ error[E0599]: no method named `poll_next` found for struct `Pin<&mut Pin>>` in the current scope --> rust/lancedb/src/utils/mod.rs:459:45 | 459 | match Pin::new(&mut self.inner).poll_next(cx) { | ^^^^^^^^^ Some errors have detailed explanations: E0046, E0107, E0191, E0271, E0277, E0282, E0308, E0407, E0425... For more information about an error, try `rustc --explain E0046`. error: could not compile `lancedb` (lib) due to 118 previous errors ``` --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a879d1f8b..78474cd16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,7 @@ env_logger = "0.11" half = { "version" = "2.7.1", default-features = false, features = [ "num-traits", ] } -futures = "0" +futures = "0.3" log = "0.4" metrics = "0.24" metrics-util = "0.19" From c5f9efefe9396c058b3ce6aa2298c0ef635477fb Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:30:55 +0800 Subject: [PATCH 30/41] test(python): cover local sync multiple-vector search (#3830) ## Summary - add regression coverage for multiple query vectors in the local synchronous Python API - verify that each query vector receives its own limited nearest-neighbor result and `query_index` ## Root cause In LanceDB v0.16, the local synchronous scanner passed a nested vector array as one query, unlike the async and remote implementations. The subsequent sync-to-async table migration supplied the correct shared runtime path, but this local sync behavior was never regression-tested and issue #1857 remained open. ## Validation - `uv run --extra tests pytest python/tests/test_query.py::test_query_multiple_vectors -q` - `uv run --project python --extra tests --extra dev ruff format --check python/python/tests/test_query.py` - `uv run --project python --extra tests --extra dev ruff check .` Fixes #1857 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_query.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/python/tests/test_query.py b/python/python/tests/test_query.py index 6840be052..d2629d1a8 100644 --- a/python/python/tests/test_query.py +++ b/python/python/tests/test_query.py @@ -570,6 +570,15 @@ def test_query_builder(table): assert all(np.array(rs[0]["vector"]) == [1, 2]) +def test_query_multiple_vectors(table): + results = table.search([np.array([1, 2]), np.array([4, 5])]).limit(1).to_list() + + assert len(results) == 2 + results_by_query = {result["query_index"]: result for result in results} + assert results_by_query[0]["id"] == 1 + assert results_by_query[1]["id"] == 2 + + def test_with_row_id(table: lancedb.table.Table): rs = table.search().with_row_id(True).to_arrow() assert "_rowid" in rs.column_names From 2922c171f7feecc6ce6ab63621fb4e3791e36420 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:31:08 +0800 Subject: [PATCH 31/41] test(rust): cover Azure table URI separators (#3837) ## Root cause The former listing-database table URI builder used OS-native `Path::join` for object-store URIs. On Windows this inserted backslashes into `az://` table paths, so `table_names` found slash-delimited objects while `open_table` addressed a different key. The production path now builds URI paths with forward slashes after the equivalent S3 report was fixed in #2575, but #1072 remained open without Azure-specific regression coverage. ## Fix - Add Azure URI regression assertions at the Rust table URI construction boundary. - Cover connection bases both with and without a trailing slash, matching the behavior reported in #1072. - Verify the resulting table URI always uses forward slashes on every platform. ## Validation - `cargo fmt --all -- --check` - `cargo test --quiet -p lancedb --lib database::listing::tests::test_table_uri` - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` - `cargo test --quiet --features remote --tests` (866 passed, 1 ignored) Fixes #1072 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/database/listing.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index fea34bb48..a4624112e 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -2342,7 +2342,7 @@ mod tests { #[tokio::test] async fn test_table_uri() { - let (_tempdir, db) = setup_database().await; + let (_tempdir, mut db) = setup_database().await; let mut pb = PathBuf::new(); pb.push(db.uri.clone()); @@ -2351,6 +2351,18 @@ mod tests { let expected = pb.to_str().unwrap(); let uri = db.table_uri("test").ok().unwrap(); assert_eq!(uri, expected); + + // URI paths always use forward slashes, even on Windows. Using + // `Path::join` here used to produce `az://container/prefix\\test.lance`, + // which Azure treated as a different object from the table returned by + // `table_names` (https://github.com/lancedb/lancedb/issues/1072). + for base_uri in ["az://container/prefix", "az://container/prefix/"] { + db.uri = base_uri.to_string(); + assert_eq!( + db.table_uri("test").unwrap(), + "az://container/prefix/test.lance" + ); + } } /// Regression: connecting via a URL-style URI (which goes through From 4048150fdd3bc19d7c651f1e4e7e16c82b6b5888 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:31:19 +0800 Subject: [PATCH 32/41] test(python): cover nullable fixed-size-list ingestion (#3812) ## Summary - add regression coverage for adding dictionary rows with a nullable fixed-size-list column - verify ordinary list columns remain aligned alongside the null fixed-size-list value ## Root cause PyArrow infers an all-`None` dictionary column as the generic `null` type. The original schema-alignment path treated the target fixed-size-list type as proof that the inferred source was also list-like and unconditionally accessed `value_field`, which raised `AttributeError`. Current alignment logic correctly falls back to the target type when the source is not list-like; this test locks in that repair for the reported ingestion path. ## Validation - `uv run --extra tests pytest python/tests/test_table.py::test_add_with_empty_fixed_size_list_drops_bad_rows python/tests/test_table.py::test_add_nullable_fixed_size_list_with_none python/tests/test_table.py::test_add_nullable_struct_with_none -q` - `uv run --with pyarrow==19.0.1 --extra tests pytest python/tests/test_table.py::test_add_nullable_fixed_size_list_with_none -q` - `uv run --project python --extra dev ruff format --check python/python/tests/test_table.py` - `uv run --project python --extra dev ruff check .` Fixes #2340 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index b2bfa2a68..e5c4ad801 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -1845,6 +1845,27 @@ def test_add_with_empty_fixed_size_list_drops_bad_rows(mem_db: DBConnection): assert np.allclose(data["embedding"].to_pylist()[0], np.array([0.1] * 16)) +def test_add_nullable_fixed_size_list_with_none(mem_db: DBConnection): + """Regression test for issue #2340.""" + table = mem_db.create_table( + "test_nullable_fixed_size_list", + schema=pa.schema( + [ + pa.field("id", pa.string()), + pa.field("feature", pa.list_(pa.float32(), 256)), + pa.field("tags", pa.list_(pa.string())), + ] + ), + ) + + table.add([{"id": "1", "feature": None, "tags": ["tag1", "tag2"]}]) + + result = table.to_arrow() + assert result.to_pylist() == [ + {"id": "1", "feature": None, "tags": ["tag1", "tag2"]} + ] + + def test_add_nullable_struct_with_none(mem_db: DBConnection): """Regression test for issue #2654: a nullable struct column whose first batch contains only None values must not crash in From fc44535ceeac0fd73ae94d0a0151c35753cb3e40 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:31:30 +0800 Subject: [PATCH 33/41] fix(python): clarify bare Vector annotations (#3809) ## Summary - raise a clear `TypeError` when `Vector` is used without a dimension - preserve normal `Vector(dim)` behavior across Pydantic v1 and v2 - add a regression test that defines a model without importing PyArrow ## Root cause Pydantic interpreted the bare `Vector` factory as a callable field type and inspected its postponed annotations in the user model's namespace. Because that namespace did not define LanceDB's internal `pa` alias, model construction failed with the misleading `NameError: name 'pa' is not defined` instead of explaining that `Vector` must be parameterized. The factory now exposes Pydantic's v1 and v2 schema hooks and rejects bare use before signature introspection with guidance to use `Vector(dim)`. ## Validation - `uvx --from 'ruff==0.15.20' ruff check .` - `uvx --from 'ruff==0.15.20' ruff format --check python/python/lancedb/pydantic.py python/python/tests/test_pydantic.py` - `cd python && uv run --extra tests pytest python/tests/test_pydantic.py::test_bare_vector_raises_clear_error -q` - `cd python && uv run --extra tests pytest python/tests/test_pydantic.py -q` - compatibility checks with Pydantic 1.10.22, 2.11.4, and 2.13.4 Fixes #2384 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/lancedb/pydantic.py | 10 ++++++++++ python/python/tests/test_pydantic.py | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/python/python/lancedb/pydantic.py b/python/python/lancedb/pydantic.py index c4dedc0e6..1ab6e6fcc 100644 --- a/python/python/lancedb/pydantic.py +++ b/python/python/lancedb/pydantic.py @@ -153,6 +153,16 @@ def Vector( return FixedSizeList +def _raise_bare_vector_error(*_args): + raise TypeError("Vector must be parameterized with a dimension, e.g. Vector(128).") + + +# Pydantic v1 and v2 otherwise treat the bare Vector factory as a field validator +# and inspect its signature, which produces misleading errors about internal types. +setattr(Vector, "__get_validators__", _raise_bare_vector_error) +setattr(Vector, "__get_pydantic_core_schema__", _raise_bare_vector_error) + + def MultiVector( dim: int, value_type: pa.DataType = pa.float32(), nullable: bool = True ) -> Type: diff --git a/python/python/tests/test_pydantic.py b/python/python/tests/test_pydantic.py index e1d533784..db93d7c64 100644 --- a/python/python/tests/test_pydantic.py +++ b/python/python/tests/test_pydantic.py @@ -415,6 +415,17 @@ def test_nullable_vector(): assert schema == pa.schema([pa.field("vec", pa.list_(pa.float32(), 16), True)]) +def test_bare_vector_raises_clear_error(): + namespace = { + "__name__": "test_model_without_pyarrow", + "LanceModel": LanceModel, + "Vector": Vector, + } + + with pytest.raises(TypeError, match=r"Vector must be parameterized.*Vector\(128\)"): + exec("class TestModel(LanceModel):\n vector: Vector", namespace) + + def test_fixed_size_list_field(): class TestModel(pydantic.BaseModel): vec: Vector(16) From ec80acb668b361e395ecfa5ee381391bd3b3b859 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:31:42 +0800 Subject: [PATCH 34/41] fix(python): expose inline types to downstream checkers (#3817) ## Summary - publish the PEP 561 `py.typed` marker so downstream type checkers consume the inline public annotations - add a Pyright contract test that distinguishes synchronous `connect` from awaited `connect_async` - verify the marker is present in the installed package ## Root cause The public Python module already annotated `lancedb.connect` as synchronous and `lancedb.connect_async` as asynchronous. The private native `_lancedb.connect` stub is intentionally awaitable because it backs `connect_async`. However, the distribution did not include a PEP 561 marker, so downstream tools such as mypy could ignore the public inline annotations and expose misleading or incomplete type information. ## Validation - `python/.venv/bin/ruff format --check python/python/tests/test_db.py python/python/type_tests/connect.py` - `python/.venv/bin/ruff check .` - `cd python && .venv/bin/pytest python/tests/test_db.py::test_package_includes_pep_561_marker -q` - `cd python && .venv/bin/pyright --pythonpath .venv/bin/python` - downstream mypy contract check for both public connection functions Fixes #2159 Co-authored-by: lancedb-gatefixer[bot] <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/pyproject.toml | 1 + python/python/lancedb/py.typed | 1 + python/python/tests/test_db.py | 5 +++++ python/python/type_tests/connect.py | 15 +++++++++++++++ 4 files changed, 22 insertions(+) create mode 100644 python/python/lancedb/py.typed create mode 100644 python/python/type_tests/connect.py diff --git a/python/pyproject.toml b/python/pyproject.toml index cb175bd6d..348058957 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -140,6 +140,7 @@ include = [ "python/lancedb/remote/errors.py", "python/lancedb/embeddings/__init__.py", "python/lancedb/_lancedb.pyi", + "python/type_tests/connect.py", ] exclude = ["python/tests/"] pythonVersion = "3.13" diff --git a/python/python/lancedb/py.typed b/python/python/lancedb/py.typed new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/python/python/lancedb/py.typed @@ -0,0 +1 @@ + diff --git a/python/python/tests/test_db.py b/python/python/tests/test_db.py index 8f4a8850c..84e78fd8f 100644 --- a/python/python/tests/test_db.py +++ b/python/python/tests/test_db.py @@ -6,6 +6,7 @@ import inspect import re import sys from datetime import timedelta +from importlib import resources import os from types import SimpleNamespace @@ -18,6 +19,10 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError from lancedb.pydantic import LanceModel, Vector +def test_package_includes_pep_561_marker(): + assert resources.files(lancedb).joinpath("py.typed").is_file() + + def test_basic(tmp_path): db = lancedb.connect(tmp_path) diff --git a/python/python/type_tests/connect.py b/python/python/type_tests/connect.py new file mode 100644 index 000000000..eb2cba37c --- /dev/null +++ b/python/python/type_tests/connect.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +from typing import assert_type + +import lancedb +from lancedb import AsyncConnection, DBConnection + + +def check_connect_type() -> None: + assert_type(lancedb.connect("memory://"), DBConnection) + + +async def check_connect_async_type() -> None: + assert_type(await lancedb.connect_async("memory://"), AsyncConnection) From dbc3687c7b9cfc2e5923d2c8a28a94cb6b9e56da Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:31:53 +0800 Subject: [PATCH 35/41] fix(node): require compatible Node.js types (#3829) ## Summary - require Node.js 18-compatible type declarations when TypeScript consumers install them - keep the type peer optional for JavaScript-only consumers - add a regression test tying the Node type peer range to the supported runtime ## Root cause LanceDB requires Node.js 18 or newer, and its public types expose Apache Arrow declarations that import built-ins through the node: scheme. The package did not declare a matching @types/node peer requirement, so npm accepted projects pinned to Node 12 declarations and TypeScript then reported that node:stream and node:fs/promises did not exist. ## Validation - pnpm lint - pnpm build - pnpm run docs - pnpm test --runInBand (678 passed, 5 skipped) - packed-package consumer probe rejects @types/node 12.20.55 and installs with @types/node 18.19.130 Fixes #1713 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/package.test.ts | 14 ++++++++++++++ nodejs/package-lock.json | 6 ++++++ nodejs/package.json | 6 ++++++ 3 files changed, 26 insertions(+) create mode 100644 nodejs/__test__/package.test.ts diff --git a/nodejs/__test__/package.test.ts b/nodejs/__test__/package.test.ts new file mode 100644 index 000000000..7743d73d6 --- /dev/null +++ b/nodejs/__test__/package.test.ts @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import packageJson = require("../package.json"); + +describe("package metadata", () => { + it("requires Node.js type declarations compatible with the runtime", () => { + expect(packageJson.engines.node).toBe(">= 18"); + expect(packageJson.peerDependencies["@types/node"]).toBe(">=18"); + expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({ + optional: true, + }); + }); +}); diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 8e30b0fab..bdbd3cf79 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -55,7 +55,13 @@ "openai": "4.29.2" }, "peerDependencies": { + "@types/node": ">=18", "apache-arrow": ">=15.0.0 <=18.1.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@aws-crypto/crc32": { diff --git a/nodejs/package.json b/nodejs/package.json index f3f719af2..671f3f94d 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -101,6 +101,12 @@ "openai": "4.29.2" }, "peerDependencies": { + "@types/node": ">=18", "apache-arrow": ">=15.0.0 <=18.1.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } } From dd5cb4d805b6cd79c7ecf9fb25754285d5d631bd Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:32:05 +0800 Subject: [PATCH 36/41] test(python): cover float16 table creation from Arrow data (#3785) ## Summary - exercise float16 sanitization through the reported direct Arrow-data table creation path - assert that the inferred fixed-size vector schema remains float16 - retain end-to-end index creation and vector search coverage ## Root cause and fix PyArrow 16 does not provide an is_nan kernel for half-float arrays, so passing float16 vector values directly to that kernel raises ArrowNotImplementedError. LanceDB's sanitizer already carries the compatibility fix from #837: it casts float16 values to float32 only for NaN detection while preserving the stored vector type. The existing end-to-end regression created an empty schema-defined table and added data afterward. This change aligns that regression with the issue reproduction by creating a table directly from a FixedSizeList Arrow table and verifying the persisted schema. ## Validation - uv run --extra tests pytest python/tests/test_table.py::test_create_f16_table_from_arrow_data -q - direct 1,000-row by 128-dimension float16 Arrow-table reproduction - PyArrow 16.1 half-float is_nan kernel reproduction - uvx ruff@0.15.20 format --check python/python/tests/test_table.py - uvx ruff@0.15.20 check . Fixes #835 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index e5c4ad801..8fc06ea69 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -2759,15 +2759,40 @@ def test_create_with_embedding_function(mem_db: DBConnection): assert actual == expected +def test_create_f16_table_from_arrow_data(mem_db: DBConnection): + dimension = 32 + num_rows = 512 + values = pa.array( + np.random.default_rng(42) + .standard_normal(num_rows * dimension) + .astype(np.float16) + ) + df = pa.table( + { + "text": [f"s-{i}" for i in range(num_rows)], + "vector": pa.FixedSizeListArray.from_arrays(values, dimension), + } + ) + table = mem_db.create_table("f16_tbl", data=df) + assert table.schema.field("vector").type == pa.list_(pa.float16(), dimension) + table.create_index(num_partitions=2, num_sub_vectors=2) + + query = df["vector"][2].as_py() + expected = table.search(query).limit(2).to_arrow() + + assert "s-2" in expected["text"].to_pylist() + + def test_create_f16_table(mem_db: DBConnection): class MyTable(LanceModel): text: str vector: Vector(32, value_type=pa.float16()) + rng = np.random.default_rng(42) df = pa.table( { "text": [f"s-{i}" for i in range(512)], - "vector": [np.random.randn(32).astype(np.float16) for _ in range(512)], + "vector": [rng.standard_normal(32).astype(np.float16) for _ in range(512)], } ) table = mem_db.create_table( From 564e5d0d56802bfd2c025e9337b8dd810d42759a Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:32:17 +0800 Subject: [PATCH 37/41] fix(python): support Polars 1.32 table scans (#3801) ## Root cause `Table.to_polars()` disabled PyArrow predicate pushdown by selecting the non-PyArrow Polars scan callback. Polars 1.32.3 invokes that callback with `batch_size` both positionally and through its partial, so collecting the returned lazy frame raises `TypeError: _scan_pyarrow_dataset_impl() got multiple values for argument batch_size`. ## Fix - Keep the compatible PyArrow callback path. - Add an identity `map_batches` barrier so predicates stay in Polars instead of reaching the LanceDB adapter as unsupported PyArrow expressions. - Extend the tested Polars range through 1.32.3 and retain lazy-frame regression coverage. ## Validation - `python/tests/test_table.py::test_polars` with Polars 1.32.3 - `python/tests/test_table.py::test_polars` with the locked Polars 1.3.0 baseline - `ruff format --check` on the changed Python files - `ruff check .` - `uv lock --check` Fixes #2619 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/pyproject.toml | 2 +- python/python/lancedb/table.py | 28 +++++++++++++++++++++++----- python/python/tests/test_table.py | 1 + python/uv.lock | 2 +- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 348058957..ce71484de 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -60,7 +60,7 @@ tests = [ "pytest-asyncio>=0.21", "duckdb>=0.9.0", "pytz>=2023.3", - "polars>=0.19, <=1.3.0", + "polars>=0.19, <=1.32.3", "pyarrow<25", "pyarrow-stubs>=16.0", "pylance==9.0.0rc1", diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index ae36bac7a..59e2650eb 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -108,6 +108,11 @@ def _should_push_down_query_table( return namespace_client is not None and "QueryTable" in pushdown_operations +def _polars_predicate_pushdown_barrier(frame: Any) -> Any: + """Return a Polars frame unchanged while blocking predicate pushdown.""" + return frame + + _MODEL_BACKED_TOKENIZER_PREFIXES = ("jieba", "lindera") _MODEL_BACKED_TOKENIZER_ERRORS = ( "unknown base tokenizer", @@ -864,12 +869,18 @@ class Table(ABC): """ raise NotImplementedError - def to_polars(self, **kwargs) -> "pl.DataFrame": - """Return the table as a polars.DataFrame. + def to_polars(self, **kwargs) -> "pl.LazyFrame": + """Return the table as a Polars LazyFrame. + + Note + ---- + The Polars streaming engine is not supported because it does not currently + implement Python PyArrow dataset scans. Use the default engine when collecting + this LazyFrame. Returns ------- - polars.DataFrame + polars.LazyFrame """ raise NotImplementedError @@ -2569,6 +2580,9 @@ class LanceTable(Table): 2. Currently we've disabled push-down of the filters from polars because polars pushdown into pyarrow uses pyarrow compute expressions rather than SQl strings (which LanceDB supports) + 3. The Polars streaming engine is not supported because it does not + currently implement Python PyArrow dataset scans. Use the default + engine when collecting this LazyFrame. Returns ------- @@ -2577,8 +2591,12 @@ class LanceTable(Table): from lancedb.integrations.pyarrow import PyarrowDatasetAdapter dataset = PyarrowDatasetAdapter(self) - return pl.scan_pyarrow_dataset( - dataset, allow_pyarrow_filter=False, batch_size=batch_size + # Polars 1.32's non-PyArrow callback path passes batch_size twice. Keep + # the compatible PyArrow path, but block predicates because this adapter + # cannot translate PyArrow expressions into LanceDB filters. + return pl.scan_pyarrow_dataset(dataset, batch_size=batch_size).map_batches( + _polars_predicate_pushdown_barrier, + predicate_pushdown=False, ) # New unified API overload diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 8fc06ea69..4ad5d7c3d 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -929,6 +929,7 @@ def test_polars(mem_db: DBConnection): # enter table to polars dataframe result = table.to_polars() + assert isinstance(result, pl.LazyFrame) assert np.allclose(result.collect()["vector"].to_list(), data["vector"]) # make sure filtering isn't broken diff --git a/python/uv.lock b/python/uv.lock index 551dc3f68..2cdcb182e 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1998,7 +1998,7 @@ requires-dist = [ { name = "pillow", marker = "extra == 'clip'", specifier = ">=12.1.1" }, { name = "pillow", marker = "extra == 'embeddings'", specifier = ">=12.1.1" }, { name = "pillow", marker = "extra == 'siglip'", specifier = ">=12.1.1" }, - { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.3.0" }, + { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.32.3" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5.0" }, { name = "pyarrow", specifier = ">=16" }, { name = "pyarrow", marker = "extra == 'tests'", specifier = "<25" }, From 607e5569276e68fc9b7bd6803b2748e6028007e1 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:32:28 +0800 Subject: [PATCH 38/41] test(python): cover search after schema merge (#3784) ## Summary - add an end-to-end regression for indexed vector search after merging a pandas column - verify unmatched rows retain a null merged value instead of failing Arrow batch assembly ## Root cause Historical Lance readers could assemble schema-evolved columns in physical data-file order. Indexed row-ID reads after a merge could therefore omit or misorder the newly merged column for unmatched rows. The currently pinned Lance release contains the reader correction, but LanceDB did not cover the reported merge-then-search path. ## Validation - uv run --extra tests pytest python/tests/test_table.py::test_merge python/tests/test_table.py::test_search_after_merge -q - uv run --project python --extra dev ruff check . - uv run --project python --extra dev ruff format --check python/python/tests/test_table.py Fixes #599 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/python/tests/test_table.py | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 4ad5d7c3d..eb6eaefaa 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -2218,6 +2218,45 @@ def test_merge(tmp_db: DBConnection, tmp_path): table.merge(other_dataset, left_on="id") +@pytest.mark.parametrize("storage_version", ["legacy", "stable"]) +def test_search_after_merge(tmp_path, storage_version): + pytest.importorskip("lance") + pd = pytest.importorskip("pandas") + + db = lancedb.connect( + tmp_path, + storage_options={"new_table_data_storage_version": storage_version}, + ) + rng = np.random.default_rng(42) + row_count = 512 + vectors = rng.standard_normal((row_count, 8)).astype(np.float32) + table = db.create_table( + "search_after_merge", + data=pd.DataFrame( + { + "id": [str(i) for i in range(row_count)], + "vector": list(vectors), + } + ), + ) + table.create_index("vector", config=IvfPq(num_partitions=1, num_sub_vectors=2)) + + links = pd.DataFrame( + { + "id": [str(i) for i in range(row_count // 2)], + "link": [f"https://example.com/{i}" for i in range(row_count // 2)], + } + ) + table.merge(links, left_on="id") + + query = table.search(vectors[-1]).refine_factor(50).limit(10) + assert "ANN" in query.explain_plan(verbose=True) + + result = query.to_arrow() + links_by_id = dict(zip(result["id"].to_pylist(), result["link"].to_pylist())) + assert links_by_id[str(row_count - 1)] is None + + def test_delete(mem_db: DBConnection): table = mem_db.create_table( "my_table", From 2ba7407dc36f4989dc720d96bd765601b94566ba Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:32:39 +0800 Subject: [PATCH 39/41] fix(node): cover non-nullable embedding schema append (#3835) ## Summary - Add an issue-specific regression for appending generated embeddings to an empty table with a non-nullable vector field. - Verify the custom embedding function produces the declared Float64 vectors and both appended rows are readable. ## Root cause In v0.4.19, records without a vector value were materialized against the explicit schema before embeddings were inserted. Apache Arrow inferred the generated batch vector field as nullable while the table retained the user-provided non-nullable field, then rejected the mismatched schemas. The current conversion path excludes the generated field from the initial record conversion and realigns the completed batch to the stored schema after embedding, but the reported empty-table append sequence lacked permanent regression coverage. ## Validation - `pnpm exec biome format --write __test__/embedding.test.ts` - `pnpm lint-ci` - `pnpm test -- --runInBand __test__/embedding.test.ts` (12 passed, 1 skipped integration test) - `pnpm build` - `pnpm run docs` Fixes #1281 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/embedding.test.ts | 60 +++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/nodejs/__test__/embedding.test.ts b/nodejs/__test__/embedding.test.ts index e56e80631..06184751e 100644 --- a/nodejs/__test__/embedding.test.ts +++ b/nodejs/__test__/embedding.test.ts @@ -11,8 +11,11 @@ import { Float16, Float32, Float64, + Int32, Schema, Utf8, + fromDataToBuffer, + tableFromIPC, } from "../lancedb/arrow"; import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding"; import { getRegistry, register } from "../lancedb/embedding/registry"; @@ -184,6 +187,63 @@ describe("embedding functions", () => { const vector0 = JSON.parse(JSON.stringify(arr[0].vector)); expect(vector0).toEqual([1, 2, 3]); }); + + it("should append generated vectors to a non-nullable schema", async () => { + @register("non_nullable_schema_test") + class MockEmbeddingFunction extends EmbeddingFunction { + ndims() { + return 3; + } + embeddingDataType(): Float { + return new Float64(); + } + async computeSourceEmbeddings(data: string[]) { + return data.map(() => [1, 2, 3]); + } + } + + const schema = new Schema([ + new Field("id", new Int32()), + new Field("text", new Utf8()), + new Field("type", new Utf8()), + new Field( + "vector", + new FixedSizeList(3, new Field("item", new Float64())), + ), + ]); + const func = new MockEmbeddingFunction(); + const db = await connect(tmpDir.name); + const table = await db.createEmptyTable("test_non_nullable", schema, { + embeddingFunction: { + function: func, + sourceColumn: "text", + }, + }); + + const data = [ + { id: 1, text: "Carrot", type: "vegetable" }, + { id: 2, text: "Apple", type: "fruit" }, + ]; + const buffer = await fromDataToBuffer( + data, + undefined, + await table.schema(), + ); + const generatedTable = tableFromIPC(buffer); + const vectorField = generatedTable.schema.fields.find( + (field) => field.name === "vector", + ); + expect(vectorField?.nullable).toBe(false); + + await table.add(data); + + const rows = await table.query().toArray(); + expect(rows).toHaveLength(2); + for (const row of rows) { + expect([...row.vector]).toEqual([1, 2, 3]); + } + }); + it("should error when appending to a table with an unregistered embedding function", async () => { @register("mock") class MockEmbeddingFunction extends EmbeddingFunction { From 11f24b1df408306d9a7801f83fe7fbed20b99e46 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:33:02 +0800 Subject: [PATCH 40/41] fix: explain unsupported object storage mounts (#3823) ## Summary - classify unsupported local-filesystem operations from Lance as a NotSupported error - explain that object-storage mounts cannot provide the safe commit operations Lance requires and direct users to native object-store URIs - preserve existing error behavior for other local I/O failures and non-local backends ## Root cause Mountpoint for Amazon S3 exposes an S3 bucket as a local path but does not implement atomic rename. Lance uses atomic rename for safe local commits, and the resulting unsupported I/O error was previously passed through as a generic Lance error, leaving Python users with an opaque low-level failure. Transparent support for such mounts is not safe; direct s3:// access remains the supported path. ## Validation - cargo test --quiet --features remote -p lancedb error::tests - cargo test --quiet --features remote -p lancedb --lib (807 passed, 1 ignored) - cargo check --quiet --features remote --tests --examples - cargo clippy --quiet --features remote --tests --examples - cargo fmt --all -- --check Fixes #2016 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- rust/lancedb/src/error.rs | 70 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/rust/lancedb/src/error.rs b/rust/lancedb/src/error.rs index f6f596f3d..4a6e6d8d9 100644 --- a/rust/lancedb/src/error.rs +++ b/rust/lancedb/src/error.rs @@ -169,6 +169,12 @@ impl From for Error { impl From for Error { fn from(source: lance::Error) -> Self { + if has_unsupported_local_filesystem_source(&source) { + return Self::NotSupported { + message: "the filesystem does not support an operation required for safe Lance commits (such as atomic rename). Object-storage mounts such as Mountpoint for Amazon S3 are not supported; use the native object-store URI (for example, s3://bucket/path) instead".to_string(), + }; + } + // Try to unwrap external errors that were wrapped by lance match source { lance::Error::Wrapped { error, .. } => Self::from_box_error(error), @@ -181,6 +187,27 @@ impl From for Error { } } +fn has_unsupported_local_filesystem_source(error: &(dyn std::error::Error + 'static)) -> bool { + let mut current = Some(error); + let mut is_local_filesystem = false; + let mut is_unsupported = false; + while let Some(error) = current { + is_local_filesystem |= error + .downcast_ref::() + .is_some_and(|error| { + matches!(error, object_store::Error::Generic { store, .. } if *store == "LocalFileSystem") + }); + is_unsupported |= error + .downcast_ref::() + .is_some_and(|error| error.kind() == std::io::ErrorKind::Unsupported); + if is_local_filesystem && is_unsupported { + return true; + } + current = error.source(); + } + false +} + impl Error { fn from_box_error(mut source: Box) -> Self { source = match source.downcast::() { @@ -270,3 +297,46 @@ impl From for Error { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_filesystem_operations_have_actionable_error() { + let object_store_error = object_store::Error::Generic { + store: "LocalFileSystem", + source: Box::new(std::io::Error::from(std::io::ErrorKind::Unsupported)), + }; + let lance_error = lance::Error::io_source(Box::new(object_store_error)); + + let error = Error::from(lance_error); + + assert!(matches!( + error, + Error::NotSupported { message } + if message.contains("Mountpoint for Amazon S3") + && message.contains("s3://bucket/path") + )); + } + + #[test] + fn other_io_errors_remain_lance_errors() { + let object_store_error = object_store::Error::Generic { + store: "LocalFileSystem", + source: Box::new(std::io::Error::from(std::io::ErrorKind::PermissionDenied)), + }; + let lance_error = lance::Error::io_source(Box::new(object_store_error)); + + assert!(matches!(Error::from(lance_error), Error::Lance { .. })); + } + + #[test] + fn unsupported_non_filesystem_errors_remain_lance_errors() { + let lance_error = lance::Error::io_source(Box::new(std::io::Error::from( + std::io::ErrorKind::Unsupported, + ))); + + assert!(matches!(Error::from(lance_error), Error::Lance { .. })); + } +} From 6ba80a960cd1a54f6d8b625124743301beaf4173 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:33:13 +0800 Subject: [PATCH 41/41] fix(node): cover offset pagination in search (#3814) ## Summary - add Node regression coverage for vector-search offset pagination - add equivalent coverage for full-text search - compare later pages with the corresponding complete-result slice and assert page sizes ## Root cause The historical query path requested only the user limit from nearest-neighbor or full-text search before applying the offset, so a page became empty when its offset reached that limit. The production query path on current main already incorporates the later fix from #2592; this change adds the missing Node binding coverage for the still-open report and protects both affected APIs from regression. ## Validation - corepack pnpm build - corepack pnpm test -- query.test.ts --runInBand --testNamePattern="Search pagination" - corepack pnpm lint-ci - corepack pnpm tsc - corepack pnpm run docs Fixes #2229 Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- nodejs/__test__/query.test.ts | 75 +++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/nodejs/__test__/query.test.ts b/nodejs/__test__/query.test.ts index da001b1eb..5f3e68b16 100644 --- a/nodejs/__test__/query.test.ts +++ b/nodejs/__test__/query.test.ts @@ -110,6 +110,81 @@ describe("Query outputSchema", () => { }); }); +describe("Search pagination", () => { + let tmpDir: tmp.DirResult; + let table: Table; + + beforeEach(async () => { + tmpDir = tmp.dirSync({ unsafeCleanup: true }); + const db = await connect(tmpDir.name); + const schema = new Schema([ + new Field("id", new Int64(), false), + new Field("text", new Utf8(), false), + new Field( + "vector", + new FixedSizeList(2, new Field("item", new Float32())), + false, + ), + ]); + const data = makeArrowTable( + [ + { id: 1n, text: "common", vector: [0, 0] }, + { id: 2n, text: "common common", vector: [1, 1] }, + { id: 3n, text: "common common common", vector: [2, 2] }, + { id: 4n, text: "common common common common", vector: [3, 3] }, + ], + { schema }, + ); + table = await db.createTable("test", data); + }); + + afterEach(() => { + tmpDir.removeCallback(); + }); + + it("applies offset after the vector search limit", async () => { + const allResults = await table + .vectorSearch([0, 0]) + .select(["id"]) + .limit(4) + .toArray(); + const secondPage = await table + .vectorSearch([0, 0]) + .select(["id"]) + .limit(2) + .offset(2) + .toArray(); + + expect(allResults).toHaveLength(4); + expect(secondPage).toHaveLength(2); + expect(secondPage.map((row) => row.id)).toEqual( + allResults.slice(2, 4).map((row) => row.id), + ); + }); + + it("applies offset after the full-text search limit", async () => { + await table.createIndex("text", { config: Index.fts() }); + + const allResults = await table + .search("common", "fts") + .select(["id"]) + .limit(4) + .toArray(); + const secondPage = await table + .search("common", "fts") + .select(["id"]) + .limit(2) + .offset(2) + .toArray(); + + expect(allResults).toHaveLength(4); + expect(secondPage).toHaveLength(2); + expect(secondPage.map((row) => row.id)).toEqual( + allResults.slice(2, 4).map((row) => row.id), + ); + }); +}); + describe("Query orderBy", () => { let tmpDir: tmp.DirResult; let table: Table;