From e12407519e1f61ca9a05159824af138cb2fe3714 Mon Sep 17 00:00:00 2001 From: Yang Cen Date: Thu, 13 Aug 2026 17:30:53 +0800 Subject: [PATCH] fix: make table existence manifest-authoritative --- Cargo.lock | 74 +- Cargo.toml | 28 +- .../examples/bench_open_missing_table.rs | 28 +- rust/lancedb/src/connection.rs | 12 +- rust/lancedb/src/database/listing.rs | 117 +- rust/lancedb/src/table.rs | 1117 +---------------- 6 files changed, 247 insertions(+), 1129 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c3a610da4..c5545ea8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1756,7 +1756,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3034,7 +3034,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3257,7 +3257,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3456,7 +3456,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4561,7 +4561,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4816,7 +4816,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arc-swap", "arrow", @@ -4891,7 +4891,7 @@ dependencies = [ [[package]] name = "lance-arrow" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow-array", "arrow-buffer", @@ -4913,7 +4913,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow-array", "arrow-buffer", @@ -4927,7 +4927,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow-array", "arrow-schema", @@ -4937,7 +4937,7 @@ dependencies = [ [[package]] name = "lance-bitpacking" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrayref", "crunchy", @@ -4948,7 +4948,7 @@ dependencies = [ [[package]] name = "lance-core" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow-array", "arrow-buffer", @@ -4989,7 +4989,7 @@ dependencies = [ [[package]] name = "lance-datafusion" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow", "arrow-array", @@ -5020,7 +5020,7 @@ dependencies = [ [[package]] name = "lance-datagen" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow", "arrow-array", @@ -5038,7 +5038,7 @@ dependencies = [ [[package]] name = "lance-derive" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "proc-macro2", "quote", @@ -5048,7 +5048,7 @@ dependencies = [ [[package]] name = "lance-encoding" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow-arith", "arrow-array", @@ -5083,7 +5083,7 @@ dependencies = [ [[package]] name = "lance-file" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow-arith", "arrow-array", @@ -5115,7 +5115,7 @@ dependencies = [ [[package]] name = "lance-index" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arc-swap", "arrow", @@ -5183,7 +5183,7 @@ dependencies = [ [[package]] name = "lance-index-core" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow-array", "arrow-schema", @@ -5206,7 +5206,7 @@ dependencies = [ [[package]] name = "lance-io" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow", "arrow-array", @@ -5243,7 +5243,7 @@ dependencies = [ [[package]] name = "lance-linalg" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow-array", "arrow-buffer", @@ -5260,7 +5260,7 @@ dependencies = [ [[package]] name = "lance-namespace" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow", "async-trait", @@ -5273,7 +5273,7 @@ dependencies = [ [[package]] name = "lance-namespace-impls" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow", "arrow-ipc", @@ -5327,7 +5327,7 @@ dependencies = [ [[package]] name = "lance-select" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow-array", "arrow-buffer", @@ -5343,7 +5343,7 @@ dependencies = [ [[package]] name = "lance-table" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow", "arrow-array", @@ -5384,7 +5384,7 @@ dependencies = [ [[package]] name = "lance-testing" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "arrow-array", "arrow-schema", @@ -5398,7 +5398,7 @@ dependencies = [ [[package]] name = "lance-tokenizer" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10" dependencies = [ "frostem", "icu_segmenter", @@ -6249,7 +6249,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7641,8 +7641,8 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.4.1", - "itertools 0.11.0", + "heck 0.5.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -7661,7 +7661,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -7932,7 +7932,7 @@ dependencies = [ "once_cell", "socket2 0.6.3", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -8710,7 +8710,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8781,7 +8781,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9337,7 +9337,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -9349,7 +9349,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -9771,7 +9771,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10748,7 +10748,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 41cb34a7b..b1eae918e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=11.0.0-beta.6", default-features = false, "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=11.0.0-beta.6", default-features = false, "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=11.0.0-beta.6", default-features = false, "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } ahash = "0.8" # Note that this one does not include pyarrow arrow = { version = "58.0.0", optional = false } diff --git a/rust/lancedb/examples/bench_open_missing_table.rs b/rust/lancedb/examples/bench_open_missing_table.rs index 7ee804362..8e6b16e11 100644 --- a/rust/lancedb/examples/bench_open_missing_table.rs +++ b/rust/lancedb/examples/bench_open_missing_table.rs @@ -1,20 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Release benchmark for opening a missing table as sibling-table cardinality grows. -//! -//! The fixture uses real `.lance` directories and marker files. Fixture creation is -//! outside the timed section. Defaults intentionally cover 1k, 10k, and 100k siblings -//! with 10 warmups and 100 distinct missing-table opens per scale: -//! -//! ```text -//! cargo run --release -p lancedb --example bench_open_missing_table -//! ``` -//! -//! `BENCH_SIBLINGS`, `BENCH_WARMUPS`, and `BENCH_TRIALS` override those defaults. -//! Reduced settings are useful only as a smoke test. Performance comparisons require -//! the same machine, filesystem, fixture sizes, settings, lockfile, and alternating -//! baseline/candidate execution order. +// Release benchmark for opening a missing table as sibling-table cardinality grows. +// +// The fixture uses real `.lance` directories and marker files. Fixture creation is +// outside the timed section. Defaults intentionally cover 1k, 10k, and 100k siblings +// with 10 warmups and 100 distinct missing-table opens per scale: +// +// ```text +// cargo run --release -p lancedb --example bench_open_missing_table +// ``` +// +// `BENCH_SIBLINGS`, `BENCH_WARMUPS`, and `BENCH_TRIALS` override those defaults. +// Reduced settings are useful only as a smoke test. Performance comparisons require +// the same machine, filesystem, fixture sizes, settings, lockfile, and alternating +// baseline/candidate execution order. use std::time::{Duration, Instant}; diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index dd53a2d2e..1f2708d4e 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -409,6 +409,11 @@ impl Connection { /// /// The names will be returned in lexicographical order (ascending) /// + /// Listing databases discover physical `*.lance` entries without opening every + /// dataset. The result is a point-in-time discovery snapshot: an entry may still be + /// under creation, may contain only uncommitted storage, or may be concurrently + /// dropped before it is opened. + /// /// The parameters `page_token` and `limit` can be used to paginate the results pub fn table_names(&self) -> TableNamesBuilder { TableNamesBuilder::new(self.internal.clone()) @@ -456,10 +461,9 @@ impl Connection { /// /// # Returns /// Created [`TableRef`], or [`Error::TableNotFound`] if the table does not exist. - /// If the table's storage is present but holds no readable dataset (for example a - /// `.lance` directory left behind by an interrupted drop and re-create, which - /// [`Self::table_names`] still lists) this returns [`Error::TableCorrupted`] - /// instead. + /// On listing databases, a committed Lance manifest is authoritative for table + /// existence. Uncommitted files or a physical `.lance` directory alone do not + /// make a table openable. pub fn open_table(&self, name: impl Into) -> OpenTableBuilder { OpenTableBuilder::new( self.internal.clone(), diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index 0ab3614e7..f284320c6 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -1291,16 +1291,21 @@ impl Database for ListingDatabase { mod tests { use super::*; use crate::Table; + use crate::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream}; use crate::connection::ConnectRequest; use crate::data::scannable::Scannable; use crate::database::{CreateTableMode, CreateTableRequest}; use crate::query::QueryRequest; use crate::table::{AnyQuery, WriteOptions}; use arrow_array::{Int32Array, RecordBatch, StringArray}; - use arrow_schema::{DataType, Field, Schema}; - use futures::TryStreamExt; + use arrow_schema::{DataType, Field, Schema, SchemaRef}; + use futures::{TryStreamExt, stream::once}; use std::path::PathBuf; + use std::sync::Arc; + use std::time::Duration; use tempfile::tempdir; + use tokio::sync::Barrier; + use tokio::time::timeout; async fn setup_database() -> (tempfile::TempDir, ListingDatabase) { let tempdir = tempdir().unwrap(); @@ -1324,6 +1329,114 @@ mod tests { (tempdir, db) } + struct BarrierScannable { + batch: RecordBatch, + barrier: Arc, + } + + impl Scannable for BarrierScannable { + fn schema(&self) -> SchemaRef { + self.batch.schema() + } + + fn scan_as_stream(&mut self) -> SendableRecordBatchStream { + let batch = self.batch.clone(); + let schema = batch.schema(); + let barrier = self.barrier.clone(); + Box::pin(SimpleRecordBatchStream { + schema, + stream: once(async move { + barrier.wait().await; + Ok(batch) + }), + }) + } + } + + fn create_request(name: &str, data: Box) -> CreateTableRequest { + CreateTableRequest { + name: name.to_string(), + namespace_path: vec![], + data, + mode: CreateTableMode::Create, + write_options: Default::default(), + location: None, + namespace_client: None, + } + } + + #[tokio::test] + async fn test_create_ignores_uncommitted_storage_without_manifest() { + let (tmp_dir, db) = setup_database().await; + let data_dir = tmp_dir.path().join("test.lance/data"); + std::fs::create_dir_all(&data_dir).unwrap(); + std::fs::write(data_dir.join("orphan.lance"), b"uncommitted").unwrap(); + + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1]))]).unwrap(); + + let table = db + .create_table(create_request("test", Box::new(batch))) + .await + .unwrap(); + assert_eq!(table.count_rows(None).await.unwrap(), 1); + } + + #[tokio::test] + async fn test_concurrent_create_is_arbitrated_by_manifest_commit() { + let uri = format!("memory:///concurrent-create-{}", uuid::Uuid::new_v4()); + let db = crate::connect(&uri).execute().await.unwrap(); + let store: Arc = + Arc::new(object_store::memory::InMemory::new()); + let table_url = url::Url::parse("memory:///database/test.lance").unwrap(); + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1]))]).unwrap(); + let barrier = Arc::new(Barrier::new(2)); + + #[allow(deprecated)] + let request = |batch, barrier| { + let mut request = create_request("test", Box::new(BarrierScannable { batch, barrier })); + request.write_options = WriteOptions { + lance_write_params: Some(lance::dataset::WriteParams { + store_params: Some(ObjectStoreParams { + object_store: Some((store.clone(), table_url.clone())), + ..Default::default() + }), + commit_handler: Some(Arc::new( + lance_table::io::commit::ConditionalPutCommitHandler, + )), + ..Default::default() + }), + }; + request + }; + + let left = db + .database() + .create_table(request(batch.clone(), barrier.clone())); + let right = db.database().create_table(request(batch, barrier)); + let (left, right) = timeout(Duration::from_secs(30), async { tokio::join!(left, right) }) + .await + .expect("concurrent creates deadlocked"); + + let results = [left, right]; + assert_eq!( + results.iter().filter(|result| result.is_ok()).count(), + 1, + "expected one successful create, got {results:?}" + ); + assert_eq!( + results + .iter() + .filter(|result| matches!(result, Err(Error::TableAlreadyExists { .. }))) + .count(), + 1, + "expected one manifest conflict, got {results:?}" + ); + } + #[tokio::test] async fn test_listing_database_root_ops_do_not_create_manifest() { let tempdir = tempdir().unwrap(); diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 28ec3d67e..32b6bcebc 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -38,7 +38,6 @@ use lance_table::format::Manifest; use lance_table::io::commit::CommitHandler; use lance_table::io::commit::ManifestNamingScheme; use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler; -use object_store::ObjectStoreExt as _; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::format; @@ -51,7 +50,6 @@ use crate::DistanceType; use crate::blob::BlobRangeRequest; use crate::data::scannable::{PeekedScannable, Scannable, estimate_write_partitions}; use crate::database::Database; -use crate::database::listing::LANCE_FILE_EXTENSION; use crate::database::read_freshness::TableFreshness; use crate::embeddings::{EmbeddingDefinition, EmbeddingRegistry, MemoryRegistry}; use crate::error::{Error, Result}; @@ -153,97 +151,6 @@ pub(crate) fn map_namespace_lance_error(err: lance::Error, table_name: &str) -> } } -/// Map a `lance::Error::DatasetNotFound` for the table at `uri` into a `lancedb::Error`. -/// -/// Lance reports "there is nothing at this location" and "there is a table directory -/// here but nothing loadable inside it" with the same error. Only the first is a -/// `TableNotFound`: a `.lance` directory left behind by an interrupted drop and -/// re-create is still reported by `Connection::table_names`, so callers need to be able -/// to tell "never existed" from "exists but is broken". -/// -/// See . -async fn map_dataset_not_found( - uri: &str, - name: &str, - params: ReadParams, - err: lance::Error, -) -> Result { - let name = name.to_string(); - let source = Box::new(err); - if table_storage_exists(uri, params).await? { - Ok(Error::TableCorrupted { name, source }) - } else { - Ok(Error::TableNotFound { name, source }) - } -} - -/// Whether storage contains an entry for `uri`, even though no dataset could be loaded. -/// -/// Local filesystems can represent an empty table directory, so inspect that exact path. -/// Object stores cannot represent empty directories: probe an exact key and then consume -/// at most the first object under the table prefix. Neither path inspects sibling tables. -async fn table_storage_exists(uri: &str, params: ReadParams) -> Result { - #[allow(deprecated)] - let uses_legacy_direct_store = params.store_options.as_ref().is_some_and(|options| { - options.object_store.is_some() && options.native_directory_path_resolver.is_none() - }); - - // Resolve the effective provider and wrapper for this exact table URI. The resolved - // store remains authoritative even when a custom provider preserves a `file` scheme. - let (object_store, path, _) = DatasetBuilder::from_uri(uri) - .with_read_params(params) - .build_object_store() - .await?; - // Only `*.lance` entries are ever reported as tables, so nothing else can produce - // the list-then-open mismatch this guards against. - if path.extension() != Some(LANCE_FILE_EXTENSION) { - return Ok(false); - } - - // The deprecated direct binding cannot identify native directory semantics unless its - // caller supplies a resolver. Preserve its established list/open agreement without - // widening this sibling-enumerating fallback to provider-resolved stores. - if uses_legacy_direct_store { - let (Some(parent), Some(dir_name)) = (path.parent(), path.filename()) else { - return Ok(false); - }; - let entries = object_store.read_dir(parent).await?; - return Ok(entries.iter().any(|entry| entry.as_str() == dir_name)); - } - - if let Some(native_path) = object_store.native_directory_path(&path) { - // A native table URI intentionally grants access to this exact path; LanceDB does - // not confine local databases beneath a separate filesystem root. nosemgrep - return match std::fs::symlink_metadata(&native_path) { - Ok(_) => Ok(true), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), - Err(error) => Err(Error::Other { - message: format!( - "Failed to inspect native table path '{}'", - native_path.display() - ), - source: Some(Box::new(error)), - }), - }; - } - - let head_error = match object_store.inner.head(&path).await { - Ok(_) => return Ok(true), - Err(error) => error, - }; - - if !matches!(head_error, object_store::Error::NotFound { .. }) { - return Err(head_error.into()); - } - - Ok(object_store - .list(Some(path)) - .next() - .await - .transpose()? - .is_some()) -} - /// Defines the type of column #[derive(Debug, Clone, Serialize, Deserialize)] pub enum ColumnKind { @@ -2463,44 +2370,30 @@ impl NativeTable { None => false, }; + let mut builder = DatasetBuilder::from_uri(uri).with_read_params(params); + // Set up commit handler when managed_versioning is enabled - let commit_handler: Option> = - if managed_versioning && let Some(ref ns_client) = namespace_client { - let external_store = LanceNamespaceExternalManifestStore::for_table_uri( - ns_client.clone(), - table_id.clone(), - uri, - )?; - Some(Arc::new(ExternalManifestCommitHandler { - external_manifest_store: Arc::new(external_store), - })) - } else { - None - }; + if managed_versioning && let Some(ref ns_client) = namespace_client { + let external_store = LanceNamespaceExternalManifestStore::for_table_uri( + ns_client.clone(), + table_id.clone(), + uri, + )?; + let commit_handler: Arc = Arc::new(ExternalManifestCommitHandler { + external_manifest_store: Arc::new(external_store), + }); + builder = builder.with_commit_handler(commit_handler); + } - let make_builder = || { - let builder = DatasetBuilder::from_uri(uri).with_read_params(params.clone()); - match &commit_handler { - Some(commit_handler) => builder.with_commit_handler(commit_handler.clone()), - None => builder, - } - }; - - let dataset = match make_builder().load().await { + let dataset = match builder.load().await { Ok(dataset) => dataset, Err(e @ lance::Error::DatasetNotFound { .. }) => { - let mapped = map_dataset_not_found(uri, name, params.clone(), e).await?; - if !matches!(mapped, Error::TableCorrupted { .. }) { - return Err(mapped); - } - - // The first manifest may have committed between the failed load and the - // storage probe. Re-open once before reporting a stale corruption result. - match make_builder().load().await { - Ok(dataset) => dataset, - Err(lance::Error::DatasetNotFound { .. }) => return Err(mapped), - Err(e) => return Err(e.into()), - } + // The manifest load is the existence check. A physical prefix may be + // from a concurrent or abandoned create, so it cannot refine this error. + return Err(Error::TableNotFound { + name: name.to_string(), + source: Box::new(e), + }); } Err(e) => return Err(e.into()), }; @@ -3779,7 +3672,6 @@ mod tests { use lance::Dataset; use lance::io::{ObjectStoreParams, WrappingObjectStore}; use lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY_POSITION; - use lance_io::object_store::NativeDirectoryPathResolver; use tempfile::tempdir; use super::*; @@ -3851,73 +3743,50 @@ mod tests { ); } - /// Write a table and then break it, leaving the `.lance` directory in place. - /// - /// `remove_all` reproduces an interrupted drop + re-create (the directory is left - /// empty); otherwise only the manifests are removed, leaving the data files behind. - async fn write_then_corrupt_table(dir: &std::path::Path, remove_all: bool) -> String { - let dataset_path = dir.join("test.lance"); - let uri = dataset_path.to_str().unwrap().to_string(); - - let batch = make_test_batches(); - let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); - Dataset::write(reader, &uri, None).await.unwrap(); - - if remove_all { - for entry in std::fs::read_dir(&dataset_path).unwrap() { - let entry = entry.unwrap(); - if entry.file_type().unwrap().is_dir() { - std::fs::remove_dir_all(entry.path()).unwrap(); - } else { - std::fs::remove_file(entry.path()).unwrap(); - } - } - assert_eq!(std::fs::read_dir(&dataset_path).unwrap().count(), 0); - } else { - let versions = dataset_path.join("_versions"); - assert!(versions.is_dir(), "expected manifests under {versions:?}"); - std::fs::remove_dir_all(&versions).unwrap(); - assert!(std::fs::read_dir(&dataset_path).unwrap().count() > 0); - } - - uri - } - #[tokio::test] - async fn test_open_corrupt_empty_dir() { + async fn test_open_not_found_when_empty_directory_exists() { let tmp_dir = tempdir().unwrap(); - let uri = write_then_corrupt_table(tmp_dir.path(), true).await; + let dataset_path = tmp_dir.path().join("test.lance"); + std::fs::create_dir(&dataset_path).unwrap(); - let err = NativeTable::open(&uri).await.unwrap_err(); + let err = NativeTable::open(dataset_path.to_str().unwrap()) + .await + .unwrap_err(); assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == "test"), + matches!(&err, Error::TableNotFound { name, .. } if name == "test"), "got {err:?}" ); } #[tokio::test] - async fn test_open_corrupt_missing_manifest() { + async fn test_open_not_found_when_only_uncommitted_storage_exists() { let tmp_dir = tempdir().unwrap(); - let uri = write_then_corrupt_table(tmp_dir.path(), false).await; + let dataset_path = tmp_dir.path().join("test.lance"); + let data_dir = dataset_path.join("data"); + std::fs::create_dir_all(&data_dir).unwrap(); + std::fs::write(data_dir.join("orphan.lance"), b"uncommitted").unwrap(); - let err = NativeTable::open(&uri).await.unwrap_err(); + let err = NativeTable::open(dataset_path.to_str().unwrap()) + .await + .unwrap_err(); assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == "test"), + matches!(&err, Error::TableNotFound { name, .. } if name == "test"), "got {err:?}" ); } - /// A table listed by `table_names()` must not be reported as missing by - /// `open_table()`. See . + /// Listing databases discover physical `*.lance` entries. That snapshot is not an + /// authoritative table-existence check: only a committed manifest makes a table + /// openable, and the entry could also be concurrently created or dropped. #[tokio::test] - async fn test_open_table_corrupt_is_still_listed() { + async fn test_table_names_may_include_uncommitted_storage() { let tmp_dir = tempdir().unwrap(); let db = connect(tmp_dir.path().to_str().unwrap()) .execute() .await .unwrap(); - write_then_corrupt_table(tmp_dir.path(), true).await; + std::fs::create_dir(tmp_dir.path().join("test.lance")).unwrap(); assert_eq!( db.table_names().execute().await.unwrap(), @@ -3925,373 +3794,27 @@ mod tests { ); let err = db.open_table("test").execute().await.unwrap_err(); assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == "test"), - "got {err:?}" - ); - assert!( - err.to_string().contains("exists but could not be loaded"), - "got {err}" + matches!(&err, Error::TableNotFound { name, .. } if name == "test"), + "physical storage without a committed manifest is not a table: {err:?}" ); } #[derive(Debug)] - struct FileSchemeMemoryProvider { - store: lance_io::object_store::ObjectStore, - } - - #[async_trait::async_trait] - impl lance_io::object_store::ObjectStoreProvider for FileSchemeMemoryProvider { - async fn new_store( - &self, - _base_path: url::Url, - _params: &ObjectStoreParams, - ) -> lance_core::Result { - Ok(self.store.clone()) - } - } - - /// Recovery must inspect the same provider that the failed dataset load used, - /// even when the table URI itself looks like a native filesystem path. - #[tokio::test] - async fn test_open_recovery_uses_custom_file_provider() { - use object_store::ObjectStoreExt as _; - - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("custom.lance"); - let uri = url::Url::from_file_path(&dataset_path).unwrap().to_string(); - assert!(!dataset_path.exists()); - - let store = lance_io::object_store::ObjectStore::memory(); - let table_path = - object_store::path::Path::from_url_path(url::Url::parse(&uri).unwrap().path()).unwrap(); - store - .inner - .put( - &table_path.clone().join("_marker"), - bytes::Bytes::new().into(), - ) - .await - .unwrap(); - - let registry = Arc::new(lance_io::object_store::ObjectStoreRegistry::default()); - registry.insert("file", Arc::new(FileSchemeMemoryProvider { store })); - let read_params = ReadParams { - session: Some(Arc::new(lance::session::Session::new(0, 0, registry))), - ..Default::default() - }; - - let err = NativeTable::open_with_params( - &uri, - "custom", - Vec::new(), - None, - Some(read_params), - None, - None, - HashSet::new(), - None, - ) - .await - .unwrap_err(); - - assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == "custom"), - "got {err:?}" - ); - } - - /// Provider provenance, not the store's reported scheme, determines whether - /// recovery may bypass the ObjectStore API for native filesystem metadata. - #[tokio::test] - async fn test_open_recovery_uses_custom_file_provider_preserving_file_scheme() { - use object_store::ObjectStoreExt as _; - - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("custom-file-scheme.lance"); - let uri = url::Url::from_file_path(&dataset_path).unwrap().to_string(); - assert!(!dataset_path.exists()); - - let inner = Arc::new(object_store::memory::InMemory::new()); - let table_path = - object_store::path::Path::from_url_path(url::Url::parse(&uri).unwrap().path()).unwrap(); - inner - .put( - &table_path.clone().join("_marker"), - bytes::Bytes::new().into(), - ) - .await - .unwrap(); - let store = lance_io::object_store::ObjectStore::new( - inner, - url::Url::parse(&uri).unwrap(), - None, - None, - false, - false, - 1, - 0, - None, - ); - assert_eq!(store.scheme(), "file"); - - let registry = Arc::new(lance_io::object_store::ObjectStoreRegistry::default()); - registry.insert("file", Arc::new(FileSchemeMemoryProvider { store })); - let session = Arc::new(lance::session::Session::new(0, 0, registry)); - let read_params = ReadParams { - session: Some(session.clone()), - ..Default::default() - }; - - let err = NativeTable::open_with_params( - &uri, - "custom-file-scheme", - Vec::new(), - None, - Some(read_params), - None, - None, - HashSet::new(), - None, - ) - .await - .unwrap_err(); - - assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == "custom-file-scheme"), - "got {err:?}" - ); - - let db = connect(tmp_dir.path().to_str().unwrap()) - .session(session) - .execute() - .await - .unwrap(); - assert_eq!( - db.table_names().execute().await.unwrap(), - vec!["custom-file-scheme".to_string()] - ); - let err = db - .open_table("custom-file-scheme") - .execute() - .await - .unwrap_err(); - assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == "custom-file-scheme"), - "got {err:?}" - ); - } - - /// `file-object-store` deliberately routes through the ObjectStore API, so its - /// provider remains authoritative even though its URI names a local path. - #[tokio::test] - async fn test_open_recovery_uses_custom_file_object_store_provider() { - use object_store::ObjectStoreExt as _; - - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("custom-object-store.lance"); - let path = dataset_path.to_str().unwrap().replace('\\', "/"); - let path_prefix = if path.starts_with('/') { "" } else { "/" }; - let url = url::Url::parse(&format!("file-object-store://{path_prefix}{path}")).unwrap(); - let uri = url.to_string(); - assert!(!dataset_path.exists()); - - let inner = Arc::new(object_store::memory::InMemory::new()); - let table_path = object_store::path::Path::from_url_path(url.path()).unwrap(); - inner - .put( - &table_path.clone().join("_marker"), - bytes::Bytes::new().into(), - ) - .await - .unwrap(); - let store = lance_io::object_store::ObjectStore::new( - inner, url, None, None, false, false, 1, 0, None, - ); - - let registry = Arc::new(lance_io::object_store::ObjectStoreRegistry::default()); - registry.insert( - "file-object-store", - Arc::new(FileSchemeMemoryProvider { store }), - ); - let read_params = ReadParams { - session: Some(Arc::new(lance::session::Session::new(0, 0, registry))), - ..Default::default() - }; - - let err = NativeTable::open_with_params( - &uri, - "custom-object-store", - Vec::new(), - None, - Some(read_params), - None, - None, - HashSet::new(), - None, - ) - .await - .unwrap_err(); - - assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == "custom-object-store"), - "got {err:?}" - ); - } - - /// The built-in file-object-store provider lists an empty `.lance` directory, - /// so opening that listed entry must preserve the corrupt-table classification. - #[tokio::test] - async fn test_open_recovery_detects_empty_file_object_store_directory() { - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("empty-object-store.lance"); - std::fs::create_dir(&dataset_path).unwrap(); - let path = tmp_dir.path().to_str().unwrap().replace('\\', "/"); - let path_prefix = if path.starts_with('/') { "" } else { "/" }; - let uri = format!("file-object-store://{path_prefix}{path}"); - let db = connect(&uri).execute().await.unwrap(); - - assert_eq!( - db.table_names().execute().await.unwrap(), - vec!["empty-object-store".to_string()] - ); - let err = db - .open_table("empty-object-store") - .execute() - .await - .unwrap_err(); - assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == "empty-object-store"), - "got {err:?}" - ); - } - - #[tokio::test] - async fn test_open_recovery_detects_empty_local_dir_with_explicit_default_session() { - let tmp_dir = tempdir().unwrap(); - let table_name = "explicit-default-session"; - std::fs::create_dir(tmp_dir.path().join(format!("{table_name}.lance"))).unwrap(); - - let db = connect(tmp_dir.path().to_str().unwrap()) - .session(Arc::new(lance::session::Session::default())) - .execute() - .await - .unwrap(); - - assert_eq!( - db.table_names().execute().await.unwrap(), - vec![table_name.to_string()] - ); - let err = db.open_table(table_name).execute().await.unwrap_err(); - assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == table_name), - "listed empty table must be corrupt, got {err:?}" - ); - } - - /// Recovery must not depend on writable process scratch space. Run the - /// assertion in a child process so changing `TMPDIR` cannot affect other tests. - #[test] - #[cfg(unix)] - fn test_open_recovery_explicit_session_without_temp_directory() { - const FIXTURE_ENV: &str = "LANCEDB_OPEN_RECOVERY_NO_TMP_FIXTURE"; - const MISSING_TMP_ENV: &str = "LANCEDB_OPEN_RECOVERY_MISSING_TMPDIR"; - const TABLE_NAME: &str = "explicit-session-no-tempfile"; - - if let Some(fixture) = std::env::var_os(FIXTURE_ENV) { - let session = Arc::new(lance::session::Session::default()); - let missing_tmpdir = std::env::var_os(MISSING_TMP_ENV).unwrap(); - // The child process isolates this environment mutation from other tests. - unsafe { std::env::set_var("TMPDIR", missing_tmpdir) }; - let runtime = tokio::runtime::Runtime::new().unwrap(); - runtime.block_on(async { - let db = connect(std::path::Path::new(&fixture).to_str().unwrap()) - .session(session) - .execute() - .await - .unwrap(); - let err = db.open_table(TABLE_NAME).execute().await.unwrap_err(); - assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == TABLE_NAME), - "listed empty table must be corrupt, got {err:?}" - ); - }); - return; - } - - let fixture = tempdir().unwrap(); - std::fs::create_dir(fixture.path().join(format!("{TABLE_NAME}.lance"))).unwrap(); - let missing_tmpdir = fixture.path().join("missing-tmpdir"); - let output = std::process::Command::new(std::env::current_exe().unwrap()) - .arg("--exact") - .arg("table::tests::test_open_recovery_explicit_session_without_temp_directory") - .arg("--nocapture") - .env(FIXTURE_ENV, fixture.path()) - .env(MISSING_TMP_ENV, missing_tmpdir) - .output() - .unwrap(); - assert!( - output.status.success(), - "child test failed:\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - - #[tokio::test] - async fn test_open_recovery_detects_empty_local_custom_location() { - let tmp_dir = tempdir().unwrap(); - let table_name = "local-location"; - let table_path = tmp_dir.path().join(format!("{table_name}.lance")); - std::fs::create_dir(&table_path).unwrap(); - - let db = connect("memory:///open-local-location-repro") - .execute() - .await - .unwrap(); - let err = db - .open_table(table_name) - .location(table_path.to_str().unwrap()) - .execute() - .await - .unwrap_err(); - assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == table_name), - "empty built-in local location must be corrupt, got {err:?}" - ); - } - - #[derive(Debug)] - struct OpenRecoveryTestStore { + struct ParentListGuardStore { inner: Arc, parent: object_store::path::Path, - sibling_count: usize, parent_list_calls: Arc, - hide_first_list: Option>, - fail_exact_head: Option, - missing_exact_head_with_directory_message: Option, - fail_target_list: Option, - target_prefix: Option, - target_object_count: usize, - target_entries_yielded: Arc, } - impl OpenRecoveryTestStore { - fn should_hide_list(&self) -> bool { - self.hide_first_list - .as_ref() - .is_some_and(|hide| hide.swap(false, Ordering::Relaxed)) - } - } - - impl std::fmt::Display for OpenRecoveryTestStore { + impl std::fmt::Display for ParentListGuardStore { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "OpenRecoveryTestStore") + f.write_str("ParentListGuardStore") } } #[async_trait::async_trait] #[deny(clippy::missing_trait_methods)] - impl object_store::ObjectStore for OpenRecoveryTestStore { + impl object_store::ObjectStore for ParentListGuardStore { async fn put_opts( &self, location: &object_store::path::Path, @@ -4314,23 +3837,6 @@ mod tests { location: &object_store::path::Path, options: object_store::GetOptions, ) -> object_store::Result { - if options.head - && self.missing_exact_head_with_directory_message.as_ref() == Some(location) - { - return Err(object_store::Error::NotFound { - path: location.to_string(), - source: Box::new(std::io::Error::new( - std::io::ErrorKind::NotFound, - "is directory", - )), - }); - } - if self.fail_exact_head.as_ref() == Some(location) { - return Err(object_store::Error::Generic { - store: "open-recovery-test", - source: Box::new(std::io::Error::other("injected exact table probe failure")), - }); - } self.inner.get_opts(location, options).await } @@ -4358,37 +3864,9 @@ mod tests { prefix: Option<&object_store::path::Path>, ) -> futures::stream::BoxStream<'static, object_store::Result> { - if self.should_hide_list() { - return Box::pin(futures::stream::empty()); - } if prefix == Some(&self.parent) { self.parent_list_calls.fetch_add(1, Ordering::Relaxed); } - if prefix == self.fail_target_list.as_ref() { - return Box::pin(futures::stream::once(async { - Err(object_store::Error::Generic { - store: "open-recovery-test", - source: Box::new(std::io::Error::other( - "injected target table list failure", - )), - }) - })); - } - if prefix == self.target_prefix.as_ref() { - let target = self.target_prefix.clone().unwrap(); - let target_entries_yielded = self.target_entries_yielded.clone(); - let objects = (0..self.target_object_count).map(move |index| { - target_entries_yielded.fetch_add(1, Ordering::Relaxed); - Ok(object_store::ObjectMeta { - location: target.clone().join(format!("data/object_{index:06}.lance")), - last_modified: chrono::Utc::now(), - size: 0, - e_tag: None, - version: None, - }) - }); - return Box::pin(futures::stream::iter(objects)); - } self.inner.list(prefix) } @@ -4398,8 +3876,8 @@ mod tests { offset: &object_store::path::Path, ) -> futures::stream::BoxStream<'static, object_store::Result> { - if self.should_hide_list() { - return Box::pin(futures::stream::empty()); + if prefix == Some(&self.parent) { + self.parent_list_calls.fetch_add(1, Ordering::Relaxed); } self.inner.list_with_offset(prefix, offset) } @@ -4410,16 +3888,6 @@ mod tests { ) -> object_store::Result { if prefix == Some(&self.parent) { self.parent_list_calls.fetch_add(1, Ordering::Relaxed); - return Ok(object_store::ListResult { - common_prefixes: (0..self.sibling_count) - .map(|index| { - self.parent - .clone() - .join(format!("sibling_{index:06}.lance")) - }) - .collect(), - objects: Vec::new(), - }); } self.inner.list_with_delimiter(prefix).await } @@ -4444,211 +3912,32 @@ mod tests { } #[derive(Debug)] - struct OpenRecoveryTestStoreWrapper { + struct ParentListGuardWrapper { parent_list_calls: Arc, - sibling_count: usize, - hide_first_list: Option>, - fail_exact_head: Option, - missing_exact_head_with_directory_message: Option, - fail_target_list: Option, - target_prefix: Option, - target_object_count: usize, - target_entries_yielded: Arc, } - impl Default for OpenRecoveryTestStoreWrapper { - fn default() -> Self { - Self { - parent_list_calls: Arc::new(AtomicUsize::new(0)), - sibling_count: 0, - hide_first_list: None, - fail_exact_head: None, - missing_exact_head_with_directory_message: None, - fail_target_list: None, - target_prefix: None, - target_object_count: 0, - target_entries_yielded: Arc::new(AtomicUsize::new(0)), - } - } - } - - impl WrappingObjectStore for OpenRecoveryTestStoreWrapper { + impl WrappingObjectStore for ParentListGuardWrapper { fn wrap( &self, _store_prefix: &str, inner: Arc, ) -> Arc { - Arc::new(OpenRecoveryTestStore { + Arc::new(ParentListGuardStore { inner, parent: object_store::path::Path::from("database"), - sibling_count: self.sibling_count, parent_list_calls: self.parent_list_calls.clone(), - hide_first_list: self.hide_first_list.clone(), - fail_exact_head: self.fail_exact_head.clone(), - missing_exact_head_with_directory_message: self - .missing_exact_head_with_directory_message - .clone(), - fail_target_list: self.fail_target_list.clone(), - target_prefix: self.target_prefix.clone(), - target_object_count: self.target_object_count, - target_entries_yielded: self.target_entries_yielded.clone(), }) } } - /// A wrapper is part of the effective store even for a local-looking URI. - /// Recovery must not bypass it by inspecting the host path directly. #[tokio::test] - async fn test_open_recovery_uses_wrapper_for_file_uri() { - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("wrapped.lance"); - let uri = url::Url::from_file_path(&dataset_path).unwrap().to_string(); - let target = - object_store::path::Path::from_url_path(url::Url::parse(&uri).unwrap().path()).unwrap(); - let read_params = ReadParams { - store_options: Some(ObjectStoreParams { - object_store_wrapper: Some(Arc::new(OpenRecoveryTestStoreWrapper { - fail_exact_head: Some(target), - ..Default::default() - })), - ..Default::default() - }), - ..Default::default() - }; - - let err = NativeTable::open_with_params( - &uri, - "wrapped", - Vec::new(), - None, - Some(read_params), - None, - None, - HashSet::new(), - None, - ) - .await - .unwrap_err(); - - assert!( - !matches!( - &err, - Error::TableNotFound { .. } | Error::TableCorrupted { .. } - ), - "wrapper probe errors must fail closed: {err:?}" - ); - assert!( - err.to_string() - .contains("injected exact table probe failure"), - "got {err}" - ); - } - - #[tokio::test] - async fn test_open_recovery_direct_capability_respects_wrapper() { - let tmp_dir = tempdir().unwrap(); - let dataset_path = tmp_dir.path().join("direct-wrapped.lance"); - std::fs::create_dir(&dataset_path).unwrap(); - let uri = url::Url::from_file_path(&dataset_path).unwrap(); - let target = object_store::path::Path::from_url_path(uri.path()).unwrap(); - let read_params = ReadParams { - store_options: Some(ObjectStoreParams { - object_store: Some(( - Arc::new(object_store::local::LocalFileSystem::new()), - uri.clone(), - )), - native_directory_path_resolver: Some(NativeDirectoryPathResolver::new(|path| { - std::path::PathBuf::from(lance_io::local::to_local_path(path)) - })), - object_store_wrapper: Some(Arc::new(OpenRecoveryTestStoreWrapper { - fail_exact_head: Some(target), - ..Default::default() - })), - ..Default::default() - }), - ..Default::default() - }; - - let err = NativeTable::open_with_params( - uri.as_ref(), - "direct-wrapped", - Vec::new(), - None, - Some(read_params), - None, - None, - HashSet::new(), - None, - ) - .await - .unwrap_err(); - - assert!( - !matches!( - &err, - Error::TableNotFound { .. } | Error::TableCorrupted { .. } - ), - "direct wrapper probe errors must fail closed: {err:?}" - ); - assert!( - err.to_string() - .contains("injected exact table probe failure"), - "got {err}" - ); - } - - #[tokio::test] - async fn test_open_missing_does_not_trust_incidental_directory_message() { - let target = object_store::path::Path::from("database/missing.lance"); - let read_params = ReadParams { - store_options: Some(ObjectStoreParams { - object_store_wrapper: Some(Arc::new(OpenRecoveryTestStoreWrapper { - missing_exact_head_with_directory_message: Some(target), - ..Default::default() - })), - ..Default::default() - }), - ..Default::default() - }; - - let err = NativeTable::open_with_params( - "memory:///database/missing.lance", - "missing", - Vec::new(), - None, - Some(read_params), - None, - None, - HashSet::new(), - None, - ) - .await - .unwrap_err(); - - assert!( - matches!(&err, Error::TableNotFound { name, .. } if name == "missing"), - "missing provider object must remain missing, got {err:?}" - ); - } - - /// Opening one missing table must not enumerate unrelated table prefixes. - /// - /// The synthetic 100,000-prefix result models production-scale result - /// cardinality. The assertion is on the I/O scope, not wall-clock timing, - /// so it is deterministic on local and CI machines. - #[tokio::test] - async fn test_open_missing_is_independent_of_sibling_count() { - const SIBLING_COUNT: usize = 100_000; - + async fn test_open_missing_never_lists_database_parent() { let parent_list_calls = Arc::new(AtomicUsize::new(0)); - let wrapper = Arc::new(OpenRecoveryTestStoreWrapper { - parent_list_calls: parent_list_calls.clone(), - sibling_count: SIBLING_COUNT, - ..Default::default() - }); - let read_params = ReadParams { + let params = ReadParams { store_options: Some(ObjectStoreParams { - object_store_wrapper: Some(wrapper), + object_store_wrapper: Some(Arc::new(ParentListGuardWrapper { + parent_list_calls: parent_list_calls.clone(), + })), ..Default::default() }), ..Default::default() @@ -4659,7 +3948,7 @@ mod tests { "missing", Vec::new(), None, - Some(read_params), + Some(params), None, None, HashSet::new(), @@ -4675,298 +3964,10 @@ mod tests { assert_eq!( parent_list_calls.load(Ordering::Relaxed), 0, - "opening one missing table must not list the database parent" + "opening one missing table must not enumerate sibling tables" ); } - #[tokio::test] - async fn test_open_probe_failure_is_not_reported_as_missing() { - let target = object_store::path::Path::from("database/failing.lance"); - let read_params = ReadParams { - store_options: Some(ObjectStoreParams { - object_store_wrapper: Some(Arc::new(OpenRecoveryTestStoreWrapper { - fail_exact_head: Some(target), - ..Default::default() - })), - ..Default::default() - }), - ..Default::default() - }; - - let err = NativeTable::open_with_params( - "memory:///database/failing.lance", - "failing", - Vec::new(), - None, - Some(read_params), - None, - None, - HashSet::new(), - None, - ) - .await - .unwrap_err(); - - assert!( - !matches!( - &err, - Error::TableNotFound { .. } | Error::TableCorrupted { .. } - ), - "storage probe errors must fail closed: {err:?}" - ); - assert!( - err.to_string() - .contains("injected exact table probe failure"), - "got {err}" - ); - } - - #[tokio::test] - async fn test_open_list_failure_is_not_reported_as_missing() { - let target = object_store::path::Path::from("database/failing-list.lance"); - let read_params = ReadParams { - store_options: Some(ObjectStoreParams { - object_store_wrapper: Some(Arc::new(OpenRecoveryTestStoreWrapper { - fail_target_list: Some(target), - ..Default::default() - })), - ..Default::default() - }), - ..Default::default() - }; - - let err = NativeTable::open_with_params( - "memory:///database/failing-list.lance", - "failing-list", - Vec::new(), - None, - Some(read_params), - None, - None, - HashSet::new(), - None, - ) - .await - .unwrap_err(); - - assert!( - !matches!( - &err, - Error::TableNotFound { .. } | Error::TableCorrupted { .. } - ), - "storage probe errors must fail closed: {err:?}" - ); - assert!( - err.to_string() - .contains("injected target table list failure"), - "got {err}" - ); - } - - /// A corrupt table may contain many artifacts, but presence classification - /// needs only one target-scoped item. - #[tokio::test] - async fn test_open_corrupt_consumes_one_target_object() { - const TARGET_OBJECT_COUNT: usize = 100_000; - - let target = object_store::path::Path::from("database/corrupt.lance"); - let target_entries_yielded = Arc::new(AtomicUsize::new(0)); - let read_params = ReadParams { - store_options: Some(ObjectStoreParams { - object_store_wrapper: Some(Arc::new(OpenRecoveryTestStoreWrapper { - target_prefix: Some(target), - target_object_count: TARGET_OBJECT_COUNT, - target_entries_yielded: target_entries_yielded.clone(), - ..Default::default() - })), - ..Default::default() - }), - ..Default::default() - }; - - let err = NativeTable::open_with_params( - "memory:///database/corrupt.lance", - "corrupt", - Vec::new(), - None, - Some(read_params), - None, - None, - HashSet::new(), - None, - ) - .await - .unwrap_err(); - - assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == "corrupt"), - "got {err:?}" - ); - assert_eq!( - target_entries_yielded.load(Ordering::Relaxed), - 1, - "presence classification must stop after the first target object" - ); - } - - #[tokio::test] - async fn test_open_corrupt_exact_table_object() { - use object_store::ObjectStoreExt as _; - - let target = object_store::path::Path::from("database/direct.lance"); - let object_store = Arc::new(object_store::memory::InMemory::new()); - object_store - .put(&target, bytes::Bytes::new().into()) - .await - .unwrap(); - let read_params = ReadParams { - store_options: Some(ObjectStoreParams { - object_store: Some(( - object_store, - url::Url::parse("memory:///database/direct.lance").unwrap(), - )), - ..Default::default() - }), - ..Default::default() - }; - - let err = NativeTable::open_with_params( - "memory:///database/direct.lance", - "direct", - Vec::new(), - None, - Some(read_params), - None, - None, - HashSet::new(), - None, - ) - .await - .unwrap_err(); - - assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == "direct"), - "got {err:?}" - ); - } - - #[tokio::test] - async fn test_open_recovery_detects_empty_direct_local_directory() { - let tmp_dir = tempdir().unwrap(); - let table_name = "direct-local"; - let table_path = tmp_dir.path().join(format!("{table_name}.lance")); - std::fs::create_dir(&table_path).unwrap(); - let uri = url::Url::from_file_path(&table_path).unwrap(); - let cases = [ - ("legacy", None), - ( - "capable", - Some(NativeDirectoryPathResolver::new(|path| { - std::path::PathBuf::from(lance_io::local::to_local_path(path)) - })), - ), - ]; - - for (case, native_directory_path_resolver) in cases { - let read_params = ReadParams { - store_options: Some(ObjectStoreParams { - object_store: Some(( - Arc::new(object_store::local::LocalFileSystem::new()), - uri.clone(), - )), - native_directory_path_resolver, - ..Default::default() - }), - ..Default::default() - }; - - let err = NativeTable::open_with_params( - uri.as_ref(), - table_name, - Vec::new(), - None, - Some(read_params), - None, - None, - HashSet::new(), - None, - ) - .await - .unwrap_err(); - - assert!( - matches!(&err, Error::TableCorrupted { name, .. } if name == table_name), - "{case} empty direct local directory must remain corrupt, got {err:?}" - ); - } - } - - /// If a concurrent create commits after the first manifest lookup misses, - /// opening should return the completed table instead of stale corruption. - #[tokio::test] - async fn test_open_recovers_from_concurrent_create_commit() { - use lance_table::io::commit::RenameCommitHandler; - - let uri = "memory:///database/test.lance"; - let inner = Arc::new(object_store::memory::InMemory::new()); - - let batch = make_test_batches(); - let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); - Dataset::write( - reader, - uri, - Some(WriteParams { - store_params: Some(ObjectStoreParams { - object_store: Some((inner.clone(), url::Url::parse(uri).unwrap())), - ..Default::default() - }), - commit_handler: Some(Arc::new(RenameCommitHandler)), - ..Default::default() - }), - ) - .await - .unwrap(); - - let hide_first_list = Arc::new(AtomicBool::new(true)); - let object_store = Arc::new(OpenRecoveryTestStore { - inner, - parent: object_store::path::Path::from("does-not-match"), - sibling_count: 0, - parent_list_calls: Arc::new(AtomicUsize::new(0)), - hide_first_list: Some(hide_first_list.clone()), - fail_exact_head: None, - missing_exact_head_with_directory_message: None, - fail_target_list: None, - target_prefix: None, - target_object_count: 0, - target_entries_yielded: Arc::new(AtomicUsize::new(0)), - }); - let read_params = ReadParams { - store_options: Some(ObjectStoreParams { - object_store: Some((object_store, url::Url::parse(uri).unwrap())), - ..Default::default() - }), - ..Default::default() - }; - - let table = NativeTable::open_with_params( - uri, - "test", - Vec::new(), - None, - Some(read_params), - None, - None, - HashSet::new(), - None, - ) - .await - .unwrap(); - - assert!(!hide_first_list.load(Ordering::Relaxed)); - assert_eq!(table.version().await.unwrap(), 1); - } - #[test] #[cfg(not(windows))] fn test_object_store_path() {