fix: make table existence manifest-authoritative

This commit is contained in:
Yang Cen
2026-08-13 17:30:53 +08:00
parent e9aed2c2ec
commit e12407519e
6 changed files with 247 additions and 1129 deletions
Generated
+37 -37
View File
@@ -1756,7 +1756,7 @@ version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3034,7 +3034,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3257,7 +3257,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3456,7 +3456,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4561,7 +4561,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4816,7 +4816,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arc-swap",
"arrow",
@@ -4891,7 +4891,7 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4913,7 +4913,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4927,7 +4927,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4937,7 +4937,7 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrayref",
"crunchy",
@@ -4948,7 +4948,7 @@ dependencies = [
[[package]]
name = "lance-core"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4989,7 +4989,7 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow",
"arrow-array",
@@ -5020,7 +5020,7 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow",
"arrow-array",
@@ -5038,7 +5038,7 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"proc-macro2",
"quote",
@@ -5048,7 +5048,7 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5083,7 +5083,7 @@ dependencies = [
[[package]]
name = "lance-file"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5115,7 +5115,7 @@ dependencies = [
[[package]]
name = "lance-index"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arc-swap",
"arrow",
@@ -5183,7 +5183,7 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5206,7 +5206,7 @@ dependencies = [
[[package]]
name = "lance-io"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow",
"arrow-array",
@@ -5243,7 +5243,7 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5260,7 +5260,7 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow",
"async-trait",
@@ -5273,7 +5273,7 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5327,7 +5327,7 @@ dependencies = [
[[package]]
name = "lance-select"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5343,7 +5343,7 @@ dependencies = [
[[package]]
name = "lance-table"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow",
"arrow-array",
@@ -5384,7 +5384,7 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5398,7 +5398,7 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=f8d628fe6af452cc168e0406c1199a91fb059cda#f8d628fe6af452cc168e0406c1199a91fb059cda"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.6#5ab688688c411111d94d279bacd037d7c319dc10"
dependencies = [
"frostem",
"icu_segmenter",
@@ -6249,7 +6249,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -7641,8 +7641,8 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
dependencies = [
"heck 0.4.1",
"itertools 0.11.0",
"heck 0.5.0",
"itertools 0.14.0",
"log",
"multimap",
"petgraph",
@@ -7661,7 +7661,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
dependencies = [
"anyhow",
"itertools 0.11.0",
"itertools 0.14.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -7932,7 +7932,7 @@ dependencies = [
"once_cell",
"socket2 0.6.3",
"tracing",
"windows-sys 0.59.0",
"windows-sys 0.60.2",
]
[[package]]
@@ -8710,7 +8710,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -8781,7 +8781,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -9337,7 +9337,7 @@ version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
dependencies = [
"heck 0.4.1",
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9349,7 +9349,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
dependencies = [
"heck 0.4.1",
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9771,7 +9771,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -10748,7 +10748,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.6", "rev" = "f8d628fe6af452cc168e0406c1199a91fb059cda", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=11.0.0-beta.6", default-features = false, "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.6", default-features = false, "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.6", default-features = false, "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.6", "tag" = "v11.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "58.0.0", optional = false }
@@ -1,20 +1,20 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Release benchmark for opening a missing table as sibling-table cardinality grows.
//!
//! The fixture uses real `.lance` directories and marker files. Fixture creation is
//! outside the timed section. Defaults intentionally cover 1k, 10k, and 100k siblings
//! with 10 warmups and 100 distinct missing-table opens per scale:
//!
//! ```text
//! cargo run --release -p lancedb --example bench_open_missing_table
//! ```
//!
//! `BENCH_SIBLINGS`, `BENCH_WARMUPS`, and `BENCH_TRIALS` override those defaults.
//! Reduced settings are useful only as a smoke test. Performance comparisons require
//! the same machine, filesystem, fixture sizes, settings, lockfile, and alternating
//! baseline/candidate execution order.
// Release benchmark for opening a missing table as sibling-table cardinality grows.
//
// The fixture uses real `.lance` directories and marker files. Fixture creation is
// outside the timed section. Defaults intentionally cover 1k, 10k, and 100k siblings
// with 10 warmups and 100 distinct missing-table opens per scale:
//
// ```text
// cargo run --release -p lancedb --example bench_open_missing_table
// ```
//
// `BENCH_SIBLINGS`, `BENCH_WARMUPS`, and `BENCH_TRIALS` override those defaults.
// Reduced settings are useful only as a smoke test. Performance comparisons require
// the same machine, filesystem, fixture sizes, settings, lockfile, and alternating
// baseline/candidate execution order.
use std::time::{Duration, Instant};
+8 -4
View File
@@ -409,6 +409,11 @@ impl Connection {
///
/// The names will be returned in lexicographical order (ascending)
///
/// Listing databases discover physical `*.lance` entries without opening every
/// dataset. The result is a point-in-time discovery snapshot: an entry may still be
/// under creation, may contain only uncommitted storage, or may be concurrently
/// dropped before it is opened.
///
/// The parameters `page_token` and `limit` can be used to paginate the results
pub fn table_names(&self) -> TableNamesBuilder {
TableNamesBuilder::new(self.internal.clone())
@@ -456,10 +461,9 @@ impl Connection {
///
/// # Returns
/// Created [`TableRef`], or [`Error::TableNotFound`] if the table does not exist.
/// If the table's storage is present but holds no readable dataset (for example a
/// `<name>.lance` directory left behind by an interrupted drop and re-create, which
/// [`Self::table_names`] still lists) this returns [`Error::TableCorrupted`]
/// instead.
/// On listing databases, a committed Lance manifest is authoritative for table
/// existence. Uncommitted files or a physical `<name>.lance` directory alone do not
/// make a table openable.
pub fn open_table(&self, name: impl Into<String>) -> OpenTableBuilder {
OpenTableBuilder::new(
self.internal.clone(),
+115 -2
View File
@@ -1291,16 +1291,21 @@ impl Database for ListingDatabase {
mod tests {
use super::*;
use crate::Table;
use crate::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream};
use crate::connection::ConnectRequest;
use crate::data::scannable::Scannable;
use crate::database::{CreateTableMode, CreateTableRequest};
use crate::query::QueryRequest;
use crate::table::{AnyQuery, WriteOptions};
use arrow_array::{Int32Array, RecordBatch, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use arrow_schema::{DataType, Field, Schema, SchemaRef};
use futures::{TryStreamExt, stream::once};
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use tempfile::tempdir;
use tokio::sync::Barrier;
use tokio::time::timeout;
async fn setup_database() -> (tempfile::TempDir, ListingDatabase) {
let tempdir = tempdir().unwrap();
@@ -1324,6 +1329,114 @@ mod tests {
(tempdir, db)
}
struct BarrierScannable {
batch: RecordBatch,
barrier: Arc<Barrier>,
}
impl Scannable for BarrierScannable {
fn schema(&self) -> SchemaRef {
self.batch.schema()
}
fn scan_as_stream(&mut self) -> SendableRecordBatchStream {
let batch = self.batch.clone();
let schema = batch.schema();
let barrier = self.barrier.clone();
Box::pin(SimpleRecordBatchStream {
schema,
stream: once(async move {
barrier.wait().await;
Ok(batch)
}),
})
}
}
fn create_request(name: &str, data: Box<dyn Scannable>) -> CreateTableRequest {
CreateTableRequest {
name: name.to_string(),
namespace_path: vec![],
data,
mode: CreateTableMode::Create,
write_options: Default::default(),
location: None,
namespace_client: None,
}
}
#[tokio::test]
async fn test_create_ignores_uncommitted_storage_without_manifest() {
let (tmp_dir, db) = setup_database().await;
let data_dir = tmp_dir.path().join("test.lance/data");
std::fs::create_dir_all(&data_dir).unwrap();
std::fs::write(data_dir.join("orphan.lance"), b"uncommitted").unwrap();
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
let batch =
RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1]))]).unwrap();
let table = db
.create_table(create_request("test", Box::new(batch)))
.await
.unwrap();
assert_eq!(table.count_rows(None).await.unwrap(), 1);
}
#[tokio::test]
async fn test_concurrent_create_is_arbitrated_by_manifest_commit() {
let uri = format!("memory:///concurrent-create-{}", uuid::Uuid::new_v4());
let db = crate::connect(&uri).execute().await.unwrap();
let store: Arc<dyn object_store::ObjectStore> =
Arc::new(object_store::memory::InMemory::new());
let table_url = url::Url::parse("memory:///database/test.lance").unwrap();
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
let batch =
RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1]))]).unwrap();
let barrier = Arc::new(Barrier::new(2));
#[allow(deprecated)]
let request = |batch, barrier| {
let mut request = create_request("test", Box::new(BarrierScannable { batch, barrier }));
request.write_options = WriteOptions {
lance_write_params: Some(lance::dataset::WriteParams {
store_params: Some(ObjectStoreParams {
object_store: Some((store.clone(), table_url.clone())),
..Default::default()
}),
commit_handler: Some(Arc::new(
lance_table::io::commit::ConditionalPutCommitHandler,
)),
..Default::default()
}),
};
request
};
let left = db
.database()
.create_table(request(batch.clone(), barrier.clone()));
let right = db.database().create_table(request(batch, barrier));
let (left, right) = timeout(Duration::from_secs(30), async { tokio::join!(left, right) })
.await
.expect("concurrent creates deadlocked");
let results = [left, right];
assert_eq!(
results.iter().filter(|result| result.is_ok()).count(),
1,
"expected one successful create, got {results:?}"
);
assert_eq!(
results
.iter()
.filter(|result| matches!(result, Err(Error::TableAlreadyExists { .. })))
.count(),
1,
"expected one manifest conflict, got {results:?}"
);
}
#[tokio::test]
async fn test_listing_database_root_ops_do_not_create_manifest() {
let tempdir = tempdir().unwrap();
+59 -1058
View File
File diff suppressed because it is too large Load Diff