rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-14 06:36:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(storage): preserve direct local directory detection

Browse Source 
Carry explicit native-path authority through direct store bindings so empty local table directories remain corrupt instead of missing.
This commit is contained in:
Yang Cen
2026-08-12 21:44:19 +08:00
parent 0d9541f4b4
commit 4ca5653367
3 changed files with 78 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+22 -22
View File
@@ -3456,7 +3456,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4816,7 +4816,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arc-swap",
"arrow",
@@ -4891,7 +4891,7 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4913,7 +4913,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4927,7 +4927,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4937,7 +4937,7 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrayref",
"crunchy",
@@ -4948,7 +4948,7 @@ dependencies = [
[[package]]
name = "lance-core"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4989,7 +4989,7 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow",
"arrow-array",
@@ -5020,7 +5020,7 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow",
"arrow-array",
@@ -5038,7 +5038,7 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"proc-macro2",
"quote",
@@ -5048,7 +5048,7 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5083,7 +5083,7 @@ dependencies = [
[[package]]
name = "lance-file"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5115,7 +5115,7 @@ dependencies = [
[[package]]
name = "lance-index"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arc-swap",
"arrow",
@@ -5183,7 +5183,7 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5206,7 +5206,7 @@ dependencies = [
[[package]]
name = "lance-io"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow",
"arrow-array",
@@ -5243,7 +5243,7 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5260,7 +5260,7 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow",
"async-trait",
@@ -5273,7 +5273,7 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5327,7 +5327,7 @@ dependencies = [
[[package]]
name = "lance-select"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5343,7 +5343,7 @@ dependencies = [
[[package]]
name = "lance-table"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow",
"arrow-array",
@@ -5384,7 +5384,7 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5398,7 +5398,7 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "11.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3"
source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5"
dependencies = [
"frostem",
"icu_segmenter",
Cargo.toml
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "58.0.0", optional = false }
rust/lancedb/src/table.rs
+42
View File
@@ -3763,6 +3763,7 @@ mod tests {
use lance::Dataset;
use lance::io::{ObjectStoreParams, WrappingObjectStore};
use lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY_POSITION;
use lance_io::object_store::NativeDirectoryPathResolver;
use tempfile::tempdir;
use super::*;
@@ -4780,6 +4781,47 @@ mod tests {
);
}
#[tokio::test]
async fn test_open_recovery_detects_empty_direct_local_directory() {
let tmp_dir = tempdir().unwrap();
let table_name = "direct-local";
let table_path = tmp_dir.path().join(format!("{table_name}.lance"));
std::fs::create_dir(&table_path).unwrap();
let uri = url::Url::from_file_path(&table_path).unwrap();
let read_params = ReadParams {
store_options: Some(ObjectStoreParams {
object_store: Some((
Arc::new(object_store::local::LocalFileSystem::new()),
uri.clone(),
)),
native_directory_path_resolver: Some(NativeDirectoryPathResolver::new(|path| {
std::path::PathBuf::from(lance_io::local::to_local_path(path))
})),
..Default::default()
}),
..Default::default()
};
let err = NativeTable::open_with_params(
uri.as_ref(),
table_name,
Vec::new(),
None,
Some(read_params),
None,
None,
HashSet::new(),
None,
)
.await
.unwrap_err();
assert!(
matches!(&err, Error::TableCorrupted { name, .. } if name == table_name),
"empty direct local directory must remain corrupt, got {err:?}"
);
}
/// If a concurrent create commits after the first manifest lookup misses,
/// opening should return the completed table instead of stale corruption.
#[tokio::test]