diff --git a/Cargo.lock b/Cargo.lock index f9c889d34..7b44a440c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3456,7 +3456,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4816,7 +4816,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arc-swap", "arrow", @@ -4891,7 +4891,7 @@ dependencies = [ [[package]] name = "lance-arrow" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow-array", "arrow-buffer", @@ -4913,7 +4913,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow-array", "arrow-buffer", @@ -4927,7 +4927,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow-array", "arrow-schema", @@ -4937,7 +4937,7 @@ dependencies = [ [[package]] name = "lance-bitpacking" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrayref", "crunchy", @@ -4948,7 +4948,7 @@ dependencies = [ [[package]] name = "lance-core" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow-array", "arrow-buffer", @@ -4989,7 +4989,7 @@ dependencies = [ [[package]] name = "lance-datafusion" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow", "arrow-array", @@ -5020,7 +5020,7 @@ dependencies = [ [[package]] name = "lance-datagen" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow", "arrow-array", @@ -5038,7 +5038,7 @@ dependencies = [ [[package]] name = "lance-derive" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "proc-macro2", "quote", @@ -5048,7 +5048,7 @@ dependencies = [ [[package]] name = "lance-encoding" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow-arith", "arrow-array", @@ -5083,7 +5083,7 @@ dependencies = [ [[package]] name = "lance-file" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow-arith", "arrow-array", @@ -5115,7 +5115,7 @@ dependencies = [ [[package]] name = "lance-index" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arc-swap", "arrow", @@ -5183,7 +5183,7 @@ dependencies = [ [[package]] name = "lance-index-core" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow-array", "arrow-schema", @@ -5206,7 +5206,7 @@ dependencies = [ [[package]] name = "lance-io" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow", "arrow-array", @@ -5243,7 +5243,7 @@ dependencies = [ [[package]] name = "lance-linalg" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow-array", "arrow-buffer", @@ -5260,7 +5260,7 @@ dependencies = [ [[package]] name = "lance-namespace" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow", "async-trait", @@ -5273,7 +5273,7 @@ dependencies = [ [[package]] name = "lance-namespace-impls" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow", "arrow-ipc", @@ -5327,7 +5327,7 @@ dependencies = [ [[package]] name = "lance-select" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow-array", "arrow-buffer", @@ -5343,7 +5343,7 @@ dependencies = [ [[package]] name = "lance-table" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow", "arrow-array", @@ -5384,7 +5384,7 @@ dependencies = [ [[package]] name = "lance-testing" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "arrow-array", "arrow-schema", @@ -5398,7 +5398,7 @@ dependencies = [ [[package]] name = "lance-tokenizer" version = "11.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?rev=c16e73b72e9d65886d01bd024f967ad909ce2fc3#c16e73b72e9d65886d01bd024f967ad909ce2fc3" +source = "git+https://github.com/lance-format/lance.git?rev=a7e3a2d2fffa5b085487bc72c3b9682df8f27db5#a7e3a2d2fffa5b085487bc72c3b9682df8f27db5" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 9636b7289..cb14ae4dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=11.0.0-beta.6", "rev" = "c16e73b72e9d65886d01bd024f967ad909ce2fc3", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=11.0.0-beta.6", default-features = false, "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=11.0.0-beta.6", "rev" = "a7e3a2d2fffa5b085487bc72c3b9682df8f27db5", "git" = "https://github.com/lance-format/lance.git" } ahash = "0.8" # Note that this one does not include pyarrow arrow = { version = "58.0.0", optional = false } diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index e8b31a60a..b00b582b7 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -3763,6 +3763,7 @@ mod tests { use lance::Dataset; use lance::io::{ObjectStoreParams, WrappingObjectStore}; use lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY_POSITION; + use lance_io::object_store::NativeDirectoryPathResolver; use tempfile::tempdir; use super::*; @@ -4780,6 +4781,47 @@ mod tests { ); } + #[tokio::test] + async fn test_open_recovery_detects_empty_direct_local_directory() { + let tmp_dir = tempdir().unwrap(); + let table_name = "direct-local"; + let table_path = tmp_dir.path().join(format!("{table_name}.lance")); + std::fs::create_dir(&table_path).unwrap(); + let uri = url::Url::from_file_path(&table_path).unwrap(); + let read_params = ReadParams { + store_options: Some(ObjectStoreParams { + object_store: Some(( + Arc::new(object_store::local::LocalFileSystem::new()), + uri.clone(), + )), + native_directory_path_resolver: Some(NativeDirectoryPathResolver::new(|path| { + std::path::PathBuf::from(lance_io::local::to_local_path(path)) + })), + ..Default::default() + }), + ..Default::default() + }; + + let err = NativeTable::open_with_params( + uri.as_ref(), + table_name, + Vec::new(), + None, + Some(read_params), + None, + None, + HashSet::new(), + None, + ) + .await + .unwrap_err(); + + assert!( + matches!(&err, Error::TableCorrupted { name, .. } if name == table_name), + "empty direct local directory must remain corrupt, got {err:?}" + ); + } + /// If a concurrent create commits after the first manifest lookup misses, /// opening should return the completed table instead of stale corruption. #[tokio::test]