From 3e4be3c4b5c2493bfbbf68bfa68d1ec967e26633 Mon Sep 17 00:00:00 2001 From: lancedb automation Date: Wed, 4 Mar 2026 02:16:34 +0000 Subject: [PATCH] chore: update lance dependency to v4.0.0-beta.7 --- Cargo.lock | 112 +++++++++++++++---------- Cargo.toml | 28 +++---- java/pom.xml | 2 +- python/src/storage_options.rs | 37 ++++---- rust/lancedb/src/database/namespace.rs | 84 +++++-------------- 5 files changed, 118 insertions(+), 145 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b6a394d09..802255dac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3088,8 +3088,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow-array", "rand 0.9.2", @@ -4260,8 +4260,8 @@ dependencies = [ [[package]] name = "lance" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow", "arrow-arith", @@ -4315,7 +4315,7 @@ dependencies = [ "semver", "serde", "serde_json", - "snafu", + "snafu 0.9.0", "tantivy", "tokio", "tokio-stream", @@ -4327,8 +4327,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow-array", "arrow-buffer", @@ -4338,6 +4338,7 @@ dependencies = [ "arrow-schema", "arrow-select", "bytes", + "futures", "getrandom 0.2.16", "half", "jsonb", @@ -4347,8 +4348,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrayref", "paste", @@ -4357,8 +4358,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow-array", "arrow-buffer", @@ -4384,7 +4385,7 @@ dependencies = [ "rand 0.9.2", "roaring", "serde_json", - "snafu", + "snafu 0.9.0", "tempfile", "tokio", "tokio-stream", @@ -4395,8 +4396,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow", "arrow-array", @@ -4419,15 +4420,15 @@ dependencies = [ "pin-project", "prost", "prost-build", - "snafu", + "snafu 0.9.0", "tokio", "tracing", ] [[package]] name = "lance-datagen" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow", "arrow-array", @@ -4445,8 +4446,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow-arith", "arrow-array", @@ -4473,7 +4474,7 @@ dependencies = [ "prost-build", "prost-types", "rand 0.9.2", - "snafu", + "snafu 0.9.0", "strum", "tokio", "tracing", @@ -4483,8 +4484,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow-arith", "arrow-array", @@ -4509,15 +4510,15 @@ dependencies = [ "prost", "prost-build", "prost-types", - "snafu", + "snafu 0.9.0", "tokio", "tracing", ] [[package]] name = "lance-index" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow", "arrow-arith", @@ -4569,7 +4570,7 @@ dependencies = [ "serde", "serde_json", "smallvec", - "snafu", + "snafu 0.9.0", "tantivy", "tempfile", "tokio", @@ -4580,8 +4581,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow", "arrow-arith", @@ -4613,7 +4614,7 @@ dependencies = [ "prost", "rand 0.9.2", "serde", - "snafu", + "snafu 0.9.0", "tempfile", "tokio", "tracing", @@ -4622,8 +4623,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow-array", "arrow-buffer", @@ -4639,21 +4640,21 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow", "async-trait", "bytes", "lance-core", "lance-namespace-reqwest-client", - "snafu", + "snafu 0.9.0", ] [[package]] name = "lance-namespace-impls" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow", "arrow-ipc", @@ -4675,7 +4676,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "snafu", + "snafu 0.9.0", "tokio", "tower", "tower-http 0.5.2", @@ -4697,8 +4698,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow", "arrow-array", @@ -4728,7 +4729,7 @@ dependencies = [ "semver", "serde", "serde_json", - "snafu", + "snafu 0.9.0", "tokio", "tracing", "url", @@ -4737,8 +4738,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "3.0.0-rc.2" -source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181" +version = "4.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71" dependencies = [ "arrow-array", "arrow-schema", @@ -4819,7 +4820,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "snafu", + "snafu 0.8.9", "tempfile", "test-log", "tokenizers", @@ -4865,7 +4866,7 @@ dependencies = [ "pyo3", "pyo3-async-runtimes", "pyo3-build-config", - "snafu", + "snafu 0.8.9", "tokio", ] @@ -7777,7 +7778,16 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" dependencies = [ - "snafu-derive", + "snafu-derive 0.8.9", +] + +[[package]] +name = "snafu" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" +dependencies = [ + "snafu-derive 0.9.0", ] [[package]] @@ -7792,6 +7802,18 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "snafu-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "socket2" version = "0.5.10" diff --git a/Cargo.toml b/Cargo.toml index 9ddddc4fb..b26437f2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,20 +15,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=3.0.0-rc.2", default-features = false, "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=3.0.0-rc.2", default-features = false, "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=3.0.0-rc.2", default-features = false, "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=4.0.0-beta.7", default-features = false, "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=4.0.0-beta.7", default-features = false, "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=4.0.0-beta.7", default-features = false, "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } ahash = "0.8" # Note that this one does not include pyarrow arrow = { version = "57.2", optional = false } diff --git a/java/pom.xml b/java/pom.xml index 28811c72b..6fa40d5a6 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 3.1.0-beta.2 + 4.0.0-beta.7 false 2.30.0 1.7 diff --git a/python/src/storage_options.rs b/python/src/storage_options.rs index d00c2aa16..7fd390413 100644 --- a/python/src/storage_options.rs +++ b/python/src/storage_options.rs @@ -66,12 +66,11 @@ impl StorageOptionsProvider for PyStorageOptionsProviderWrapper { .inner .bind(py) .call_method0("fetch_storage_options") - .map_err(|e| lance_core::Error::IO { - source: Box::new(std::io::Error::other(format!( + .map_err(|e| { + lance_core::Error::io_source(Box::new(std::io::Error::other(format!( "Failed to call fetch_storage_options: {}", e - ))), - location: snafu::location!(), + )))) })?; // If result is None, return None @@ -81,26 +80,25 @@ impl StorageOptionsProvider for PyStorageOptionsProviderWrapper { // Extract the result dict - should be a flat Map let result_dict = result.downcast::().map_err(|_| { - lance_core::Error::InvalidInput { - source: "fetch_storage_options() must return None or a dict of string key-value pairs".into(), - location: snafu::location!(), - } + lance_core::Error::invalid_input( + "fetch_storage_options() must return None or a dict of string key-value pairs", + ) })?; // Convert all entries to HashMap let mut storage_options = HashMap::new(); for (key, value) in result_dict.iter() { let key_str: String = key.extract().map_err(|e| { - lance_core::Error::InvalidInput { - source: format!("Storage option key must be a string: {}", e).into(), - location: snafu::location!(), - } + lance_core::Error::invalid_input(format!( + "Storage option key must be a string: {}", + e + )) })?; let value_str: String = value.extract().map_err(|e| { - lance_core::Error::InvalidInput { - source: format!("Storage option value must be a string: {}", e).into(), - location: snafu::location!(), - } + lance_core::Error::invalid_input(format!( + "Storage option value must be a string: {}", + e + )) })?; storage_options.insert(key_str, value_str); } @@ -109,13 +107,10 @@ impl StorageOptionsProvider for PyStorageOptionsProviderWrapper { }) }) .await - .map_err(|e| lance_core::Error::IO { - source: Box::new(std::io::Error::other(format!( + .map_err(|e| lance_core::Error::io_source(Box::new(std::io::Error::other(format!( "Task join error: {}", e - ))), - location: snafu::location!(), - })? + )))))? } fn provider_id(&self) -> String { diff --git a/rust/lancedb/src/database/namespace.rs b/rust/lancedb/src/database/namespace.rs index 90b5c19bd..d22caa945 100644 --- a/rust/lancedb/src/database/namespace.rs +++ b/rust/lancedb/src/database/namespace.rs @@ -11,14 +11,13 @@ use lance_io::object_store::{ObjectStoreParams, StorageOptionsAccessor}; use lance_namespace::{ LanceNamespace, models::{ - CreateEmptyTableRequest, CreateNamespaceRequest, CreateNamespaceResponse, - DeclareTableRequest, DescribeNamespaceRequest, DescribeNamespaceResponse, - DescribeTableRequest, DropNamespaceRequest, DropNamespaceResponse, DropTableRequest, - ListNamespacesRequest, ListNamespacesResponse, ListTablesRequest, ListTablesResponse, + CreateNamespaceRequest, CreateNamespaceResponse, DeclareTableRequest, + DescribeNamespaceRequest, DescribeNamespaceResponse, DescribeTableRequest, + DropNamespaceRequest, DropNamespaceResponse, DropTableRequest, ListNamespacesRequest, + ListNamespacesResponse, ListTablesRequest, ListTablesResponse, }, }; use lance_namespace_impls::ConnectBuilder; -use log::warn; use crate::database::ReadConsistency; use crate::error::{Error, Result}; @@ -206,70 +205,27 @@ impl Database for LanceNamespaceDatabase { let mut table_id = request.namespace.clone(); table_id.push(request.name.clone()); - // Try declare_table first, falling back to create_empty_table for backwards - // compatibility with older namespace clients that don't support declare_table + // Declare table metadata through lance-namespace before creating data files. let declare_request = DeclareTableRequest { id: Some(table_id.clone()), ..Default::default() }; - let (location, initial_storage_options) = - match self.namespace.declare_table(declare_request).await { - Ok(response) => { - let loc = response.location.ok_or_else(|| Error::Runtime { - message: "Table location is missing from declare_table response" - .to_string(), - })?; - // Use storage options from response, fall back to self.storage_options - let opts = response - .storage_options - .or_else(|| Some(self.storage_options.clone())) - .filter(|o| !o.is_empty()); - (loc, opts) - } - Err(e) => { - // Check if the error is "not supported" and try create_empty_table as fallback - let err_str = e.to_string().to_lowercase(); - if err_str.contains("not supported") || err_str.contains("not implemented") { - warn!( - "declare_table is not supported by the namespace client, \ - falling back to deprecated create_empty_table. \ - create_empty_table is deprecated and will be removed in Lance 3.0.0. \ - Please upgrade your namespace client to support declare_table." - ); - #[allow(deprecated)] - let create_empty_request = CreateEmptyTableRequest { - id: Some(table_id.clone()), - ..Default::default() - }; - - #[allow(deprecated)] - let create_response = self - .namespace - .create_empty_table(create_empty_request) - .await - .map_err(|e| Error::Runtime { - message: format!("Failed to create empty table: {}", e), - })?; - - let loc = create_response.location.ok_or_else(|| Error::Runtime { - message: "Table location is missing from create_empty_table response" - .to_string(), - })?; - // For deprecated path, use self.storage_options - let opts = if self.storage_options.is_empty() { - None - } else { - Some(self.storage_options.clone()) - }; - (loc, opts) - } else { - return Err(Error::Runtime { - message: format!("Failed to declare table: {}", e), - }); - } - } - }; + let declare_response = self + .namespace + .declare_table(declare_request) + .await + .map_err(|e| Error::Runtime { + message: format!("Failed to declare table: {}", e), + })?; + let location = declare_response.location.ok_or_else(|| Error::Runtime { + message: "Table location is missing from declare_table response".to_string(), + })?; + // Use storage options from response, fall back to self.storage_options + let initial_storage_options = declare_response + .storage_options + .or_else(|| Some(self.storage_options.clone())) + .filter(|o| !o.is_empty()); let write_params = if let Some(storage_opts) = initial_storage_options { let mut params = request.write_options.lance_write_params.unwrap_or_default();