From 1f0b39cc8df830faeec50d375b0fdc980e4525d5 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 18 Nov 2022 14:17:38 +0800 Subject: [PATCH] chore: Bump OpenDAL to v0.20 (#569) Signed-off-by: Xuanwo --- Cargo.lock | 57 ++++++++++++++------- src/catalog/Cargo.toml | 4 +- src/object-store/Cargo.toml | 2 +- src/object-store/src/backend/azblob.rs | 2 +- src/object-store/src/backend/fs.rs | 2 +- src/object-store/src/backend/memory.rs | 2 +- src/object-store/src/backend/s3.rs | 2 +- src/object-store/src/lib.rs | 4 +- src/object-store/src/util.rs | 4 +- src/object-store/tests/object_store_test.rs | 4 +- src/storage/src/manifest/storage.rs | 6 +-- 11 files changed, 55 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49930fc549..d39c873fe7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -507,9 +507,9 @@ dependencies = [ [[package]] name = "backon" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62fde477410f4e8a92e2fd4b8536b16ad98270478211d1516cded50bad7208e" +checksum = "6cd1a59bc091e593ee9ed62df4e4a07115e00a0e0a52fd7e0e04540773939b80" dependencies = [ "futures", "pin-project", @@ -571,6 +571,25 @@ dependencies = [ "serde", ] +[[package]] +name = "bincode" +version = "2.0.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb50c5a2ef4b9b1e7ae73e3a73b52ea24b20312d629f9c4df28260b7ad2c3c4" +dependencies = [ + "bincode_derive", + "serde", +] + +[[package]] +name = "bincode_derive" +version = "2.0.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a45a23389446d2dd25dc8e73a7a3b3c43522b630cac068927f0649d43d719d2" +dependencies = [ + "virtue", +] + [[package]] name = "bindgen" version = "0.59.2" @@ -3014,12 +3033,6 @@ dependencies = [ "digest", ] -[[package]] -name = "md5" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" - [[package]] name = "memchr" version = "2.5.0" @@ -3563,20 +3576,22 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "opendal" -version = "0.17.4" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1ac2fba66b4b574259a4e1c6a1b57b8e649682ed7da167c9cde120c1495f2cd" +checksum = "63b17b778cf11d10fbaaae4a5a0f82d5c6f527f96a9e4843f4e2dd6cd0dbe580" dependencies = [ "anyhow", "async-compat", "async-trait", + "backon", "base64", + "bincode 2.0.0-rc.2", "bytes", "flagset", "futures", "http", "log", - "md5", + "md-5", "once_cell", "parking_lot", "percent-encoding", @@ -4377,9 +4392,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.25.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58e21a144a0ffb5fad7b464babcdab934a325ad69b7c0373bcfef5cbd9799ca9" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" dependencies = [ "memchr", "serde", @@ -4583,9 +4598,9 @@ dependencies = [ [[package]] name = "reqsign" -version = "0.4.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6e28b79573bf9e503b514799a5533935fa63bec22d582d4d11cab1eab7a040d" +checksum = "e22524be78041476bf8673f2720fa1000f34432b384d9ad5846b024569a4b150" dependencies = [ "anyhow", "backon", @@ -4811,7 +4826,7 @@ name = "rustpython-bytecode" version = "0.1.2" source = "git+https://github.com/RustPython/RustPython?rev=02a1d1d#02a1d1d7db57afbb78049599c2585cc7cd59e6d3" dependencies = [ - "bincode", + "bincode 1.3.3", "bitflags", "bstr", "itertools", @@ -5973,6 +5988,7 @@ dependencies = [ "itoa 1.0.3", "libc", "num_threads", + "serde", "time-macros", ] @@ -6407,7 +6423,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand 0.8.5", + "rand 0.4.6", "static_assertions", ] @@ -6584,7 +6600,6 @@ checksum = "b97acb4c28a254fd7a4aeec976c46a7fa404eac4d7c134b30c75144846d7cb8f" dependencies = [ "base64", "chunked_transfer", - "flate2", "log", "once_cell", "rustls", @@ -6644,6 +6659,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "virtue" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b60dcd6a64dd45abf9bd426970c9843726da7fc08f44cd6fcebf68c21220a63" + [[package]] name = "volatile" version = "0.3.0" diff --git a/src/catalog/Cargo.toml b/src/catalog/Cargo.toml index b4b88d966e..ad52271a4e 100644 --- a/src/catalog/Cargo.toml +++ b/src/catalog/Cargo.toml @@ -27,7 +27,7 @@ futures = "0.3" futures-util = "0.3" lazy_static = "1.4" meta-client = { path = "../meta-client" } -opendal = "0.17" +opendal = "0.20" regex = "1.6" serde = "1.0" serde_json = "1.0" @@ -40,7 +40,7 @@ tokio = { version = "1.18", features = ["full"] } chrono = "0.4" log-store = { path = "../log-store" } object-store = { path = "../object-store" } -opendal = "0.17" +opendal = "0.20" storage = { path = "../storage" } mito = { path = "../mito" } tempdir = "0.3" diff --git a/src/object-store/Cargo.toml b/src/object-store/Cargo.toml index 91e9eb3bb2..e7e63109e1 100644 --- a/src/object-store/Cargo.toml +++ b/src/object-store/Cargo.toml @@ -6,7 +6,7 @@ license = "Apache-2.0" [dependencies] futures = { version = "0.3" } -opendal = "0.17" +opendal = "0.20" tokio = { version = "1.0", features = ["full"] } [dev-dependencies] diff --git a/src/object-store/src/backend/azblob.rs b/src/object-store/src/backend/azblob.rs index 6a70ef95d3..2ee88a2628 100644 --- a/src/object-store/src/backend/azblob.rs +++ b/src/object-store/src/backend/azblob.rs @@ -12,4 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub use opendal::services::azblob::{Backend, Builder}; +pub use opendal::services::azblob::Builder; diff --git a/src/object-store/src/backend/fs.rs b/src/object-store/src/backend/fs.rs index 78c03a24bb..4e46c52356 100644 --- a/src/object-store/src/backend/fs.rs +++ b/src/object-store/src/backend/fs.rs @@ -12,4 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub use opendal::services::fs::{Backend, Builder}; +pub use opendal::services::fs::Builder; diff --git a/src/object-store/src/backend/memory.rs b/src/object-store/src/backend/memory.rs index 7f9e2a4acd..5ec43d5c29 100644 --- a/src/object-store/src/backend/memory.rs +++ b/src/object-store/src/backend/memory.rs @@ -12,4 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub use opendal::services::memory::{Backend, Builder}; +pub use opendal::services::memory::Builder; diff --git a/src/object-store/src/backend/s3.rs b/src/object-store/src/backend/s3.rs index 918e84a6b8..d419967a56 100644 --- a/src/object-store/src/backend/s3.rs +++ b/src/object-store/src/backend/s3.rs @@ -12,4 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub use opendal::services::s3::{Backend, Builder}; +pub use opendal::services::s3::Builder; diff --git a/src/object-store/src/lib.rs b/src/object-store/src/lib.rs index 88702ebf3a..2be43fa5c7 100644 --- a/src/object-store/src/lib.rs +++ b/src/object-store/src/lib.rs @@ -14,8 +14,8 @@ pub use opendal::io_util::SeekableReader; pub use opendal::{ - layers, services, Accessor, DirEntry, DirStreamer, Layer, Object, ObjectMetadata, ObjectMode, - Operator as ObjectStore, + layers, services, Accessor, Layer, Object, ObjectEntry, ObjectMetadata, ObjectMode, + ObjectStreamer, Operator as ObjectStore, }; pub mod backend; pub mod util; diff --git a/src/object-store/src/util.rs b/src/object-store/src/util.rs index 05447f0594..01bb9e5360 100644 --- a/src/object-store/src/util.rs +++ b/src/object-store/src/util.rs @@ -14,9 +14,9 @@ use futures::TryStreamExt; -use crate::{DirEntry, DirStreamer}; +use crate::{ObjectEntry, ObjectStreamer}; -pub async fn collect(stream: DirStreamer) -> Result, std::io::Error> { +pub async fn collect(stream: ObjectStreamer) -> Result, std::io::Error> { stream.try_collect::>().await } diff --git a/src/object-store/tests/object_store_test.rs b/src/object-store/tests/object_store_test.rs index 01dd839106..27fa76262b 100644 --- a/src/object-store/tests/object_store_test.rs +++ b/src/object-store/tests/object_store_test.rs @@ -17,7 +17,7 @@ use std::env; use anyhow::Result; use common_telemetry::logging; use object_store::backend::{fs, s3}; -use object_store::{util, DirStreamer, Object, ObjectMode, ObjectStore}; +use object_store::{util, Object, ObjectMode, ObjectStore, ObjectStreamer}; use tempdir::TempDir; async fn test_object_crud(store: &ObjectStore) -> Result<()> { @@ -61,7 +61,7 @@ async fn test_object_list(store: &ObjectStore) -> Result<()> { // List objects let o: Object = store.object("/"); - let obs: DirStreamer = o.list().await?; + let obs: ObjectStreamer = o.list().await?; let objects = util::collect(obs).await?; assert_eq!(3, objects.len()); diff --git a/src/storage/src/manifest/storage.rs b/src/storage/src/manifest/storage.rs index 4edbd02a81..27c924e56e 100644 --- a/src/storage/src/manifest/storage.rs +++ b/src/storage/src/manifest/storage.rs @@ -19,7 +19,7 @@ use async_trait::async_trait; use common_telemetry::logging; use futures::TryStreamExt; use lazy_static::lazy_static; -use object_store::{util, DirEntry, ObjectStore}; +use object_store::{util, ObjectEntry, ObjectStore}; use regex::Regex; use serde::{Deserialize, Serialize}; use snafu::{ensure, ResultExt}; @@ -63,7 +63,7 @@ pub fn is_delta_file(file_name: &str) -> bool { } pub struct ObjectStoreLogIterator { - iter: Box + Send + Sync>, + iter: Box + Send + Sync>, } #[async_trait] @@ -156,7 +156,7 @@ impl ManifestLogStorage for ManifestObjectStore { .await .context(ListObjectsSnafu { path: &self.path })?; - let mut entries: Vec<(ManifestVersion, DirEntry)> = streamer + let mut entries: Vec<(ManifestVersion, ObjectEntry)> = streamer .try_filter_map(|e| async move { let file_name = e.name(); if is_delta_file(file_name) {