From 6b08a5f94e119cc0cd6aeeefd63b58f409dcbe45 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 31 May 2023 11:12:40 +0800 Subject: [PATCH] chore: Bump OpenDAL to v0.36 (#1678) * chore: Bump OpenDAL to v0.36 Signed-off-by: Xuanwo * Fix Signed-off-by: Xuanwo --------- Signed-off-by: Xuanwo --- Cargo.lock | 108 ++++++++++++----------- src/datanode/Cargo.toml | 1 - src/datanode/src/store.rs | 3 +- src/object-store/Cargo.toml | 2 +- src/object-store/src/layers/lru_cache.rs | 19 ++-- 5 files changed, 70 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fb97d4bf19..a2ea7730a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1938,7 +1938,7 @@ dependencies = [ "nom", "pathdiff", "ron", - "rust-ini", + "rust-ini 0.18.0", "serde", "serde_json", "toml", @@ -2574,7 +2574,6 @@ dependencies = [ "humantime-serde", "hyper", "key-lock", - "log", "log-store", "meta-client", "meta-srv", @@ -2628,17 +2627,6 @@ dependencies = [ "snafu", ] -[[package]] -name = "der" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" -dependencies = [ - "const-oid", - "pem-rfc7468", - "zeroize", -] - [[package]] name = "der" version = "0.7.4" @@ -2646,6 +2634,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b14af2045fa69ed2b7a48934bebb842d0f33e73e96e78766ecb14bb5347a11" dependencies = [ "const-oid", + "pem-rfc7468", "zeroize", ] @@ -2787,6 +2776,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" +[[package]] +name = "dlv-list" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d529fd73d344663edfd598ccb3f344e46034db51ebd103518eae34338248ad73" +dependencies = [ + "const-random", +] + [[package]] name = "dns-lookup" version = "1.0.8" @@ -5657,9 +5655,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "opendal" -version = "0.33.1" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c582b11500656b4f6210f868a0a26395cdf30183c5597c45d0a34525dd94512" +checksum = "3555168d4cc9a83c332e1416ff00e3be36a6d78447dff472829962afbc91bb3d" dependencies = [ "anyhow", "async-compat", @@ -5684,6 +5682,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2", "tokio", "tracing", "uuid", @@ -5808,10 +5807,20 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" dependencies = [ - "dlv-list", + "dlv-list 0.3.0", "hashbrown 0.12.3", ] +[[package]] +name = "ordered-multimap" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ed8acf08e98e744e5384c8bc63ceb0364e68a6854187221c18df61c4797690e" +dependencies = [ + "dlv-list 0.5.0", + "hashbrown 0.13.2", +] + [[package]] name = "os_str_bytes" version = "6.5.0" @@ -6002,9 +6011,9 @@ dependencies = [ [[package]] name = "pem-rfc7468" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d159833a9105500e0398934e205e0773f0b27529557134ecfc51c27646adac" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" dependencies = [ "base64ct", ] @@ -6185,24 +6194,23 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkcs1" -version = "0.4.1" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff33bdbdfc54cc98a2eca766ebdec3e1b8fb7387523d5c9c9a2891da856f719" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der 0.6.1", + "der", "pkcs8", - "spki 0.6.0", - "zeroize", + "spki", ] [[package]] name = "pkcs8" -version = "0.9.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.6.1", - "spki 0.6.0", + "der", + "spki", ] [[package]] @@ -7010,14 +7018,13 @@ dependencies = [ [[package]] name = "reqsign" -version = "0.9.4" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f753b768a9d691395e7db3dc74452cc39a5da55293c3fa2241e2aeaf9d94028" +checksum = "b04f5fccb94d61c154f0d8520ec42e79afdc145f4b1a392faa269874995fda66" dependencies = [ "anyhow", "async-trait", "base64 0.21.0", - "bytes", "chrono", "form_urlencoded", "hex", @@ -7032,7 +7039,7 @@ dependencies = [ "rand", "reqwest", "rsa", - "rust-ini", + "rust-ini 0.19.0", "serde", "serde_json", "sha1", @@ -7179,11 +7186,12 @@ dependencies = [ [[package]] name = "rsa" -version = "0.8.2" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55a77d189da1fee555ad95b7e50e7457d91c0e089ec68ca69ad2989413bbdab4" +checksum = "6ab43bb47d23c1a631b4b680199a45255dce26fa9ab2fa902581f624ff13e6a8" dependencies = [ "byteorder", + "const-oid", "digest", "num-bigint-dig", "num-integer", @@ -7192,8 +7200,8 @@ dependencies = [ "pkcs1", "pkcs8", "rand_core", - "sha2", "signature", + "spki", "subtle", "zeroize", ] @@ -7277,7 +7285,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" dependencies = [ "cfg-if 1.0.0", - "ordered-multimap", + "ordered-multimap 0.4.3", +] + +[[package]] +name = "rust-ini" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e2a3bcec1f113553ef1c88aae6c020a369d03d55b58de9869a0908930385091" +dependencies = [ + "cfg-if 1.0.0", + "ordered-multimap 0.6.0", ] [[package]] @@ -8465,22 +8483,12 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "spki" -version = "0.6.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" dependencies = [ "base64ct", - "der 0.6.1", -] - -[[package]] -name = "spki" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a5be806ab6f127c3da44b7378837ebf01dadca8510a0e572460216b228bd0e" -dependencies = [ - "base64ct", - "der 0.7.4", + "der", ] [[package]] @@ -10595,12 +10603,12 @@ dependencies = [ "bcder", "bytes", "chrono", - "der 0.7.4", + "der", "hex", "pem 1.1.1", "ring", "signature", - "spki 0.7.1", + "spki", "thiserror", ] diff --git a/src/datanode/Cargo.toml b/src/datanode/Cargo.toml index aff160835a..b31c89a8f6 100644 --- a/src/datanode/Cargo.toml +++ b/src/datanode/Cargo.toml @@ -37,7 +37,6 @@ futures-util.workspace = true key-lock = "0.1" hyper = { version = "0.14", features = ["full"] } humantime-serde = "1.1" -log = "0.4" log-store = { path = "../log-store" } meta-client = { path = "../meta-client" } meta-srv = { path = "../meta-srv", features = ["mock"] } diff --git a/src/datanode/src/store.rs b/src/datanode/src/store.rs index 99d3faa8fd..a245b70e54 100644 --- a/src/datanode/src/store.rs +++ b/src/datanode/src/store.rs @@ -56,7 +56,8 @@ pub(crate) async fn new_object_store(store_config: &ObjectStoreConfig) -> Result LoggingLayer::default() // Print the expected error only in DEBUG level. // See https://docs.rs/opendal/latest/opendal/layers/struct.LoggingLayer.html#method.with_error_level - .with_error_level(Some(log::Level::Debug)), + .with_error_level(Some("debug")) + .expect("input error level must be valid"), ) .layer(TracingLayer)) } diff --git a/src/object-store/Cargo.toml b/src/object-store/Cargo.toml index 47e44f1ae8..4fae116499 100644 --- a/src/object-store/Cargo.toml +++ b/src/object-store/Cargo.toml @@ -11,7 +11,7 @@ bytes = "1.4" futures = { version = "0.3" } md5 = "0.7" metrics.workspace = true -opendal = { version = "0.33", features = ["layers-tracing", "layers-metrics"] } +opendal = { version = "0.36", features = ["layers-tracing", "layers-metrics"] } pin-project = "1.0" tokio.workspace = true diff --git a/src/object-store/src/layers/lru_cache.rs b/src/object-store/src/layers/lru_cache.rs index 6fba7faf78..5a3b2bf89b 100644 --- a/src/object-store/src/layers/lru_cache.rs +++ b/src/object-store/src/layers/lru_cache.rs @@ -19,9 +19,11 @@ use std::sync::Arc; use async_trait::async_trait; use lru::LruCache; use metrics::increment_counter; -use opendal::ops::{OpDelete, OpList, OpRead, OpScan, OpWrite}; use opendal::raw::oio::{Page, Read, Reader, Write}; -use opendal::raw::{Accessor, Layer, LayeredAccessor, RpDelete, RpList, RpRead, RpScan, RpWrite}; +use opendal::raw::{ + Accessor, Layer, LayeredAccessor, OpAppend, OpDelete, OpList, OpRead, OpWrite, RpAppend, + RpDelete, RpList, RpRead, RpWrite, +}; use opendal::{ErrorKind, Result}; use tokio::sync::Mutex; @@ -108,6 +110,7 @@ impl LayeredAccessor for LruCacheAccessor { type BlockingWriter = I::BlockingWriter; type Pager = I::Pager; type BlockingPager = I::BlockingPager; + type Appender = I::Appender; fn inner(&self) -> &Self::Inner { &self.inner @@ -167,6 +170,10 @@ impl LayeredAccessor for LruCacheAccessor { self.inner.write(path, args).await } + async fn append(&self, path: &str, args: OpAppend) -> Result<(RpAppend, Self::Appender)> { + self.inner.append(path, args).await + } + async fn delete(&self, path: &str, args: OpDelete) -> Result { let cache_path = md5::compute(path); let lru_cache = &self.lru_cache; @@ -194,10 +201,6 @@ impl LayeredAccessor for LruCacheAccessor { self.inner.list(path, args).await } - async fn scan(&self, path: &str, args: OpScan) -> Result<(RpScan, Self::Pager)> { - self.inner.scan(path, args).await - } - fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> { self.inner.blocking_read(path, args) } @@ -209,10 +212,6 @@ impl LayeredAccessor for LruCacheAccessor { fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingPager)> { self.inner.blocking_list(path, args) } - - fn blocking_scan(&self, path: &str, args: OpScan) -> Result<(RpScan, Self::BlockingPager)> { - self.inner.blocking_scan(path, args) - } } #[inline]