chore: update opendal to 0.57 (#8204)

* chore: update opendal to 0.57 Signed-off-by: shuiyisong <xixing.sys@gmail.com> * chore: remove duplicate compat to use opendal's Signed-off-by: shuiyisong <xixing.sys@gmail.com> --------- Signed-off-by: shuiyisong <xixing.sys@gmail.com>
2026-06-02 05:10:40 +00:00 · 2026-06-01 15:45:44 +08:00
parent 33ffa41ac2
commit a796dbf1a0
13 changed files with 98 additions and 1105 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2278,6 +2278,7 @@ dependencies = [
 "futures",
 "lazy_static",
 "object-store",
+ "object_store_opendal",
 "orc-rust",
 "parquet",
 "paste",
@@ -5102,6 +5103,7 @@ dependencies = [
 "datatypes",
 "futures",
 "object-store",
+ "object_store_opendal",
 "serde",
 "serde_json",
 "snafu 0.8.6",
@@ -9074,8 +9076,9 @@ dependencies = [

 [[package]]
 name = "object_store_opendal"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8"
 dependencies = [
 "async-trait",
 "bytes",
@@ -9162,8 +9165,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"

 [[package]]
 name = "opendal"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1"
 dependencies = [
 "ctor",
 "opendal-core",
@@ -9183,8 +9187,9 @@ dependencies = [

 [[package]]
 name = "opendal-core"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309"
 dependencies = [
 "anyhow",
 "base64 0.22.1",
@@ -9210,8 +9215,9 @@ dependencies = [

 [[package]]
 name = "opendal-layer-concurrent-limit"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f"
 dependencies = [
 "futures",
 "http 1.3.1",
@@ -9221,8 +9227,9 @@ dependencies = [

 [[package]]
 name = "opendal-layer-logging"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2"
 dependencies = [
 "log",
 "opendal-core",
@@ -9230,8 +9237,9 @@ dependencies = [

 [[package]]
 name = "opendal-layer-observe-metrics-common"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "628b0228fdbd13c3d9d50eee4341f2eb82ca5b44991e4c68f07c84cc823e2d12"
 dependencies = [
 "futures",
 "http 1.3.1",
@@ -9240,8 +9248,9 @@ dependencies = [

 [[package]]
 name = "opendal-layer-prometheus"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0487bdb1357097ec8654781bad03ef310282517738e2864ebde69e27aaafc5ec"
 dependencies = [
 "opendal-core",
 "opendal-layer-observe-metrics-common",
@@ -9250,8 +9259,9 @@ dependencies = [

 [[package]]
 name = "opendal-layer-retry"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7"
 dependencies = [
 "backon",
 "log",
@@ -9260,8 +9270,9 @@ dependencies = [

 [[package]]
 name = "opendal-layer-timeout"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3"
 dependencies = [
 "opendal-core",
 "tokio",
@@ -9269,8 +9280,9 @@ dependencies = [

 [[package]]
 name = "opendal-layer-tracing"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90c6fc9df6da1f0dafbdf55fa48525f1643aefbe7da8f46936e869e2a5b8a34f"
 dependencies = [
 "futures",
 "http 1.3.1",
@@ -9280,8 +9292,9 @@ dependencies = [

 [[package]]
 name = "opendal-service-azblob"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a"
 dependencies = [
 "base64 0.22.1",
 "bytes",
@@ -9300,8 +9313,9 @@ dependencies = [

 [[package]]
 name = "opendal-service-azure-common"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051"
 dependencies = [
 "http 1.3.1",
 "opendal-core",
@@ -9309,8 +9323,9 @@ dependencies = [

 [[package]]
 name = "opendal-service-fs"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e89a665fef0e6bd249cf5ea47fc174b7ba892159bee4b9382528b1ca873a2c"
 dependencies = [
 "bytes",
 "log",
@@ -9322,8 +9337,9 @@ dependencies = [

 [[package]]
 name = "opendal-service-gcs"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47"
 dependencies = [
 "async-trait",
 "bytes",
@@ -9342,8 +9358,9 @@ dependencies = [

 [[package]]
 name = "opendal-service-http"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb6af628a0bf14075b957179444927e1df40dc7addef382b585a05ef015a077b"
 dependencies = [
 "http 1.3.1",
 "log",
@@ -9353,8 +9370,9 @@ dependencies = [

 [[package]]
 name = "opendal-service-oss"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace"
 dependencies = [
 "bytes",
 "http 1.3.1",
@@ -9369,8 +9387,9 @@ dependencies = [

 [[package]]
 name = "opendal-service-s3"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb"
 dependencies = [
 "base64 0.22.1",
 "bytes",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -178,7 +178,7 @@ nalgebra = "0.33"
 nix = { version = "0.30.1", default-features = false, features = ["event", "fs", "process"] }
 notify = "8.0"
 num_cpus = "1.16"
-object_store_opendal = { git = "https://github.com/apache/opendal.git", rev = "4ad2d85296ffa6fdc2882f97d3c760ee243913f7" }
+object_store_opendal = "0.57"
 once_cell = "1.18"
 opentelemetry-proto = { version = "0.31", features = [
    "gen-tonic",
--- a/src/common/datasource/Cargo.toml
+++ b/src/common/datasource/Cargo.toml
@@ -33,6 +33,7 @@ datatypes.workspace = true
 futures.workspace = true
 lazy_static.workspace = true
 object-store.workspace = true
+object_store_opendal.workspace = true
 orc-rust = { version = "0.8", default-features = false, features = ["async"] }
 parquet.workspace = true
 paste.workspace = true
--- a/src/common/datasource/src/file_format.rs
+++ b/src/common/datasource/src/file_format.rs
@@ -316,7 +316,7 @@ pub async fn file_to_stream(
            .with_file_compression_type(df_compression)
            .build();

-    let store = Arc::new(object_store::compat::OpendalStore::new(store.clone()));
+    let store = Arc::new(object_store_opendal::OpendalStore::new(store.clone()));
    let file_opener = config.file_source().create_file_opener(store, &config, 0)?;
    let stream = FileStream::new(&config, 0, file_opener, &ExecutionPlanMetricsSet::new())?;

--- a/src/common/datasource/src/file_format/tests.rs
+++ b/src/common/datasource/src/file_format/tests.rs
@@ -44,7 +44,7 @@ struct Test<'a> {

 impl Test<'_> {
    async fn run(self, store: &ObjectStore) {
-        let store = Arc::new(object_store::compat::OpendalStore::new(store.clone()));
+        let store = Arc::new(object_store_opendal::OpendalStore::new(store.clone()));
        let file_opener = self
            .file_source
            .create_file_opener(store, &self.config, 0)
--- a/src/common/datasource/src/object_store/oss.rs
+++ b/src/common/datasource/src/object_store/oss.rs
@@ -27,12 +27,14 @@ const ACCESS_KEY_ID: &str = "access_key_id";
 const ACCESS_KEY_SECRET: &str = "access_key_secret";
 const ROOT: &str = "root";
 const ALLOW_ANONYMOUS: &str = "allow_anonymous";
+const SKIP_SIGNATURE: &str = "skip_signature";

 /// Check if the key is supported in OSS configuration.
 pub fn is_supported_in_oss(key: &str) -> bool {
    [
        ROOT,
        ALLOW_ANONYMOUS,
+        SKIP_SIGNATURE,
        BUCKET,
        ENDPOINT,
        ACCESS_KEY_ID,
@@ -61,18 +63,23 @@ pub fn build_oss_backend(
        builder = builder.access_key_secret(access_key_secret);
    }

-    if let Some(allow_anonymous) = connection.get(ALLOW_ANONYMOUS) {
-        let allow = allow_anonymous.as_str().parse::<bool>().map_err(|e| {
+    if let Some((key, value)) = connection
+        .get(SKIP_SIGNATURE)
+        .map(|value| (SKIP_SIGNATURE, value))
+        .or_else(|| {
+            connection
+                .get(ALLOW_ANONYMOUS)
+                .map(|value| (ALLOW_ANONYMOUS, value))
+        })
+    {
+        let skip_signature = value.as_str().parse::<bool>().map_err(|e| {
            error::InvalidConnectionSnafu {
-                msg: format!(
-                    "failed to parse the option {}={}, {}",
-                    ALLOW_ANONYMOUS, allow_anonymous, e
-                ),
+                msg: format!("failed to parse the option {}={}, {}", key, value, e),
            }
            .build()
        })?;
-        if allow {
-            builder = builder.allow_anonymous();
+        if skip_signature {
+            builder = builder.skip_signature();
        }
    }

@@ -93,6 +100,7 @@ mod tests {
    fn test_is_supported_in_oss() {
        assert!(is_supported_in_oss(ROOT));
        assert!(is_supported_in_oss(ALLOW_ANONYMOUS));
+        assert!(is_supported_in_oss(SKIP_SIGNATURE));
        assert!(is_supported_in_oss(BUCKET));
        assert!(is_supported_in_oss(ENDPOINT));
        assert!(is_supported_in_oss(ACCESS_KEY_ID));
--- a/src/common/datasource/src/test_util.rs
+++ b/src/common/datasource/src/test_util.rs
@@ -103,7 +103,7 @@ pub async fn setup_stream_to_json_test(origin_path: &str, threshold: impl Fn(usi
        test_util::TEST_BATCH_SIZE,
        schema.clone(),
        FileCompressionType::UNCOMPRESSED,
-        Arc::new(object_store::compat::OpendalStore::new(store.clone())),
+        Arc::new(object_store_opendal::OpendalStore::new(store.clone())),
        true,
    );

@@ -157,7 +157,7 @@ pub async fn setup_stream_to_csv_test(

    let csv_opener = csv_source
        .create_file_opener(
-            Arc::new(object_store::compat::OpendalStore::new(store.clone())),
+            Arc::new(object_store_opendal::OpendalStore::new(store.clone())),
            &config,
            0,
        )
--- a/src/file-engine/Cargo.toml
+++ b/src/file-engine/Cargo.toml
@@ -29,6 +29,7 @@ datafusion-expr.workspace = true
 datatypes.workspace = true
 futures.workspace = true
 object-store.workspace = true
+object_store_opendal.workspace = true
 serde = { version = "1.0", features = ["derive"] }
 serde_json.workspace = true
 snafu.workspace = true
--- a/src/file-engine/src/query/file_stream.rs
+++ b/src/file-engine/src/query/file_stream.rs
@@ -61,7 +61,7 @@ fn build_record_batch_stream(
            .with_file_group(FileGroup::new(files))
            .build();

-    let store = Arc::new(object_store::compat::OpendalStore::new(
+    let store = Arc::new(object_store_opendal::OpendalStore::new(
        scan_plan_config.store.clone(),
    ));

--- a/src/object-store/Cargo.toml
+++ b/src/object-store/Cargo.toml
@@ -24,7 +24,7 @@ derive_builder = { workspace = true, optional = true }
 futures.workspace = true
 humantime-serde.workspace = true
 lazy_static.workspace = true
-opendal = { git = "https://github.com/apache/opendal.git", rev = "4ad2d85296ffa6fdc2882f97d3c760ee243913f7", features = [
+opendal = { version = "0.57", features = [
    "layers-tracing",
    "layers-prometheus",
    "services-azblob",
--- a/src/object-store/src/compat.rs
+++ b/src/object-store/src/compat.rs
--- a/src/object-store/src/layers/mock.rs
+++ b/src/object-store/src/layers/mock.rs
@@ -21,7 +21,7 @@ pub use opendal::raw::{
    Access, Layer, LayeredAccess, OpDelete, OpList, OpRead, OpWrite, RpDelete, RpList, RpRead,
    RpWrite, oio,
 };
-use opendal::raw::{OpCopy, RpCopy};
+use opendal::raw::{OpCopier, OpCopy, RpCopy};
 pub use opendal::{Buffer, Error, ErrorKind, Metadata, Result};

 pub type MockWriterFactory = Arc<dyn Fn(&str, OpWrite, oio::Writer) -> oio::Writer + Send + Sync>;
@@ -146,6 +146,7 @@ impl<A: Access> LayeredAccess for MockAccessor<A> {
    type Writer = MockWriter;
    type Lister = MockLister;
    type Deleter = MockDeleter;
+    type Copier = oio::Copier;

    fn inner(&self) -> &Self::Inner {
        &self.inner
@@ -222,15 +223,24 @@ impl<A: Access> LayeredAccess for MockAccessor<A> {
        }
    }

-    async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
-        let Some(copy_interceptor) = self.copy_interceptor.as_ref() else {
-            return self.inner.copy(from, to, args).await;
-        };
+    async fn copy(
+        &self,
+        from: &str,
+        to: &str,
+        args: OpCopy,
+        opts: OpCopier,
+    ) -> Result<(RpCopy, Self::Copier)> {
+        if let Some(result) = self
+            .copy_interceptor
+            .as_ref()
+            .and_then(|copy_interceptor| copy_interceptor(from, to, args.clone()))
+        {
+            return result.map(|rp_copy| (rp_copy, Box::new(()) as oio::Copier));
+        }

-        let Some(result) = copy_interceptor(from, to, args.clone()) else {
-            return self.inner.copy(from, to, args).await;
-        };
-
-        result
+        self.inner
+            .copy(from, to, args, opts)
+            .await
+            .map(|(rp_copy, copier)| (rp_copy, Box::new(copier) as oio::Copier))
    }
 }
--- a/src/object-store/src/lib.rs
+++ b/src/object-store/src/lib.rs
@@ -18,7 +18,6 @@ pub use opendal::{
    FuturesAsyncWriter, Lister, Operator as ObjectStore, Reader, Result, Writer, services,
 };

-pub mod compat;
 pub mod config;
 pub mod error;
 pub mod factory;