From c088d361a4df2b08d8d464dcda7c7a0e0956eded Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Thu, 18 Dec 2025 19:55:08 +0800 Subject: [PATCH] chore: expose `disable_ec2_metadata` option (#7439) chore: add option for disable ec2 metadata Signed-off-by: shuiyisong --- config/datanode.example.toml | 1 + config/standalone.example.toml | 1 + src/cli/src/common/object_store.rs | 2 ++ src/common/datasource/src/object_store/s3.rs | 18 +++++++++++++++ src/object-store/src/config.rs | 23 ++++++++++---------- tests-integration/tests/http.rs | 1 + 6 files changed, 35 insertions(+), 11 deletions(-) diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 47b6cc8cec..0a1358a6e7 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -225,6 +225,7 @@ overwrite_entry_start_id = false # endpoint = "https://s3.amazonaws.com" # region = "us-west-2" # enable_virtual_host_style = false +# disable_ec2_metadata = false # Example of using Oss as the storage. # [storage] diff --git a/config/standalone.example.toml b/config/standalone.example.toml index 47be8f6334..ec5b6301ff 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -332,6 +332,7 @@ max_running_procedures = 128 # endpoint = "https://s3.amazonaws.com" # region = "us-west-2" # enable_virtual_host_style = false +# disable_ec2_metadata = false # Example of using Oss as the storage. # [storage] diff --git a/src/cli/src/common/object_store.rs b/src/cli/src/common/object_store.rs index bfe9662b8d..7a61784251 100644 --- a/src/cli/src/common/object_store.rs +++ b/src/cli/src/common/object_store.rs @@ -255,6 +255,8 @@ wrap_with_clap_prefix! { region: Option, #[doc = "Enable virtual host style for the object store."] enable_virtual_host_style: bool = Default::default(), + #[doc = "Disable EC2 metadata service for the object store."] + disable_ec2_metadata: bool = Default::default(), } } diff --git a/src/common/datasource/src/object_store/s3.rs b/src/common/datasource/src/object_store/s3.rs index 356b6fcf75..36762ff0ac 100644 --- a/src/common/datasource/src/object_store/s3.rs +++ b/src/common/datasource/src/object_store/s3.rs @@ -27,6 +27,7 @@ const SECRET_ACCESS_KEY: &str = "secret_access_key"; const SESSION_TOKEN: &str = "session_token"; const REGION: &str = "region"; const ENABLE_VIRTUAL_HOST_STYLE: &str = "enable_virtual_host_style"; +const DISABLE_EC2_METADATA: &str = "disable_ec2_metadata"; pub fn is_supported_in_s3(key: &str) -> bool { [ @@ -36,6 +37,7 @@ pub fn is_supported_in_s3(key: &str) -> bool { SESSION_TOKEN, REGION, ENABLE_VIRTUAL_HOST_STYLE, + DISABLE_EC2_METADATA, ] .contains(&key) } @@ -82,6 +84,21 @@ pub fn build_s3_backend( } } + if let Some(disable_str) = connection.get(DISABLE_EC2_METADATA) { + let disable = disable_str.as_str().parse::().map_err(|e| { + error::InvalidConnectionSnafu { + msg: format!( + "failed to parse the option {}={}, {}", + DISABLE_EC2_METADATA, disable_str, e + ), + } + .build() + })?; + if disable { + builder = builder.disable_ec2_metadata(); + } + } + // TODO(weny): Consider finding a better way to eliminate duplicate code. Ok(ObjectStore::new(builder) .context(error::BuildBackendSnafu)? @@ -109,6 +126,7 @@ mod tests { assert!(is_supported_in_s3(SESSION_TOKEN)); assert!(is_supported_in_s3(REGION)); assert!(is_supported_in_s3(ENABLE_VIRTUAL_HOST_STYLE)); + assert!(is_supported_in_s3(DISABLE_EC2_METADATA)); assert!(!is_supported_in_s3("foo")) } } diff --git a/src/object-store/src/config.rs b/src/object-store/src/config.rs index ac976da8f0..5b408a9b34 100644 --- a/src/object-store/src/config.rs +++ b/src/object-store/src/config.rs @@ -16,7 +16,6 @@ use std::time::Duration; use common_base::readable_size::ReadableSize; use common_base::secrets::{ExposeSecret, SecretString}; -use common_telemetry::tracing::warn; use opendal::services::{Azblob, Gcs, Oss, S3}; use serde::{Deserialize, Serialize}; @@ -118,23 +117,25 @@ pub struct S3Connection { /// By default, opendal will send API to https://s3.us-east-1.amazonaws.com/bucket_name /// Enabled, opendal will send API to https://bucket_name.s3.us-east-1.amazonaws.com pub enable_virtual_host_style: bool, + /// Disable EC2 metadata service. + /// By default, opendal will use EC2 metadata service to load credentials from the instance metadata, + /// when access key id and secret access key are not provided. + /// If enabled, opendal will *NOT* use EC2 metadata service. + pub disable_ec2_metadata: bool, } impl From<&S3Connection> for S3 { fn from(connection: &S3Connection) -> Self { let root = util::normalize_dir(&connection.root); - let mut builder = S3::default().root(&root).bucket(&connection.bucket); + let mut builder = S3::default() + .root(&root) + .bucket(&connection.bucket) + .access_key_id(connection.access_key_id.expose_secret()) + .secret_access_key(connection.secret_access_key.expose_secret()); - if !connection.access_key_id.expose_secret().is_empty() - && !connection.secret_access_key.expose_secret().is_empty() - { - builder = builder - .access_key_id(connection.access_key_id.expose_secret()) - .secret_access_key(connection.secret_access_key.expose_secret()); - } else { - warn!("No access key id or secret access key provided, using anonymous access"); - builder = builder.allow_anonymous().disable_ec2_metadata(); + if connection.disable_ec2_metadata { + builder = builder.disable_ec2_metadata(); } if let Some(endpoint) = &connection.endpoint { diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 912bb303b9..902373ebb8 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -1586,6 +1586,7 @@ fn drop_lines_with_inconsistent_results(input: String) -> String { "endpoint =", "region =", "enable_virtual_host_style =", + "disable_ec2_metadata =", "cache_path =", "cache_capacity =", "memory_pool_size =",