chore: expose disable_ec2_metadata option (#7439)

chore: add option for disable ec2 metadata

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

config/datanode.example.toml

@@ -225,6 +225,7 @@ overwrite_entry_start_id = false
 # endpoint = "https://s3.amazonaws.com"
 # region = "us-west-2"
 # enable_virtual_host_style = false
+# disable_ec2_metadata = false
 
 # Example of using Oss as the storage.
 # [storage]

config/standalone.example.toml

@@ -332,6 +332,7 @@ max_running_procedures = 128
 # endpoint = "https://s3.amazonaws.com"
 # region = "us-west-2"
 # enable_virtual_host_style = false
+# disable_ec2_metadata = false
 
 # Example of using Oss as the storage.
 # [storage]

src/cli/src/common/object_store.rs

@@ -255,6 +255,8 @@ wrap_with_clap_prefix! {
         region: Option<String>,
         #[doc = "Enable virtual host style for the object store."]
         enable_virtual_host_style: bool = Default::default(),
+        #[doc = "Disable EC2 metadata service for the object store."]
+        disable_ec2_metadata: bool = Default::default(),
     }
 }
 

src/common/datasource/src/object_store/s3.rs

@@ -27,6 +27,7 @@ const SECRET_ACCESS_KEY: &str = "secret_access_key";
 const SESSION_TOKEN: &str = "session_token";
 const REGION: &str = "region";
 const ENABLE_VIRTUAL_HOST_STYLE: &str = "enable_virtual_host_style";
+const DISABLE_EC2_METADATA: &str = "disable_ec2_metadata";
 
 pub fn is_supported_in_s3(key: &str) -> bool {
     [
@@ -36,6 +37,7 @@ pub fn is_supported_in_s3(key: &str) -> bool {
         SESSION_TOKEN,
         REGION,
         ENABLE_VIRTUAL_HOST_STYLE,
+        DISABLE_EC2_METADATA,
     ]
     .contains(&key)
 }
@@ -82,6 +84,21 @@ pub fn build_s3_backend(
         }
     }
 
+    if let Some(disable_str) = connection.get(DISABLE_EC2_METADATA) {
+        let disable = disable_str.as_str().parse::<bool>().map_err(|e| {
+            error::InvalidConnectionSnafu {
+                msg: format!(
+                    "failed to parse the option {}={}, {}",
+                    DISABLE_EC2_METADATA, disable_str, e
+                ),
+            }
+            .build()
+        })?;
+        if disable {
+            builder = builder.disable_ec2_metadata();
+        }
+    }
+
     // TODO(weny): Consider finding a better way to eliminate duplicate code.
     Ok(ObjectStore::new(builder)
         .context(error::BuildBackendSnafu)?
@@ -109,6 +126,7 @@ mod tests {
         assert!(is_supported_in_s3(SESSION_TOKEN));
         assert!(is_supported_in_s3(REGION));
         assert!(is_supported_in_s3(ENABLE_VIRTUAL_HOST_STYLE));
+        assert!(is_supported_in_s3(DISABLE_EC2_METADATA));
         assert!(!is_supported_in_s3("foo"))
     }
 }

src/object-store/src/config.rs

@@ -16,7 +16,6 @@ use std::time::Duration;
 
 use common_base::readable_size::ReadableSize;
 use common_base::secrets::{ExposeSecret, SecretString};
-use common_telemetry::tracing::warn;
 use opendal::services::{Azblob, Gcs, Oss, S3};
 use serde::{Deserialize, Serialize};
 
@@ -118,23 +117,25 @@ pub struct S3Connection {
     /// By default, opendal will send API to https://s3.us-east-1.amazonaws.com/bucket_name
     /// Enabled, opendal will send API to https://bucket_name.s3.us-east-1.amazonaws.com
     pub enable_virtual_host_style: bool,
+    /// Disable EC2 metadata service.
+    /// By default, opendal will use EC2 metadata service to load credentials from the instance metadata,
+    /// when access key id and secret access key are not provided.
+    /// If enabled, opendal will *NOT* use EC2 metadata service.
+    pub disable_ec2_metadata: bool,
 }
 
 impl From<&S3Connection> for S3 {
     fn from(connection: &S3Connection) -> Self {
         let root = util::normalize_dir(&connection.root);
 
-        let mut builder = S3::default().root(&root).bucket(&connection.bucket);
+        let mut builder = S3::default()
+            .root(&root)
+            .bucket(&connection.bucket)
+            .access_key_id(connection.access_key_id.expose_secret())
+            .secret_access_key(connection.secret_access_key.expose_secret());
 
-        if !connection.access_key_id.expose_secret().is_empty()
-            && !connection.secret_access_key.expose_secret().is_empty()
-        {
-            builder = builder
-                .access_key_id(connection.access_key_id.expose_secret())
-                .secret_access_key(connection.secret_access_key.expose_secret());
-        } else {
-            warn!("No access key id or secret access key provided, using anonymous access");
-            builder = builder.allow_anonymous().disable_ec2_metadata();
+        if connection.disable_ec2_metadata {
+            builder = builder.disable_ec2_metadata();
         }
 
         if let Some(endpoint) = &connection.endpoint {

tests-integration/tests/http.rs

@@ -1586,6 +1586,7 @@ fn drop_lines_with_inconsistent_results(input: String) -> String {
         "endpoint =",
         "region =",
         "enable_virtual_host_style =",
+        "disable_ec2_metadata =",
         "cache_path =",
         "cache_capacity =",
         "memory_pool_size =",