feat(storage): Add skip_ssl_validation option for object storage HTTP client (#6358)

* feat(storage): Add skip_ssl_validation option for object storage HTTP client

Signed-off-by: rgidda <rgidda@hitachivantara.com>

* fix(test): Broken test case for - Add skip_ssl_validation option for object storage HTTP client

Signed-off-by: rgidda <rgidda@hitachivantara.com>

* fix: test

* fix: test

---------

Signed-off-by: rgidda <rgidda@hitachivantara.com>
Co-authored-by: rgidda <rgidda@hitachivantara.com>
Co-authored-by: dennis zhuang <killme2008@gmail.com>

config/config.md

@@ -123,6 +123,7 @@
 | `storage.http_client.connect_timeout` | String | `30s` | The timeout for only the connect phase of a http client. |
 | `storage.http_client.timeout` | String | `30s` | The total request timeout, applied from when the request starts connecting until the response body has finished.<br/>Also considered a total deadline. |
 | `storage.http_client.pool_idle_timeout` | String | `90s` | The timeout for idle sockets being kept-alive. |
+| `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification<br/>**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. |
 | `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines. |
 | `region_engine.mito` | -- | -- | The Mito engine options. |
 | `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. |
@@ -471,6 +472,7 @@
 | `storage.http_client.connect_timeout` | String | `30s` | The timeout for only the connect phase of a http client. |
 | `storage.http_client.timeout` | String | `30s` | The total request timeout, applied from when the request starts connecting until the response body has finished.<br/>Also considered a total deadline. |
 | `storage.http_client.pool_idle_timeout` | String | `90s` | The timeout for idle sockets being kept-alive. |
+| `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification<br/>**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. |
 | `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines. |
 | `region_engine.mito` | -- | -- | The Mito engine options. |
 | `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. |

config/datanode.example.toml

@@ -367,6 +367,10 @@ timeout = "30s"
 ## The timeout for idle sockets being kept-alive.
 pool_idle_timeout = "90s"
 
+## To skip the ssl verification
+## **Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks.
+skip_ssl_validation = false
+
 # Custom storage options
 # [[storage.providers]]
 # name = "S3"

config/standalone.example.toml

@@ -458,6 +458,10 @@ timeout = "30s"
 ## The timeout for idle sockets being kept-alive.
 pool_idle_timeout = "90s"
 
+## To skip the ssl verification
+## **Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks.
+skip_ssl_validation = false
+
 # Custom storage options
 # [[storage.providers]]
 # name = "S3"

src/datanode/src/config.rs

@@ -144,6 +144,9 @@ pub struct HttpClientConfig {
     /// The timeout for idle sockets being kept-alive.
     #[serde(with = "humantime_serde")]
     pub(crate) pool_idle_timeout: Duration,
+
+    /// Skip SSL certificate validation (insecure)
+    pub skip_ssl_validation: bool,
 }
 
 impl Default for HttpClientConfig {
@@ -153,6 +156,7 @@ impl Default for HttpClientConfig {
             connect_timeout: Duration::from_secs(30),
             timeout: Duration::from_secs(30),
             pool_idle_timeout: Duration::from_secs(90),
+            skip_ssl_validation: false,
         }
     }
 }
@@ -514,4 +518,48 @@ mod tests {
             _ => unreachable!(),
         }
     }
+    #[test]
+    fn test_skip_ssl_validation_config() {
+        // Test with skip_ssl_validation = true
+        let toml_str_true = r#"
+            [storage]
+            type = "S3"
+            [storage.http_client]
+            skip_ssl_validation = true
+        "#;
+        let opts: DatanodeOptions = toml::from_str(toml_str_true).unwrap();
+        match &opts.storage.store {
+            ObjectStoreConfig::S3(cfg) => {
+                assert!(cfg.http_client.skip_ssl_validation);
+            }
+            _ => panic!("Expected S3 config"),
+        }
+
+        // Test with skip_ssl_validation = false
+        let toml_str_false = r#"
+            [storage]
+            type = "S3"
+            [storage.http_client]
+            skip_ssl_validation = false
+        "#;
+        let opts: DatanodeOptions = toml::from_str(toml_str_false).unwrap();
+        match &opts.storage.store {
+            ObjectStoreConfig::S3(cfg) => {
+                assert!(!cfg.http_client.skip_ssl_validation);
+            }
+            _ => panic!("Expected S3 config"),
+        }
+        // Test default value (should be false)
+        let toml_str_default = r#"
+            [storage]
+            type = "S3"
+        "#;
+        let opts: DatanodeOptions = toml::from_str(toml_str_default).unwrap();
+        match &opts.storage.store {
+            ObjectStoreConfig::S3(cfg) => {
+                assert!(!cfg.http_client.skip_ssl_validation);
+            }
+            _ => panic!("Expected S3 config"),
+        }
+    }
 }

src/datanode/src/store.rs

@@ -212,6 +212,10 @@ pub(crate) fn build_http_client(config: &HttpClientConfig) -> Result<HttpClient>
         .connect_timeout(config.connect_timeout)
         .pool_idle_timeout(config.pool_idle_timeout)
         .timeout(config.timeout)
+        .danger_accept_invalid_certs({
+            info!("skip_ssl_validation: {}", config.skip_ssl_validation);
+            config.skip_ssl_validation
+        })
         .build()
         .context(BuildHttpClientSnafu)?;
     Ok(HttpClient::with(client))

tests-integration/tests/http.rs

@@ -991,7 +991,8 @@ providers = []
 pool_max_idle_per_host = 1024
 connect_timeout = "30s"
 timeout = "30s"
-pool_idle_timeout = "1m 30s""#,
+pool_idle_timeout = "1m 30s"
+skip_ssl_validation = false"#,
             store_type
         )
     } else {