From e026f766d2d6a84730425ac1dabbaa3904e96971 Mon Sep 17 00:00:00 2001 From: rgidda Date: Fri, 20 Jun 2025 13:38:19 +0530 Subject: [PATCH] feat(storage): Add skip_ssl_validation option for object storage HTTP client (#6358) * feat(storage): Add skip_ssl_validation option for object storage HTTP client Signed-off-by: rgidda * fix(test): Broken test case for - Add skip_ssl_validation option for object storage HTTP client Signed-off-by: rgidda * fix: test * fix: test --------- Signed-off-by: rgidda Co-authored-by: rgidda Co-authored-by: dennis zhuang --- config/config.md | 2 ++ config/datanode.example.toml | 4 +++ config/standalone.example.toml | 4 +++ src/datanode/src/config.rs | 48 +++++++++++++++++++++++++++++++++ src/datanode/src/store.rs | 4 +++ tests-integration/tests/http.rs | 3 ++- 6 files changed, 64 insertions(+), 1 deletion(-) diff --git a/config/config.md b/config/config.md index 11b5579949..37cd5d1bac 100644 --- a/config/config.md +++ b/config/config.md @@ -123,6 +123,7 @@ | `storage.http_client.connect_timeout` | String | `30s` | The timeout for only the connect phase of a http client. | | `storage.http_client.timeout` | String | `30s` | The total request timeout, applied from when the request starts connecting until the response body has finished.
Also considered a total deadline. | | `storage.http_client.pool_idle_timeout` | String | `90s` | The timeout for idle sockets being kept-alive. | +| `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification
**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. | | `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines. | | `region_engine.mito` | -- | -- | The Mito engine options. | | `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. | @@ -471,6 +472,7 @@ | `storage.http_client.connect_timeout` | String | `30s` | The timeout for only the connect phase of a http client. | | `storage.http_client.timeout` | String | `30s` | The total request timeout, applied from when the request starts connecting until the response body has finished.
Also considered a total deadline. | | `storage.http_client.pool_idle_timeout` | String | `90s` | The timeout for idle sockets being kept-alive. | +| `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification
**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. | | `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines. | | `region_engine.mito` | -- | -- | The Mito engine options. | | `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 507858383a..ac9e9487f5 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -367,6 +367,10 @@ timeout = "30s" ## The timeout for idle sockets being kept-alive. pool_idle_timeout = "90s" +## To skip the ssl verification +## **Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. +skip_ssl_validation = false + # Custom storage options # [[storage.providers]] # name = "S3" diff --git a/config/standalone.example.toml b/config/standalone.example.toml index d31b70412b..269878b61a 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -458,6 +458,10 @@ timeout = "30s" ## The timeout for idle sockets being kept-alive. pool_idle_timeout = "90s" +## To skip the ssl verification +## **Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. +skip_ssl_validation = false + # Custom storage options # [[storage.providers]] # name = "S3" diff --git a/src/datanode/src/config.rs b/src/datanode/src/config.rs index d53a4d56f6..59709c1285 100644 --- a/src/datanode/src/config.rs +++ b/src/datanode/src/config.rs @@ -144,6 +144,9 @@ pub struct HttpClientConfig { /// The timeout for idle sockets being kept-alive. #[serde(with = "humantime_serde")] pub(crate) pool_idle_timeout: Duration, + + /// Skip SSL certificate validation (insecure) + pub skip_ssl_validation: bool, } impl Default for HttpClientConfig { @@ -153,6 +156,7 @@ impl Default for HttpClientConfig { connect_timeout: Duration::from_secs(30), timeout: Duration::from_secs(30), pool_idle_timeout: Duration::from_secs(90), + skip_ssl_validation: false, } } } @@ -514,4 +518,48 @@ mod tests { _ => unreachable!(), } } + #[test] + fn test_skip_ssl_validation_config() { + // Test with skip_ssl_validation = true + let toml_str_true = r#" + [storage] + type = "S3" + [storage.http_client] + skip_ssl_validation = true + "#; + let opts: DatanodeOptions = toml::from_str(toml_str_true).unwrap(); + match &opts.storage.store { + ObjectStoreConfig::S3(cfg) => { + assert!(cfg.http_client.skip_ssl_validation); + } + _ => panic!("Expected S3 config"), + } + + // Test with skip_ssl_validation = false + let toml_str_false = r#" + [storage] + type = "S3" + [storage.http_client] + skip_ssl_validation = false + "#; + let opts: DatanodeOptions = toml::from_str(toml_str_false).unwrap(); + match &opts.storage.store { + ObjectStoreConfig::S3(cfg) => { + assert!(!cfg.http_client.skip_ssl_validation); + } + _ => panic!("Expected S3 config"), + } + // Test default value (should be false) + let toml_str_default = r#" + [storage] + type = "S3" + "#; + let opts: DatanodeOptions = toml::from_str(toml_str_default).unwrap(); + match &opts.storage.store { + ObjectStoreConfig::S3(cfg) => { + assert!(!cfg.http_client.skip_ssl_validation); + } + _ => panic!("Expected S3 config"), + } + } } diff --git a/src/datanode/src/store.rs b/src/datanode/src/store.rs index 6b9d85f239..3a2f626141 100644 --- a/src/datanode/src/store.rs +++ b/src/datanode/src/store.rs @@ -212,6 +212,10 @@ pub(crate) fn build_http_client(config: &HttpClientConfig) -> Result .connect_timeout(config.connect_timeout) .pool_idle_timeout(config.pool_idle_timeout) .timeout(config.timeout) + .danger_accept_invalid_certs({ + info!("skip_ssl_validation: {}", config.skip_ssl_validation); + config.skip_ssl_validation + }) .build() .context(BuildHttpClientSnafu)?; Ok(HttpClient::with(client)) diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 91c3afd22f..ad241f85a2 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -991,7 +991,8 @@ providers = [] pool_max_idle_per_host = 1024 connect_timeout = "30s" timeout = "30s" -pool_idle_timeout = "1m 30s""#, +pool_idle_timeout = "1m 30s" +skip_ssl_validation = false"#, store_type ) } else {