From 77628738423d5f14bd21c48097301cbb2b7700b1 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Wed, 4 Jan 2023 11:24:24 +0800 Subject: [PATCH] feat: endpoint and region config for s3 storage (#820) * feat: adds serde default attribute to options * feat: adds endpoint and region for s3 config --- src/cmd/src/standalone.rs | 1 + src/cmd/src/toml_loader.rs | 26 +++++++++++++++++++++++ src/datanode/src/datanode.rs | 3 +++ src/datanode/src/instance.rs | 33 ++++++++++++++++++++++-------- src/frontend/src/frontend.rs | 1 + src/meta-srv/src/metasrv.rs | 1 + tests-integration/src/test_util.rs | 2 ++ 7 files changed, 59 insertions(+), 8 deletions(-) diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index f8c17a2cba..bc16e3ea0a 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -63,6 +63,7 @@ impl SubCommand { } #[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(default)] pub struct StandaloneOptions { pub http_options: Option, pub grpc_options: Option, diff --git a/src/cmd/src/toml_loader.rs b/src/cmd/src/toml_loader.rs index 011171104a..2f1ba739a5 100644 --- a/src/cmd/src/toml_loader.rs +++ b/src/cmd/src/toml_loader.rs @@ -37,12 +37,23 @@ mod tests { use crate::error::Result; #[derive(Clone, PartialEq, Debug, Deserialize, Serialize)] + #[serde(default)] struct MockConfig { path: String, port: u32, host: String, } + impl Default for MockConfig { + fn default() -> Self { + Self { + path: "test".to_string(), + port: 0, + host: "localhost".to_string(), + } + } + } + #[test] fn test_from_file() -> Result<()> { let config = MockConfig { @@ -63,6 +74,21 @@ mod tests { let loaded_config: MockConfig = from_file!(&test_file)?; assert_eq!(loaded_config, config); + // Only host in file + let mut file = File::create(&test_file).unwrap(); + file.write_all("host='greptime.test'\n".as_bytes()).unwrap(); + + let loaded_config: MockConfig = from_file!(&test_file)?; + assert_eq!(loaded_config.host, "greptime.test"); + assert_eq!(loaded_config.port, 0); + assert_eq!(loaded_config.path, "test"); + + // Truncate the file. + let file = File::create(&test_file).unwrap(); + file.set_len(0).unwrap(); + let loaded_config: MockConfig = from_file!(&test_file)?; + assert_eq!(loaded_config, MockConfig::default()); + Ok(()) } } diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs index b5f15592da..cd5c7e7240 100644 --- a/src/datanode/src/datanode.rs +++ b/src/datanode/src/datanode.rs @@ -34,6 +34,8 @@ pub enum ObjectStoreConfig { root: String, access_key_id: String, secret_access_key: String, + endpoint: Option, + region: Option, }, } @@ -46,6 +48,7 @@ impl Default for ObjectStoreConfig { } #[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(default)] pub struct DatanodeOptions { pub node_id: Option, pub rpc_addr: String, diff --git a/src/datanode/src/instance.rs b/src/datanode/src/instance.rs index e12b1c8bee..704d98adc0 100644 --- a/src/datanode/src/instance.rs +++ b/src/datanode/src/instance.rs @@ -202,28 +202,45 @@ pub(crate) async fn new_object_store(store_config: &ObjectStoreConfig) -> Result } pub(crate) async fn new_s3_object_store(store_config: &ObjectStoreConfig) -> Result { - let (root, secret_key, key_id, bucket) = match store_config { + let (root, secret_key, key_id, bucket, endpoint, region) = match store_config { ObjectStoreConfig::S3 { bucket, root, access_key_id, secret_access_key, - } => (root, secret_access_key, access_key_id, bucket), + endpoint, + region, + } => ( + root, + secret_access_key, + access_key_id, + bucket, + endpoint, + region, + ), _ => unreachable!(), }; let root = util::normalize_dir(root); info!("The s3 storage bucket is: {}, root is: {}", bucket, &root); - let accessor = S3Builder::default() + let mut builder = S3Builder::default(); + let mut builder = builder .root(&root) .bucket(bucket) .access_key_id(key_id) - .secret_access_key(secret_key) - .build() - .with_context(|_| error::InitBackendSnafu { - config: store_config.clone(), - })?; + .secret_access_key(secret_key); + + if let Some(endpoint) = endpoint { + builder = builder.endpoint(endpoint); + } + if let Some(region) = region { + builder = builder.region(region); + } + + let accessor = builder.build().with_context(|_| error::InitBackendSnafu { + config: store_config.clone(), + })?; Ok(ObjectStore::new(accessor)) } diff --git a/src/frontend/src/frontend.rs b/src/frontend/src/frontend.rs index 563d392434..00a3b4fbe9 100644 --- a/src/frontend/src/frontend.rs +++ b/src/frontend/src/frontend.rs @@ -33,6 +33,7 @@ use crate::server::Services; use crate::Plugins; #[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(default)] pub struct FrontendOptions { pub http_options: Option, pub grpc_options: Option, diff --git a/src/meta-srv/src/metasrv.rs b/src/meta-srv/src/metasrv.rs index 5e6245ac5c..4359b04d1d 100644 --- a/src/meta-srv/src/metasrv.rs +++ b/src/meta-srv/src/metasrv.rs @@ -32,6 +32,7 @@ use crate::service::store::kv::KvStoreRef; pub const TABLE_ID_SEQ: &str = "table_id"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] pub struct MetaSrvOptions { pub bind_addr: String, pub server_addr: String, diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs index bf244f0162..4a49f7feae 100644 --- a/tests-integration/src/test_util.rs +++ b/tests-integration/src/test_util.rs @@ -99,6 +99,8 @@ fn get_test_store_config( bucket, access_key_id: key_id, secret_access_key: secret_key, + endpoint: None, + region: None, }; let store = ObjectStore::new(accessor);