feat: endpoint and region config for s3 storage (#820)

* feat: adds serde default attribute to options

* feat: adds endpoint and region for s3 config

src/cmd/src/standalone.rs

@@ -63,6 +63,7 @@ impl SubCommand {
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(default)]
 pub struct StandaloneOptions {
     pub http_options: Option<HttpOptions>,
     pub grpc_options: Option<GrpcOptions>,

src/cmd/src/toml_loader.rs

@@ -37,12 +37,23 @@ mod tests {
     use crate::error::Result;
 
     #[derive(Clone, PartialEq, Debug, Deserialize, Serialize)]
+    #[serde(default)]
     struct MockConfig {
         path: String,
         port: u32,
         host: String,
     }
 
+    impl Default for MockConfig {
+        fn default() -> Self {
+            Self {
+                path: "test".to_string(),
+                port: 0,
+                host: "localhost".to_string(),
+            }
+        }
+    }
+
     #[test]
     fn test_from_file() -> Result<()> {
         let config = MockConfig {
@@ -63,6 +74,21 @@ mod tests {
         let loaded_config: MockConfig = from_file!(&test_file)?;
         assert_eq!(loaded_config, config);
 
+        // Only host in file
+        let mut file = File::create(&test_file).unwrap();
+        file.write_all("host='greptime.test'\n".as_bytes()).unwrap();
+
+        let loaded_config: MockConfig = from_file!(&test_file)?;
+        assert_eq!(loaded_config.host, "greptime.test");
+        assert_eq!(loaded_config.port, 0);
+        assert_eq!(loaded_config.path, "test");
+
+        // Truncate the file.
+        let file = File::create(&test_file).unwrap();
+        file.set_len(0).unwrap();
+        let loaded_config: MockConfig = from_file!(&test_file)?;
+        assert_eq!(loaded_config, MockConfig::default());
+
         Ok(())
     }
 }

src/datanode/src/datanode.rs

@@ -34,6 +34,8 @@ pub enum ObjectStoreConfig {
         root: String,
         access_key_id: String,
         secret_access_key: String,
+        endpoint: Option<String>,
+        region: Option<String>,
     },
 }
 
@@ -46,6 +48,7 @@ impl Default for ObjectStoreConfig {
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(default)]
 pub struct DatanodeOptions {
     pub node_id: Option<u64>,
     pub rpc_addr: String,

src/datanode/src/instance.rs

@@ -202,28 +202,45 @@ pub(crate) async fn new_object_store(store_config: &ObjectStoreConfig) -> Result
 }
 
 pub(crate) async fn new_s3_object_store(store_config: &ObjectStoreConfig) -> Result<ObjectStore> {
-    let (root, secret_key, key_id, bucket) = match store_config {
+    let (root, secret_key, key_id, bucket, endpoint, region) = match store_config {
         ObjectStoreConfig::S3 {
             bucket,
             root,
             access_key_id,
             secret_access_key,
-        } => (root, secret_access_key, access_key_id, bucket),
+            endpoint,
+            region,
+        } => (
+            root,
+            secret_access_key,
+            access_key_id,
+            bucket,
+            endpoint,
+            region,
+        ),
         _ => unreachable!(),
     };
 
     let root = util::normalize_dir(root);
     info!("The s3 storage bucket is: {}, root is: {}", bucket, &root);
 
-    let accessor = S3Builder::default()
+    let mut builder = S3Builder::default();
+    let mut builder = builder
         .root(&root)
         .bucket(bucket)
         .access_key_id(key_id)
-        .secret_access_key(secret_key)
-        .build()
-        .with_context(|_| error::InitBackendSnafu {
-            config: store_config.clone(),
-        })?;
+        .secret_access_key(secret_key);
+
+    if let Some(endpoint) = endpoint {
+        builder = builder.endpoint(endpoint);
+    }
+    if let Some(region) = region {
+        builder = builder.region(region);
+    }
+
+    let accessor = builder.build().with_context(|_| error::InitBackendSnafu {
+        config: store_config.clone(),
+    })?;
 
     Ok(ObjectStore::new(accessor))
 }

src/frontend/src/frontend.rs

@@ -33,6 +33,7 @@ use crate::server::Services;
 use crate::Plugins;
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(default)]
 pub struct FrontendOptions {
     pub http_options: Option<HttpOptions>,
     pub grpc_options: Option<GrpcOptions>,

src/meta-srv/src/metasrv.rs

@@ -32,6 +32,7 @@ use crate::service::store::kv::KvStoreRef;
 pub const TABLE_ID_SEQ: &str = "table_id";
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(default)]
 pub struct MetaSrvOptions {
     pub bind_addr: String,
     pub server_addr: String,

tests-integration/src/test_util.rs

@@ -99,6 +99,8 @@ fn get_test_store_config(
                 bucket,
                 access_key_id: key_id,
                 secret_access_key: secret_key,
+                endpoint: None,
+                region: None,
             };
 
             let store = ObjectStore::new(accessor);