chore: Rename MetaClientOpts to MetaClientOptions (#1075)

* fix: Serialize FrontendOptions to toml

* fix: Serialize DatanodeOptions to toml

* fix: Serialize StandaloneOptions to toml

See https://users.rust-lang.org/t/why-toml-to-string-get-error-valueaftertable/85903/2

* chore!: Rename MetaClientOpts to MetaClientOptions

BREAKING CHANGE: Change the meta_client_opts in the config file to
meta_client_options

Cargo.lock

@@ -2208,6 +2208,7 @@ dependencies = [
  "tempdir",
  "tokio",
  "tokio-stream",
+ "toml",
  "tonic",
  "tower",
  "tower-http",
@@ -2708,6 +2709,7 @@ dependencies = [
  "table",
  "tempdir",
  "tokio",
+ "toml",
  "tonic",
  "tower",
 ]

config/datanode.example.toml

@@ -19,7 +19,7 @@ sync_write = false
 type = 'File'
 data_dir = '/tmp/greptimedb/data/'
 
-[meta_client_opts]
+[meta_client_options]
 metasrv_addrs = ['127.0.0.1:3002']
 timeout_millis = 3000
 connect_timeout_millis = 5000

config/frontend.example.toml

@@ -5,7 +5,7 @@ datanode_rpc_addr = '127.0.0.1:3001'
 addr = '127.0.0.1:4000'
 timeout = "30s"
 
-[meta_client_opts]
+[meta_client_options]
 metasrv_addrs = ['127.0.0.1:3002']
 timeout_millis = 3000
 connect_timeout_millis = 5000

src/cmd/src/datanode.rs

@@ -15,7 +15,7 @@
 use clap::Parser;
 use common_telemetry::logging;
 use datanode::datanode::{Datanode, DatanodeOptions, FileConfig, ObjectStoreConfig};
-use meta_client::MetaClientOpts;
+use meta_client::MetaClientOptions;
 use servers::Mode;
 use snafu::ResultExt;
 
@@ -110,8 +110,8 @@ impl TryFrom<StartCommand> for DatanodeOptions {
         }
 
         if let Some(meta_addr) = cmd.metasrv_addr {
-            opts.meta_client_opts
-                .get_or_insert_with(MetaClientOpts::default)
+            opts.meta_client_options
+                .get_or_insert_with(MetaClientOptions::default)
                 .metasrv_addrs = meta_addr
                 .split(',')
                 .map(&str::trim)
@@ -162,12 +162,12 @@ mod tests {
         assert_eq!("/tmp/greptimedb/wal".to_string(), options.wal.dir);
         assert_eq!("127.0.0.1:4406".to_string(), options.mysql_addr);
         assert_eq!(4, options.mysql_runtime_size);
-        let MetaClientOpts {
+        let MetaClientOptions {
             metasrv_addrs: metasrv_addr,
             timeout_millis,
             connect_timeout_millis,
             tcp_nodelay,
-        } = options.meta_client_opts.unwrap();
+        } = options.meta_client_options.unwrap();
 
         assert_eq!(vec!["127.0.0.1:3002".to_string()], metasrv_addr);
         assert_eq!(5000, connect_timeout_millis);
@@ -240,12 +240,12 @@ mod tests {
         assert_eq!(1024 * 1024 * 1024 * 50, dn_opts.wal.purge_threshold.0);
         assert!(!dn_opts.wal.sync_write);
         assert_eq!(Some(42), dn_opts.node_id);
-        let MetaClientOpts {
+        let MetaClientOptions {
             metasrv_addrs: metasrv_addr,
             timeout_millis,
             connect_timeout_millis,
             tcp_nodelay,
-        } = dn_opts.meta_client_opts.unwrap();
+        } = dn_opts.meta_client_options.unwrap();
         assert_eq!(vec!["127.0.0.1:3002".to_string()], metasrv_addr);
         assert_eq!(3000, timeout_millis);
         assert_eq!(5000, connect_timeout_millis);

src/cmd/src/frontend.rs

@@ -23,7 +23,7 @@ use frontend::instance::Instance;
 use frontend::mysql::MysqlOptions;
 use frontend::opentsdb::OpentsdbOptions;
 use frontend::postgres::PostgresOptions;
-use meta_client::MetaClientOpts;
+use meta_client::MetaClientOptions;
 use servers::auth::UserProviderRef;
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
@@ -158,8 +158,8 @@ impl TryFrom<StartCommand> for FrontendOptions {
             opts.influxdb_options = Some(InfluxdbOptions { enable });
         }
         if let Some(metasrv_addr) = cmd.metasrv_addr {
-            opts.meta_client_opts
-                .get_or_insert_with(MetaClientOpts::default)
+            opts.meta_client_options
+                .get_or_insert_with(MetaClientOptions::default)
                 .metasrv_addrs = metasrv_addr
                 .split(',')
                 .map(&str::trim)

src/cmd/src/standalone.rs

@@ -68,6 +68,8 @@ impl SubCommand {
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(default)]
 pub struct StandaloneOptions {
+    pub mode: Mode,
+    pub enable_memory_catalog: bool,
     pub http_options: Option<HttpOptions>,
     pub grpc_options: Option<GrpcOptions>,
     pub mysql_options: Option<MysqlOptions>,
@@ -76,16 +78,16 @@ pub struct StandaloneOptions {
     pub influxdb_options: Option<InfluxdbOptions>,
     pub prometheus_options: Option<PrometheusOptions>,
     pub promql_options: Option<PromqlOptions>,
-    pub mode: Mode,
     pub wal: WalConfig,
     pub storage: ObjectStoreConfig,
     pub compaction: CompactionConfig,
-    pub enable_memory_catalog: bool,
 }
 
 impl Default for StandaloneOptions {
     fn default() -> Self {
         Self {
+            mode: Mode::Standalone,
+            enable_memory_catalog: false,
             http_options: Some(HttpOptions::default()),
             grpc_options: Some(GrpcOptions::default()),
             mysql_options: Some(MysqlOptions::default()),
@@ -94,11 +96,9 @@ impl Default for StandaloneOptions {
             influxdb_options: Some(InfluxdbOptions::default()),
             prometheus_options: Some(PrometheusOptions::default()),
             promql_options: Some(PromqlOptions::default()),
-            mode: Mode::Standalone,
             wal: WalConfig::default(),
             storage: ObjectStoreConfig::default(),
             compaction: CompactionConfig::default(),
-            enable_memory_catalog: false,
         }
     }
 }
@@ -106,6 +106,7 @@ impl Default for StandaloneOptions {
 impl StandaloneOptions {
     fn frontend_options(self) -> FrontendOptions {
         FrontendOptions {
+            mode: self.mode,
             http_options: self.http_options,
             grpc_options: self.grpc_options,
             mysql_options: self.mysql_options,
@@ -114,16 +115,15 @@ impl StandaloneOptions {
             influxdb_options: self.influxdb_options,
             prometheus_options: self.prometheus_options,
             promql_options: self.promql_options,
-            mode: self.mode,
-            meta_client_opts: None,
+            meta_client_options: None,
         }
     }
 
     fn datanode_options(self) -> DatanodeOptions {
         DatanodeOptions {
+            enable_memory_catalog: self.enable_memory_catalog,
             wal: self.wal,
             storage: self.storage,
-            enable_memory_catalog: self.enable_memory_catalog,
             compaction: self.compaction,
             ..Default::default()
         }
@@ -367,4 +367,11 @@ mod tests {
             .await;
         assert!(result.is_ok());
     }
+
+    #[test]
+    fn test_toml() {
+        let opts = StandaloneOptions::default();
+        let toml_string = toml::to_string(&opts).unwrap();
+        let _parsed: StandaloneOptions = toml::from_str(&toml_string).unwrap();
+    }
 }

src/datanode/Cargo.toml

@@ -64,3 +64,4 @@ client = { path = "../client" }
 common-query = { path = "../common/query" }
 datafusion-common.workspace = true
 tempdir = "0.3"
+toml = "0.5"

src/datanode/src/datanode.rs

@@ -17,7 +17,7 @@ use std::time::Duration;
 
 use common_base::readable_size::ReadableSize;
 use common_telemetry::info;
-use meta_client::MetaClientOpts;
+use meta_client::MetaClientOptions;
 use serde::{Deserialize, Serialize};
 use servers::Mode;
 use storage::config::EngineConfig as StorageEngineConfig;
@@ -147,35 +147,35 @@ impl From<&DatanodeOptions> for StorageEngineConfig {
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(default)]
 pub struct DatanodeOptions {
+    pub mode: Mode,
+    pub enable_memory_catalog: bool,
     pub node_id: Option<u64>,
     pub rpc_addr: String,
     pub rpc_hostname: Option<String>,
     pub rpc_runtime_size: usize,
     pub mysql_addr: String,
     pub mysql_runtime_size: usize,
-    pub meta_client_opts: Option<MetaClientOpts>,
+    pub meta_client_options: Option<MetaClientOptions>,
     pub wal: WalConfig,
     pub storage: ObjectStoreConfig,
-    pub enable_memory_catalog: bool,
     pub compaction: CompactionConfig,
-    pub mode: Mode,
 }
 
 impl Default for DatanodeOptions {
     fn default() -> Self {
         Self {
+            mode: Mode::Standalone,
+            enable_memory_catalog: false,
             node_id: None,
             rpc_addr: "127.0.0.1:3001".to_string(),
             rpc_hostname: None,
             rpc_runtime_size: 8,
             mysql_addr: "127.0.0.1:4406".to_string(),
             mysql_runtime_size: 2,
-            meta_client_opts: None,
+            meta_client_options: None,
             wal: WalConfig::default(),
             storage: ObjectStoreConfig::default(),
-            enable_memory_catalog: false,
             compaction: CompactionConfig::default(),
-            mode: Mode::Standalone,
         }
     }
 }
@@ -218,3 +218,15 @@ impl Datanode {
         self.instance.clone()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_toml() {
+        let opts = DatanodeOptions::default();
+        let toml_string = toml::to_string(&opts).unwrap();
+        let _parsed: DatanodeOptions = toml::from_str(&toml_string).unwrap();
+    }
+}

src/datanode/src/instance.rs

@@ -25,7 +25,7 @@ use common_telemetry::logging::info;
 use log_store::raft_engine::log_store::RaftEngineLogStore;
 use log_store::LogConfig;
 use meta_client::client::{MetaClient, MetaClientBuilder};
-use meta_client::MetaClientOpts;
+use meta_client::MetaClientOptions;
 use mito::config::EngineConfig as TableEngineConfig;
 use mito::engine::MitoEngine;
 use object_store::cache_policy::LruCacheLayer;
@@ -83,7 +83,7 @@ impl Instance {
             Mode::Distributed => {
                 let meta_client = new_metasrv_client(
                     opts.node_id.context(MissingNodeIdSnafu)?,
-                    opts.meta_client_opts
+                    opts.meta_client_options
                         .as_ref()
                         .context(MissingMetasrvOptsSnafu)?,
                 )
@@ -352,7 +352,7 @@ pub(crate) async fn new_fs_object_store(store_config: &ObjectStoreConfig) -> Res
 }
 
 /// Create metasrv client instance and spawn heartbeat loop.
-async fn new_metasrv_client(node_id: u64, meta_config: &MetaClientOpts) -> Result<MetaClient> {
+async fn new_metasrv_client(node_id: u64, meta_config: &MetaClientOptions) -> Result<MetaClient> {
     let cluster_id = 0; // TODO(hl): read from config
     let member_id = node_id;
 

src/frontend/Cargo.toml

@@ -53,4 +53,5 @@ futures = "0.3"
 meta-srv = { path = "../meta-srv", features = ["mock"] }
 strfmt = "0.2"
 tempdir = "0.3"
+toml = "0.5"
 tower = "0.4"

src/frontend/src/error.rs

@@ -253,7 +253,7 @@ pub enum Error {
         source: datanode::error::Error,
     },
 
-    #[snafu(display("Missing meta_client_opts section in config"))]
+    #[snafu(display("Missing meta_client_options section in config"))]
     MissingMetasrvOpts { backtrace: Backtrace },
 
     #[snafu(display("Failed to convert AlterExpr to AlterRequest, source: {}", source))]

src/frontend/src/frontend.rs

@@ -15,7 +15,7 @@
 use std::sync::Arc;
 
 use common_base::Plugins;
-use meta_client::MetaClientOpts;
+use meta_client::MetaClientOptions;
 use serde::{Deserialize, Serialize};
 use servers::http::HttpOptions;
 use servers::Mode;
@@ -35,6 +35,7 @@ use crate::server::Services;
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(default)]
 pub struct FrontendOptions {
+    pub mode: Mode,
     pub http_options: Option<HttpOptions>,
     pub grpc_options: Option<GrpcOptions>,
     pub mysql_options: Option<MysqlOptions>,
@@ -43,13 +44,13 @@ pub struct FrontendOptions {
     pub influxdb_options: Option<InfluxdbOptions>,
     pub prometheus_options: Option<PrometheusOptions>,
     pub promql_options: Option<PromqlOptions>,
-    pub mode: Mode,
-    pub meta_client_opts: Option<MetaClientOpts>,
+    pub meta_client_options: Option<MetaClientOptions>,
 }
 
 impl Default for FrontendOptions {
     fn default() -> Self {
         Self {
+            mode: Mode::Standalone,
             http_options: Some(HttpOptions::default()),
             grpc_options: Some(GrpcOptions::default()),
             mysql_options: Some(MysqlOptions::default()),
@@ -58,8 +59,7 @@ impl Default for FrontendOptions {
             influxdb_options: Some(InfluxdbOptions::default()),
             prometheus_options: Some(PrometheusOptions::default()),
             promql_options: Some(PromqlOptions::default()),
-            mode: Mode::Standalone,
-            meta_client_opts: None,
+            meta_client_options: None,
         }
     }
 }
@@ -97,3 +97,15 @@ impl<T: FrontendInstance> Frontend<T> {
         Services::start(&self.opts, instance, self.plugins.clone()).await
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_toml() {
+        let opts = FrontendOptions::default();
+        let toml_string = toml::to_string(&opts).unwrap();
+        let _parsed: FrontendOptions = toml::from_str(&toml_string).unwrap();
+    }
+}

src/frontend/src/instance.rs

@@ -42,7 +42,7 @@ use datanode::instance::InstanceRef as DnInstanceRef;
 use datatypes::schema::Schema;
 use distributed::DistInstance;
 use meta_client::client::{MetaClient, MetaClientBuilder};
-use meta_client::MetaClientOpts;
+use meta_client::MetaClientOptions;
 use partition::manager::PartitionRuleManager;
 use partition::route::TableRoutes;
 use query::parser::PromQuery;
@@ -148,7 +148,7 @@ impl Instance {
 
     async fn create_meta_client(opts: &FrontendOptions) -> Result<Arc<MetaClient>> {
         let metasrv_addr = &opts
-            .meta_client_opts
+            .meta_client_options
             .as_ref()
             .context(MissingMetasrvOptsSnafu)?
             .metasrv_addrs;
@@ -157,7 +157,7 @@ impl Instance {
             metasrv_addr
         );
 
-        let meta_config = MetaClientOpts::default();
+        let meta_config = MetaClientOptions::default();
         let channel_config = ChannelConfig::new()
             .timeout(Duration::from_millis(meta_config.timeout_millis))
             .connect_timeout(Duration::from_millis(meta_config.connect_timeout_millis))

src/meta-client/src/lib.rs

@@ -22,14 +22,14 @@ pub mod rpc;
 
 // Options for meta client in datanode instance.
 #[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct MetaClientOpts {
+pub struct MetaClientOptions {
     pub metasrv_addrs: Vec<String>,
     pub timeout_millis: u64,
     pub connect_timeout_millis: u64,
     pub tcp_nodelay: bool,
 }
 
-impl Default for MetaClientOpts {
+impl Default for MetaClientOptions {
     fn default() -> Self {
         Self {
             metasrv_addrs: vec!["127.0.0.1:3002".to_string()],

tests/conf/datanode-test.toml.template

@@ -16,7 +16,7 @@ sync_write = false
 type = 'File'
 data_dir = '{data_dir}'
 
-[meta_client_opts]
+[meta_client_options]
 metasrv_addrs = ['127.0.0.1:3002']
 timeout_millis = 3000
 connect_timeout_millis = 5000