From f6e871708a11150ad4140aa61152dfa8bb0ec855 Mon Sep 17 00:00:00 2001 From: Yingwen Date: Fri, 24 Feb 2023 16:28:38 +0800 Subject: [PATCH] chore: Rename MetaClientOpts to MetaClientOptions (#1075) * fix: Serialize FrontendOptions to toml * fix: Serialize DatanodeOptions to toml * fix: Serialize StandaloneOptions to toml See https://users.rust-lang.org/t/why-toml-to-string-get-error-valueaftertable/85903/2 * chore!: Rename MetaClientOpts to MetaClientOptions BREAKING CHANGE: Change the meta_client_opts in the config file to meta_client_options --- Cargo.lock | 2 ++ config/datanode.example.toml | 2 +- config/frontend.example.toml | 2 +- src/cmd/src/datanode.rs | 14 +++++++------- src/cmd/src/frontend.rs | 6 +++--- src/cmd/src/standalone.rs | 21 ++++++++++++++------- src/datanode/Cargo.toml | 1 + src/datanode/src/datanode.rs | 26 +++++++++++++++++++------- src/datanode/src/instance.rs | 6 +++--- src/frontend/Cargo.toml | 1 + src/frontend/src/error.rs | 2 +- src/frontend/src/frontend.rs | 22 +++++++++++++++++----- src/frontend/src/instance.rs | 6 +++--- src/meta-client/src/lib.rs | 4 ++-- tests/conf/datanode-test.toml.template | 2 +- 15 files changed, 76 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d29c0b0136..bd28e140c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2208,6 +2208,7 @@ dependencies = [ "tempdir", "tokio", "tokio-stream", + "toml", "tonic", "tower", "tower-http", @@ -2708,6 +2709,7 @@ dependencies = [ "table", "tempdir", "tokio", + "toml", "tonic", "tower", ] diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 57654e3f35..83bcfc4d5d 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -19,7 +19,7 @@ sync_write = false type = 'File' data_dir = '/tmp/greptimedb/data/' -[meta_client_opts] +[meta_client_options] metasrv_addrs = ['127.0.0.1:3002'] timeout_millis = 3000 connect_timeout_millis = 5000 diff --git a/config/frontend.example.toml b/config/frontend.example.toml index a26112ba22..a8022a1113 100644 --- a/config/frontend.example.toml +++ b/config/frontend.example.toml @@ -5,7 +5,7 @@ datanode_rpc_addr = '127.0.0.1:3001' addr = '127.0.0.1:4000' timeout = "30s" -[meta_client_opts] +[meta_client_options] metasrv_addrs = ['127.0.0.1:3002'] timeout_millis = 3000 connect_timeout_millis = 5000 diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs index e67e0533cc..1dc71d3313 100644 --- a/src/cmd/src/datanode.rs +++ b/src/cmd/src/datanode.rs @@ -15,7 +15,7 @@ use clap::Parser; use common_telemetry::logging; use datanode::datanode::{Datanode, DatanodeOptions, FileConfig, ObjectStoreConfig}; -use meta_client::MetaClientOpts; +use meta_client::MetaClientOptions; use servers::Mode; use snafu::ResultExt; @@ -110,8 +110,8 @@ impl TryFrom for DatanodeOptions { } if let Some(meta_addr) = cmd.metasrv_addr { - opts.meta_client_opts - .get_or_insert_with(MetaClientOpts::default) + opts.meta_client_options + .get_or_insert_with(MetaClientOptions::default) .metasrv_addrs = meta_addr .split(',') .map(&str::trim) @@ -162,12 +162,12 @@ mod tests { assert_eq!("/tmp/greptimedb/wal".to_string(), options.wal.dir); assert_eq!("127.0.0.1:4406".to_string(), options.mysql_addr); assert_eq!(4, options.mysql_runtime_size); - let MetaClientOpts { + let MetaClientOptions { metasrv_addrs: metasrv_addr, timeout_millis, connect_timeout_millis, tcp_nodelay, - } = options.meta_client_opts.unwrap(); + } = options.meta_client_options.unwrap(); assert_eq!(vec!["127.0.0.1:3002".to_string()], metasrv_addr); assert_eq!(5000, connect_timeout_millis); @@ -240,12 +240,12 @@ mod tests { assert_eq!(1024 * 1024 * 1024 * 50, dn_opts.wal.purge_threshold.0); assert!(!dn_opts.wal.sync_write); assert_eq!(Some(42), dn_opts.node_id); - let MetaClientOpts { + let MetaClientOptions { metasrv_addrs: metasrv_addr, timeout_millis, connect_timeout_millis, tcp_nodelay, - } = dn_opts.meta_client_opts.unwrap(); + } = dn_opts.meta_client_options.unwrap(); assert_eq!(vec!["127.0.0.1:3002".to_string()], metasrv_addr); assert_eq!(3000, timeout_millis); assert_eq!(5000, connect_timeout_millis); diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index cf6587ca3d..875fcf358c 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -23,7 +23,7 @@ use frontend::instance::Instance; use frontend::mysql::MysqlOptions; use frontend::opentsdb::OpentsdbOptions; use frontend::postgres::PostgresOptions; -use meta_client::MetaClientOpts; +use meta_client::MetaClientOptions; use servers::auth::UserProviderRef; use servers::http::HttpOptions; use servers::tls::{TlsMode, TlsOption}; @@ -158,8 +158,8 @@ impl TryFrom for FrontendOptions { opts.influxdb_options = Some(InfluxdbOptions { enable }); } if let Some(metasrv_addr) = cmd.metasrv_addr { - opts.meta_client_opts - .get_or_insert_with(MetaClientOpts::default) + opts.meta_client_options + .get_or_insert_with(MetaClientOptions::default) .metasrv_addrs = metasrv_addr .split(',') .map(&str::trim) diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 060c7e1739..1be6ad8979 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -68,6 +68,8 @@ impl SubCommand { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default)] pub struct StandaloneOptions { + pub mode: Mode, + pub enable_memory_catalog: bool, pub http_options: Option, pub grpc_options: Option, pub mysql_options: Option, @@ -76,16 +78,16 @@ pub struct StandaloneOptions { pub influxdb_options: Option, pub prometheus_options: Option, pub promql_options: Option, - pub mode: Mode, pub wal: WalConfig, pub storage: ObjectStoreConfig, pub compaction: CompactionConfig, - pub enable_memory_catalog: bool, } impl Default for StandaloneOptions { fn default() -> Self { Self { + mode: Mode::Standalone, + enable_memory_catalog: false, http_options: Some(HttpOptions::default()), grpc_options: Some(GrpcOptions::default()), mysql_options: Some(MysqlOptions::default()), @@ -94,11 +96,9 @@ impl Default for StandaloneOptions { influxdb_options: Some(InfluxdbOptions::default()), prometheus_options: Some(PrometheusOptions::default()), promql_options: Some(PromqlOptions::default()), - mode: Mode::Standalone, wal: WalConfig::default(), storage: ObjectStoreConfig::default(), compaction: CompactionConfig::default(), - enable_memory_catalog: false, } } } @@ -106,6 +106,7 @@ impl Default for StandaloneOptions { impl StandaloneOptions { fn frontend_options(self) -> FrontendOptions { FrontendOptions { + mode: self.mode, http_options: self.http_options, grpc_options: self.grpc_options, mysql_options: self.mysql_options, @@ -114,16 +115,15 @@ impl StandaloneOptions { influxdb_options: self.influxdb_options, prometheus_options: self.prometheus_options, promql_options: self.promql_options, - mode: self.mode, - meta_client_opts: None, + meta_client_options: None, } } fn datanode_options(self) -> DatanodeOptions { DatanodeOptions { + enable_memory_catalog: self.enable_memory_catalog, wal: self.wal, storage: self.storage, - enable_memory_catalog: self.enable_memory_catalog, compaction: self.compaction, ..Default::default() } @@ -367,4 +367,11 @@ mod tests { .await; assert!(result.is_ok()); } + + #[test] + fn test_toml() { + let opts = StandaloneOptions::default(); + let toml_string = toml::to_string(&opts).unwrap(); + let _parsed: StandaloneOptions = toml::from_str(&toml_string).unwrap(); + } } diff --git a/src/datanode/Cargo.toml b/src/datanode/Cargo.toml index 10c6430c02..355da9c046 100644 --- a/src/datanode/Cargo.toml +++ b/src/datanode/Cargo.toml @@ -64,3 +64,4 @@ client = { path = "../client" } common-query = { path = "../common/query" } datafusion-common.workspace = true tempdir = "0.3" +toml = "0.5" diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs index 8d2a9f25a2..daa8822d5c 100644 --- a/src/datanode/src/datanode.rs +++ b/src/datanode/src/datanode.rs @@ -17,7 +17,7 @@ use std::time::Duration; use common_base::readable_size::ReadableSize; use common_telemetry::info; -use meta_client::MetaClientOpts; +use meta_client::MetaClientOptions; use serde::{Deserialize, Serialize}; use servers::Mode; use storage::config::EngineConfig as StorageEngineConfig; @@ -147,35 +147,35 @@ impl From<&DatanodeOptions> for StorageEngineConfig { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default)] pub struct DatanodeOptions { + pub mode: Mode, + pub enable_memory_catalog: bool, pub node_id: Option, pub rpc_addr: String, pub rpc_hostname: Option, pub rpc_runtime_size: usize, pub mysql_addr: String, pub mysql_runtime_size: usize, - pub meta_client_opts: Option, + pub meta_client_options: Option, pub wal: WalConfig, pub storage: ObjectStoreConfig, - pub enable_memory_catalog: bool, pub compaction: CompactionConfig, - pub mode: Mode, } impl Default for DatanodeOptions { fn default() -> Self { Self { + mode: Mode::Standalone, + enable_memory_catalog: false, node_id: None, rpc_addr: "127.0.0.1:3001".to_string(), rpc_hostname: None, rpc_runtime_size: 8, mysql_addr: "127.0.0.1:4406".to_string(), mysql_runtime_size: 2, - meta_client_opts: None, + meta_client_options: None, wal: WalConfig::default(), storage: ObjectStoreConfig::default(), - enable_memory_catalog: false, compaction: CompactionConfig::default(), - mode: Mode::Standalone, } } } @@ -218,3 +218,15 @@ impl Datanode { self.instance.clone() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_toml() { + let opts = DatanodeOptions::default(); + let toml_string = toml::to_string(&opts).unwrap(); + let _parsed: DatanodeOptions = toml::from_str(&toml_string).unwrap(); + } +} diff --git a/src/datanode/src/instance.rs b/src/datanode/src/instance.rs index a684dd42c2..7cfc10b3b3 100644 --- a/src/datanode/src/instance.rs +++ b/src/datanode/src/instance.rs @@ -25,7 +25,7 @@ use common_telemetry::logging::info; use log_store::raft_engine::log_store::RaftEngineLogStore; use log_store::LogConfig; use meta_client::client::{MetaClient, MetaClientBuilder}; -use meta_client::MetaClientOpts; +use meta_client::MetaClientOptions; use mito::config::EngineConfig as TableEngineConfig; use mito::engine::MitoEngine; use object_store::cache_policy::LruCacheLayer; @@ -83,7 +83,7 @@ impl Instance { Mode::Distributed => { let meta_client = new_metasrv_client( opts.node_id.context(MissingNodeIdSnafu)?, - opts.meta_client_opts + opts.meta_client_options .as_ref() .context(MissingMetasrvOptsSnafu)?, ) @@ -352,7 +352,7 @@ pub(crate) async fn new_fs_object_store(store_config: &ObjectStoreConfig) -> Res } /// Create metasrv client instance and spawn heartbeat loop. -async fn new_metasrv_client(node_id: u64, meta_config: &MetaClientOpts) -> Result { +async fn new_metasrv_client(node_id: u64, meta_config: &MetaClientOptions) -> Result { let cluster_id = 0; // TODO(hl): read from config let member_id = node_id; diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml index 716c51e539..53bb74eb35 100644 --- a/src/frontend/Cargo.toml +++ b/src/frontend/Cargo.toml @@ -53,4 +53,5 @@ futures = "0.3" meta-srv = { path = "../meta-srv", features = ["mock"] } strfmt = "0.2" tempdir = "0.3" +toml = "0.5" tower = "0.4" diff --git a/src/frontend/src/error.rs b/src/frontend/src/error.rs index 86f89d372b..3691504498 100644 --- a/src/frontend/src/error.rs +++ b/src/frontend/src/error.rs @@ -253,7 +253,7 @@ pub enum Error { source: datanode::error::Error, }, - #[snafu(display("Missing meta_client_opts section in config"))] + #[snafu(display("Missing meta_client_options section in config"))] MissingMetasrvOpts { backtrace: Backtrace }, #[snafu(display("Failed to convert AlterExpr to AlterRequest, source: {}", source))] diff --git a/src/frontend/src/frontend.rs b/src/frontend/src/frontend.rs index 8cf492873b..2dced030e1 100644 --- a/src/frontend/src/frontend.rs +++ b/src/frontend/src/frontend.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use common_base::Plugins; -use meta_client::MetaClientOpts; +use meta_client::MetaClientOptions; use serde::{Deserialize, Serialize}; use servers::http::HttpOptions; use servers::Mode; @@ -35,6 +35,7 @@ use crate::server::Services; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default)] pub struct FrontendOptions { + pub mode: Mode, pub http_options: Option, pub grpc_options: Option, pub mysql_options: Option, @@ -43,13 +44,13 @@ pub struct FrontendOptions { pub influxdb_options: Option, pub prometheus_options: Option, pub promql_options: Option, - pub mode: Mode, - pub meta_client_opts: Option, + pub meta_client_options: Option, } impl Default for FrontendOptions { fn default() -> Self { Self { + mode: Mode::Standalone, http_options: Some(HttpOptions::default()), grpc_options: Some(GrpcOptions::default()), mysql_options: Some(MysqlOptions::default()), @@ -58,8 +59,7 @@ impl Default for FrontendOptions { influxdb_options: Some(InfluxdbOptions::default()), prometheus_options: Some(PrometheusOptions::default()), promql_options: Some(PromqlOptions::default()), - mode: Mode::Standalone, - meta_client_opts: None, + meta_client_options: None, } } } @@ -97,3 +97,15 @@ impl Frontend { Services::start(&self.opts, instance, self.plugins.clone()).await } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_toml() { + let opts = FrontendOptions::default(); + let toml_string = toml::to_string(&opts).unwrap(); + let _parsed: FrontendOptions = toml::from_str(&toml_string).unwrap(); + } +} diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index feb4a8d326..c55837776e 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -42,7 +42,7 @@ use datanode::instance::InstanceRef as DnInstanceRef; use datatypes::schema::Schema; use distributed::DistInstance; use meta_client::client::{MetaClient, MetaClientBuilder}; -use meta_client::MetaClientOpts; +use meta_client::MetaClientOptions; use partition::manager::PartitionRuleManager; use partition::route::TableRoutes; use query::parser::PromQuery; @@ -148,7 +148,7 @@ impl Instance { async fn create_meta_client(opts: &FrontendOptions) -> Result> { let metasrv_addr = &opts - .meta_client_opts + .meta_client_options .as_ref() .context(MissingMetasrvOptsSnafu)? .metasrv_addrs; @@ -157,7 +157,7 @@ impl Instance { metasrv_addr ); - let meta_config = MetaClientOpts::default(); + let meta_config = MetaClientOptions::default(); let channel_config = ChannelConfig::new() .timeout(Duration::from_millis(meta_config.timeout_millis)) .connect_timeout(Duration::from_millis(meta_config.connect_timeout_millis)) diff --git a/src/meta-client/src/lib.rs b/src/meta-client/src/lib.rs index 153161cc3b..1754380f2b 100644 --- a/src/meta-client/src/lib.rs +++ b/src/meta-client/src/lib.rs @@ -22,14 +22,14 @@ pub mod rpc; // Options for meta client in datanode instance. #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct MetaClientOpts { +pub struct MetaClientOptions { pub metasrv_addrs: Vec, pub timeout_millis: u64, pub connect_timeout_millis: u64, pub tcp_nodelay: bool, } -impl Default for MetaClientOpts { +impl Default for MetaClientOptions { fn default() -> Self { Self { metasrv_addrs: vec!["127.0.0.1:3002".to_string()], diff --git a/tests/conf/datanode-test.toml.template b/tests/conf/datanode-test.toml.template index 99be1f3ee1..3a03784d31 100644 --- a/tests/conf/datanode-test.toml.template +++ b/tests/conf/datanode-test.toml.template @@ -16,7 +16,7 @@ sync_write = false type = 'File' data_dir = '{data_dir}' -[meta_client_opts] +[meta_client_options] metasrv_addrs = ['127.0.0.1:3002'] timeout_millis = 3000 connect_timeout_millis = 5000