rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-01-06 05:12:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: support parsing env list (#1595)

Browse Source 
* refactor: support parse env list

* refactor: set 'multiple = true' for metasrv_addr cli option and remove duplicated parsing
This commit is contained in:
zyy17
2023-05-17 14:37:08 +08:00
committed by GitHub
parent 715e1a321f
commit 4920836021
7 changed files with 118 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/cmd/src/datanode.rs
View File

@@ -86,8 +86,8 @@ struct StartCommand {
rpc_hostname: Option<String>,
#[clap(long)]
mysql_addr: Option<String>,
#[clap(long)]
metasrv_addr: Option<String>,
#[clap(long, multiple = true, value_delimiter = ',')]
metasrv_addr: Option<Vec<String>>,
#[clap(short, long)]
config_file: Option<String>,
#[clap(long)]
@@ -104,8 +104,11 @@ struct StartCommand {
impl StartCommand {
fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
let mut opts: DatanodeOptions =
Options::load_layered_options(self.config_file.as_deref(), self.env_prefix.as_ref())?;
let mut opts: DatanodeOptions = Options::load_layered_options(
self.config_file.as_deref(),
self.env_prefix.as_ref(),
DatanodeOptions::env_list_keys(),
)?;
if let Some(dir) = top_level_opts.log_dir {
opts.logging.dir = dir;
@@ -130,15 +133,10 @@ impl StartCommand {
opts.node_id = Some(node_id);
}
if let Some(meta_addr) = &self.metasrv_addr {
if let Some(metasrv_addrs) = &self.metasrv_addr {
opts.meta_client_options
.get_or_insert_with(MetaClientOptions::default)
.metasrv_addrs = meta_addr
.clone()
.split(',')
.map(&str::trim)
.map(&str::to_string)
.collect::<_>();
.metasrv_addrs = metasrv_addrs.clone();
opts.mode = Mode::Distributed;
}
@@ -316,7 +314,7 @@ mod tests {
if let Options::Datanode(opt) = (StartCommand {
node_id: Some(42),
metasrv_addr: Some("127.0.0.1:3002".to_string()),
metasrv_addr: Some(vec!["127.0.0.1:3002".to_string()]),
..Default::default()
})
.load_options(TopLevelOptions::default())
@@ -326,7 +324,7 @@ mod tests {
}
assert!((StartCommand {
metasrv_addr: Some("127.0.0.1:3002".to_string()),
metasrv_addr: Some(vec!["127.0.0.1:3002".to_string()]),
..Default::default()
})
.load_options(TopLevelOptions::default())
@@ -371,7 +369,6 @@ mod tests {
mysql_runtime_size = 2
[meta_client_options]
metasrv_addrs = ["127.0.0.1:3002"]
timeout_millis = 3000
connect_timeout_millis = 5000
tcp_nodelay = true
@@ -427,6 +424,16 @@ mod tests {
.join(ENV_VAR_SEP),
Some("99"),
),
(
// meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
vec![
env_prefix.to_string(),
"meta_client_options".to_uppercase(),
"metasrv_addrs".to_uppercase(),
]
.join(ENV_VAR_SEP),
Some("127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003"),
),
],
|| {
let command = StartCommand {
@@ -444,6 +451,14 @@ mod tests {
opts.storage.manifest.gc_duration,
Some(Duration::from_secs(9))
);
assert_eq!(
opts.meta_client_options.unwrap().metasrv_addrs,
vec![
"127.0.0.1:3001".to_string(),
"127.0.0.1:3002".to_string(),
"127.0.0.1:3003".to_string()
]
);
// Should be read from config file, config file > env > default values.
assert_eq!(opts.storage.compaction.max_purge_tasks, 32);

43
src/cmd/src/frontend.rs
View File

@@ -102,8 +102,8 @@ pub struct StartCommand {
config_file: Option<String>,
#[clap(short, long)]
influxdb_enable: Option<bool>,
#[clap(long)]
metasrv_addr: Option<String>,
#[clap(long, multiple = true, value_delimiter = ',')]
metasrv_addr: Option<Vec<String>>,
#[clap(long)]
tls_mode: Option<TlsMode>,
#[clap(long)]
@@ -120,8 +120,11 @@ pub struct StartCommand {
impl StartCommand {
fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
let mut opts: FrontendOptions =
Options::load_layered_options(self.config_file.as_deref(), self.env_prefix.as_ref())?;
let mut opts: FrontendOptions = Options::load_layered_options(
self.config_file.as_deref(),
self.env_prefix.as_ref(),
FrontendOptions::env_list_keys(),
)?;
if let Some(dir) = top_level_opts.log_dir {
opts.logging.dir = dir;
@@ -182,15 +185,10 @@ impl StartCommand {
opts.influxdb_options = Some(InfluxdbOptions { enable });
}
if let Some(metasrv_addr) = &self.metasrv_addr {
if let Some(metasrv_addrs) = &self.metasrv_addr {
opts.meta_client_options
.get_or_insert_with(MetaClientOptions::default)
.metasrv_addrs = metasrv_addr
.clone()
.split(',')
.map(&str::trim)
.map(&str::to_string)
.collect::<Vec<_>>();
.metasrv_addrs = metasrv_addrs.clone();
opts.mode = Mode::Distributed;
}
@@ -374,6 +372,11 @@ mod tests {
[http_options]
addr = "127.0.0.1:4000"
[meta_client_options]
timeout_millis = 3000
connect_timeout_millis = 5000
tcp_nodelay = true
[mysql_options]
addr = "127.0.0.1:4002"
"#;
@@ -412,6 +415,16 @@ mod tests {
.join(ENV_VAR_SEP),
Some("127.0.0.1:24000"),
),
(
// meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
vec![
env_prefix.to_string(),
"meta_client_options".to_uppercase(),
"metasrv_addrs".to_uppercase(),
]
.join(ENV_VAR_SEP),
Some("127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003"),
),
],
|| {
let command = StartCommand {
@@ -430,6 +443,14 @@ mod tests {
// Should be read from env, env > default values.
assert_eq!(fe_opts.mysql_options.as_ref().unwrap().runtime_size, 11);
assert_eq!(
fe_opts.meta_client_options.unwrap().metasrv_addrs,
vec![
"127.0.0.1:3001".to_string(),
"127.0.0.1:3002".to_string(),
"127.0.0.1:3003".to_string()
]
);
// Should be read from config file, config file > env > default values.
assert_eq!(

7
src/cmd/src/metasrv.rs
View File

@@ -102,8 +102,11 @@ struct StartCommand {
impl StartCommand {
fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
let mut opts: MetaSrvOptions =
Options::load_layered_options(self.config_file.as_deref(), self.env_prefix.as_ref())?;
let mut opts: MetaSrvOptions = Options::load_layered_options(
self.config_file.as_deref(),
self.env_prefix.as_ref(),
None,
)?;
if let Some(dir) = top_level_opts.log_dir {
opts.logging.dir = dir;

39
src/cmd/src/options.rs
View File

@@ -23,6 +23,7 @@ use snafu::ResultExt;
use crate::error::{LoadLayeredConfigSnafu, Result};
pub const ENV_VAR_SEP: &str = "__";
pub const ENV_LIST_SEP: &str = ",";
pub struct MixOptions {
pub fe_opts: FrontendOptions,
@@ -60,9 +61,12 @@ impl Options {
/// `env_prefix` is the prefix of environment variables, e.g. "FRONTEND__xxx".
/// The function will use dunder(double underscore) `__` as the separator for environment variables, for example:
/// `DATANODE__STORAGE__MANIFEST__CHECKPOINT_MARGIN` will be mapped to `DatanodeOptions.storage.manifest.checkpoint_margin` field in the configuration.
/// `list_keys` is the list of keys that should be parsed as a list, for example, you can pass `Some(&["meta_client_options.metasrv_addrs"]` to parse `GREPTIMEDB_METASRV__META_CLIENT_OPTIONS__METASRV_ADDRS` as a list.
/// The function will use comma `,` as the separator for list values, for example: `127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003`.
pub fn load_layered_options<'de, T: Serialize + Deserialize<'de> + Default>(
config_file: Option<&str>,
env_prefix: &str,
list_keys: Option<&[&str]>,
) -> Result<T> {
let default_opts = T::default();
@@ -73,6 +77,13 @@ impl Options {
env = env.prefix(env_prefix);
}
if let Some(list_keys) = list_keys {
env = env.list_separator(ENV_LIST_SEP);
for key in list_keys {
env = env.with_list_parse_key(key);
}
}
env.try_parsing(true)
.separator(ENV_VAR_SEP)
.ignore_empty(true)
@@ -121,7 +132,6 @@ mod tests {
mysql_runtime_size = 2
[meta_client_options]
metasrv_addrs = ["127.0.0.1:3002"]
timeout_millis = 3000
connect_timeout_millis = 5000
tcp_nodelay = true
@@ -212,11 +222,24 @@ mod tests {
.join(ENV_VAR_SEP),
Some("/other/wal/dir"),
),
(
// meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
vec![
env_prefix.to_string(),
"meta_client_options".to_uppercase(),
"metasrv_addrs".to_uppercase(),
]
.join(ENV_VAR_SEP),
Some("127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003"),
),
],
|| {
let opts: DatanodeOptions =
Options::load_layered_options(Some(file.path().to_str().unwrap()), env_prefix)
.unwrap();
let opts: DatanodeOptions = Options::load_layered_options(
Some(file.path().to_str().unwrap()),
env_prefix,
DatanodeOptions::env_list_keys(),
)
.unwrap();
// Check the configs from environment variables.
assert_eq!(opts.storage.manifest.checkpoint_margin, Some(99));
@@ -231,6 +254,14 @@ mod tests {
Some(Duration::from_secs(42))
);
assert!(opts.storage.manifest.checkpoint_on_startup);
assert_eq!(
opts.meta_client_options.unwrap().metasrv_addrs,
vec![
"127.0.0.1:3001".to_string(),
"127.0.0.1:3002".to_string(),
"127.0.0.1:3003".to_string()
]
);
// Should be the values from config file, not environment variables.
assert_eq!(opts.wal.dir, "/tmp/greptimedb/wal".to_string());

7
src/cmd/src/standalone.rs
View File

@@ -217,8 +217,11 @@ struct StartCommand {
impl StartCommand {
fn load_options(&self, top_level_options: TopLevelOptions) -> Result<Options> {
let mut opts: StandaloneOptions =
Options::load_layered_options(self.config_file.as_deref(), self.env_prefix.as_ref())?;
let mut opts: StandaloneOptions = Options::load_layered_options(
self.config_file.as_deref(),
self.env_prefix.as_ref(),
None,
)?;
opts.enable_memory_catalog = self.enable_memory_catalog;

6
src/datanode/src/datanode.rs
View File

@@ -323,6 +323,12 @@ impl Default for DatanodeOptions {
}
}
impl DatanodeOptions {
pub fn env_list_keys() -> Option<&'static [&'static str]> {
Some(&["meta_client_options.metasrv_addrs"])
}
}
/// Datanode service.
pub struct Datanode {
opts: DatanodeOptions,

6
src/frontend/src/frontend.rs
View File

@@ -60,6 +60,12 @@ impl Default for FrontendOptions {
}
}
impl FrontendOptions {
pub fn env_list_keys() -> Option<&'static [&'static str]> {
Some(&["meta_client_options.metasrv_addrs"])
}
}
#[cfg(test)]
mod tests {
use super::*;