rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-01-05 21:02:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: refactor options (#2476)

Browse Source
This commit is contained in:
JeremyHi
2023-09-24 10:12:33 +08:00
committed by GitHub
parent 73842f10e7
commit 7310ec0bb3
7 changed files with 86 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@@ -1612,12 +1612,14 @@ dependencies = [
"datatypes",
"either",
"etcd-client",
"file-engine",
"frontend",
"futures",
"lazy_static",
"meta-client",
"meta-srv",
"metrics",
"mito2",
"nu-ansi-term",
"partition",
"prost",

2
config/frontend.example.toml
View File

@@ -8,7 +8,7 @@ interval_millis = 5000
retry_interval_millis = 5000
# HTTP server options, see `standalone.example.toml`.
[http_options]
[http]
addr = "127.0.0.1:4000"
timeout = "30s"
body_limit = "64MB"

2
src/cmd/Cargo.toml
View File

@@ -38,12 +38,14 @@ datanode = { workspace = true }
datatypes = { workspace = true }
either = "1.8"
etcd-client.workspace = true
file-engine = { workspace = true }
frontend = { workspace = true }
futures.workspace = true
lazy_static.workspace = true
meta-client = { workspace = true }
meta-srv = { workspace = true }
metrics.workspace = true
mito2 = { workspace = true }
nu-ansi-term = "0.46"
partition = { workspace = true }
prost.workspace = true

2
src/cmd/src/frontend.rs
View File

@@ -383,7 +383,7 @@ mod tests {
Some("11"),
),
(
// http_options.addr = 127.0.0.1:24000
// http.addr = 127.0.0.1:24000
[
env_prefix.to_string(),
"http".to_uppercase(),

12
src/cmd/src/options.rs
View File

@@ -29,11 +29,11 @@ pub const ENV_LIST_SEP: &str = ",";
/// Options mixed up from datanode, frontend and metasrv.
pub struct MixOptions {
pub data_home: String,
pub procedure_cfg: ProcedureConfig,
pub kv_store_cfg: KvStoreConfig,
pub fe_opts: FrontendOptions,
pub dn_opts: DatanodeOptions,
pub logging_opts: LoggingOptions,
pub procedure: ProcedureConfig,
pub kv_store: KvStoreConfig,
pub frontend: FrontendOptions,
pub datanode: DatanodeOptions,
pub logging: LoggingOptions,
}
pub enum Options {
@@ -56,7 +56,7 @@ impl Options {
Options::Datanode(opts) => &opts.logging,
Options::Frontend(opts) => &opts.logging,
Options::Metasrv(opts) => &opts.logging,
Options::Standalone(opts) => &opts.logging_opts,
Options::Standalone(opts) => &opts.logging,
Options::Cli(opts) => opts,
}
}

140
src/cmd/src/standalone.rs
View File

@@ -24,14 +24,16 @@ use common_meta::kv_backend::KvBackendRef;
use common_procedure::ProcedureManagerRef;
use common_telemetry::info;
use common_telemetry::logging::LoggingOptions;
use datanode::config::{DatanodeOptions, ProcedureConfig, StorageConfig};
use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
use datanode::datanode::{Datanode, DatanodeBuilder};
use datanode::region_server::RegionServer;
use file_engine::config::EngineConfig as FileEngineConfig;
use frontend::frontend::FrontendOptions;
use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
use frontend::service_config::{
GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
};
use mito2::config::MitoConfig;
use serde::{Deserialize, Serialize};
use servers::http::HttpOptions;
use servers::tls::{TlsMode, TlsOption};
@@ -85,18 +87,20 @@ impl SubCommand {
pub struct StandaloneOptions {
pub mode: Mode,
pub enable_telemetry: bool,
pub http_options: HttpOptions,
pub grpc_options: GrpcOptions,
pub mysql_options: MysqlOptions,
pub postgres_options: PostgresOptions,
pub opentsdb_options: OpentsdbOptions,
pub influxdb_options: InfluxdbOptions,
pub prom_store_options: PromStoreOptions,
pub http: HttpOptions,
pub grpc: GrpcOptions,
pub mysql: MysqlOptions,
pub postgres: PostgresOptions,
pub opentsdb: OpentsdbOptions,
pub influxdb: InfluxdbOptions,
pub prom_store: PromStoreOptions,
pub wal: WalConfig,
pub storage: StorageConfig,
pub kv_store: KvStoreConfig,
pub procedure: ProcedureConfig,
pub logging: LoggingOptions,
/// Options for different store engines.
pub region_engine: Vec<RegionEngineConfig>,
}
impl Default for StandaloneOptions {
@@ -104,18 +108,22 @@ impl Default for StandaloneOptions {
Self {
mode: Mode::Standalone,
enable_telemetry: true,
http_options: HttpOptions::default(),
grpc_options: GrpcOptions::default(),
mysql_options: MysqlOptions::default(),
postgres_options: PostgresOptions::default(),
opentsdb_options: OpentsdbOptions::default(),
influxdb_options: InfluxdbOptions::default(),
prom_store_options: PromStoreOptions::default(),
http: HttpOptions::default(),
grpc: GrpcOptions::default(),
mysql: MysqlOptions::default(),
postgres: PostgresOptions::default(),
opentsdb: OpentsdbOptions::default(),
influxdb: InfluxdbOptions::default(),
prom_store: PromStoreOptions::default(),
wal: WalConfig::default(),
storage: StorageConfig::default(),
kv_store: KvStoreConfig::default(),
procedure: ProcedureConfig::default(),
logging: LoggingOptions::default(),
region_engine: vec![
RegionEngineConfig::Mito(MitoConfig::default()),
RegionEngineConfig::File(FileEngineConfig::default()),
],
}
}
}
@@ -124,13 +132,13 @@ impl StandaloneOptions {
fn frontend_options(self) -> FrontendOptions {
FrontendOptions {
mode: self.mode,
http: self.http_options,
grpc: self.grpc_options,
mysql: self.mysql_options,
postgres: self.postgres_options,
opentsdb: self.opentsdb_options,
influxdb: self.influxdb_options,
prom_store: self.prom_store_options,
http: self.http,
grpc: self.grpc,
mysql: self.mysql,
postgres: self.postgres,
opentsdb: self.opentsdb,
influxdb: self.influxdb,
prom_store: self.prom_store,
meta_client: None,
logging: self.logging,
..Default::default()
@@ -143,6 +151,7 @@ impl StandaloneOptions {
enable_telemetry: self.enable_telemetry,
wal: self.wal,
storage: self.storage,
region_engine: self.region_engine,
..Default::default()
}
}
@@ -232,7 +241,7 @@ impl StartCommand {
);
if let Some(addr) = &self.http_addr {
opts.http_options.addr = addr.clone()
opts.http.addr = addr.clone()
}
if let Some(addr) = &self.rpc_addr {
@@ -246,42 +255,42 @@ impl StartCommand {
}
.fail();
}
opts.grpc_options.addr = addr.clone()
opts.grpc.addr = addr.clone()
}
if let Some(addr) = &self.mysql_addr {
opts.mysql_options.enable = true;
opts.mysql_options.addr = addr.clone();
opts.mysql_options.tls = tls_opts.clone();
opts.mysql.enable = true;
opts.mysql.addr = addr.clone();
opts.mysql.tls = tls_opts.clone();
}
if let Some(addr) = &self.postgres_addr {
opts.postgres_options.enable = true;
opts.postgres_options.addr = addr.clone();
opts.postgres_options.tls = tls_opts;
opts.postgres.enable = true;
opts.postgres.addr = addr.clone();
opts.postgres.tls = tls_opts;
}
if let Some(addr) = &self.opentsdb_addr {
opts.opentsdb_options.enable = true;
opts.opentsdb_options.addr = addr.clone();
opts.opentsdb.enable = true;
opts.opentsdb.addr = addr.clone();
}
if self.influxdb_enable {
opts.influxdb_options.enable = self.influxdb_enable;
opts.influxdb.enable = self.influxdb_enable;
}
let kv_store_cfg = opts.kv_store.clone();
let procedure_cfg = opts.procedure.clone();
let fe_opts = opts.clone().frontend_options();
let logging_opts = opts.logging.clone();
let dn_opts = opts.datanode_options();
let kv_store = opts.kv_store.clone();
let procedure = opts.procedure.clone();
let frontend = opts.clone().frontend_options();
let logging = opts.logging.clone();
let datanode = opts.datanode_options();
Ok(Options::Standalone(Box::new(MixOptions {
procedure_cfg,
kv_store_cfg,
data_home: dn_opts.storage.data_home.to_string(),
fe_opts,
dn_opts,
logging_opts,
procedure,
kv_store,
data_home: datanode.storage.data_home.to_string(),
frontend,
datanode,
logging,
})))
}
@@ -290,8 +299,8 @@ impl StartCommand {
#[allow(clippy::diverging_sub_expression)]
async fn build(self, opts: MixOptions) -> Result<Instance> {
let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
let fe_opts = opts.fe_opts;
let dn_opts = opts.dn_opts;
let fe_opts = opts.frontend;
let dn_opts = opts.datanode;
info!("Standalone start command: {:#?}", self);
info!(
@@ -300,13 +309,10 @@ impl StartCommand {
);
let kv_dir = kv_store_dir(&opts.data_home);
let (kv_store, procedure_manager) = FeInstance::try_build_standalone_components(
kv_dir,
opts.kv_store_cfg,
opts.procedure_cfg,
)
.await
.context(StartFrontendSnafu)?;
let (kv_store, procedure_manager) =
FeInstance::try_build_standalone_components(kv_dir, opts.kv_store, opts.procedure)
.await
.context(StartFrontendSnafu)?;
let datanode =
DatanodeBuilder::new(dn_opts.clone(), Some(kv_store.clone()), plugins.clone())
@@ -436,9 +442,9 @@ mod tests {
checkpoint_margin = 9
gc_duration = '7s'
[http_options]
[http]
addr = "127.0.0.1:4000"
timeout = "30s"
timeout = "33s"
body_limit = "128MB"
[logging]
@@ -456,12 +462,12 @@ mod tests {
else {
unreachable!()
};
let fe_opts = options.fe_opts;
let dn_opts = options.dn_opts;
let logging_opts = options.logging_opts;
let fe_opts = options.frontend;
let dn_opts = options.datanode;
let logging_opts = options.logging;
assert_eq!(Mode::Standalone, fe_opts.mode);
assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
assert_eq!(Duration::from_secs(30), fe_opts.http.timeout);
assert_eq!(Duration::from_secs(33), fe_opts.http.timeout);
assert_eq!(ReadableSize::mb(128), fe_opts.http.body_limit);
assert_eq!("127.0.0.1:4001".to_string(), fe_opts.grpc.addr);
assert!(fe_opts.mysql.enable);
@@ -503,8 +509,8 @@ mod tests {
unreachable!()
};
assert_eq!("/tmp/greptimedb/test/logs", opts.logging_opts.dir);
assert_eq!("debug", opts.logging_opts.level.unwrap());
assert_eq!("/tmp/greptimedb/test/logs", opts.logging.dir);
assert_eq!("debug", opts.logging.level.unwrap());
}
#[test]
@@ -548,7 +554,7 @@ mod tests {
// http.addr = 127.0.0.1:24000
[
env_prefix.to_string(),
"http_options".to_uppercase(),
"http".to_uppercase(),
"addr".to_uppercase(),
]
.join(ENV_VAR_SEP),
@@ -573,17 +579,17 @@ mod tests {
};
// Should be read from env, env > default values.
assert_eq!(opts.logging_opts.dir, "/other/log/dir");
assert_eq!(opts.logging.dir, "/other/log/dir");
// Should be read from config file, config file > env > default values.
assert_eq!(opts.logging_opts.level.as_ref().unwrap(), "debug");
assert_eq!(opts.logging.level.as_ref().unwrap(), "debug");
// Should be read from cli, cli > config file > env > default values.
assert_eq!(opts.fe_opts.http.addr, "127.0.0.1:14000");
assert_eq!(ReadableSize::mb(64), opts.fe_opts.http.body_limit);
assert_eq!(opts.frontend.http.addr, "127.0.0.1:14000");
assert_eq!(ReadableSize::mb(64), opts.frontend.http.body_limit);
// Should be default value.
assert_eq!(opts.fe_opts.grpc.addr, GrpcOptions::default().addr);
assert_eq!(opts.frontend.grpc.addr, GrpcOptions::default().addr);
},
);
}

2
src/servers/tests/http/http_handler_test.rs
View File

@@ -401,7 +401,7 @@ async fn test_config() {
let toml_str = r#"
mode = "distributed"
[http_options]
[http]
addr = "127.0.0.1:4000"
timeout = "30s"
body_limit = "2GB"