From 144344bcfc8f71cd48763bc2d0f5b882cb68fcd4 Mon Sep 17 00:00:00 2001 From: mbecker20 Date: Thu, 11 Apr 2024 00:15:05 -0700 Subject: [PATCH] full env core configuration --- bin/core/Dockerfile | 1 + bin/core/src/config.rs | 209 +++++++++++++++++++----- bin/core/src/main.rs | 4 +- config_example/core.config.example.toml | 122 +++++++------- config_example/core.env.example | 52 +++++- 5 files changed, 278 insertions(+), 110 deletions(-) diff --git a/bin/core/Dockerfile b/bin/core/Dockerfile index ed7d828da..826e2d59d 100644 --- a/bin/core/Dockerfile +++ b/bin/core/Dockerfile @@ -14,6 +14,7 @@ RUN cd frontend && yarn link @monitor/client && yarn && yarn build # Final Image FROM gcr.io/distroless/cc +COPY ./config_example/core.config.example.toml /config/config.toml COPY --from=core-builder /builder/target/release/core / COPY --from=frontend-builder /builder/frontend/dist /frontend EXPOSE 9000 diff --git a/bin/core/src/config.rs b/bin/core/src/config.rs index 5af75b690..b428480f0 100644 --- a/bin/core/src/config.rs +++ b/bin/core/src/config.rs @@ -1,45 +1,91 @@ use std::sync::OnceLock; +use anyhow::Context; use logger::LogConfig; use merge_config_files::parse_config_file; use monitor_client::entities::Timelength; use serde::Deserialize; -pub fn env() -> &'static Env { - static ENV: OnceLock = OnceLock::new(); - ENV.get_or_init(|| { - envy::from_env().expect("failed to parse environment") +pub fn frontend_path() -> &'static String { + #[derive(Deserialize)] + struct FrontendEnv { + #[serde(default = "default_frontend_path")] + monitor_frontend_path: String, + } + + fn default_frontend_path() -> String { + "/frontend".to_string() + } + + static FRONTEND_PATH: OnceLock = OnceLock::new(); + FRONTEND_PATH.get_or_init(|| { + let FrontendEnv { + monitor_frontend_path, + } = envy::from_env() + .context("failed to parse FrontendEnv") + .unwrap(); + monitor_frontend_path }) } -#[derive(Deserialize, Debug)] -pub struct Env { - #[serde(default = "default_frontend_path")] - pub frontend_path: String, - - #[serde(default = "default_config_path")] - config_path: String, - - // Config overrides - port: Option, - log_level: Option, - stdio_log_mode: Option, - loki_url: Option, -} - -fn default_config_path() -> String { - "/config/config.toml".to_string() -} - -fn default_frontend_path() -> String { - "/frontend".to_string() -} - pub fn core_config() -> &'static CoreConfig { + #[derive(Deserialize)] + struct OverrideEnv { + #[serde(default = "default_config_path")] + monitor_config_path: String, + + monitor_title: Option, + monitor_host: Option, + monitor_port: Option, + monitor_passkey: Option, + monitor_jwt_valid_for: Option, + monitor_monitoring_interval: Option, + monitor_keep_stats_for_days: Option, + monitor_keep_alerts_for_days: Option, + monitor_github_webhook_secret: Option, + monitor_github_webhook_base_url: Option, + monitor_docker_organizations: Option>, + + // logging + monitor_logging_level: Option, + monitor_logging_stdio: Option, + monitor_logging_loki_url: Option, + + monitor_local_auth: Option, + + // github + monitor_github_oauth_enabled: Option, + monitor_github_oauth_id: Option, + monitor_github_oauth_secret: Option, + + // google + monitor_google_oauth_enabled: Option, + monitor_google_oauth_id: Option, + monitor_google_oauth_secret: Option, + + // mongo + monitor_mongo_uri: Option, + monitor_mongo_address: Option, + monitor_mongo_username: Option, + monitor_mongo_password: Option, + monitor_mongo_app_name: Option, + monitor_mongo_db_name: Option, + + // aws + monitor_aws_access_key_id: Option, + monitor_aws_secret_access_key: Option, + } + + fn default_config_path() -> String { + "/config/config.toml".to_string() + } + static CORE_CONFIG: OnceLock = OnceLock::new(); CORE_CONFIG.get_or_init(|| { - let env = env(); - let config_path = &env.config_path; + let env: OverrideEnv = envy::from_env() + .context("failed to parse OverrideEnv") + .unwrap(); + let config_path = &env.monitor_config_path; let mut config = parse_config_file::(config_path.as_str()) .unwrap_or_else(|e| { @@ -47,13 +93,79 @@ pub fn core_config() -> &'static CoreConfig { }); // Overrides - config.port = env.port.unwrap_or(config.port); + config.title = env.monitor_title.unwrap_or(config.title); + config.host = env.monitor_host.unwrap_or(config.host); + config.port = env.monitor_port.unwrap_or(config.port); + config.passkey = env.monitor_passkey.unwrap_or(config.passkey); + config.jwt_valid_for = + env.monitor_jwt_valid_for.unwrap_or(config.jwt_valid_for); + config.monitoring_interval = env + .monitor_monitoring_interval + .unwrap_or(config.monitoring_interval); + config.keep_stats_for_days = env + .monitor_keep_stats_for_days + .unwrap_or(config.keep_stats_for_days); + config.keep_alerts_for_days = env + .monitor_keep_alerts_for_days + .unwrap_or(config.keep_alerts_for_days); + config.github_webhook_secret = env + .monitor_github_webhook_secret + .unwrap_or(config.github_webhook_secret); + config.github_webhook_base_url = env + .monitor_github_webhook_base_url + .or(config.github_webhook_base_url); + config.docker_organizations = env + .monitor_docker_organizations + .unwrap_or(config.docker_organizations); + config.logging.level = - env.log_level.unwrap_or(config.logging.level); + env.monitor_logging_level.unwrap_or(config.logging.level); config.logging.stdio = - env.stdio_log_mode.unwrap_or(config.logging.stdio); + env.monitor_logging_stdio.unwrap_or(config.logging.stdio); config.logging.loki_url = - env.loki_url.clone().or(config.logging.loki_url); + env.monitor_logging_loki_url.or(config.logging.loki_url); + + config.local_auth = + env.monitor_local_auth.unwrap_or(config.local_auth); + + config.github_oauth.enabled = env + .monitor_github_oauth_enabled + .unwrap_or(config.github_oauth.enabled); + config.github_oauth.id = env + .monitor_github_oauth_id + .unwrap_or(config.github_oauth.id); + config.github_oauth.secret = env + .monitor_github_oauth_secret + .unwrap_or(config.github_oauth.secret); + + config.google_oauth.enabled = env + .monitor_google_oauth_enabled + .unwrap_or(config.google_oauth.enabled); + config.google_oauth.id = env + .monitor_google_oauth_id + .unwrap_or(config.google_oauth.id); + config.google_oauth.secret = env + .monitor_google_oauth_secret + .unwrap_or(config.google_oauth.secret); + + config.mongo.uri = env.monitor_mongo_uri.or(config.mongo.uri); + config.mongo.address = + env.monitor_mongo_address.or(config.mongo.address); + config.mongo.username = + env.monitor_mongo_username.or(config.mongo.username); + config.mongo.password = + env.monitor_mongo_password.or(config.mongo.password); + config.mongo.app_name = + env.monitor_mongo_app_name.unwrap_or(config.mongo.app_name); + config.mongo.db_name = + env.monitor_mongo_db_name.unwrap_or(config.mongo.db_name); + + config.aws.access_key_id = env + .monitor_aws_access_key_id + .unwrap_or(config.aws.access_key_id); + config.aws.secret_access_key = env + .monitor_aws_secret_access_key + .unwrap_or(config.aws.secret_access_key); config }) @@ -75,10 +187,6 @@ pub struct CoreConfig { /// Sent in auth header with req to periphery pub passkey: String, - /// Configure logging - #[serde(default)] - pub logging: LogConfig, - /// Control how long distributed JWT remain valid for. Default is 1-day #[serde(default = "default_jwt_valid_for")] pub jwt_valid_for: Timelength, @@ -87,11 +195,13 @@ pub struct CoreConfig { #[serde(default = "default_monitoring_interval")] pub monitoring_interval: Timelength, - /// number of days to keep stats, or 0 to disable pruning. stats older than this number of days are deleted on a daily cycle + /// Number of days to keep stats, or 0 to disable pruning. stats older than this number of days are deleted on a daily cycle + /// Default is 0 (no pruning) #[serde(default)] pub keep_stats_for_days: u64, - /// number of days to keep alerts, or 0 to disable pruning. alerts older than this number of days are deleted on a daily cycle + /// Number of days to keep alerts, or 0 to disable pruning. alerts older than this number of days are deleted on a daily cycle + /// Default is 0 (no pruning) #[serde(default)] pub keep_alerts_for_days: u64, @@ -106,6 +216,10 @@ pub struct CoreConfig { #[serde(default)] pub docker_organizations: Vec, + /// Configure logging + #[serde(default)] + pub logging: LogConfig, + /// enable login with local auth #[serde(default)] pub local_auth: bool, @@ -123,7 +237,7 @@ pub struct CoreConfig { } fn default_title() -> String { - String::from("monitor") + String::from("Monitor") } fn default_core_port() -> u16 { @@ -148,7 +262,7 @@ pub struct OauthCredentials { pub secret: String, } -#[derive(Deserialize, Debug, Clone, Default)] +#[derive(Deserialize, Debug, Clone)] pub struct MongoConfig { pub uri: Option, pub address: Option, @@ -168,6 +282,19 @@ fn default_core_mongo_db_name() -> String { "monitor".to_string() } +impl Default for MongoConfig { + fn default() -> Self { + Self { + uri: None, + address: Some("localhost:27017".to_string()), + username: None, + password: None, + app_name: default_core_mongo_app_name(), + db_name: default_core_mongo_db_name(), + } + } +} + #[derive(Deserialize, Debug, Clone, Default)] pub struct AwsCredentials { pub access_key_id: String, diff --git a/bin/core/src/main.rs b/bin/core/src/main.rs index cb2cf2b71..a642de03c 100644 --- a/bin/core/src/main.rs +++ b/bin/core/src/main.rs @@ -11,7 +11,7 @@ use tower_http::{ services::{ServeDir, ServeFile}, }; -use crate::config::{core_config, env, Env}; +use crate::config::{core_config, frontend_path}; mod api; mod auth; @@ -36,7 +36,7 @@ async fn app() -> anyhow::Result<()> { prune::spawn_prune_loop(); // Setup static frontend services - let Env { frontend_path, .. } = env(); + let frontend_path = frontend_path(); let frontend_index = ServeFile::new(format!("{frontend_path}/index.html")); let serve_dir = ServeDir::new(frontend_path) diff --git a/config_example/core.config.example.toml b/config_example/core.config.example.toml index 9279f704f..f2c528d5f 100644 --- a/config_example/core.config.example.toml +++ b/config_example/core.config.example.toml @@ -1,88 +1,88 @@ -# this will be the document title on the web page (shows up as text in the browser tab). -# default: 'monitor' -title = "monitor" +## this will be the document title on the web page (shows up as text in the browser tab). +## default: 'Monitor' +title = "Monitor" -# required for oauth functionality. this should be the url used to access monitor in browser, potentially behind DNS. -# eg https://monitor.mogh.tech or http://12.34.56.78:9000. this should match the address configured in your oauth app. -# no default -host = "https://monitor.mogh.tech" +## required for oauth functionality. this should be the url used to access monitor in browser, potentially behind DNS. +## eg https://monitor.dev or http://12.34.56.78:9000. this should match the address configured in your oauth app. +## no default +host = "https://monitor.dev" -# the port the core system will run on. if running core in docker container, leave as this port as 9000 and use port bind eg. -p 9001:9000 -# default: 9000 +## the port the core system will run on. if running core in docker container, leave as this port as 9000 and use port bind eg. -p 9001:9000 +## default: 9000 port = 9000 -# required to match a passkey in periphery config. token used to authenticate core requests to periphery -# no default +## required to match a passkey in periphery config. token used to authenticate core requests to periphery +## no default passkey = "a_random_passkey" -# specify the log level of the monitor core application -# default: info -# options: off, error, warn, info, debug, trace +## specify the log level of the monitor core application +## default: info +## options: off, error, warn, info, debug, trace logging.level = "info" -# specify the logging format for stdout / stderr. -# default: standard -# options: standard, json, none +## specify the logging format for stdout / stderr. +## default: standard +## options: standard, json, none logging.stdio = "standard" -# specify a loki endpoint to send tracing logs to -# optional, default unassigned -#logging.loki_url = "http://localhost:3100" +## specify a loki endpoint to send tracing logs to +## optional, default unassigned +# logging.loki_url = "http://localhost:3100" -# specify how long an issued jwt stays valid. all jwts are invalidated on application restart. -# default: 1-day. -# options: 1-hr, 12-hr, 1-day, 3-day, 1-wk, 2-wk, 30-day +## specify how long an issued jwt stays valid. all jwts are invalidated on application restart. +## default: 1-day. +## options: 1-hr, 12-hr, 1-day, 3-day, 1-wk, 2-wk, 30-day jwt_valid_for = "1-day" -# controls the granularity of the system stats collection by monitor core -# default: 15-sec -# options: 5-sec, 15-sec, 30-sec, 1-min, 2-min, 5-min, 15-min +## controls the granularity of the system stats collection by monitor core +## default: 15-sec +## options: 5-sec, 15-sec, 30-sec, 1-min, 2-min, 5-min, 15-min monitoring_interval = "15-sec" -# number of days to keep stats around, or 0 to disable pruning. -# stats older than this number of days are deleted daily -# default: 0 (pruning disabled) +## number of days to keep stats around, or 0 to disable pruning. +## stats older than this number of days are deleted daily +## default: 0 (pruning disabled) keep_stats_for_days = 0 -# token that has to be given to github during repo webhook config as the secret -# default: empty (none) +## token that has to be given to github during repo webhook config as the secret +## default: empty (none) github_webhook_secret = "your_random_webhook_secret" -# an alternate base url that is used to recieve github webhook requests -# if empty or not specified, will use 'host' address as base -# default: empty (none) -github_webhook_base_url = "https://monitor-github-webhook.mogh.tech" +## an alternate base url that is used to recieve github webhook requests +## if empty or not specified, will use 'host' address as base +## default: empty (none) +# github_webhook_base_url = "https://github-webhook.monitor.dev" -# these will be used by the GUI to attach to builds. -# when attached to build, image will be pushed to repo under the specified organization. -# if empty, the "docker organization" config option will not be shown. -# default: empty -docker_organizations = ["your_docker_org1", "your_docker_org_2"] +## these will be used by the GUI to attach to builds. +## when attached to build, image will be pushed to repo under the specified organization. +## if empty, the "docker organization" config option will not be shown. +## default: empty +# docker_organizations = ["your_docker_org1", "your_docker_org_2"] -# allow or deny user login with username / password -# default: false -local_auth = true +## allow or deny user login with username / password +## default: false +# local_auth = true -[github_oauth] -enabled = true # default: false -id = "your_github_client_id" -secret = "your_github_client_secret" +# github_oauth.enabled = true +# github_oauth.id = "your_github_client_id" +# github_oauth.secret = "your_github_client_secret" -[google_oauth] -enabled = true # default: false -id = "your_google_client_id" -secret = "your_google_client_secret" +# google_oauth.enabled = true +# google_oauth.id = "your_google_client_id" +# google_oauth.secret = "your_google_client_secret" [mongo] -uri = "mongodb://username:password@localhost:27017" -# ==== or ==== +# uri = "mongodb://username:password@localhost:27017" +## ==== or ==== address = "localhost:27017" -username = "username" -password = "password" -# ==== other ==== -db_name = "monitor" # default: monitor. this is the name of the mongo database that monitor will create its collections in. -app_name = "monitor_core" # default: monitor_core. this is the assigned app_name of the mongo client +# username = "username" +# password = "password" +## ==== other ==== +## default: monitor. this is the name of the mongo database that monitor will create its collections in. +db_name = "monitor" +## default: monitor_core. this is the assigned app_name of the mongo client +app_name = "monitor_core" -[aws] -access_key_id = "your_aws_key_id" -secret_access_key = "your_aws_secret_key" +# [aws] +# access_key_id = "your_aws_key_id" +# secret_access_key = "your_aws_secret_key" diff --git a/config_example/core.env.example b/config_example/core.env.example index 10983919d..9e8340c7a 100644 --- a/config_example/core.env.example +++ b/config_example/core.env.example @@ -1,8 +1,48 @@ -# optional. default is /config/config.toml -CONFIG_PATH=/config/config.toml - # optional. default is /frontend -FRONTEND_PATH=/frontend +MONITOR_FRONTEND_PATH=/frontend -# optional. default is 9000. -PORT=9000 \ No newline at end of file +# optional. default is /config/config.toml +MONITOR_CONFIG_PATH=/config/config.toml + +## All config file fields are optionally available to override on environment +## The config fields are prefixed with 'MONITOR_' +## Nested config fields are set by converting nesting to 'MONITOR_{field}', where field is uppercase + +## Note. ALL the following are optional, and could also be specified in the config file +MONITOR_TITLE=Monitor +MONITOR_HOST=https://monitor.dev +MONITOR_PORT=9000 +MONITOR_PASSKEY=asdfasdf +MONITOR_JWT_VALID_FOR=1-day +MONITOR_MONITORING_INTERVAL=15-sec +MONITOR_KEEP_STATS_FOR_DAYS=0 +MONITOR_KEEP_ALERTS_FOR_DAYS=0 +MONITOR_GITHUB_WEBHOOK_SECRET=asdfasdf +MONITOR_GITHUB_WEBHOOK_BASE_URL=https://github-listener.monitor.dev +MONITOR_DOCKER_ORGANIZATIONS=org1,org2 + +MONITOR_LOGGING_LEVEL=info +MONITOR_LOGGING_STDIO=standard +MONITOR_LOGGING_LOKI_URL=http://localhost:3100 + +MONITOR_LOCAL_AUTH=true + +MONITOR_GITHUB_OAUTH_ENABLED=true +MONITOR_GITHUB_OAUTH_ID=asdfasdf +MONITOR_GITHUB_OAUTH_SECRET=asdfasdf + +MONITOR_GOOGLE_OAUTH_ENABLED=true +MONITOR_GOOGLE_OAUTH_ID=asdfasdf +MONITOR_GOOGLE_OAUTH_SECRET=asdfasdf + +MONITOR_MONGO_URI=mongodb://admin:admin@localhost:27017 +# or +MONITOR_MONGO_ADDRESS=localhost:27017 +MONITOR_MONGO_USERNAME=admin +MONITOR_MONGO_PASSWORD=admin + +MONITOR_MONGO_APP_NAME=monitor_core +MONITOR_MONGO_DB_NAME=monitor + +MONITOR_AWS_ACCESS_KEY_ID=asdfasdf +MONITOR_AWS_SECRET_ACCESS_KEY=asdfasdf