diff --git a/Cargo.lock b/Cargo.lock index 09adf0161..647ca94b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2044,9 +2044,9 @@ dependencies = [ [[package]] name = "mungos" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5bab215e70538484f1a21ef673a8c5b49d39f4ba91886852fe60173a5fd2951" +checksum = "ed3fd16ef4c4dc182aa3e058259747a4660736e87dd42d2edbd371a1a8a7b385" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 54dfdcacf..7683ae419 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ derive_variants = "0.1.1" make_option = "0.1.7" resolver_api = "0.1.6" parse_csl = "0.1.0" -mungos = "0.4.18" +mungos = "0.4.19" svi = "0.1.4" # external tokio = { version = "1.32.0", features = ["full"] } diff --git a/bin/core/src/helpers/alert.rs b/bin/core/src/helpers/alert.rs index d3068be1f..1ea0485ae 100644 --- a/bin/core/src/helpers/alert.rs +++ b/bin/core/src/helpers/alert.rs @@ -4,7 +4,7 @@ use monitor_types::entities::{ alert::{Alert, AlertData}, alerter::*, deployment::DockerContainerState, - server::stats::SystemProcess, + server::stats::{SeverityLevel, SystemProcess}, }; use reqwest::StatusCode; use slack::types::Block; @@ -48,25 +48,29 @@ pub async fn send_alert(alerters: &[Alerter], alert: &Alert) { } pub async fn send_slack_alert(url: &str, alert: &Alert) -> anyhow::Result<()> { - let level = alert.level; + let level = fmt_level(alert.level); let (text, blocks): (_, Option<_>) = match &alert.data { AlertData::ServerUnreachable { name, region, .. } => { let region = fmt_region(region); - let text = format!("CRITICAL 🚨 | *{name}*{region} is *unreachable* ❌"); - let blocks = vec![ - Block::header("CRITICAL 🚨"), - Block::section(format!("*{name}*{region} is *unreachable* ❌")), - ]; - (text, blocks.into()) - } - AlertData::ServerReachable { name, region, .. } => { - let region = fmt_region(region); - let text = format!("OK ✅ | *{name}*{region} is now *reachable*"); - let blocks = vec![ - Block::header("OK ✅"), - Block::section(format!("*{name}*{region} is now *reachable*")), - ]; - (text, blocks.into()) + match alert.level { + SeverityLevel::Ok => { + let text = format!("{level} | *{name}*{region} is now *reachable*"); + let blocks = vec![ + Block::header(level), + Block::section(format!("*{name}*{region} is now *reachable*")), + ]; + (text, blocks.into()) + } + SeverityLevel::Critical => { + let text = format!("{level} | *{name}*{region} is *unreachable* ❌"); + let blocks = vec![ + Block::header(level), + Block::section(format!("*{name}*{region} is *unreachable* ❌")), + ]; + (text, blocks.into()) + } + _ => unreachable!(), + } } AlertData::ServerCpu { name, @@ -76,15 +80,19 @@ pub async fn send_slack_alert(url: &str, alert: &Alert) -> anyhow::Result<()> { .. } => { let region = fmt_region(region); - let text = - format!("{level} 🚨 | *{name}*{region} cpu usage at *{percentage:.1}%* 📈 🚨"); - let blocks = vec![ + let text = format!("{level} | *{name}*{region} cpu usage at *{percentage:.1}%* 📈 🚨"); + let mut blocks = vec![ Block::header(format!("{level} 🚨")), Block::section(format!( "*{name}*{region} cpu usage at *{percentage:.1}%* 📈 🚨" )), - Block::section(format!("*top cpu processes*{}", fmt_top_procs(top_procs))), ]; + if alert.level != SeverityLevel::Ok { + blocks.push(Block::section(format!( + "*top cpu processes*{}", + fmt_top_procs(top_procs) + ))); + } (text, blocks.into()) } AlertData::ServerMem { @@ -98,15 +106,20 @@ pub async fn send_slack_alert(url: &str, alert: &Alert) -> anyhow::Result<()> { let region = fmt_region(region); let percentage = 100.0 * used_gb / total_gb; let text = - format!("{level} 🚨 | *{name}*{region} memory usage at *{percentage:.1}%* 💾 🚨"); - let blocks = vec![ - Block::header(format!("{level} 🚨")), + format!("{level} | *{name}*{region} memory usage at *{percentage:.1}%* 💾 🚨"); + let mut blocks = vec![ + Block::header(level), Block::section(format!( "*{name}*{region} memory usage at *{percentage:.1}%* 💾 🚨" )), Block::section(format!("using *{used_gb:.1} GiB* / *{total_gb:.1} GiB*")), - Block::section(format!("*top cpu processes*{}", fmt_top_procs(top_procs))), ]; + if alert.level != SeverityLevel::Ok { + blocks.push(Block::section(format!( + "*top mem processes*{}", + fmt_top_procs(top_procs) + ))); + } (text, blocks.into()) } AlertData::ServerDisk { @@ -120,9 +133,9 @@ pub async fn send_slack_alert(url: &str, alert: &Alert) -> anyhow::Result<()> { let region = fmt_region(region); let percentage = 100.0 * used_gb / total_gb; let text = - format!("{level} 🚨 | *{name}*{region} disk usage at *{percentage:.1}%* | mount point: *{path}* 💿 🚨"); + format!("{level} | *{name}*{region} disk usage at *{percentage:.1}%* | mount point: *{path}* 💿 🚨"); let blocks = vec![ - Block::header(format!("{level} 🚨")), + Block::header(level), Block::section(format!( "*{name}*{region} disk usage at *{percentage:.1}%* 💿 🚨" )), @@ -140,11 +153,10 @@ pub async fn send_slack_alert(url: &str, alert: &Alert) -> anyhow::Result<()> { .. } => { let region = fmt_region(region); - let text = format!( - "{level} 🚨 | *{name}*{region} temp at {temp:.0} °C (max: {max:.0} °C) 🌡️ 🚨" - ); + let text = + format!("{level} | *{name}*{region} temp at {temp:.0} °C (max: {max:.0} °C) 🌡️ 🚨"); let blocks = vec![ - Block::header(format!("{level} 🚨")), + Block::header(level), Block::section(format!( "*{name}*{region} temp at {temp:.0} °C (max: {max:.0} °C) 🌡️ 🚨" )), @@ -226,3 +238,11 @@ fn fmt_docker_container_state(state: &DockerContainerState) -> String { _ => state.to_string(), } } + +fn fmt_level(level: SeverityLevel) -> &'static str { + match level { + SeverityLevel::Critical => "CRITICAL 🚨", + SeverityLevel::Warning => "WARNING 🚨", + SeverityLevel::Ok => "OK ✅", + } +} diff --git a/bin/core/src/monitor/alert/server.rs b/bin/core/src/monitor/alert/server.rs index 065473c06..ff561dce8 100644 --- a/bin/core/src/monitor/alert/server.rs +++ b/bin/core/src/monitor/alert/server.rs @@ -77,20 +77,18 @@ impl State { } tokio::join!( - self.open_alerts(alerts_to_open), + self.open_alerts(&alerts_to_open), self.resolve_alerts(&alert_ids_to_close) ); } - async fn open_alerts(&self, alerts: Vec) { + async fn open_alerts(&self, alerts: &[Alert]) { let open = || async { - // self.db.alerters.create_many(alerts).await?; + self.db.alerts.create_many(alerts).await?; anyhow::Ok(()) }; - let alert = || async { todo!() }; - - let (res, _) = tokio::join!(open(), alert()); + let (res, _) = tokio::join!(open(), self.send_alerts(alerts)); if let Err(e) = res { error!("failed to create alerts on db | {e:#?}"); @@ -106,7 +104,7 @@ impl State { self.db .alerts .update_many( - doc! { "_id": { "$in": alert_ids } }, + doc! { "_id": { "$in": &alert_ids } }, doc! { "$set": { "resolved": "true", @@ -116,17 +114,23 @@ impl State { ) .await .context("failed to resolve alerts on db")?; + let mut closed = self + .db + .alerts + .get_some(doc! { "_id": { "$in": &alert_ids } }, None) + .await + .context("failed to get closed alerts from db")?; + + for closed in &mut closed { + closed.level = SeverityLevel::Ok; + } + + self.send_alerts(&closed).await; + anyhow::Ok(()) }; - let alert = || async { - // alert that alert has be resolved - todo!() - }; - - let (res, _) = tokio::join!(close(), alert()); - - if let Err(e) = res { + if let Err(e) = close().await { error!("failed to resolve alerts | {e:#?}"); } } diff --git a/lib/types/src/entities/alert.rs b/lib/types/src/entities/alert.rs index fb396c480..0045ef089 100644 --- a/lib/types/src/entities/alert.rs +++ b/lib/types/src/entities/alert.rs @@ -63,11 +63,6 @@ pub enum AlertData { name: String, region: Option, }, - ServerReachable { - id: String, - name: String, - region: Option, - }, ServerCpu { id: String, name: String, diff --git a/lib/types/src/entities/server/stats.rs b/lib/types/src/entities/server/stats.rs index 2b65a251e..301519e84 100644 --- a/lib/types/src/entities/server/stats.rs +++ b/lib/types/src/entities/server/stats.rs @@ -143,7 +143,9 @@ pub struct SystemComponent { } #[typeshare] -#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, Display, EnumString)] +#[derive( + Serialize, Deserialize, Debug, Clone, Copy, Default, Display, EnumString, PartialEq, Eq, +)] #[serde(rename_all = "UPPERCASE")] #[strum(serialize_all = "UPPERCASE")] pub enum SeverityLevel {