diff --git a/bin/core/src/monitor/alert/server.rs b/bin/core/src/monitor/alert/server.rs index ff561dce8..63b29510a 100644 --- a/bin/core/src/monitor/alert/server.rs +++ b/bin/core/src/monitor/alert/server.rs @@ -46,6 +46,10 @@ impl State { } let server = server.unwrap(); let server_alerts = alerts.get(&ResourceTarget::Server(v.id.clone())); + + // =================== + // SERVER HEALTH + // =================== let health_alert = server_alerts .as_ref() .and_then(|alerts| alerts.get(&AlertDataVariant::ServerUnreachable)); @@ -74,6 +78,56 @@ impl State { } _ => {} } + + if v.health.is_none() { + continue; + } + + let health = v.health.as_ref().unwrap(); + + // =================== + // SERVER CPU + // =================== + let cpu_alert = server_alerts + .as_ref() + .and_then(|alerts| alerts.get(&AlertDataVariant::ServerCpu)); + match (health.cpu, cpu_alert) { + (SeverityLevel::Warning | SeverityLevel::Critical, None) => { + // open alert + } + (SeverityLevel::Warning | SeverityLevel::Critical, Some(alert)) => { + // modify alert level + } + (SeverityLevel::Ok, Some(alert)) => { + // resolve alert + } + _ => {} + } + + // =================== + // SERVER MEM + // =================== + let mem_alert = server_alerts + .as_ref() + .and_then(|alerts| alerts.get(&AlertDataVariant::ServerMem)); + match (health.mem, mem_alert) { + (SeverityLevel::Warning | SeverityLevel::Critical, None) => { + // open alert + } + (SeverityLevel::Warning | SeverityLevel::Critical, Some(alert)) => { + // modify alert level + } + (SeverityLevel::Ok, Some(alert)) => { + // resolve alert + } + _ => {} + } + + // =================== + // SERVER DISK + // =================== + + // alerts possible on multiple disks make this complicated (multiple ServerDisk alerts possible on same server) } tokio::join!( diff --git a/bin/core/src/requests/read/alert.rs b/bin/core/src/requests/read/alert.rs new file mode 100644 index 000000000..b899ccf89 --- /dev/null +++ b/bin/core/src/requests/read/alert.rs @@ -0,0 +1,62 @@ +use anyhow::Context; +use async_trait::async_trait; +use monitor_types::{ + entities::server::Server, + requests::read::{ListAlerts, ListAlertsResponse}, +}; +use mungos::mongodb::{ + bson::{doc, Document}, + options::FindOptions, +}; +use resolver_api::Resolve; + +use crate::{auth::RequestUser, helpers::resource::StateResource, state::State}; + +const NUM_ALERTS_PER_PAGE: u64 = 10; + +#[async_trait] +impl Resolve for State { + async fn resolve( + &self, + ListAlerts { + page, + include_resolved, + }: ListAlerts, + user: RequestUser, + ) -> anyhow::Result { + let mut query = Document::new(); + if !include_resolved { + query.insert("resolved", true); + } + if !user.is_admin { + let server_ids = + >::get_resource_ids_for_non_admin(self, &user.id) + .await?; + query.insert("target.type", "Server"); + query.insert("target.id", doc! { "$in": server_ids }); + } + let alerts = self + .db + .alerts + .get_some( + query, + FindOptions::builder() + .sort(doc! { "ts": -1 }) + .limit(NUM_ALERTS_PER_PAGE as i64) + .skip(page * NUM_ALERTS_PER_PAGE) + .build(), + ) + .await + .context("failed to get alerts from db")?; + + let next_page = if alerts.len() < NUM_ALERTS_PER_PAGE as usize { + None + } else { + Some((page + 1) as i64) + }; + + let res = ListAlertsResponse { next_page, alerts }; + + Ok(res) + } +} diff --git a/bin/core/src/requests/read/mod.rs b/bin/core/src/requests/read/mod.rs index 480e96d76..50625ef37 100644 --- a/bin/core/src/requests/read/mod.rs +++ b/bin/core/src/requests/read/mod.rs @@ -26,6 +26,7 @@ mod server; mod tag; mod update; mod user; +mod alert; #[typeshare] #[derive(Serialize, Deserialize, Debug, Clone, Resolver)] @@ -98,6 +99,9 @@ enum ReadRequest { GetUpdate(GetUpdate), ListUpdates(ListUpdates), + // ==== ALERT ==== + ListAlerts(ListAlerts), + // ==== SERVER STATS ==== #[to_string_resolver] GetAllSystemStats(GetAllSystemStats), diff --git a/client/ts/generate_types.mjs b/client/ts/generate_types.mjs index 50550e1cc..ae74fec14 100644 --- a/client/ts/generate_types.mjs +++ b/client/ts/generate_types.mjs @@ -25,6 +25,7 @@ function fix_types() { const contents = readFileSync(types_path); const fixed = contents .toString() - .replaceAll("ResourceTargetVariant", 'ResourceTarget["type"]'); + .replaceAll("ResourceTargetVariant", 'ResourceTarget["type"]') + .replaceAll("AlertDataVariant", 'AlertData["type"]'); writeFileSync(types_path, fixed); } diff --git a/client/ts/src/responses.ts b/client/ts/src/responses.ts index 3e5534153..0d4fdd882 100644 --- a/client/ts/src/responses.ts +++ b/client/ts/src/responses.ts @@ -74,6 +74,9 @@ export type ReadResponses = { GetUpdate: Types.Update; ListUpdates: Types.ListUpdatesResponse; + // ==== ALERT ==== + ListAlerts: Types.ListAlertsResponse; + // ==== SERVER STATS ==== GetAllSystemStats: Types.AllSystemStats; GetBasicSystemStats: Types.BasicSystemStats; diff --git a/client/ts/src/types.ts b/client/ts/src/types.ts index 28f28a595..b54a5d15e 100644 --- a/client/ts/src/types.ts +++ b/client/ts/src/types.ts @@ -259,6 +259,78 @@ export type _PartialServerConfig = Partial; export type _PartialCustomTag = Partial; +export enum SeverityLevel { + Ok = "OK", + Warning = "WARNING", + Critical = "CRITICAL", +} + +export type ResourceTarget = + | { type: "System", id: string } + | { type: "Build", id: string } + | { type: "Builder", id: string } + | { type: "Deployment", id: string } + | { type: "Server", id: string } + | { type: "Repo", id: string } + | { type: "Alerter", id: string }; + +export type AlertData = + | { type: "ServerUnreachable", data: { + id: string; + name: string; + region?: string; +}} + | { type: "ServerCpu", data: { + id: string; + name: string; + region?: string; + percentage: number; + top_procs: SystemProcess[]; +}} + | { type: "ServerMem", data: { + id: string; + name: string; + region?: string; + used_gb: number; + total_gb: number; + top_procs: SystemProcess[]; +}} + | { type: "ServerDisk", data: { + id: string; + name: string; + region?: string; + path: string; + used_gb: number; + total_gb: number; +}} + | { type: "ServerTemp", data: { + id: string; + name: string; + region?: string; + temp: number; + max: number; +}} + | { type: "ContainerStateChange", data: { + id: string; + name: string; + server: string; + from: DockerContainerState; + to: DockerContainerState; +}} + | { type: "None", data: { +}}; + +export interface Alert { + _id?: MongoId; + ts: I64; + resolved: boolean; + level: SeverityLevel; + target: ResourceTarget; + variant: AlertData["type"]; + data: AlertData; + resolved_ts?: I64; +} + export interface CustomAlerterConfig { url: string; } @@ -525,18 +597,12 @@ export interface AllSystemStats { refresh_list_ts: I64; } -export enum StatsState { - Ok = "OK", - Warning = "WARNING", - Critical = "CRITICAL", -} - export interface ServerHealth { - cpu: StatsState; - mem: StatsState; - disk: StatsState; - disks: Record; - temps: Record; + cpu: SeverityLevel; + mem: SeverityLevel; + disk: SeverityLevel; + disks: Record; + temps: Record; } export enum TagColor { @@ -598,15 +664,6 @@ export enum Operation { AutoPull = "AutoPull", } -export type ResourceTarget = - | { type: "System", id: string } - | { type: "Build", id: string } - | { type: "Builder", id: string } - | { type: "Deployment", id: string } - | { type: "Server", id: string } - | { type: "Repo", id: string } - | { type: "Alerter", id: string }; - export interface Log { stage: string; command: string; @@ -769,6 +826,16 @@ export interface PruneDockerContainers { server_id: string; } +export interface ListAlerts { + page?: U64; + include_resolved?: boolean; +} + +export interface ListAlertsResponse { + alerts: Alert[]; + next_page?: I64; +} + export interface GetAlerter { id: string; } @@ -1391,6 +1458,7 @@ export type ReadRequest = | { type: "ListTags", params: ListTags } | { type: "GetUpdate", params: GetUpdate } | { type: "ListUpdates", params: ListUpdates } + | { type: "ListAlerts", params: ListAlerts } | { type: "GetAllSystemStats", params: GetAllSystemStats } | { type: "GetBasicSystemStats", params: GetBasicSystemStats } | { type: "GetCpuUsage", params: GetCpuUsage } @@ -1439,54 +1507,6 @@ export type WriteRequest = | { type: "DeleteTag", params: DeleteTag } | { type: "UpdateTag", params: UpdateTag }; -export type Alert = - | { type: "ServerUnreachable", data: { - id: string; - name: string; - region?: string; -}} - | { type: "ServerCpu", data: { - id: string; - name: string; - region?: string; - state: StatsState; - percentage: number; - top_procs: SystemProcess[]; -}} - | { type: "ServerMem", data: { - id: string; - name: string; - region?: string; - state: StatsState; - used_gb: number; - total_gb: number; - top_procs: SystemProcess[]; -}} - | { type: "ServerDisk", data: { - id: string; - name: string; - region?: string; - state: StatsState; - path: string; - used_gb: number; - total_gb: number; -}} - | { type: "ServerTemp", data: { - id: string; - name: string; - region?: string; - state: StatsState; - temp: number; - max: number; -}} - | { type: "ContainerStateChange", data: { - id: string; - name: string; - server: string; - from: DockerContainerState; - to: DockerContainerState; -}}; - export type Tag = | { type: "ResourceType", params: { resource: ResourceTarget["type"]; diff --git a/lib/types/src/requests/read/alert.rs b/lib/types/src/requests/read/alert.rs new file mode 100644 index 000000000..e812ea4e6 --- /dev/null +++ b/lib/types/src/requests/read/alert.rs @@ -0,0 +1,22 @@ +use resolver_api::derive::Request; +use serde::{Deserialize, Serialize}; +use typeshare::typeshare; + +use crate::{entities::alert::Alert, I64, U64}; + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone, Request)] +#[response(ListAlertsResponse)] +pub struct ListAlerts { + #[serde(default)] + pub page: U64, + #[serde(default)] + pub include_resolved: bool, +} + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ListAlertsResponse { + pub alerts: Vec, + pub next_page: Option, +} diff --git a/lib/types/src/requests/read/mod.rs b/lib/types/src/requests/read/mod.rs index 4caaaae0e..3df82f538 100644 --- a/lib/types/src/requests/read/mod.rs +++ b/lib/types/src/requests/read/mod.rs @@ -2,6 +2,7 @@ use resolver_api::derive::Request; use serde::{Deserialize, Serialize}; use typeshare::typeshare; +mod alert; mod alerter; mod build; mod builder; @@ -12,6 +13,7 @@ mod server; mod tag; mod update; +pub use alert::*; pub use alerter::*; pub use build::*; pub use builder::*;