diff --git a/src/cli/basic/cli.rs b/src/cli/basic/cli.rs index 625417bd2d..be269ee3eb 100644 --- a/src/cli/basic/cli.rs +++ b/src/cli/basic/cli.rs @@ -161,7 +161,7 @@ pub async fn cli() -> Result { db::user::reset().await?; } "alert" => { - db::alerts::reset().await?; + db::alerts::alert::reset().await?; } "dashboard" => { db::dashboards::reset().await?; diff --git a/src/common/infra/config.rs b/src/common/infra/config.rs index 74c167edc6..f732922431 100644 --- a/src/common/infra/config.rs +++ b/src/common/infra/config.rs @@ -24,7 +24,7 @@ use vector_enrichment::TableRegistry; use crate::{ common::meta::{ - alerts, + alerts::{alert::Alert, destinations::Destination, templates::Template}, dashboards::reports, functions::{StreamFunctionsList, Transform}, maxmind::MaxmindClient, @@ -60,14 +60,11 @@ pub static METRIC_CLUSTER_MAP: Lazy>>> = Lazy::new(|| Arc::new(tokio::sync::RwLock::new(HashMap::new()))); pub static METRIC_CLUSTER_LEADER: Lazy>> = Lazy::new(|| Arc::new(tokio::sync::RwLock::new(HashMap::new()))); -pub static STREAM_ALERTS: Lazy>> = - Lazy::new(Default::default); +pub static STREAM_ALERTS: Lazy>> = Lazy::new(Default::default); pub static REALTIME_ALERT_TRIGGERS: Lazy> = Lazy::new(Default::default); -pub static ALERTS_TEMPLATES: Lazy> = - Lazy::new(Default::default); -pub static ALERTS_DESTINATIONS: Lazy> = - Lazy::new(Default::default); +pub static ALERTS_TEMPLATES: Lazy> = Lazy::new(Default::default); +pub static ALERTS_DESTINATIONS: Lazy> = Lazy::new(Default::default); pub static DASHBOARD_REPORTS: Lazy> = Lazy::new(Default::default); pub static SYSLOG_ROUTES: Lazy> = Lazy::new(Default::default); diff --git a/src/common/meta/alerts/alert.rs b/src/common/meta/alerts/alert.rs new file mode 100644 index 0000000000..40abb80396 --- /dev/null +++ b/src/common/meta/alerts/alert.rs @@ -0,0 +1,80 @@ +// Copyright 2024 Zinc Labs Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use config::meta::stream::StreamType; +use hashbrown::HashMap; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +use super::{QueryCondition, TriggerCondition}; + +#[derive(Clone, Debug, Serialize, Deserialize, ToSchema)] +pub struct Alert { + #[serde(default)] + pub name: String, + #[serde(default)] + pub org_id: String, + #[serde(default)] + pub stream_type: StreamType, + #[serde(default)] + pub stream_name: String, + #[serde(default)] + pub is_real_time: bool, + #[serde(default)] + pub query_condition: QueryCondition, + #[serde(default)] + pub trigger_condition: TriggerCondition, + pub destinations: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub context_attributes: Option>, + #[serde(default)] + pub row_template: String, + #[serde(default)] + pub description: String, + #[serde(default)] + pub enabled: bool, + #[serde(default)] + /// Timezone offset in minutes. + /// The negative secs means the Western Hemisphere + pub tz_offset: i32, +} + +impl PartialEq for Alert { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + && self.stream_type == other.stream_type + && self.stream_name == other.stream_name + } +} + +impl Default for Alert { + fn default() -> Self { + Self { + name: "".to_string(), + org_id: "".to_string(), + stream_type: StreamType::default(), + stream_name: "".to_string(), + is_real_time: false, + query_condition: QueryCondition::default(), + trigger_condition: TriggerCondition::default(), + destinations: vec![], + context_attributes: None, + row_template: "".to_string(), + description: "".to_string(), + enabled: false, + tz_offset: 0, // UTC + } + } +} diff --git a/src/common/meta/alerts/derived_streams.rs b/src/common/meta/alerts/derived_streams.rs new file mode 100644 index 0000000000..6f0e380040 --- /dev/null +++ b/src/common/meta/alerts/derived_streams.rs @@ -0,0 +1,66 @@ +// Copyright 2024 Zinc Labs Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use hashbrown::HashMap; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +use crate::common::meta::{ + alerts::{QueryCondition, TriggerCondition}, + stream::StreamParams, +}; + +#[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)] +pub struct DerivedStreamMeta { + #[serde(default)] + pub name: String, + #[serde(default)] + pub source: StreamParams, + #[serde(default)] + pub destination: StreamParams, + #[serde(default)] + pub is_real_time: bool, + #[serde(default)] + pub query_condition: QueryCondition, + #[serde(default)] + pub trigger_condition: TriggerCondition, // Frequency type only supports minutes + #[serde(skip_serializing_if = "Option::is_none")] + pub context_attributes: Option>, + #[serde(default)] + pub description: String, + #[serde(default)] + pub enabled: bool, + #[serde(default)] + /// Timezone offset in minutes. + /// The negative secs means the Western Hemisphere + pub tz_offset: i32, +} + +impl Default for DerivedStreamMeta { + fn default() -> Self { + Self { + name: "".to_string(), + source: StreamParams::default(), + destination: StreamParams::default(), + is_real_time: false, + query_condition: QueryCondition::default(), + trigger_condition: TriggerCondition::default(), + context_attributes: None, + description: "".to_string(), + enabled: true, + tz_offset: 0, // UTC + } + } +} diff --git a/src/common/meta/alerts/mod.rs b/src/common/meta/alerts/mod.rs index 88cff93e73..af4a184523 100644 --- a/src/common/meta/alerts/mod.rs +++ b/src/common/meta/alerts/mod.rs @@ -13,74 +13,16 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use config::{meta::stream::StreamType, utils::json::Value}; -use hashbrown::HashMap; +use config::{meta::search::SearchEventType, utils::json::Value}; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; +pub mod alert; +pub mod derived_streams; pub mod destinations; pub mod templates; -#[derive(Clone, Debug, Serialize, Deserialize, ToSchema)] -pub struct Alert { - #[serde(default)] - pub name: String, - #[serde(default)] - pub org_id: String, - #[serde(default)] - pub stream_type: StreamType, - #[serde(default)] - pub stream_name: String, - #[serde(default)] - pub is_real_time: bool, - #[serde(default)] - pub query_condition: QueryCondition, - #[serde(default)] - pub trigger_condition: TriggerCondition, - pub destinations: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub context_attributes: Option>, - #[serde(default)] - pub row_template: String, - #[serde(default)] - pub description: String, - #[serde(default)] - pub enabled: bool, - #[serde(default)] - /// Timezone offset in minutes. - /// The negative secs means the Western Hemisphere - pub tz_offset: i32, -} - -impl PartialEq for Alert { - fn eq(&self, other: &Self) -> bool { - self.name == other.name - && self.stream_type == other.stream_type - && self.stream_name == other.stream_name - } -} - -impl Default for Alert { - fn default() -> Self { - Self { - name: "".to_string(), - org_id: "".to_string(), - stream_type: StreamType::default(), - stream_name: "".to_string(), - is_real_time: false, - query_condition: QueryCondition::default(), - trigger_condition: TriggerCondition::default(), - destinations: vec![], - context_attributes: None, - row_template: "".to_string(), - description: "".to_string(), - enabled: false, - tz_offset: 0, // UTC - } - } -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize, ToSchema)] +#[derive(Clone, Debug, Default, Serialize, Deserialize, ToSchema, PartialEq)] pub struct TriggerCondition { pub period: i64, // 10 minutes #[serde(default)] @@ -92,13 +34,13 @@ pub struct TriggerCondition { #[serde(default)] pub cron: String, // Cron Expression #[serde(default)] - pub frequency_type: AlertFrequencyType, + pub frequency_type: FrequencyType, #[serde(default)] pub silence: i64, // silence for 10 minutes after fire an alert } #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, ToSchema)] -pub enum AlertFrequencyType { +pub enum FrequencyType { #[serde(rename = "cron")] Cron, #[serde(rename = "minutes")] @@ -106,7 +48,7 @@ pub enum AlertFrequencyType { Minutes, } -#[derive(Clone, Debug, Default, Serialize, Deserialize, ToSchema)] +#[derive(Clone, Debug, Default, Serialize, Deserialize, ToSchema, PartialEq)] pub struct QueryCondition { #[serde(default)] #[serde(rename = "type")] @@ -118,9 +60,11 @@ pub struct QueryCondition { pub aggregation: Option, #[serde(default)] pub vrl_function: Option, + #[serde(default)] + pub search_event_type: Option, } -#[derive(Clone, Debug, Serialize, Deserialize, ToSchema)] +#[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)] pub struct Aggregation { pub group_by: Option>, pub function: AggFunction, diff --git a/src/common/meta/dashboards/v4/mod.rs b/src/common/meta/dashboards/v4/mod.rs index 7b0d6d69ad..347e36f09e 100644 --- a/src/common/meta/dashboards/v4/mod.rs +++ b/src/common/meta/dashboards/v4/mod.rs @@ -214,7 +214,6 @@ pub struct DrillDown { data: Option, } - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct MarkLine { diff --git a/src/common/meta/organization.rs b/src/common/meta/organization.rs index 70b993847e..3297950e57 100644 --- a/src/common/meta/organization.rs +++ b/src/common/meta/organization.rs @@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize}; use utoipa::ToSchema; -use super::{alerts::Alert, functions::Transform}; +use super::{alerts::alert::Alert, functions::Transform}; pub const DEFAULT_ORG: &str = "default"; pub const CUSTOM: &str = "custom"; diff --git a/src/common/meta/pipelines.rs b/src/common/meta/pipelines.rs index 73cfd2c5d2..5bcf9d6010 100644 --- a/src/common/meta/pipelines.rs +++ b/src/common/meta/pipelines.rs @@ -22,7 +22,9 @@ use config::{ use serde::{Deserialize, Serialize}; use utoipa::ToSchema; -use crate::common::meta::functions::StreamFunctionsList; +use crate::common::meta::{ + alerts::derived_streams::DerivedStreamMeta, functions::StreamFunctionsList, +}; #[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)] pub struct PipeLine { @@ -36,6 +38,8 @@ pub struct PipeLine { #[serde(skip_serializing_if = "Option::is_none")] pub routing: Option>>, #[serde(skip_serializing_if = "Option::is_none")] + pub derived_streams: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub meta: Option>, } @@ -47,6 +51,7 @@ impl PipeLine { stream_name: self.stream_name, stream_type: self.stream_type, routing: self.routing, + derived_streams: self.derived_streams, functions, meta: self.meta, } @@ -67,6 +72,8 @@ pub struct PipeLineResponse { #[serde(skip_serializing_if = "Option::is_none")] pub functions: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub derived_streams: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub meta: Option>, } diff --git a/src/common/meta/stream.rs b/src/common/meta/stream.rs index 9babe9e948..f684b7c689 100644 --- a/src/common/meta/stream.rs +++ b/src/common/meta/stream.rs @@ -64,13 +64,31 @@ pub struct ListStream { pub list: Vec, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct StreamParams { pub org_id: faststr::FastStr, pub stream_name: faststr::FastStr, pub stream_type: StreamType, } +impl PartialEq for StreamParams { + fn eq(&self, other: &Self) -> bool { + self.org_id == other.org_id + && self.stream_name == other.stream_name + && self.stream_type == other.stream_type + } +} + +impl Default for StreamParams { + fn default() -> Self { + Self { + org_id: String::default().into(), + stream_name: String::default().into(), + stream_type: StreamType::default(), + } + } +} + impl StreamParams { pub fn new(org_id: &str, stream_name: &str, stream_type: StreamType) -> Self { Self { @@ -79,6 +97,10 @@ impl StreamParams { stream_type, } } + + pub fn is_valid(&self) -> bool { + !(self.org_id.is_empty() || self.stream_name.is_empty()) + } } pub struct SchemaEvolution { diff --git a/src/common/utils/http.rs b/src/common/utils/http.rs index 0f624ccaee..6aaddfc946 100644 --- a/src/common/utils/http.rs +++ b/src/common/utils/http.rs @@ -62,6 +62,7 @@ pub(crate) fn get_search_type_from_request( "alerts" => Some(SearchEventType::Alerts), "values" => Some(SearchEventType::Values), "rum" => Some(SearchEventType::RUM), + "derived_stream" => Some(SearchEventType::DerivedStream), _ => { return Err(Error::new( ErrorKind::Other, diff --git a/src/config/src/config.rs b/src/config/src/config.rs index 884c64ca13..5973cce978 100644 --- a/src/config/src/config.rs +++ b/src/config/src/config.rs @@ -872,6 +872,8 @@ pub struct Limit { pub alert_schedule_timeout: i64, #[env_config(name = "ZO_REPORT_SCHEDULE_TIMEOUT", default = 300)] // seconds pub report_schedule_timeout: i64, + #[env_config(name = "ZO_DERIVED_STREAM_SCHEDULE_INTERVAL", default = 300)] // seconds + pub derived_stream_schedule_interval: i64, #[env_config(name = "ZO_SCHEDULER_MAX_RETRIES", default = 3)] pub scheduler_max_retries: i32, #[env_config(name = "ZO_SCHEDULER_CLEAN_INTERVAL", default = 30)] // seconds diff --git a/src/config/src/meta/cluster.rs b/src/config/src/meta/cluster.rs index d9927560f0..f7654895e3 100644 --- a/src/config/src/meta/cluster.rs +++ b/src/config/src/meta/cluster.rs @@ -164,7 +164,9 @@ impl From<&str> for RoleGroup { impl From for RoleGroup { fn from(value: SearchEventType) -> Self { match value { - SearchEventType::Reports | SearchEventType::Alerts => RoleGroup::Background, + SearchEventType::Reports | SearchEventType::Alerts | SearchEventType::DerivedStream => { + RoleGroup::Background + } _ => RoleGroup::Interactive, } } diff --git a/src/config/src/meta/search.rs b/src/config/src/meta/search.rs index 178e5374eb..d189358086 100644 --- a/src/config/src/meta/search.rs +++ b/src/config/src/meta/search.rs @@ -506,18 +506,20 @@ pub enum SearchEventType { Values, Other, RUM, + DerivedStream, } impl std::fmt::Display for SearchEventType { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - SearchEventType::UI => write!(f, "UI"), - SearchEventType::Dashboards => write!(f, "Dashboards"), - SearchEventType::Reports => write!(f, "Reports"), - SearchEventType::Alerts => write!(f, "Alerts"), - SearchEventType::Other => write!(f, "Other"), + SearchEventType::UI => write!(f, "ui"), + SearchEventType::Dashboards => write!(f, "dashboards"), + SearchEventType::Reports => write!(f, "reports"), + SearchEventType::Alerts => write!(f, "alerts"), SearchEventType::Values => write!(f, "_values"), - SearchEventType::RUM => write!(f, "RUM"), + SearchEventType::Other => write!(f, "other"), + SearchEventType::RUM => write!(f, "rum"), + SearchEventType::DerivedStream => write!(f, "derived_stream"), } } } @@ -531,9 +533,10 @@ impl FromStr for SearchEventType { "dashboards" => Ok(SearchEventType::Dashboards), "reports" => Ok(SearchEventType::Reports), "alerts" => Ok(SearchEventType::Alerts), - "values" => Ok(SearchEventType::Values), + "values" | "_values" => Ok(SearchEventType::Values), "other" => Ok(SearchEventType::Other), "rum" => Ok(SearchEventType::RUM), + "derived_stream" | "derivedstream" => Ok(SearchEventType::DerivedStream), _ => Err(format!("Invalid search event type: {s}")), } } diff --git a/src/config/src/meta/stream.rs b/src/config/src/meta/stream.rs index 4f5be64a88..07f97dceff 100644 --- a/src/config/src/meta/stream.rs +++ b/src/config/src/meta/stream.rs @@ -80,6 +80,20 @@ impl From<&str> for StreamType { } } +impl From for cluster_rpc::StreamType { + fn from(value: StreamType) -> Self { + match value { + StreamType::Logs => cluster_rpc::StreamType::Logs, + StreamType::Metrics => cluster_rpc::StreamType::Metrics, + StreamType::Traces => cluster_rpc::StreamType::Traces, + StreamType::EnrichmentTables => cluster_rpc::StreamType::EnrichmentTables, + StreamType::Filelist => cluster_rpc::StreamType::Filelist, + StreamType::Metadata => cluster_rpc::StreamType::Metadata, + StreamType::Index => cluster_rpc::StreamType::Index, + } + } +} + impl std::fmt::Display for StreamType { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { diff --git a/src/config/src/meta/usage.rs b/src/config/src/meta/usage.rs index 26a7376db3..5a480f27b5 100644 --- a/src/config/src/meta/usage.rs +++ b/src/config/src/meta/usage.rs @@ -41,6 +41,8 @@ pub enum TriggerDataType { Report, #[serde(rename = "alert")] Alert, + #[serde(rename = "derived_stream")] + DerivedStream, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] diff --git a/src/handler/grpc/request/ingest.rs b/src/handler/grpc/request/ingest.rs new file mode 100644 index 0000000000..7c6e60601e --- /dev/null +++ b/src/handler/grpc/request/ingest.rs @@ -0,0 +1,101 @@ +// Copyright 2024 Zinc Labs Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use async_trait::async_trait; +use config::utils::json; +use proto::cluster_rpc::{ingest_server::Ingest, IngestionRequest, IngestionResponse, StreamType}; +use tonic::{Request, Response, Status}; + +use crate::service::ingestion::create_log_ingestion_req; + +#[derive(Debug, Default)] +pub struct Ingester; + +#[async_trait] +impl Ingest for Ingester { + async fn ingest( + &self, + request: Request, + ) -> Result, Status> { + // let metadata = request.metadata().clone(); + let req = request.into_inner(); + let org_id = req.org_id; + let stream_name = req.stream_name; + let in_data = req.data.unwrap_or_default(); + + let resp = match StreamType::try_from(req.stream_type) { + Ok(StreamType::Logs) => { + let log_ingestion_type = req.ingestion_type.unwrap_or_default(); + let data = bytes::Bytes::from(in_data.data); + match create_log_ingestion_req(log_ingestion_type, &data) { + Err(e) => Err(e), + Ok(ingestion_req) => crate::service::logs::ingest::ingest( + &org_id, + &stream_name, + ingestion_req, + "", + None, + ) + .await + .map_or_else(Err, |_| Ok(())), + } + } + Ok(StreamType::EnrichmentTables) => { + let json_records: Vec> = + json::from_slice(&in_data.data).unwrap_or({ + let vec_value: Vec = json::from_slice(&in_data.data).unwrap(); + vec_value + .into_iter() + .filter_map(|v| match v { + json::Value::Object(map) => Some(map), + _ => None, + }) + .collect() + }); + crate::service::enrichment_table::save_enrichment_data( + &org_id, + &stream_name, + json_records, + false, + ) + .await + .map_or_else( + |e| { + Err(anyhow::anyhow!( + "Internal gPRC ingestion service errors saving enrichment data: {}", + e.to_string() + )) + }, + |_| Ok(()), + ) + } + _ => Err(anyhow::anyhow!( + "Internal gPRC ingestion service currently only supports Logs and EnrichmentTables", + )), + }; + + let reply = match resp { + Ok(_) => IngestionResponse { + status_code: 200, + message: "OK".to_string(), + }, + Err(err) => IngestionResponse { + status_code: 500, + message: err.to_string(), + }, + }; + Ok(Response::new(reply)) + } +} diff --git a/src/handler/grpc/request/metrics/ingester.rs b/src/handler/grpc/request/metrics/ingester.rs index 1829e260a9..0b953b6433 100644 --- a/src/handler/grpc/request/metrics/ingester.rs +++ b/src/handler/grpc/request/metrics/ingester.rs @@ -21,10 +21,10 @@ use opentelemetry_proto::tonic::collector::metrics::v1::{ use tonic::{Response, Status}; #[derive(Default)] -pub struct Ingester; +pub struct MetricsIngester; #[async_trait] -impl MetricsService for Ingester { +impl MetricsService for MetricsIngester { async fn export( &self, request: tonic::Request, diff --git a/src/handler/grpc/request/metrics/querier.rs b/src/handler/grpc/request/metrics/querier.rs index 94fa944444..9909293b25 100644 --- a/src/handler/grpc/request/metrics/querier.rs +++ b/src/handler/grpc/request/metrics/querier.rs @@ -41,10 +41,10 @@ use crate::{ service::{promql::search as SearchService, search::match_source}, }; -pub struct Querier; +pub struct MetricsQuerier; #[tonic::async_trait] -impl Metrics for Querier { +impl Metrics for MetricsQuerier { #[tracing::instrument(name = "grpc:metrics:query", skip_all, fields(org_id = req.get_ref().org_id))] async fn query( &self, diff --git a/src/handler/grpc/request/mod.rs b/src/handler/grpc/request/mod.rs index bc9ea36b91..c96c638895 100644 --- a/src/handler/grpc/request/mod.rs +++ b/src/handler/grpc/request/mod.rs @@ -17,6 +17,7 @@ use opentelemetry::propagation::Extractor; pub mod event; pub mod file_list; +pub mod ingest; pub mod logs; pub mod metrics; pub mod query_cache; diff --git a/src/handler/http/request/alerts/alert.rs b/src/handler/http/request/alerts/alert.rs new file mode 100644 index 0000000000..ed16238063 --- /dev/null +++ b/src/handler/http/request/alerts/alert.rs @@ -0,0 +1,373 @@ +// Copyright 2024 Zinc Labs Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::{collections::HashMap, io::Error}; + +use actix_web::{delete, get, http, post, put, web, HttpRequest, HttpResponse}; + +use crate::{ + common::{ + meta::{alerts::alert::Alert, http::HttpResponse as MetaHttpResponse}, + utils::http::get_stream_type_from_request, + }, + service::alerts::alert, +}; + +/// CreateAlert +#[utoipa::path( + context_path = "/api", + tag = "Alerts", + operation_id = "SaveAlert", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ("stream_name" = String, Path, description = "Stream name"), + ), + request_body(content = Alert, description = "Alert data", content_type = "application/json"), + responses( + (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), + (status = 400, description = "Error", content_type = "application/json", body = HttpResponse), + ) +)] +#[post("/{org_id}/{stream_name}/alerts")] +pub async fn save_alert( + path: web::Path<(String, String)>, + alert: web::Json, +) -> Result { + let (org_id, stream_name) = path.into_inner(); + + // Hack for frequency: convert minutes to seconds + let mut alert = alert.into_inner(); + alert.trigger_condition.frequency *= 60; + + match alert::save(&org_id, &stream_name, "", alert, true).await { + Ok(_) => Ok(MetaHttpResponse::ok("Alert saved")), + Err(e) => Ok(MetaHttpResponse::bad_request(e)), + } +} + +/// UpdateAlert +#[utoipa::path( + context_path = "/api", + tag = "Alerts", + operation_id = "UpdateAlert", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ("stream_name" = String, Path, description = "Stream name"), + ("alert_name" = String, Path, description = "Alert name"), + ), + request_body(content = Alert, description = "Alert data", content_type = "application/json"), + responses( + (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), + (status = 400, description = "Error", content_type = "application/json", body = HttpResponse), + ) +)] +#[put("/{org_id}/{stream_name}/alerts/{alert_name}")] +pub async fn update_alert( + path: web::Path<(String, String, String)>, + alert: web::Json, +) -> Result { + let (org_id, stream_name, name) = path.into_inner(); + + // Hack for frequency: convert minutes to seconds + let mut alert = alert.into_inner(); + alert.trigger_condition.frequency *= 60; + match alert::save(&org_id, &stream_name, &name, alert, false).await { + Ok(_) => Ok(MetaHttpResponse::ok("Alert Updated")), + Err(e) => Ok(MetaHttpResponse::bad_request(e)), + } +} + +/// ListStreamAlerts +#[utoipa::path( + context_path = "/api", + tag = "Alerts", + operation_id = "ListStreamAlerts", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ("stream_name" = String, Path, description = "Stream name"), + ), + responses( + (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), + (status = 400, description = "Error", content_type = "application/json", body = HttpResponse), + ) +)] +#[get("/{org_id}/{stream_name}/alerts")] +async fn list_stream_alerts( + path: web::Path<(String, String)>, + req: HttpRequest, +) -> Result { + let (org_id, stream_name) = path.into_inner(); + let query = web::Query::>::from_query(req.query_string()).unwrap(); + let stream_type = match get_stream_type_from_request(&query) { + Ok(v) => v, + Err(e) => { + return Ok(MetaHttpResponse::bad_request(e)); + } + }; + match alert::list(&org_id, stream_type, Some(stream_name.as_str()), None).await { + Ok(mut data) => { + // Hack for frequency: convert seconds to minutes + for alert in data.iter_mut() { + alert.trigger_condition.frequency /= 60; + } + + let mut mapdata = HashMap::new(); + mapdata.insert("list", data); + Ok(MetaHttpResponse::json(mapdata)) + } + Err(e) => Ok(MetaHttpResponse::bad_request(e)), + } +} + +/// ListAlerts +#[utoipa::path( + context_path = "/api", + tag = "Alerts", + operation_id = "ListAlerts", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ), + responses( + (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), + ) +)] +#[get("/{org_id}/alerts")] +async fn list_alerts(path: web::Path, _req: HttpRequest) -> Result { + let org_id = path.into_inner(); + + let mut _alert_list_from_rbac = None; + // Get List of allowed objects + #[cfg(feature = "enterprise")] + { + let user_id = _req.headers().get("user_id").unwrap(); + match crate::handler::http::auth::validator::list_objects_for_user( + &org_id, + user_id.to_str().unwrap(), + "GET", + "alert", + ) + .await + { + Ok(stream_list) => { + _alert_list_from_rbac = stream_list; + } + Err(e) => { + return Ok(crate::common::meta::http::HttpResponse::forbidden( + e.to_string(), + )); + } + } + // Get List of allowed objects ends + } + + match alert::list(&org_id, None, None, _alert_list_from_rbac).await { + Ok(mut data) => { + // Hack for frequency: convert seconds to minutes + for alert in data.iter_mut() { + alert.trigger_condition.frequency /= 60; + } + + let mut mapdata = HashMap::new(); + mapdata.insert("list", data); + Ok(MetaHttpResponse::json(mapdata)) + } + Err(e) => Ok(MetaHttpResponse::bad_request(e)), + } +} + +/// GetAlertByName +#[utoipa::path( + context_path = "/api", + tag = "Alerts", + operation_id = "GetAlert", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ("stream_name" = String, Path, description = "Stream name"), + ("alert_name" = String, Path, description = "Alert name"), + ), + responses( + (status = 200, description = "Success", content_type = "application/json", body = Alert), + (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse), + ) +)] +#[get("/{org_id}/{stream_name}/alerts/{alert_name}")] +async fn get_alert( + path: web::Path<(String, String, String)>, + req: HttpRequest, +) -> Result { + let (org_id, stream_name, name) = path.into_inner(); + let query = web::Query::>::from_query(req.query_string()).unwrap(); + let stream_type = match get_stream_type_from_request(&query) { + Ok(v) => v.unwrap_or_default(), + Err(e) => { + return Ok(MetaHttpResponse::bad_request(e)); + } + }; + match alert::get(&org_id, stream_type, &stream_name, &name).await { + Ok(mut data) => { + // Hack for frequency: convert seconds to minutes + if let Some(ref mut data) = data { + data.trigger_condition.frequency /= 60; + } + Ok(MetaHttpResponse::json(data)) + } + Err(e) => Ok(MetaHttpResponse::not_found(e)), + } +} + +/// DeleteAlert +#[utoipa::path( + context_path = "/api", + tag = "Alerts", + operation_id = "DeleteAlert", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ("stream_name" = String, Path, description = "Stream name"), + ("alert_name" = String, Path, description = "Alert name"), + ), + responses( + (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), + (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse), + (status = 500, description = "Failure", content_type = "application/json", body = HttpResponse), + ) +)] +#[delete("/{org_id}/{stream_name}/alerts/{alert_name}")] +async fn delete_alert( + path: web::Path<(String, String, String)>, + req: HttpRequest, +) -> Result { + let (org_id, stream_name, name) = path.into_inner(); + let query = web::Query::>::from_query(req.query_string()).unwrap(); + let stream_type = match get_stream_type_from_request(&query) { + Ok(v) => v.unwrap_or_default(), + Err(e) => { + return Ok(MetaHttpResponse::bad_request(e)); + } + }; + match alert::delete(&org_id, stream_type, &stream_name, &name).await { + Ok(_) => Ok(MetaHttpResponse::ok("Alert deleted")), + Err(e) => match e { + (http::StatusCode::NOT_FOUND, e) => Ok(MetaHttpResponse::not_found(e)), + (_, e) => Ok(MetaHttpResponse::internal_error(e)), + }, + } +} + +/// EnableAlert +#[utoipa::path( + context_path = "/api", + tag = "Alerts", + operation_id = "EnableAlert", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ("stream_name" = String, Path, description = "Stream name"), + ("alert_name" = String, Path, description = "Alert name"), + ("value" = bool, Query, description = "Enable or disable alert"), + ), + responses( + (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), + (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse), + (status = 500, description = "Failure", content_type = "application/json", body = HttpResponse), + ) +)] +#[put("/{org_id}/{stream_name}/alerts/{alert_name}/enable")] +async fn enable_alert( + path: web::Path<(String, String, String)>, + req: HttpRequest, +) -> Result { + let (org_id, stream_name, name) = path.into_inner(); + let query = web::Query::>::from_query(req.query_string()).unwrap(); + let stream_type = match get_stream_type_from_request(&query) { + Ok(v) => v.unwrap_or_default(), + Err(e) => { + return Ok(MetaHttpResponse::bad_request(e)); + } + }; + let enable = match query.get("value") { + Some(v) => v.parse::().unwrap_or_default(), + None => false, + }; + let mut resp = HashMap::new(); + resp.insert("enabled".to_string(), enable); + match alert::enable(&org_id, stream_type, &stream_name, &name, enable).await { + Ok(_) => Ok(MetaHttpResponse::json(resp)), + Err(e) => match e { + (http::StatusCode::NOT_FOUND, e) => Ok(MetaHttpResponse::not_found(e)), + (_, e) => Ok(MetaHttpResponse::internal_error(e)), + }, + } +} + +/// TriggerAlert +#[utoipa::path( + context_path = "/api", + tag = "Alerts", + operation_id = "TriggerAlert", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ("stream_name" = String, Path, description = "Stream name"), + ("alert_name" = String, Path, description = "Alert name"), + ), + responses( + (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), + (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse), + (status = 500, description = "Failure", content_type = "application/json", body = HttpResponse), + ) +)] +#[put("/{org_id}/{stream_name}/alerts/{alert_name}/trigger")] +async fn trigger_alert( + path: web::Path<(String, String, String)>, + req: HttpRequest, +) -> Result { + let (org_id, stream_name, name) = path.into_inner(); + let query = web::Query::>::from_query(req.query_string()).unwrap(); + let stream_type = match get_stream_type_from_request(&query) { + Ok(v) => v.unwrap_or_default(), + Err(e) => { + return Ok(MetaHttpResponse::bad_request(e)); + } + }; + match alert::trigger(&org_id, stream_type, &stream_name, &name).await { + Ok(_) => Ok(MetaHttpResponse::ok("Alert triggered")), + Err(e) => match e { + (http::StatusCode::NOT_FOUND, e) => Ok(MetaHttpResponse::not_found(e)), + (_, e) => Ok(MetaHttpResponse::internal_error(e)), + }, + } +} diff --git a/src/handler/http/request/alerts/mod.rs b/src/handler/http/request/alerts/mod.rs index 603dcf532f..3ad68947cd 100644 --- a/src/handler/http/request/alerts/mod.rs +++ b/src/handler/http/request/alerts/mod.rs @@ -13,364 +13,6 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{collections::HashMap, io::Error}; - -use actix_web::{delete, get, http, post, put, web, HttpRequest, HttpResponse}; - -use crate::{ - common::{ - meta::{alerts::Alert, http::HttpResponse as MetaHttpResponse}, - utils::http::get_stream_type_from_request, - }, - service::alerts, -}; - +pub mod alert; pub mod destinations; pub mod templates; - -/// CreateAlert -#[utoipa::path( - context_path = "/api", - tag = "Alerts", - operation_id = "SaveAlert", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ("stream_name" = String, Path, description = "Stream name"), - ), - request_body(content = Alert, description = "Alert data", content_type = "application/json"), - responses( - (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), - (status = 400, description = "Error", content_type = "application/json", body = HttpResponse), - ) -)] -#[post("/{org_id}/{stream_name}/alerts")] -pub async fn save_alert( - path: web::Path<(String, String)>, - alert: web::Json, -) -> Result { - let (org_id, stream_name) = path.into_inner(); - - // Hack for frequency: convert minutes to seconds - let mut alert = alert.into_inner(); - alert.trigger_condition.frequency *= 60; - - match alerts::save(&org_id, &stream_name, "", alert, true).await { - Ok(_) => Ok(MetaHttpResponse::ok("Alert saved")), - Err(e) => Ok(MetaHttpResponse::bad_request(e)), - } -} - -/// UpdateAlert -#[utoipa::path( - context_path = "/api", - tag = "Alerts", - operation_id = "UpdateAlert", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ("stream_name" = String, Path, description = "Stream name"), - ("alert_name" = String, Path, description = "Alert name"), - ), - request_body(content = Alert, description = "Alert data", content_type = "application/json"), - responses( - (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), - (status = 400, description = "Error", content_type = "application/json", body = HttpResponse), - ) -)] -#[put("/{org_id}/{stream_name}/alerts/{alert_name}")] -pub async fn update_alert( - path: web::Path<(String, String, String)>, - alert: web::Json, -) -> Result { - let (org_id, stream_name, name) = path.into_inner(); - - // Hack for frequency: convert minutes to seconds - let mut alert = alert.into_inner(); - alert.trigger_condition.frequency *= 60; - match alerts::save(&org_id, &stream_name, &name, alert, false).await { - Ok(_) => Ok(MetaHttpResponse::ok("Alert Updated")), - Err(e) => Ok(MetaHttpResponse::bad_request(e)), - } -} - -/// ListStreamAlerts -#[utoipa::path( - context_path = "/api", - tag = "Alerts", - operation_id = "ListStreamAlerts", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ("stream_name" = String, Path, description = "Stream name"), - ), - responses( - (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), - (status = 400, description = "Error", content_type = "application/json", body = HttpResponse), - ) -)] -#[get("/{org_id}/{stream_name}/alerts")] -async fn list_stream_alerts( - path: web::Path<(String, String)>, - req: HttpRequest, -) -> Result { - let (org_id, stream_name) = path.into_inner(); - let query = web::Query::>::from_query(req.query_string()).unwrap(); - let stream_type = match get_stream_type_from_request(&query) { - Ok(v) => v, - Err(e) => { - return Ok(MetaHttpResponse::bad_request(e)); - } - }; - match alerts::list(&org_id, stream_type, Some(stream_name.as_str()), None).await { - Ok(mut data) => { - // Hack for frequency: convert seconds to minutes - for alert in data.iter_mut() { - alert.trigger_condition.frequency /= 60; - } - - let mut mapdata = HashMap::new(); - mapdata.insert("list", data); - Ok(MetaHttpResponse::json(mapdata)) - } - Err(e) => Ok(MetaHttpResponse::bad_request(e)), - } -} - -/// ListAlerts -#[utoipa::path( - context_path = "/api", - tag = "Alerts", - operation_id = "ListAlerts", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ), - responses( - (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), - ) -)] -#[get("/{org_id}/alerts")] -async fn list_alerts(path: web::Path, _req: HttpRequest) -> Result { - let org_id = path.into_inner(); - - let mut _alert_list_from_rbac = None; - // Get List of allowed objects - #[cfg(feature = "enterprise")] - { - let user_id = _req.headers().get("user_id").unwrap(); - match crate::handler::http::auth::validator::list_objects_for_user( - &org_id, - user_id.to_str().unwrap(), - "GET", - "alert", - ) - .await - { - Ok(stream_list) => { - _alert_list_from_rbac = stream_list; - } - Err(e) => { - return Ok(crate::common::meta::http::HttpResponse::forbidden( - e.to_string(), - )); - } - } - // Get List of allowed objects ends - } - - match alerts::list(&org_id, None, None, _alert_list_from_rbac).await { - Ok(mut data) => { - // Hack for frequency: convert seconds to minutes - for alert in data.iter_mut() { - alert.trigger_condition.frequency /= 60; - } - - let mut mapdata = HashMap::new(); - mapdata.insert("list", data); - Ok(MetaHttpResponse::json(mapdata)) - } - Err(e) => Ok(MetaHttpResponse::bad_request(e)), - } -} - -/// GetAlertByName -#[utoipa::path( - context_path = "/api", - tag = "Alerts", - operation_id = "GetAlert", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ("stream_name" = String, Path, description = "Stream name"), - ("alert_name" = String, Path, description = "Alert name"), - ), - responses( - (status = 200, description = "Success", content_type = "application/json", body = Alert), - (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse), - ) -)] -#[get("/{org_id}/{stream_name}/alerts/{alert_name}")] -async fn get_alert( - path: web::Path<(String, String, String)>, - req: HttpRequest, -) -> Result { - let (org_id, stream_name, name) = path.into_inner(); - let query = web::Query::>::from_query(req.query_string()).unwrap(); - let stream_type = match get_stream_type_from_request(&query) { - Ok(v) => v.unwrap_or_default(), - Err(e) => { - return Ok(MetaHttpResponse::bad_request(e)); - } - }; - match alerts::get(&org_id, stream_type, &stream_name, &name).await { - Ok(mut data) => { - // Hack for frequency: convert seconds to minutes - if let Some(ref mut data) = data { - data.trigger_condition.frequency /= 60; - } - Ok(MetaHttpResponse::json(data)) - } - Err(e) => Ok(MetaHttpResponse::not_found(e)), - } -} - -/// DeleteAlert -#[utoipa::path( - context_path = "/api", - tag = "Alerts", - operation_id = "DeleteAlert", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ("stream_name" = String, Path, description = "Stream name"), - ("alert_name" = String, Path, description = "Alert name"), - ), - responses( - (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), - (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse), - (status = 500, description = "Failure", content_type = "application/json", body = HttpResponse), - ) -)] -#[delete("/{org_id}/{stream_name}/alerts/{alert_name}")] -async fn delete_alert( - path: web::Path<(String, String, String)>, - req: HttpRequest, -) -> Result { - let (org_id, stream_name, name) = path.into_inner(); - let query = web::Query::>::from_query(req.query_string()).unwrap(); - let stream_type = match get_stream_type_from_request(&query) { - Ok(v) => v.unwrap_or_default(), - Err(e) => { - return Ok(MetaHttpResponse::bad_request(e)); - } - }; - match alerts::delete(&org_id, stream_type, &stream_name, &name).await { - Ok(_) => Ok(MetaHttpResponse::ok("Alert deleted")), - Err(e) => match e { - (http::StatusCode::NOT_FOUND, e) => Ok(MetaHttpResponse::not_found(e)), - (_, e) => Ok(MetaHttpResponse::internal_error(e)), - }, - } -} - -/// EnableAlert -#[utoipa::path( - context_path = "/api", - tag = "Alerts", - operation_id = "EnableAlert", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ("stream_name" = String, Path, description = "Stream name"), - ("alert_name" = String, Path, description = "Alert name"), - ("value" = bool, Query, description = "Enable or disable alert"), - ), - responses( - (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), - (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse), - (status = 500, description = "Failure", content_type = "application/json", body = HttpResponse), - ) -)] -#[put("/{org_id}/{stream_name}/alerts/{alert_name}/enable")] -async fn enable_alert( - path: web::Path<(String, String, String)>, - req: HttpRequest, -) -> Result { - let (org_id, stream_name, name) = path.into_inner(); - let query = web::Query::>::from_query(req.query_string()).unwrap(); - let stream_type = match get_stream_type_from_request(&query) { - Ok(v) => v.unwrap_or_default(), - Err(e) => { - return Ok(MetaHttpResponse::bad_request(e)); - } - }; - let enable = match query.get("value") { - Some(v) => v.parse::().unwrap_or_default(), - None => false, - }; - let mut resp = HashMap::new(); - resp.insert("enabled".to_string(), enable); - match alerts::enable(&org_id, stream_type, &stream_name, &name, enable).await { - Ok(_) => Ok(MetaHttpResponse::json(resp)), - Err(e) => match e { - (http::StatusCode::NOT_FOUND, e) => Ok(MetaHttpResponse::not_found(e)), - (_, e) => Ok(MetaHttpResponse::internal_error(e)), - }, - } -} - -/// TriggerAlert -#[utoipa::path( - context_path = "/api", - tag = "Alerts", - operation_id = "TriggerAlert", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ("stream_name" = String, Path, description = "Stream name"), - ("alert_name" = String, Path, description = "Alert name"), - ), - responses( - (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), - (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse), - (status = 500, description = "Failure", content_type = "application/json", body = HttpResponse), - ) -)] -#[put("/{org_id}/{stream_name}/alerts/{alert_name}/trigger")] -async fn trigger_alert( - path: web::Path<(String, String, String)>, - req: HttpRequest, -) -> Result { - let (org_id, stream_name, name) = path.into_inner(); - let query = web::Query::>::from_query(req.query_string()).unwrap(); - let stream_type = match get_stream_type_from_request(&query) { - Ok(v) => v.unwrap_or_default(), - Err(e) => { - return Ok(MetaHttpResponse::bad_request(e)); - } - }; - match alerts::trigger(&org_id, stream_type, &stream_name, &name).await { - Ok(_) => Ok(MetaHttpResponse::ok("Alert triggered")), - Err(e) => match e { - (http::StatusCode::NOT_FOUND, e) => Ok(MetaHttpResponse::not_found(e)), - (_, e) => Ok(MetaHttpResponse::internal_error(e)), - }, - } -} diff --git a/src/handler/http/request/enrichment_table/mod.rs b/src/handler/http/request/enrichment_table/mod.rs index ab878af084..ba327ec932 100644 --- a/src/handler/http/request/enrichment_table/mod.rs +++ b/src/handler/http/request/enrichment_table/mod.rs @@ -22,7 +22,7 @@ use hashbrown::HashMap; use crate::{ common::meta::http::HttpResponse as MetaHttpResponse, - service::enrichment_table::save_enrichment_data, + service::enrichment_table::{extract_multipart, save_enrichment_data}, }; /// CreateEnrichmentTable @@ -81,7 +81,8 @@ pub async fn save_enrichment_table( Some(append_data) => append_data.parse::().unwrap_or(false), None => false, }; - save_enrichment_data(&org_id, &table_name, payload, append_data).await + let json_record = extract_multipart(payload).await?; + save_enrichment_data(&org_id, &table_name, json_record, append_data).await } else { Ok(MetaHttpResponse::bad_request( "Bad Request, content-type must be multipart/form-data", diff --git a/src/handler/http/request/pipelines.rs b/src/handler/http/request/pipelines.rs index 4fdfeaadd1..b698b170b0 100644 --- a/src/handler/http/request/pipelines.rs +++ b/src/handler/http/request/pipelines.rs @@ -49,11 +49,11 @@ pub async fn save_pipeline( req: HttpRequest, ) -> Result { let (org_id, stream_name) = path.into_inner(); + let query = web::Query::>::from_query(req.query_string()).unwrap(); let mut pipeline = pipeline.into_inner(); pipeline.name = pipeline.name.trim().to_string(); pipeline.stream_name = stream_name; - let query = web::Query::>::from_query(req.query_string()).unwrap(); - let stream_type = match get_stream_type_from_request(&query) { + pipeline.stream_type = match get_stream_type_from_request(&query) { Ok(v) => v.unwrap_or_default(), Err(e) => { return Ok( @@ -80,7 +80,6 @@ pub async fn save_pipeline( routing.insert(formatted_key, value); } } - pipeline.stream_type = stream_type; crate::service::pipelines::save_pipeline(org_id, pipeline).await } @@ -162,7 +161,8 @@ async fn delete_pipeline( return Ok(crate::common::meta::http::HttpResponse::bad_request(e)); } }; - crate::service::pipelines::delete_pipeline(&org_id, stream_type, &stream_name, &name).await + let pipeline_source = meta::stream::StreamParams::new(&org_id, &stream_name, stream_type); + crate::service::pipelines::delete_pipeline(&name, pipeline_source).await } /// UpdatePipeline @@ -200,8 +200,8 @@ pub async fn update_pipeline( let name = name.trim(); let mut pipeline = pipeline.into_inner(); pipeline.name = name.to_string(); - pipeline.stream_name = stream_name; pipeline.stream_type = stream_type; + pipeline.stream_name = stream_name; if let Some(ref mut routing) = &mut pipeline.routing { let keys_to_update: Vec<_> = routing.keys().cloned().collect(); @@ -219,5 +219,5 @@ pub async fn update_pipeline( routing.insert(formatted_key, value); } } - crate::service::pipelines::update_pipeline(&org_id, name, pipeline).await + crate::service::pipelines::update_pipeline(&org_id, pipeline).await } diff --git a/src/handler/http/router/mod.rs b/src/handler/http/router/mod.rs index 84f6bcd27b..d0b5dca8e1 100644 --- a/src/handler/http/router/mod.rs +++ b/src/handler/http/router/mod.rs @@ -420,14 +420,14 @@ pub fn get_service_routes(cfg: &mut web::ServiceConfig) { .service(dashboards::reports::delete_report) .service(dashboards::reports::enable_report) .service(dashboards::reports::trigger_report) - .service(alerts::save_alert) - .service(alerts::update_alert) - .service(alerts::get_alert) - .service(alerts::list_alerts) - .service(alerts::list_stream_alerts) - .service(alerts::delete_alert) - .service(alerts::enable_alert) - .service(alerts::trigger_alert) + .service(alerts::alert::save_alert) + .service(alerts::alert::update_alert) + .service(alerts::alert::get_alert) + .service(alerts::alert::list_alerts) + .service(alerts::alert::list_stream_alerts) + .service(alerts::alert::delete_alert) + .service(alerts::alert::enable_alert) + .service(alerts::alert::trigger_alert) .service(alerts::templates::save_template) .service(alerts::templates::update_template) .service(alerts::templates::get_template) diff --git a/src/handler/http/router/openapi.rs b/src/handler/http/router/openapi.rs index cad9ff0f39..c6864c11f4 100644 --- a/src/handler/http/router/openapi.rs +++ b/src/handler/http/router/openapi.rs @@ -86,14 +86,14 @@ use crate::{common::meta, handler::http::request}; request::dashboards::folders::get_folder, request::dashboards::folders::update_folder, request::dashboards::move_dashboard, - request::alerts::save_alert, - request::alerts::update_alert, - request::alerts::list_stream_alerts, - request::alerts::list_alerts, - request::alerts::get_alert, - request::alerts::delete_alert, - request::alerts::enable_alert, - request::alerts::trigger_alert, + request::alerts::alert::save_alert, + request::alerts::alert::update_alert, + request::alerts::alert::list_stream_alerts, + request::alerts::alert::list_alerts, + request::alerts::alert::get_alert, + request::alerts::alert::delete_alert, + request::alerts::alert::enable_alert, + request::alerts::alert::trigger_alert, request::alerts::templates::list_templates, request::alerts::templates::get_template, request::alerts::templates::save_template, @@ -168,14 +168,14 @@ use crate::{common::meta, handler::http::request}; meta::saved_view::DeleteViewResponse, meta::saved_view::CreateViewResponse, meta::saved_view::UpdateViewRequest, - meta::alerts::Alert, + meta::alerts::alert::Alert, meta::alerts::Condition, meta::alerts::Operator, meta::alerts::Aggregation, meta::alerts::AggFunction, meta::alerts::QueryType, meta::alerts::TriggerCondition, - meta::alerts::AlertFrequencyType, + meta::alerts::FrequencyType, meta::alerts::QueryCondition, meta::alerts::destinations::Destination, meta::alerts::destinations::DestinationWithTemplate, diff --git a/src/infra/src/scheduler/mod.rs b/src/infra/src/scheduler/mod.rs index 6427cddae0..a5bed44931 100644 --- a/src/infra/src/scheduler/mod.rs +++ b/src/infra/src/scheduler/mod.rs @@ -81,6 +81,7 @@ pub enum TriggerModule { Report, #[default] Alert, + DerivedStream, } impl std::fmt::Display for TriggerModule { @@ -88,6 +89,7 @@ impl std::fmt::Display for TriggerModule { match self { TriggerModule::Alert => write!(f, "alert"), TriggerModule::Report => write!(f, "report"), + TriggerModule::DerivedStream => write!(f, "derived_stream"), } } } diff --git a/src/job/alert_manager.rs b/src/job/alert_manager.rs index 0d511877e9..cd0e133812 100644 --- a/src/job/alert_manager.rs +++ b/src/job/alert_manager.rs @@ -70,7 +70,7 @@ async fn run_schedule_jobs() -> Result<(), anyhow::Error> { interval.tick().await; // trigger the first run loop { interval.tick().await; - if let Err(e) = service::alerts::alert_manager::run().await { + if let Err(e) = service::alerts::scheduler::run().await { log::error!("[ALERT MANAGER] run schedule jobs error: {}", e); } } diff --git a/src/job/mod.rs b/src/job/mod.rs index da94d81c96..82037a30e6 100644 --- a/src/job/mod.rs +++ b/src/job/mod.rs @@ -109,7 +109,7 @@ pub async fn init() -> Result<(), anyhow::Error> { tokio::task::spawn(async move { db::alerts::templates::watch().await }); tokio::task::spawn(async move { db::alerts::destinations::watch().await }); tokio::task::spawn(async move { db::alerts::realtime_triggers::watch().await }); - tokio::task::spawn(async move { db::alerts::watch().await }); + tokio::task::spawn(async move { db::alerts::alert::watch().await }); tokio::task::spawn(async move { db::dashboards::reports::watch().await }); tokio::task::spawn(async move { db::organization::watch().await }); #[cfg(feature = "enterprise")] @@ -147,7 +147,9 @@ pub async fn init() -> Result<(), anyhow::Error> { db::alerts::realtime_triggers::cache() .await .expect("alerts realtime triggers cache failed"); - db::alerts::cache().await.expect("alerts cache failed"); + db::alerts::alert::cache() + .await + .expect("alerts cache failed"); db::dashboards::reports::cache() .await .expect("reports cache failed"); diff --git a/src/main.rs b/src/main.rs index ab127bbd71..b89590a498 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,8 +32,9 @@ use openobserve::{ request::{ event::Eventer, file_list::Filelister, + ingest::Ingester, logs::LogsServer, - metrics::{ingester::Ingester, querier::Querier}, + metrics::{ingester::MetricsIngester, querier::MetricsQuerier}, query_cache::QueryCacheServerImpl, traces::TraceServer, usage::UsageServerImpl, @@ -53,8 +54,9 @@ use opentelemetry_proto::tonic::collector::{ }; use opentelemetry_sdk::{propagation::TraceContextPropagator, trace as sdktrace, Resource}; use proto::cluster_rpc::{ - event_server::EventServer, filelist_server::FilelistServer, metrics_server::MetricsServer, - query_cache_server::QueryCacheServer, search_server::SearchServer, usage_server::UsageServer, + event_server::EventServer, filelist_server::FilelistServer, ingest_server::IngestServer, + metrics_server::MetricsServer, query_cache_server::QueryCacheServer, + search_server::SearchServer, usage_server::UsageServer, }; #[cfg(feature = "profiling")] use pyroscope::PyroscopeAgent; @@ -380,10 +382,10 @@ async fn init_common_grpc_server( let filelist_svc = FilelistServer::new(Filelister) .send_compressed(CompressionEncoding::Gzip) .accept_compressed(CompressionEncoding::Gzip); - let metrics_svc = MetricsServer::new(Querier) + let metrics_svc = MetricsServer::new(MetricsQuerier) .send_compressed(CompressionEncoding::Gzip) .accept_compressed(CompressionEncoding::Gzip); - let metrics_ingest_svc = MetricsServiceServer::new(Ingester) + let metrics_ingest_svc = MetricsServiceServer::new(MetricsIngester) .send_compressed(CompressionEncoding::Gzip) .accept_compressed(CompressionEncoding::Gzip); let usage_svc = UsageServer::new(UsageServerImpl) @@ -399,6 +401,9 @@ async fn init_common_grpc_server( let query_cache_svc = QueryCacheServer::new(QueryCacheServerImpl) .send_compressed(CompressionEncoding::Gzip) .accept_compressed(CompressionEncoding::Gzip); + let ingest_svc = IngestServer::new(Ingester) + .send_compressed(CompressionEncoding::Gzip) + .accept_compressed(CompressionEncoding::Gzip); log::info!("starting gRPC server at {}", gaddr); init_tx.send(()).ok(); @@ -413,6 +418,7 @@ async fn init_common_grpc_server( .add_service(usage_svc) .add_service(logs_svc) .add_service(query_cache_svc) + .add_service(ingest_svc) .serve_with_shutdown(gaddr, async { shutdown_rx.await.ok(); log::info!("gRPC server starts shutting down"); diff --git a/src/proto/build.rs b/src/proto/build.rs index 023c9ef9a9..e51b8fcf9e 100644 --- a/src/proto/build.rs +++ b/src/proto/build.rs @@ -58,6 +58,7 @@ fn main() -> Result<()> { "proto/cluster/filelist.proto", "proto/cluster/metrics.proto", "proto/cluster/search.proto", + "proto/cluster/ingest.proto", "proto/cluster/usage.proto", "proto/cluster/querycache.proto", "proto/cluster/plan.proto", diff --git a/src/proto/proto/cluster/common.proto b/src/proto/proto/cluster/common.proto index 853676de49..8085a18526 100644 --- a/src/proto/proto/cluster/common.proto +++ b/src/proto/proto/cluster/common.proto @@ -23,12 +23,14 @@ message FileMeta { } enum StreamType { - LOGS = 0; - METRICS = 1; - TRACES = 2; - METADATA = 3; - FILELIST = 4; - INDEX = 5; + LOGS = 0; + METRICS = 1; + TRACES = 2; + METADATA = 3; + EnrichmentTables = 4; + FILELIST = 5; + INDEX = 6; + } // Search request type diff --git a/src/proto/proto/cluster/ingest.proto b/src/proto/proto/cluster/ingest.proto new file mode 100644 index 0000000000..bc592ee603 --- /dev/null +++ b/src/proto/proto/cluster/ingest.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "org.openobserve.cluster"; +option java_outer_classname = "ingestProto"; + +package cluster; + +import "cluster/common.proto"; + +service Ingest { + rpc Ingest (IngestionRequest) returns (IngestionResponse) {} +} + +message IngestionData { + bytes data = 1; +} + +message IngestionRequest { + string org_id = 1; + string stream_name = 2; + IngestionData data = 3; + StreamType stream_type = 4; + optional IngestionType ingestion_type = 5; +} + +enum IngestionType { + JSON = 0; + MULTI = 1; + GCP = 2; + KINESISFH = 3; + RUM = 4; + USAGE = 5; +} + +message IngestionResponse { + int32 status_code = 1; + string message = 2; +} \ No newline at end of file diff --git a/src/proto/src/generated/cluster.rs b/src/proto/src/generated/cluster.rs index aefb6761eb..d6cc74d010 100644 --- a/src/proto/src/generated/cluster.rs +++ b/src/proto/src/generated/cluster.rs @@ -96,8 +96,9 @@ pub enum StreamType { Metrics = 1, Traces = 2, Metadata = 3, - Filelist = 4, - Index = 5, + EnrichmentTables = 4, + Filelist = 5, + Index = 6, } impl StreamType { /// String value of the enum field names used in the ProtoBuf definition. @@ -110,6 +111,7 @@ impl StreamType { StreamType::Metrics => "METRICS", StreamType::Traces => "TRACES", StreamType::Metadata => "METADATA", + StreamType::EnrichmentTables => "EnrichmentTables", StreamType::Filelist => "FILELIST", StreamType::Index => "INDEX", } @@ -121,6 +123,7 @@ impl StreamType { "METRICS" => Some(Self::Metrics), "TRACES" => Some(Self::Traces), "METADATA" => Some(Self::Metadata), + "EnrichmentTables" => Some(Self::EnrichmentTables), "FILELIST" => Some(Self::Filelist), "INDEX" => Some(Self::Index), _ => None, @@ -1926,6 +1929,338 @@ pub mod search_server { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionData { + #[prost(bytes = "vec", tag = "1")] + pub data: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionRequest { + #[prost(string, tag = "1")] + pub org_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub stream_name: ::prost::alloc::string::String, + #[prost(message, optional, tag = "3")] + pub data: ::core::option::Option, + #[prost(enumeration = "StreamType", tag = "4")] + pub stream_type: i32, + #[prost(enumeration = "IngestionType", optional, tag = "5")] + pub ingestion_type: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionResponse { + #[prost(int32, tag = "1")] + pub status_code: i32, + #[prost(string, tag = "2")] + pub message: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum IngestionType { + Json = 0, + Multi = 1, + Gcp = 2, + Kinesisfh = 3, + Rum = 4, + Usage = 5, +} +impl IngestionType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + IngestionType::Json => "JSON", + IngestionType::Multi => "MULTI", + IngestionType::Gcp => "GCP", + IngestionType::Kinesisfh => "KINESISFH", + IngestionType::Rum => "RUM", + IngestionType::Usage => "USAGE", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "JSON" => Some(Self::Json), + "MULTI" => Some(Self::Multi), + "GCP" => Some(Self::Gcp), + "KINESISFH" => Some(Self::Kinesisfh), + "RUM" => Some(Self::Rum), + "USAGE" => Some(Self::Usage), + _ => None, + } + } +} +/// Generated client implementations. +pub mod ingest_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::{http::Uri, *}; + #[derive(Debug, Clone)] + pub struct IngestClient { + inner: tonic::client::Grpc, + } + impl IngestClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl IngestClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> IngestClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + >>::Error: + Into + Send + Sync, + { + IngestClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn ingest( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/cluster.Ingest/Ingest"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("cluster.Ingest", "Ingest")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod ingest_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with + /// IngestServer. + #[async_trait] + pub trait Ingest: Send + Sync + 'static { + async fn ingest( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct IngestServer { + inner: _Inner, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + struct _Inner(Arc); + impl IngestServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + let inner = _Inner(inner); + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for IngestServer + where + T: Ingest, + B: Body + Send + 'static, + B::Error: Into + Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/cluster.Ingest/Ingest" => { + #[allow(non_camel_case_types)] + struct IngestSvc(pub Arc); + impl tonic::server::UnaryService for IngestSvc { + type Response = super::IngestionResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::ingest(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = IngestSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + Ok(http::Response::builder() + .status(200) + .header("grpc-status", "12") + .header("content-type", "application/grpc") + .body(empty_body()) + .unwrap()) + }), + } + } + } + impl Clone for IngestServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + impl Clone for _Inner { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } + } + impl std::fmt::Debug for _Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } + } + impl tonic::server::NamedService for IngestServer { + const NAME: &'static str = "cluster.Ingest"; + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct UsageData { #[prost(bytes = "vec", tag = "1")] pub data: ::prost::alloc::vec::Vec, diff --git a/src/proto/src/lib.rs b/src/proto/src/lib.rs index fc068f60e4..e3247eb547 100644 --- a/src/proto/src/lib.rs +++ b/src/proto/src/lib.rs @@ -25,3 +25,11 @@ impl From> for cluster_rpc::UsageData { } } } + +impl From> for cluster_rpc::IngestionData { + fn from(usages: Vec) -> Self { + Self { + data: serde_json::to_vec(&usages).unwrap(), + } + } +} diff --git a/src/service/alerts/alert.rs b/src/service/alerts/alert.rs new file mode 100644 index 0000000000..4b09ae8b84 --- /dev/null +++ b/src/service/alerts/alert.rs @@ -0,0 +1,910 @@ +// Copyright 2024 Zinc Labs Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::{ + collections::{HashMap, HashSet}, + str::FromStr, +}; + +use actix_web::http; +use chrono::{Duration, Local, TimeZone, Timelike, Utc}; +use config::{ + get_config, + meta::stream::StreamType, + utils::{ + base64, + json::{Map, Value}, + }, + SMTP_CLIENT, +}; +use cron::Schedule; +use lettre::{message::SinglePart, AsyncTransport, Message}; + +use crate::{ + common::{ + meta::{ + alerts::{ + alert::Alert, + destinations::{DestinationType, DestinationWithTemplate, HTTPType}, + FrequencyType, Operator, QueryType, + }, + authz::Authz, + stream::StreamParams, + }, + utils::auth::{remove_ownership, set_ownership}, + }, + service::{ + alerts::{build_sql, destinations}, + db, + }, +}; + +pub async fn save( + org_id: &str, + stream_name: &str, + name: &str, + mut alert: Alert, + create: bool, +) -> Result<(), anyhow::Error> { + if !name.is_empty() { + alert.name = name.to_string(); + } + alert.name = alert.name.trim().to_string(); + alert.org_id = org_id.to_string(); + let stream_type = alert.stream_type; + alert.stream_name = stream_name.to_string(); + alert.row_template = alert.row_template.trim().to_string(); + + match db::alerts::alert::get(org_id, stream_type, stream_name, &alert.name).await { + Ok(Some(_)) => { + if create { + return Err(anyhow::anyhow!("Alert already exists")); + } + } + Ok(None) => { + if !create { + return Err(anyhow::anyhow!("Alert not found")); + } + } + Err(e) => { + return Err(e); + } + } + + if alert.trigger_condition.frequency_type == FrequencyType::Cron { + let cron_exp = alert.trigger_condition.cron.clone(); + if cron_exp.starts_with("* ") { + let (_, rest) = cron_exp.split_once(" ").unwrap(); + let now = Utc::now().second().to_string(); + alert.trigger_condition.cron = format!("{now} {rest}"); + log::debug!( + "New cron expression for alert {}: {}", + alert.name, + alert.trigger_condition.cron + ); + } + // Check the cron expression + Schedule::from_str(&alert.trigger_condition.cron)?; + } else if alert.trigger_condition.frequency == 0 { + // default frequency is 60 seconds + alert.trigger_condition.frequency = + std::cmp::max(10, get_config().limit.alert_schedule_interval); + } + + if alert.name.is_empty() || alert.stream_name.is_empty() { + return Err(anyhow::anyhow!("Alert name is required")); + } + if alert.name.contains('/') { + return Err(anyhow::anyhow!("Alert name cannot contain '/'")); + } + + if let Some(vrl) = alert.query_condition.vrl_function.as_ref() { + match base64::decode_url(vrl) { + Ok(vrl) => { + if !vrl.ends_with('.') { + let vrl = base64::encode_url(&format!("{vrl} \n .")); + alert.query_condition.vrl_function = Some(vrl); + } + } + Err(e) => { + return Err(anyhow::anyhow!( + "Error decoding vrl function for alert: {e}" + )); + } + } + } + + // before saving alert check alert destination + if alert.destinations.is_empty() { + return Err(anyhow::anyhow!("Alert destinations is required")); + } + for dest in alert.destinations.iter() { + if db::alerts::destinations::get(org_id, dest).await.is_err() { + return Err(anyhow::anyhow!("Alert destination {dest} not found")); + }; + } + + // before saving alert check alert context attributes + if alert.context_attributes.is_some() { + let attrs = alert.context_attributes.as_ref().unwrap(); + let mut new_attrs = hashbrown::HashMap::with_capacity(attrs.len()); + for key in attrs.keys() { + let new_key = key.trim().to_string(); + if !new_key.is_empty() { + new_attrs.insert(new_key, attrs.get(key).unwrap().to_string()); + } + } + alert.context_attributes = Some(new_attrs); + } + + // before saving alert check column type to decide numeric condition + let schema = infra::schema::get(org_id, stream_name, stream_type).await?; + if stream_name.is_empty() || schema.fields().is_empty() { + return Err(anyhow::anyhow!("Stream {stream_name} not found")); + } + + if alert.is_real_time && alert.query_condition.query_type != QueryType::Custom { + return Err(anyhow::anyhow!( + "Realtime alert should use Custom query type" + )); + } + + match alert.query_condition.query_type { + QueryType::Custom => { + if alert.query_condition.aggregation.is_some() { + // if it has result we should fire the alert when enable aggregation + alert.trigger_condition.operator = Operator::GreaterThanEquals; + alert.trigger_condition.threshold = 1; + } + } + QueryType::SQL => { + if alert.query_condition.sql.is_none() + || alert.query_condition.sql.as_ref().unwrap().is_empty() + { + return Err(anyhow::anyhow!("Alert with SQL mode should have a query")); + } + } + QueryType::PromQL => { + if alert.query_condition.promql.is_none() + || alert.query_condition.promql.as_ref().unwrap().is_empty() + || alert.query_condition.promql_condition.is_none() + { + return Err(anyhow::anyhow!( + "Alert with PromQL mode should have a query" + )); + } + } + } + + // test the alert + if let Err(e) = &alert.evaluate(None).await { + return Err(anyhow::anyhow!("Alert test failed: {}", e)); + } + + // save the alert + match db::alerts::alert::set(org_id, stream_type, stream_name, &alert, create).await { + Ok(_) => { + if name.is_empty() { + set_ownership(org_id, "alerts", Authz::new(&alert.name)).await; + } + Ok(()) + } + Err(e) => Err(e), + } +} + +pub async fn get( + org_id: &str, + stream_type: StreamType, + stream_name: &str, + name: &str, +) -> Result, anyhow::Error> { + db::alerts::alert::get(org_id, stream_type, stream_name, name).await +} + +pub async fn list( + org_id: &str, + stream_type: Option, + stream_name: Option<&str>, + permitted: Option>, +) -> Result, anyhow::Error> { + match db::alerts::alert::list(org_id, stream_type, stream_name).await { + Ok(alerts) => { + let mut result = Vec::new(); + for alert in alerts { + if permitted.is_none() + || permitted + .as_ref() + .unwrap() + .contains(&format!("alert:{}", alert.name)) + || permitted + .as_ref() + .unwrap() + .contains(&format!("alert:_all_{}", org_id)) + { + result.push(alert); + } + } + Ok(result) + } + Err(e) => Err(e), + } +} + +pub async fn delete( + org_id: &str, + stream_type: StreamType, + stream_name: &str, + name: &str, +) -> Result<(), (http::StatusCode, anyhow::Error)> { + if db::alerts::alert::get(org_id, stream_type, stream_name, name) + .await + .is_err() + { + return Err(( + http::StatusCode::NOT_FOUND, + anyhow::anyhow!("Alert not found"), + )); + } + match db::alerts::alert::delete(org_id, stream_type, stream_name, name).await { + Ok(_) => { + remove_ownership(org_id, "alerts", Authz::new(name)).await; + Ok(()) + } + Err(e) => Err((http::StatusCode::INTERNAL_SERVER_ERROR, e)), + } +} + +pub async fn enable( + org_id: &str, + stream_type: StreamType, + stream_name: &str, + name: &str, + value: bool, +) -> Result<(), (http::StatusCode, anyhow::Error)> { + let mut alert = match db::alerts::alert::get(org_id, stream_type, stream_name, name).await { + Ok(Some(alert)) => alert, + _ => { + return Err(( + http::StatusCode::NOT_FOUND, + anyhow::anyhow!("Alert not found"), + )); + } + }; + alert.enabled = value; + db::alerts::alert::set(org_id, stream_type, stream_name, &alert, false) + .await + .map_err(|e| (http::StatusCode::INTERNAL_SERVER_ERROR, e)) +} + +pub async fn trigger( + org_id: &str, + stream_type: StreamType, + stream_name: &str, + name: &str, +) -> Result<(bool, String), (http::StatusCode, anyhow::Error)> { + let alert = match db::alerts::alert::get(org_id, stream_type, stream_name, name).await { + Ok(Some(alert)) => alert, + _ => { + return Err(( + http::StatusCode::NOT_FOUND, + anyhow::anyhow!("Alert not found"), + )); + } + }; + alert + .send_notification(&[]) + .await + .map_err(|e| (http::StatusCode::INTERNAL_SERVER_ERROR, e)) +} + +impl Alert { + /// Returns the evaluated row data and the end time of the search timerange, + /// for realtime this is 0 + pub async fn evaluate( + &self, + row: Option<&Map>, + ) -> Result<(Option>>, i64), anyhow::Error> { + if self.is_real_time { + self.query_condition.evaluate_realtime(row).await + } else { + self.query_condition + .evaluate_scheduled( + &self.get_stream_params(), + &self.trigger_condition, + &self.query_condition, + ) + .await + } + } + + /// Returns a tuple containing a boolean - if all the send notification jobs succeeded + /// and the error message if any + pub async fn send_notification( + &self, + rows: &[Map], + ) -> Result<(bool, String), anyhow::Error> { + let mut message = "".to_string(); + let mut no_of_error = 0; + for dest in self.destinations.iter() { + let dest = destinations::get_with_template(&self.org_id, dest).await?; + if let Err(e) = send_notification(self, &dest, rows).await { + log::error!( + "Error sending notification for {}/{}/{}/{} for destination {} err: {}", + self.org_id, + self.stream_type, + self.stream_name, + self.name, + dest.name, + e + ); + no_of_error += 1; + message = format!( + "{message} Error sending notification for destination {} err: {e};", + dest.name + ); + } + } + if no_of_error == self.destinations.len() { + Err(anyhow::anyhow!(message)) + } else if no_of_error != 0 { + // Some send notification job failed + Ok((false, message)) + } else { + Ok((true, "".to_owned())) + } + } + + pub fn get_stream_params(&self) -> StreamParams { + StreamParams { + org_id: self.org_id.clone().into(), + stream_name: self.stream_name.clone().into(), + stream_type: self.stream_type, + } + } +} + +pub async fn send_notification( + alert: &Alert, + dest: &DestinationWithTemplate, + rows: &[Map], +) -> Result<(), anyhow::Error> { + let rows_tpl_val = if alert.row_template.is_empty() { + vec!["".to_string()] + } else { + process_row_template(&alert.row_template, alert, rows) + }; + let msg: String = process_dest_template(&dest.template.body, alert, rows, &rows_tpl_val).await; + + match dest.destination_type { + DestinationType::Http => send_http_notification(dest, msg.clone()).await, + DestinationType::Email => send_email_notification(&alert.name, dest, msg).await, + } +} + +pub async fn send_http_notification( + dest: &DestinationWithTemplate, + msg: String, +) -> Result<(), anyhow::Error> { + let client = if dest.skip_tls_verify { + reqwest::Client::builder() + .danger_accept_invalid_certs(true) + .build()? + } else { + reqwest::Client::new() + }; + let url = url::Url::parse(&dest.url)?; + let mut req = match dest.method { + HTTPType::POST => client.post(url), + HTTPType::PUT => client.put(url), + HTTPType::GET => client.get(url), + }; + + // Add additional headers if any from destination description + let mut has_context_type = false; + if let Some(headers) = &dest.headers { + for (key, value) in headers.iter() { + if !key.is_empty() && !value.is_empty() { + if key.to_lowercase().trim() == "content-type" { + has_context_type = true; + } + req = req.header(key, value); + } + } + }; + // set default content type + if !has_context_type { + req = req.header("Content-type", "application/json"); + } + + let resp = req.body(msg.clone()).send().await?; + if !resp.status().is_success() { + log::error!("Alert body: {}", msg); + return Err(anyhow::anyhow!( + "sent error status: {}, err: {:?}", + resp.status(), + resp.bytes().await + )); + } + + Ok(()) +} + +pub async fn send_email_notification( + alert_name: &str, + dest: &DestinationWithTemplate, + msg: String, +) -> Result<(), anyhow::Error> { + let cfg = get_config(); + if !cfg.smtp.smtp_enabled { + return Err(anyhow::anyhow!("SMTP configuration not enabled")); + } + + let mut recepients = vec![]; + for recepient in &dest.emails { + recepients.push(recepient); + } + + let mut email = Message::builder() + .from(cfg.smtp.smtp_from_email.parse()?) + .subject(format!("Openobserve Alert - {}", alert_name)); + + for recepient in recepients { + email = email.to(recepient.parse()?); + } + + if !cfg.smtp.smtp_reply_to.is_empty() { + email = email.reply_to(cfg.smtp.smtp_reply_to.parse()?); + } + + let email = email.singlepart(SinglePart::html(msg)).unwrap(); + + // Send the email + match SMTP_CLIENT.as_ref().unwrap().send(email).await { + Ok(_) => Ok(()), + Err(e) => Err(anyhow::anyhow!("Error sending email: {e}")), + } +} + +fn process_row_template(tpl: &String, alert: &Alert, rows: &[Map]) -> Vec { + let alert_type = if alert.is_real_time { + "realtime" + } else { + "scheduled" + }; + let alert_count = rows.len(); + let mut rows_tpl = Vec::with_capacity(rows.len()); + for row in rows.iter() { + let mut resp = tpl.to_string(); + let mut alert_start_time = 0; + let mut alert_end_time = 0; + for (key, value) in row.iter() { + let value = if value.is_string() { + value.as_str().unwrap_or_default().to_string() + } else if value.is_f64() { + format!("{:.2}", value.as_f64().unwrap_or_default()) + } else { + value.to_string() + }; + process_variable_replace(&mut resp, key, &VarValue::Str(&value)); + + // calculate start and end time + if key == &get_config().common.column_timestamp { + let val = value.parse::().unwrap_or_default(); + if alert_start_time == 0 || val < alert_start_time { + alert_start_time = val; + } + if alert_end_time == 0 || val > alert_end_time { + alert_end_time = val; + } + } + if key == "zo_sql_min_time" { + let val = value.parse::().unwrap_or_default(); + if alert_start_time == 0 || val < alert_start_time { + alert_start_time = val; + } + } + if key == "zo_sql_max_time" { + let val = value.parse::().unwrap_or_default(); + if alert_end_time == 0 || val > alert_end_time { + alert_end_time = val; + } + } + } + let alert_start_time_str = if alert_start_time > 0 { + Local + .timestamp_nanos(alert_start_time * 1000) + .format("%Y-%m-%dT%H:%M:%S") + .to_string() + } else { + String::from("N/A") + }; + let alert_end_time_str = if alert_end_time > 0 { + Local + .timestamp_nanos(alert_end_time * 1000) + .format("%Y-%m-%dT%H:%M:%S") + .to_string() + } else { + String::from("N/A") + }; + + resp = resp + .replace("{org_name}", &alert.org_id) + .replace("{stream_type}", &alert.stream_type.to_string()) + .replace("{stream_name}", &alert.stream_name) + .replace("{alert_name}", &alert.name) + .replace("{alert_type}", alert_type) + .replace( + "{alert_period}", + &alert.trigger_condition.period.to_string(), + ) + .replace( + "{alert_operator}", + &alert.trigger_condition.operator.to_string(), + ) + .replace( + "{alert_threshold}", + &alert.trigger_condition.threshold.to_string(), + ) + .replace("{alert_count}", &alert_count.to_string()) + .replace("{alert_start_time}", &alert_start_time_str) + .replace("{alert_end_time}", &alert_end_time_str); + + if let Some(contidion) = &alert.query_condition.promql_condition { + resp = resp + .replace("{alert_promql_operator}", &contidion.operator.to_string()) + .replace("{alert_promql_value}", &contidion.value.to_string()); + } + + if let Some(attrs) = &alert.context_attributes { + for (key, value) in attrs.iter() { + process_variable_replace(&mut resp, key, &VarValue::Str(value)); + } + } + + rows_tpl.push(resp); + } + + rows_tpl +} + +async fn process_dest_template( + tpl: &str, + alert: &Alert, + rows: &[Map], + rows_tpl_val: &[String], +) -> String { + let cfg = get_config(); + // format values + let alert_count = rows.len(); + let mut vars = HashMap::with_capacity(rows.len()); + for row in rows.iter() { + for (key, value) in row.iter() { + let value = if value.is_string() { + value.as_str().unwrap_or_default().to_string() + } else if value.is_f64() { + format!("{:.2}", value.as_f64().unwrap_or_default()) + } else { + value.to_string() + }; + let entry = vars.entry(key.to_string()).or_insert_with(HashSet::new); + entry.insert(value); + } + } + + // calculate start and end time + let (alert_start_time, alert_end_time) = + get_alert_start_end_time(&vars, alert.trigger_condition.period); + + let alert_start_time_str = if alert_start_time > 0 { + Local + .timestamp_nanos(alert_start_time * 1000) + .format("%Y-%m-%dT%H:%M:%S") + .to_string() + } else { + String::from("N/A") + }; + let alert_end_time_str = if alert_end_time > 0 { + Local + .timestamp_nanos(alert_end_time * 1000) + .format("%Y-%m-%dT%H:%M:%S") + .to_string() + } else { + String::from("N/A") + }; + + let alert_type = if alert.is_real_time { + "realtime" + } else { + "scheduled" + }; + + let mut alert_query = String::new(); + let alert_url = if alert.query_condition.query_type == QueryType::PromQL { + if let Some(promql) = &alert.query_condition.promql { + let condition = alert.query_condition.promql_condition.as_ref().unwrap(); + alert_query = format!( + "({}) {} {}", + promql, + match condition.operator { + Operator::EqualTo => "==".to_string(), + _ => condition.operator.to_string(), + }, + to_float(&condition.value) + ); + } + // http://localhost:5080/web/metrics?stream=zo_http_response_time_bucket&from=1705248000000000&to=1705334340000000&query=em9faHR0cF9yZXNwb25zZV90aW1lX2J1Y2tldHt9&org_identifier=default + format!( + "{}{}/web/metrics?stream_type={}&stream={}&stream_value={}&from={}&to={}&query={}&org_identifier={}", + cfg.common.web_url, + cfg.common.base_uri, + alert.stream_type, + alert.stream_name, + alert.stream_name, + alert_start_time, + alert_end_time, + base64::encode_url(&alert_query).replace('+', "%2B"), + alert.org_id, + ) + } else { + match alert.query_condition.query_type { + QueryType::SQL => { + if let Some(sql) = &alert.query_condition.sql { + alert_query = sql.clone(); + } + } + QueryType::Custom => { + if let Some(conditions) = &alert.query_condition.conditions { + if let Ok(v) = build_sql( + &alert.get_stream_params(), + &alert.query_condition, + conditions, + ) + .await + { + alert_query = v; + } + } + } + _ => unreachable!(), + }; + // http://localhost:5080/web/logs?stream_type=logs&stream=test&from=1708416534519324&to=1708416597898186&sql_mode=true&query=U0VMRUNUICogRlJPTSAidGVzdCIgd2hlcmUgbGV2ZWwgPSAnaW5mbyc=&org_identifier=default + format!( + "{}{}/web/logs?stream_type={}&stream={}&stream_value={}&from={}&to={}&sql_mode=true&query={}&org_identifier={}", + cfg.common.web_url, + cfg.common.base_uri, + alert.stream_type, + alert.stream_name, + alert.stream_name, + alert_start_time, + alert_end_time, + base64::encode_url(&alert_query), + alert.org_id, + ) + }; + + let mut resp = tpl + .replace("{org_name}", &alert.org_id) + .replace("{stream_type}", &alert.stream_type.to_string()) + .replace("{stream_name}", &alert.stream_name) + .replace("{alert_name}", &alert.name) + .replace("{alert_type}", alert_type) + .replace( + "{alert_period}", + &alert.trigger_condition.period.to_string(), + ) + .replace( + "{alert_operator}", + &alert.trigger_condition.operator.to_string(), + ) + .replace( + "{alert_threshold}", + &alert.trigger_condition.threshold.to_string(), + ) + .replace("{alert_count}", &alert_count.to_string()) + .replace("{alert_start_time}", &alert_start_time_str) + .replace("{alert_end_time}", &alert_end_time_str) + .replace("{alert_url}", &alert_url); + + if let Some(contidion) = &alert.query_condition.promql_condition { + resp = resp + .replace("{alert_promql_operator}", &contidion.operator.to_string()) + .replace("{alert_promql_value}", &contidion.value.to_string()); + } + + process_variable_replace(&mut resp, "rows", &VarValue::Vector(rows_tpl_val)); + for (key, value) in vars.iter() { + if resp.contains(&format!("{{{key}}}")) { + let val = value.iter().cloned().collect::>(); + process_variable_replace(&mut resp, key, &VarValue::Str(&val.join(", "))); + } + } + if let Some(attrs) = &alert.context_attributes { + for (key, value) in attrs.iter() { + process_variable_replace(&mut resp, key, &VarValue::Str(value)); + } + } + + resp +} + +fn process_variable_replace(tpl: &mut String, var_name: &str, var_val: &VarValue) { + let pattern = "{".to_owned() + var_name + "}"; + if tpl.contains(&pattern) { + *tpl = tpl.replace(&pattern, &var_val.to_string_with_length(0)); + return; + } + let pattern = "{".to_owned() + var_name + ":"; + if let Some(start) = tpl.find(&pattern) { + // find } start from position v + let p = start + pattern.len(); + if let Some(end) = tpl[p..].find('}') { + let len = tpl[p..p + end].parse::().unwrap_or_default(); + if len > 0 { + *tpl = tpl.replace( + &tpl[start..p + end + 1], + &var_val.to_string_with_length(len), + ); + } + } + } +} + +pub fn get_row_column_map(rows: &[Map]) -> HashMap> { + let mut vars = HashMap::with_capacity(rows.len()); + for row in rows.iter() { + for (key, value) in row.iter() { + let value = if value.is_string() { + value.as_str().unwrap_or_default().to_string() + } else if value.is_f64() { + format!("{:.2}", value.as_f64().unwrap_or_default()) + } else { + value.to_string() + }; + let entry = vars.entry(key.to_string()).or_insert_with(HashSet::new); + entry.insert(value); + } + } + vars +} + +pub fn get_alert_start_end_time( + vars: &HashMap>, + period: i64, +) -> (i64, i64) { + let cfg = get_config(); + + // calculate start and end time + let mut alert_start_time = 0; + let mut alert_end_time = 0; + if let Some(values) = vars.get(&cfg.common.column_timestamp) { + for val in values { + let val = val.parse::().unwrap_or_default(); + if alert_start_time == 0 || val < alert_start_time { + alert_start_time = val; + } + if alert_end_time == 0 || val > alert_end_time { + alert_end_time = val; + } + } + } + if let Some(values) = vars.get("zo_sql_min_time") { + for val in values { + let val = val.parse::().unwrap_or_default(); + if alert_start_time == 0 || val < alert_start_time { + alert_start_time = val; + } + } + } + if let Some(values) = vars.get("zo_sql_max_time") { + for val in values { + let val = val.parse::().unwrap_or_default(); + if alert_end_time == 0 || val > alert_end_time { + alert_end_time = val; + } + } + } + + // Hack time range for alert url + alert_end_time = if alert_end_time == 0 { + Utc::now().timestamp_micros() + } else { + // the frontend will drop the second, so we add 1 minute to the end time + alert_end_time + + Duration::try_minutes(1) + .unwrap() + .num_microseconds() + .unwrap() + }; + if alert_start_time == 0 { + alert_start_time = alert_end_time + - Duration::try_minutes(period) + .unwrap() + .num_microseconds() + .unwrap(); + } + if alert_end_time - alert_start_time + < Duration::try_minutes(1) + .unwrap() + .num_microseconds() + .unwrap() + { + alert_start_time = alert_end_time + - Duration::try_minutes(period) + .unwrap() + .num_microseconds() + .unwrap(); + } + (alert_start_time, alert_end_time) +} + +fn format_variable_value(val: String) -> String { + val.replace('\n', "\\n") + .replace('\r', "\\r") + .replace('\"', "\\\"") +} + +pub(super) fn to_float(val: &Value) -> f64 { + if val.is_number() { + val.as_f64().unwrap_or_default() + } else { + val.as_str().unwrap_or_default().parse().unwrap_or_default() + } +} + +enum VarValue<'a> { + Str(&'a str), + Vector(&'a [String]), +} + +impl<'a> VarValue<'a> { + fn len(&self) -> usize { + match self { + VarValue::Str(v) => v.chars().count(), + VarValue::Vector(v) => v.len(), + } + } + + fn to_string_with_length(&self, n: usize) -> String { + let n = if n > 0 && n < self.len() { + n + } else { + self.len() + }; + match self { + VarValue::Str(v) => format_variable_value(v.chars().take(n).collect()), + VarValue::Vector(v) => v[0..n].join("\\n"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_alert_create() { + let org_id = "default"; + let stream_name = "default"; + let alert_name = "abc/alert"; + let alert = Alert { + name: alert_name.to_string(), + ..Default::default() + }; + let ret = save(org_id, stream_name, alert_name, alert, true).await; + // alert name should not contain / + assert!(ret.is_err()); + } +} diff --git a/src/service/alerts/derived_streams.rs b/src/service/alerts/derived_streams.rs new file mode 100644 index 0000000000..2e6bf77451 --- /dev/null +++ b/src/service/alerts/derived_streams.rs @@ -0,0 +1,197 @@ +// Copyright 2024 Zinc Labs Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::str::FromStr; + +use chrono::{Duration, Utc}; +use config::{ + get_config, + utils::json::{Map, Value}, +}; +use cron::Schedule; +use hashbrown::HashMap; + +use crate::{ + common::{ + meta::{ + // authz::Authz, + alerts::{derived_streams::DerivedStreamMeta, FrequencyType, QueryType}, + }, + // utils::auth::{remove_ownership, set_ownership}, + }, + service::db, +}; + +pub async fn save( + mut derived_stream: DerivedStreamMeta, + pipeline_name: &str, +) -> Result<(), anyhow::Error> { + derived_stream.name = derived_stream.name.trim().to_string(); + // 1. Start validate DerivedStream + if !derived_stream.is_valid() { + return Err(anyhow::anyhow!( + "DerivedStream Name, destination, and Trigger period required" + )); + } + + // query_type for realtime + if derived_stream.is_real_time && derived_stream.query_condition.query_type != QueryType::Custom + { + return Err(anyhow::anyhow!( + "Realtime DerivedStream should use Custom query type" + )); + } + + // other checks for query type + match derived_stream.query_condition.query_type { + QueryType::SQL => { + if derived_stream + .query_condition + .sql + .as_ref() + .map_or(false, |sql| sql.is_empty()) + { + return Err(anyhow::anyhow!( + "DerivedStreams with SQL mode should have a query" + )); + } + } + QueryType::PromQL => { + if derived_stream + .query_condition + .promql + .as_ref() + .map_or(false, |promql| promql.is_empty()) + || derived_stream.query_condition.promql_condition.is_none() + { + return Err(anyhow::anyhow!( + "DerivedStreams with SQL mode should have a query" + )); + } + } + _ => {} + } + + // check source stream schema + let schema = infra::schema::get( + &derived_stream.source.org_id, + &derived_stream.source.stream_name, + derived_stream.source.stream_type, + ) + .await?; + if schema.fields().is_empty() { + return Err(anyhow::anyhow!( + "Source Stream {}/{} schema is empty.", + derived_stream.source.org_id, + derived_stream.source.stream_name, + )); + } + // End input validation + + // 2. update the frequency + if derived_stream.trigger_condition.frequency_type == FrequencyType::Cron { + // Check the cron expression + Schedule::from_str(&derived_stream.trigger_condition.cron)?; + } else if derived_stream.trigger_condition.frequency == 0 { + // default 3 mins, set min at 1 minutes + derived_stream.trigger_condition.frequency = + std::cmp::max(1, get_config().limit.derived_stream_schedule_interval / 60); + } + + // 3. clean up DerivedStream context attributes + if let Some(attrs) = &derived_stream.context_attributes { + let mut new_attrs = HashMap::with_capacity(attrs.len()); + for (key, val) in attrs.iter() { + new_attrs.insert(key.trim().to_string(), val.to_string()); + } + derived_stream.context_attributes = Some(new_attrs); + } + + // test derived_stream + if let Err(e) = &derived_stream.evaluate(None).await { + return Err(anyhow::anyhow!( + "DerivedStream not saved due to failed test run caused by {}", + e.to_string() + )); + }; + + // Save the trigger to db + let next_run_at = Utc::now().timestamp_micros() + + Duration::try_minutes(derived_stream.trigger_condition.frequency) + .unwrap() + .num_microseconds() + .unwrap(); + let trigger = db::scheduler::Trigger { + org: derived_stream.source.org_id.to_string(), + module: db::scheduler::TriggerModule::DerivedStream, + module_key: derived_stream.get_scheduler_module_key(pipeline_name), + next_run_at, + is_realtime: derived_stream.is_real_time, + is_silenced: false, + ..Default::default() + }; + + match db::scheduler::get(&trigger.org, trigger.module.clone(), &trigger.module_key).await { + Ok(_) => db::scheduler::update_trigger(trigger) + .await + .map_err(|_| anyhow::anyhow!("Trigger already exists, but failed to update")), + Err(_) => db::scheduler::push(trigger) + .await + .map_err(|e| anyhow::anyhow!("Error save DerivedStream trigger: {}", e)), + } +} + +pub async fn delete( + derived_stream: DerivedStreamMeta, + pipeline_name: &str, +) -> Result<(), anyhow::Error> { + db::scheduler::delete( + &derived_stream.source.org_id, + db::scheduler::TriggerModule::DerivedStream, + &derived_stream.get_scheduler_module_key(pipeline_name), + ) + .await + .map_err(|e| anyhow::anyhow!("Error deleting derived stream trigger: {e}")) +} + +impl DerivedStreamMeta { + pub fn is_valid(&self) -> bool { + !self.name.is_empty() + && !self.is_real_time // TODO(taiming): support realtime DerivedStream + && self.source.is_valid() + && self.destination.is_valid() + && self.trigger_condition.period != 0 + } + + pub fn get_scheduler_module_key(&self, pipeline_name: &str) -> String { + format!( + "{}/{}/{}/{}", + self.source.stream_type, self.source.stream_name, pipeline_name, self.name + ) + } + + pub async fn evaluate( + &self, + row: Option<&Map>, + ) -> Result<(Option>>, i64), anyhow::Error> { + if self.is_real_time { + self.query_condition.evaluate_realtime(row).await + } else { + self.query_condition + .evaluate_scheduled(&self.source, &self.trigger_condition, &self.query_condition) + .await + } + } +} diff --git a/src/service/alerts/mod.rs b/src/service/alerts/mod.rs index aaf95573c4..5ada97891d 100644 --- a/src/service/alerts/mod.rs +++ b/src/service/alerts/mod.rs @@ -13,387 +13,65 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{ - collections::{HashMap, HashSet}, - str::FromStr, -}; - -use actix_web::http; +use alert::to_float; use arrow_schema::DataType; -use chrono::{Duration, Local, TimeZone, Timelike, Utc}; +use chrono::{Duration, Utc}; use config::{ get_config, ider, - meta::{search::SearchEventType, stream::StreamType}, + meta::search::SearchEventType, utils::{ base64, json::{Map, Value}, }, - SMTP_CLIENT, }; -use cron::Schedule; -use lettre::{message::SinglePart, AsyncTransport, Message}; use super::promql; use crate::{ - common::{ - meta::{ - alerts::{ - destinations::{DestinationType, DestinationWithTemplate, HTTPType}, - AggFunction, Alert, AlertFrequencyType, Condition, Operator, QueryCondition, - QueryType, - }, - authz::Authz, - }, - utils::auth::{remove_ownership, set_ownership}, + common::meta::{ + alerts::{AggFunction, Condition, Operator, QueryCondition, QueryType, TriggerCondition}, + stream::StreamParams, }, - service::{db, search as SearchService}, + service::search as SearchService, }; -pub mod alert_manager; +pub mod alert; +pub mod derived_streams; pub mod destinations; +pub mod scheduler; pub mod templates; -pub async fn save( - org_id: &str, - stream_name: &str, - name: &str, - mut alert: Alert, - create: bool, -) -> Result<(), anyhow::Error> { - if !name.is_empty() { - alert.name = name.to_string(); - } - alert.name = alert.name.trim().to_string(); - alert.org_id = org_id.to_string(); - let stream_type = alert.stream_type; - alert.stream_name = stream_name.to_string(); - alert.row_template = alert.row_template.trim().to_string(); - - match db::alerts::get(org_id, stream_type, stream_name, &alert.name).await { - Ok(Some(_)) => { - if create { - return Err(anyhow::anyhow!("Alert already exists")); - } - } - Ok(None) => { - if !create { - return Err(anyhow::anyhow!("Alert not found")); - } - } - Err(e) => { - return Err(e); - } - } - - if alert.trigger_condition.frequency_type == AlertFrequencyType::Cron { - let cron_exp = alert.trigger_condition.cron.clone(); - if cron_exp.starts_with("* ") { - let (_, rest) = cron_exp.split_once(" ").unwrap(); - let now = Utc::now().second().to_string(); - alert.trigger_condition.cron = format!("{now} {rest}"); - log::debug!( - "New cron expression for alert {}: {}", - alert.name, - alert.trigger_condition.cron - ); - } - // Check the cron expression - Schedule::from_str(&alert.trigger_condition.cron)?; - } else if alert.trigger_condition.frequency == 0 { - // default frequency is 60 seconds - alert.trigger_condition.frequency = - std::cmp::max(10, get_config().limit.alert_schedule_interval); - } - - if alert.name.is_empty() || alert.stream_name.is_empty() { - return Err(anyhow::anyhow!("Alert name is required")); - } - if alert.name.contains('/') { - return Err(anyhow::anyhow!("Alert name cannot contain '/'")); - } - - if let Some(vrl) = alert.query_condition.vrl_function.as_ref() { - match base64::decode_url(vrl) { - Ok(vrl) => { - if !vrl.ends_with('.') { - let vrl = base64::encode_url(&format!("{vrl} \n .")); - alert.query_condition.vrl_function = Some(vrl); - } - } - Err(e) => { - return Err(anyhow::anyhow!( - "Error decoding vrl function for alert: {e}" - )); - } - } - } - - // before saving alert check alert destination - if alert.destinations.is_empty() { - return Err(anyhow::anyhow!("Alert destinations is required")); - } - for dest in alert.destinations.iter() { - if db::alerts::destinations::get(org_id, dest).await.is_err() { - return Err(anyhow::anyhow!("Alert destination {dest} not found")); - }; - } - - // before saving alert check alert context attributes - if alert.context_attributes.is_some() { - let attrs = alert.context_attributes.as_ref().unwrap(); - let mut new_attrs = hashbrown::HashMap::with_capacity(attrs.len()); - for key in attrs.keys() { - let new_key = key.trim().to_string(); - if !new_key.is_empty() { - new_attrs.insert(new_key, attrs.get(key).unwrap().to_string()); - } - } - alert.context_attributes = Some(new_attrs); - } - - // before saving alert check column type to decide numeric condition - let schema = infra::schema::get(org_id, stream_name, stream_type).await?; - if stream_name.is_empty() || schema.fields().is_empty() { - return Err(anyhow::anyhow!("Stream {stream_name} not found")); - } - - if alert.is_real_time && alert.query_condition.query_type != QueryType::Custom { - return Err(anyhow::anyhow!( - "Realtime alert should use Custom query type" - )); - } - - match alert.query_condition.query_type { - QueryType::Custom => { - if alert.query_condition.aggregation.is_some() { - // if it has result we should fire the alert when enable aggregation - alert.trigger_condition.operator = Operator::GreaterThanEquals; - alert.trigger_condition.threshold = 1; - } - } - QueryType::SQL => { - if alert.query_condition.sql.is_none() - || alert.query_condition.sql.as_ref().unwrap().is_empty() - { - return Err(anyhow::anyhow!("Alert with SQL mode should have a query")); - } - } - QueryType::PromQL => { - if alert.query_condition.promql.is_none() - || alert.query_condition.promql.as_ref().unwrap().is_empty() - || alert.query_condition.promql_condition.is_none() - { - return Err(anyhow::anyhow!( - "Alert with PromQL mode should have a query" - )); - } - } - } - - // test the alert - if let Err(e) = &alert.evaluate(None).await { - return Err(anyhow::anyhow!("Alert test failed: {}", e)); - } - - // save the alert - match db::alerts::set(org_id, stream_type, stream_name, &alert, create).await { - Ok(_) => { - if name.is_empty() { - set_ownership(org_id, "alerts", Authz::new(&alert.name)).await; - } - Ok(()) - } - Err(e) => Err(e), - } -} - -pub async fn get( - org_id: &str, - stream_type: StreamType, - stream_name: &str, - name: &str, -) -> Result, anyhow::Error> { - db::alerts::get(org_id, stream_type, stream_name, name).await -} - -pub async fn list( - org_id: &str, - stream_type: Option, - stream_name: Option<&str>, - permitted: Option>, -) -> Result, anyhow::Error> { - match db::alerts::list(org_id, stream_type, stream_name).await { - Ok(alerts) => { - let mut result = Vec::new(); - for alert in alerts { - if permitted.is_none() - || permitted - .as_ref() - .unwrap() - .contains(&format!("alert:{}", alert.name)) - || permitted - .as_ref() - .unwrap() - .contains(&format!("alert:_all_{}", org_id)) - { - result.push(alert); - } - } - Ok(result) - } - Err(e) => Err(e), - } -} - -pub async fn delete( - org_id: &str, - stream_type: StreamType, - stream_name: &str, - name: &str, -) -> Result<(), (http::StatusCode, anyhow::Error)> { - if db::alerts::get(org_id, stream_type, stream_name, name) - .await - .is_err() - { - return Err(( - http::StatusCode::NOT_FOUND, - anyhow::anyhow!("Alert not found"), - )); - } - match db::alerts::delete(org_id, stream_type, stream_name, name).await { - Ok(_) => { - remove_ownership(org_id, "alerts", Authz::new(name)).await; - Ok(()) - } - Err(e) => Err((http::StatusCode::INTERNAL_SERVER_ERROR, e)), - } -} - -pub async fn enable( - org_id: &str, - stream_type: StreamType, - stream_name: &str, - name: &str, - value: bool, -) -> Result<(), (http::StatusCode, anyhow::Error)> { - let mut alert = match db::alerts::get(org_id, stream_type, stream_name, name).await { - Ok(Some(alert)) => alert, - _ => { - return Err(( - http::StatusCode::NOT_FOUND, - anyhow::anyhow!("Alert not found"), - )); - } - }; - alert.enabled = value; - db::alerts::set(org_id, stream_type, stream_name, &alert, false) - .await - .map_err(|e| (http::StatusCode::INTERNAL_SERVER_ERROR, e)) -} - -pub async fn trigger( - org_id: &str, - stream_type: StreamType, - stream_name: &str, - name: &str, -) -> Result<(bool, String), (http::StatusCode, anyhow::Error)> { - let alert = match db::alerts::get(org_id, stream_type, stream_name, name).await { - Ok(Some(alert)) => alert, - _ => { - return Err(( - http::StatusCode::NOT_FOUND, - anyhow::anyhow!("Alert not found"), - )); - } - }; - alert - .send_notification(&[]) - .await - .map_err(|e| (http::StatusCode::INTERNAL_SERVER_ERROR, e)) -} - -impl Alert { - /// Returns the evaluated row data and the end time of the search timerange, - /// for realtime this is 0 - pub async fn evaluate( - &self, - row: Option<&Map>, - ) -> Result<(Option>>, i64), anyhow::Error> { - if self.is_real_time { - let rows = self.query_condition.evaluate_realtime(row).await?; - Ok((rows, Utc::now().timestamp_micros())) - } else { - self.query_condition.evaluate_scheduled(self).await - } - } - - /// Returns a tuple containing a boolean - if all the send notification jobs succeeded - /// and the error message if any - pub async fn send_notification( - &self, - rows: &[Map], - ) -> Result<(bool, String), anyhow::Error> { - let mut message = "".to_string(); - let mut no_of_error = 0; - for dest in self.destinations.iter() { - let dest = destinations::get_with_template(&self.org_id, dest).await?; - if let Err(e) = send_notification(self, &dest, rows).await { - log::error!( - "Error sending notification for {}/{}/{}/{} for destination {} err: {}", - self.org_id, - self.stream_type, - self.stream_name, - self.name, - dest.name, - e - ); - no_of_error += 1; - message = format!( - "{message} Error sending notification for destination {} err: {e};", - dest.name - ); - } - } - if no_of_error == self.destinations.len() { - Err(anyhow::anyhow!(message)) - } else if no_of_error != 0 { - // Some send notification job failed - Ok((false, message)) - } else { - Ok((true, "".to_owned())) - } - } -} - impl QueryCondition { pub async fn evaluate_realtime( &self, row: Option<&Map>, - ) -> Result>>, anyhow::Error> { + ) -> Result<(Option>>, i64), anyhow::Error> { + let now = Utc::now().timestamp_micros(); let row = match row { Some(row) => row, None => { - return Ok(None); + return Ok((None, now)); } }; if self.conditions.is_none() { - return Ok(None); + return Ok((None, now)); } let conditions = self.conditions.as_ref().unwrap(); if conditions.is_empty() { - return Ok(None); + return Ok((None, now)); } for condition in conditions.iter() { if !condition.evaluate(row).await { - return Ok(None); + return Ok((None, now)); } } - Ok(Some(vec![row.to_owned()])) + Ok((Some(vec![row.to_owned()]), now)) } pub async fn evaluate_scheduled( &self, - alert: &Alert, + stream_param: &StreamParams, + trigger_condition: &TriggerCondition, + query_condition: &QueryCondition, ) -> Result<(Option>>, i64), anyhow::Error> { let now = Utc::now().timestamp_micros(); let sql = match self.query_type { @@ -401,7 +79,7 @@ impl QueryCondition { let Some(v) = self.conditions.as_ref() else { return Ok((None, now)); }; - build_sql(alert, v).await? + build_sql(stream_param, self, v).await? } QueryType::SQL => { let Some(v) = self.sql.as_ref() else { @@ -421,7 +99,7 @@ impl QueryCondition { return Ok((None, now)); } let start = now - - Duration::try_minutes(alert.trigger_condition.period) + - Duration::try_minutes(trigger_condition.period) .unwrap() .num_microseconds() .unwrap(); @@ -444,7 +122,7 @@ impl QueryCondition { (end - start) / promql::MAX_DATA_POINTS, ), }; - let resp = match promql::search::search(&alert.org_id, &req, 0, "").await { + let resp = match promql::search::search(&stream_param.org_id, &req, 0, "").await { Ok(v) => v, Err(_) => { return Ok((None, now)); @@ -461,10 +139,10 @@ impl QueryCondition { // TODO calculate the sample in a row, suddenly a sample can be ignored let value = value .into_iter() - .filter(|f| f.samples.len() >= alert.trigger_condition.threshold as usize) + .filter(|f| f.samples.len() >= trigger_condition.threshold as usize) .collect::>(); return if value.is_empty() { - Ok((None, now)) + return Ok((None, now)); } else { Ok(( Some( @@ -495,13 +173,17 @@ impl QueryCondition { }; // fire the query - let req: config::meta::search::Request = config::meta::search::Request { + let req = config::meta::search::Request { query: config::meta::search::Query { sql: sql.clone(), from: 0, - size: std::cmp::max(100, alert.trigger_condition.threshold), + size: if self.search_event_type.is_some() { + -1 + } else { + std::cmp::max(100, trigger_condition.threshold) + }, start_time: now - - Duration::try_minutes(alert.trigger_condition.period) + - Duration::try_minutes(trigger_condition.period) .unwrap() .num_microseconds() .unwrap(), @@ -511,12 +193,13 @@ impl QueryCondition { query_type: "".to_string(), track_total_hits: false, uses_zo_fn: false, - query_fn: if alert.query_condition.vrl_function.is_some() { - match base64::decode_url(alert.query_condition.vrl_function.as_ref().unwrap()) { + query_fn: if query_condition.vrl_function.is_some() { + match base64::decode_url(query_condition.vrl_function.as_ref().unwrap()) { Ok(query_fn) => Some(query_fn), Err(e) => { return Err(anyhow::anyhow!( - "Error decoding alert vrl query function: {e}" + "Error decoding alert vrl query function: {e}" /* TODO: update + * error msg */ )); } } @@ -529,23 +212,29 @@ impl QueryCondition { regions: vec![], clusters: vec![], timeout: 0, - search_type: Some(SearchEventType::Alerts), + search_type: Some(SearchEventType::Alerts), /* TODO(taiming): change the name to + * scheduled & inform FE */ }; let trace_id = ider::uuid(); - let resp = - match SearchService::search(&trace_id, &alert.org_id, alert.stream_type, None, &req) - .await - { - Ok(v) => v, - Err(e) => { - if let infra::errors::Error::ErrorCode(e) = e { - return Err(anyhow::anyhow!("{}", e.get_message())); - } else { - return Err(anyhow::anyhow!("{}", e)); - } + let resp = match SearchService::search( + &trace_id, + &stream_param.org_id, + stream_param.stream_type, + None, + &req, + ) + .await + { + Ok(v) => v, + Err(e) => { + if let infra::errors::Error::ErrorCode(e) = e { + return Err(anyhow::anyhow!("{}", e.get_message())); + } else { + return Err(anyhow::anyhow!("{}", e)); } - }; - if resp.total < alert.trigger_condition.threshold as usize { + } + }; + if self.search_event_type.is_none() && resp.total < trigger_condition.threshold as usize { Ok((None, now)) } else { Ok(( @@ -627,8 +316,17 @@ impl Condition { } } -async fn build_sql(alert: &Alert, conditions: &[Condition]) -> Result { - let schema = infra::schema::get(&alert.org_id, &alert.stream_name, alert.stream_type).await?; +async fn build_sql( + stream_params: &StreamParams, + query_condition: &QueryCondition, + conditions: &[Condition], +) -> Result { + let schema = infra::schema::get( + &stream_params.org_id, + &stream_params.stream_name, + stream_params.stream_type, + ) + .await?; let mut wheres = Vec::with_capacity(conditions.len()); for cond in conditions.iter() { let data_type = match schema.field_with_name(&cond.column) { @@ -637,7 +335,7 @@ async fn build_sql(alert: &Alert, conditions: &[Condition]) -> Result Result field.data_type(), @@ -666,7 +364,7 @@ async fn build_sql(alert: &Alert, conditions: &[Condition]) -> Result Result Result], -) -> Result<(), anyhow::Error> { - let rows_tpl_val = if alert.row_template.is_empty() { - vec!["".to_string()] - } else { - process_row_template(&alert.row_template, alert, rows) - }; - let msg: String = process_dest_template(&dest.template.body, alert, rows, &rows_tpl_val).await; - - match dest.destination_type { - DestinationType::Http => send_http_notification(dest, msg.clone()).await, - DestinationType::Email => send_email_notification(&alert.name, dest, msg).await, - } -} - -pub async fn send_http_notification( - dest: &DestinationWithTemplate, - msg: String, -) -> Result<(), anyhow::Error> { - let client = if dest.skip_tls_verify { - reqwest::Client::builder() - .danger_accept_invalid_certs(true) - .build()? - } else { - reqwest::Client::new() - }; - let url = url::Url::parse(&dest.url)?; - let mut req = match dest.method { - HTTPType::POST => client.post(url), - HTTPType::PUT => client.put(url), - HTTPType::GET => client.get(url), - }; - - // Add additional headers if any from destination description - let mut has_context_type = false; - if let Some(headers) = &dest.headers { - for (key, value) in headers.iter() { - if !key.is_empty() && !value.is_empty() { - if key.to_lowercase().trim() == "content-type" { - has_context_type = true; - } - req = req.header(key, value); - } - } - }; - // set default content type - if !has_context_type { - req = req.header("Content-type", "application/json"); - } - - let resp = req.body(msg.clone()).send().await?; - if !resp.status().is_success() { - log::error!("Alert body: {}", msg); - return Err(anyhow::anyhow!( - "sent error status: {}, err: {:?}", - resp.status(), - resp.bytes().await - )); - } - - Ok(()) -} - -pub async fn send_email_notification( - alert_name: &str, - dest: &DestinationWithTemplate, - msg: String, -) -> Result<(), anyhow::Error> { - let cfg = get_config(); - if !cfg.smtp.smtp_enabled { - return Err(anyhow::anyhow!("SMTP configuration not enabled")); - } - - let mut recepients = vec![]; - for recepient in &dest.emails { - recepients.push(recepient); - } - - let mut email = Message::builder() - .from(cfg.smtp.smtp_from_email.parse()?) - .subject(format!("Openobserve Alert - {}", alert_name)); - - for recepient in recepients { - email = email.to(recepient.parse()?); - } - - if !cfg.smtp.smtp_reply_to.is_empty() { - email = email.reply_to(cfg.smtp.smtp_reply_to.parse()?); - } - - let email = email.singlepart(SinglePart::html(msg)).unwrap(); - - // Send the email - match SMTP_CLIENT.as_ref().unwrap().send(email).await { - Ok(_) => Ok(()), - Err(e) => Err(anyhow::anyhow!("Error sending email: {e}")), - } -} - -fn process_row_template(tpl: &String, alert: &Alert, rows: &[Map]) -> Vec { - let alert_type = if alert.is_real_time { - "realtime" - } else { - "scheduled" - }; - let alert_count = rows.len(); - let mut rows_tpl = Vec::with_capacity(rows.len()); - for row in rows.iter() { - let mut resp = tpl.to_string(); - let mut alert_start_time = 0; - let mut alert_end_time = 0; - for (key, value) in row.iter() { - let value = if value.is_string() { - value.as_str().unwrap_or_default().to_string() - } else if value.is_f64() { - format!("{:.2}", value.as_f64().unwrap_or_default()) - } else { - value.to_string() - }; - process_variable_replace(&mut resp, key, &VarValue::Str(&value)); - - // calculate start and end time - if key == &get_config().common.column_timestamp { - let val = value.parse::().unwrap_or_default(); - if alert_start_time == 0 || val < alert_start_time { - alert_start_time = val; - } - if alert_end_time == 0 || val > alert_end_time { - alert_end_time = val; - } - } - if key == "zo_sql_min_time" { - let val = value.parse::().unwrap_or_default(); - if alert_start_time == 0 || val < alert_start_time { - alert_start_time = val; - } - } - if key == "zo_sql_max_time" { - let val = value.parse::().unwrap_or_default(); - if alert_end_time == 0 || val > alert_end_time { - alert_end_time = val; - } - } - } - let alert_start_time_str = if alert_start_time > 0 { - Local - .timestamp_nanos(alert_start_time * 1000) - .format("%Y-%m-%dT%H:%M:%S") - .to_string() - } else { - String::from("N/A") - }; - let alert_end_time_str = if alert_end_time > 0 { - Local - .timestamp_nanos(alert_end_time * 1000) - .format("%Y-%m-%dT%H:%M:%S") - .to_string() - } else { - String::from("N/A") - }; - - resp = resp - .replace("{org_name}", &alert.org_id) - .replace("{stream_type}", &alert.stream_type.to_string()) - .replace("{stream_name}", &alert.stream_name) - .replace("{alert_name}", &alert.name) - .replace("{alert_type}", alert_type) - .replace( - "{alert_period}", - &alert.trigger_condition.period.to_string(), - ) - .replace( - "{alert_operator}", - &alert.trigger_condition.operator.to_string(), - ) - .replace( - "{alert_threshold}", - &alert.trigger_condition.threshold.to_string(), - ) - .replace("{alert_count}", &alert_count.to_string()) - .replace("{alert_start_time}", &alert_start_time_str) - .replace("{alert_end_time}", &alert_end_time_str); - - if let Some(contidion) = &alert.query_condition.promql_condition { - resp = resp - .replace("{alert_promql_operator}", &contidion.operator.to_string()) - .replace("{alert_promql_value}", &contidion.value.to_string()); - } - - if let Some(attrs) = &alert.context_attributes { - for (key, value) in attrs.iter() { - process_variable_replace(&mut resp, key, &VarValue::Str(value)); - } - } - - rows_tpl.push(resp); - } - - rows_tpl -} - -async fn process_dest_template( - tpl: &str, - alert: &Alert, - rows: &[Map], - rows_tpl_val: &[String], -) -> String { - let cfg = get_config(); - // format values - let alert_count = rows.len(); - let mut vars = HashMap::with_capacity(rows.len()); - for row in rows.iter() { - for (key, value) in row.iter() { - let value = if value.is_string() { - value.as_str().unwrap_or_default().to_string() - } else if value.is_f64() { - format!("{:.2}", value.as_f64().unwrap_or_default()) - } else { - value.to_string() - }; - let entry = vars.entry(key.to_string()).or_insert_with(HashSet::new); - entry.insert(value); - } - } - - // calculate start and end time - let (alert_start_time, alert_end_time) = - get_alert_start_end_time(&vars, alert.trigger_condition.period); - - let alert_start_time_str = if alert_start_time > 0 { - Local - .timestamp_nanos(alert_start_time * 1000) - .format("%Y-%m-%dT%H:%M:%S") - .to_string() - } else { - String::from("N/A") - }; - let alert_end_time_str = if alert_end_time > 0 { - Local - .timestamp_nanos(alert_end_time * 1000) - .format("%Y-%m-%dT%H:%M:%S") - .to_string() - } else { - String::from("N/A") - }; - - let alert_type = if alert.is_real_time { - "realtime" - } else { - "scheduled" - }; - - let mut alert_query = String::new(); - let alert_url = if alert.query_condition.query_type == QueryType::PromQL { - if let Some(promql) = &alert.query_condition.promql { - let condition = alert.query_condition.promql_condition.as_ref().unwrap(); - alert_query = format!( - "({}) {} {}", - promql, - match condition.operator { - Operator::EqualTo => "==".to_string(), - _ => condition.operator.to_string(), - }, - to_float(&condition.value) - ); - } - // http://localhost:5080/web/metrics?stream=zo_http_response_time_bucket&from=1705248000000000&to=1705334340000000&query=em9faHR0cF9yZXNwb25zZV90aW1lX2J1Y2tldHt9&org_identifier=default - format!( - "{}{}/web/metrics?stream_type={}&stream={}&stream_value={}&from={}&to={}&query={}&org_identifier={}", - cfg.common.web_url, - cfg.common.base_uri, - alert.stream_type, - alert.stream_name, - alert.stream_name, - alert_start_time, - alert_end_time, - base64::encode_url(&alert_query).replace('+', "%2B"), - alert.org_id, - ) - } else { - match alert.query_condition.query_type { - QueryType::SQL => { - if let Some(sql) = &alert.query_condition.sql { - alert_query = sql.clone(); - } - } - QueryType::Custom => { - if let Some(conditions) = &alert.query_condition.conditions { - if let Ok(v) = build_sql(alert, conditions).await { - alert_query = v; - } - } - } - _ => unreachable!(), - }; - // http://localhost:5080/web/logs?stream_type=logs&stream=test&from=1708416534519324&to=1708416597898186&sql_mode=true&query=U0VMRUNUICogRlJPTSAidGVzdCIgd2hlcmUgbGV2ZWwgPSAnaW5mbyc=&org_identifier=default - format!( - "{}{}/web/logs?stream_type={}&stream={}&stream_value={}&from={}&to={}&sql_mode=true&query={}&org_identifier={}", - cfg.common.web_url, - cfg.common.base_uri, - alert.stream_type, - alert.stream_name, - alert.stream_name, - alert_start_time, - alert_end_time, - base64::encode_url(&alert_query), - alert.org_id, - ) - }; - - let mut resp = tpl - .replace("{org_name}", &alert.org_id) - .replace("{stream_type}", &alert.stream_type.to_string()) - .replace("{stream_name}", &alert.stream_name) - .replace("{alert_name}", &alert.name) - .replace("{alert_type}", alert_type) - .replace( - "{alert_period}", - &alert.trigger_condition.period.to_string(), - ) - .replace( - "{alert_operator}", - &alert.trigger_condition.operator.to_string(), - ) - .replace( - "{alert_threshold}", - &alert.trigger_condition.threshold.to_string(), - ) - .replace("{alert_count}", &alert_count.to_string()) - .replace("{alert_start_time}", &alert_start_time_str) - .replace("{alert_end_time}", &alert_end_time_str) - .replace("{alert_url}", &alert_url); - - if let Some(contidion) = &alert.query_condition.promql_condition { - resp = resp - .replace("{alert_promql_operator}", &contidion.operator.to_string()) - .replace("{alert_promql_value}", &contidion.value.to_string()); - } - - process_variable_replace(&mut resp, "rows", &VarValue::Vector(rows_tpl_val)); - for (key, value) in vars.iter() { - if resp.contains(&format!("{{{key}}}")) { - let val = value.iter().cloned().collect::>(); - process_variable_replace(&mut resp, key, &VarValue::Str(&val.join(", "))); - } - } - if let Some(attrs) = &alert.context_attributes { - for (key, value) in attrs.iter() { - process_variable_replace(&mut resp, key, &VarValue::Str(value)); - } - } - - resp -} - -fn process_variable_replace(tpl: &mut String, var_name: &str, var_val: &VarValue) { - let pattern = "{".to_owned() + var_name + "}"; - if tpl.contains(&pattern) { - *tpl = tpl.replace(&pattern, &var_val.to_string_with_length(0)); - return; - } - let pattern = "{".to_owned() + var_name + ":"; - if let Some(start) = tpl.find(&pattern) { - // find } start from position v - let p = start + pattern.len(); - if let Some(end) = tpl[p..].find('}') { - let len = tpl[p..p + end].parse::().unwrap_or_default(); - if len > 0 { - *tpl = tpl.replace( - &tpl[start..p + end + 1], - &var_val.to_string_with_length(len), - ); - } - } - } -} - -pub fn get_row_column_map(rows: &[Map]) -> HashMap> { - let mut vars = HashMap::with_capacity(rows.len()); - for row in rows.iter() { - for (key, value) in row.iter() { - let value = if value.is_string() { - value.as_str().unwrap_or_default().to_string() - } else if value.is_f64() { - format!("{:.2}", value.as_f64().unwrap_or_default()) - } else { - value.to_string() - }; - let entry = vars.entry(key.to_string()).or_insert_with(HashSet::new); - entry.insert(value); - } - } - vars -} - -pub fn get_alert_start_end_time( - vars: &HashMap>, - period: i64, -) -> (i64, i64) { - let cfg = get_config(); - - // calculate start and end time - let mut alert_start_time = 0; - let mut alert_end_time = 0; - if let Some(values) = vars.get(&cfg.common.column_timestamp) { - for val in values { - let val = val.parse::().unwrap_or_default(); - if alert_start_time == 0 || val < alert_start_time { - alert_start_time = val; - } - if alert_end_time == 0 || val > alert_end_time { - alert_end_time = val; - } - } - } - if let Some(values) = vars.get("zo_sql_min_time") { - for val in values { - let val = val.parse::().unwrap_or_default(); - if alert_start_time == 0 || val < alert_start_time { - alert_start_time = val; - } - } - } - if let Some(values) = vars.get("zo_sql_max_time") { - for val in values { - let val = val.parse::().unwrap_or_default(); - if alert_end_time == 0 || val > alert_end_time { - alert_end_time = val; - } - } - } - - // Hack time range for alert url - alert_end_time = if alert_end_time == 0 { - Utc::now().timestamp_micros() - } else { - // the frontend will drop the second, so we add 1 minute to the end time - alert_end_time - + Duration::try_minutes(1) - .unwrap() - .num_microseconds() - .unwrap() - }; - if alert_start_time == 0 { - alert_start_time = alert_end_time - - Duration::try_minutes(period) - .unwrap() - .num_microseconds() - .unwrap(); - } - if alert_end_time - alert_start_time - < Duration::try_minutes(1) - .unwrap() - .num_microseconds() - .unwrap() - { - alert_start_time = alert_end_time - - Duration::try_minutes(period) - .unwrap() - .num_microseconds() - .unwrap(); - } - (alert_start_time, alert_end_time) -} - -fn format_variable_value(val: String) -> String { - val.replace('\n', "\\n") - .replace('\r', "\\r") - .replace('\"', "\\\"") -} - -fn to_float(val: &Value) -> f64 { - if val.is_number() { - val.as_f64().unwrap_or_default() - } else { - val.as_str().unwrap_or_default().parse().unwrap_or_default() - } -} - -enum VarValue<'a> { - Str(&'a str), - Vector(&'a [String]), -} - -impl<'a> VarValue<'a> { - fn len(&self) -> usize { - match self { - VarValue::Str(v) => v.chars().count(), - VarValue::Vector(v) => v.len(), - } - } - - fn to_string_with_length(&self, n: usize) -> String { - let n = if n > 0 && n < self.len() { - n - } else { - self.len() - }; - match self { - VarValue::Str(v) => format_variable_value(v.chars().take(n).collect()), - VarValue::Vector(v) => v[0..n].join("\\n"), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_alert_create() { - let org_id = "default"; - let stream_name = "default"; - let alert_name = "abc/alert"; - let alert = Alert { - name: alert_name.to_string(), - ..Default::default() - }; - let ret = save(org_id, stream_name, alert_name, alert, true).await; - // alert name should not contain / - assert!(ret.is_err()); - } -} diff --git a/src/service/alerts/alert_manager.rs b/src/service/alerts/scheduler.rs similarity index 64% rename from src/service/alerts/alert_manager.rs rename to src/service/alerts/scheduler.rs index 050fe2ff3e..c05922c064 100644 --- a/src/service/alerts/alert_manager.rs +++ b/src/service/alerts/scheduler.rs @@ -22,15 +22,18 @@ use config::{ stream::StreamType, usage::{TriggerData, TriggerDataStatus, TriggerDataType}, }, + utils::json, }; use cron::Schedule; use futures::future::try_join_all; +use proto::cluster_rpc; use crate::{ - common::meta::{alerts::AlertFrequencyType, dashboards::reports::ReportFrequencyType}, + common::meta::{alerts::FrequencyType, dashboards::reports::ReportFrequencyType}, service::{ - alerts::{get_alert_start_end_time, get_row_column_map}, + alerts::alert::{get_alert_start_end_time, get_row_column_map}, db, + ingestion::ingestion_service, usage::publish_triggers_usage, }, }; @@ -55,13 +58,13 @@ pub async fn run() -> Result<(), anyhow::Error> { for trigger in triggers { let task = tokio::task::spawn(async move { if let Err(e) = handle_triggers(trigger).await { - log::error!("[ALERT_MANAGER] Error handling trigger: {}", e); + log::error!("[SCHEDULER] Error handling trigger: {}", e); } }); tasks.push(task); } if let Err(e) = try_join_all(tasks).await { - log::error!("[ALERT_MANAGER] Error handling triggers: {}", e); + log::error!("[SCHEDULER] Error handling triggers: {}", e); } Ok(()) } @@ -70,6 +73,9 @@ pub async fn handle_triggers(trigger: db::scheduler::Trigger) -> Result<(), anyh match trigger.module { db::scheduler::TriggerModule::Report => handle_report_triggers(trigger).await, db::scheduler::TriggerModule::Alert => handle_alert_triggers(trigger).await, + db::scheduler::TriggerModule::DerivedStream => { + handle_derived_stream_triggers(trigger).await + } } } @@ -105,7 +111,7 @@ async fn handle_alert_triggers(trigger: db::scheduler::Trigger) -> Result<(), an return Ok(()); } - let alert = match super::get(org_id, stream_type, stream_name, alert_name).await? { + let alert = match super::alert::get(org_id, stream_type, stream_name, alert_name).await? { Some(alert) => alert, None => { return Err(anyhow::anyhow!( @@ -145,7 +151,7 @@ async fn handle_alert_triggers(trigger: db::scheduler::Trigger) -> Result<(), an ); } if ret.is_some() && alert.trigger_condition.silence > 0 { - if alert.trigger_condition.frequency_type == AlertFrequencyType::Cron { + if alert.trigger_condition.frequency_type == FrequencyType::Cron { let schedule = Schedule::from_str(&alert.trigger_condition.cron)?; let silence = Utc::now() + Duration::try_minutes(alert.trigger_condition.silence).unwrap(); @@ -163,7 +169,7 @@ async fn handle_alert_triggers(trigger: db::scheduler::Trigger) -> Result<(), an .unwrap(); } new_trigger.is_silenced = true; - } else if alert.trigger_condition.frequency_type == AlertFrequencyType::Cron { + } else if alert.trigger_condition.frequency_type == FrequencyType::Cron { let schedule = Schedule::from_str(&alert.trigger_condition.cron)?; // tz_offset is in minutes let tz_offset = FixedOffset::east_opt(alert.tz_offset * 60).unwrap(); @@ -425,3 +431,207 @@ async fn handle_report_triggers(trigger: db::scheduler::Trigger) -> Result<(), a Ok(()) } + +async fn handle_derived_stream_triggers( + trigger: db::scheduler::Trigger, +) -> Result<(), anyhow::Error> { + log::debug!( + "Inside handle_derived_stream_triggers processing trigger: {}", + trigger.module_key + ); + + // module_key format: stream_type/stream_name/pipeline_name/derived_stream_name + let columns = trigger.module_key.split('/').collect::>(); + assert_eq!(columns.len(), 4); + let org_id = &trigger.org; + let stream_type: StreamType = columns[0].into(); + let stream_name = columns[1]; + let pipeline_name = columns[2]; + let name = columns[3]; + + let is_real_time = trigger.is_realtime; + let is_silenced = trigger.is_silenced; + if is_real_time && is_silenced { + log::debug!( + "Realtime derived_stream needs to wake up, {}/{}", + org_id, + trigger.module_key + ); + let new_trigger = db::scheduler::Trigger { + next_run_at: Utc::now().timestamp_micros(), + is_silenced: false, + status: db::scheduler::TriggerStatus::Waiting, + ..trigger.clone() + }; + db::scheduler::update_trigger(new_trigger).await?; + return Ok(()); + } + + let Ok(pipeline) = db::pipelines::get(org_id, stream_type, stream_name, pipeline_name).await + else { + return Err(anyhow::anyhow!( + "Pipeline associated with trigger not found: {}/{}/{}/{}", + org_id, + stream_name, + stream_type, + pipeline_name + )); + }; + + let Some(derived_stream) = pipeline + .derived_streams + .and_then(|ds| ds.into_iter().find(|ds| ds.name == name)) + else { + return Err(anyhow::anyhow!( + "DerivedStream associated with the trigger not found in pipeline: {}/{}/{}/{}", + org_id, + stream_name, + stream_type, + name, + )); + }; + + let mut new_trigger = db::scheduler::Trigger { + next_run_at: Utc::now().timestamp_micros(), + is_silenced: false, + status: db::scheduler::TriggerStatus::Waiting, + ..trigger.clone() + }; + + // evaluate trigger and configure trigger next run time + let (ret, _) = derived_stream.evaluate(None).await?; + if ret.is_some() { + log::info!( + "DerivedStream conditions satisfied, org: {}, module_key: {}", + new_trigger.org, + new_trigger.module_key + ); + } + if ret.is_some() && derived_stream.trigger_condition.silence > 0 { + if derived_stream.trigger_condition.frequency_type == FrequencyType::Cron { + let schedule = Schedule::from_str(&derived_stream.trigger_condition.cron)?; + let silence = Utc::now() + + Duration::try_minutes(derived_stream.trigger_condition.silence).unwrap(); + let silence = silence.with_timezone( + FixedOffset::east_opt(derived_stream.tz_offset * 60) + .as_ref() + .unwrap(), + ); + // Check for the cron timestamp after the silence period + new_trigger.next_run_at = schedule.after(&silence).next().unwrap().timestamp_micros(); + } else { + new_trigger.next_run_at += + Duration::try_minutes(derived_stream.trigger_condition.silence) + .unwrap() + .num_microseconds() + .unwrap(); + } + new_trigger.is_silenced = true; + } else if derived_stream.trigger_condition.frequency_type == FrequencyType::Cron { + let schedule = Schedule::from_str(&derived_stream.trigger_condition.cron)?; + // tz_offset is in minutes + let tz_offset = FixedOffset::east_opt(derived_stream.tz_offset * 60).unwrap(); + new_trigger.next_run_at = schedule + .upcoming(tz_offset) + .next() + .unwrap() + .timestamp_micros(); + } else { + new_trigger.next_run_at += + Duration::try_minutes(derived_stream.trigger_condition.frequency) + .unwrap() + .num_microseconds() + .unwrap(); + } + + let mut trigger_data_stream = TriggerData { + _timestamp: trigger.start_time.unwrap_or_default(), + org: trigger.org, + module: TriggerDataType::DerivedStream, + key: trigger.module_key.clone(), + next_run_at: new_trigger.next_run_at, + is_realtime: trigger.is_realtime, + is_silenced: trigger.is_silenced, + status: TriggerDataStatus::Completed, + start_time: trigger.start_time.unwrap_or_default(), + end_time: trigger.end_time.unwrap_or_default(), + retries: trigger.retries, + error: None, + }; + + // ingest evaluation result into destination + if let Some(data) = ret { + let local_val = data + .into_iter() + .map(json::Value::Object) + .collect::>(); + // Ingest result into destination stream + let (org_id, stream_name, stream_type): (String, String, i32) = { + ( + derived_stream.destination.org_id.into(), + derived_stream.destination.stream_name.into(), + cluster_rpc::StreamType::from(derived_stream.destination.stream_type).into(), + ) + }; + let req = cluster_rpc::IngestionRequest { + org_id: org_id.clone(), + stream_name: stream_name.clone(), + stream_type, + data: Some(cluster_rpc::IngestionData::from(local_val)), + ingestion_type: Some(cluster_rpc::IngestionType::Json.into()), /* TODO(taiming): finalize IngestionType for derived_stream */ + }; + match ingestion_service::ingest(&org_id, req).await { + Ok(_) => { + log::info!( + "DerivedStream result ingested to destination {org_id}/{stream_name}/{stream_type}", + ); + db::scheduler::update_trigger(new_trigger).await?; + } + Err(e) => { + log::error!( + "Error in ingesting DerivedStream result to destination {:?}, org: {}, module_key: {}", + e, + new_trigger.org, + new_trigger.module_key + ); + if trigger.retries + 1 >= get_config().limit.scheduler_max_retries { + // It has been tried the maximum time, just update the + // next_run_at to the next expected trigger time + log::debug!( + "This DerivedStream trigger: {}/{} has reached maximum retries", + &new_trigger.org, + &new_trigger.module_key + ); + db::scheduler::update_trigger(new_trigger).await?; + } else { + // Otherwise update its status only + db::scheduler::update_status( + &new_trigger.org, + new_trigger.module, + &new_trigger.module_key, + db::scheduler::TriggerStatus::Waiting, + trigger.retries + 1, + ) + .await?; + } + trigger_data_stream.status = TriggerDataStatus::Failed; + trigger_data_stream.error = + Some(format!("error sending notification for alert: {e}")); + } + } + } else { + log::debug!( + "DerivedStream conditions not satisfied, org: {}, module_key: {}", + &new_trigger.org, + &new_trigger.module_key + ); + db::scheduler::update_trigger(new_trigger).await?; + trigger_data_stream.status = TriggerDataStatus::ConditionNotSatisfied; + } + + // publish the triggers as stream + trigger_data_stream.end_time = Utc::now().timestamp_micros(); + publish_triggers_usage(trigger_data_stream).await; + + Ok(()) +} diff --git a/src/service/db/alerts/alert.rs b/src/service/db/alerts/alert.rs new file mode 100644 index 0000000000..a5d9d61293 --- /dev/null +++ b/src/service/db/alerts/alert.rs @@ -0,0 +1,257 @@ +// Copyright 2024 Zinc Labs Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::sync::Arc; + +use config::{meta::stream::StreamType, utils::json}; + +use crate::{ + common::{infra::config::STREAM_ALERTS, meta::alerts::alert::Alert}, + service::db, +}; + +pub async fn get( + org_id: &str, + stream_type: StreamType, + stream_name: &str, + name: &str, +) -> Result, anyhow::Error> { + let stream_key = format!("{org_id}/{stream_type}/{stream_name}"); + let value: Option = if let Some(v) = STREAM_ALERTS.read().await.get(&stream_key) { + v.iter().find(|x| x.name.eq(name)).cloned() + } else { + let key = format!("/alerts/{org_id}/{stream_type}/{stream_name}/{name}"); + match db::get(&key).await { + Ok(val) => json::from_slice(&val)?, + Err(_) => None, + } + }; + if value.is_none() { Ok(None) } else { Ok(value) } +} + +pub async fn set( + org_id: &str, + stream_type: StreamType, + stream_name: &str, + alert: &Alert, + create: bool, +) -> Result<(), anyhow::Error> { + let schedule_key = format!("{stream_type}/{stream_name}/{}", alert.name); + let key = format!("/alerts/{org_id}/{}", &schedule_key); + match db::put( + &key, + json::to_vec(alert).unwrap().into(), + db::NEED_WATCH, + None, + ) + .await + { + Ok(_) => { + let trigger = db::scheduler::Trigger { + org: org_id.to_string(), + module_key: schedule_key, + next_run_at: chrono::Utc::now().timestamp_micros(), + is_realtime: alert.is_real_time, + is_silenced: false, + ..Default::default() + }; + if create { + match db::scheduler::push(trigger).await { + Ok(_) => Ok(()), + Err(e) => { + log::error!("Failed to save trigger: {}", e); + Ok(()) + } + } + } else { + match db::scheduler::update_trigger(trigger).await { + Ok(_) => Ok(()), + Err(e) => { + log::error!("Failed to update trigger: {}", e); + Ok(()) + } + } + } + } + Err(e) => Err(anyhow::anyhow!("Error save alert: {}", e)), + } +} + +pub async fn delete( + org_id: &str, + stream_type: StreamType, + stream_name: &str, + name: &str, +) -> Result<(), anyhow::Error> { + let schedule_key = format!("{stream_type}/{stream_name}/{name}"); + let key = format!("/alerts/{org_id}/{}", &schedule_key); + match db::delete(&key, false, db::NEED_WATCH, None).await { + Ok(_) => { + match db::scheduler::delete(org_id, db::scheduler::TriggerModule::Alert, &schedule_key) + .await + { + Ok(_) => Ok(()), + Err(e) => { + log::error!("Failed to delete trigger: {}", e); + Ok(()) + } + } + } + Err(e) => Err(anyhow::anyhow!("Error deleting alert: {e}")), + } +} + +pub async fn list( + org_id: &str, + stream_type: Option, + stream_name: Option<&str>, +) -> Result, anyhow::Error> { + let loc_stream_type = stream_type.unwrap_or_default(); + let key = match stream_name { + Some(stream_name) => format!("/alerts/{org_id}/{loc_stream_type}/{stream_name}"), + None => format!("/alerts/{org_id}/"), + }; + let ret = db::list_values(&key).await?; + let mut items: Vec = Vec::with_capacity(ret.len()); + for item_value in ret { + let json_val = json::from_slice(&item_value)?; + items.push(json_val) + } + items.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(items) +} + +pub async fn watch() -> Result<(), anyhow::Error> { + let key = "/alerts/"; + let cluster_coordinator = db::get_coordinator().await; + let mut events = cluster_coordinator.watch(key).await?; + let events = Arc::get_mut(&mut events).unwrap(); + log::info!("Start watching alerts"); + loop { + let ev = match events.recv().await { + Some(ev) => ev, + None => { + log::error!("watch_alerts: event channel closed"); + break; + } + }; + match ev { + db::Event::Put(ev) => { + let item_key = ev.key.strip_prefix(key).unwrap(); + let stream_key = item_key[0..item_key.rfind('/').unwrap()].to_string(); + let item_value: Alert = if config::get_config().common.meta_store_external { + match db::get(&ev.key).await { + Ok(val) => match json::from_slice(&val) { + Ok(val) => val, + Err(e) => { + log::error!("Error getting value: {}", e); + continue; + } + }, + Err(e) => { + log::error!("Error getting value: {}", e); + continue; + } + } + } else { + json::from_slice(&ev.value.unwrap()).unwrap() + }; + let mut cacher = STREAM_ALERTS.write().await; + let group = cacher.entry(stream_key.to_string()).or_default(); + if group.contains(&item_value) { + let idx = group.iter().position(|x| x.eq(&item_value)).unwrap(); + let _ = std::mem::replace(&mut group[idx], item_value); + } else { + group.push(item_value); + } + drop(cacher); + } + db::Event::Delete(ev) => { + let item_key = ev.key.strip_prefix(key).unwrap(); + let stream_key = item_key[0..item_key.rfind('/').unwrap()].to_string(); + let alert_name = item_key[item_key.rfind('/').unwrap() + 1..].to_string(); + let mut cacher = STREAM_ALERTS.write().await; + if stream_key.contains('/') { + let group = match cacher.get_mut(&stream_key) { + Some(v) => v, + None => continue, + }; + group.retain(|v| !v.name.eq(&alert_name)); + } else { + cacher.remove(&stream_key); + } + drop(cacher); + } + db::Event::Empty => {} + } + } + Ok(()) +} + +pub async fn cache() -> Result<(), anyhow::Error> { + let key = "/alerts/"; + let ret = db::list(key).await?; + for (item_key, item_value) in ret { + let new_key = item_key.strip_prefix(key).unwrap(); + let alert: Alert = match json::from_slice(&item_value) { + Ok(v) => v, + Err(_) => { + // HACK: for old version, write it back to up + let data: json::Value = json::from_slice(&item_value).unwrap(); + let data = data.as_object().unwrap(); + let destinations = if let Some(dest) = data.get("destination") { + vec![dest.as_str().unwrap().to_string()] + } else { + vec![] + }; + let alert = Alert { + name: data.get("name").unwrap().as_str().unwrap().to_string(), + stream_type: data.get("stream_type").unwrap().as_str().unwrap().into(), + stream_name: match data.get("stream") { + Some(v) => v.as_str().unwrap().to_string(), + None => data + .get("stream_name") + .unwrap() + .as_str() + .unwrap() + .to_string(), + }, + destinations, + ..Default::default() + }; + _ = db::put( + &item_key, + json::to_vec(&alert).unwrap().into(), + db::NO_NEED_WATCH, + None, + ) + .await; + alert + } + }; + let stream_key = &new_key[0..new_key.rfind('/').unwrap()]; + + let mut cacher = STREAM_ALERTS.write().await; + let group = cacher.entry(stream_key.to_string()).or_default(); + group.push(alert); + } + log::info!("Alerts Cached"); + Ok(()) +} + +pub async fn reset() -> Result<(), anyhow::Error> { + let key = "/alerts/"; + Ok(db::delete(key, true, db::NO_NEED_WATCH, None).await?) +} diff --git a/src/service/db/alerts/mod.rs b/src/service/db/alerts/mod.rs index d84fdf5b14..8fc40371bb 100644 --- a/src/service/db/alerts/mod.rs +++ b/src/service/db/alerts/mod.rs @@ -13,249 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::sync::Arc; - -use config::{meta::stream::StreamType, utils::json}; - -use crate::{ - common::{infra::config::STREAM_ALERTS, meta::alerts::Alert}, - service::db, -}; - +pub mod alert; pub mod destinations; pub mod realtime_triggers; pub mod templates; - -pub async fn get( - org_id: &str, - stream_type: StreamType, - stream_name: &str, - name: &str, -) -> Result, anyhow::Error> { - let stream_key = format!("{org_id}/{stream_type}/{stream_name}"); - let value: Option = if let Some(v) = STREAM_ALERTS.read().await.get(&stream_key) { - v.iter().find(|x| x.name.eq(name)).cloned() - } else { - let key = format!("/alerts/{org_id}/{stream_type}/{stream_name}/{name}"); - match db::get(&key).await { - Ok(val) => json::from_slice(&val)?, - Err(_) => None, - } - }; - if value.is_none() { Ok(None) } else { Ok(value) } -} - -pub async fn set( - org_id: &str, - stream_type: StreamType, - stream_name: &str, - alert: &Alert, - create: bool, -) -> Result<(), anyhow::Error> { - let schedule_key = format!("{stream_type}/{stream_name}/{}", alert.name); - let key = format!("/alerts/{org_id}/{}", &schedule_key); - match db::put( - &key, - json::to_vec(alert).unwrap().into(), - db::NEED_WATCH, - None, - ) - .await - { - Ok(_) => { - let trigger = db::scheduler::Trigger { - org: org_id.to_string(), - module_key: schedule_key, - next_run_at: chrono::Utc::now().timestamp_micros(), - is_realtime: alert.is_real_time, - is_silenced: false, - ..Default::default() - }; - if create { - match db::scheduler::push(trigger).await { - Ok(_) => Ok(()), - Err(e) => { - log::error!("Failed to save trigger: {}", e); - Ok(()) - } - } - } else { - match db::scheduler::update_trigger(trigger).await { - Ok(_) => Ok(()), - Err(e) => { - log::error!("Failed to update trigger: {}", e); - Ok(()) - } - } - } - } - Err(e) => Err(anyhow::anyhow!("Error save alert: {}", e)), - } -} - -pub async fn delete( - org_id: &str, - stream_type: StreamType, - stream_name: &str, - name: &str, -) -> Result<(), anyhow::Error> { - let schedule_key = format!("{stream_type}/{stream_name}/{name}"); - let key = format!("/alerts/{org_id}/{}", &schedule_key); - match db::delete(&key, false, db::NEED_WATCH, None).await { - Ok(_) => { - match db::scheduler::delete(org_id, db::scheduler::TriggerModule::Alert, &schedule_key) - .await - { - Ok(_) => Ok(()), - Err(e) => { - log::error!("Failed to delete trigger: {}", e); - Ok(()) - } - } - } - Err(e) => Err(anyhow::anyhow!("Error deleting alert: {e}")), - } -} - -pub async fn list( - org_id: &str, - stream_type: Option, - stream_name: Option<&str>, -) -> Result, anyhow::Error> { - let loc_stream_type = stream_type.unwrap_or_default(); - let key = match stream_name { - Some(stream_name) => format!("/alerts/{org_id}/{loc_stream_type}/{stream_name}"), - None => format!("/alerts/{org_id}/"), - }; - let ret = db::list_values(&key).await?; - let mut items: Vec = Vec::with_capacity(ret.len()); - for item_value in ret { - let json_val = json::from_slice(&item_value)?; - items.push(json_val) - } - items.sort_by(|a, b| a.name.cmp(&b.name)); - Ok(items) -} - -pub async fn watch() -> Result<(), anyhow::Error> { - let key = "/alerts/"; - let cluster_coordinator = db::get_coordinator().await; - let mut events = cluster_coordinator.watch(key).await?; - let events = Arc::get_mut(&mut events).unwrap(); - log::info!("Start watching alerts"); - loop { - let ev = match events.recv().await { - Some(ev) => ev, - None => { - log::error!("watch_alerts: event channel closed"); - break; - } - }; - match ev { - db::Event::Put(ev) => { - let item_key = ev.key.strip_prefix(key).unwrap(); - let stream_key = item_key[0..item_key.rfind('/').unwrap()].to_string(); - let item_value: Alert = if config::get_config().common.meta_store_external { - match db::get(&ev.key).await { - Ok(val) => match json::from_slice(&val) { - Ok(val) => val, - Err(e) => { - log::error!("Error getting value: {}", e); - continue; - } - }, - Err(e) => { - log::error!("Error getting value: {}", e); - continue; - } - } - } else { - json::from_slice(&ev.value.unwrap()).unwrap() - }; - let mut cacher = STREAM_ALERTS.write().await; - let group = cacher.entry(stream_key.to_string()).or_default(); - if group.contains(&item_value) { - let idx = group.iter().position(|x| x.eq(&item_value)).unwrap(); - let _ = std::mem::replace(&mut group[idx], item_value); - } else { - group.push(item_value); - } - drop(cacher); - } - db::Event::Delete(ev) => { - let item_key = ev.key.strip_prefix(key).unwrap(); - let stream_key = item_key[0..item_key.rfind('/').unwrap()].to_string(); - let alert_name = item_key[item_key.rfind('/').unwrap() + 1..].to_string(); - let mut cacher = STREAM_ALERTS.write().await; - if stream_key.contains('/') { - let group = match cacher.get_mut(&stream_key) { - Some(v) => v, - None => continue, - }; - group.retain(|v| !v.name.eq(&alert_name)); - } else { - cacher.remove(&stream_key); - } - drop(cacher); - } - db::Event::Empty => {} - } - } - Ok(()) -} - -pub async fn cache() -> Result<(), anyhow::Error> { - let key = "/alerts/"; - let ret = db::list(key).await?; - for (item_key, item_value) in ret { - let new_key = item_key.strip_prefix(key).unwrap(); - let alert: Alert = match json::from_slice(&item_value) { - Ok(v) => v, - Err(_) => { - // HACK: for old version, write it back to up - let data: json::Value = json::from_slice(&item_value).unwrap(); - let data = data.as_object().unwrap(); - let destinations = if let Some(dest) = data.get("destination") { - vec![dest.as_str().unwrap().to_string()] - } else { - vec![] - }; - let alert = Alert { - name: data.get("name").unwrap().as_str().unwrap().to_string(), - stream_type: data.get("stream_type").unwrap().as_str().unwrap().into(), - stream_name: match data.get("stream") { - Some(v) => v.as_str().unwrap().to_string(), - None => data - .get("stream_name") - .unwrap() - .as_str() - .unwrap() - .to_string(), - }, - destinations, - ..Default::default() - }; - _ = db::put( - &item_key, - json::to_vec(&alert).unwrap().into(), - db::NO_NEED_WATCH, - None, - ) - .await; - alert - } - }; - let stream_key = &new_key[0..new_key.rfind('/').unwrap()]; - - let mut cacher = STREAM_ALERTS.write().await; - let group = cacher.entry(stream_key.to_string()).or_default(); - group.push(alert); - } - log::info!("Alerts Cached"); - Ok(()) -} - -pub async fn reset() -> Result<(), anyhow::Error> { - let key = "/alerts/"; - Ok(db::delete(key, true, db::NO_NEED_WATCH, None).await?) -} diff --git a/src/service/db/pipelines.rs b/src/service/db/pipelines.rs index 8747de6cfd..aad421e98a 100644 --- a/src/service/db/pipelines.rs +++ b/src/service/db/pipelines.rs @@ -22,10 +22,10 @@ use crate::{ service::db, }; -pub async fn set(org_id: &str, name: &str, pipeline: &PipeLine) -> Result<(), anyhow::Error> { +pub async fn set(org_id: &str, pipeline: &PipeLine) -> Result<(), anyhow::Error> { let key = format!( - "/pipeline/{org_id}/{}/{}/{name}", - pipeline.stream_type, pipeline.stream_name + "/pipeline/{org_id}/{}/{}/{}", + pipeline.stream_type, pipeline.stream_name, pipeline.name ); match db::put( &key, diff --git a/src/service/enrichment_table/mod.rs b/src/service/enrichment_table/mod.rs index 2448806fa7..b3c994ea74 100644 --- a/src/service/enrichment_table/mod.rs +++ b/src/service/enrichment_table/mod.rs @@ -56,7 +56,7 @@ pub mod geoip; pub async fn save_enrichment_data( org_id: &str, table_name: &str, - mut payload: Multipart, + payload: Vec>, append_data: bool, ) -> Result { let start = std::time::Instant::now(); @@ -118,77 +118,44 @@ pub async fn save_enrichment_data( .parse::() .unwrap() }; - while let Ok(Some(mut field)) = payload.try_next().await { - let content_disposition = field.content_disposition(); - let filename = content_disposition.get_filename(); - let mut data = bytes::Bytes::new(); - if filename.is_some() { - while let Some(chunk) = field.next().await { - let chunked_data = chunk.unwrap(); - // Reconstruct entire CSV data bytes here to prevent fragmentation of values. - data = Bytes::from([data.as_ref(), chunked_data.as_ref()].concat()); - } - let mut rdr = csv::Reader::from_reader(data.as_ref()); - let headers: csv::StringRecord = rdr - .headers()? - .iter() - .map(|x| { - let mut x = x.trim().to_string(); - format_key(&mut x); - x - }) - .collect::>() - .into(); + for mut json_record in payload { + json_record.insert( + get_config().common.column_timestamp.clone(), + json::Value::Number(timestamp.into()), + ); - for result in rdr.records() { - // The iterator yields Result, so we check the - // error here. - let record = result?; - // Transform the record to a JSON value - let mut json_record = json::Map::new(); - - for (header, field) in headers.iter().zip(record.iter()) { - json_record.insert(header.into(), json::Value::String(field.into())); - } - json_record.insert( - get_config().common.column_timestamp.clone(), - json::Value::Number(timestamp.into()), - ); - - // check for schema evolution - if !schema_evolved - && check_for_schema( - org_id, - stream_name, - StreamType::EnrichmentTables, - &mut stream_schema_map, - vec![&json_record], - timestamp, - ) - .await - .is_ok() - { - schema_evolved = true; - } - - if records.is_empty() { - let schema = stream_schema_map.get(stream_name).unwrap(); - let schema_key = schema.hash_key(); - hour_key = super::ingestion::get_wal_time_key( - timestamp, - &vec![], - PartitionTimeLevel::Unset, - &json_record, - Some(schema_key), - ); - } - let record = json::Value::Object(json_record); - let record_size = json::estimate_json_bytes(&record); - records.push(Arc::new(record)); - records_size += record_size; - } + // check for schema evolution + if !schema_evolved + && check_for_schema( + org_id, + stream_name, + StreamType::EnrichmentTables, + &mut stream_schema_map, + vec![&json_record], + timestamp, + ) + .await + .is_ok() + { + schema_evolved = true; } + + if records.is_empty() { + let schema = stream_schema_map.get(stream_name).unwrap(); + let schema_key = schema.hash_key(); + hour_key = super::ingestion::get_wal_time_key( + timestamp, + &vec![], + PartitionTimeLevel::Unset, + &json_record, + Some(schema_key), + ); + } + let record = json::Value::Object(json_record); + let record_size = json::estimate_json_bytes(&record); + records.push(Arc::new(record)); + records_size += record_size; } if records.is_empty() { @@ -280,3 +247,51 @@ async fn delete_enrichment_table(org_id: &str, stream_name: &str, stream_type: S stats::remove_stream_stats(org_id, stream_name, stream_type); log::info!("deleted enrichment table {stream_name}"); } + +pub async fn extract_multipart( + mut payload: Multipart, +) -> Result>, Error> { + let mut records = Vec::new(); + while let Ok(Some(mut field)) = payload.try_next().await { + let content_disposition = field.content_disposition(); + let filename = content_disposition.get_filename(); + let mut data = bytes::Bytes::new(); + + if filename.is_some() { + while let Some(chunk) = field.next().await { + let chunked_data = chunk.unwrap(); + // Reconstruct entire CSV data bytes here to prevent fragmentation of values. + data = Bytes::from([data.as_ref(), chunked_data.as_ref()].concat()); + } + let mut rdr = csv::Reader::from_reader(data.as_ref()); + let headers: csv::StringRecord = rdr + .headers()? + .iter() + .map(|x| { + let mut x = x.trim().to_string(); + format_key(&mut x); + x + }) + .collect::>() + .into(); + + for result in rdr.records() { + // The iterator yields Result, so we check the + // error here. + let record = result?; + // Transform the record to a JSON value + let mut json_record = json::Map::new(); + + for (header, field) in headers.iter().zip(record.iter()) { + json_record.insert(header.into(), json::Value::String(field.into())); + } + + if !json_record.is_empty() { + records.push(json_record); + } + } + } + } + + Ok(records) +} diff --git a/src/service/ingestion/ingestion_service.rs b/src/service/ingestion/ingestion_service.rs new file mode 100644 index 0000000000..249758417f --- /dev/null +++ b/src/service/ingestion/ingestion_service.rs @@ -0,0 +1,68 @@ +// Copyright 2024 Zinc Labs Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use anyhow::Error; +use proto::cluster_rpc; +use tonic::{ + codec::CompressionEncoding, + metadata::{MetadataKey, MetadataValue}, + Request, +}; + +use crate::{common::infra::cluster, router::grpc::ingest::get_ingester_channel}; + +pub async fn ingest( + dest_org_id: &str, + req: cluster_rpc::IngestionRequest, +) -> Result { + let cfg = config::get_config(); + let org_header_key: MetadataKey<_> = cfg + .grpc + .org_header_key + .parse() + .map_err(|_| Error::msg("invalid org_header_key".to_string()))?; + let token: MetadataValue<_> = cluster::get_internal_grpc_token() + .parse() + .map_err(|_| Error::msg("invalid token".to_string()))?; + let channel = get_ingester_channel().await?; + let mut client = cluster_rpc::ingest_client::IngestClient::with_interceptor( + channel, + move |mut req: Request<()>| { + req.metadata_mut().insert("authorization", token.clone()); + req.metadata_mut() + .insert(org_header_key.clone(), dest_org_id.parse().unwrap()); + Ok(req) + }, + ); + client = client + .send_compressed(CompressionEncoding::Gzip) + .accept_compressed(CompressionEncoding::Gzip) + .max_decoding_message_size(cfg.grpc.max_message_size * 1024 * 1024) + .max_encoding_message_size(cfg.grpc.max_message_size * 1024 * 1024); + let res: cluster_rpc::IngestionResponse = match client.ingest(req).await { + Ok(res) => res.into_inner(), + Err(err) => { + log::error!( + "[InternalIngestion] export partial_success response: {:?}", + err + ); + if err.code() == tonic::Code::Internal { + return Err(err.into()); + } + return Err(Error::msg("Ingest node error")); + } + }; + Ok(res) +} diff --git a/src/service/ingestion/mod.rs b/src/service/ingestion/mod.rs index 30d94ab110..a75bd41f82 100644 --- a/src/service/ingestion/mod.rs +++ b/src/service/ingestion/mod.rs @@ -30,6 +30,7 @@ use config::{ utils::{flatten, json::*}, SIZE_IN_MB, }; +use proto::cluster_rpc::IngestionType; use vector_enrichment::TableRegistry; use vrl::{ compiler::{runtime::Runtime, CompilationResult, TargetValueRef}, @@ -43,8 +44,9 @@ use crate::{ REALTIME_ALERT_TRIGGERS, STREAM_ALERTS, STREAM_FUNCTIONS, STREAM_PIPELINES, }, meta::{ - alerts::Alert, + alerts::alert::Alert, functions::{StreamTransform, VRLResultResolver, VRLRuntimeConfig}, + ingestion::IngestionRequest, stream::{SchemaRecords, StreamParams}, }, utils::functions::get_vrl_compiler_config, @@ -53,6 +55,7 @@ use crate::{ }; pub mod grpc; +pub mod ingestion_service; pub type TriggerAlertData = Vec<(Alert, Vec>)>; @@ -575,6 +578,19 @@ pub async fn get_user_defined_schema( } } +pub fn create_log_ingestion_req( + ingestion_type: i32, + data: &actix_web::web::Bytes, +) -> Result { + match IngestionType::try_from(ingestion_type) { + Ok(IngestionType::Json) => Ok(IngestionRequest::JSON(data)), + Ok(IngestionType::Multi) => Ok(IngestionRequest::Multi(data)), + Ok(IngestionType::Usage) => Ok(IngestionRequest::Usage(data)), + Ok(IngestionType::Rum) => Ok(IngestionRequest::RUM(data)), + _ => Err(anyhow::anyhow!("Not yet supported")), + } +} + #[cfg(test)] mod tests { use infra::schema::{unwrap_stream_settings, STREAM_SETTINGS}; diff --git a/src/service/logs/mod.rs b/src/service/logs/mod.rs index 71704b7e9e..8d44ccdd9b 100644 --- a/src/service/logs/mod.rs +++ b/src/service/logs/mod.rs @@ -43,7 +43,7 @@ use super::{ }; use crate::{ common::meta::{ - alerts::Alert, + alerts::alert::Alert, ingestion::IngestionStatus, stream::{SchemaRecords, StreamParams}, }, diff --git a/src/service/metrics/otlp_grpc.rs b/src/service/metrics/otlp_grpc.rs index 74bb0329cc..030a2241a6 100644 --- a/src/service/metrics/otlp_grpc.rs +++ b/src/service/metrics/otlp_grpc.rs @@ -39,7 +39,7 @@ use prost::Message; use crate::{ common::meta::{ - alerts, + alerts::alert, http::HttpResponse as MetaHttpResponse, prom::*, stream::{SchemaRecords, StreamParams}, @@ -96,7 +96,7 @@ pub async fn handle_grpc_request( let mut metric_data_map: HashMap> = HashMap::new(); let mut metric_schema_map: HashMap = HashMap::new(); let mut schema_evolved: HashMap = HashMap::new(); - let mut stream_alerts_map: HashMap> = HashMap::new(); + let mut stream_alerts_map: HashMap> = HashMap::new(); let mut stream_trigger_map: HashMap> = HashMap::new(); let mut stream_partitioning_map: HashMap = HashMap::new(); diff --git a/src/service/metrics/otlp_http.rs b/src/service/metrics/otlp_http.rs index 688b511cd6..c31fdceec6 100644 --- a/src/service/metrics/otlp_http.rs +++ b/src/service/metrics/otlp_http.rs @@ -40,7 +40,7 @@ use prost::Message; use crate::{ common::meta::{ self, - alerts::Alert, + alerts::alert::Alert, http::HttpResponse as MetaHttpResponse, prom::{self, MetricType, HASH_LABEL, NAME_LABEL, VALUE_LABEL}, stream::{SchemaRecords, StreamParams}, diff --git a/src/service/metrics/prom.rs b/src/service/metrics/prom.rs index 6ae5631957..45eff5140f 100644 --- a/src/service/metrics/prom.rs +++ b/src/service/metrics/prom.rs @@ -43,7 +43,7 @@ use crate::{ common::{ infra::config::{METRIC_CLUSTER_LEADER, METRIC_CLUSTER_MAP}, meta::{ - alerts, + alerts::alert, functions::StreamTransform, prom::*, stream::{SchemaRecords, StreamParams}, @@ -95,7 +95,7 @@ pub async fn remote_write( let mut metric_data_map: HashMap> = HashMap::new(); let mut metric_schema_map: HashMap = HashMap::new(); let mut schema_evolved: HashMap = HashMap::new(); - let mut stream_alerts_map: HashMap> = HashMap::new(); + let mut stream_alerts_map: HashMap> = HashMap::new(); let mut stream_trigger_map: HashMap> = HashMap::new(); let mut stream_transform_map: HashMap> = HashMap::new(); let mut stream_partitioning_map: HashMap = HashMap::new(); diff --git a/src/service/organization.rs b/src/service/organization.rs index e2ad7e872d..a56b49c12f 100644 --- a/src/service/organization.rs +++ b/src/service/organization.rs @@ -35,7 +35,7 @@ use crate::{ pub async fn get_summary(org_id: &str) -> OrgSummary { let streams = get_streams(org_id, None, false, None).await; let functions = db::functions::list(org_id).await.unwrap(); - let alerts = db::alerts::list(org_id, None, None).await.unwrap(); + let alerts = db::alerts::alert::list(org_id, None, None).await.unwrap(); let mut num_streams = 0; let mut total_storage_size = 0.00; let mut total_compressed_size = 0.00; diff --git a/src/service/pipelines.rs b/src/service/pipelines.rs index 7c68b241b7..c5e41735af 100644 --- a/src/service/pipelines.rs +++ b/src/service/pipelines.rs @@ -19,7 +19,7 @@ use actix_web::{ http::{self, StatusCode}, HttpResponse, }; -use config::meta::stream::StreamType; +use config::meta::{search::SearchEventType, stream::StreamType}; use super::db; use crate::common::{ @@ -27,24 +27,52 @@ use crate::common::{ meta::{ http::HttpResponse as MetaHttpResponse, pipelines::{PipeLine, PipeLineList}, + stream::StreamParams, }, }; #[tracing::instrument(skip(pipeline))] -pub async fn save_pipeline(org_id: String, pipeline: PipeLine) -> Result { - if let Some(_existing_pipeline) = check_existing_pipeline( +pub async fn save_pipeline(org_id: String, mut pipeline: PipeLine) -> Result { + if check_existing_pipeline( &org_id, pipeline.stream_type, &pipeline.stream_name, &pipeline.name, ) .await + .is_some() { - Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( + return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( StatusCode::BAD_REQUEST.into(), "Pipeline already exits".to_string(), - ))) - } else if let Err(error) = db::pipelines::set(&org_id, &pipeline.name, &pipeline).await { + ))); + } + + // Save DerivedStream details if there's any + if let Some(ref mut derived_streams) = &mut pipeline.derived_streams { + for derived_stream in derived_streams { + derived_stream.source = + StreamParams::new(&org_id, &pipeline.stream_name, pipeline.stream_type); + derived_stream.query_condition.search_event_type = Some(SearchEventType::DerivedStream); + if !derived_stream.is_valid() { + return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( + http::StatusCode::BAD_REQUEST.into(), + "Invalid DerivedStream details. Name, destination, and Trigger Period required" + .to_string(), + ))); + } + if let Err(e) = + super::alerts::derived_streams::save(derived_stream.clone(), &pipeline.name).await + { + return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( + http::StatusCode::BAD_REQUEST.into(), + format!("Failed to save DerivedStream details error: {}", e), + ))); + } + } + } + + if let Err(error) = db::pipelines::set(&org_id, &pipeline).await { return Ok( HttpResponse::InternalServerError().json(MetaHttpResponse::message( http::StatusCode::INTERNAL_SERVER_ERROR.into(), @@ -60,16 +88,12 @@ pub async fn save_pipeline(org_id: String, pipeline: PipeLine) -> Result Result { +pub async fn update_pipeline(org_id: &str, mut pipeline: PipeLine) -> Result { let existing_pipeline = match check_existing_pipeline( org_id, pipeline.stream_type, &pipeline.stream_name, - pipeline_name, + &pipeline.name, ) .await { @@ -85,7 +109,32 @@ pub async fn update_pipeline( return Ok(HttpResponse::Ok().json(pipeline)); } - if let Err(error) = db::pipelines::set(org_id, &pipeline.name, &pipeline).await { + // Update DerivedStream details if there's any + if let Some(ref mut derived_streams) = &mut pipeline.derived_streams { + for derived_stream in derived_streams { + derived_stream.source = + StreamParams::new(org_id, &pipeline.stream_name, pipeline.stream_type); + derived_stream.query_condition.search_event_type = Some(SearchEventType::DerivedStream); + if !derived_stream.is_valid() { + return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( + http::StatusCode::BAD_REQUEST.into(), + "Invalid DerivedStream details. Name, destination, and Trigger Period required" + .to_string(), + ))); + } + // derived_streams::save updates existing triggers when found + if let Err(e) = + super::alerts::derived_streams::save(derived_stream.clone(), &pipeline.name).await + { + return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( + http::StatusCode::BAD_REQUEST.into(), + format!("Failed to update DerivedStream details with error {}", e), + ))); + } + } + } + + if let Err(error) = db::pipelines::set(org_id, &pipeline).await { return Ok( HttpResponse::InternalServerError().json(MetaHttpResponse::message( http::StatusCode::INTERNAL_SERVER_ERROR.into(), @@ -135,12 +184,50 @@ pub async fn list_pipelines( #[tracing::instrument] pub async fn delete_pipeline( - org_id: &str, - stream_type: StreamType, - stream_name: &str, pipeline_name: &str, + source: StreamParams, ) -> Result { - let result = db::pipelines::delete(org_id, stream_type, stream_name, pipeline_name).await; + let existing_pipeline = match check_existing_pipeline( + &source.org_id, + source.stream_type, + &source.stream_name, + pipeline_name, + ) + .await + { + Some(pipeline) => pipeline, + None => { + return Ok(HttpResponse::NotFound().json(MetaHttpResponse::error( + StatusCode::NOT_FOUND.into(), + "Pipeline not found".to_string(), + ))); + } + }; + + // delete DerivedStream details if there's any + if let Some(derived_streams) = existing_pipeline.derived_streams { + for derived_stream in derived_streams { + if let Err(error) = + super::alerts::derived_streams::delete(derived_stream, &existing_pipeline.name) + .await + { + return Ok( + HttpResponse::InternalServerError().json(MetaHttpResponse::message( + http::StatusCode::INTERNAL_SERVER_ERROR.into(), + error.to_string(), + )), + ); + } + } + } + + let result = db::pipelines::delete( + &source.org_id, + source.stream_type, + &source.stream_name, + pipeline_name, + ) + .await; match result { Ok(_) => Ok(HttpResponse::Ok().json(MetaHttpResponse::message( http::StatusCode::OK.into(), diff --git a/src/service/search/datafusion/plan.rs b/src/service/search/datafusion/plan.rs index 5209a0bf0e..6eb17595f2 100644 --- a/src/service/search/datafusion/plan.rs +++ b/src/service/search/datafusion/plan.rs @@ -55,7 +55,8 @@ pub fn get_partial_plan( if cplan.name() == "RepartitionExec" { let plan = cplan.as_any().downcast_ref::().unwrap(); if let Partitioning::RoundRobinBatch(_) = plan.partitioning() { - return Ok(None); + let child = *cplan.children().first().unwrap(); + return get_partial_plan(child); } } if DISTRIBUTED_PLAN_NAMES.contains(&cplan.name()) { diff --git a/src/service/search/mod.rs b/src/service/search/mod.rs index b9eb39c9e3..c3429e90f2 100644 --- a/src/service/search/mod.rs +++ b/src/service/search/mod.rs @@ -160,6 +160,7 @@ pub async fn search( search::SearchEventType::Dashboards => (true, in_req.search_type), search::SearchEventType::Reports => (true, in_req.search_type), search::SearchEventType::Alerts => (true, in_req.search_type), + search::SearchEventType::DerivedStream => (true, in_req.search_type), search::SearchEventType::RUM => (true, in_req.search_type), search::SearchEventType::Values => (false, None), search::SearchEventType::Other => (false, None), diff --git a/src/service/traces/mod.rs b/src/service/traces/mod.rs index cb5fd9a711..0c59a93f80 100644 --- a/src/service/traces/mod.rs +++ b/src/service/traces/mod.rs @@ -41,7 +41,7 @@ use prost::Message; use crate::{ common::meta::{ - alerts::Alert, + alerts::alert::Alert, http::HttpResponse as MetaHttpResponse, stream::{SchemaRecords, StreamParams}, traces::{Event, Span, SpanLink, SpanLinkContext, SpanRefType}, diff --git a/web/src/components/alerts/AlertTypeSelector.vue b/web/src/components/alerts/AlertTypeSelector.vue deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/web/src/components/alerts/FieldsInput.vue b/web/src/components/alerts/FieldsInput.vue index cc14977c0d..8981d11040 100644 --- a/web/src/components/alerts/FieldsInput.vue +++ b/web/src/components/alerts/FieldsInput.vue @@ -1,3 +1,19 @@ + +